pax_global_header00006660000000000000000000000064144033344000014505gustar00rootroot0000000000000052 comment=9cceb0dde8d7d224707a1cf2ff0d642ad6c51ce6 beanstalkd-1.13/000077500000000000000000000000001440333440000135415ustar00rootroot00000000000000beanstalkd-1.13/.codecov.yml000066400000000000000000000013661440333440000157720ustar00rootroot00000000000000codecov: notify: require_ci_to_pass: yes coverage: precision: 2 round: down range: "65...90" status: project: default: threshold: 0.5 # Allow the coverage to drop by threshold %, and posting a success status. patch: default: target: 0% # trial operation changes: no parsers: gcov: branch_detection: conditional: yes loop: yes method: no macro: no comment: layout: "header, diff" behavior: default require_changes: no ignore: - ".git" - "*.yml" - "*.md" # ignore test files - "test*.c" # ignore sd-daemon.* since it's vendored as external lib - "sd-daemon.*" # ignore folders and all its contents - "adm/.*" - "ct/.*" - "doc/.*" - "pkg/.*" beanstalkd-1.13/.dockerignore000066400000000000000000000000541440333440000162140ustar00rootroot00000000000000*.gcda *.gcno *.o /vers.c /beanstalkd /News beanstalkd-1.13/.github/000077500000000000000000000000001440333440000151015ustar00rootroot00000000000000beanstalkd-1.13/.github/workflows/000077500000000000000000000000001440333440000171365ustar00rootroot00000000000000beanstalkd-1.13/.github/workflows/build-latest.yaml000066400000000000000000000014051440333440000224130ustar00rootroot00000000000000--- name: Continous integration on: push: branches: - master jobs: tests: name: Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Tests run: make check build: name: Docker build runs-on: ubuntu-latest needs: - tests steps: - uses: actions/checkout@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - run: docker build . -t `echo ghcr.io/${{ github.repository }}:latest | tr '[:upper:]' '[:lower:]'` - run: docker push `echo ghcr.io/${{ github.repository }}:latest | tr '[:upper:]' '[:lower:]'` beanstalkd-1.13/.github/workflows/prs.yaml000066400000000000000000000003531440333440000206270ustar00rootroot00000000000000--- name: Checks for PRs on: pull_request: push: branches-ignore: - master jobs: tests: name: Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Tests run: make checkbeanstalkd-1.13/.github/workflows/test-clients.yaml000066400000000000000000000061471440333440000224500ustar00rootroot00000000000000name: Testing clients on: - push - pull_request jobs: build: name: Build beanstalkd runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - run: make - uses: actions/upload-artifact@v3 with: name: beanstalkd path: beanstalkd pheanstalk: name: Test Pheanstalk PHP runs-on: ubuntu-latest continue-on-error: true needs: - build steps: - name: Set up PHP uses: shivammathur/setup-php@v2 with: php-version: '8.1' tools: phpunit - uses: actions/download-artifact@v3 with: name: beanstalkd - name: Start beanstalkd run: chmod +x ./beanstalkd && ./beanstalkd & - uses: actions/checkout@v3 with: repository: pheanstalk/pheanstalk ref: v5 - name: Install dependencies including dev-dependencies run: composer install - name: Run tests run: phpunit env: SERVER_HOST: localhost greenstalk: # Disable this test suite until https://github.com/justinmayhew/greenstalk/issues/9 is fixed if: ${{ false }} name: Test Greenstalk Python runs-on: ubuntu-latest continue-on-error: true strategy: matrix: python-version: [ 3.9 ] needs: - build steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - uses: actions/checkout@v3 with: repository: justinmayhew/greenstalk - uses: actions/download-artifact@v3 id: download with: name: beanstalkd - name: Make beanstalkd executable run: chmod +x ${{steps.download.outputs.download-path}}/beanstalkd - name: Run beanstalkd -v run: ${{steps.download.outputs.download-path}}/beanstalkd -v - name: Install dependencies run: | pip install pytest pip install . - name: Run tests env: BEANSTALKD_PATH: ${{steps.download.outputs.download-path}}/beanstalkd run: make test pystalk: name: Test Pystalk Python runs-on: ubuntu-latest continue-on-error: true strategy: matrix: python-version: [ "3.10" ] needs: - build steps: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - uses: actions/checkout@v3 with: repository: EasyPost/pystalk - uses: actions/download-artifact@v3 id: download with: name: beanstalkd - name: Make beanstalkd executable run: chmod +x ${{steps.download.outputs.download-path}}/beanstalkd - name: Run beanstalkd -v run: ${{steps.download.outputs.download-path}}/beanstalkd -v - name: Install dependencies run: python -m pip install -r requirements-tests.txt -e . - name: Run tests env: BEANSTALKD_PATH: ${{steps.download.outputs.download-path}}/beanstalkd run: pytest tests/ beanstalkd-1.13/.gitignore000066400000000000000000000000541440333440000155300ustar00rootroot00000000000000*.gcda *.gcno *.o /vers.c /beanstalkd /News beanstalkd-1.13/.travis.yml000066400000000000000000000035121440333440000156530ustar00rootroot00000000000000branches: only: - master git: quiet: true depth: 5 language: c addons: apt: packages: - lcov compiler: - clang - gcc os: - linux - osx env: global: - MAKEJOBS="-j$(getconf _NPROCESSORS_ONLN)" - TRAVIS_COMMIT_LOG=`git log --format=fuller -2` - COVERAGE=OFF # Currently works only with gcc on linux before_script: - '[[ "${TRAVIS_OS_NAME}-${TRAVIS_COMPILER}" != "linux-gcc" ]] || export COVERAGE="ON"' script: - | if [ $COVERAGE = "ON" ]; then export LDFLAGS=" -lgcov --coverage" export CFLAGS="-O0 -ggdb -fprofile-arcs -ftest-coverage" fi - make $MAKEJOBS || ( echo "Build failure. Verbose build follows." && make V=1 ; false ) - make check -j1 VERBOSE=1 jobs: include: - stage: Benchmark os: linux compiler: gcc before_script: - export COVERAGE=OFF script: - make $MAKEJOBS - make $MAKEJOBS bench - stage: Benchmark os: linux compiler: gcc env: - CFLAGS="-march=native -mtune=native -O3" before_script: - export COVERAGE=OFF script: - make $MAKEJOBS - make $MAKEJOBS bench - stage: Benchmark os: osx compiler: clang env: - CFLAGS="-march=native -mtune=native -O3" before_script: - export COVERAGE=OFF script: - make $MAKEJOBS - make $MAKEJOBS bench - stage: Benchmark os: osx compiler: clang before_script: - export COVERAGE=OFF script: - make $MAKEJOBS - make $MAKEJOBS bench after_success: - '[[ "$COVERAGE" != "ON" ]] || bash <(curl -s https://codecov.io/bash)' after_script: - printf "$TRAVIS_COMMIT_RANGE\n" - printf "$TRAVIS_COMMIT_LOG\n" beanstalkd-1.13/CHANGELOG.md000066400000000000000000000022031440333440000153470ustar00rootroot00000000000000# Changelog All notable changes to this project will be documented in this file. ## [Unreleased] ## [1.12] - 2020-06-04 - add support of UNIX domain sockets - add support of Solaris/illumos - add the "reserve-job" command - add draining status to the "stats" command - make fsync turned on by default when binlog is used: it's synced every 50ms instead of never - replace vendored systemd files with libsystemd - systemd usage can be controlled with USE_SYSTEMD=yes/no - specify C99 as required compiler ## [1.11] - 2019-06-29 - add automated testing via TravisCI - add System V init script - enable code coverage - misc. fixes and documentation improvements ## [1.10] - 2014-08-05 - fix crash on suspend or other EINTR (#220) - document touch command’s TTR reset (#188) - add some basic benchmark tests - add DESTDIR support to Makefile [unreleased]: https://github.com/beanstalkd/beanstalkd/compare/v1.12...HEAD [1.12]: https://github.com/beanstalkd/beanstalkd/compare/v1.11...v1.12 [1.11]: https://github.com/beanstalkd/beanstalkd/compare/v1.10...v1.11 [1.10]: https://github.com/beanstalkd/beanstalkd/compare/v1.9...v1.10 beanstalkd-1.13/CONTRIBUTING.md000066400000000000000000000031141440333440000157710ustar00rootroot00000000000000# Contributing to beanstalkd Greetings. Firstly, if you're thinking of contributing to beanstalkd, thank you! It's the hard work of people like you that keeps beanstalkd a high-quality codebase and running smoothly in the demanding, high-volume production environment of the servers of many organizations around the world. Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See CodeOfConduct.txt for details. ## General This is a mature project, so it rarely takes on new features. We mostly focus on stability, bug fixing, clarity, and performance, in that order. ## Issues When reporting a bug, please describe: - which version of beanstalkd you're using - steps to reproduce the bug - the behavior you saw - the behavior you expected If you're not using the latest version, please consider also testing with the latest. There's a good chance the bug you found has already been fixed. ## Good commit messages Please see how to write good commit messages in the Go contributing guide [here](https://golang.org/doc/contribute.html#commit_messages). ## Pull Requests When opening a pull request, try to keep the changes focused on one topic and avoid unrelated changes (even small things, like editing punctuation or whitespace in comments). If you're making big changes, consider discussing it on the mailing list first. You might save yourself a lot of time if it turns out that the changes you want to make aren't a good fit for the project. This is especially true if you are adding new functionality. beanstalkd-1.13/CodeOfConduct.txt000066400000000000000000000063461440333440000167720ustar00rootroot00000000000000Contributor Covenant Code of Conduct Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Our Standards Examples of behavior that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behavior by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others’ private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. Scope This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at beanstalk-team@googlegroups.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. Attribution This Code of Conduct is adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq beanstalkd-1.13/Dockerfile000066400000000000000000000004451440333440000155360ustar00rootroot00000000000000ARG BASE=alpine FROM alpine as builder RUN apk add --no-cache build-base git COPY . /tmp/beanstalkd RUN cd /tmp/beanstalkd && make ################################ ARG BASE FROM ${BASE} COPY --from=builder /tmp/beanstalkd/beanstalkd /usr/bin/ EXPOSE 11300 ENTRYPOINT ["/usr/bin/beanstalkd"] beanstalkd-1.13/LICENSE000066400000000000000000000022331440333440000145460ustar00rootroot00000000000000Copyright (c) 2007 The authors of beanstalkd. Copyright in contributions to beanstalkd is retained by the original copyright holder of each contribution. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. beanstalkd-1.13/Makefile000066400000000000000000000037211440333440000152040ustar00rootroot00000000000000PREFIX?=/usr/local BINDIR=$(DESTDIR)$(PREFIX)/bin override CFLAGS+=-Wall -Werror -Wformat=2 -g override LDFLAGS?= LDLIBS?= OS?=$(shell uname | tr 'A-Z' 'a-z') INSTALL?=install PKG_CONFIG?=pkg-config ifeq ($(OS),sunos) override LDFLAGS += -lxnet -lsocket -lnsl endif VERS=$(shell ./vers.sh) TARG=beanstalkd MOFILE=main.o OFILES=\ $(OS).o\ conn.o\ file.o\ heap.o\ job.o\ ms.o\ net.o\ primes.o\ prot.o\ serv.o\ time.o\ tube.o\ util.o\ vers.o\ walg.o\ TOFILES=\ testheap.o\ testjobs.o\ testms.o\ testserv.o\ testutil.o\ HFILES=\ dat.h\ ifeq ($(OS),linux) LDLIBS+=-lrt endif # systemd support can be configured via USE_SYSTEMD: # no: disabled # yes: enabled, build fails if libsystemd is not found # otherwise: enabled if libsystemd is found ifneq ($(USE_SYSTEMD),no) ifeq ($(shell $(PKG_CONFIG) --exists libsystemd && echo $$?),0) LDLIBS+=$(shell $(PKG_CONFIG) --libs libsystemd) CPPFLAGS+=-DHAVE_LIBSYSTEMD else ifeq ($(USE_SYSTEMD),yes) $(error USE_SYSTEMD is set to "$(USE_SYSTEMD)", but $(PKG_CONFIG) cannot find libsystemd) endif endif endif CLEANFILES=\ vers.c\ $(wildcard *.gc*) .PHONY: all all: $(TARG) $(TARG): $(OFILES) $(MOFILE) $(LINK.o) -o $@ $^ $(LDLIBS) .PHONY: install install: $(BINDIR)/$(TARG) $(BINDIR)/%: % $(INSTALL) -d $(dir $@) $(INSTALL) $< $@ CLEANFILES+=$(TARG) $(OFILES) $(MOFILE): $(HFILES) CLEANFILES+=$(wildcard *.o) .PHONY: clean clean: rm -f $(CLEANFILES) .PHONY: check check: ct/_ctcheck ct/_ctcheck .PHONY: bench bench: ct/_ctcheck ct/_ctcheck -b ct/_ctcheck: ct/_ctcheck.o ct/ct.o $(OFILES) $(TOFILES) ct/_ctcheck.c: $(TOFILES) ct/gen ct/gen $(TOFILES) >$@.part mv $@.part $@ ct/ct.o ct/_ctcheck.o: ct/ct.h ct/internal.h $(TOFILES): $(HFILES) ct/ct.h CLEANFILES+=$(wildcard ct/_* ct/*.o ct/*.gc*) ifneq ($(shell ./verc.sh),$(shell cat vers.c 2>/dev/null)) .PHONY: vers.c endif vers.c: ./verc.sh >vers.c doc/beanstalkd.1 doc/beanstalkd.1.html: doc/beanstalkd.ronn ronn $< freebsd.o: darwin.c beanstalkd-1.13/News000066400000000000000000000025641440333440000144070ustar00rootroot00000000000000This is a maintenance release. The list of merged PRs: Add Missing Stats (#643) Add pystalk client (#638) ci: run tests of some beanstalk clients (#630) ci: add a workflow for running tests on PRs or pushes to non-master branch (#635) ci: update actions to their latest versions (#634) Fix test regression (#632) set up CI using GH actions (#613) Update beanstalkd.service (#623) Quote string values in yaml dictionaries (#610) (#611) clean up Dockerfile and support dynamic base image via build args (#616) use read_u32 to safely read reserve-with-timeout argument (#607) fix pause-tube parameter (#604) refactor code using tube_find and remove_ready_job functions (#600) testserv: fix endianness issue (#594) Make sure to quote os value in stats cmd (#592) add dockerfile (#586) exit when SIGTERM is received and pid is 1 (#585) server: remove redundant listen (#583) Updates changelog with 1.12 release (#584) Full list of changes (includes authorship information): Our Urls -------- Download the 1.13 tarball directly: Learn all about beanstalk: Talk about beanstalk development or use at: Bugs ---- Please report any bugs to: beanstalkd-1.13/README.md000066400000000000000000000027061440333440000150250ustar00rootroot00000000000000[![Build Status](https://github.com/beanstalkd/beanstalkd/actions/workflows/build-latest.yaml/badge.svg)](https://github.com/beanstalkd/beanstalkd/actions/workflows/build-latest.yaml) [![codecov](https://codecov.io/gh/beanstalkd/beanstalkd/branch/master/graph/badge.svg)](https://codecov.io/gh/beanstalkd/beanstalkd) # beanstalkd Simple and fast general purpose work queue. https://beanstalkd.github.io/ See [doc/protocol.txt](https://github.com/beanstalkd/beanstalkd/blob/master/doc/protocol.txt) for details of the network protocol. Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See CodeOfConduct.txt for details. ## Quick Start $ make $ ./beanstalkd also try, $ ./beanstalkd -h $ ./beanstalkd -VVV $ make CFLAGS=-O2 $ make CC=clang $ make check $ make install $ make install PREFIX=/usr Requires Linux (2.6.17 or later), Mac OS X, FreeBSD, or Illumos. Currently beanstalkd is tested with GCC and clang, but it should work with any compiler that supports C99. Uses ronn to generate the manual. See http://github.com/rtomayko/ronn. ## Subdirectories - `adm` - files useful for system administrators - `ct` - testing tool; vendored from https://github.com/kr/ct - `doc` - documentation - `pkg` - scripts to make releases ## Tests Unit tests are in test*.c. See https://github.com/kr/ct for information on how to write them. beanstalkd-1.13/adm/000077500000000000000000000000001440333440000143025ustar00rootroot00000000000000beanstalkd-1.13/adm/Readme000066400000000000000000000030101440333440000154140ustar00rootroot00000000000000The usual way to run beanstalkd is to type its name in a Unix shell prompt, like this: $ beanstalkd This will start up the process and give you control over it. You can control its output (by default output is printed to the screen; you can arrange to have output go into file b.log by typing ">b.log" at the end of the command line), pause and restart the process (by pressing Control-Z and typing "fg"), and kill it (by pressing Control-C). This is most convenient while writing programs that use beanstalkd (or when working on beanstalkd itself), since you might want to start and stop it many times and regularly inspect its output. If you want beanstalkd to start when your operating system boots, the mechanism varies. Traditionally, you must add a command line to the shell script in /etc/rc (which is read by init when the system boots), using the "&" notation to run beanstalkd in the background. This would suffice for most situations, but it isn't always possible. These days, many popular operating systems have a replacement init program with its own configuration language. Example configuration files for several of these are included in subdirectories here, but the most common is probably "System V init", which reads /etc/inittab for lines describing commands to run at various times. If this file exists, you can add a line something like bean:345:respawn:su nobody -c 'exec /usr/bin/beanstalkd' and type "telinit q" to tell init to reread its configuration. Type "man 5 inittab" for details of this notation. beanstalkd-1.13/adm/launchd/000077500000000000000000000000001440333440000157205ustar00rootroot00000000000000beanstalkd-1.13/adm/launchd/beanstalkd.plist000066400000000000000000000007621440333440000211120ustar00rootroot00000000000000 Label beanstalkd UserName nobody ProgramArguments /usr/local/bin/beanstalkd KeepAlive beanstalkd-1.13/adm/systemd/000077500000000000000000000000001440333440000157725ustar00rootroot00000000000000beanstalkd-1.13/adm/systemd/beanstalkd.service000066400000000000000000000002261440333440000214640ustar00rootroot00000000000000[Unit] Description=Beanstalkd is a simple, fast work queue [Service] User=nobody ExecStart=/usr/bin/beanstalkd [Install] WantedBy=multi-user.target beanstalkd-1.13/adm/systemd/beanstalkd.socket000066400000000000000000000002041440333440000213100ustar00rootroot00000000000000[Unit] Description=Socket for beanstalkd, a simple, fast work queue [Socket] ListenStream=11300 [Install] WantedBy=sockets.target beanstalkd-1.13/adm/systemv/000077500000000000000000000000001440333440000160145ustar00rootroot00000000000000beanstalkd-1.13/adm/systemv/beanstalkd.init000077500000000000000000000051371440333440000210220ustar00rootroot00000000000000#!/bin/sh # # System V init script in charge of starting/stopping beanstalkd # # chkconfig: - 57 47 # description: beanstalkd is a simple, fast work queue # processname: beanstalkd # config: /etc/sysconfig/beanstalkd # pidfile: /var/run/beanstalkd/beanstalkd.pid ### BEGIN INIT INFO # Provides: beanstalkd # Required-Start: $local_fs $network $remote_fs # Required-Stop: $local_fs $network $remote_fs # Default-Stop: 0 1 2 6 # Short-Description: start and stop beanstalkd # Description: beanstalkd is a simple, fast work queue ### END INIT INFO # Source function library . /etc/rc.d/init.d/functions # Source networking configuration . /etc/sysconfig/network # Check that networking is up [ "$NETWORKING" = "no" ] && exit exec="/usr/bin/beanstalkd" prog=$(basename $exec) # Default options, overruled by items in sysconfig BEANSTALKD_ADDR=127.0.0.1 BEANSTALKD_PORT=11300 BEANSTALKD_USER=beanstalkd [ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog lockfile=/var/lock/subsys/$prog start() { [ -x $exec ] || exit 5 echo -n $"Starting $prog: " options="-l $BEANSTALKD_ADDR -p $BEANSTALKD_PORT -u $BEANSTALKD_USER" [ -n "$BEANSTALKD_MAX_JOB_SIZE" ] && options="$options -z $BEANSTALKD_MAX_JOB_SIZE" if [ -n "$BEANSTALKD_BINLOG_DIR" ]; then if [ ! -d "$BEANSTALKD_BINLOG_DIR" ]; then echo "Creating binlog directory ($BEANSTALKD_BINLOG_DIR)" mkdir -p $BEANSTALKD_BINLOG_DIR chown $BEANSTALKD_USER:$BEANSTALKD_USER $BEANSTALKD_BINLOG_DIR fi options="$options -b $BEANSTALKD_BINLOG_DIR" if [ -n "$BEANSTALKD_BINLOG_FSYNC_PERIOD" ]; then options="$options -f $BEANSTALKD_BINLOG_FSYNC_PERIOD" else options="$options -F" fi if [ -n "$BEANSTALKD_BINLOG_SIZE" ]; then options="$options -s $BEANSTALKD_BINLOG_SIZE" fi fi daemon "nohup $exec $options > /dev/null 2>&1 &" retval=$? echo [ $retval -eq 0 ] && touch $lockfile return $retval } stop() { echo -n $"Stopping $prog: " killproc $prog retval=$? echo [ $retval -eq 0 ] && rm -f $lockfile return $retval } restart() { stop start } reload() { restart } force_reload() { restart } rh_status() { # Run checks to determine if the service is running or use generic status status $prog } rh_status_q() { rh_status >/dev/null 2>&1 } case "$1" in start) rh_status_q && exit 0 $1 ;; stop) rh_status_q || exit 0 $1 ;; restart) $1 ;; reload) rh_status_q || exit 7 $1 ;; force-reload) force_reload ;; status) rh_status ;; condrestart|try-restart) rh_status_q || exit 0 restart ;; *) echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" exit 2 esac exit $? beanstalkd-1.13/adm/upstart/000077500000000000000000000000001440333440000160045ustar00rootroot00000000000000beanstalkd-1.13/adm/upstart/beanstalkd.conf000066400000000000000000000002301440333440000207560ustar00rootroot00000000000000description "simple, fast work queue" start on filesystem stop on runlevel [!2345] respawn respawn limit 5 2 setuid nobody exec /usr/bin/beanstalkd beanstalkd-1.13/beanstalkd.spec000066400000000000000000000047321440333440000165330ustar00rootroot00000000000000%define beanstalkd_user beanstalkd %define beanstalkd_group %{beanstalkd_user} %define beanstalkd_home %{_localstatedir}/lib/beanstalkd %define beanstalkd_logdir %{_localstatedir}/log/beanstalkd Name: beanstalkd Version: 1.13 Release: 0%{?dist} Summary: A simple, fast workqueue service Group: System Environment/Daemons License: GPLv3+ URL: http://xph.us/software/%{name}/ Source0: http://xph.us/dist/%{name}/rel/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires(pre): %{_sbindir}/useradd Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %description Beanstalk is a simple, fast workqueue service. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. %prep %setup -q if [ ! -e configure ]; then sh buildconf.sh fi %build %configure --disable-rpath --docdir=%{_defaultdocdir}/%{name}-%{version} make %{?_smp_mflags} %install rm -rf $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT %{__install} -p -D -m 0644 doc/%{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} %{__install} -p -D -m 0644 scripts/%{name}.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} %clean rm -rf $RPM_BUILD_ROOT %pre %{_sbindir}/useradd -c "beanstalkd user" -s /bin/false -r -m -d %{beanstalkd_home} %{beanstalkd_user} 2>/dev/null || : %post /sbin/chkconfig --add %{name} %preun if [ $1 = 0 ]; then /sbin/service %{name} stop >/dev/null 2>&1 /sbin/chkconfig --del %{name} fi %postun if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi %files %defattr(-,root,root,-) %doc %{_defaultdocdir}/%{name}-%{version}/protocol.txt %doc README COPYING doc/protocol.txt %{_initrddir}/%{name} %{_bindir}/%{name} %{_mandir}/man1/%{name}.1.gz %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %changelog * Thu Oct 1 2009 Keith Rarick - 1.4-0 - Convert this file to an autoconf template - Tweak the summary and description * Sun Jan 4 2009 Ask Bjørn Hansen - 1.2-0 - 1.2-tobe - Use man page and .init/sysconfig scripts from .tar.gz * Sat Nov 22 2008 Jeremy Hinegardner - 1.1-1 - initial spec creation beanstalkd-1.13/conn.c000066400000000000000000000122211440333440000146400ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #define SAFETY_MARGIN (1000000000) /* 1 second */ static int cur_conn_ct = 0, cur_worker_ct = 0, cur_producer_ct = 0; static uint tot_conn_ct = 0; int verbose = 0; static void on_watch(Ms *a, Tube *t, size_t i) { UNUSED_PARAMETER(a); UNUSED_PARAMETER(i); tube_iref(t); t->watching_ct++; } static void on_ignore(Ms *a, Tube *t, size_t i) { UNUSED_PARAMETER(a); UNUSED_PARAMETER(i); t->watching_ct--; tube_dref(t); } Conn * make_conn(int fd, char start_state, Tube *use, Tube *watch) { Conn *c = new(Conn); if (!c) { twarn("OOM"); return NULL; } ms_init(&c->watch, (ms_event_fn) on_watch, (ms_event_fn) on_ignore); if (!ms_append(&c->watch, watch)) { free(c); twarn("OOM"); return NULL; } TUBE_ASSIGN(c->use, use); use->using_ct++; c->state = start_state; c->pending_timeout = -1; c->tickpos = 0; // Does not mean anything if in_conns is set to 0. c->in_conns = 0; // The list is empty. job_list_reset(&c->reserved_jobs); /* stats */ cur_conn_ct++; tot_conn_ct++; return c; } void connsetproducer(Conn *c) { if (c->type & CONN_TYPE_PRODUCER) return; c->type |= CONN_TYPE_PRODUCER; cur_producer_ct++; /* stats */ } void connsetworker(Conn *c) { if (c->type & CONN_TYPE_WORKER) return; c->type |= CONN_TYPE_WORKER; cur_worker_ct++; /* stats */ } int count_cur_conns() { return cur_conn_ct; } uint count_tot_conns() { return tot_conn_ct; } int count_cur_producers() { return cur_producer_ct; } int count_cur_workers() { return cur_worker_ct; } static int has_reserved_job(Conn *c) { return !job_list_is_empty(&c->reserved_jobs); } // Returns positive nanoseconds when c should tick, 0 otherwise. static int64 conntickat(Conn *c) { int margin = 0, should_timeout = 0; int64 t = INT64_MAX; if (conn_waiting(c)) { margin = SAFETY_MARGIN; } if (has_reserved_job(c)) { t = connsoonestjob(c)->r.deadline_at - nanoseconds() - margin; should_timeout = 1; } if (c->pending_timeout >= 0) { t = min(t, ((int64)c->pending_timeout) * 1000000000); should_timeout = 1; } if (should_timeout) { return nanoseconds() + t; } return 0; } // Remove c from the c->srv heap and reschedule it using the value // returned by conntickat if there is an outstanding timeout in the c. void connsched(Conn *c) { if (c->in_conns) { heapremove(&c->srv->conns, c->tickpos); c->in_conns = 0; } c->tickat = conntickat(c); if (c->tickat) { heapinsert(&c->srv->conns, c); c->in_conns = 1; } } // conn_set_soonestjob updates c->soonest_job with j // if j should be handled sooner than c->soonest_job. static void conn_set_soonestjob(Conn *c, Job *j) { if (!c->soonest_job || j->r.deadline_at < c->soonest_job->r.deadline_at) { c->soonest_job = j; } } // Return the reserved job with the earliest deadline, // or NULL if there's no reserved job. Job * connsoonestjob(Conn *c) { // use cached value and bail out. if (c->soonest_job != NULL) return c->soonest_job; Job *j = NULL; for (j = c->reserved_jobs.next; j != &c->reserved_jobs; j = j->next) { conn_set_soonestjob(c, j); } return c->soonest_job; } void conn_reserve_job(Conn *c, Job *j) { j->tube->stat.reserved_ct++; j->r.reserve_ct++; j->r.deadline_at = nanoseconds() + j->r.ttr; j->r.state = Reserved; job_list_insert(&c->reserved_jobs, j); j->reserver = c; c->pending_timeout = -1; conn_set_soonestjob(c, j); } // Return true if c has a reserved job with less than one second until its // deadline. int conndeadlinesoon(Conn *c) { int64 t = nanoseconds(); Job *j = connsoonestjob(c); return j && t >= j->r.deadline_at - SAFETY_MARGIN; } int conn_ready(Conn *c) { size_t i; for (i = 0; i < c->watch.len; i++) { if (((Tube *) c->watch.items[i])->ready.len) return 1; } return 0; } int conn_less(void *ca, void *cb) { Conn *a = (Conn *)ca; Conn *b = (Conn *)cb; return a->tickat < b->tickat; } void conn_setpos(void *c, size_t i) { ((Conn *)c)->tickpos = i; } void connclose(Conn *c) { sockwant(&c->sock, 0); close(c->sock.fd); if (verbose) { printf("close %d\n", c->sock.fd); } job_free(c->in_job); /* was this a peek or stats command? */ if (c->out_job && c->out_job->r.state == Copy) job_free(c->out_job); c->in_job = c->out_job = NULL; c->in_job_read = 0; if (c->type & CONN_TYPE_PRODUCER) cur_producer_ct--; /* stats */ if (c->type & CONN_TYPE_WORKER) cur_worker_ct--; /* stats */ cur_conn_ct--; /* stats */ remove_waiting_conn(c); if (has_reserved_job(c)) enqueue_reserved_jobs(c); ms_clear(&c->watch); c->use->using_ct--; TUBE_ASSIGN(c->use, NULL); if (c->in_conns) { heapremove(&c->srv->conns, c->tickpos); c->in_conns = 0; } free(c); } beanstalkd-1.13/ct/000077500000000000000000000000001440333440000141475ustar00rootroot00000000000000beanstalkd-1.13/ct/.gitignore000066400000000000000000000000111440333440000161270ustar00rootroot00000000000000/*.o /_* beanstalkd-1.13/ct/License000066400000000000000000000020471440333440000154570ustar00rootroot00000000000000Copyright © 2010–2013 Keith Rarick Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. beanstalkd-1.13/ct/ct.c000066400000000000000000000253101440333440000147220ustar00rootroot00000000000000/* CT - simple-minded unit testing for C */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "internal.h" #include "ct.h" static char *curdir; static int rjobfd = -1, wjobfd = -1; int fail = 0; /* bool */ static int64 bstart, bdur; static int btiming; /* bool */ static int64 bbytes; enum { Second = 1000 * 1000 * 1000 }; enum { BenchTime = Second }; enum { MaxN = 1000 * 1000 * 1000 }; #ifdef __MACH__ # include static int64 nstime() { return (int64)mach_absolute_time(); } #else # include static int64 nstime() { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return (int64)(t.tv_sec)*Second + t.tv_nsec; } #endif void ctlogpn(const char *p, int n, const char *fmt, ...) { va_list arg; printf("%s:%d: ", p, n); va_start(arg, fmt); vprintf(fmt, arg); va_end(arg); putchar('\n'); } void ctfail(void) { fail = 1; } void ctfailnow(void) { fflush(NULL); abort(); } char * ctdir(void) { return curdir; } void ctresettimer(void) { bdur = 0; bstart = nstime(); } void ctstarttimer(void) { if (!btiming) { bstart = nstime(); btiming = 1; } } void ctstoptimer(void) { if (btiming) { bdur += nstime() - bstart; btiming = 0; } } void ctsetbytes(int n) { bbytes = (int64)n; } static void die(int code, int err, const char *msg) { putc('\n', stderr); if (msg && *msg) { fputs(msg, stderr); fputs(": ", stderr); } fputs(strerror(err), stderr); putc('\n', stderr); exit(code); } static int tmpfd(void) { FILE *f = tmpfile(); if (!f) { die(1, errno, "tmpfile"); } return fileno(f); } static int failed(int s) { return WIFSIGNALED(s) && (WTERMSIG(s) == SIGABRT); } static void waittest(Test *ts) { Test *t; int pid, stat; pid = wait3(&stat, 0, 0); if (pid == -1) { die(3, errno, "wait"); } killpg(pid, SIGKILL); for (t=ts; t->f; t++) { if (t->pid == pid) { t->status = stat; if (!t->status) { putchar('.'); } else if (failed(t->status)) { putchar('F'); } else { putchar('E'); } fflush(stdout); } } } static void start(Test *t) { t->fd = tmpfd(); strcpy(t->dir, TmpDirPat); if (mkdtemp(t->dir) == NULL) { die(1, errno, "mkdtemp"); } fflush(NULL); t->pid = fork(); if (t->pid < 0) { die(1, errno, "fork"); } else if (!t->pid) { setpgid(0, 0); if (dup2(t->fd, 1) == -1) { die(3, errno, "dup2"); } if (close(t->fd) == -1) { die(3, errno, "fclose"); } if (dup2(1, 2) == -1) { die(3, errno, "dup2"); } curdir = t->dir; t->f(); if (fail) { ctfailnow(); } exit(0); } setpgid(t->pid, t->pid); } static void runalltest(Test *ts, int limit) { int nrun = 0; Test *t; for (t=ts; t->f; t++) { if (nrun >= limit) { waittest(ts); nrun--; } start(t); nrun++; } for (; nrun; nrun--) { waittest(ts); } } static void copyfd(FILE *out, int in) { ssize_t n; char buf[1024]; /* arbitrary size */ while ((n = read(in, buf, sizeof(buf))) != 0) { if (fwrite(buf, 1, n, out) != (size_t)n) { die(3, errno, "fwrite"); } } } /* Removes path and all of its children. Writes errors to stderr and keeps going. If path doesn't exist, rmtree returns silently. */ static void rmtree(char *path) { int r = unlink(path); if (r == 0 || errno == ENOENT) { return; /* success */ } int unlinkerr = errno; DIR *d = opendir(path); if (!d) { if (errno == ENOTDIR) { fprintf(stderr, "ct: unlink: %s\n", strerror(unlinkerr)); } else { perror("ct: opendir"); } fprintf(stderr, "ct: path %s\n", path); return; } struct dirent *ent; while ((ent = readdir(d))) { if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { continue; } int n = strlen(path) + 1 + strlen(ent->d_name); char s[n+1]; sprintf(s, "%s/%s", path, ent->d_name); rmtree(s); } closedir(d); r = rmdir(path); if (r == -1) { perror("ct: rmdir"); fprintf(stderr, "ct: path %s\n", path); } } static void runbenchn(Benchmark *b, int n) { int outfd = tmpfd(); int durfd = tmpfd(); strcpy(b->dir, TmpDirPat); if (mkdtemp(b->dir) == NULL) { die(1, errno, "mkdtemp"); } fflush(NULL); int pid = fork(); if (pid < 0) { die(1, errno, "fork"); } else if (!pid) { setpgid(0, 0); if (dup2(outfd, 1) == -1) { die(3, errno, "dup2"); } if (close(outfd) == -1) { die(3, errno, "fclose"); } if (dup2(1, 2) == -1) { die(3, errno, "dup2"); } curdir = b->dir; ctstarttimer(); b->f(n); ctstoptimer(); if (write(durfd, &bdur, sizeof bdur) != sizeof bdur) { die(3, errno, "write"); } if (write(durfd, &bbytes, sizeof bbytes) != sizeof bbytes) { die(3, errno, "write"); } exit(0); } setpgid(pid, pid); pid = waitpid(pid, &b->status, 0); if (pid == -1) { die(3, errno, "wait"); } killpg(pid, SIGKILL); rmtree(b->dir); if (b->status != 0) { putchar('\n'); lseek(outfd, 0, SEEK_SET); copyfd(stdout, outfd); return; } lseek(durfd, 0, SEEK_SET); int r = read(durfd, &b->dur, sizeof b->dur); if (r != sizeof b->dur) { perror("read"); b->status = 1; } r = read(durfd, &b->bytes, sizeof b->bytes); if (r != sizeof b->bytes) { perror("read"); b->status = 1; } } /* rounddown10 rounds a number down to the nearest power of 10. */ static int rounddown10(int n) { int tens = 0; /* tens = floor(log_10(n)) */ while (n >= 10) { n = n / 10; tens++; } /* result = 10**tens */ int i, result = 1; for (i = 0; i < tens; i++) { result *= 10; } return result; } /* roundup rounds n up to a number of the form [1eX, 2eX, 5eX]. */ static int roundup(int n) { int base = rounddown10(n); if (n <= base) return base; if (n <= 2*base) return 2*base; if (n <= 3*base) return 3*base; if (n <= 5*base) return 5*base; return 10*base; } static int min(int a, int b) { if (a < b) { return a; } return b; } static int max(int a, int b) { if (a > b) { return a; } return b; } static void runbench(Benchmark *b) { printf("%s\t", b->name); fflush(stdout); int n = 1; runbenchn(b, n); while (b->status == 0 && b->dur < BenchTime && n < MaxN) { int last = n; /* Predict iterations/sec. */ int nsop = b->dur / n; if (nsop == 0) { n = MaxN; } else { n = BenchTime / nsop; } /* Run more iterations than we think we'll need for a second (1.2x). Don't grow too fast in case we had timing errors previously. Be sure to run at least one more than last time. */ n = max(min(n+n/5, 100*last), last+1); /* Round up to something easy to read. */ n = roundup(n); runbenchn(b, n); } if (b->status == 0) { printf("%8d\t%10" PRId64 " ns/op", n, b->dur/n); if (b->bytes > 0) { double mbs = 0; if (b->dur > 0) { int64 sec = b->dur / 1000L / 1000L / 1000L; int64 nsec = b->dur % 1000000000L; double dur = (double)sec + (double)nsec*.0000000001; mbs = ((double)b->bytes * (double)n / 1000000) / dur; } printf("\t%7.2f MB/s", mbs); } putchar('\n'); } else { if (failed(b->status)) { printf("failure"); } else { printf("error"); if (WIFEXITED(b->status)) { printf(" (exit status %d)", WEXITSTATUS(b->status)); } if (WIFSIGNALED(b->status)) { printf(" (signal %d)", WTERMSIG(b->status)); } } putchar('\n'); } } static void runallbench(Benchmark *b) { for (; b->f; b++) { runbench(b); } } static int report(Test *t) { int nfail = 0, nerr = 0; putchar('\n'); for (; t->f; t++) { rmtree(t->dir); if (!t->status) { continue; } printf("\n%s: ", t->name); if (failed(t->status)) { nfail++; printf("failure"); } else { nerr++; printf("error"); if (WIFEXITED(t->status)) { printf(" (exit status %d)", WEXITSTATUS(t->status)); } if (WIFSIGNALED(t->status)) { printf(" (signal %d)", WTERMSIG(t->status)); } } putchar('\n'); lseek(t->fd, 0, SEEK_SET); copyfd(stdout, t->fd); } if (nfail || nerr) { printf("\n%d failures; %d errors.\n", nfail, nerr); } else { printf("\nPASS\n"); } return nfail || nerr; } static int readtokens() { int n = 1; char c, *s; char *v = getenv("MAKEFLAGS"); if (v == NULL) return n; if ((s = strstr(v, " --jobserver-fds="))) { rjobfd = (int)strtol(s+17, &s, 10); /* skip " --jobserver-fds=" */ wjobfd = (int)strtol(s+1, NULL, 10); /* skip comma */ } if (rjobfd >= 0) { fcntl(rjobfd, F_SETFL, fcntl(rjobfd, F_GETFL)|O_NONBLOCK); while (read(rjobfd, &c, 1) > 0) { n++; } } return n; } static void writetokens(int n) { char c = '+'; if (wjobfd >= 0) { fcntl(wjobfd, F_SETFL, fcntl(wjobfd, F_GETFL)|O_NONBLOCK); for (; n>1; n--) { if (write(wjobfd, &c, 1) != 1) { /* ignore error; nothing we can do anyway */ } } } } int main(int argc, char **argv) { int n = readtokens(); runalltest(ctmaintest, n); writetokens(n); int code = report(ctmaintest); if (code != 0) { return code; } if (argc == 2 && strcmp(argv[1], "-b") == 0) { runallbench(ctmainbench); } return 0; } beanstalkd-1.13/ct/ct.h000066400000000000000000000010051440333440000147220ustar00rootroot00000000000000char *ctdir(void); void ctfail(void); void ctfailnow(void); void ctresettimer(void); void ctstarttimer(void); void ctstoptimer(void); void ctsetbytes(int); void ctlogpn(const char*, int, const char*, ...) __attribute__((format(printf, 3, 4))); #define ctlog(...) ctlogpn(__FILE__, __LINE__, __VA_ARGS__) #define assert(x) do if (!(x)) {\ ctlog("%s", "test: " #x);\ ctfailnow();\ } while (0) #define assertf(x, ...) do if (!(x)) {\ ctlog("%s", "test: " #x);\ ctlog(__VA_ARGS__);\ ctfailnow();\ } while (0) beanstalkd-1.13/ct/gen000077500000000000000000000014311440333440000146450ustar00rootroot00000000000000#!/bin/sh set -e fixsyms() { if test "`uname -s|tr A-Z a-z`" = darwin then egrep -v [.] | egrep ^_ | sed s/^_// else cat fi } syms() { prefix=$1 shift for f in "$@" do nm $f done | cut -d ' ' -f 3 | fixsyms | egrep ^$prefix } ts=`syms cttest "$@" || true` bs=`syms ctbench "$@" || true` printf '#include \n' printf '#include "internal.h"\n' for t in $ts do printf 'void %s(void);\n' $t done for b in $bs do printf 'void %s(int);\n' $b done printf 'Test ctmaintest[] = {\n' for t in $ts do printf ' {%s, "%s", 0, 0, 0, TmpDirPat},\n' $t $t done printf ' {.f = 0},\n' printf '};\n' printf 'Benchmark ctmainbench[] = {\n' for b in $bs do printf ' {%s, "%s", 0, 0, 0, TmpDirPat},\n' $b $b done printf ' {.f = 0},\n' printf '};\n' beanstalkd-1.13/ct/internal.h000066400000000000000000000007551440333440000161430ustar00rootroot00000000000000/* include */ #define TmpDirPat "/tmp/ct.XXXXXX" typedef int64_t int64; typedef struct Test Test; typedef struct Benchmark Benchmark; struct Test { void (*f)(void); const char *name; int status; int fd; int pid; char dir[sizeof TmpDirPat]; }; struct Benchmark { void (*f)(int); const char *name; int status; int64 dur; int64 bytes; char dir[sizeof TmpDirPat]; }; extern Test ctmaintest[]; extern Benchmark ctmainbench[]; beanstalkd-1.13/darwin.c000066400000000000000000000034201440333440000151700ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #include enum { Infinity = 1 << 30 }; static int kq; int sockinit(void) { kq = kqueue(); if (kq == -1) { twarn("kqueue"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int n = 0; struct kevent evs[2] = {{0}}; struct kevent *ev = evs; struct timespec ts = {.tv_sec = 0, .tv_nsec = 0}; if (s->added) { ev->ident = s->fd; ev->filter = s->added; ev->flags = EV_DELETE; ev++; n++; } if (rw) { ev->ident = s->fd; switch (rw) { case 'r': ev->filter = EVFILT_READ; break; case 'w': ev->filter = EVFILT_WRITE; break; default: // check only for hangup ev->filter = EVFILT_READ; ev->fflags = NOTE_LOWAT; ev->data = Infinity; } ev->flags = EV_ADD; ev->udata = s; s->added = ev->filter; ev++; n++; } return kevent(kq, evs, n, NULL, 0, &ts); } int socknext(Socket **s, int64 timeout) { int r; struct kevent ev; static struct timespec ts; ts.tv_sec = timeout / 1000000000; ts.tv_nsec = timeout % 1000000000; r = kevent(kq, NULL, 0, &ev, 1, &ts); if (r == -1 && errno != EINTR) { twarn("kevent"); return -1; } if (r > 0) { *s = ev.udata; if (ev.flags & EV_EOF) { return 'h'; } switch (ev.filter) { case EVFILT_READ: return 'r'; case EVFILT_WRITE: return 'w'; } } return 0; } beanstalkd-1.13/dat.h000066400000000000000000000334721440333440000144730ustar00rootroot00000000000000#include #include typedef unsigned char uchar; typedef uchar byte; typedef unsigned int uint; typedef int32_t int32; typedef uint32_t uint32; typedef int64_t int64; typedef uint64_t uint64; typedef struct Ms Ms; typedef struct Job Job; typedef struct Tube Tube; typedef struct Conn Conn; typedef struct Heap Heap; typedef struct Jobrec Jobrec; typedef struct File File; typedef struct Socket Socket; typedef struct Server Server; typedef struct Wal Wal; typedef void(*Handle)(void*, int rw); typedef int(FAlloc)(int, int); // NUM_PRIMES is used in the jobs hashing. #if _LP64 #define NUM_PRIMES 48 #else #define NUM_PRIMES 19 #endif /* Some compilers (e.g. gcc on SmartOS) define NULL as 0. * This is allowed by the C standard, but is unhelpful when * using NULL in most pointer contexts with errors turned on. */ #if (defined(sun) || defined(__sun)) && (defined(__SVR4) || defined(__svr4__)) #ifdef NULL #undef NULL #endif #define NULL ((void*)0) #endif // The name of a tube cannot be longer than MAX_TUBE_NAME_LEN-1 #define MAX_TUBE_NAME_LEN 201 // A command can be at most LINE_BUF_SIZE chars, including "\r\n". This value // MUST be enough to hold the longest possible command ("pause-tube a{200} 4294967295\r\n") // or reply line ("USING a{200}\r\n"). #define LINE_BUF_SIZE (11 + MAX_TUBE_NAME_LEN + 12) #define min(a,b) ((a)<(b)?(a):(b)) // Jobs with priority less than URGENT_THRESHOLD are counted as urgent. #define URGENT_THRESHOLD 1024 // The default maximum job size. #define JOB_DATA_SIZE_LIMIT_DEFAULT ((1 << 16) - 1) // The maximum value that job_data_size_limit can be set to via "-z". // It could be up to INT32_MAX-2 (~2GB), but set it to 1024^3 (1GB). // The width is restricted by Jobrec.body_size that is int32. #define JOB_DATA_SIZE_LIMIT_MAX 1073741824 // The default value for the fsync (-f) parameter, milliseconds. #define DEFAULT_FSYNC_MS 50 // Use this macro to designate unused parameters in functions. #define UNUSED_PARAMETER(x) (void)(x) // version is defined in vers.c, see vers.sh for details. extern const char version[]; // verbose holds the count of -V parameters; it's a verbosity level. extern int verbose; extern struct Server srv; // Replaced by tests to simulate failures. extern FAlloc *falloc; // stats structure holds counters for operations, both globally and per tube. struct stats { uint64 urgent_ct; uint64 waiting_ct; uint64 buried_ct; uint64 reserved_ct; uint64 pause_ct; uint64 total_delete_ct; uint64 total_jobs_ct; }; // less_fn is used by the binary heap to determine the order of elements. typedef int(*less_fn)(void*, void*); // setpos_fn is used by the binary heap to record the new positions of elements // whenever they get moved or inserted. typedef void(*setpos_fn)(void*, size_t); struct Heap { size_t cap; // capacity of the heap size_t len; // amount of elements in the heap void **data; // actual elements less_fn less; setpos_fn setpos; }; int heapinsert(Heap *h, void *x); void* heapremove(Heap *h, size_t k); struct Socket { // Descriptor for the socket. int fd; // f can point to srvaccept or prothandle. Handle f; // x is passed as first parameter to f. void *x; // added value is platform dependend: on OSX it can be > 1. // Value of 1 - socket was already added to event notifications, // otherwise it is 0. int added; }; int sockinit(void); // sockwant updates event filter for the socket s. rw designates // the kind of event we should be notified about: // 'r' - read // 'w' - write // 'h' - hangup (closed connection) // 0 - ignore this socket int sockwant(Socket *s, int rw); // socknext waits for the next event at most timeout nanoseconds. // If event happens before timeout then s points to the corresponding socket, // and the kind of event is returned. In case of timeout, 0 is returned. int socknext(Socket **s, int64 timeout); // ms_event_fn is called with the element being inserted/removed and its position. typedef void(*ms_event_fn)(Ms *a, void *item, size_t i); // Resizable multiset struct Ms { size_t len; // amount of stored elements size_t cap; // capacity size_t last; // position of last taken element void **items; ms_event_fn oninsert; // called on insertion of an element ms_event_fn onremove; // called on removal of an element }; void ms_init(Ms *a, ms_event_fn oninsert, ms_event_fn onremove); void ms_clear(Ms *a); int ms_append(Ms *a, void *item); int ms_remove(Ms *a, void *item); int ms_contains(Ms *a, void *item); void *ms_take(Ms *a); enum // Jobrec.state { Invalid, Ready, Reserved, Buried, Delayed, Copy }; enum { Walver = 7 }; // If you modify Jobrec struct, you must increment Walver above. // // This workflow is expected: // 1. If any change needs to be made to the format, first increment Walver. // 2. If and only if this is the first such change since the last release: // a. Copy-paste relevant file-reading functions in file.c and // add the old version number to their names. For example, // if you are incrementing Walver from 7 to 8, copy readrec to readrec7. // (Currently, there is only one such function, readrec. But if // a future readrec calls other version-specific functions, // those will have to be copied too.) // 3. Add a switch case to fileread for the old version. // 4. Modify the current reading function (readrec) to reflect your change. // // Incrementing Walver for every change, even if not every version // will be released, is helpful even if it "wastes" version numbers. // It is a really easy thing to do and it means during development // you won't have to worry about misinterpreting the contents of a binlog // that you generated with a dev copy of beanstalkd. struct Jobrec { uint64 id; uint32 pri; int64 delay; int64 ttr; int32 body_size; int64 created_at; // deadline_at is a timestamp, in nsec, that points to: // * time when job will become ready for delayed job, // * time when TTR is about to expire for reserved job, // * undefined otherwise. int64 deadline_at; uint32 reserve_ct; uint32 timeout_ct; uint32 release_ct; uint32 bury_ct; uint32 kick_ct; byte state; }; struct Job { // persistent fields; these get written to the wal Jobrec r; // bookeeping fields; these are in-memory only char pad[6]; Tube *tube; Job *prev, *next; // linked list of jobs Job *ht_next; // Next job in a hash table list size_t heap_index; // where is this job in its current heap File *file; Job *fnext; Job *fprev; void *reserver; int walresv; int walused; char *body; // written separately to the wal }; struct Tube { uint refs; char name[MAX_TUBE_NAME_LEN]; Heap ready; Heap delay; Ms waiting_conns; // conns waiting for the job at this moment struct stats stat; uint using_ct; uint watching_ct; // pause is set to the duration of the current pause, otherwise 0, in nsec. int64 pause; // unpause_at is a timestamp when to unpause the tube, in nsec. int64 unpause_at; Job buried; // linked list header }; // Prints warning message on stderr in the format: // : FILE:LINE in FUNC: : #define twarn(...) __twarn(__VA_ARGS__, "") // Hack to quiet the compiler. When VA_ARGS in twarn() has one element, // e.g. twarn("OOM"), its replaced with __twarn("OOM", ""), // thus VA_ARGS is expanded to at least one element in warn(). #define __twarn(fmt, ...) \ warn("%s:%d in %s: " fmt "%s", __FILE__, __LINE__, __func__, __VA_ARGS__) // Prints warning message on stderr in the format: // : FILE:LINE in FUNC: #define twarnx(...) __twarnx(__VA_ARGS__, "") // See __twarn macro. #define __twarnx(fmt, ...) \ warnx("%s:%d in %s: " fmt "%s", __FILE__, __LINE__, __func__, __VA_ARGS__) void warn(const char *fmt, ...) __attribute__((format(printf, 1, 2))); void warnx(const char *fmt, ...) __attribute__((format(printf, 1, 2))); char* fmtalloc(char *fmt, ...) __attribute__((format(printf, 1, 2))); void* zalloc(int n); #define new(T) zalloc(sizeof(T)) void optparse(Server*, char**); extern const char *progname; int64 nanoseconds(void); int rawfalloc(int fd, int len); // Take ID for a jobs from next_id and allocate and store the job. #define make_job(pri,delay,ttr,body_size,tube) \ make_job_with_id(pri,delay,ttr,body_size,tube,0) Job *allocate_job(int body_size); Job *make_job_with_id(uint pri, int64 delay, int64 ttr, int body_size, Tube *tube, uint64 id); void job_free(Job *j); /* Lookup a job by job ID */ Job *job_find(uint64 job_id); /* the void* parameters are really job pointers */ void job_setpos(void *j, size_t pos); int job_pri_less(void *ja, void *jb); int job_delay_less(void *ja, void *jb); Job *job_copy(Job *j); const char * job_state(Job *j); void job_list_reset(Job *head); int job_list_is_empty(Job *head); Job *job_list_remove(Job *j); void job_list_insert(Job *head, Job *j); /* for unit tests */ size_t get_all_jobs_used(void); extern struct Ms tubes; Tube *make_tube(const char *name); void tube_dref(Tube *t); void tube_iref(Tube *t); Tube *tube_find(Ms *tubeset, const char *name); Tube *tube_find_or_make(const char *name); #define TUBE_ASSIGN(a,b) (tube_dref(a), (a) = (b), tube_iref(a)) Conn *make_conn(int fd, char start_state, Tube *use, Tube *watch); int count_cur_conns(void); uint count_tot_conns(void); int count_cur_producers(void); int count_cur_workers(void); extern size_t primes[]; extern size_t job_data_size_limit; void prot_init(void); int64 prottick(Server *s); void remove_waiting_conn(Conn *c); void enqueue_reserved_jobs(Conn *c); void enter_drain_mode(int sig); void h_accept(const int fd, const short which, Server *s); int prot_replay(Server *s, Job *list); int make_server_socket(char *host, char *port); // CONN_TYPE_* are bit masks used to track the type of connection. // A put command adds the PRODUCER type, "reserve*" adds the WORKER type. // If connection awaits for data, then it has WAITING type. #define CONN_TYPE_PRODUCER 1 #define CONN_TYPE_WORKER 2 #define CONN_TYPE_WAITING 4 struct Conn { Server *srv; Socket sock; char state; // see the STATE_* description char type; // combination of CONN_TYPE_* values Conn *next; // only used in epollq functions Tube *use; // tube currently in use int64 tickat; // time at which to do more work; determines pos in heap size_t tickpos; // position in srv->conns, stale when in_conns=0 byte in_conns; // 1 if the conn is in srv->conns heap, 0 otherwise Job *soonest_job;// memoization of the soonest job int rw; // currently want: 'r', 'w', or 'h' // How long client should "wait" for the next job; -1 means forever. int pending_timeout; // Used to inform state machine that client no longer waits for the data. char halfclosed; char cmd[LINE_BUF_SIZE]; // this string is NOT NUL-terminated size_t cmd_len; int cmd_read; char *reply; int reply_len; int reply_sent; char reply_buf[LINE_BUF_SIZE]; // this string IS NUL-terminated // How many bytes of in_job->body have been read so far. If in_job is NULL // while in_job_read is nonzero, we are in bit bucket mode and // in_job_read's meaning is inverted -- then it counts the bytes that // remain to be thrown away. int64 in_job_read; Job *in_job; // a job to be read from the client Job *out_job; // a job to be sent to the client int out_job_sent; // how many bytes of *out_job were sent already Ms watch; // the set of watched tubes by the connection Job reserved_jobs; // linked list header }; int conn_less(void *ca, void *cb); void conn_setpos(void *c, size_t i); void connsched(Conn *c); void connclose(Conn *c); void connsetproducer(Conn *c); void connsetworker(Conn *c); Job *connsoonestjob(Conn *c); int conndeadlinesoon(Conn *c); int conn_ready(Conn *c); void conn_reserve_job(Conn *c, Job *j); #define conn_waiting(c) ((c)->type & CONN_TYPE_WAITING) enum { Filesizedef = (10 << 20) }; struct Wal { int filesize; int use; char *dir; File *head; File *cur; File *tail; int nfile; int next; int64 resv; // bytes reserved int64 alive; // bytes in use int64 nmig; // migrations int64 nrec; // records written ever int wantsync; // do we sync to disk? int64 syncrate; // how often we sync to disk, in nanoseconds int64 lastsync; }; int waldirlock(Wal*); void walinit(Wal*, Job *list); int walwrite(Wal*, Job*); void walmaint(Wal*); int walresvput(Wal*, Job*); int walresvupdate(Wal*); void walgc(Wal*); struct File { File *next; uint refs; int seq; int iswopen; // is open for writing int fd; int free; int resv; char *path; Wal *w; Job jlist; // jobs written in this file }; int fileinit(File*, Wal*, int); Wal* fileadd(File*, Wal*); void fileincref(File*); void filedecref(File*); void fileaddjob(File*, Job*); void filermjob(File*, Job*); int fileread(File*, Job *list); void filewopen(File*); void filewclose(File*); int filewrjobshort(File*, Job*); int filewrjobfull(File*, Job*); #define Portdef "11300" struct Server { char *port; char *addr; char *user; Wal wal; Socket sock; // Connections that must produce deadline or timeout, ordered by the time. Heap conns; }; void srv_acquire_wal(Server *s); void srvserve(Server *s); void srvaccept(Server *s, int ev); beanstalkd-1.13/doc/000077500000000000000000000000001440333440000143065ustar00rootroot00000000000000beanstalkd-1.13/doc/beanstalkd.1000066400000000000000000000067751440333440000165170ustar00rootroot00000000000000.\" generated with Ronn-NG/v0.8.2 .\" http://github.com/apjanke/ronn-ng/tree/0.8.2 .TH "BEANSTALKD" "1" "June 2020" "" "" .SH "NAME" \fBbeanstalkd\fR \- simple, fast work queue .SH "SYNOPSIS" \fBbeanstalkd\fR [\fIoptions\fR] .SH "DESCRIPTION" \fBBeanstalkd\fR is a simple work\-queue service\. Its interface is generic, though it was originally designed for reducing the latency of page views in high\-volume web applications by running time\-consuming tasks asynchronously\. .P When started, \fBbeanstalkd\fR opens a socket (or uses a file descriptor provided by the init(1) system, see \fI\%#ENVIRONMENT\fR) and listens for incoming connections\. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done\. See file \fBdoc/protocol\.txt\fR in the \fBbeanstalkd\fR distribution for a thorough description of the meaning and format of the \fBbeanstalkd\fR protocol\. .SH "OPTIONS" .TP \fB\-b\fR \fIpath\fR Use a binlog to keep jobs on persistent storage in directory \fIpath\fR\. Upon startup, \fBbeanstalkd\fR will recover any binlog that is present in \fIpath\fR, then, during normal operation, append new jobs and changes in state to the binlog\. .TP \fB\-f\fR \fIms\fR Call fsync(2) at most once every \fIms\fR milliseconds\. Larger values for \fIms\fR reduce disk activity and improve speed at the cost of safety\. A power failure could result in the loss of up to \fIms\fR milliseconds of history\. .IP A \fIms\fR value of 0 will cause \fBbeanstalkd\fR to call fsync every time it writes to the binlog\. .IP The default behavior is to sync every 50 ms\. .IP (This option has no effect without \fB\-b\fR\.) .TP \fB\-F\fR Never call fsync(2)\. Equivalent to \fB\-f\fR with an infinite \fIms\fR value\. .IP (This option has no effect without \fB\-b\fR\.) .TP \fB\-h\fR Show a brief help message and exit\. .TP \fB\-l\fR \fIaddr\fR Listen on address \fIaddr\fR (default is 0\.0\.0\.0)\. .IP When \fIaddr\fR starts with "unix:", the unprefixed value of it will be used as the local filesystem path to create a UNIX socket instead of a TCP socket\. In this case the value of \fB\-p\fR will be ignored\. .IP (Option \fB\-l\fR has no effect if sd\-daemon(5) socket activation is being used\. See also \fI\%#ENVIRONMENT\fR\.) .TP \fB\-p\fR \fIport\fR Listen on TCP port \fIport\fR (default is 11300)\. .IP (Option \fB\-p\fR has no effect if sd\-daemon(5) socket activation is being used\. See also \fI\%#ENVIRONMENT\fR\.) .TP \fB\-s\fR \fIbytes\fR The size in bytes of each binlog file\. .IP (This option has no effect without \fB\-b\fR\.) .TP \fB\-u\fR \fIuser\fR Become the user \fIuser\fR and its primary group\. .TP \fB\-V\fR Increase verbosity\. May be used more than once to produce more verbose output\. The output format is subject to change\. .TP \fB\-v\fR Print the version string and exit\. .TP \fB\-z\fR \fIbytes\fR The maximum size in bytes of a job\. .TP \fB\-c\fR This flag has no effect\. It is kept for historical compatibility only\. .TP \fB\-n\fR This flag has no effect\. It is kept for historical compatibility only\. .SH "ENVIRONMENT" .TP \fBLISTEN_PID\fR, \fBLISTEN_FDS\fR These variables can be set by init(1)\. See sd_listen_fds(3) for details\. .SH "SEE ALSO" sd\-daemon(3), sd_listen_fds(3) .P Files \fBREADME\.md\fR and \fBdoc/protocol\.txt\fR in the \fBbeanstalkd\fR distribution\. .P \fI\%https://beanstalkd\.github\.io/\fR .SH "AUTHOR" \fBBeanstalkd\fR is written by Keith Rarick and maintained by the community at \fI\%https://github\.com/beanstalkd/beanstalkd/issues\fR beanstalkd-1.13/doc/beanstalkd.1.html000066400000000000000000000175171440333440000174560ustar00rootroot00000000000000 beanstalkd(1) - simple, fast work queue
  1. beanstalkd(1)
  3. beanstalkd(1)

NAME

beanstalkd - simple, fast work queue

SYNOPSIS

beanstalkd [options]

DESCRIPTION

Beanstalkd is a simple work-queue service. Its interface is generic, though it was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously.

When started, beanstalkd opens a socket (or uses a file descriptor provided by the init(1) system, see ENVIRONMENT) and listens for incoming connections. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done. See file doc/protocol.txt in the beanstalkd distribution for a thorough description of the meaning and format of the beanstalkd protocol.

OPTIONS

-b path
Use a binlog to keep jobs on persistent storage in directory path. Upon startup, beanstalkd will recover any binlog that is present in path, then, during normal operation, append new jobs and changes in state to the binlog.
-f ms
Call fsync(2) at most once every ms milliseconds. Larger values for ms reduce disk activity and improve speed at the cost of safety. A power failure could result in the loss of up to ms milliseconds of history.

A ms value of 0 will cause beanstalkd to call fsync every time it writes to the binlog.

The default behavior is to sync every 50 ms.

(This option has no effect without -b.)

-F
Never call fsync(2). Equivalent to -f with an infinite ms value.

(This option has no effect without -b.)

-h
Show a brief help message and exit.
-l addr
Listen on address addr (default is 0.0.0.0).

When addr starts with "unix:", the unprefixed value of it will be used as the local filesystem path to create a UNIX socket instead of a TCP socket. In this case the value of -p will be ignored.

(Option -l has no effect if sd-daemon(5) socket activation is being used. See also ENVIRONMENT.)

-p port
Listen on TCP port port (default is 11300).

(Option -p has no effect if sd-daemon(5) socket activation is being used. See also ENVIRONMENT.)

-s bytes
The size in bytes of each binlog file.

(This option has no effect without -b.)

-u user
Become the user user and its primary group.
-V
Increase verbosity. May be used more than once to produce more verbose output. The output format is subject to change.
-v
Print the version string and exit.
-z bytes
The maximum size in bytes of a job.
-c
This flag has no effect. It is kept for historical compatibility only.
-n
This flag has no effect. It is kept for historical compatibility only.

ENVIRONMENT

LISTEN_PID, LISTEN_FDS
These variables can be set by init(1). See sd_listen_fds(3) for details.

SEE ALSO

sd-daemon(3), sd_listen_fds(3)

Files README.md and doc/protocol.txt in the beanstalkd distribution.

https://beanstalkd.github.io/

AUTHOR

Beanstalkd is written by Keith Rarick and maintained by the community at https://github.com/beanstalkd/beanstalkd/issues

  2. June 2020
  3. beanstalkd(1)
beanstalkd-1.13/doc/beanstalkd.ronn000066400000000000000000000062041440333440000173160ustar00rootroot00000000000000beanstalkd(1) -- simple, fast work queue ======================================== ## SYNOPSIS `beanstalkd` [] ## DESCRIPTION `Beanstalkd` is a simple work-queue service. Its interface is generic, though it was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. When started, `beanstalkd` opens a socket (or uses a file descriptor provided by the init(1) system, see [ENVIRONMENT][]) and listens for incoming connections. For each connection, it reads a sequence of commands to create, reserve, delete, and otherwise manipulate "jobs", units of work to be done. See file `doc/protocol.txt` in the `beanstalkd` distribution for a thorough description of the meaning and format of the `beanstalkd` protocol. ## OPTIONS * `-b` : Use a binlog to keep jobs on persistent storage in directory . Upon startup, `beanstalkd` will recover any binlog that is present in , then, during normal operation, append new jobs and changes in state to the binlog. * `-f` : Call fsync(2) at most once every milliseconds. Larger values for reduce disk activity and improve speed at the cost of safety. A power failure could result in the loss of up to milliseconds of history. A value of 0 will cause `beanstalkd` to call fsync every time it writes to the binlog. The default behavior is to sync every 50 ms. (This option has no effect without `-b`.) * `-F`: Never call fsync(2). Equivalent to `-f` with an infinite value. (This option has no effect without `-b`.) * `-h`: Show a brief help message and exit. * `-l` : Listen on address (default is 0.0.0.0). When starts with "unix:", the unprefixed value of it will be used as the local filesystem path to create a UNIX socket instead of a TCP socket. In this case the value of `-p` will be ignored. (Option `-l` has no effect if sd-daemon(5) socket activation is being used. See also [ENVIRONMENT][].) * `-p` : Listen on TCP port (default is 11300). (Option `-p` has no effect if sd-daemon(5) socket activation is being used. See also [ENVIRONMENT][].) * `-s` : The size in bytes of each binlog file. (This option has no effect without `-b`.) * `-u` : Become the user and its primary group. * `-V`: Increase verbosity. May be used more than once to produce more verbose output. The output format is subject to change. * `-v`: Print the version string and exit. * `-z` : The maximum size in bytes of a job. * `-c`: This flag has no effect. It is kept for historical compatibility only. * `-n`: This flag has no effect. It is kept for historical compatibility only. ## ENVIRONMENT * `LISTEN_PID`, `LISTEN_FDS`: These variables can be set by init(1). See sd_listen_fds(3) for details. ## SEE ALSO sd-daemon(3), sd_listen_fds(3) Files `README.md` and `doc/protocol.txt` in the `beanstalkd` distribution. ## AUTHOR `Beanstalkd` is written by Keith Rarick and maintained by the community at beanstalkd-1.13/doc/protocol.txt000066400000000000000000000627061440333440000167230ustar00rootroot00000000000000= Beanstalk Protocol = Protocol -------- The beanstalk protocol runs over TCP using ASCII encoding. Clients connect, send commands and data, wait for responses, and close the connection. For each connection, the server processes commands serially in the order in which they were received and sends responses in the same order. All integers in the protocol are formatted in decimal and (unless otherwise indicated) nonnegative. Names, in this protocol, are ASCII strings. They may contain letters (A-Z and a-z), numerals (0-9), hyphen ("-"), plus ("+"), slash ("/"), semicolon (";"), dot ("."), dollar-sign ("$"), underscore ("_"), and parentheses ("(" and ")"), but they may not begin with a hyphen. They are terminated by white space (either a space char or end of line). Each name must be at least one character long. The protocol contains two kinds of data: text lines and unstructured chunks of data. Text lines are used for client commands and server responses. Chunks are used to transfer job bodies and stats information. Each job body is an opaque sequence of bytes. The server never inspects or modifies a job body and always sends it back in its original form. It is up to the clients to agree on a meaningful interpretation of job bodies. The client may issue the "quit" command, or simply close the TCP connection when it no longer has use for the server. However, beanstalkd performs very well with a large number of open connections, so it is usually better for the client to keep its connection open and reuse it as much as possible. This also avoids the overhead of establishing new TCP connections. If a client violates the protocol (such as by sending a request that is not well-formed or a command that does not exist) or if the server has an error, the server will reply with one of the following error messages: - "OUT_OF_MEMORY\r\n" The server cannot allocate enough memory for the job. The client should try again later. - "INTERNAL_ERROR\r\n" This indicates a bug in the server. It should never happen. If it does happen, please report it at http://groups.google.com/group/beanstalk-talk. - "BAD_FORMAT\r\n" The client sent a command line that was not well-formed. This can happen if the line's length exceeds 224 bytes including \r\n, if the name of a tube exceeds 200 bytes, if non-numeric characters occur where an integer is expected, if the wrong number of arguments are present, or if the command line is mal-formed in any other way. - "UNKNOWN_COMMAND\r\n" The client sent a command that the server does not know. These error responses will not be listed in this document for individual commands in the following sections, but they are implicitly included in the description of all commands. Clients should be prepared to receive an error response after any command. As a last resort, if the server has a serious error that prevents it from continuing service to the current client, the server will close the connection. Job Lifecycle ------------- A job in beanstalk gets created by a client with the "put" command. During its life it can be in one of four states: "ready", "reserved", "delayed", or "buried". After the put command, a job typically starts out ready. It waits in the ready queue until a worker comes along and runs the "reserve" command. If this job is next in the queue, it will be reserved for the worker. The worker will execute the job; when it is finished the worker will send a "delete" command to delete the job. Here is a picture of the typical job lifecycle: put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* Here is a picture with more possibilities: put with delay release with delay ----------------> [DELAYED] <------------. | | | (time passes) | | | put v reserve | delete -----------------> [READY] ---------> [RESERVED] --------> *poof* ^ ^ | | | \ release | | | `-------------' | | | | kick | | | | bury | [BURIED] <---------------' | | delete `--------> *poof* The system has one or more tubes. Each tube consists of a ready queue and a delay queue. Each job spends its entire life in one tube. Consumers can show interest in tubes by sending the "watch" command; they can show disinterest by sending the "ignore" command. This set of interesting tubes is said to be a consumer's "watch list". When a client reserves a job, it may come from any of the tubes in its watch list. When a client connects, its watch list is initially just the tube named "default". If it submits jobs without having sent a "use" command, they will live in the tube named "default". Tubes are created on demand whenever they are referenced. If a tube is empty (that is, it contains no ready, delayed, or buried jobs) and no client refers to it, it will be deleted. Producer Commands ----------------- The "put" command is for any process that wants to insert a job into the queue. It comprises a command line followed by the job body: put \r\n \r\n It inserts a job into the client's currently used tube (see the "use" command below). - is an integer < 2**32. Jobs with smaller priority values will be scheduled before jobs with larger priorities. The most urgent priority is 0; the least urgent priority is 4,294,967,295. - is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. Maximum delay is 2**32-1. - -- time to run -- is an integer number of seconds to allow a worker to run this job. This time is counted from the moment a worker reserves this job. If the worker does not delete, release, or bury the job within seconds, the job will time out and the server will release the job. The minimum ttr is 1. If the client sends 0, the server will silently increase the ttr to 1. Maximum ttr is 2**32-1. - is an integer indicating the size of the job body, not including the trailing "\r\n". This value must be less than max-job-size (default: 2**16). - is the job body -- a sequence of bytes of length from the previous line. After sending the command line and body, the client waits for a reply, which may be: - "INSERTED \r\n" to indicate success. - is the integer id of the new job - "BURIED \r\n" if the server ran out of memory trying to grow the priority queue data structure. - is the integer id of the new job - "EXPECTED_CRLF\r\n" The job body must be followed by a CR-LF pair, that is, "\r\n". These two bytes are not counted in the job size given by the client in the put command line. - "JOB_TOO_BIG\r\n" The client has requested to put a job with a body larger than max-job-size bytes. - "DRAINING\r\n" This means that the server has been put into "drain mode" and is no longer accepting new jobs. The client should try another server or disconnect and try again later. To put the server in drain mode, send the SIGUSR1 signal to the process. The "use" command is for producers. Subsequent put commands will put jobs into the tube specified by this command. If no use command has been issued, jobs will be put into the tube named "default". use \r\n - is a name at most 200 bytes. It specifies the tube to use. If the tube does not exist, it will be created. The only reply is: USING \r\n - is the name of the tube now being used. Worker Commands --------------- A process that wants to consume jobs from the queue uses "reserve", "delete", "release", and "bury". The first worker command, "reserve", looks like this: reserve\r\n Alternatively, you can specify a timeout as follows: reserve-with-timeout \r\n This will return a newly-reserved job. If no job is available to be reserved, beanstalkd will wait to send a response until one becomes available. Once a job is reserved for the client, the client has limited time to run (TTR) the job before the job times out. When the job times out, the server will put the job back into the ready queue. Both the TTR and the actual time left can be found in response to the stats-job command. If more than one job is ready, beanstalkd will choose the one with the smallest priority value. Within each priority, it will choose the one that was received first. A timeout value of 0 will cause the server to immediately return either a response or TIMED_OUT. A positive value of timeout will limit the amount of time the client will block on the reserve request until a job becomes available. During the TTR of a reserved job, the last second is kept by the server as a safety margin, during which the client will not be made to wait for another job. If the client issues a reserve command during the safety margin, or if the safety margin arrives while the client is waiting on a reserve command, the server will respond with: DEADLINE_SOON\r\n This gives the client a chance to delete or release its reserved job before the server automatically releases it. TIMED_OUT\r\n If a non-negative timeout was specified and the timeout exceeded before a job became available, or if the client's connection is half-closed, the server will respond with TIMED_OUT. Otherwise, the only other response to this command is a successful reservation in the form of a text line followed by the job body: RESERVED \r\n \r\n - is the job id -- an integer unique to this job in this instance of beanstalkd. - is an integer indicating the size of the job body, not including the trailing "\r\n". - is the job body -- a sequence of bytes of length from the previous line. This is a verbatim copy of the bytes that were originally sent to the server in the put command for this job. A job can be reserved by its id. Once a job is reserved for the client, the client has limited time to run (TTR) the job before the job times out. When the job times out, the server will put the job back into the ready queue. The command looks like this: reserve-job \r\n - is the job id to reserve This should immediately return one of these responses: - "NOT_FOUND\r\n" if the job does not exist or reserved by a client or is not either ready, buried or delayed. - "RESERVED \r\n\r\n". See the description for the reserve command. The delete command removes a job from the server entirely. It is normally used by the client when the job has successfully run to completion. A client can delete jobs that it has reserved, ready jobs, delayed jobs, and jobs that are buried. The delete command looks like this: delete \r\n - is the job id to delete. The client then waits for one line of response, which may be: - "DELETED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not either reserved by the client, ready, or buried. This could happen if the job timed out before the client sent the delete command. The release command puts a reserved job back into the ready queue (and marks its state as "ready") to be run by any client. It is normally used when the job fails because of a transitory error. It looks like this: release \r\n - is the job id to release. - is a new priority to assign to the job. - is an integer number of seconds to wait before putting the job in the ready queue. The job will be in the "delayed" state during this time. The client expects one line of response, which may be: - "RELEASED\r\n" to indicate success. - "BURIED\r\n" if the server ran out of memory trying to grow the priority queue data structure. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The bury command puts a job into the "buried" state. Buried jobs are put into a FIFO linked list and will not be touched by the server again until a client kicks them with the "kick" command. The bury command looks like this: bury \r\n - is the job id to bury. - is a new priority to assign to the job. There are two possible responses: - "BURIED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The "touch" command allows a worker to request more time to work on a job. This is useful for jobs that potentially take a long time, but you still want the benefits of a TTR pulling a job away from an unresponsive worker. A worker may periodically tell the server that it's still alive and processing a job (e.g. it may do this on DEADLINE_SOON). The command postpones the auto release of a reserved job until TTR seconds from when the command is issued. The touch command looks like this: touch \r\n - is the ID of a job reserved by the current connection. There are two possible responses: - "TOUCHED\r\n" to indicate success. - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client. The "watch" command adds the named tube to the watch list for the current connection. A reserve command will take a job from any of the tubes in the watch list. For each new connection, the watch list initially consists of one tube, named "default". watch \r\n - is a name at most 200 bytes. It specifies a tube to add to the watch list. If the tube doesn't exist, it will be created. The reply is: WATCHING \r\n - is the integer number of tubes currently in the watch list. The "ignore" command is for consumers. It removes the named tube from the watch list for the current connection. ignore \r\n The reply is one of: - "WATCHING \r\n" to indicate success. - is the integer number of tubes currently in the watch list. - "NOT_IGNORED\r\n" if the client attempts to ignore the only tube in its watch list. Other Commands -------------- The peek commands let the client inspect a job in the system. There are four variations. All but the first operate only on the currently used tube. - "peek \r\n" - return job . - "peek-ready\r\n" - return the next ready job. - "peek-delayed\r\n" - return the delayed job with the shortest delay left. - "peek-buried\r\n" - return the next job in the list of buried jobs. There are two possible responses, either a single line: - "NOT_FOUND\r\n" if the requested job doesn't exist or there are no jobs in the requested state. Or a line followed by a chunk of data, if the command was successful: FOUND \r\n \r\n - is the job id. - is an integer indicating the size of the job body, not including the trailing "\r\n". - is the job body -- a sequence of bytes of length from the previous line. The kick command applies only to the currently used tube. It moves jobs into the ready queue. If there are any buried jobs, it will only kick buried jobs. Otherwise it will kick delayed jobs. It looks like: kick \r\n - is an integer upper bound on the number of jobs to kick. The server will kick no more than jobs. The response is of the form: KICKED \r\n - is an integer indicating the number of jobs actually kicked. The kick-job command is a variant of kick that operates with a single job identified by its job id. If the given job id exists and is in a buried or delayed state, it will be moved to the ready queue of the the same tube where it currently belongs. The syntax is: kick-job \r\n - is the job id to kick. The response is one of: - "NOT_FOUND\r\n" if the job does not exist or is not in a kickable state. This can also happen upon internal errors. - "KICKED\r\n" when the operation succeeded. The stats-job command gives statistical information about the specified job if it exists. Its form is: stats-job \r\n - is a job id. The response is one of: - "NOT_FOUND\r\n" if the job does not exist. - "OK \r\n\r\n" - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented by a dictionary. The stats-job data is a YAML file representing a single dictionary of string keys to scalar values. It contains these keys: - "id" is the job id - "tube" is the name of the tube that contains this job - "state" is "ready" or "delayed" or "reserved" or "buried" - "pri" is the priority value set by the put, release, or bury commands. - "age" is the time in seconds since the put command that created this job. - "delay" is the integer number of seconds to wait before putting this job in the ready queue. - "ttr" -- time to run -- is the integer number of seconds a worker is allowed to run this job. - "time-left" is the number of seconds left until the server puts this job into the ready queue. This number is only meaningful if the job is reserved or delayed. If the job is reserved and this amount of time elapses before its state changes, it is considered to have timed out. - "file" is the number of the earliest binlog file containing this job. If -b wasn't used, this will be 0. - "reserves" is the number of times this job has been reserved. - "timeouts" is the number of times this job has timed out during a reservation. - "releases" is the number of times a client has released this job from a reservation. - "buries" is the number of times this job has been buried. - "kicks" is the number of times this job has been kicked. The stats-tube command gives statistical information about the specified tube if it exists. Its form is: stats-tube \r\n - is a name at most 200 bytes. Stats will be returned for this tube. The response is one of: - "NOT_FOUND\r\n" if the tube does not exist. - "OK \r\n\r\n" - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented by a dictionary. The stats-tube data is a YAML file representing a single dictionary of string keys to scalar values. It contains these keys: - "name" is the tube's name. - "current-jobs-urgent" is the number of ready jobs with priority < 1024 in this tube. - "current-jobs-ready" is the number of jobs in the ready queue in this tube. - "current-jobs-reserved" is the number of jobs reserved by all clients in this tube. - "current-jobs-delayed" is the number of delayed jobs in this tube. - "current-jobs-buried" is the number of buried jobs in this tube. - "total-jobs" is the cumulative count of jobs created in this tube in the current beanstalkd process. - "current-using" is the number of open connections that are currently using this tube. - "current-waiting" is the number of open connections that have issued a reserve command while watching this tube but not yet received a response. - "current-watching" is the number of open connections that are currently watching this tube. - "pause" is the number of seconds the tube has been paused for. - "cmd-delete" is the cumulative number of delete commands for this tube - "cmd-pause-tube" is the cumulative number of pause-tube commands for this tube. - "pause-time-left" is the number of seconds until the tube is un-paused. The stats command gives statistical information about the system as a whole. Its form is: stats\r\n The server will respond: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file with statistical information represented by a dictionary. The stats data for the system is a YAML file representing a single dictionary of string keys to scalar values. Entries described as "cumulative" are reset when the beanstalkd process starts; they are not stored on disk with the -b flag. - "current-jobs-urgent" is the number of ready jobs with priority < 1024. - "current-jobs-ready" is the number of jobs in the ready queue. - "current-jobs-reserved" is the number of jobs reserved by all clients. - "current-jobs-delayed" is the number of delayed jobs. - "current-jobs-buried" is the number of buried jobs. - "cmd-put" is the cumulative number of put commands. - "cmd-peek" is the cumulative number of peek commands. - "cmd-peek-ready" is the cumulative number of peek-ready commands. - "cmd-peek-delayed" is the cumulative number of peek-delayed commands. - "cmd-peek-buried" is the cumulative number of peek-buried commands. - "cmd-reserve" is the cumulative number of reserve commands. - "cmd-reserve-with-timeout" is the cumulative number of reserve-with-timeout commands. - "cmd-touch" is the cumulative number of touch commands. - "cmd-use" is the cumulative number of use commands. - "cmd-watch" is the cumulative number of watch commands. - "cmd-ignore" is the cumulative number of ignore commands. - "cmd-delete" is the cumulative number of delete commands. - "cmd-release" is the cumulative number of release commands. - "cmd-bury" is the cumulative number of bury commands. - "cmd-kick" is the cumulative number of kick commands. - "cmd-stats" is the cumulative number of stats commands. - "cmd-stats-job" is the cumulative number of stats-job commands. - "cmd-stats-tube" is the cumulative number of stats-tube commands. - "cmd-list-tubes" is the cumulative number of list-tubes commands. - "cmd-list-tube-used" is the cumulative number of list-tube-used commands. - "cmd-list-tubes-watched" is the cumulative number of list-tubes-watched commands. - "cmd-pause-tube" is the cumulative number of pause-tube commands. - "job-timeouts" is the cumulative count of times a job has timed out. - "total-jobs" is the cumulative count of jobs created. - "max-job-size" is the maximum number of bytes in a job. - "current-tubes" is the number of currently-existing tubes. - "current-connections" is the number of currently open connections. - "current-producers" is the number of open connections that have each issued at least one put command. - "current-workers" is the number of open connections that have each issued at least one reserve command. - "current-waiting" is the number of open connections that have issued a reserve command but not yet received a response. - "total-connections" is the cumulative count of connections. - "pid" is the process id of the server. - "version" is the version string of the server. - "rusage-utime" is the cumulative user CPU time of this process in seconds and microseconds. - "rusage-stime" is the cumulative system CPU time of this process in seconds and microseconds. - "uptime" is the number of seconds since this server process started running. - "binlog-oldest-index" is the index of the oldest binlog file needed to store the current jobs. - "binlog-current-index" is the index of the current binlog file being written to. If binlog is not active this value will be 0. - "binlog-max-size" is the maximum size in bytes a binlog file is allowed to get before a new binlog file is opened. - "binlog-records-written" is the cumulative number of records written to the binlog. - "binlog-records-migrated" is the cumulative number of records written as part of compaction. - "draining" is set to "true" if the server is in drain mode, "false" otherwise. - "id" is a random id string for this server process, generated every time beanstalkd process starts. - "hostname" is the hostname of the machine as determined by uname. - "os" is the OS version as determined by uname - "platform" is the machine architecture as determined by uname The list-tubes command returns a list of all existing tubes. Its form is: list-tubes\r\n The response is: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file containing all tube names as a list of strings. The list-tube-used command returns the tube currently being used by the client. Its form is: list-tube-used\r\n The response is: USING \r\n - is the name of the tube being used. The list-tubes-watched command returns a list tubes currently being watched by the client. Its form is: list-tubes-watched\r\n The response is: OK \r\n \r\n - is the size of the following data section in bytes. - is a sequence of bytes of length from the previous line. It is a YAML file containing watched tube names as a list of strings. The quit command simply closes the connection. Its form is: quit\r\n The pause-tube command can delay any new job being reserved for a given time. Its form is: pause-tube \r\n - is the tube to pause - is an integer number of seconds < 2**32 to wait before reserving any more jobs from the queue There are two possible responses: - "PAUSED\r\n" to indicate success. - "NOT_FOUND\r\n" if the tube does not exist. beanstalkd-1.13/doc/protocol.zh-CN.md000066400000000000000000000430721440333440000174150ustar00rootroot00000000000000# Beanstalkd中文协议 ### 总括 `beanstalkd` 协议基于 ASCII 编码运行在 tcp 上. 客户端连接服务器并发送指令和数据，然后等待响应并关闭连接. 对于每个连接，服务器按照接收命令的序列依次处理并响应. 所有整型值都非负的十进制数，除非有特别声明. ### 名称约定 所有名称必须是 ASCII 码字符串，即包括： * **字母** (A-Z and a-z) * **数字** (0-9) * **连字符** ("-") * **加号** ("+") * **斜线** ("/") * **分号** (";") * **点** (".") * **美元符** ("$") * **下划线** ("_") * **括号** ("*(*" and "*)*") **注意**：名称不能以连字符开始，并且是以空白字符结束，每个名称至少包含一个字符. ### 错误说明 | 返回的错误 | 描述 | | --------------------------------------- | -------- | | `OUT_OF_MEMORY\r\n` | 服务器没有足够的内存分配给特定的 job，客户端应该稍后重试 | | `INTERNAL_ERROR\r\n` | 服务器内部错误，该错误不应该发生，如果发生了，请报告: [http://groups.google.com/group/beanstalk-talk](http://groups.google.com/group/beanstalk-talk).| | `BAD_FORMAT\r\n` | 格式不正确，客户端发送的指令格式出错，有可能不是以 `\r\n` 结尾，或者要求整型值等等 | | `UNKNOWN_COMMAND\r\n` | 未知的命令，客户端发送的指令服务器不理解 | ### job 的生命周期 Client 使用 put 命令创建一个工作任务 job. 在整个生命周期中 job 可能有四个工作状态：ready、reserved、delayed、buried. 在 put 操作之后，一个 job 的典型状态是 ready，在 ready 队列中，它将等待一个 worker 取出此 job 并设置为其为 reserved 状态. worker占有此 job 并执行，当 job 执行完毕，worker 可以发送一个 delete 指令删除此 job. | Status | Description | | --------------------| ------------- | | `ready` | 等待被取出并处理 | | `reserved` | 如果 job 被 worker 取出，将被此 worker 预订，worker 将执行此 job | | `delayed` | 等待特定时间之后，状态再迁移为 `ready` 状态 | | `buried` | 等待唤醒，通常在 job 处理失败时进入该状态 #### job 典型的生命周期 ```text put reserve delete -----> [READY] ---------> [RESERVED] --------> *poof* ``` #### job 可能的状态迁移 ```text put with delay release with delay ----------------> [DELAYED] <------------. | | kick | (time passes) | | | put v reserve | delete -----------------> [READY] ---------> [RESERVED] --------> *poof* ^ ^ | | | \ release | | | `-------------' | | | | kick | | | | bury | [BURIED] <---------------' | | delete `--------> *poof* ``` ## Tubes 一个 beanstalkd 实例服务可能有一个或者多个 tube，用来储存统一类型的 job.每个 tube 由一个就绪 (`ready`) 队列与延迟 (`delayed`) 队列组成.每个 job 所有的状态迁移在一个 tube 中完成.通过发送 `watch` 指令, 消费者 consumers 可以监控感兴趣的 tube.通过发送 `ignore` 指令, 消费者 consumers 可以取消监控 tube.通过 `list-tubes-watched`命令返回所有监控的 tubes，当客户端预订 (`reserved`) 一个 job，此 job 可能来自任何一个它监控的 tube. 当一个客户端连接上服务器时，客户端监控的 tube 默认为 `defaut`，如果客户端提交 job 时，没有使用 `use` 命令，那么这些 job 就存于名为`default`的 tube 中. tube 按需求创建，无论他们在什么时候被引用到.如果一个 tube 变为空（即 no ready jobs，no delayed jobs，no buried jobs）和没有任何客户端引用(being watched)，它将会被**自动删除**. ### 指令说明（Commands） #### 生产者指令说明（Producer Commands） #### `put` 插入一个 job 到队列 **指令格式** ```text put \r\n \r\n ``` * `` 整型值, 为优先级, 可以为0-2^32 (4,294,967,295) 值越小优先级越高, 默认为1024. * `` 整型值，延迟`ready`的秒数，在这段时间 job 为 `delayed` 状态. 最大 delay 值为 2^32-1. * `` -- time to run --整型值，允许 worker 执行的最大秒数，如果 worker 在这段时间不能 delete，release，bury job，那么当 job 超时，服务器将**自动** release 此job，此 job 的状态迁移为`ready`. 最小为 1 秒，如果客户端指定为 0 将会被重置为 1. 最大 ttr 值为 2^32-1. * `` 整型值，job body的长度，不包含`\r\n`，这个值必须小于 `max-job-size`，默认为 2^16. * `` job body **响应** ```text INSERTED \r\n ``` 表示插入 job 成功，id 为新 job 的任务标识，整型值 (uint64) ```text BURIED \r\n ``` 如服务器为了增加队列的优先级而，内存不足时返回，id 为新 job 的任务标识，整型值 (uint64) ```text EXPECTED_CRLF\r\n ``` job body 必须以 `\r\n` 结尾 ```text JOB_TOO_BIG\r\n ``` job body 的长度超过 `max-job-size` ```text DRAINING\r\n ``` 表示服务器资源耗尽，表示服务器已经进入了`drain mode`，服务器再也不能接受连接，客户端应该使用其他服务器或者断开稍后重试 #### `use` producer 生产者使用，之后使用的 put 指令，都将会把 job 放置于 use 的 tube 中，如果没有指定 use 的 tube， 任务 job 将会进入默认名称为 `default` 的 tube **指令格式** ```text use \r\n tube tube 的名称，最大为 200 字节，不存在时将自动创建 ``` **响应** ```text USING \r\n tube 为正在使用的tube名称 ``` #### 消费者指令说明（Worker Commands） #### `reserve` 预订(reserved) job 等待处理. beanstalkd 将返回一个新预订的 job，如果没有 job，beanstalkd 将一直等待到有 job 时才发送响应. 一旦 job 状态迁移为 `reserved`, 取出 job 的 client 被限制在指定的时间（如果设置了ttr）完成，否则将超时，job 状态重装迁移为ready. **指令格式** ```text reserve\r\n ``` 可选的一个相似的命令 `reserve-with-timeout \r\n` 设置取 job 的超时时间，timeout 设置为 0 时，服务器立即响应或者 TIMED_OUT，积极的设置超时，将会限制客户端阻塞在取 job 的请求的时间. ##### 失败响应 ```text DEADLINE_SOON\r\n ``` * 在一个预定的任务 job 的运行时间内, **最后一秒**会被服务器保持为一个**安全边际**，在这个**时间间隔** (1s) 中，client 将无法获取其他任务. 如果客户端在安全隔离期间发出一个预留 (reserve) 指令，或者客户端在等候一个预定 (reserve) 指令返回结果时，client 安全隔离期到达时，将会收到 `DEADLINE_SOON` 回复 * `DEADLINE_SOON` 的返回结果提示 client 这是一个 delete 或者 touch 它所预订(reserved) 的任务 job 的时机，之后 beanstalkd 服务端将会自动释放 `ttr` 到期的 job ```text TIMED_OUT\r\n 超时 ``` ##### 成功响应 ```text RESERVED \r\n \r\n ``` 成功取出 job: * `` 为 job id,整型值 * `` 为 job body 的长度，不包含`\r\n`， * `` 为job body #### `delete` 从队列中删除一个job **指令格式** ```text delete \r\n ``` * `` 为 job id **响应** ```text DELETED\r\n ``` * 删除成功 ```text NOT_FOUND\r\n ``` * job 不存在时，或者 job 并不为当前的 client 所 reserved; * job 的状态不为 `ready`和 `buried`（这种情况是在 job 被其他 client 所预订(reserved) 且还未执行超时，此时当前 client 发送了 delete 指令就会收到 `NOT_FOUND` 回复） #### `release` release 指令将一个`reserved`的 job 恢复为`ready`. 它通常在 job 执行失败时使用. **指令格式** ```text release \r\n ``` * `` 为job id * `` 为 job 的优先级 * `` 为延迟`ready`的秒数 **响应** ```text RELEASED\r\n 表明成功 BURIED\r\n 如服务器为了增加队列的优先级而，内存不足时返回 bury \r\n ``` * `` 为 job id * `` 为优先级 **响应** ```text BURIED\r\n 表明成功 NOT_FOUND\r\n 如果 job 不存在或者 client 没有预订此 job ``` #### `touch` 允许 worker 请求更多的时间执行 job；当 job 需要更长的时间来执行，这个指令就将会起作用，worker 可用周期性的告诉服务器它仍然在执行job（可以被 `DEADLINE_SOON` 触发） **指令格式** ```text touch \r\n ``` * `` 为 job id **响应** ```text TOUCHED\r\n 表明成功 NOT_FOUND\r\n 如果 job 不存在或者 client 没有预订此 job ``` #### `watch` 添加监控的 tube 到 watch list 列表，reserve 指令将会从监控的 tube 列表获取 job，对于每个连接，监控的列表默认为 `default` **指令格式** ```text watch \r\n ``` * `` 为监控的 tube 名称，名称最大为 200 字节，如果 tube 不存在会**自动创建** **响应** ```text WATCHING \r\n 表明成功 ``` * `` 整型值，已监控的 tube 数量 #### `ignore` 从已监控的 watch list 列表中移出特定的 tube **指令格式** ```text ignore \r\n ``` * `` 为移出的 tube 名称，名称最多为 200 字节，如果 tube 不存在会自动创建 **响应** ```text WATCHING \r\n 表明成功 ``` * `` 整型值，已监控的tube数量 ```text NOT_IGNORED\r\n ``` * 如果 client 尝试忽略其仅有的tube时的响应 #### 其他指令说明（Other Command） #### `peek` 让 client 在系统中检查 job，有四种形式的命令，其中第一种形式的指令是针对当前使用 (use) 的 tube **指令格式** ```text peek \r\n 返回 id 对应的 job peek-ready\r\n 返回下一个 ready job peek-delayed\r\n 返回下一个延迟剩余时间最短的 job peek-buried\r\n 返回下一个在 buried 列表中的 job ``` **响应** ```text NOT_FOUND\r\n ``` * 如果 job 不存在，或者没有对应状态的 job ```text FOUND \r\n \r\n ``` * `` 为对应的 job id * `` job body 的字节数 * `` 为 job body #### `kick` 此指令应用在当前使用 (use) 的 tube 中，它将 job 的状态迁移为`ready`或者`delayed` **指令格式** ```text kick \r\n ``` * `` 整型值，唤醒的 job 上限 **响应** ``` KICKED \r\n ``` * `` 为真实唤醒的job数量 #### kick-job kick 指令的一个变体，可以使单个 job 被唤醒，使一个状态为`buried`或者`delayed`的 job迁移为`ready`，所有的状态迁移都在相同的 tube 中完成 **指令格式** ```text kick-job \r\n ``` * `` 为job id **响应** `NOT_FOUND\r\n` 如果 job 不存在，或者 job 是不可唤醒的状态 `KICKED\r\n` 表明成功 #### `stats-job` 统计 job 的相关信息 **指令格式** ```text stats-job \r\n ``` * `` 为 job id **响应** ```text NOT_FOUND\r\n 如果job不存在 OK \r\n\r\n ``` * `` 为接下来的 data 区块的长度 * `` 为 YAML file 的统计信息 其中 YAML file 包括的 key 有: * `id` 表示 job id * `tube` 表示 tube 的名称 * `state` 表示 job 的当前状态 * `pri` 表示 job 的优先级 * `age` 表示 job 创建的时间单位秒 * `time-left` 表示 job 的状态迁移为 ready 的时间，仅在 job 状态为`reserved`或者`delayed`时有意义，当 job 状态为`reserved`时表示剩余的超时时间. * `file` 表示包含此 job 的`binlog`序号，如果没有开启它将为 0 * `reserves` 表示 job 被`reserved`的次数 * `timeouts` 表示 job 处理的超时时间 * `releases` 表示 job 被`released`的次数 * `buries` 表示 job 被`buried`的次数 * `kicks` 表示 job 被`kiced`的次数 #### `stats-tube` 统计 tube 的相关信息 **指令格式** ```text stats-tube \r\n ``` * `` 为对应的 tube 的名称，最多为 200 字节 **响应** ```text NOT_FOUND\r\n ``` * 如果tube不存在 ```text OK \r\n \r\n ``` * `` 为接下来的 data 区块的长度 * `` 为 YAML file的统计信息 其中 YAML file 包括的 key 有： * `name` 表示tube的名称 * `current-jobs-urgent` 此 tube 中优先级小于 1024 状态为`ready`的 job 数量 * `current-jobs-ready` 此 tube 中状态为`ready`的 job 数量 * `current-jobs-reserved` 此 tube 中状态为`reserved`的 job 数量 * `current-jobs-delayed` 此 tube 中状态为`delayed`的 job 数量 * `current-jobs-bureid` 此 tube 中状态为`buried`的job数量 * `total-jobs` 此 tube 中创建的所有job数量 * `current-using` 使用此 tube 打开的连接数 * `current-wating` 使用此 tube 打开连接并且等待响应的连接数 * `current-watching` 打开的连接监控此 tube 的数量 * `pause` 此 tube 暂停的秒数 * `cmd-delete` 此 tube 中总共执行的`delete`指令的次数 * `cmd-pause-tube` 此 tube 中总共执行`pause-tube`指令的次数 * `pause-time-left` 此 tube 暂停剩余的秒数 #### `stats` 返回整个消息队列系统的整体信息 **指令格式** ```text stats\r\n ``` **响应** ```text OK \r\n \r\n ``` * `` 为接下来的 data 区块的长度 * `` 为 YAML file 的统计信息 从 beanstalkd 进程启动以来，所有的信息都累积的，这些信息不储存在 binlog 中 其中 YAML file 包括的key有： * `current-jobs-urgent` 优先级小于 1024 状态为`ready`的 job 数量 * `current-jobs-ready` 状态为`ready`的 job 数量 * `current-jobs-reserved` 状态为`reserved`的 job 数量 * `current-jobs-delayed` 状态为`delayed`的 job 数量 * `current-jobs-bureid` 状态为`buried`的 job 数量 * `cmd-put` 总共执行`put`指令的次数 * `cmd-peek` 总共执行`peek`指令的次数 * `cmd-peek-ready` 总共执行`peek-ready`指令的次数 * `cmd-peek-delayed` 总共执行`peek-delayed`指令的次数 * `cmd-peek-buried` 总共执行`peek-buried`指令的次数 * `cmd-reserve` 总共执行`reserve`指令的次数 * `cmd-use` 总共执行`use`指令的次数 * `cmd-watch` 总共执行`watch`指令的次数 * `cmd-ignore` 总共执行`ignore`指令的次数 * `cmd-release` 总共执行`release`指令的次数 * `cmd-bury` 总共执行`bury`指令的次数 * `cmd-kick` 总共执行`kick`指令的次数 * `cmd-stats` 总共执行`stats`指令的次数 * `cmd-stats-job` 总共执行`stats-job`指令的次数 * `cmd-stats-tube` 总共执行`stats-tube`指令的次数 * `cmd-list-tubes` 总共执行`list-tubes`指令的次数 * `cmd-list-tube-used` 总共执行`list-tube-used`指令的次数 * `cmd-list-tubes-watched` 总共执行`list-tubes-watched`指令的次数 * `cmd-pause-tube` 总共执行`pause-tube`指令的次数 * `job-timeouts` 所有超时的 job 的总共数量 * `total-jobs` 创建的所有 job 数量 * `max-job-size` job 的数据部分最大长度 * `current-tubes` 当前存在的 tube 数量 * `current-connections` 当前打开的连接数 * `current-producers` 当前所有的打开的连接中至少执行一次 put 指令的连接数量 * `current-workers` 当前所有的打开的连接中至少执行一次 reserve 指令的连接数量 * `current-waiting` 当前所有的打开的连接中执行 reserve 指令但是未响应的连接数量 * `total-connections` 总共处理的连接数 * `pid` 服务器进程的 id * `version` 服务器版本号 * `rusage-utime` 进程总共占用的用户 CPU 时间 * `rusage-stime` 进程总共占用的系统 CPU 时间 * `uptime` 服务器进程运行的秒数 * `binlog-oldest-index` 开始储存 jobs 的 binlog 索引号 * `binlog-current-index` 当前储存 jobs 的 binlog 索引号 * `binlog-max-size` binlog 的最大容量 * `binlog-records-written` binlog 累积写入的记录数 * `binlog-records-migrated` is the cumulative number of records written as part of compaction. * `id` 一个随机字符串，在 beanstalkd 进程启动时产生 * `hostname` 主机名 #### `list-tubes` 列出当前 beanstalkd 所有存在的 tubes **指令格式** ```text list-tubes\r\n ``` **响应** ```text OK \r\n \r\n ``` * `` 为接下来的 data 区块的长度 * `` 为 YAML file，包含所有的 tube 名称 #### `list-tube-used` 列出当前 client 正在 use 的 tube **指令格式** ```text list-tube-used\r\n ``` **响应** ```text USING \r\n ``` * `` 为 tube 名称 #### `list-tubes-watched` 列出当前 client 所 watch 的 tubes **指令格式** ```text list-tubes-watched\r\n ``` **响应** ```text OK \r\n \r\n ``` * `` 为接下来的 data 区块的长度 * `` 为 YAML file，包含所有的 tube 名称 #### `quit` client 向 beanstalkd 发送 `quit` 报文，并关闭连接，beanstalkd 收到该报文后主动关闭连接 **指令格式** ```text quit\r\n ``` 无响应 #### `pause-tube` 此指令针对特定的 tube 内所有新的 job 延迟指定的秒数 **指令格式** ```text pause-tube \r\n ``` * `` 延迟的时间 **响应** ```text PAUSED\r\n 表示成功 NOT_FOUND\r\n tube 不存在 ``` > Translated by PHPBoy :http://www.phpboy.net/ and fzb.me Revised by Pseudocodes: https://github.com/pseudocodes beanstalkd-1.13/file.c000066400000000000000000000316011440333440000146250ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #include #include #include #include #include static int readrec(File*, Job *, int*); static int readrec5(File*, Job *, int*); static int readfull(File*, void*, int, int*, char*); static void warnpos(File*, int, char*, ...) __attribute__((format(printf, 3, 4))); FAlloc *falloc = &rawfalloc; enum { Walver5 = 5 }; typedef struct Jobrec5 Jobrec5; struct Jobrec5 { uint64 id; uint32 pri; uint64 delay; // usec uint64 ttr; // usec int32 body_size; uint64 created_at; // usec uint64 deadline_at; // usec uint32 reserve_ct; uint32 timeout_ct; uint32 release_ct; uint32 bury_ct; uint32 kick_ct; byte state; char pad[1]; }; enum { Jobrec5size = offsetof(Jobrec5, pad) }; // rawfalloc allocates disk space of len bytes. // It expects fd's offset to be 0; may also reset fd's offset to 0. // Returns 0 on success, and a positive errno otherwise. int rawfalloc(int fd, int len) { // We do not use ftruncate() because it might extend the file // with a sequence of null bytes or a hole. // posix_fallocate() is not portable enough, might fail for NFS. static char buf[4096] = {0}; int i, w; for (i = 0; i < len; i += w) { w = write(fd, buf, sizeof buf); if (w == -1) return errno; } lseek(fd, 0, 0); // do not care if this fails return 0; } void fileincref(File *f) { if (!f) return; f->refs++; } void filedecref(File *f) { if (!f) return; f->refs--; if (f->refs < 1) { walgc(f->w); } } void fileaddjob(File *f, Job *j) { Job *h; h = &f->jlist; if (!h->fprev) h->fprev = h; j->file = f; j->fprev = h->fprev; j->fnext = h; h->fprev->fnext = j; h->fprev = j; fileincref(f); } void filermjob(File *f, Job *j) { if (!f) return; if (f != j->file) return; j->fnext->fprev = j->fprev; j->fprev->fnext = j->fnext; j->fnext = 0; j->fprev = 0; j->file = NULL; f->w->alive -= j->walused; j->walused = 0; filedecref(f); } // Fileread reads jobs from f->path into list. // It returns 0 on success, or 1 if any errors occurred. int fileread(File *f, Job *list) { int err = 0, v; if (!readfull(f, &v, sizeof(v), &err, "version")) { return err; } switch (v) { case Walver: fileincref(f); while (readrec(f, list, &err)); filedecref(f); return err; case Walver5: fileincref(f); while (readrec5(f, list, &err)); filedecref(f); return err; } warnx("%s: unknown version: %d", f->path, v); return 1; } // Readrec reads a record from f->fd into linked list l. // If an error occurs, it sets *err to 1. // Readrec returns the number of records read, either 1 or 0. static int readrec(File *f, Job *l, int *err) { int r, sz = 0; int namelen; Jobrec jr; Job *j; Tube *t; char tubename[MAX_TUBE_NAME_LEN]; r = read(f->fd, &namelen, sizeof(int)); if (r == -1) { twarn("read"); warnpos(f, 0, "error"); *err = 1; return 0; } if (r != sizeof(int)) { return 0; } sz += r; if (namelen >= MAX_TUBE_NAME_LEN) { warnpos(f, -r, "namelen %d exceeds maximum of %d", namelen, MAX_TUBE_NAME_LEN - 1); *err = 1; return 0; } if (namelen < 0) { warnpos(f, -r, "namelen %d is negative", namelen); *err = 1; return 0; } if (namelen) { r = readfull(f, tubename, namelen, err, "tube name"); if (!r) { return 0; } sz += r; } tubename[namelen] = '\0'; r = readfull(f, &jr, sizeof(Jobrec), err, "job struct"); if (!r) { return 0; } sz += r; // are we reading trailing zeroes? if (!jr.id) return 0; j = job_find(jr.id); if (!(j || namelen)) { // We read a short record without having seen a // full record for this job, so the full record // was in an earlier file that has been deleted. // Therefore the job itself has either been // deleted or migrated; either way, this record // should be ignored. return 1; } switch (jr.state) { case Reserved: jr.state = Ready; /* Falls through */ case Ready: case Buried: case Delayed: if (!j) { if ((size_t)jr.body_size > job_data_size_limit) { warnpos(f, -r, "job %"PRIu64" is too big (%"PRId32" > %zu)", jr.id, jr.body_size, job_data_size_limit); goto Error; } t = tube_find_or_make(tubename); j = make_job_with_id(jr.pri, jr.delay, jr.ttr, jr.body_size, t, jr.id); job_list_reset(j); j->r.created_at = jr.created_at; } j->r = jr; job_list_insert(l, j); // full record; read the job body if (namelen) { if (jr.body_size != j->r.body_size) { warnpos(f, -r, "job %"PRIu64" size changed", j->r.id); warnpos(f, -r, "was %d, now %d", j->r.body_size, jr.body_size); goto Error; } r = readfull(f, j->body, j->r.body_size, err, "job body"); if (!r) { goto Error; } sz += r; // since this is a full record, we can move // the file pointer and decref the old // file, if any filermjob(j->file, j); fileaddjob(f, j); } j->walused += sz; f->w->alive += sz; return 1; case Invalid: if (j) { job_list_remove(j); filermjob(j->file, j); job_free(j); } return 1; } Error: *err = 1; if (j) { job_list_remove(j); filermjob(j->file, j); job_free(j); } return 0; } // Readrec5 is like readrec, but it reads a record in "version 5" // of the log format. static int readrec5(File *f, Job *l, int *err) { int r, sz = 0; size_t namelen; Jobrec5 jr; Job *j; Tube *t; char tubename[MAX_TUBE_NAME_LEN]; r = read(f->fd, &namelen, sizeof(namelen)); if (r == -1) { twarn("read"); warnpos(f, 0, "error"); *err = 1; return 0; } if (r != sizeof(namelen)) { return 0; } sz += r; if (namelen >= MAX_TUBE_NAME_LEN) { warnpos(f, -r, "namelen %zu exceeds maximum of %d", namelen, MAX_TUBE_NAME_LEN - 1); *err = 1; return 0; } if (namelen) { r = readfull(f, tubename, namelen, err, "v5 tube name"); if (!r) { return 0; } sz += r; } tubename[namelen] = '\0'; r = readfull(f, &jr, Jobrec5size, err, "v5 job struct"); if (!r) { return 0; } sz += r; // are we reading trailing zeroes? if (!jr.id) return 0; j = job_find(jr.id); if (!(j || namelen)) { // We read a short record without having seen a // full record for this job, so the full record // was in an eariler file that has been deleted. // Therefore the job itself has either been // deleted or migrated; either way, this record // should be ignored. return 1; } switch (jr.state) { case Reserved: jr.state = Ready; /* Falls through */ case Ready: case Buried: case Delayed: if (!j) { if ((size_t)jr.body_size > job_data_size_limit) { warnpos(f, -r, "job %"PRIu64" is too big (%"PRId32" > %zu)", jr.id, jr.body_size, job_data_size_limit); goto Error; } t = tube_find_or_make(tubename); j = make_job_with_id(jr.pri, jr.delay, jr.ttr, jr.body_size, t, jr.id); job_list_reset(j); } j->r.id = jr.id; j->r.pri = jr.pri; j->r.delay = jr.delay * 1000; // us => ns j->r.ttr = jr.ttr * 1000; // us => ns j->r.body_size = jr.body_size; j->r.created_at = jr.created_at * 1000; // us => ns j->r.deadline_at = jr.deadline_at * 1000; // us => ns j->r.reserve_ct = jr.reserve_ct; j->r.timeout_ct = jr.timeout_ct; j->r.release_ct = jr.release_ct; j->r.bury_ct = jr.bury_ct; j->r.kick_ct = jr.kick_ct; j->r.state = jr.state; job_list_insert(l, j); // full record; read the job body if (namelen) { if (jr.body_size != j->r.body_size) { warnpos(f, -r, "job %"PRIu64" size changed", j->r.id); warnpos(f, -r, "was %"PRId32", now %"PRId32, j->r.body_size, jr.body_size); goto Error; } r = readfull(f, j->body, j->r.body_size, err, "v5 job body"); if (!r) { goto Error; } sz += r; // since this is a full record, we can move // the file pointer and decref the old // file, if any filermjob(j->file, j); fileaddjob(f, j); } j->walused += sz; f->w->alive += sz; return 1; case Invalid: if (j) { job_list_remove(j); filermjob(j->file, j); job_free(j); } return 1; } Error: *err = 1; if (j) { job_list_remove(j); filermjob(j->file, j); job_free(j); } return 0; } static int readfull(File *f, void *c, int n, int *err, char *desc) { int r; r = read(f->fd, c, n); if (r == -1) { twarn("read"); warnpos(f, 0, "error reading %s", desc); *err = 1; return 0; } if (r != n) { warnpos(f, -r, "unexpected EOF reading %d bytes (got %d): %s", n, r, desc); *err = 1; return 0; } return r; } static void warnpos(File *f, int adj, char *fmt, ...) { int off; va_list ap; off = lseek(f->fd, 0, SEEK_CUR); fprintf(stderr, "%s:%d: ", f->path, off+adj); va_start(ap, fmt); vfprintf(stderr, fmt, ap); va_end(ap); fputc('\n', stderr); } // Opens f for writing, writes a header, and initializes // f->free and f->resv. // Sets f->iswopen if successful. void filewopen(File *f) { int fd, r; int n; int ver = Walver; fd = open(f->path, O_WRONLY|O_CREAT, 0400); if (fd < 0) { twarn("open %s", f->path); return; } r = falloc(fd, f->w->filesize); if (r) { if (close(fd) == -1) twarn("close"); errno = r; twarn("falloc %s", f->path); r = unlink(f->path); if (r) { twarn("unlink %s", f->path); } return; } n = write(fd, &ver, sizeof(int)); if (n < 0 || (size_t)n < sizeof(int)) { twarn("write %s", f->path); if (close(fd) == -1) twarn("close"); return; } f->fd = fd; f->iswopen = 1; fileincref(f); f->free = f->w->filesize - n; f->resv = 0; } static int filewrite(File *f, Job *j, void *buf, int len) { int r; r = write(f->fd, buf, len); if (r != len) { twarn("write"); return 0; } f->w->resv -= r; f->resv -= r; j->walresv -= r; j->walused += r; f->w->alive += r; return 1; } int filewrjobshort(File *f, Job *j) { int r, nl; nl = 0; // name len 0 indicates short record r = filewrite(f, j, &nl, sizeof nl) && filewrite(f, j, &j->r, sizeof j->r); if (!r) return 0; if (j->r.state == Invalid) { filermjob(j->file, j); } return r; } int filewrjobfull(File *f, Job *j) { int nl; fileaddjob(f, j); nl = strlen(j->tube->name); return filewrite(f, j, &nl, sizeof nl) && filewrite(f, j, j->tube->name, nl) && filewrite(f, j, &j->r, sizeof j->r) && filewrite(f, j, j->body, j->r.body_size); } void filewclose(File *f) { if (!f) return; if (!f->iswopen) return; if (f->free) { errno = 0; if (ftruncate(f->fd, f->w->filesize - f->free) != 0) { twarn("ftruncate"); } } if (close(f->fd) == -1) twarn("close"); f->iswopen = 0; filedecref(f); } int fileinit(File *f, Wal *w, int n) { f->w = w; f->seq = n; f->path = fmtalloc("%s/binlog.%d", w->dir, n); return !!f->path; } // Adds f to the linked list in w, // updating w->tail and w->head as necessary. Wal* fileadd(File *f, Wal *w) { if (w->tail) { w->tail->next = f; } w->tail = f; if (!w->head) { w->head = f; } w->nfile++; return w; } beanstalkd-1.13/freebsd.c000066400000000000000000000000241440333440000153130ustar00rootroot00000000000000#include "darwin.c" beanstalkd-1.13/heap.c000066400000000000000000000035111440333440000146220ustar00rootroot00000000000000#include "dat.h" #include #include #include static void set(Heap *h, size_t k, void *x) { h->data[k] = x; h->setpos(x, k); } static void swap(Heap *h, size_t a, size_t b) { void *tmp; tmp = h->data[a]; set(h, a, h->data[b]); set(h, b, tmp); } static int less(Heap *h, size_t a, size_t b) { return h->less(h->data[a], h->data[b]); } static void siftdown(Heap *h, size_t k) { for (;;) { size_t p = (k-1) / 2; /* parent */ if (k == 0 || less(h, p, k)) { return; } swap(h, k, p); k = p; } } static void siftup(Heap *h, size_t k) { for (;;) { size_t l = k*2 + 1; /* left child */ size_t r = k*2 + 2; /* right child */ /* find the smallest of the three */ size_t s = k; if (l < h->len && less(h, l, s)) s = l; if (r < h->len && less(h, r, s)) s = r; if (s == k) { return; /* satisfies the heap property */ } swap(h, k, s); k = s; } } // Heapinsert inserts x into heap h according to h->less. // It returns 1 on success, otherwise 0. int heapinsert(Heap *h, void *x) { if (h->len == h->cap) { void **ndata; size_t ncap = (h->len+1) * 2; /* allocate twice what we need */ ndata = malloc(sizeof(void*) * ncap); if (!ndata) { return 0; } memcpy(ndata, h->data, sizeof(void*) * h->len); free(h->data); h->data = ndata; h->cap = ncap; } size_t k = h->len; h->len++; set(h, k, x); siftdown(h, k); return 1; } void * heapremove(Heap *h, size_t k) { if (k >= h->len) { return 0; } void *x = h->data[k]; h->len--; set(h, k, h->data[h->len]); siftdown(h, k); siftup(h, k); return x; } beanstalkd-1.13/job.c000066400000000000000000000123431440333440000144620ustar00rootroot00000000000000#include "dat.h" #include #include #include static uint64 next_id = 1; static int cur_prime = 0; static Job *all_jobs_init[12289] = {0}; static Job **all_jobs = all_jobs_init; static size_t all_jobs_cap = 12289; /* == primes[0] */ static size_t all_jobs_used = 0; static int hash_table_was_oom = 0; static void rehash(int); static int _get_job_hash_index(uint64 job_id) { return job_id % all_jobs_cap; } static void store_job(Job *j) { int index = 0; index = _get_job_hash_index(j->r.id); j->ht_next = all_jobs[index]; all_jobs[index] = j; all_jobs_used++; /* accept a load factor of 4 */ if (all_jobs_used > (all_jobs_cap << 2)) rehash(1); } static void rehash(int is_upscaling) { Job **old = all_jobs; size_t old_cap = all_jobs_cap, old_used = all_jobs_used, i; int old_prime = cur_prime; int d = is_upscaling ? 1 : -1; if (cur_prime + d >= NUM_PRIMES) return; if (cur_prime + d < 0) return; if (is_upscaling && hash_table_was_oom) return; cur_prime += d; all_jobs_cap = primes[cur_prime]; all_jobs = calloc(all_jobs_cap, sizeof(Job *)); if (!all_jobs) { twarnx("Failed to allocate %zu new hash buckets", all_jobs_cap); hash_table_was_oom = 1; cur_prime = old_prime; all_jobs = old; all_jobs_cap = old_cap; all_jobs_used = old_used; return; } all_jobs_used = 0; hash_table_was_oom = 0; for (i = 0; i < old_cap; i++) { while (old[i]) { Job *j = old[i]; old[i] = j->ht_next; j->ht_next = NULL; store_job(j); } } if (old != all_jobs_init) { free(old); } } Job * job_find(uint64 job_id) { int index = _get_job_hash_index(job_id); Job *jh = all_jobs[index]; while (jh && jh->r.id != job_id) jh = jh->ht_next; return jh; } Job * allocate_job(int body_size) { Job *j; j = malloc(sizeof(Job) + body_size); if (!j) { twarnx("OOM"); return (Job *) 0; } memset(j, 0, sizeof(Job)); j->r.created_at = nanoseconds(); j->r.body_size = body_size; j->body = (char *)j + sizeof(Job); job_list_reset(j); return j; } Job * make_job_with_id(uint32 pri, int64 delay, int64 ttr, int body_size, Tube *tube, uint64 id) { Job *j; j = allocate_job(body_size); if (!j) { twarnx("OOM"); return (Job *) 0; } if (id) { j->r.id = id; if (id >= next_id) next_id = id + 1; } else { j->r.id = next_id++; } j->r.pri = pri; j->r.delay = delay; j->r.ttr = ttr; store_job(j); TUBE_ASSIGN(j->tube, tube); return j; } static void job_hash_free(Job *j) { Job **slot; slot = &all_jobs[_get_job_hash_index(j->r.id)]; while (*slot && *slot != j) slot = &(*slot)->ht_next; if (*slot) { *slot = (*slot)->ht_next; --all_jobs_used; } // Downscale when the hashmap is too sparse if (all_jobs_used < (all_jobs_cap >> 4)) rehash(0); } void job_free(Job *j) { if (j) { TUBE_ASSIGN(j->tube, NULL); if (j->r.state != Copy) job_hash_free(j); } free(j); } void job_setpos(void *j, size_t pos) { ((Job *)j)->heap_index = pos; } int job_pri_less(void *ja, void *jb) { Job *a = (Job *)ja; Job *b = (Job *)jb; if (a->r.pri < b->r.pri) return 1; if (a->r.pri > b->r.pri) return 0; return a->r.id < b->r.id; } int job_delay_less(void *ja, void *jb) { Job *a = ja; Job *b = jb; if (a->r.deadline_at < b->r.deadline_at) return 1; if (a->r.deadline_at > b->r.deadline_at) return 0; return a->r.id < b->r.id; } Job * job_copy(Job *j) { if (!j) return NULL; Job *n = malloc(sizeof(Job) + j->r.body_size); if (!n) { twarnx("OOM"); return (Job *) 0; } memcpy(n, j, sizeof(Job) + j->r.body_size); job_list_reset(n); n->file = NULL; /* copies do not have refcnt on the wal */ n->tube = 0; /* Don't use memcpy for the tube, which we must refcount. */ TUBE_ASSIGN(n->tube, j->tube); /* Mark this job as a copy so it can be appropriately freed later on */ n->r.state = Copy; return n; } const char * job_state(Job *j) { if (j->r.state == Ready) return "ready"; if (j->r.state == Reserved) return "reserved"; if (j->r.state == Buried) return "buried"; if (j->r.state == Delayed) return "delayed"; return "invalid"; } // job_list_reset detaches head from the list, // marking the list starting in head pointing to itself. void job_list_reset(Job *head) { head->prev = head; head->next = head; } int job_list_is_empty(Job *head) { return head->next == head && head->prev == head; } Job * job_list_remove(Job *j) { if (!j) return NULL; if (job_list_is_empty(j)) return NULL; /* not in a doubly-linked list */ j->next->prev = j->prev; j->prev->next = j->next; job_list_reset(j); return j; } void job_list_insert(Job *head, Job *j) { if (!job_list_is_empty(j)) return; /* already in a linked list */ j->prev = head->prev; j->next = head; head->prev->next = j; head->prev = j; } /* for unit tests */ size_t get_all_jobs_used() { return all_jobs_used; } beanstalkd-1.13/linux.c000066400000000000000000000027311440333440000150470ustar00rootroot00000000000000#define _XOPEN_SOURCE 600 #include "dat.h" #include #include #include #include #include #include #include #ifndef EPOLLRDHUP #define EPOLLRDHUP 0x2000 #endif static int epfd; int sockinit(void) { epfd = epoll_create(1); if (epfd == -1) { twarn("epoll_create"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int op; if (!s->added && !rw) { return 0; } else if (!s->added && rw) { s->added = 1; op = EPOLL_CTL_ADD; } else if (!rw) { op = EPOLL_CTL_DEL; } else { op = EPOLL_CTL_MOD; } struct epoll_event ev = {.events=0}; switch (rw) { case 'r': ev.events = EPOLLIN; break; case 'w': ev.events = EPOLLOUT; break; } ev.events |= EPOLLRDHUP | EPOLLPRI; ev.data.ptr = s; return epoll_ctl(epfd, op, s->fd, &ev); } int socknext(Socket **s, int64 timeout) { int r; struct epoll_event ev = {.events=0}; r = epoll_wait(epfd, &ev, 1, (int)(timeout/1000000)); if (r == -1 && errno != EINTR) { twarn("epoll_wait"); exit(1); } if (r > 0) { *s = ev.data.ptr; if (ev.events & (EPOLLHUP|EPOLLRDHUP)) { return 'h'; } else if (ev.events & EPOLLIN) { return 'r'; } else if (ev.events & EPOLLOUT) { return 'w'; } } return 0; } beanstalkd-1.13/main.c000066400000000000000000000042211440333440000146300ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #include #include static void su(const char *user) { errno = 0; struct passwd *pwent = getpwnam(user); if (errno) { twarn("getpwnam(\"%s\")", user); exit(32); } if (!pwent) { twarnx("getpwnam(\"%s\"): no such user", user); exit(33); } int r = setgid(pwent->pw_gid); if (r == -1) { twarn("setgid(%d \"%s\")", pwent->pw_gid, user); exit(34); } r = setuid(pwent->pw_uid); if (r == -1) { twarn("setuid(%d \"%s\")", pwent->pw_uid, user); exit(34); } } static void handle_sigterm_pid1() { exit(143); } static void set_sig_handlers() { struct sigaction sa; sa.sa_handler = SIG_IGN; sa.sa_flags = 0; int r = sigemptyset(&sa.sa_mask); if (r == -1) { twarn("sigemptyset()"); exit(111); } r = sigaction(SIGPIPE, &sa, 0); if (r == -1) { twarn("sigaction(SIGPIPE)"); exit(111); } sa.sa_handler = enter_drain_mode; r = sigaction(SIGUSR1, &sa, 0); if (r == -1) { twarn("sigaction(SIGUSR1)"); exit(111); } // Workaround for running the server with pid=1 in Docker. // Handle SIGTERM so the server is killed immediately and // not after 10 seconds timeout. See issue #527. if (getpid() == 1) { sa.sa_handler = handle_sigterm_pid1; r = sigaction(SIGTERM, &sa, 0); if (r == -1) { twarn("sigaction(SIGTERM)"); exit(111); } } } int main(int argc, char **argv) { UNUSED_PARAMETER(argc); progname = argv[0]; setlinebuf(stdout); optparse(&srv, argv+1); if (verbose) { printf("pid %d\n", getpid()); } int r = make_server_socket(srv.addr, srv.port); if (r == -1) { twarnx("make_server_socket()"); exit(111); } srv.sock.fd = r; prot_init(); if (srv.user) su(srv.user); set_sig_handlers(); srv_acquire_wal(&srv); srvserve(&srv); exit(0); } beanstalkd-1.13/ms.c000066400000000000000000000036061440333440000143310ustar00rootroot00000000000000#include "dat.h" #include #include #include void ms_init(Ms *a, ms_event_fn oninsert, ms_event_fn onremove) { a->len = a->cap = a->last = 0; a->items = NULL; a->oninsert = oninsert; a->onremove = onremove; } static int grow(Ms *a) { void **nitems; size_t ncap = a->cap << 1; if (!ncap) ncap = 1; nitems = malloc(ncap * sizeof(void *)); if (!nitems) return 0; memcpy(nitems, a->items, a->len * sizeof(void *)); free(a->items); a->items = nitems; a->cap = ncap; return 1; } int ms_append(Ms *a, void *item) { if (a->len >= a->cap && !grow(a)) return 0; a->items[a->len++] = item; if (a->oninsert) a->oninsert(a, item, a->len - 1); return 1; } static int ms_delete(Ms *a, size_t i) { void *item; if (i >= a->len) return 0; item = a->items[i]; a->items[i] = a->items[--a->len]; /* it has already been removed now */ if (a->onremove) a->onremove(a, item, i); return 1; } void ms_clear(Ms *a) { while (ms_delete(a, 0)); free(a->items); ms_init(a, a->oninsert, a->onremove); } int ms_remove(Ms *a, void *item) { size_t i; for (i = 0; i < a->len; i++) { if (a->items[i] == item) return ms_delete(a, i); } return 0; } int ms_contains(Ms *a, void *item) { size_t i; for (i = 0; i < a->len; i++) { if (a->items[i] == item) return 1; } return 0; } void * ms_take(Ms *a) { void *item; if (!a->len) return NULL; // The result of last behaviour is that ms_take returns the oldest elements // first, exception is a row of multiple take calls without inserts on ms // of even number of elements. See the test. a->last = a->last % a->len; item = a->items[a->last]; ms_delete(a, a->last); ++a->last; return item; } beanstalkd-1.13/net.c000066400000000000000000000147341440333440000145040ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #include #include #include #include #ifdef HAVE_LIBSYSTEMD #include #endif static int set_nonblocking(int fd) { int flags, r; flags = fcntl(fd, F_GETFL, 0); if (flags < 0) { twarn("getting flags"); return -1; } r = fcntl(fd, F_SETFL, flags | O_NONBLOCK); if (r == -1) { twarn("setting O_NONBLOCK"); return -1; } return 0; } static int make_inet_socket(char *host, char *port) { int fd = -1, flags, r; struct linger linger = {0, 0}; struct addrinfo *airoot, *ai, hints; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; r = getaddrinfo(host, port, &hints, &airoot); if (r != 0) { twarnx("getaddrinfo(): %s", gai_strerror(r)); return -1; } for (ai = airoot; ai; ai = ai->ai_next) { fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol); if (fd == -1) { twarn("socket()"); continue; } r = set_nonblocking(fd); if (r == -1) { close(fd); continue; } flags = 1; r = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof flags); if (r == -1) { twarn("setting SO_REUSEADDR on fd %d", fd); close(fd); continue; } r = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &flags, sizeof flags); if (r == -1) { twarn("setting SO_KEEPALIVE on fd %d", fd); close(fd); continue; } r = setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger, sizeof linger); if (r == -1) { twarn("setting SO_LINGER on fd %d", fd); close(fd); continue; } r = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof flags); if (r == -1) { twarn("setting TCP_NODELAY on fd %d", fd); close(fd); continue; } if (host == NULL && ai->ai_family == AF_INET6) { flags = 0; r = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &flags, sizeof(flags)); if (r == -1) { twarn("setting IPV6_V6ONLY on fd %d", fd); close(fd); continue; } } r = bind(fd, ai->ai_addr, ai->ai_addrlen); if (r == -1) { twarn("bind()"); close(fd); continue; } if (verbose) { char hbuf[NI_MAXHOST], pbuf[NI_MAXSERV], *h = host, *p = port; struct sockaddr_in addr; socklen_t addrlen; addrlen = sizeof(addr); r = getsockname(fd, (struct sockaddr *) &addr, &addrlen); if (!r) { r = getnameinfo((struct sockaddr *) &addr, addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf), NI_NUMERICHOST|NI_NUMERICSERV); if (!r) { h = hbuf; p = pbuf; } } if (ai->ai_family == AF_INET6) { printf("bind %d [%s]:%s\n", fd, h, p); } else { printf("bind %d %s:%s\n", fd, h, p); } } r = listen(fd, 1024); if (r == -1) { twarn("listen()"); close(fd); continue; } break; } freeaddrinfo(airoot); if(ai == NULL) fd = -1; return fd; } static int make_unix_socket(char *path) { int fd = -1, r; struct stat st; struct sockaddr_un addr; const size_t maxlen = sizeof(addr.sun_path) - 1; // Reserve the last position for '\0' memset(&addr, 0, sizeof(struct sockaddr_un)); addr.sun_family = AF_UNIX; if (strlen(path) > maxlen) { warnx("socket path %s is too long (%ld characters), where maximum allowed is %ld", path, strlen(path), maxlen); return -1; } strncpy(addr.sun_path, path, maxlen); r = stat(path, &st); if (r == 0) { if (S_ISSOCK(st.st_mode)) { warnx("removing existing local socket to replace it"); r = unlink(path); if (r == -1) { twarn("unlink"); return -1; } } else { twarnx("another file already exists in the given path"); return -1; } } fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd == -1) { twarn("socket()"); return -1; } r = set_nonblocking(fd); if (r == -1) { close(fd); return -1; } r = bind(fd, (struct sockaddr *) &addr, sizeof(struct sockaddr_un)); if (r == -1) { twarn("bind()"); close(fd); return -1; } if (verbose) { printf("bind %d %s\n", fd, path); } r = listen(fd, 1024); if (r == -1) { twarn("listen()"); close(fd); return -1; } return fd; } int make_server_socket(char *host, char *port) { #ifdef HAVE_LIBSYSTEMD int fd = -1, r; /* See if we got a listen fd from systemd. If so, all socket options etc * are already set, so we check that the fd is a TCP or UNIX listen socket * and return. */ r = sd_listen_fds(1); if (r < 0) { twarn("sd_listen_fds"); return -1; } if (r > 0) { if (r > 1) { twarnx("inherited more than one listen socket;" " ignoring all but the first"); } fd = SD_LISTEN_FDS_START; r = sd_is_socket_inet(fd, 0, SOCK_STREAM, 1, 0); if (r < 0) { twarn("sd_is_socket_inet"); errno = -r; return -1; } if (r == 0) { r = sd_is_socket_unix(fd, SOCK_STREAM, 1, NULL, 0); if (r < 0) { twarn("sd_is_socket_unix"); errno = -r; return -1; } if (r == 0) { twarnx("inherited fd is not a TCP or UNIX listening socket"); return -1; } } return fd; } #endif if (host && !strncmp(host, "unix:", 5)) { return make_unix_socket(&host[5]); } else { return make_inet_socket(host, port); } } beanstalkd-1.13/pkg/000077500000000000000000000000001440333440000143225ustar00rootroot00000000000000beanstalkd-1.13/pkg/README.md000066400000000000000000000003761440333440000156070ustar00rootroot00000000000000How to make a release: 1. Check out master 2. Tag the commit to release devX.Y: git tag dev1.11 3. Write a change log in file News (don't commit it) 4. Run pkg/dist.sh and follow the intructions. 5. Push tags and beanstalkd.github.io 6. Run pkg/mail.sh beanstalkd-1.13/pkg/beanstalkd.spec.in000066400000000000000000000047371440333440000177260ustar00rootroot00000000000000%define beanstalkd_user beanstalkd %define beanstalkd_group %{beanstalkd_user} %define beanstalkd_home %{_localstatedir}/lib/beanstalkd %define beanstalkd_logdir %{_localstatedir}/log/beanstalkd Name: beanstalkd Version: @VERSION@ Release: 0%{?dist} Summary: A simple, fast workqueue service Group: System Environment/Daemons License: GPLv3+ URL: http://xph.us/software/%{name}/ Source0: http://xph.us/dist/%{name}/rel/%{name}-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) Requires(pre): %{_sbindir}/useradd Requires(post): /sbin/chkconfig Requires(preun): /sbin/chkconfig, /sbin/service Requires(postun): /sbin/service %description Beanstalk is a simple, fast workqueue service. Its interface is generic, but was originally designed for reducing the latency of page views in high-volume web applications by running time-consuming tasks asynchronously. %prep %setup -q if [ ! -e configure ]; then sh buildconf.sh fi %build %configure --disable-rpath --docdir=%{_defaultdocdir}/%{name}-%{version} make %{?_smp_mflags} %install rm -rf $RPM_BUILD_ROOT make install DESTDIR=$RPM_BUILD_ROOT %{__install} -p -D -m 0644 doc/%{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} %{__install} -p -D -m 0644 scripts/%{name}.sysconfig %{buildroot}%{_sysconfdir}/sysconfig/%{name} %clean rm -rf $RPM_BUILD_ROOT %pre %{_sbindir}/useradd -c "beanstalkd user" -s /bin/false -r -m -d %{beanstalkd_home} %{beanstalkd_user} 2>/dev/null || : %post /sbin/chkconfig --add %{name} %preun if [ $1 = 0 ]; then /sbin/service %{name} stop >/dev/null 2>&1 /sbin/chkconfig --del %{name} fi %postun if [ $1 -ge 1 ]; then /sbin/service %{name} condrestart > /dev/null 2>&1 || : fi %files %defattr(-,root,root,-) %doc %{_defaultdocdir}/%{name}-%{version}/protocol.txt %doc README COPYING doc/protocol.txt %{_initrddir}/%{name} %{_bindir}/%{name} %{_mandir}/man1/%{name}.1.gz %config(noreplace) %{_sysconfdir}/sysconfig/%{name} %changelog * Thu Oct 1 2009 Keith Rarick - 1.4-0 - Convert this file to an autoconf template - Tweak the summary and description * Sun Jan 4 2009 Ask Bjørn Hansen - 1.2-0 - 1.2-tobe - Use man page and .init/sysconfig scripts from .tar.gz * Sat Nov 22 2008 Jeremy Hinegardner - 1.1-1 - initial spec creation beanstalkd-1.13/pkg/bloghead.in000066400000000000000000000002761440333440000164240ustar00rootroot00000000000000--- layout: post title: Beanstalkd @VERSION@ Release Notes version: @VERSION@ dist: https://github.com/beanstalkd/beanstalkd/archive/v@VERSION@.tar.gz file: beanstalkd-@VERSION@.tar.gz --- beanstalkd-1.13/pkg/dist.sh000077500000000000000000000024151440333440000156260ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -o pipefail exp() { sed s/@VERSION@/$ver/ | sed s/@PARENT@/$prev/ } clean() { rm -f "$GIT_INDEX_FILE" } mkobj() { git hash-object -w --stdin } die() { echo >&2 "$@" exit 2 } ver=`./vers.sh` case $ver in *+*) die bad ver $ver ;; esac prev=`git describe --abbrev=0 --match=dev* --tags dev$ver^|sed s/^dev//` test -n "$prev" || die no prev ver test -f News || die no News export GIT_INDEX_FILE GIT_INDEX_FILE=`mktemp -t beanstalkd-dist-index-XXX` trap clean EXIT git read-tree dev$ver newsobj=`cat News pkg/newstail.in|exp|mkobj` versobj=`echo "printf '$ver'"|mkobj` specobj=`exp $postfile echo "Now, run these commands to update the blog:" echo "(mv $postfile ../beanstalkd.github.io/_posts/ && cd ../beanstalkd.github.io && git add . && git commit -m \"announce release $ver\")" beanstalkd-1.13/pkg/mail.sh000077500000000000000000000005131440333440000156020ustar00rootroot00000000000000#!/usr/bin/env bash set -e set -o pipefail die() { echo >&2 "$@" exit 2 } addr=beanstalk-talk@googlegroups.com ver=`./vers.sh` case $ver in *+*) die bad ver $ver ;; esac (cat < Subject: [ANN] beanstalkd $ver end beanstalkd-1.13/pkg/newstail.in000066400000000000000000000007651440333440000165100ustar00rootroot00000000000000Full list of changes (includes authorship information): Our Urls -------- Download the @VERSION@ tarball directly: Learn all about beanstalk: Talk about beanstalk development or use at: Bugs ---- Please report any bugs to: beanstalkd-1.13/primes.c000066400000000000000000000020071440333440000152030ustar00rootroot00000000000000#include // prime // downscale treshold / upscale treshold size_t primes[] = { 12289, // NA / 3072 24593, // 1537 / 6148 49193, // 3074 / 12298 98387, // 6149 / 24596 196799, // etc 393611, 787243, 1574491, 3148987, 6297979, 12595991, 25191989, 50383981, 100767977, 201535967, 403071937, 806143879, 1612287763, 3224575537UL, #if _LP64 6449151103, 12898302233, 25796604473, 51593208973, 103186417951, 206372835917, 412745671837, 825491343683, 1650982687391, 3301965374803, 6603930749621, 13207861499251, 26415722998507, 52831445997037, 105662891994103, 211325783988211, 422651567976461, 845303135952931, 1690606271905871, 3381212543811743, 6762425087623523, 13524850175247127, 27049700350494287, 54099400700988593, 108198801401977301, 216397602803954641, 432795205607909293, 865590411215818597, 1731180822431637217, #endif }; beanstalkd-1.13/prot.c000066400000000000000000001675741440333440000147150ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* job body cannot be greater than this many bytes long */ size_t job_data_size_limit = JOB_DATA_SIZE_LIMIT_DEFAULT; #define NAME_CHARS \ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" \ "abcdefghijklmnopqrstuvwxyz" \ "0123456789-+/;.$_()" #define CMD_PUT "put " #define CMD_PEEKJOB "peek " #define CMD_PEEK_READY "peek-ready" #define CMD_PEEK_DELAYED "peek-delayed" #define CMD_PEEK_BURIED "peek-buried" #define CMD_RESERVE "reserve" #define CMD_RESERVE_TIMEOUT "reserve-with-timeout " #define CMD_RESERVE_JOB "reserve-job " #define CMD_DELETE "delete " #define CMD_RELEASE "release " #define CMD_BURY "bury " #define CMD_KICK "kick " #define CMD_KICKJOB "kick-job " #define CMD_TOUCH "touch " #define CMD_STATS "stats" #define CMD_STATSJOB "stats-job " #define CMD_USE "use " #define CMD_WATCH "watch " #define CMD_IGNORE "ignore " #define CMD_LIST_TUBES "list-tubes" #define CMD_LIST_TUBE_USED "list-tube-used" #define CMD_LIST_TUBES_WATCHED "list-tubes-watched" #define CMD_STATS_TUBE "stats-tube " #define CMD_QUIT "quit" #define CMD_PAUSE_TUBE "pause-tube" #define CONSTSTRLEN(m) (sizeof(m) - 1) #define CMD_PEEK_READY_LEN CONSTSTRLEN(CMD_PEEK_READY) #define CMD_PEEK_DELAYED_LEN CONSTSTRLEN(CMD_PEEK_DELAYED) #define CMD_PEEK_BURIED_LEN CONSTSTRLEN(CMD_PEEK_BURIED) #define CMD_PEEKJOB_LEN CONSTSTRLEN(CMD_PEEKJOB) #define CMD_RESERVE_LEN CONSTSTRLEN(CMD_RESERVE) #define CMD_RESERVE_TIMEOUT_LEN CONSTSTRLEN(CMD_RESERVE_TIMEOUT) #define CMD_RESERVE_JOB_LEN CONSTSTRLEN(CMD_RESERVE_JOB) #define CMD_DELETE_LEN CONSTSTRLEN(CMD_DELETE) #define CMD_RELEASE_LEN CONSTSTRLEN(CMD_RELEASE) #define CMD_BURY_LEN CONSTSTRLEN(CMD_BURY) #define CMD_KICK_LEN CONSTSTRLEN(CMD_KICK) #define CMD_KICKJOB_LEN CONSTSTRLEN(CMD_KICKJOB) #define CMD_TOUCH_LEN CONSTSTRLEN(CMD_TOUCH) #define CMD_STATS_LEN CONSTSTRLEN(CMD_STATS) #define CMD_STATSJOB_LEN CONSTSTRLEN(CMD_STATSJOB) #define CMD_USE_LEN CONSTSTRLEN(CMD_USE) #define CMD_WATCH_LEN CONSTSTRLEN(CMD_WATCH) #define CMD_IGNORE_LEN CONSTSTRLEN(CMD_IGNORE) #define CMD_LIST_TUBES_LEN CONSTSTRLEN(CMD_LIST_TUBES) #define CMD_LIST_TUBE_USED_LEN CONSTSTRLEN(CMD_LIST_TUBE_USED) #define CMD_LIST_TUBES_WATCHED_LEN CONSTSTRLEN(CMD_LIST_TUBES_WATCHED) #define CMD_STATS_TUBE_LEN CONSTSTRLEN(CMD_STATS_TUBE) #define CMD_PAUSE_TUBE_LEN CONSTSTRLEN(CMD_PAUSE_TUBE) #define MSG_FOUND "FOUND" #define MSG_NOTFOUND "NOT_FOUND\r\n" #define MSG_RESERVED "RESERVED" #define MSG_DEADLINE_SOON "DEADLINE_SOON\r\n" #define MSG_TIMED_OUT "TIMED_OUT\r\n" #define MSG_DELETED "DELETED\r\n" #define MSG_RELEASED "RELEASED\r\n" #define MSG_BURIED "BURIED\r\n" #define MSG_KICKED "KICKED\r\n" #define MSG_TOUCHED "TOUCHED\r\n" #define MSG_BURIED_FMT "BURIED %"PRIu64"\r\n" #define MSG_INSERTED_FMT "INSERTED %"PRIu64"\r\n" #define MSG_NOT_IGNORED "NOT_IGNORED\r\n" #define MSG_OUT_OF_MEMORY "OUT_OF_MEMORY\r\n" #define MSG_INTERNAL_ERROR "INTERNAL_ERROR\r\n" #define MSG_DRAINING "DRAINING\r\n" #define MSG_BAD_FORMAT "BAD_FORMAT\r\n" #define MSG_UNKNOWN_COMMAND "UNKNOWN_COMMAND\r\n" #define MSG_EXPECTED_CRLF "EXPECTED_CRLF\r\n" #define MSG_JOB_TOO_BIG "JOB_TOO_BIG\r\n" // Connection can be in one of these states: #define STATE_WANT_COMMAND 0 // conn expects a command from the client #define STATE_WANT_DATA 1 // conn expects a job data #define STATE_SEND_JOB 2 // conn sends job to the client #define STATE_SEND_WORD 3 // conn sends a line reply #define STATE_WAIT 4 // client awaits for the job reservation #define STATE_BITBUCKET 5 // conn discards content #define STATE_CLOSE 6 // conn should be closed #define STATE_WANT_ENDLINE 7 // skip until the end of a line #define OP_UNKNOWN 0 #define OP_PUT 1 #define OP_PEEKJOB 2 #define OP_RESERVE 3 #define OP_DELETE 4 #define OP_RELEASE 5 #define OP_BURY 6 #define OP_KICK 7 #define OP_STATS 8 #define OP_STATSJOB 9 #define OP_PEEK_BURIED 10 #define OP_USE 11 #define OP_WATCH 12 #define OP_IGNORE 13 #define OP_LIST_TUBES 14 #define OP_LIST_TUBE_USED 15 #define OP_LIST_TUBES_WATCHED 16 #define OP_STATS_TUBE 17 #define OP_PEEK_READY 18 #define OP_PEEK_DELAYED 19 #define OP_RESERVE_TIMEOUT 20 #define OP_TOUCH 21 #define OP_QUIT 22 #define OP_PAUSE_TUBE 23 #define OP_KICKJOB 24 #define OP_RESERVE_JOB 25 #define TOTAL_OPS 26 #define STATS_FMT "---\n" \ "current-jobs-urgent: %" PRIu64 "\n" \ "current-jobs-ready: %" PRIu64 "\n" \ "current-jobs-reserved: %" PRIu64 "\n" \ "current-jobs-delayed: %u\n" \ "current-jobs-buried: %" PRIu64 "\n" \ "cmd-put: %" PRIu64 "\n" \ "cmd-peek: %" PRIu64 "\n" \ "cmd-peek-ready: %" PRIu64 "\n" \ "cmd-peek-delayed: %" PRIu64 "\n" \ "cmd-peek-buried: %" PRIu64 "\n" \ "cmd-reserve: %" PRIu64 "\n" \ "cmd-reserve-with-timeout: %" PRIu64 "\n" \ "cmd-delete: %" PRIu64 "\n" \ "cmd-release: %" PRIu64 "\n" \ "cmd-use: %" PRIu64 "\n" \ "cmd-watch: %" PRIu64 "\n" \ "cmd-ignore: %" PRIu64 "\n" \ "cmd-bury: %" PRIu64 "\n" \ "cmd-kick: %" PRIu64 "\n" \ "cmd-touch: %" PRIu64 "\n" \ "cmd-stats: %" PRIu64 "\n" \ "cmd-stats-job: %" PRIu64 "\n" \ "cmd-stats-tube: %" PRIu64 "\n" \ "cmd-list-tubes: %" PRIu64 "\n" \ "cmd-list-tube-used: %" PRIu64 "\n" \ "cmd-list-tubes-watched: %" PRIu64 "\n" \ "cmd-pause-tube: %" PRIu64 "\n" \ "job-timeouts: %" PRIu64 "\n" \ "total-jobs: %" PRIu64 "\n" \ "max-job-size: %zu\n" \ "current-tubes: %zu\n" \ "current-connections: %u\n" \ "current-producers: %u\n" \ "current-workers: %u\n" \ "current-waiting: %" PRIu64 "\n" \ "total-connections: %u\n" \ "pid: %ld\n" \ "version: \"%s\"\n" \ "rusage-utime: %d.%06d\n" \ "rusage-stime: %d.%06d\n" \ "uptime: %u\n" \ "binlog-oldest-index: %d\n" \ "binlog-current-index: %d\n" \ "binlog-records-migrated: %" PRId64 "\n" \ "binlog-records-written: %" PRId64 "\n" \ "binlog-max-size: %d\n" \ "draining: %s\n" \ "id: %s\n" \ "hostname: \"%s\"\n" \ "os: \"%s\"\n" \ "platform: \"%s\"\n" \ "\r\n" #define STATS_TUBE_FMT "---\n" \ "name: \"%s\"\n" \ "current-jobs-urgent: %" PRIu64 "\n" \ "current-jobs-ready: %zu\n" \ "current-jobs-reserved: %" PRIu64 "\n" \ "current-jobs-delayed: %zu\n" \ "current-jobs-buried: %" PRIu64 "\n" \ "total-jobs: %" PRIu64 "\n" \ "current-using: %u\n" \ "current-watching: %u\n" \ "current-waiting: %" PRIu64 "\n" \ "cmd-delete: %" PRIu64 "\n" \ "cmd-pause-tube: %" PRIu64 "\n" \ "pause: %" PRIu64 "\n" \ "pause-time-left: %" PRId64 "\n" \ "\r\n" #define STATS_JOB_FMT "---\n" \ "id: %" PRIu64 "\n" \ "tube: \"%s\"\n" \ "state: %s\n" \ "pri: %u\n" \ "age: %" PRId64 "\n" \ "delay: %" PRId64 "\n" \ "ttr: %" PRId64 "\n" \ "time-left: %" PRId64 "\n" \ "file: %d\n" \ "reserves: %u\n" \ "timeouts: %u\n" \ "releases: %u\n" \ "buries: %u\n" \ "kicks: %u\n" \ "\r\n" // The size of the throw-away (BITBUCKET) buffer. Arbitrary. #define BUCKET_BUF_SIZE 1024 static uint64 ready_ct = 0; static uint64 timeout_ct = 0; static uint64 op_ct[TOTAL_OPS] = {0}; static struct stats global_stat = {0}; static Tube *default_tube; // If drain_mode is 1, then server does not accept new jobs. // Variable is set by the SIGUSR1 handler. static volatile sig_atomic_t drain_mode = 0; static int64 started_at; enum { instance_id_bytes = 8 }; static char instance_hex[instance_id_bytes * 2 + 1]; // hex-encoded len of instance_id_bytes static struct utsname node_info; // Single linked list with connections that require updates // in the event notification mechanism. static Conn *epollq; static const char * op_names[] = { "", CMD_PUT, CMD_PEEKJOB, CMD_RESERVE, CMD_DELETE, CMD_RELEASE, CMD_BURY, CMD_KICK, CMD_STATS, CMD_STATSJOB, CMD_PEEK_BURIED, CMD_USE, CMD_WATCH, CMD_IGNORE, CMD_LIST_TUBES, CMD_LIST_TUBE_USED, CMD_LIST_TUBES_WATCHED, CMD_STATS_TUBE, CMD_PEEK_READY, CMD_PEEK_DELAYED, CMD_RESERVE_TIMEOUT, CMD_TOUCH, CMD_QUIT, CMD_PAUSE_TUBE, CMD_KICKJOB, CMD_RESERVE_JOB, }; static Job *remove_ready_job(Job *j); static Job *remove_buried_job(Job *j); // epollq_add schedules connection c in the s->conns heap, adds c // to the epollq list to change expected operation in event notifications. // rw='w' means to notify when socket is writeable, 'r' - readable, 'h' - closed. static void epollq_add(Conn *c, char rw) { c->rw = rw; connsched(c); c->next = epollq; epollq = c; } // epollq_rmconn removes connection c from the epollq. static void epollq_rmconn(Conn *c) { Conn *x, *newhead = NULL; while (epollq) { // x as next element from epollq. x = epollq; epollq = epollq->next; x->next = NULL; // put x back into newhead list. if (x != c) { x->next = newhead; newhead = x; } } epollq = newhead; } // Propagate changes to event notification mechanism about expected operations // in connections' sockets. Clear the epollq list. static void epollq_apply() { Conn *c; while (epollq) { c = epollq; epollq = epollq->next; c->next = NULL; int r = sockwant(&c->sock, c->rw); if (r == -1) { twarn("sockwant"); connclose(c); } } } #define reply_msg(c, m) \ reply((c), (m), CONSTSTRLEN(m), STATE_SEND_WORD) #define reply_serr(c, e) \ (twarnx("server error: %s", (e)), reply_msg((c), (e))) static void reply(Conn *c, char *line, int len, int state) { if (!c) return; epollq_add(c, 'w'); c->reply = line; c->reply_len = len; c->reply_sent = 0; c->state = state; if (verbose >= 2) { printf(">%d reply %.*s\n", c->sock.fd, len-2, line); } } static void reply_line(Conn*, int, const char*, ...) __attribute__((format(printf, 3, 4))); // reply_line prints *fmt into c->reply_buffer and // calls reply() for the string and state. static void reply_line(Conn *c, int state, const char *fmt, ...) { int r; va_list ap; va_start(ap, fmt); r = vsnprintf(c->reply_buf, LINE_BUF_SIZE, fmt, ap); va_end(ap); /* Make sure the buffer was big enough. If not, we have a bug. */ if (r >= LINE_BUF_SIZE) { reply_serr(c, MSG_INTERNAL_ERROR); return; } reply(c, c->reply_buf, r, state); } // reply_job tells the connection c which job to send, // and replies with this line: . static void reply_job(Conn *c, Job *j, const char *msg) { c->out_job = j; c->out_job_sent = 0; reply_line(c, STATE_SEND_JOB, "%s %"PRIu64" %u\r\n", msg, j->r.id, j->r.body_size - 2); } // remove_waiting_conn unsets CONN_TYPE_WAITING for the connection, // removes it from the waiting_conns set of every tube it's watching. // Noop if connection is not waiting. void remove_waiting_conn(Conn *c) { if (!conn_waiting(c)) return; c->type &= ~CONN_TYPE_WAITING; global_stat.waiting_ct--; size_t i; for (i = 0; i < c->watch.len; i++) { Tube *t = c->watch.items[i]; t->stat.waiting_ct--; ms_remove(&t->waiting_conns, c); } } // enqueue_waiting_conn sets CONN_TYPE_WAITING for the connection, // adds it to the waiting_conns set of every tube it's watching. static void enqueue_waiting_conn(Conn *c) { c->type |= CONN_TYPE_WAITING; global_stat.waiting_ct++; size_t i; for (i = 0; i < c->watch.len; i++) { Tube *t = c->watch.items[i]; t->stat.waiting_ct++; ms_append(&t->waiting_conns, c); } } // next_awaited_job iterates through all the tubes with awaiting connections, // returns the next ready job with the smallest priority. // If jobs has the same priority it picks the job with smaller id. // All tubes with expired pause are unpaused. static Job * next_awaited_job(int64 now) { size_t i; Job *j = NULL; for (i = 0; i < tubes.len; i++) { Tube *t = tubes.items[i]; if (t->pause) { if (t->unpause_at > now) continue; t->pause = 0; } if (t->waiting_conns.len && t->ready.len) { Job *candidate = t->ready.data[0]; if (!j || job_pri_less(candidate, j)) { j = candidate; } } } return j; } // process_queue performs reservation for every jobs that is awaited for. static void process_queue() { Job *j = NULL; int64 now = nanoseconds(); while ((j = next_awaited_job(now))) { j = remove_ready_job(j); if (j == NULL) { twarnx("job not ready"); continue; } Conn *c = ms_take(&j->tube->waiting_conns); if (c == NULL) { twarnx("waiting_conns is empty"); continue; } global_stat.reserved_ct++; remove_waiting_conn(c); conn_reserve_job(c, j); reply_job(c, j, MSG_RESERVED); } } // soonest_delayed_job returns the delayed job // with the smallest deadline_at among all tubes. static Job * soonest_delayed_job() { Job *j = NULL; size_t i; for (i = 0; i < tubes.len; i++) { Tube *t = tubes.items[i]; if (t->delay.len == 0) { continue; } Job *nj = t->delay.data[0]; if (!j || nj->r.deadline_at < j->r.deadline_at) j = nj; } return j; } // enqueue_job inserts job j in the tube, returns 1 on success, otherwise 0. // If update_store then it writes an entry to WAL. // On success it processes the queue. // BUG: If maintenance of WAL has failed, it is not reported as error. static int enqueue_job(Server *s, Job *j, int64 delay, char update_store) { int r; j->reserver = NULL; if (delay) { j->r.deadline_at = nanoseconds() + delay; r = heapinsert(&j->tube->delay, j); if (!r) return 0; j->r.state = Delayed; } else { r = heapinsert(&j->tube->ready, j); if (!r) return 0; j->r.state = Ready; ready_ct++; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct++; j->tube->stat.urgent_ct++; } } if (update_store) { if (!walwrite(&s->wal, j)) { return 0; } walmaint(&s->wal); } // The call below makes this function do too much. // TODO: refactor this call outside so the call is explicit (not hidden)? process_queue(); return 1; } static int bury_job(Server *s, Job *j, char update_store) { if (update_store) { int z = walresvupdate(&s->wal); if (!z) return 0; j->walresv += z; } job_list_insert(&j->tube->buried, j); global_stat.buried_ct++; j->tube->stat.buried_ct++; j->r.state = Buried; j->reserver = NULL; j->r.bury_ct++; if (update_store) { if (!walwrite(&s->wal, j)) { return 0; } walmaint(&s->wal); } return 1; } void enqueue_reserved_jobs(Conn *c) { while (!job_list_is_empty(&c->reserved_jobs)) { Job *j = job_list_remove(c->reserved_jobs.next); int r = enqueue_job(c->srv, j, 0, 0); if (r < 1) bury_job(c->srv, j, 0); global_stat.reserved_ct--; j->tube->stat.reserved_ct--; c->soonest_job = NULL; } } static int kick_buried_job(Server *s, Job *j) { int r; int z; z = walresvupdate(&s->wal); if (!z) return 0; j->walresv += z; remove_buried_job(j); j->r.kick_ct++; r = enqueue_job(s, j, 0, 1); if (r == 1) return 1; /* ready queue is full, so bury it */ bury_job(s, j, 0); return 0; } static uint get_delayed_job_ct() { size_t i; uint count = 0; for (i = 0; i < tubes.len; i++) { Tube *t = tubes.items[i]; count += t->delay.len; } return count; } static int kick_delayed_job(Server *s, Job *j) { int r; int z; z = walresvupdate(&s->wal); if (!z) return 0; j->walresv += z; heapremove(&j->tube->delay, j->heap_index); j->r.kick_ct++; r = enqueue_job(s, j, 0, 1); if (r == 1) return 1; /* ready queue is full, so delay it again */ r = enqueue_job(s, j, j->r.delay, 0); if (r == 1) return 0; /* last resort */ bury_job(s, j, 0); return 0; } static int buried_job_p(Tube *t) { // this function does not do much. inline? return !job_list_is_empty(&t->buried); } /* return the number of jobs successfully kicked */ static uint kick_buried_jobs(Server *s, Tube *t, uint n) { uint i; for (i = 0; (i < n) && buried_job_p(t); ++i) { kick_buried_job(s, t->buried.next); } return i; } /* return the number of jobs successfully kicked */ static uint kick_delayed_jobs(Server *s, Tube *t, uint n) { uint i; for (i = 0; (i < n) && (t->delay.len > 0); ++i) { kick_delayed_job(s, (Job *)t->delay.data[0]); } return i; } static uint kick_jobs(Server *s, Tube *t, uint n) { if (buried_job_p(t)) return kick_buried_jobs(s, t, n); return kick_delayed_jobs(s, t, n); } // remove_buried_job returns non-NULL value if job j was in the buried state. // It excludes the job from the buried list and updates counters. static Job * remove_buried_job(Job *j) { if (!j || j->r.state != Buried) return NULL; j = job_list_remove(j); if (j) { global_stat.buried_ct--; j->tube->stat.buried_ct--; } return j; } // remove_delayed_job returns non-NULL value if job j was in the delayed state. // It removes the job from the tube delayed heap. static Job * remove_delayed_job(Job *j) { if (!j || j->r.state != Delayed) return NULL; heapremove(&j->tube->delay, j->heap_index); return j; } // remove_ready_job returns non-NULL value if job j was in the ready state. // It removes the job from the tube ready heap and updates counters. static Job * remove_ready_job(Job *j) { if (!j || j->r.state != Ready) return NULL; heapremove(&j->tube->ready, j->heap_index); ready_ct--; if (j->r.pri < URGENT_THRESHOLD) { global_stat.urgent_ct--; j->tube->stat.urgent_ct--; } return j; } static bool is_job_reserved_by_conn(Conn *c, Job *j) { return j && j->reserver == c && j->r.state == Reserved; } static bool touch_job(Conn *c, Job *j) { if (is_job_reserved_by_conn(c, j)) { j->r.deadline_at = nanoseconds() + j->r.ttr; c->soonest_job = NULL; return true; } return false; } static void check_err(Conn *c, const char *s) { if (errno == EAGAIN) return; if (errno == EINTR) return; if (errno == EWOULDBLOCK) return; twarn("%s", s); c->state = STATE_CLOSE; } /* Scan the given string for the sequence "\r\n" and return the line length. * Always returns at least 2 if a match is found. Returns 0 if no match. */ static size_t scan_line_end(const char *s, int size) { char *match; match = memchr(s, '\r', size - 1); if (!match) return 0; /* this is safe because we only scan size - 1 chars above */ if (match[1] == '\n') return match - s + 2; return 0; } /* parse the command line */ static int which_cmd(Conn *c) { #define TEST_CMD(s,c,o) if (strncmp((s), (c), CONSTSTRLEN(c)) == 0) return (o); TEST_CMD(c->cmd, CMD_PUT, OP_PUT); TEST_CMD(c->cmd, CMD_PEEKJOB, OP_PEEKJOB); TEST_CMD(c->cmd, CMD_PEEK_READY, OP_PEEK_READY); TEST_CMD(c->cmd, CMD_PEEK_DELAYED, OP_PEEK_DELAYED); TEST_CMD(c->cmd, CMD_PEEK_BURIED, OP_PEEK_BURIED); TEST_CMD(c->cmd, CMD_RESERVE_TIMEOUT, OP_RESERVE_TIMEOUT); TEST_CMD(c->cmd, CMD_RESERVE_JOB, OP_RESERVE_JOB); TEST_CMD(c->cmd, CMD_RESERVE, OP_RESERVE); TEST_CMD(c->cmd, CMD_DELETE, OP_DELETE); TEST_CMD(c->cmd, CMD_RELEASE, OP_RELEASE); TEST_CMD(c->cmd, CMD_BURY, OP_BURY); TEST_CMD(c->cmd, CMD_KICK, OP_KICK); TEST_CMD(c->cmd, CMD_KICKJOB, OP_KICKJOB); TEST_CMD(c->cmd, CMD_TOUCH, OP_TOUCH); TEST_CMD(c->cmd, CMD_STATSJOB, OP_STATSJOB); TEST_CMD(c->cmd, CMD_STATS_TUBE, OP_STATS_TUBE); TEST_CMD(c->cmd, CMD_STATS, OP_STATS); TEST_CMD(c->cmd, CMD_USE, OP_USE); TEST_CMD(c->cmd, CMD_WATCH, OP_WATCH); TEST_CMD(c->cmd, CMD_IGNORE, OP_IGNORE); TEST_CMD(c->cmd, CMD_LIST_TUBES_WATCHED, OP_LIST_TUBES_WATCHED); TEST_CMD(c->cmd, CMD_LIST_TUBE_USED, OP_LIST_TUBE_USED); TEST_CMD(c->cmd, CMD_LIST_TUBES, OP_LIST_TUBES); TEST_CMD(c->cmd, CMD_QUIT, OP_QUIT); TEST_CMD(c->cmd, CMD_PAUSE_TUBE, OP_PAUSE_TUBE); return OP_UNKNOWN; } /* Copy up to body_size trailing bytes into the job, then the rest into the cmd * buffer. If c->in_job exists, this assumes that c->in_job->body is empty. * This function is idempotent(). */ static void fill_extra_data(Conn *c) { if (!c->sock.fd) return; /* the connection was closed */ if (!c->cmd_len) return; /* we don't have a complete command */ /* how many extra bytes did we read? */ int64 extra_bytes = c->cmd_read - c->cmd_len; int64 job_data_bytes = 0; /* how many bytes should we put into the job body? */ if (c->in_job) { job_data_bytes = min(extra_bytes, c->in_job->r.body_size); memcpy(c->in_job->body, c->cmd + c->cmd_len, job_data_bytes); c->in_job_read = job_data_bytes; } else if (c->in_job_read) { /* we are in bit-bucket mode, throwing away data */ job_data_bytes = min(extra_bytes, c->in_job_read); c->in_job_read -= job_data_bytes; } /* how many bytes are left to go into the future cmd? */ int64 cmd_bytes = extra_bytes - job_data_bytes; memmove(c->cmd, c->cmd + c->cmd_len + job_data_bytes, cmd_bytes); c->cmd_read = cmd_bytes; c->cmd_len = 0; /* we no longer know the length of the new command */ } #define skip(conn,n,msg) (_skip(conn, n, msg, CONSTSTRLEN(msg))) static void _skip(Conn *c, int64 n, char *msg, int msglen) { /* Invert the meaning of in_job_read while throwing away data -- it * counts the bytes that remain to be thrown away. */ c->in_job = 0; c->in_job_read = n; fill_extra_data(c); if (c->in_job_read == 0) { reply(c, msg, msglen, STATE_SEND_WORD); return; } c->reply = msg; c->reply_len = msglen; c->reply_sent = 0; c->state = STATE_BITBUCKET; } static void enqueue_incoming_job(Conn *c) { int r; Job *j = c->in_job; c->in_job = NULL; /* the connection no longer owns this job */ c->in_job_read = 0; /* check if the trailer is present and correct */ if (memcmp(j->body + j->r.body_size - 2, "\r\n", 2)) { job_free(j); reply_msg(c, MSG_EXPECTED_CRLF); return; } if (verbose >= 2) { printf("<%d job %"PRIu64"\n", c->sock.fd, j->r.id); } if (drain_mode) { job_free(j); reply_serr(c, MSG_DRAINING); return; } if (j->walresv) { reply_serr(c, MSG_INTERNAL_ERROR); return; } j->walresv = walresvput(&c->srv->wal, j); if (!j->walresv) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } /* we have a complete job, so let's stick it in the pqueue */ r = enqueue_job(c->srv, j, j->r.delay, 1); // Dead code: condition cannot happen, r can take 1 or 0 values only. if (r < 0) { reply_serr(c, MSG_INTERNAL_ERROR); return; } global_stat.total_jobs_ct++; j->tube->stat.total_jobs_ct++; if (r == 1) { reply_line(c, STATE_SEND_WORD, MSG_INSERTED_FMT, j->r.id); return; } /* out of memory trying to grow the queue, so it gets buried */ bury_job(c->srv, j, 0); reply_line(c, STATE_SEND_WORD, MSG_BURIED_FMT, j->r.id); } static uint uptime() { return (nanoseconds() - started_at) / 1000000000; } static int fmt_stats(char *buf, size_t size, void *x) { int whead = 0, wcur = 0; Server *s = x; struct rusage ru; s = x; if (s->wal.head) { whead = s->wal.head->seq; } if (s->wal.cur) { wcur = s->wal.cur->seq; } getrusage(RUSAGE_SELF, &ru); /* don't care if it fails */ return snprintf(buf, size, STATS_FMT, global_stat.urgent_ct, ready_ct, global_stat.reserved_ct, get_delayed_job_ct(), global_stat.buried_ct, op_ct[OP_PUT], op_ct[OP_PEEKJOB], op_ct[OP_PEEK_READY], op_ct[OP_PEEK_DELAYED], op_ct[OP_PEEK_BURIED], op_ct[OP_RESERVE], op_ct[OP_RESERVE_TIMEOUT], op_ct[OP_DELETE], op_ct[OP_RELEASE], op_ct[OP_USE], op_ct[OP_WATCH], op_ct[OP_IGNORE], op_ct[OP_BURY], op_ct[OP_KICK], op_ct[OP_TOUCH], op_ct[OP_STATS], op_ct[OP_STATSJOB], op_ct[OP_STATS_TUBE], op_ct[OP_LIST_TUBES], op_ct[OP_LIST_TUBE_USED], op_ct[OP_LIST_TUBES_WATCHED], op_ct[OP_PAUSE_TUBE], timeout_ct, global_stat.total_jobs_ct, job_data_size_limit, tubes.len, count_cur_conns(), count_cur_producers(), count_cur_workers(), global_stat.waiting_ct, count_tot_conns(), (long) getpid(), version, (int) ru.ru_utime.tv_sec, (int) ru.ru_utime.tv_usec, (int) ru.ru_stime.tv_sec, (int) ru.ru_stime.tv_usec, uptime(), whead, wcur, s->wal.nmig, s->wal.nrec, s->wal.filesize, drain_mode ? "true" : "false", instance_hex, node_info.nodename, node_info.version, node_info.machine); } /* Read an integer from the given buffer and place it in num. * Parsed integer should fit into uint64. * Update end to point to the address after the last character consumed. * num and end can be NULL. If they are both NULL, read_u64() will do the * conversion and return the status code but not update any values. * This is an easy way to check for errors. * If end is NULL, read_u64() will also check that the entire input string * was consumed and return an error code otherwise. * Return 0 on success, or nonzero on failure. * If a failure occurs, num and end are not modified. */ static int read_u64(uint64 *num, const char *buf, char **end) { uintmax_t tnum; char *tend; errno = 0; while (buf[0] == ' ') buf++; if (buf[0] < '0' || '9' < buf[0]) return -1; tnum = strtoumax(buf, &tend, 10); if (tend == buf) return -1; if (errno) return -1; if (!end && tend[0] != '\0') return -1; if (tnum > UINT64_MAX) return -1; if (num) *num = (uint64)tnum; if (end) *end = tend; return 0; } // Indentical to read_u64() but instead reads into uint32. static int read_u32(uint32 *num, const char *buf, char **end) { uintmax_t tnum; char *tend; errno = 0; while (buf[0] == ' ') buf++; if (buf[0] < '0' || '9' < buf[0]) return -1; tnum = strtoumax(buf, &tend, 10); if (tend == buf) return -1; if (errno) return -1; if (!end && tend[0] != '\0') return -1; if (tnum > UINT32_MAX) return -1; if (num) *num = (uint32)tnum; if (end) *end = tend; return 0; } /* Read a delay value in seconds from the given buffer and place it in duration in nanoseconds. The interface and behavior are analogous to read_u32(). */ static int read_duration(int64 *duration, const char *buf, char **end) { int r; uint32 dur_sec; r = read_u32(&dur_sec, buf, end); if (r) return r; *duration = ((int64) dur_sec) * 1000000000; return 0; } /* Read a tube name from the given buffer moving the buffer to the name start */ static int read_tube_name(char **tubename, char *buf, char **end) { size_t len; while (buf[0] == ' ') buf++; len = strspn(buf, NAME_CHARS); if (len == 0) return -1; if (tubename) *tubename = buf; if (end) *end = buf + len; return 0; } static void wait_for_job(Conn *c, int timeout) { c->state = STATE_WAIT; enqueue_waiting_conn(c); /* Set the pending timeout to the requested timeout amount */ c->pending_timeout = timeout; // only care if they hang up epollq_add(c, 'h'); } typedef int(*fmt_fn)(char *, size_t, void *); static void do_stats(Conn *c, fmt_fn fmt, void *data) { int r, stats_len; /* first, measure how big a buffer we will need */ stats_len = fmt(NULL, 0, data) + 16; c->out_job = allocate_job(stats_len); /* fake job to hold stats data */ if (!c->out_job) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } /* Mark this job as a copy so it can be appropriately freed later on */ c->out_job->r.state = Copy; /* now actually format the stats data */ r = fmt(c->out_job->body, stats_len, data); /* and set the actual body size */ c->out_job->r.body_size = r; if (r > stats_len) { reply_serr(c, MSG_INTERNAL_ERROR); return; } c->out_job_sent = 0; reply_line(c, STATE_SEND_JOB, "OK %d\r\n", r - 2); } static void do_list_tubes(Conn *c, Ms *l) { char *buf; Tube *t; size_t i, resp_z; /* first, measure how big a buffer we will need */ resp_z = 6; /* initial "---\n" and final "\r\n" */ for (i = 0; i < l->len; i++) { t = l->items[i]; resp_z += 3 + strlen(t->name); /* including "- " and "\n" */ } c->out_job = allocate_job(resp_z); /* fake job to hold response data */ if (!c->out_job) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } /* Mark this job as a copy so it can be appropriately freed later on */ c->out_job->r.state = Copy; /* now actually format the response */ buf = c->out_job->body; buf += snprintf(buf, 5, "---\n"); for (i = 0; i < l->len; i++) { t = l->items[i]; buf += snprintf(buf, 4 + strlen(t->name), "- %s\n", t->name); } buf[0] = '\r'; buf[1] = '\n'; c->out_job_sent = 0; reply_line(c, STATE_SEND_JOB, "OK %zu\r\n", resp_z - 2); } static int fmt_job_stats(char *buf, size_t size, Job *j) { int64 t; int64 time_left; int file = 0; t = nanoseconds(); if (j->r.state == Reserved || j->r.state == Delayed) { time_left = (j->r.deadline_at - t) / 1000000000; } else { time_left = 0; } if (j->file) { file = j->file->seq; } return snprintf(buf, size, STATS_JOB_FMT, j->r.id, j->tube->name, job_state(j), j->r.pri, (t - j->r.created_at) / 1000000000, j->r.delay / 1000000000, j->r.ttr / 1000000000, time_left, file, j->r.reserve_ct, j->r.timeout_ct, j->r.release_ct, j->r.bury_ct, j->r.kick_ct); } static int fmt_stats_tube(char *buf, size_t size, Tube *t) { uint64 time_left; if (t->pause > 0) { time_left = (t->unpause_at - nanoseconds()) / 1000000000; } else { time_left = 0; } return snprintf(buf, size, STATS_TUBE_FMT, t->name, t->stat.urgent_ct, t->ready.len, t->stat.reserved_ct, t->delay.len, t->stat.buried_ct, t->stat.total_jobs_ct, t->using_ct, t->watching_ct, t->stat.waiting_ct, t->stat.total_delete_ct, t->stat.pause_ct, t->pause / 1000000000, time_left); } static void maybe_enqueue_incoming_job(Conn *c) { Job *j = c->in_job; /* do we have a complete job? */ if (c->in_job_read == j->r.body_size) { enqueue_incoming_job(c); return; } /* otherwise we have incomplete data, so just keep waiting */ c->state = STATE_WANT_DATA; } /* j can be NULL */ static Job * remove_this_reserved_job(Conn *c, Job *j) { j = job_list_remove(j); if (j) { global_stat.reserved_ct--; j->tube->stat.reserved_ct--; j->reserver = NULL; } c->soonest_job = NULL; return j; } static Job * remove_reserved_job(Conn *c, Job *j) { if (!is_job_reserved_by_conn(c, j)) return NULL; return remove_this_reserved_job(c, j); } static bool is_valid_tube(const char *name, size_t max) { size_t len = strlen(name); return 0 < len && len <= max && strspn(name, NAME_CHARS) == len && name[0] != '-'; } static void dispatch_cmd(Conn *c) { int r, timeout = -1; uint i; uint count; Job *j = 0; byte type; char *size_buf, *delay_buf, *ttr_buf, *pri_buf, *end_buf, *name; uint32 pri; uint32 body_size; int64 delay, ttr; uint64 id; Tube *t = NULL; /* NUL-terminate this string so we can use strtol and friends */ c->cmd[c->cmd_len - 2] = '\0'; /* check for possible maliciousness */ if (strlen(c->cmd) != c->cmd_len - 2) { reply_msg(c, MSG_BAD_FORMAT); return; } type = which_cmd(c); if (verbose >= 2) { printf("<%d command %s\n", c->sock.fd, op_names[type]); } switch (type) { case OP_PUT: if (read_u32(&pri, c->cmd + 4, &delay_buf) || read_duration(&delay, delay_buf, &ttr_buf) || read_duration(&ttr, ttr_buf, &size_buf) || read_u32(&body_size, size_buf, &end_buf)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; if (body_size > job_data_size_limit) { /* throw away the job body and respond with JOB_TOO_BIG */ skip(c, (int64)body_size + 2, MSG_JOB_TOO_BIG); return; } /* don't allow trailing garbage */ if (end_buf[0] != '\0') { reply_msg(c, MSG_BAD_FORMAT); return; } connsetproducer(c); if (ttr < 1000000000) { ttr = 1000000000; } c->in_job = make_job(pri, delay, ttr, body_size + 2, c->use); /* OOM? */ if (!c->in_job) { /* throw away the job body and respond with OUT_OF_MEMORY */ twarnx("server error: " MSG_OUT_OF_MEMORY); skip(c, body_size + 2, MSG_OUT_OF_MEMORY); return; } fill_extra_data(c); /* it's possible we already have a complete job */ maybe_enqueue_incoming_job(c); return; case OP_PEEK_READY: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_READY_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; if (c->use->ready.len) { j = job_copy(c->use->ready.data[0]); } if (!j) { reply_msg(c, MSG_NOTFOUND); return; } reply_job(c, j, MSG_FOUND); return; case OP_PEEK_DELAYED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_DELAYED_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; if (c->use->delay.len) { j = job_copy(c->use->delay.data[0]); } if (!j) { reply_msg(c, MSG_NOTFOUND); return; } reply_job(c, j, MSG_FOUND); return; case OP_PEEK_BURIED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_PEEK_BURIED_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; if (buried_job_p(c->use)) j = job_copy(c->use->buried.next); else j = NULL; if (!j) { reply_msg(c, MSG_NOTFOUND); return; } reply_job(c, j, MSG_FOUND); return; case OP_PEEKJOB: if (read_u64(&id, c->cmd + CMD_PEEKJOB_LEN, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; /* So, peek is annoying, because some other connection might free the * job while we are still trying to write it out. So we copy it and * free the copy when it's done sending, in the "conn_want_command" function. */ j = job_copy(job_find(id)); if (!j) { reply_msg(c, MSG_NOTFOUND); return; } reply_job(c, j, MSG_FOUND); return; case OP_RESERVE_TIMEOUT: errno = 0; uint32 utimeout = 0; if (read_u32(&utimeout, c->cmd + CMD_RESERVE_TIMEOUT_LEN, &end_buf) != 0 || utimeout > INT_MAX) { reply_msg(c, MSG_BAD_FORMAT); return; } timeout = (int)utimeout; /* Falls through */ case OP_RESERVE: /* don't allow trailing garbage */ if (type == OP_RESERVE && c->cmd_len != CMD_RESERVE_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; connsetworker(c); if (conndeadlinesoon(c) && !conn_ready(c)) { reply_msg(c, MSG_DEADLINE_SOON); return; } /* try to get a new job for this guy */ wait_for_job(c, timeout); process_queue(); return; case OP_RESERVE_JOB: if (read_u64(&id, c->cmd + CMD_RESERVE_JOB_LEN, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; // This command could produce "deadline soon" if // the connection has a reservation about to expire. // We choose not to do it, because this command does not block // for an arbitrary amount of time as reserve and reserve-with-timeout. j = job_find(id); if (!j) { reply_msg(c, MSG_NOTFOUND); return; } // Check if this job is already reserved. if (j->r.state == Reserved || j->r.state == Invalid) { reply_msg(c, MSG_NOTFOUND); return; } // Job can be in ready, buried or delayed states. if (j->r.state == Ready) { j = remove_ready_job(j); } else if (j->r.state == Buried) { j = remove_buried_job(j); } else if (j->r.state == Delayed) { j = remove_delayed_job(j); } else { reply_serr(c, MSG_INTERNAL_ERROR); return; } connsetworker(c); global_stat.reserved_ct++; conn_reserve_job(c, j); reply_job(c, j, MSG_RESERVED); return; case OP_DELETE: if (read_u64(&id, c->cmd + CMD_DELETE_LEN, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; { Job *jf = job_find(id); j = remove_reserved_job(c, jf); if (!j) j = remove_ready_job(jf); if (!j) j = remove_buried_job(jf); if (!j) j = remove_delayed_job(jf); } if (!j) { reply_msg(c, MSG_NOTFOUND); return; } j->tube->stat.total_delete_ct++; j->r.state = Invalid; r = walwrite(&c->srv->wal, j); walmaint(&c->srv->wal); job_free(j); if (!r) { reply_serr(c, MSG_INTERNAL_ERROR); return; } reply_msg(c, MSG_DELETED); return; case OP_RELEASE: if (read_u64(&id, c->cmd + CMD_RELEASE_LEN, &pri_buf) || read_u32(&pri, pri_buf, &delay_buf) || read_duration(&delay, delay_buf, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; j = remove_reserved_job(c, job_find(id)); if (!j) { reply_msg(c, MSG_NOTFOUND); return; } /* We want to update the delay deadline on disk, so reserve space for * that. */ if (delay) { int z = walresvupdate(&c->srv->wal); if (!z) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } j->walresv += z; } j->r.pri = pri; j->r.delay = delay; j->r.release_ct++; r = enqueue_job(c->srv, j, delay, !!delay); if (r < 0) { reply_serr(c, MSG_INTERNAL_ERROR); return; } if (r == 1) { reply_msg(c, MSG_RELEASED); return; } /* out of memory trying to grow the queue, so it gets buried */ bury_job(c->srv, j, 0); reply_msg(c, MSG_BURIED); return; case OP_BURY: if (read_u64(&id, c->cmd + CMD_BURY_LEN, &pri_buf) || read_u32(&pri, pri_buf, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; j = remove_reserved_job(c, job_find(id)); if (!j) { reply_msg(c, MSG_NOTFOUND); return; } j->r.pri = pri; r = bury_job(c->srv, j, 1); if (!r) { reply_serr(c, MSG_INTERNAL_ERROR); return; } reply_msg(c, MSG_BURIED); return; case OP_KICK: errno = 0; count = strtoul(c->cmd + CMD_KICK_LEN, &end_buf, 10); if (end_buf == c->cmd + CMD_KICK_LEN || errno) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; i = kick_jobs(c->srv, c->use, count); reply_line(c, STATE_SEND_WORD, "KICKED %u\r\n", i); return; case OP_KICKJOB: if (read_u64(&id, c->cmd + CMD_KICKJOB_LEN, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; j = job_find(id); if (!j) { reply_msg(c, MSG_NOTFOUND); return; } if ((j->r.state == Buried && kick_buried_job(c->srv, j)) || (j->r.state == Delayed && kick_delayed_job(c->srv, j))) { reply_msg(c, MSG_KICKED); } else { reply_msg(c, MSG_NOTFOUND); } return; case OP_TOUCH: if (read_u64(&id, c->cmd + CMD_TOUCH_LEN, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; if (touch_job(c, job_find(id))) { reply_msg(c, MSG_TOUCHED); } else { reply_msg(c, MSG_NOTFOUND); } return; case OP_STATS: /* don't allow trailing garbage */ if (c->cmd_len != CMD_STATS_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; do_stats(c, fmt_stats, c->srv); return; case OP_STATSJOB: if (read_u64(&id, c->cmd + CMD_STATSJOB_LEN, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; j = job_find(id); if (!j) { reply_msg(c, MSG_NOTFOUND); return; } if (!j->tube) { reply_serr(c, MSG_INTERNAL_ERROR); return; } do_stats(c, (fmt_fn) fmt_job_stats, j); return; case OP_STATS_TUBE: name = c->cmd + CMD_STATS_TUBE_LEN; if (!is_valid_tube(name, MAX_TUBE_NAME_LEN - 1)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; t = tube_find(&tubes, name); if (!t) { reply_msg(c, MSG_NOTFOUND); return; } do_stats(c, (fmt_fn) fmt_stats_tube, t); t = NULL; return; case OP_LIST_TUBES: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBES_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; do_list_tubes(c, &tubes); return; case OP_LIST_TUBE_USED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBE_USED_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; reply_line(c, STATE_SEND_WORD, "USING %s\r\n", c->use->name); return; case OP_LIST_TUBES_WATCHED: /* don't allow trailing garbage */ if (c->cmd_len != CMD_LIST_TUBES_WATCHED_LEN + 2) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; do_list_tubes(c, &c->watch); return; case OP_USE: name = c->cmd + CMD_USE_LEN; if (!is_valid_tube(name, MAX_TUBE_NAME_LEN - 1)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; TUBE_ASSIGN(t, tube_find_or_make(name)); if (!t) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } c->use->using_ct--; TUBE_ASSIGN(c->use, t); TUBE_ASSIGN(t, NULL); c->use->using_ct++; reply_line(c, STATE_SEND_WORD, "USING %s\r\n", c->use->name); return; case OP_WATCH: name = c->cmd + CMD_WATCH_LEN; if (!is_valid_tube(name, MAX_TUBE_NAME_LEN - 1)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; TUBE_ASSIGN(t, tube_find_or_make(name)); if (!t) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } r = 1; if (!ms_contains(&c->watch, t)) r = ms_append(&c->watch, t); TUBE_ASSIGN(t, NULL); if (!r) { reply_serr(c, MSG_OUT_OF_MEMORY); return; } reply_line(c, STATE_SEND_WORD, "WATCHING %zu\r\n", c->watch.len); return; case OP_IGNORE: name = c->cmd + CMD_IGNORE_LEN; if (!is_valid_tube(name, MAX_TUBE_NAME_LEN - 1)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; t = tube_find(&c->watch, name); if (t && c->watch.len < 2) { reply_msg(c, MSG_NOT_IGNORED); return; } if (t) ms_remove(&c->watch, t); /* may free t if refcount => 0 */ t = NULL; reply_line(c, STATE_SEND_WORD, "WATCHING %zu\r\n", c->watch.len); return; case OP_QUIT: c->state = STATE_CLOSE; return; case OP_PAUSE_TUBE: if (read_tube_name(&name, c->cmd + CMD_PAUSE_TUBE_LEN, &delay_buf) || read_duration(&delay, delay_buf, NULL)) { reply_msg(c, MSG_BAD_FORMAT); return; } op_ct[type]++; *delay_buf = '\0'; if (!is_valid_tube(name, MAX_TUBE_NAME_LEN - 1)) { reply_msg(c, MSG_BAD_FORMAT); return; } t = tube_find(&tubes, name); if (!t) { reply_msg(c, MSG_NOTFOUND); return; } // Always pause for a positive amount of time, to make sure // that waiting clients wake up when the deadline arrives. if (delay == 0) { delay = 1; } t->unpause_at = nanoseconds() + delay; t->pause = delay; t->stat.pause_ct++; reply_line(c, STATE_SEND_WORD, "PAUSED\r\n"); return; default: reply_msg(c, MSG_UNKNOWN_COMMAND); } } /* There are three reasons this function may be called. We need to check for * all of them. * * 1. A reserved job has run out of time. * 2. A waiting client's reserved job has entered the safety margin. * 3. A waiting client's requested timeout has occurred. * * If any of these happen, we must do the appropriate thing. */ static void conn_timeout(Conn *c) { int should_timeout = 0; Job *j; /* Check if the client was trying to reserve a job. */ if (conn_waiting(c) && conndeadlinesoon(c)) should_timeout = 1; /* Check if any reserved jobs have run out of time. We should do this * whether or not the client is waiting for a new reservation. */ while ((j = connsoonestjob(c))) { if (j->r.deadline_at >= nanoseconds()) break; /* This job is in the middle of being written out. If we return it to * the ready queue, someone might free it before we finish writing it * out to the socket. So we'll copy it here and free the copy when it's * done sending. */ if (j == c->out_job) { c->out_job = job_copy(c->out_job); } timeout_ct++; /* stats */ j->r.timeout_ct++; int r = enqueue_job(c->srv, remove_this_reserved_job(c, j), 0, 0); if (r < 1) bury_job(c->srv, j, 0); /* out of memory, so bury it */ connsched(c); } if (should_timeout) { remove_waiting_conn(c); reply_msg(c, MSG_DEADLINE_SOON); } else if (conn_waiting(c) && c->pending_timeout >= 0) { c->pending_timeout = -1; remove_waiting_conn(c); reply_msg(c, MSG_TIMED_OUT); } } void enter_drain_mode(int sig) { UNUSED_PARAMETER(sig); drain_mode = 1; } static void conn_want_command(Conn *c) { epollq_add(c, 'r'); /* was this a peek or stats command? */ if (c->out_job && c->out_job->r.state == Copy) job_free(c->out_job); c->out_job = NULL; c->reply_sent = 0; /* now that we're done, reset this */ c->state = STATE_WANT_COMMAND; } static void conn_process_io(Conn *c) { int r; int64 to_read; Job *j; struct iovec iov[2]; switch (c->state) { case STATE_WANT_COMMAND: r = read(c->sock.fd, c->cmd + c->cmd_read, LINE_BUF_SIZE - c->cmd_read); if (r == -1) { check_err(c, "read()"); return; } if (r == 0) { c->state = STATE_CLOSE; return; } c->cmd_read += r; c->cmd_len = scan_line_end(c->cmd, c->cmd_read); if (c->cmd_len) { // We found complete command line. Bail out to h_conn. return; } // c->cmd_read > LINE_BUF_SIZE can't happen if (c->cmd_read == LINE_BUF_SIZE) { // Command line too long. // Put connection into special state that discards // the command line until the end line is found. c->cmd_read = 0; // discard the input so far c->state = STATE_WANT_ENDLINE; } // We have an incomplete line, so just keep waiting. return; case STATE_WANT_ENDLINE: r = read(c->sock.fd, c->cmd + c->cmd_read, LINE_BUF_SIZE - c->cmd_read); if (r == -1) { check_err(c, "read()"); return; } if (r == 0) { c->state = STATE_CLOSE; return; } c->cmd_read += r; c->cmd_len = scan_line_end(c->cmd, c->cmd_read); if (c->cmd_len) { // Found the EOL. Reply and reuse whatever was read afer the EOL. reply_msg(c, MSG_BAD_FORMAT); fill_extra_data(c); return; } // c->cmd_read > LINE_BUF_SIZE can't happen if (c->cmd_read == LINE_BUF_SIZE) { // Keep discarding the input since no EOL was found. c->cmd_read = 0; } return; case STATE_BITBUCKET: { /* Invert the meaning of in_job_read while throwing away data -- it * counts the bytes that remain to be thrown away. */ static char bucket[BUCKET_BUF_SIZE]; to_read = min(c->in_job_read, BUCKET_BUF_SIZE); r = read(c->sock.fd, bucket, to_read); if (r == -1) { check_err(c, "read()"); return; } if (r == 0) { c->state = STATE_CLOSE; return; } c->in_job_read -= r; /* we got some bytes */ /* (c->in_job_read < 0) can't happen */ if (c->in_job_read == 0) { reply(c, c->reply, c->reply_len, STATE_SEND_WORD); } return; } case STATE_WANT_DATA: j = c->in_job; r = read(c->sock.fd, j->body + c->in_job_read, j->r.body_size -c->in_job_read); if (r == -1) { check_err(c, "read()"); return; } if (r == 0) { c->state = STATE_CLOSE; return; } c->in_job_read += r; /* we got some bytes */ /* (j->in_job_read > j->r.body_size) can't happen */ maybe_enqueue_incoming_job(c); return; case STATE_SEND_WORD: r= write(c->sock.fd, c->reply + c->reply_sent, c->reply_len - c->reply_sent); if (r == -1) { check_err(c, "write()"); return; } if (r == 0) { c->state = STATE_CLOSE; return; } c->reply_sent += r; /* we got some bytes */ /* (c->reply_sent > c->reply_len) can't happen */ if (c->reply_sent == c->reply_len) { conn_want_command(c); return; } /* otherwise we sent an incomplete reply, so just keep waiting */ break; case STATE_SEND_JOB: j = c->out_job; iov[0].iov_base = (void *)(c->reply + c->reply_sent); iov[0].iov_len = c->reply_len - c->reply_sent; /* maybe 0 */ iov[1].iov_base = j->body + c->out_job_sent; iov[1].iov_len = j->r.body_size - c->out_job_sent; r = writev(c->sock.fd, iov, 2); if (r == -1) { check_err(c, "writev()"); return; } if (r == 0) { c->state = STATE_CLOSE; return; } /* update the sent values */ c->reply_sent += r; if (c->reply_sent >= c->reply_len) { c->out_job_sent += c->reply_sent - c->reply_len; c->reply_sent = c->reply_len; } /* (c->out_job_sent > j->r.body_size) can't happen */ /* are we done? */ if (c->out_job_sent == j->r.body_size) { if (verbose >= 2) { printf(">%d job %"PRIu64"\n", c->sock.fd, j->r.id); } conn_want_command(c); return; } /* otherwise we sent incomplete data, so just keep waiting */ break; case STATE_WAIT: if (c->halfclosed) { c->pending_timeout = -1; remove_waiting_conn(c); reply_msg(c, MSG_TIMED_OUT); return; } break; } } #define want_command(c) ((c)->sock.fd && ((c)->state == STATE_WANT_COMMAND)) #define cmd_data_ready(c) (want_command(c) && (c)->cmd_read) static void h_conn(const int fd, const short which, Conn *c) { if (fd != c->sock.fd) { twarnx("Argh! event fd doesn't match conn fd."); close(fd); connclose(c); epollq_apply(); return; } if (which == 'h') { c->halfclosed = 1; } conn_process_io(c); while (cmd_data_ready(c) && (c->cmd_len = scan_line_end(c->cmd, c->cmd_read))) { dispatch_cmd(c); fill_extra_data(c); } if (c->state == STATE_CLOSE) { epollq_rmconn(c); connclose(c); } epollq_apply(); } static void prothandle(Conn *c, int ev) { h_conn(c->sock.fd, ev, c); } // prottick returns nanoseconds till the next work. int64 prottick(Server *s) { Job *j; int64 now; Tube *t; int64 period = 0x34630B8A000LL; /* 1 hour in nanoseconds */ int64 d; now = nanoseconds(); // Enqueue all jobs that are no longer delayed. // Capture the smallest period from the soonest delayed job. while ((j = soonest_delayed_job())) { d = j->r.deadline_at - now; if (d > 0) { period = min(period, d); break; } heapremove(&j->tube->delay, j->heap_index); int r = enqueue_job(s, j, 0, 0); if (r < 1) bury_job(s, j, 0); /* out of memory */ } // Unpause every possible tube and process the queue. // Capture the smallest period from the soonest pause deadline. size_t i; for (i = 0; i < tubes.len; i++) { t = tubes.items[i]; d = t->unpause_at - now; if (t->pause && d <= 0) { t->pause = 0; process_queue(); } else if (d > 0) { period = min(period, d); } } // Process connections with pending timeouts. Release jobs with expired ttr. // Capture the smallest period from the soonest connection. while (s->conns.len) { Conn *c = s->conns.data[0]; d = c->tickat - now; if (d > 0) { period = min(period, d); break; } heapremove(&s->conns, 0); c->in_conns = 0; conn_timeout(c); } epollq_apply(); return period; } void h_accept(const int fd, const short which, Server *s) { UNUSED_PARAMETER(which); struct sockaddr_storage addr; socklen_t addrlen = sizeof addr; int cfd = accept(fd, (struct sockaddr *)&addr, &addrlen); if (cfd == -1) { if (errno != EAGAIN && errno != EWOULDBLOCK) twarn("accept()"); epollq_apply(); return; } if (verbose) { printf("accept %d\n", cfd); } int flags = fcntl(cfd, F_GETFL, 0); if (flags < 0) { twarn("getting flags"); close(cfd); if (verbose) { printf("close %d\n", cfd); } epollq_apply(); return; } int r = fcntl(cfd, F_SETFL, flags | O_NONBLOCK); if (r < 0) { twarn("setting O_NONBLOCK"); close(cfd); if (verbose) { printf("close %d\n", cfd); } epollq_apply(); return; } Conn *c = make_conn(cfd, STATE_WANT_COMMAND, default_tube, default_tube); if (!c) { twarnx("make_conn() failed"); close(cfd); if (verbose) { printf("close %d\n", cfd); } epollq_apply(); return; } c->srv = s; c->sock.x = c; c->sock.f = (Handle)prothandle; c->sock.fd = cfd; r = sockwant(&c->sock, 'r'); if (r == -1) { twarn("sockwant"); close(cfd); if (verbose) { printf("close %d\n", cfd); } } epollq_apply(); } void prot_init() { started_at = nanoseconds(); memset(op_ct, 0, sizeof(op_ct)); int dev_random = open("/dev/urandom", O_RDONLY); if (dev_random < 0) { twarn("open /dev/urandom"); exit(50); } int i, r; byte rand_data[instance_id_bytes]; r = read(dev_random, &rand_data, instance_id_bytes); if (r != instance_id_bytes) { twarn("read /dev/urandom"); exit(50); } for (i = 0; i < instance_id_bytes; i++) { sprintf(instance_hex + (i * 2), "%02x", rand_data[i]); } close(dev_random); if (uname(&node_info) == -1) { warn("uname"); exit(50); } ms_init(&tubes, NULL, NULL); TUBE_ASSIGN(default_tube, tube_find_or_make("default")); if (!default_tube) twarnx("Out of memory during startup!"); } // For each job in list, inserts the job into the appropriate data // structures and adds it to the log. // // Returns 1 on success, 0 on failure. int prot_replay(Server *s, Job *list) { Job *j, *nj; int64 t; int r; for (j = list->next ; j != list ; j = nj) { nj = j->next; job_list_remove(j); int z = walresvupdate(&s->wal); if (!z) { twarnx("failed to reserve space"); return 0; } int64 delay = 0; switch (j->r.state) { case Buried: { bury_job(s, j, 0); break; } case Delayed: t = nanoseconds(); if (t < j->r.deadline_at) { delay = j->r.deadline_at - t; } /* Falls through */ default: r = enqueue_job(s, j, delay, 0); if (r < 1) twarnx("error recovering job %"PRIu64, j->r.id); } } return 1; } beanstalkd-1.13/serv.c000066400000000000000000000032451440333440000146700ustar00rootroot00000000000000#include "dat.h" #include #include #include struct Server srv = { .port = Portdef, .wal = { .filesize = Filesizedef, .wantsync = 1, .syncrate = DEFAULT_FSYNC_MS * 1000000, }, }; // srv_acquire_wal tries to lock the wal dir specified by s->wal and // replay entries from it to initialize the s state with jobs. // On errors it exits from the program. void srv_acquire_wal(Server *s) { if (s->wal.use) { // We want to make sure that only one beanstalkd tries // to use the wal directory at a time. So acquire a lock // now and never release it. if (!waldirlock(&s->wal)) { twarnx("failed to lock wal dir %s", s->wal.dir); exit(10); } Job list = {.prev=NULL, .next=NULL}; list.prev = list.next = &list; walinit(&s->wal, &list); int ok = prot_replay(s, &list); if (!ok) { twarnx("failed to replay log"); exit(1); } } } void srvserve(Server *s) { Socket *sock; if (sockinit() == -1) { twarnx("sockinit"); exit(1); } s->sock.x = s; s->sock.f = (Handle)srvaccept; s->conns.less = conn_less; s->conns.setpos = conn_setpos; if (sockwant(&s->sock, 'r') == -1) { twarn("sockwant"); exit(2); } for (;;) { int64 period = prottick(s); int rw = socknext(&sock, period); if (rw == -1) { twarnx("socknext"); exit(1); } if (rw) { sock->f(sock->x, rw); } } } void srvaccept(Server *s, int ev) { h_accept(s->sock.fd, ev, s); } beanstalkd-1.13/sunos.c000066400000000000000000000034671440333440000150660ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include "dat.h" static int portfd; int sockinit(void) { portfd = port_create(); if (portfd == -1) { twarn("port_create"); return -1; } return 0; } int sockwant(Socket *s, int rw) { int events = 0; if (rw) { switch (rw) { case 'r': events |= POLLIN; break; case 'w': events |= POLLOUT; break; } } events |= POLLPRI; if (!s->added && !rw) { return 0; } else if (!s->added && rw) { s->added = 1; return port_associate(portfd, PORT_SOURCE_FD, s->fd, events, (void *)s); } else if (!rw) { return port_dissociate(portfd, PORT_SOURCE_FD, s->fd); } else { port_dissociate(portfd, PORT_SOURCE_FD, s->fd); return port_associate(portfd, PORT_SOURCE_FD, s->fd, events, (void *)s); } } int socknext(Socket **s, int64 timeout) { int r; uint_t n = 1; struct port_event pe; struct timespec ts; ts.tv_sec = timeout / 1000000000; ts.tv_nsec = timeout % 1000000000; r = port_getn(portfd, &pe, 1, &n, &ts); if (r == -1 && errno != ETIME && errno != EINTR) { twarn("port_getn"); return -1; } if (r == 0) { *s = pe.portev_user; if (pe.portev_events & POLLHUP) { return 'h'; } else if (pe.portev_events & POLLIN) { if (sockwant(*s, 'r') == -1) { return -1; } return 'r'; } else if (pe.portev_events & POLLOUT) { if (sockwant(*s, 'w') == -1) { return -1; } return 'w'; } } return 0; } beanstalkd-1.13/testheap.c000066400000000000000000000144221440333440000155250ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include "ct/ct.h" void cttest_heap_insert_one() { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; Job *j = make_job(1, 0, 1, 0, 0); assertf(j, "allocate job"); heapinsert(&h, j); assertf(h.len == 1, "h should contain one item."); assertf(j->heap_index == 0, "should match"); assert(heapremove(&h, 0)); job_free(j); free(h.data); } void cttest_heap_insert_and_remove_one() { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; Job *j1 = make_job(1, 0, 1, 0, 0); assertf(j1, "allocate job"); int r = heapinsert(&h, j1); assertf(r, "insert should succeed"); Job *got = heapremove(&h, 0); assertf(got == j1, "j1 should come back out"); assertf(h.len == 0, "h should be empty."); free(h.data); job_free(j1); } void cttest_heap_priority() { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; Job *j, *j1, *j2, *j3; j1 = make_job(1, 0, 1, 0, 0); j2 = make_job(2, 0, 1, 0, 0); j3 = make_job(3, 0, 1, 0, 0); assertf(j1, "allocate job"); assertf(j2, "allocate job"); assertf(j3, "allocate job"); int r = heapinsert(&h, j2); assertf(r, "insert should succeed"); assertf(j2->heap_index == 0, "should match"); r = heapinsert(&h, j3); assertf(r, "insert should succeed"); assertf(j2->heap_index == 0, "should match"); assertf(j3->heap_index == 1, "should match"); r = heapinsert(&h, j1); assertf(r, "insert should succeed"); assertf(j1->heap_index == 0, "should match"); assertf(j2->heap_index == 2, "should match"); assertf(j3->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j1, "j1 should come out first."); assertf(j2->heap_index == 0, "should match"); assertf(j3->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j2, "j2 should come out second."); assertf(j3->heap_index == 0, "should match"); j = heapremove(&h, 0); assertf(j == j3, "j3 should come out third."); free(h.data); job_free(j1); job_free(j2); job_free(j3); } void cttest_heap_fifo_property() { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; Job *j, *j3a, *j3b, *j3c; j3a = make_job(3, 0, 1, 0, 0); j3b = make_job(3, 0, 1, 0, 0); j3c = make_job(3, 0, 1, 0, 0); assertf(j3a, "allocate job"); assertf(j3b, "allocate job"); assertf(j3c, "allocate job"); int r = heapinsert(&h, j3a); assertf(r, "insert should succeed"); assertf(h.data[0] == j3a, "j3a should be in pos 0"); assertf(j3a->heap_index == 0, "should match"); r = heapinsert(&h, j3b); assertf(r, "insert should succeed"); assertf(h.data[1] == j3b, "j3b should be in pos 1"); assertf(j3a->heap_index == 0, "should match"); assertf(j3b->heap_index == 1, "should match"); r = heapinsert(&h, j3c); assertf(r, "insert should succeed"); assertf(h.data[2] == j3c, "j3c should be in pos 2"); assertf(j3a->heap_index == 0, "should match"); assertf(j3b->heap_index == 1, "should match"); assertf(j3c->heap_index == 2, "should match"); j = heapremove(&h, 0); assertf(j == j3a, "j3a should come out first."); assertf(j3b->heap_index == 0, "should match"); assertf(j3c->heap_index == 1, "should match"); j = heapremove(&h, 0); assertf(j == j3b, "j3b should come out second."); assertf(j3c->heap_index == 0, "should match"); j = heapremove(&h, 0); assertf(j == j3c, "j3c should come out third."); free(h.data); job_free(j3a); job_free(j3b); job_free(j3c); } void cttest_heap_many_jobs() { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; const int n = 20; Job *j; int i; for (i = 0; i < n; i++) { j = make_job(1 + rand() % 8192, 0, 1, 0, 0); assertf(j, "allocation"); int r = heapinsert(&h, j); assertf(r, "heapinsert"); } uint last_pri = 0; for (i = 0; i < n; i++) { j = heapremove(&h, 0); assertf(j->r.pri >= last_pri, "should come out in order"); last_pri = j->r.pri; assert(j); job_free(j); } free(h.data); } void cttest_heap_remove_k() { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; const int n = 50; const int mid = 25; int c, i; for (c = 0; c < 50; c++) { for (i = 0; i < n; i++) { Job *j = make_job(1 + rand() % 8192, 0, 1, 0, 0); assertf(j, "allocation"); int r = heapinsert(&h, j); assertf(r, "heapinsert"); } /* remove one from the middle */ Job *j0 = heapremove(&h, mid); assertf(j0, "j0 should not be NULL"); job_free(j0); /* now make sure the rest are still a valid heap */ uint last_pri = 0; for (i = 1; i < n; i++) { Job *j = heapremove(&h, 0); assertf(j->r.pri >= last_pri, "should come out in order"); last_pri = j->r.pri; assertf(j, "j should not be NULL"); job_free(j); } } free(h.data); } void ctbench_heap_insert(int n) { Job **j = calloc(n, sizeof *j); int i; for (i = 0; i < n; i++) { j[i] = make_job(1, 0, 1, 0, 0); assert(j[i]); j[i]->r.pri = -j[i]->r.id; } Heap h = { .less = job_pri_less, .setpos = job_setpos, }; ctresettimer(); for (i = 0; i < n; i++) { heapinsert(&h, j[i]); } ctstoptimer(); for (i = 0; i < n; i++) job_free(heapremove(&h, 0)); free(h.data); free(j); } void ctbench_heap_remove(int n) { Heap h = { .less = job_pri_less, .setpos = job_setpos, }; int i; for (i = 0; i < n; i++) { Job *j = make_job(1, 0, 1, 0, 0); assertf(j, "allocate job"); heapinsert(&h, j); } Job **jj = calloc(n, sizeof(Job *)); // temp storage to deallocate jobs later ctresettimer(); for (i = 0; i < n; i++) { jj[i] = (Job *)heapremove(&h, 0); } ctstoptimer(); free(h.data); for (i = 0; i < n; i++) job_free(jj[i]); free(jj); } beanstalkd-1.13/testjobs.c000066400000000000000000000065771440333440000155610ustar00rootroot00000000000000#include "ct/ct.h" #include "dat.h" #include #include #include #include #include static Tube *default_tube; void cttest_job_creation() { Job *j; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job(1, 0, 1, 0, default_tube); assertf(j->r.pri == 1, "priority should match"); } void cttest_job_cmp_pris() { Job *a, *b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(1 << 27, 0, 1, 0, default_tube); assertf(job_pri_less(a, b), "should be less"); } void cttest_job_cmp_ids() { Job *a, *b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(1, 0, 1, 0, default_tube); b->r.id <<= 49; assertf(job_pri_less(a, b), "should be less"); } void cttest_job_large_pris() { Job *a, *b; TUBE_ASSIGN(default_tube, make_tube("default")); a = make_job(1, 0, 1, 0, default_tube); b = make_job(-5, 0, 1, 0, default_tube); assertf(job_pri_less(a, b), "should be less"); a = make_job(-5, 0, 1, 0, default_tube); b = make_job(1, 0, 1, 0, default_tube); assertf(!job_pri_less(a, b), "should not be less"); } void cttest_job_hash_free() { Job *j; uint64 jid = 83; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job_with_id(0, 0, 1, 0, default_tube, jid); job_free(j); assertf(!job_find(jid), "job should be missing"); } void cttest_job_hash_free_next() { Job *a, *b; uint64 aid = 97, bid = 12386; TUBE_ASSIGN(default_tube, make_tube("default")); b = make_job_with_id(0, 0, 1, 0, default_tube, bid); a = make_job_with_id(0, 0, 1, 0, default_tube, aid); assertf(a->ht_next == b, "b should be chained to a"); job_free(b); assertf(a->ht_next == NULL, "job should be missing"); } void cttest_job_all_jobs_used() { Job *j, *x; TUBE_ASSIGN(default_tube, make_tube("default")); j = make_job(0, 0, 1, 0, default_tube); assertf(get_all_jobs_used() == 1, "should match"); x = allocate_job(10); assertf(get_all_jobs_used() == 1, "should match"); job_free(x); assertf(get_all_jobs_used() == 1, "should match"); job_free(j); assertf(get_all_jobs_used() == 0, "should match"); } void cttest_job_100_000_jobs() { int i; TUBE_ASSIGN(default_tube, make_tube("default")); for (i = 0; i < 100000; i++) { make_job(0, 0, 1, 0, default_tube); } assertf(get_all_jobs_used() == 100000, "should match"); for (i = 1; i <= 100000; i++) { job_free(job_find(i)); } fprintf(stderr, "get_all_jobs_used() => %zu\n", get_all_jobs_used()); assertf(get_all_jobs_used() == 0, "should match"); } void ctbench_job_make(int n) { int i; Job **j = calloc(n, sizeof *j); TUBE_ASSIGN(default_tube, make_tube("default")); ctresettimer(); for (i = 0; i < n; i++) { j[i] = make_job(0, 0, 1, 0, default_tube); } ctstoptimer(); for (i = 0; i < n; i++) { job_free(j[i]); } free(j); } void ctbench_job_free(int n) { int i; Job **j = calloc(n, sizeof *j); TUBE_ASSIGN(default_tube, make_tube("default")); for (i = 0; i < n; i++) { j[i] = make_job(0, 0, 1, 0, default_tube); } ctresettimer(); for (i = 0; i < n; i++) { job_free(j[i]); } ctstoptimer(); free(j); } beanstalkd-1.13/testms.c000066400000000000000000000045261440333440000152330ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include "ct/ct.h" void cttest_ms_append() { Ms *a = new(struct Ms); ms_init(a, NULL, NULL); int i = 10; int ok = ms_append(a, &i); assertf(a->len == 1, "a should contain one item"); assertf(ok, "should be added"); ok = ms_append(a, &i); assertf(a->len == 2, "a should contain two items"); assertf(ok, "should be added"); ms_clear(a); assertf(a->len == 0, "a should be empty"); free(a->items); free(a); } void cttest_ms_remove() { Ms *a = new(struct Ms); ms_init(a, NULL, NULL); int i = 1; ms_append(a, &i); int j = 2; int ok = ms_remove(a, &j); assertf(!ok, "j should not be removed"); ok = ms_remove(a, &i); assertf(ok, "i should be removed"); ok = ms_remove(a, &i); assertf(!ok, "i was already removed"); assertf(a->len == 0, "a should be empty"); free(a->items); free(a); } void cttest_ms_contains() { Ms *a = new(struct Ms); ms_init(a, NULL, NULL); int i = 1; ms_append(a, &i); int ok = ms_contains(a, &i); assertf(ok, "i should be in a"); int j = 2; ok = ms_contains(a, &j); assertf(!ok, "j should not be in a"); ms_clear(a); free(a->items); free(a); } void cttest_ms_clear_empty() { Ms *a = new(struct Ms); ms_init(a, NULL, NULL); ms_clear(a); assertf(a->len == 0, "a should be empty"); free(a->items); free(a); } void cttest_ms_take() { Ms *a = new(struct Ms); ms_init(a, NULL, NULL); int i = 10; int j = 20; ms_append(a, &i); ms_append(a, &j); int *n; n = (int *)ms_take(a); assertf(n == &i, "n should point to i"); n = (int *)ms_take(a); assertf(n == &j, "n should point to j"); n = (int *)ms_take(a); assertf(n == NULL, "n should be NULL; ms is empty"); free(a->items); free(a); } void cttest_ms_take_sequence() { size_t i; int s[] = {1, 2, 3, 4, 5, 6}; int e[] = {1, 2, 3, 6, 5, 4}; Ms *a = new(struct Ms); ms_init(a, NULL, NULL); size_t n = sizeof(s)/sizeof(s[0]); for (i = 0; i < n; i++) ms_append(a, &s[i]); for (i = 0; i < n; i++) { int *got = (int *)ms_take(a); assert(*got == e[i]); } free(a->items); free(a); } beanstalkd-1.13/testserv.c000066400000000000000000001476221440333440000156000ustar00rootroot00000000000000#include "ct/ct.h" #include "dat.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int srvpid, size; // Global timeout set for reading response in tests; 5sec. static int64 timeout = 5000000000LL; // Allocation pattern for wrapfalloc that replaces falloc in tests. // Zero value at N-th element means that N-th call to the falloc // should fail with ENOSPC result. static byte fallocpat[3]; static int exist(char *path) { struct stat s; int r = stat(path, &s); return r != -1; } static int wrapfalloc(int fd, int size) { static size_t c = 0; printf("\nwrapfalloc: fd=%d size=%d\n", fd, size); if (c >= sizeof(fallocpat) || !fallocpat[c++]) { return ENOSPC; } return rawfalloc(fd, size); } static void muststart(char *a0, char *a1, char *a2, char *a3, char *a4) { srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { printf("%s %s %s %s %s\n", a0, a1, a2, a3, a4); printf("start server pid=%d\n", srvpid); usleep(100000); // .1s; time for the child to bind to its port return; } /* now in child */ execlp(a0, a0, a1, a2, a3, a4, NULL); } static int mustdiallocal(int port) { struct sockaddr_in addr = { .sin_family = AF_INET, .sin_port = htons(port), }; int r = inet_aton("127.0.0.1", &addr.sin_addr); if (!r) { errno = EINVAL; twarn("inet_aton"); exit(1); } int fd = socket(AF_INET, SOCK_STREAM, 0); if (fd == -1) { twarn("socket"); exit(1); } // Fix of the benchmarking issue on Linux. See issue #430. int flags = 1; if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &flags, sizeof(int))) { twarn("setting TCP_NODELAY on fd %d", fd); exit(1); } r = connect(fd, (struct sockaddr *)&addr, sizeof addr); if (r == -1) { twarn("connect"); exit(1); } return fd; } static int mustdialunix(char *socket_file) { struct sockaddr_un addr; const size_t maxlen = sizeof(addr.sun_path); addr.sun_family = AF_UNIX; snprintf(addr.sun_path, maxlen, "%s", socket_file); int fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd == -1) { twarn("socket"); exit(1); } int r = connect(fd, (struct sockaddr *)&addr, sizeof addr); if (r == -1) { twarn("connect"); exit(1); } return fd; } static void exit_process(int signum) { UNUSED_PARAMETER(signum); exit(0); } static void set_sig_handler() { struct sigaction sa; sa.sa_flags = 0; int r = sigemptyset(&sa.sa_mask); if (r == -1) { twarn("sigemptyset()"); exit(111); } // This is required to trigger gcov on exit. See issue #443. sa.sa_handler = exit_process; r = sigaction(SIGTERM, &sa, 0); if (r == -1) { twarn("sigaction(SIGTERM)"); exit(111); } } // Kill the srvpid (child process) with SIGTERM to give it a chance // to write gcov data to the filesystem before ct kills it with SIGKILL. // Do nothing in case of srvpid==0; child was already killed. static void kill_srvpid(void) { if (!srvpid) return; kill(srvpid, SIGTERM); waitpid(srvpid, 0, 0); srvpid = 0; } #define SERVER() (progname=__func__, mustforksrv()) #define SERVER_UNIX() (progname=__func__, mustforksrv_unix()) // Forks the server storing the pid in srvpid. // The parent process returns port assigned. // The child process serves until the SIGTERM is received by it. static int mustforksrv(void) { struct sockaddr_in addr; srv.sock.fd = make_server_socket("127.0.0.1", "0"); if (srv.sock.fd == -1) { puts("mustforksrv failed"); exit(1); } socklen_t len = sizeof(addr); int r = getsockname(srv.sock.fd, (struct sockaddr *)&addr, &len); if (r == -1 || len > sizeof(addr)) { puts("mustforksrv failed"); exit(1); } int port = ntohs(addr.sin_port); srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { // On exit the parent (test) sends SIGTERM to the child. atexit(kill_srvpid); printf("start server port=%d pid=%d\n", port, srvpid); return port; } /* now in child */ set_sig_handler(); prot_init(); srv_acquire_wal(&srv); srvserve(&srv); /* does not return */ exit(1); /* satisfy the compiler */ } static char * mustforksrv_unix(void) { static char path[90]; char name[95]; snprintf(path, sizeof(path), "%s/socket", ctdir()); snprintf(name, sizeof(name), "unix:%s", path); srv.sock.fd = make_server_socket(name, NULL); if (srv.sock.fd == -1) { puts("mustforksrv_unix failed"); exit(1); } srvpid = fork(); if (srvpid < 0) { twarn("fork"); exit(1); } if (srvpid > 0) { // On exit the parent (test) sends SIGTERM to the child. atexit(kill_srvpid); printf("start server socket=%s\n", path); assert(exist(path)); return path; } /* now in child */ set_sig_handler(); prot_init(); srv_acquire_wal(&srv); srvserve(&srv); /* does not return */ exit(1); /* satisfy the compiler */ } static char * readline(int fd) { char c = 0, p = 0; static char buf[1024]; fd_set rfd; struct timeval tv; printf("<%d ", fd); fflush(stdout); size_t i = 0; for (;;) { FD_ZERO(&rfd); FD_SET(fd, &rfd); tv.tv_sec = timeout / 1000000000; tv.tv_usec = (timeout/1000) % 1000000; int r = select(fd+1, &rfd, NULL, NULL, &tv); switch (r) { case 1: break; case 0: fputs("timeout", stderr); exit(8); case -1: perror("select"); exit(1); default: fputs("unknown error", stderr); exit(3); } // TODO: try reading into a buffer to improve performance. // See related issue #430. r = read(fd, &c, 1); if (r == -1) { perror("write"); exit(1); } if (i >= sizeof(buf)-1) { fputs("response too big", stderr); exit(4); } putc(c, stdout); fflush(stdout); buf[i++] = c; if (p == '\r' && c == '\n') { break; } p = c; } buf[i] = '\0'; return buf; } static void ckresp(int fd, char *exp) { char *line = readline(fd); assertf(strcmp(exp, line) == 0, "\"%s\" != \"%s\"", exp, line); } static void ckrespsub(int fd, char *sub) { char *line = readline(fd); assertf(strstr(line, sub), "\"%s\" not in \"%s\"", sub, line); } static void writefull(int fd, char *s, int n) { int c; for (; n; n -= c) { c = write(fd, s, n); if (c == -1) { perror("write"); exit(1); } s += c; } } static void mustsend(int fd, char *s) { writefull(fd, s, strlen(s)); printf(">%d %s", fd, s); fflush(stdout); } static int filesize(char *path) { struct stat s; int r = stat(path, &s); if (r == -1) { twarn("stat"); exit(1); } return s.st_size; } void cttest_unknown_command() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "nont10knowncommand\r\n"); ckresp(fd, "UNKNOWN_COMMAND\r\n"); } void cttest_too_long_commandline() { int port = SERVER(); int fd = mustdiallocal(port); int i; for (i = 0; i < 10; i++) mustsend(fd, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); // 50 bytes mustsend(fd, "\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); // Issue another command and check that reponse is not "UNKNOWN_COMMAND" // as described in issue #337 mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "A\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttest_put_in_drain() { enter_drain_mode(SIGUSR1); int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "DRAINING\r\n"); } void cttest_peek_ok() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "peek 1\r\n"); ckresp(fd, "FOUND 1 1\r\n"); ckresp(fd, "a\r\n"); } void cttest_peek_not_found() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "peek 2\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "peek 18446744073709551615\r\n"); // UINT64_MAX ckresp(fd, "NOT_FOUND\r\n"); } void cttest_peek_ok_unix() { char *name = SERVER_UNIX(); int fd = mustdialunix(name); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "a\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "peek 1\r\n"); ckresp(fd, "FOUND 1 1\r\n"); ckresp(fd, "a\r\n"); unlink(name); } void cttest_unix_auto_removal() { // Twice, to trigger autoremoval SERVER_UNIX(); kill_srvpid(); SERVER_UNIX(); } void cttest_peek_bad_format() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "peek 18446744073709551616\r\n"); // UINT64_MAX+1 ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "peek 184467440737095516160000000000000000000000000000\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "peek foo111\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "peek 111foo\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); } void cttest_peek_delayed() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "peek-delayed\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "A\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 99 1 1\r\n"); mustsend(fd, "B\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "put 0 1 1 1\r\n"); mustsend(fd, "C\r\n"); ckresp(fd, "INSERTED 3\r\n"); mustsend(fd, "peek-delayed\r\n"); ckresp(fd, "FOUND 3 1\r\n"); ckresp(fd, "C\r\n"); mustsend(fd, "delete 3\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "peek-delayed\r\n"); ckresp(fd, "FOUND 2 1\r\n"); ckresp(fd, "B\r\n"); mustsend(fd, "delete 2\r\n"); ckresp(fd, "DELETED\r\n"); mustsend(fd, "peek-delayed\r\n"); ckresp(fd, "NOT_FOUND\r\n"); } void cttest_peek_buried_kick() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "A\r\n"); ckresp(fd, "INSERTED 1\r\n"); // cannot bury unreserved job mustsend(fd, "bury 1 0\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "peek-buried\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "reserve-with-timeout 0\r\n"); ckresp(fd, "RESERVED 1 1\r\n"); ckresp(fd, "A\r\n"); // now we can bury mustsend(fd, "bury 1 0\r\n"); ckresp(fd, "BURIED\r\n"); mustsend(fd, "peek-buried\r\n"); ckresp(fd, "FOUND 1 1\r\n"); ckresp(fd, "A\r\n"); // kick and verify the job is ready mustsend(fd, "kick 1\r\n"); ckresp(fd, "KICKED 1\r\n"); mustsend(fd, "peek-buried\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "peek-ready\r\n"); ckresp(fd, "FOUND 1 1\r\n"); ckresp(fd, "A\r\n"); // nothing is left to kick mustsend(fd, "kick 1\r\n"); ckresp(fd, "KICKED 0\r\n"); } void cttest_touch_bad_format() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "touch a111\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "touch 111a\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "touch !@#!@#\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); } void cttest_touch_not_found() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "touch 1\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "touch 100000000000000\r\n"); ckresp(fd, "NOT_FOUND\r\n"); } void cttest_bury_bad_format() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "bury 111abc 2\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "bury 111\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "bury 111 222abc\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); } void cttest_kickjob_bad_format() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "kick-job a111\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "kick-job 111a\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); mustsend(fd, "kick-job !@#!@#\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); } void cttest_kickjob_buried() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "A\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 1\r\n"); ckresp(fd, "A\r\n"); mustsend(fd, "bury 1 0\r\n"); ckresp(fd, "BURIED\r\n"); mustsend(fd, "kick-job 100\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "kick-job 1\r\n"); ckresp(fd, "KICKED\r\n"); mustsend(fd, "kick-job 1\r\n"); ckresp(fd, "NOT_FOUND\r\n"); } void cttest_kickjob_delayed() { int port = SERVER(); int fd = mustdiallocal(port); // jid=1 - no delay, jid=2 - delay mustsend(fd, "put 0 0 1 1\r\n"); mustsend(fd, "A\r\n"); ckresp(fd, "INSERTED 1\r\n"); mustsend(fd, "put 0 10 1 1\r\n"); mustsend(fd, "B\r\n"); ckresp(fd, "INSERTED 2\r\n"); mustsend(fd, "kick-job 1\r\n"); ckresp(fd, "NOT_FOUND\r\n"); mustsend(fd, "kick-job 2\r\n"); ckresp(fd, "KICKED\r\n"); mustsend(fd, "kick-job 2\r\n"); ckresp(fd, "NOT_FOUND\r\n"); } void cttest_pause() { int64 s; int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "INSERTED 1\r\n"); s = nanoseconds(); mustsend(fd, "pause-tube default 1\r\n"); ckresp(fd, "PAUSED\r\n"); mustsend(fd, "reserve\r\n"); ckresp(fd, "RESERVED 1 1\r\n"); ckresp(fd, "x\r\n"); assert(nanoseconds() - s >= 1000000000); // 1s } void cttest_underscore() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "use x_y\r\n"); ckresp(fd, "USING x_y\r\n"); } void cttest_2cmdpacket() { int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "use a\r\nuse b\r\n"); ckresp(fd, "USING a\r\n"); ckresp(fd, "USING b\r\n"); } void cttest_too_big() { job_data_size_limit = 10; int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 11\r\n"); mustsend(fd, "delete 9999\r\n"); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "JOB_TOO_BIG\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttest_job_size_invalid() { job_data_size_limit = JOB_DATA_SIZE_LIMIT_MAX; int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 4294967296\r\n"); mustsend(fd, "put 0 0 0 10b\r\n"); mustsend(fd, "put 0 0 0 --!@#$%^&&**()0b\r\n"); mustsend(fd, "put 0 0 0 1\r\n"); mustsend(fd, "x\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); ckresp(fd, "BAD_FORMAT\r\n"); ckresp(fd, "INSERTED 1\r\n"); } void cttest_job_size_max_plus_1() { /* verify that server reject the job larger than maximum allowed. */ job_data_size_limit = JOB_DATA_SIZE_LIMIT_MAX; int port = SERVER(); int fd = mustdiallocal(port); mustsend(fd, "put 0 0 0 1073741825\r\n"); const int len = 1024*1024; char body[len+1]; memset(body, 'a', len); body[len] = 0; int i; for (i=0; i 0) { srv.wal.dir = ctdir(); srv.wal.use = 1; srv.wal.filesize = walsize; srv.wal.syncrate = syncrate_ms * 1000000; srv.wal.wantsync = sync; } job_data_size_limit = JOB_DATA_SIZE_LIMIT_MAX; int port = SERVER(); int fd = mustdiallocal(port); char buf[50], put[50]; char body[size+1]; memset(body, 'a', size); body[size] = 0; ctsetbytes(size); sprintf(put, "put 0 0 0 %d\r\n", size); ctresettimer(); int i; for (i = 0; i < n; i++) { mustsend(fd, put); mustsend(fd, body); mustsend(fd, "\r\n"); ckrespsub(fd, "INSERTED "); sprintf(buf, "delete %d\r\n", i + 1); mustsend(fd, buf); ckresp(fd, "DELETED\r\n"); } ctstoptimer(); } void ctbench_put_delete_0008(int n) { bench_put_delete_size(n, 8, 0, 0, 0); } void ctbench_put_delete_1024(int n) { bench_put_delete_size(n, 1024, 0, 0, 0); } void ctbench_put_delete_8192(int n) { bench_put_delete_size(n, 8192, 0, 0, 0); } void ctbench_put_delete_81920(int n) { bench_put_delete_size(n, 81920, 0, 0, 0); } void ctbench_put_delete_wal_1024_fsync_000ms(int n) { bench_put_delete_size(n, 1024, 512000, 1, 0); } void ctbench_put_delete_wal_1024_fsync_050ms(int n) { bench_put_delete_size(n, 1024, 512000, 1, 50); } void ctbench_put_delete_wal_1024_fsync_200ms(int n) { bench_put_delete_size(n, 1024, 512000, 1, 200); } void ctbench_put_delete_wal_1024_no_fsync(int n) { bench_put_delete_size(n, 1024, 512000, 0, 0); } void ctbench_put_delete_wal_8192_fsync_000ms(int n) { bench_put_delete_size(n, 8192, 512000, 1, 0); } void ctbench_put_delete_wal_8192_fsync_050ms(int n) { bench_put_delete_size(n, 8192, 512000, 1, 50); } void ctbench_put_delete_wal_8192_fsync_200ms(int n) { bench_put_delete_size(n, 8192, 512000, 1, 200); } void ctbench_put_delete_wal_8192_no_fsync(int n) { bench_put_delete_size(n, 8192, 512000, 0, 0); } beanstalkd-1.13/testutil.c000066400000000000000000000064241440333440000155700ustar00rootroot00000000000000#include "ct/ct.h" #include "dat.h" #include #include #include #include #include #include void cttest_allocf() { char *got; got = fmtalloc("hello, %s %d", "world", 5); assertf(strcmp("hello, world 5", got) == 0, "got \"%s\"", got); free(got); } void cttest_opt_none() { char *args[] = { NULL, }; optparse(&srv, args); assert(strcmp(srv.port, Portdef) == 0); assert(srv.addr == NULL); assert(job_data_size_limit == JOB_DATA_SIZE_LIMIT_DEFAULT); assert(srv.wal.filesize == Filesizedef); assert(srv.wal.wantsync == 1); assert(srv.wal.syncrate == DEFAULT_FSYNC_MS*1000000); assert(srv.user == NULL); assert(srv.wal.dir == NULL); assert(srv.wal.use == 0); assert(verbose == 0); } static void success(void) { _exit(0); } void cttest_optminus() { char *args[] = { "-", NULL, }; atexit(success); optparse(&srv, args); assertf(0, "optparse failed to call exit"); } void cttest_optp() { char *args[] = { "-p1234", NULL, }; optparse(&srv, args); assert(strcmp(srv.port, "1234") == 0); } void cttest_optl() { char *args[] = { "-llocalhost", NULL, }; optparse(&srv, args); assert(strcmp(srv.addr, "localhost") == 0); } void cttest_optlseparate() { char *args[] = { "-l", "localhost", NULL, }; optparse(&srv, args); assert(strcmp(srv.addr, "localhost") == 0); } void cttest_optz() { char *args[] = { "-z1234", NULL, }; optparse(&srv, args); assert(job_data_size_limit == 1234); } void cttest_optz_more_than_max() { char *args[] = { "-z1073741825", NULL, }; optparse(&srv, args); assert(job_data_size_limit == 1073741824); } void cttest_opts() { char *args[] = { "-s1234", NULL, }; optparse(&srv, args); assert(srv.wal.filesize == 1234); } void cttest_optf() { char *args[] = { "-f1234", NULL, }; optparse(&srv, args); assert(srv.wal.syncrate == 1234000000); assert(srv.wal.wantsync == 1); } void cttest_optF() { char *args[] = { "-f1234", "-F", NULL, }; optparse(&srv, args); assert(srv.wal.wantsync == 0); } void cttest_optu() { char *args[] = { "-ukr", NULL, }; optparse(&srv, args); assert(strcmp(srv.user, "kr") == 0); } void cttest_optb() { char *args[] = { "-bfoo", NULL, }; optparse(&srv, args); assert(strcmp(srv.wal.dir, "foo") == 0); assert(srv.wal.use == 1); } void cttest_optV() { char *args[] = { "-V", NULL, }; optparse(&srv, args); assert(verbose == 1); } void cttest_optV_V() { char *args[] = { "-V", "-V", NULL, }; optparse(&srv, args); assert(verbose == 2); } void cttest_optVVV() { char *args[] = { "-VVV", NULL, }; optparse(&srv, args); assert(verbose == 3); } void cttest_optVFVu() { char *args[] = { "-VFVukr", NULL, }; optparse(&srv, args); assert(verbose == 2); assert(srv.wal.wantsync == 0); assert(strcmp(srv.user, "kr") == 0); } beanstalkd-1.13/time.c000066400000000000000000000004651440333440000146500ustar00rootroot00000000000000#include "dat.h" #include #include #include int64 nanoseconds(void) { int r; struct timeval tv; r = gettimeofday(&tv, 0); if (r != 0) return warnx("gettimeofday"), -1; // can't happen return ((int64)tv.tv_sec)*1000000000 + ((int64)tv.tv_usec)*1000; } beanstalkd-1.13/tube.c000066400000000000000000000035201440333440000146440ustar00rootroot00000000000000#include "dat.h" #include #include #include struct Ms tubes; Tube * make_tube(const char *name) { Tube *t = new(Tube); if (!t) return NULL; strncpy(t->name, name, MAX_TUBE_NAME_LEN); if (t->name[MAX_TUBE_NAME_LEN - 1] != '\0') { t->name[MAX_TUBE_NAME_LEN - 1] = '\0'; twarnx("truncating tube name"); } t->ready.less = job_pri_less; t->delay.less = job_delay_less; t->ready.setpos = job_setpos; t->delay.setpos = job_setpos; Job j = {.tube = NULL}; t->buried = j; t->buried.prev = t->buried.next = &t->buried; ms_init(&t->waiting_conns, NULL, NULL); return t; } static void tube_free(Tube *t) { ms_remove(&tubes, t); free(t->ready.data); free(t->delay.data); ms_clear(&t->waiting_conns); free(t); } void tube_dref(Tube *t) { if (!t) return; if (t->refs < 1) { twarnx("refs is zero for tube: %s", t->name); return; } --t->refs; if (t->refs < 1) tube_free(t); } void tube_iref(Tube *t) { if (!t) return; ++t->refs; } static Tube * make_and_insert_tube(const char *name) { int r; Tube *t = NULL; t = make_tube(name); if (!t) return NULL; /* We want this global tube list to behave like "weak" refs, so don't * increment the ref count. */ r = ms_append(&tubes, t); if (!r) return tube_dref(t), (Tube *) 0; return t; } Tube * tube_find(Ms *tubeset, const char *name) { size_t i; for (i = 0; i < tubeset->len; i++) { Tube *t = tubeset->items[i]; if (strncmp(t->name, name, MAX_TUBE_NAME_LEN) == 0) return t; } return NULL; } Tube * tube_find_or_make(const char *name) { Tube *t = tube_find(&tubes, name); if (t) return t; return make_and_insert_tube(name); } beanstalkd-1.13/util.c000066400000000000000000000131411440333440000146620ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #ifdef HAVE_LIBSYSTEMD #include #endif const char *progname; static void vwarnx(const char *err, const char *fmt, va_list args) __attribute__((format(printf, 2, 0))); static void vwarnx(const char *err, const char *fmt, va_list args) { fprintf(stderr, "%s: ", progname); if (fmt) { vfprintf(stderr, fmt, args); if (err) fprintf(stderr, ": %s", err); } fputc('\n', stderr); } void warn(const char *fmt, ...) { char *err = strerror(errno); /* must be done first thing */ va_list args; va_start(args, fmt); vwarnx(err, fmt, args); va_end(args); } void warnx(const char *fmt, ...) { va_list args; va_start(args, fmt); vwarnx(NULL, fmt, args); va_end(args); } char* fmtalloc(char *fmt, ...) { int n; char *buf; va_list ap; // find out how much space is needed va_start(ap, fmt); n = vsnprintf(0, 0, fmt, ap) + 1; // include space for trailing NUL va_end(ap); buf = malloc(n); if (buf) { va_start(ap, fmt); vsnprintf(buf, n, fmt, ap); va_end(ap); } return buf; } // Zalloc allocates n bytes of zeroed memory and // returns a pointer to it. // If insufficient memory is available, zalloc returns 0. void* zalloc(int n) { void *p; p = malloc(n); if (p) { memset(p, 0, n); } return p; } static void warn_systemd_ignored_option(char *opt, char *arg) { #ifdef HAVE_LIBSYSTEMD if (sd_listen_fds(0) > 0) { warnx("inherited listen fd; ignoring option: %s %s", opt, arg); } #endif } static void usage(int code) __attribute__ ((noreturn)); static void usage(int code) { fprintf(stderr, "Use: %s [OPTIONS]\n" "\n" "Options:\n" " -b DIR write-ahead log directory\n" " -f MS fsync at most once every MS milliseconds (default is %dms);\n" " use -f0 for \"always fsync\"\n" " -F never fsync\n" " -l ADDR listen on address (default is 0.0.0.0)\n" " -p PORT listen on port (default is " Portdef ")\n" " -u USER become user and group\n" " -z BYTES set the maximum job size in bytes (default is %d);\n" " max allowed is %d bytes\n" " -s BYTES set the size of each write-ahead log file (default is %d);\n" " will be rounded up to a multiple of 4096 bytes\n" " -v show version information\n" " -V increase verbosity\n" " -h show this help\n", progname, DEFAULT_FSYNC_MS, JOB_DATA_SIZE_LIMIT_DEFAULT, JOB_DATA_SIZE_LIMIT_MAX, Filesizedef); exit(code); } static char *flagusage(char *flag) __attribute__ ((noreturn)); static char * flagusage(char *flag) { warnx("flag requires an argument: %s", flag); usage(5); } static size_t parse_size_t(char *str) { char r, x; size_t size; r = sscanf(str, "%zu%c", &size, &x); if (1 != r) { warnx("invalid size: %s", str); usage(5); } return size; } void optparse(Server *s, char **argv) { int64 ms; char *arg, *tmp; # define EARGF(x) (*arg ? (tmp=arg,arg="",tmp) : *argv ? *argv++ : (x)) while ((arg = *argv++) && *arg++ == '-' && *arg) { char c; while ((c = *arg++)) { switch (c) { case 'p': s->port = EARGF(flagusage("-p")); warn_systemd_ignored_option("-p", s->port); break; case 'l': s->addr = EARGF(flagusage("-l")); warn_systemd_ignored_option("-l", s->addr); break; case 'z': job_data_size_limit = parse_size_t(EARGF(flagusage("-z"))); if (job_data_size_limit > JOB_DATA_SIZE_LIMIT_MAX) { warnx("maximum job size was set to %d", JOB_DATA_SIZE_LIMIT_MAX); job_data_size_limit = JOB_DATA_SIZE_LIMIT_MAX; } break; case 's': s->wal.filesize = parse_size_t(EARGF(flagusage("-s"))); break; case 'c': warnx("-c flag was removed. binlog is always compacted."); break; case 'n': warnx("-n flag was removed. binlog is always compacted."); break; case 'f': ms = (int64)parse_size_t(EARGF(flagusage("-f"))); s->wal.syncrate = ms * 1000000; s->wal.wantsync = 1; break; case 'F': s->wal.wantsync = 0; break; case 'u': s->user = EARGF(flagusage("-u")); break; case 'b': s->wal.dir = EARGF(flagusage("-b")); s->wal.use = 1; break; case 'h': usage(0); case 'v': printf("beanstalkd %s\n", version); exit(0); case 'V': verbose++; break; default: warnx("unknown flag: %s", arg-2); usage(5); } } } if (arg) { warnx("unknown argument: %s", arg-1); usage(5); } } beanstalkd-1.13/verc.sh000077500000000000000000000001051440333440000150330ustar00rootroot00000000000000#!/bin/sh printf 'const char version[] = "' ./vers.sh printf '";\n' beanstalkd-1.13/vers.sh000077500000000000000000000000161440333440000150540ustar00rootroot00000000000000printf '1.13' beanstalkd-1.13/walg.c000066400000000000000000000220521440333440000146400ustar00rootroot00000000000000#include "dat.h" #include #include #include #include #include #include #include #include #include #include #include static int reserve(Wal *w, int n); // Reads w->dir for files matching binlog.NNN, // sets w->next to the next unused number, and // returns the minimum number. // If no files are found, sets w->next to 1 and // returns a large number. static int walscandir(Wal *w) { static char base[] = "binlog."; static const int len = sizeof(base) - 1; DIR *d; struct dirent *e; int min = 1<<30; int max = 0; int n; char *p; d = opendir(w->dir); if (!d) return min; while ((e = readdir(d))) { if (strncmp(e->d_name, base, len) == 0) { n = strtol(e->d_name+len, &p, 10); if (p && *p == '\0') { if (n > max) max = n; if (n < min) min = n; } } } closedir(d); w->next = max + 1; return min; } void walgc(Wal *w) { File *f; while (w->head && !w->head->refs) { f = w->head; w->head = f->next; if (w->tail == f) { w->tail = f->next; // also, f->next == NULL } w->nfile--; unlink(f->path); free(f->path); free(f); } } // returns 1 on success, 0 on error. static int usenext(Wal *w) { File *f; f = w->cur; if (!f->next) { twarnx("there is no next wal file"); return 0; } w->cur = f->next; filewclose(f); return 1; } static int ratio(Wal *w) { int64 n, d; d = w->alive + w->resv; n = (int64)w->nfile * (int64)w->filesize - d; if (!d) return 0; return n / d; } // Returns the number of bytes reserved or 0 on error. static int walresvmigrate(Wal *w, Job *j) { int z = 0; // reserve only space for the migrated full job record // space for the delete is already reserved z += sizeof(int); z += strlen(j->tube->name); z += sizeof(Jobrec); z += j->r.body_size; return reserve(w, z); } static void moveone(Wal *w) { Job *j; if (w->head == w->cur || w->head->next == w->cur) { // no point in moving a job return; } j = w->head->jlist.fnext; if (!j || j == &w->head->jlist) { // head holds no jlist; can't happen twarnx("head holds no jlist"); return; } if (!walresvmigrate(w, j)) { // it will not fit, so we'll try again later return; } filermjob(w->head, j); w->nmig++; walwrite(w, j); } static void walcompact(Wal *w) { int r; for (r=ratio(w); r>=2; r--) { moveone(w); } } static void walsync(Wal *w) { int64 now; now = nanoseconds(); if (w->wantsync && now >= w->lastsync+w->syncrate) { w->lastsync = now; if (fsync(w->cur->fd) == -1) { twarn("fsync"); } } } // Walwrite writes j to the log w (if w is enabled). // On failure, walwrite disables w and returns 0; on success, it returns 1. // Unlke walresv*, walwrite should never fail because of a full disk. // If w is disabled, then walwrite takes no action and returns 1. int walwrite(Wal *w, Job *j) { int r = 0; if (!w->use) return 1; if (w->cur->resv > 0 || usenext(w)) { if (j->file) { r = filewrjobshort(w->cur, j); } else { r = filewrjobfull(w->cur, j); } } if (!r) { filewclose(w->cur); w->use = 0; } w->nrec++; return r; } void walmaint(Wal *w) { if (w->use) { walcompact(w); walsync(w); } } static int makenextfile(Wal *w) { File *f; f = new(File); if (!f) { twarnx("OOM"); return 0; } if (!fileinit(f, w, w->next)) { free(f); twarnx("OOM"); return 0; } filewopen(f); if (!f->iswopen) { free(f->path); free(f); return 0; } w->next++; fileadd(f, w); return 1; } static void moveresv(File *to, File *from, int n) { from->resv -= n; from->free += n; to->resv += n; to->free -= n; } static int needfree(Wal *w, int n) { if (w->tail->free >= n) return n; if (makenextfile(w)) return n; return 0; } // Ensures: // 1. b->resv is congruent to n (mod z). // 2. x->resv is congruent to 0 (mod z) for each future file x. // Assumes (and preserves) that b->resv >= n. // Reserved space is conserved (neither created nor destroyed); // we just move it around to preserve the invariant. // We might have to allocate a new file. // Returns 1 on success, otherwise 0. If there was a failure, // w->tail is not updated. static int balancerest(Wal *w, File *b, int n) { int rest, c, r; static const int z = sizeof(int) + sizeof(Jobrec); if (!b) return 1; rest = b->resv - n; r = rest % z; if (r == 0) return balancerest(w, b->next, 0); c = z - r; if (w->tail->resv >= c && b->free >= c) { moveresv(b, w->tail, c); return balancerest(w, b->next, 0); } if (needfree(w, r) != r) { twarnx("needfree"); return 0; } moveresv(w->tail, b, r); return balancerest(w, b->next, 0); } // Ensures: // 1. w->cur->resv >= n. // 2. w->cur->resv is congruent to n (mod z). // 3. x->resv is congruent to 0 (mod z) for each future file x. // (where z is the size of a delete record in the wal). // Reserved space is conserved (neither created nor destroyed); // we just move it around to preserve the invariant. // We might have to allocate a new file. // Returns 1 on success, otherwise 0. If there was a failure, // w->tail is not updated. static int balance(Wal *w, int n) { // Invariant 1 // (this loop will run at most once) while (w->cur->resv < n) { int m = w->cur->resv; int r = needfree(w, m); if (r != m) { twarnx("needfree"); return 0; } moveresv(w->tail, w->cur, m); usenext(w); } // Invariants 2 and 3 return balancerest(w, w->cur, n); } // Returns the number of bytes successfully reserved: either 0 or n. static int reserve(Wal *w, int n) { int r; // return value must be nonzero but is otherwise ignored if (!w->use) return 1; if (w->cur->free >= n) { w->cur->free -= n; w->cur->resv += n; w->resv += n; return n; } r = needfree(w, n); if (r != n) { twarnx("needfree"); return 0; } w->tail->free -= n; w->tail->resv += n; w->resv += n; if (!balance(w, n)) { // error; undo the reservation w->resv -= n; w->tail->resv -= n; w->tail->free += n; return 0; } return n; } // Returns the number of bytes reserved or 0 on error. int walresvput(Wal *w, Job *j) { int z = 0; // reserve space for the initial job record z += sizeof(int); z += strlen(j->tube->name); z += sizeof(Jobrec); z += j->r.body_size; // plus space for a delete to come later z += sizeof(int); z += sizeof(Jobrec); return reserve(w, z); } // Returns the number of bytes reserved or 0 on error. int walresvupdate(Wal *w) { int z = 0; z +=sizeof(int); z +=sizeof(Jobrec); return reserve(w, z); } // Returns the number of locks acquired: either 0 or 1. int waldirlock(Wal *w) { int r; int fd; struct flock lk; char *path; size_t path_length; path_length = strlen(w->dir) + strlen("/lock") + 1; if ((path = malloc(path_length)) == NULL) { twarn("malloc"); return 0; } snprintf(path, path_length, "%s/lock", w->dir); fd = open(path, O_WRONLY|O_CREAT, 0600); free(path); if (fd == -1) { twarn("open"); return 0; } lk.l_type = F_WRLCK; lk.l_whence = SEEK_SET; lk.l_start = 0; lk.l_len = 0; r = fcntl(fd, F_SETLK, &lk); if (r) { twarn("fcntl"); return 0; } // intentionally leak fd, since we never want to close it // and we'll never need it again return 1; } void walread(Wal *w, Job *list, int min) { int i; int err = 0; for (i = min; i < w->next; i++) { File *f = new(File); if (!f) { twarnx("OOM"); exit(1); } if (!fileinit(f, w, i)) { free(f); twarnx("OOM"); exit(1); } int fd = open(f->path, O_RDONLY); if (fd < 0) { twarn("open %s", f->path); free(f->path); free(f); continue; } f->fd = fd; fileadd(f, w); err |= fileread(f, list); if (close(fd) == -1) twarn("close"); } if (err) { warnx("Errors reading one or more WAL files."); warnx("Continuing. You may be missing data."); } } void walinit(Wal *w, Job *list) { int min; min = walscandir(w); walread(w, list, min); // first writable file if (!makenextfile(w)) { twarnx("makenextfile"); exit(1); } w->cur = w->tail; }