pax_global_header 0000666 0000000 0000000 00000000064 14743672320 0014522 g ustar 00root root 0000000 0000000 52 comment=9621d6d3d443131adeba126c0fba400ea3a21939
multicoretests-0.7/ 0000775 0000000 0000000 00000000000 14743672320 0014456 5 ustar 00root root 0000000 0000000 multicoretests-0.7/.gitattributes 0000664 0000000 0000000 00000000021 14743672320 0017342 0 ustar 00root root 0000000 0000000 *.sh text eol=lf
multicoretests-0.7/.github/ 0000775 0000000 0000000 00000000000 14743672320 0016016 5 ustar 00root root 0000000 0000000 multicoretests-0.7/.github/runner.sh 0000775 0000000 0000000 00000011736 14743672320 0017676 0 ustar 00root root 0000000 0000000 #!/bin/sh
set -e
OCAMLDIR=ocaml
DUNEDIR=dune
MULTICORETESTSDIR=multicoretests
fatal() {
printf %s "$1"
exit 1
}
compiler_sha() {
# Expect $COMPILER_REPO and $COMPILER_REF to be set
# Note: only GitHub-hosted compiler forks are supported for now
git ls-remote "https://github.com/$COMPILER_REPO.git" "$COMPILER_REF" \
| cut -f 1
}
setup() {
if [ -n "$GITHUB_ENV" ] && [ -n "$GITHUB_PATH" ] ; then
sha="$(compiler_sha)" || fatal "Cannot find compiler's SHA"
arch="$(uname -m)"
opts="$(printf %s "$OCAML_OPTIONS" | tr " " -)"
opts="${opts:+-}$opts"
echo "cache_key=ocaml-$sha-$OCAML_PLATFORM-$arch$opts" >> "$GITHUB_ENV"
case "$OCAML_PLATFORM" in
mingw|msvc|cygwin)
PREFIX='D:\ocaml'
bin='D:\ocaml\bin'
;;
*)
PREFIX="$HOME/local"
bin="$HOME/local/bin"
;;
esac
printf "PREFIX=%s\n" "$PREFIX" >> "$GITHUB_ENV"
printf "%s\n" "$bin" >> "$GITHUB_PATH"
if [ -z "$JOBS" ] ; then
if command -v nproc > /dev/null; then
echo "JOBS=$(nproc)" >> "$GITHUB_ENV"
elif command -v sysctl > /dev/null; then
echo "JOBS=$(sysctl -n hw.ncpu)" >> "$GITHUB_ENV"
fi
fi
echo Environment set up:
cat "$GITHUB_ENV"
echo PATH addition:
cat "$GITHUB_PATH"
fi
case "$OCAML_PLATFORM,$OCAML_OPTIONS" in
linux,*32bit*)
sudo dpkg --add-architecture i386
sudo apt-get update
sudo apt-get install pkg-config:i386 libzstd1:i386 libzstd-dev:i386 \
libgcc-s1:i386 gcc-multilib g++-multilib
;;
linux,*musl*)
sudo apt-get update
sudo apt-get install musl-tools
;;
esac
}
build_ocaml() {
echo "${LOGBEGINGRP}Building OCaml"
# We let standard OCaml CI test for warnings
opts="--disable-warn-error \
--disable-stdlib-manpages \
--disable-ocamltest \
--disable-ocamldoc"
case "$OCAML_OPTIONS" in
*fp*)
opts="$opts --enable-frame-pointers"
;;
esac
case "$OCAML_OPTIONS" in
*bytecode-only*)
opts="$opts --disable-native-compiler"
;;
esac
case "$OCAML_PLATFORM" in
msvc|mingw|cygwin)
opts="$opts --prefix=/cygdrive/d/ocaml"
;;
*)
opts="$opts --prefix=$PREFIX"
;;
esac
cd "$OCAMLDIR"
case "$OCAML_PLATFORM,$OCAML_OPTIONS" in
msvc,*32bit*)
eval $(tools/msvs-promote-path)
printf 'Running: %s\n' "./configure --host=i686-pc-windows $opts"
if ! ./configure --host=i686-pc-windows $opts ; then
cat config.log
exit 1
fi
;;
msvc,*)
eval $(tools/msvs-promote-path)
printf 'Running: %s\n' "./configure --host=x86_64-pc-windows $opts"
if ! ./configure --host=x86_64-pc-windows $opts ; then
cat config.log
exit 1
fi
;;
mingw,*)
printf 'Running: %s\n' "./configure --host=x86_64-w64-mingw32 $opts"
if ! ./configure --host=x86_64-w64-mingw32 $opts ; then
cat config.log
exit 1
fi
;;
cygwin,*)
case $COMPILER_REF in
*/5.1*)
git -C flexdll fetch origin 0.43
git -C flexdll checkout FETCH_HEAD
;;
esac
printf 'Running: %s\n' "./configure $opts"
if ! ./configure $opts ; then
cat config.log
exit 1
fi
;;
linux,*32bit*)
printf 'Running: %s\n' \
"./configure --host=i386-linux \"CC=gcc -m32\" $opts"
if ! ./configure --host=i386-linux "CC=gcc -m32" $opts ; then
cat config.log
exit 1
fi
;;
linux,*musl*)
printf 'Running: %s\n' \
"./configure \"CC=musl-gcc\" \"CFLAGS=-Os\" \"ASPP=musl-gcc -c\" $opts"
if ! ./configure "CC=musl-gcc" "CFLAGS=-Os" "ASPP=musl-gcc -c" $opts ; then
cat config.log
exit 1
fi
;;
*) # linux, macos, default options
printf 'Running: %s\n' "./configure $opts"
if ! ./configure $opts ; then
cat config.log
exit 1
fi
;;
esac
make -j"$JOBS"
make install
echo "$LOGENDGRP"
}
build_dune() {
echo "${LOGBEGINGRP}Building dune"
case "$OCAML_PLATFORM" in
msvc)
eval $("$OCAMLDIR"/tools/msvs-promote-path)
;;
esac
cd "$DUNEDIR"
make release
make install PREFIX="$PREFIX"
echo "$LOGENDGRP"
}
show_config() {
set -x
ocamlc -config
dune --version
}
build_testsuite() {
case "$OCAML_PLATFORM" in
msvc)
eval $("$OCAMLDIR"/tools/msvs-promote-path)
;;
esac
cd "$MULTICORETESTSDIR"
dune build
dune build test/
}
case "$1" in
setup)
setup
;;
ocaml)
build_ocaml
;;
dune)
build_dune
;;
show_config)
show_config
;;
build)
build_testsuite
;;
testsuite)
cd "$MULTICORETESTSDIR"
dune build @ci -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice
;;
internaltests)
cd "$MULTICORETESTSDIR"
dune build @internaltests -j1 --no-buffer --display=quiet --cache=disabled --error-reporting=twice
;;
*)
fatal "Unknown command '$1'"
;;
esac
multicoretests-0.7/.github/workflows/ 0000775 0000000 0000000 00000000000 14743672320 0020053 5 ustar 00root root 0000000 0000000 multicoretests-0.7/.github/workflows/common.yml 0000664 0000000 0000000 00000011610 14743672320 0022065 0 ustar 00root root 0000000 0000000 name: Common CI workflow
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
on:
workflow_call:
inputs:
runs_on:
description: 'Type of machine + OS on which to run the tests'
type: string
default: 'ubuntu-latest'
options:
description: >-
Configuration options for the compiler.
Space-separated list of '32bit', 'bytecode-only', 'fp', 'musl'.
type: string
default: ''
platform:
description: 'Platform. One of: linux, macos, msvc, mingw, cygwin.'
type: string
default: 'linux'
timeout:
description: 'Timeout'
type: number
default: 180
dune_profile:
description: 'Dune profile to use'
type: string
default: 'dev'
runparam:
description: 'OCAMLRUNPARAM to use'
type: string
default: ''
dune_alias:
description: 'dune alias that should be built in the main step'
type: string
default: 'testsuite'
compiler_repository:
description: 'Repository from which to fetch the compiler'
type: string
default: 'ocaml/ocaml'
compiler_ref:
description: 'Git reference to use'
type: string
required: true
permissions: {}
jobs:
test:
env:
QCHECK_MSG_INTERVAL: '60'
OCAML_OPTIONS: ${{ inputs.options }}
OCAML_PLATFORM: ${{ inputs.platform }}
DUNE_PROFILE: ${{ inputs.dune_profile }}
OCAMLRUNPARAM: ${{ inputs.runparam }}
DUNE_CI_ALIAS: ${{ inputs.dune_alias }}
COMPILER_REPO: ${{ inputs.compiler_repository }}
COMPILER_REF: ${{ inputs.compiler_ref }}
LOGBEGINGRP: "::group::"
LOGENDGRP: "::endgroup::"
# For the record, PR 12345 of the compiler can be tested simply by setting
# COMPILER_REF: 'refs/pull/12345/head'
runs-on: ${{ inputs.runs_on }}
timeout-minutes: ${{ inputs.timeout }}
steps:
- name: Configure EOLs on Cygwin
run: |
# Ensure that .expected files are not modified by check out
# as, in Cygwin, the .expected should use LF line endings,
# rather than Windows’ CRLF
git config --global core.autocrlf input
if: inputs.platform == 'cygwin'
- name: Checkout code
uses: actions/checkout@v4
with:
path: multicoretests
- name: Fetch QCheck
uses: actions/checkout@v4
with:
repository: c-cube/qcheck
ref: v0.23
path: multicoretests/qcheck
- name: Pre-Setup
run: |
bash multicoretests/.github/runner.sh setup
- name: Set up MSVC
uses: ilammy/msvc-dev-cmd@v1
if: inputs.platform == 'msvc'
- name: Restore cache
uses: actions/cache/restore@v4
id: cache
with:
path: |
${{ env.PREFIX }}
C:\cygwin-packages
key: ${{ env.cache_key }}
- name: Install Cygwin (Windows only)
uses: cygwin/cygwin-install-action@v4
with:
packages: make,bash${{ inputs.platform == 'mingw' && ',mingw64-x86_64-gcc-core,mingw64-x86_64-gcc-g++' || '' }}${{ inputs.platform == 'cygwin' && ',gcc-core' }}
install-dir: 'D:\cygwin'
if: runner.os == 'Windows'
- name: Fetch OCaml
uses: actions/checkout@v4
with:
repository: ${{ env.COMPILER_REPO }}
ref: ${{ env.COMPILER_REF }}
path: ocaml
submodules: true
if: steps.cache.outputs.cache-hit != 'true' || inputs.platform == 'msvc'
# We need to fetch OCaml in all cases for MSVC for msvs-promote-path
- name: Fetch dune
uses: actions/checkout@v4
with:
repository: ocaml/dune
ref: 3.16.0
path: dune
if: steps.cache.outputs.cache-hit != 'true'
- name: Build and install OCaml and dune
run: |
bash multicoretests/.github/runner.sh ocaml
bash multicoretests/.github/runner.sh dune
if: steps.cache.outputs.cache-hit != 'true'
- name: Save cache
uses: actions/cache/save@v4
with:
path: |
${{ env.PREFIX }}
C:\cygwin-packages
key: ${{ env.cache_key }}
if: steps.cache.outputs.cache-hit != 'true'
- name: Show the configuration
run: |
bash multicoretests/.github/runner.sh show_config
- name: Build the test suite
run: |
bash multicoretests/.github/runner.sh build
- name: Run the internal package tests
run: |
bash multicoretests/.github/runner.sh internaltests
- name: Run the multicore test suite
run: |
bash multicoretests/.github/runner.sh testsuite
multicoretests-0.7/.github/workflows/cygwin-52x.yml 0000664 0000000 0000000 00000000447 14743672320 0022517 0 ustar 00root root 0000000 0000000 name: Cygwin 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: cygwin
compiler_ref: refs/tags/5.2.1
timeout: 240
multicoretests-0.7/.github/workflows/cygwin-530-trunk.yml 0000664 0000000 0000000 00000000531 14743672320 0023543 0 ustar 00root root 0000000 0000000 name: Cygwin 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: cygwin
compiler_ref: refs/heads/5.3
timeout: 240
multicoretests-0.7/.github/workflows/cygwin-540-trunk.yml 0000664 0000000 0000000 00000000535 14743672320 0023550 0 ustar 00root root 0000000 0000000 name: Cygwin trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: cygwin
compiler_ref: refs/heads/trunk
timeout: 240
multicoretests-0.7/.github/workflows/gh-pages.yml 0000664 0000000 0000000 00000001732 14743672320 0022274 0 ustar 00root root 0000000 0000000 name: github pages
on:
push:
branches:
- main # Set a branch name to trigger deployment
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Cache opam
id: cache-opam
uses: actions/cache@v3
with:
path: ~/.opam
key: opam-ubuntu-latest-5.0.0
- uses: avsm/setup-ocaml@v3
with:
ocaml-compiler: 'ocaml-base-compiler.5.0.0'
default: https://github.com/ocaml/opam-repository.git
- name: Pin packages
run: opam pin -n .
- name: Install dependencies
run: opam install -d . --deps-only
- name: Build
run: opam exec -- dune build @doc
- name: Deploy
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: ./_build/default/_doc/_html/
destination_dir: dev
enable_jekyll: true
multicoretests-0.7/.github/workflows/linux-52x-32bit.yml 0000664 0000000 0000000 00000000406 14743672320 0023272 0 ustar 00root root 0000000 0000000 name: 32bit 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/tags/5.2.1
options: 32bit
timeout: 240
multicoretests-0.7/.github/workflows/linux-52x-bytecode.yml 0000664 0000000 0000000 00000000421 14743672320 0024142 0 ustar 00root root 0000000 0000000 name: Bytecode 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/tags/5.2.1
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/linux-52x-debug.yml 0000664 0000000 0000000 00000000470 14743672320 0023436 0 ustar 00root root 0000000 0000000 name: Linux 5.2 debug
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/tags/5.2.1
dune_profile: 'debug-runtime'
runparam: 's=4096,V=1'
timeout: 240
multicoretests-0.7/.github/workflows/linux-52x-fp.yml 0000664 0000000 0000000 00000000400 14743672320 0022746 0 ustar 00root root 0000000 0000000 name: FP 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/tags/5.2.1
options: fp
timeout: 240
multicoretests-0.7/.github/workflows/linux-52x-musl.yml 0000664 0000000 0000000 00000000404 14743672320 0023325 0 ustar 00root root 0000000 0000000 name: musl 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/tags/5.2.1
options: musl
timeout: 240
multicoretests-0.7/.github/workflows/linux-52x.yml 0000664 0000000 0000000 00000000336 14743672320 0022353 0 ustar 00root root 0000000 0000000 name: Linux 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/tags/5.2.1
multicoretests-0.7/.github/workflows/linux-530-trunk-32bit.yml 0000664 0000000 0000000 00000000470 14743672320 0024325 0 ustar 00root root 0000000 0000000 name: 32bit 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/5.3
options: 32bit
timeout: 240
multicoretests-0.7/.github/workflows/linux-530-trunk-bytecode.yml 0000664 0000000 0000000 00000000503 14743672320 0025175 0 ustar 00root root 0000000 0000000 name: Bytecode 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/5.3
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/linux-530-trunk-debug.yml 0000664 0000000 0000000 00000000552 14743672320 0024471 0 ustar 00root root 0000000 0000000 name: Linux 5.3 debug
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/5.3
dune_profile: 'debug-runtime'
runparam: 's=4096,V=1'
timeout: 240
multicoretests-0.7/.github/workflows/linux-530-trunk-fp.yml 0000664 0000000 0000000 00000000462 14743672320 0024010 0 ustar 00root root 0000000 0000000 name: FP 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/5.3
options: fp
timeout: 240
multicoretests-0.7/.github/workflows/linux-530-trunk-musl.yml 0000664 0000000 0000000 00000000466 14743672320 0024367 0 ustar 00root root 0000000 0000000 name: musl 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/5.3
options: musl
timeout: 240
multicoretests-0.7/.github/workflows/linux-530-trunk.yml 0000664 0000000 0000000 00000000420 14743672320 0023377 0 ustar 00root root 0000000 0000000 name: Linux 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/5.3
multicoretests-0.7/.github/workflows/linux-540-trunk-32bit.yml 0000664 0000000 0000000 00000000474 14743672320 0024332 0 ustar 00root root 0000000 0000000 name: 32bit trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/trunk
options: 32bit
timeout: 240
multicoretests-0.7/.github/workflows/linux-540-trunk-bytecode.yml 0000664 0000000 0000000 00000000507 14743672320 0025202 0 ustar 00root root 0000000 0000000 name: Bytecode trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/trunk
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/linux-540-trunk-debug.yml 0000664 0000000 0000000 00000000556 14743672320 0024476 0 ustar 00root root 0000000 0000000 name: Linux trunk debug
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/trunk
dune_profile: 'debug-runtime'
runparam: 's=4096,V=1'
timeout: 240
multicoretests-0.7/.github/workflows/linux-540-trunk-fp.yml 0000664 0000000 0000000 00000000466 14743672320 0024015 0 ustar 00root root 0000000 0000000 name: FP trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/trunk
options: fp
timeout: 240
multicoretests-0.7/.github/workflows/linux-540-trunk-musl.yml 0000664 0000000 0000000 00000000472 14743672320 0024365 0 ustar 00root root 0000000 0000000 name: musl trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/trunk
options: musl
timeout: 240
multicoretests-0.7/.github/workflows/linux-540-trunk.yml 0000664 0000000 0000000 00000000424 14743672320 0023404 0 ustar 00root root 0000000 0000000 name: Linux trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
compiler_ref: refs/heads/trunk
multicoretests-0.7/.github/workflows/macosx-arm64-52x.yml 0000664 0000000 0000000 00000000424 14743672320 0023433 0 ustar 00root root 0000000 0000000 name: macOS-ARM64 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: 'macos-14'
platform: macos
compiler_ref: refs/tags/5.2.1
multicoretests-0.7/.github/workflows/macosx-arm64-530-trunk.yml 0000664 0000000 0000000 00000000506 14743672320 0024466 0 ustar 00root root 0000000 0000000 name: macOS-ARM64 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: 'macos-14'
platform: macos
compiler_ref: refs/heads/5.3
multicoretests-0.7/.github/workflows/macosx-arm64-540-trunk.yml 0000664 0000000 0000000 00000000512 14743672320 0024464 0 ustar 00root root 0000000 0000000 name: macOS-ARM64 trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: 'macos-14'
platform: macos
compiler_ref: refs/heads/trunk
multicoretests-0.7/.github/workflows/macosx-intel-52x.yml 0000664 0000000 0000000 00000000424 14743672320 0023615 0 ustar 00root root 0000000 0000000 name: macOS-intel 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: 'macos-13'
platform: macos
compiler_ref: refs/tags/5.2.1
multicoretests-0.7/.github/workflows/macosx-intel-530-trunk.yml 0000664 0000000 0000000 00000000506 14743672320 0024650 0 ustar 00root root 0000000 0000000 name: macOS-intel 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: 'macos-13'
platform: macos
compiler_ref: refs/heads/5.3
multicoretests-0.7/.github/workflows/macosx-intel-540-trunk.yml 0000664 0000000 0000000 00000000512 14743672320 0024646 0 ustar 00root root 0000000 0000000 name: macOS-intel trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: 'macos-13'
platform: macos
compiler_ref: refs/heads/trunk
multicoretests-0.7/.github/workflows/mingw-52x-bytecode.yml 0000664 0000000 0000000 00000000513 14743672320 0024126 0 ustar 00root root 0000000 0000000 name: MinGW bytecode 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: mingw
compiler_ref: refs/tags/5.2.1
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/mingw-52x.yml 0000664 0000000 0000000 00000000445 14743672320 0022336 0 ustar 00root root 0000000 0000000 name: MinGW 5.2
on:
schedule:
# Every Monday morning, at 1:11 UTC
- cron: '11 1 * * 1'
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: mingw
compiler_ref: refs/tags/5.2.1
timeout: 240
multicoretests-0.7/.github/workflows/mingw-530-trunk-bytecode.yml 0000664 0000000 0000000 00000000575 14743672320 0025170 0 ustar 00root root 0000000 0000000 name: MinGW bytecode 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: mingw
compiler_ref: refs/heads/5.3
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/mingw-530-trunk.yml 0000664 0000000 0000000 00000000527 14743672320 0023371 0 ustar 00root root 0000000 0000000 name: MinGW 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: mingw
compiler_ref: refs/heads/5.3
timeout: 240
multicoretests-0.7/.github/workflows/mingw-540-trunk-bytecode.yml 0000664 0000000 0000000 00000000601 14743672320 0025157 0 ustar 00root root 0000000 0000000 name: MinGW bytecode trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: mingw
compiler_ref: refs/heads/trunk
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/mingw-540-trunk.yml 0000664 0000000 0000000 00000000533 14743672320 0023367 0 ustar 00root root 0000000 0000000 name: MinGW trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: mingw
compiler_ref: refs/heads/trunk
timeout: 240
multicoretests-0.7/.github/workflows/msvc-530-trunk-bytecode.yml 0000664 0000000 0000000 00000000573 14743672320 0025015 0 ustar 00root root 0000000 0000000 name: MSVC bytecode 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: msvc
compiler_ref: refs/heads/5.3
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/msvc-530-trunk.yml 0000664 0000000 0000000 00000000502 14743672320 0023211 0 ustar 00root root 0000000 0000000 name: MSVC 5.3
on:
schedule:
# Every Monday morning, at 2:22 UTC
- cron: '22 2 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: msvc
compiler_ref: refs/heads/5.3
multicoretests-0.7/.github/workflows/msvc-540-trunk-bytecode.yml 0000664 0000000 0000000 00000000577 14743672320 0025022 0 ustar 00root root 0000000 0000000 name: MSVC bytecode trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: msvc
compiler_ref: refs/heads/trunk
options: bytecode-only
timeout: 240
multicoretests-0.7/.github/workflows/msvc-540-trunk.yml 0000664 0000000 0000000 00000000506 14743672320 0023216 0 ustar 00root root 0000000 0000000 name: MSVC trunk
on:
schedule:
# Every Monday morning, at 3:33 UTC
- cron: '33 3 * * 1'
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/common.yml
with:
runs_on: windows-latest
platform: msvc
compiler_ref: refs/heads/trunk
multicoretests-0.7/.github/workflows/opam.yml 0000664 0000000 0000000 00000002206 14743672320 0021532 0 ustar 00root root 0000000 0000000 name: OPAM installation
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
on:
pull_request:
push:
branches:
- main
workflow_dispatch:
jobs:
build-and-test:
env:
QCHECK_MSG_INTERVAL: '60'
strategy:
matrix:
ocaml-compiler:
- 4.12.x
- 4.13.x
- 4.14.x
- 5.0.0
- 5.1.0
- 5.2.0
- 5.3.0
- ocaml-variants.5.4.0+trunk
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Install OCaml compiler
uses: ocaml/setup-ocaml@v3
with:
ocaml-compiler: ${{ matrix.ocaml-compiler }}
- name: Test installation of the OPAM packages
run: |
opam install --with-test ./qcheck-multicoretests-util.opam ./qcheck-lin.opam ./qcheck-stm.opam
- name: Show configuration
run: |
opam exec -- ocamlc -config
opam config list
opam exec -- dune printenv
opam list --columns=name,installed-version,repository,synopsis-or-target
multicoretests-0.7/.gitignore 0000664 0000000 0000000 00000000012 14743672320 0016437 0 ustar 00root root 0000000 0000000 _build
*~
multicoretests-0.7/AUTHORS 0000664 0000000 0000000 00000000216 14743672320 0015525 0 ustar 00root root 0000000 0000000 The following people have contributed to multicoretests:
Jan Midtgaard
Olivier Nicole
Nicolas Osborne
Naomi Spargo
Samuel Hym
Charlène Gros
multicoretests-0.7/CHANGES.md 0000664 0000000 0000000 00000005623 14743672320 0016056 0 ustar 00root root 0000000 0000000 # Changes
## 0.7
- #509: Change/Fix to use a symmetric barrier to synchronize domains
- #511: Introduce extended specs to allow wrapping command sequences
- #517: Add `Lin` combinators `seq_small`, `array_small`, and `list_small`
## 0.6
- No changes to the opam-released library packages.
Two significant additions to the test suite in the `multicoretests` opam package:
- #463: Dynarray tests
- #469: Add gc tests
## 0.5
- #492: Also use the new, upstreamed `Gen.exponential` combinator in STM
- #491: Require `qcheck.0.23`, simplify show functions by utilizing it, and update
expect outputs accordingly
- #486: Add `Util.Pp.pp_fun_` printer for generated `QCheck.fun_` functions
## 0.4
- #415: Remove `--verbose` in internal `mutable_set_v5` expect test to avoid
a test failure on a slow machine
- #443: Add `Lin_domain.stress_test` as a lighter stress test, not
requiring an interleaving search.
- #462: Add `STM_domain.stress_test_par`, similar to `Lin_domain.stress_test`
for STM models.
- #472: Switch `arb_cmds` to use an exponential distribution with a
mean of 10, avoiding lists of up to 10000 cmds in `STM_sequential`
(reported by @nikolaushuber).
## 0.3
- #400: Catch and delay exceptions in `STM`'s `next_state` for a nicer UX
- #387: Reduce needless allocations in `Lin`'s sequential consistency
search, as part of an `Out_channel` test cleanup
- #379: Extend the set of `Util.Pp` pretty-printers and teach them to
add break hints similar to `ppx_deriving.show`; teach `to_show` to
generate truncated strings when `$MCTUTILS_TRUNCATE` environment
variable is set
- #368: Switch `STM_domain.agree_prop_par_asym` from using
`Semaphore.Binary` to using an `int Atomic.t` which improves
the error rate across platforms and backends
## 0.2
- #342: Add two submodules of combinators in `Util`:
- `Pp` to pretty-print values back to valid OCaml syntax
- `Equal` to test equality of values
- #337: Add 3 `Bytes.t` combinators to `Lin`: `bytes`, `bytes_small`, `bytes_small_printable`
- #329,340,352: Support `qcheck-lin` and `qcheck-stm` on OCaml 4.12.x, 4.13.x and 4.14.x
without the `Domain` and `Effect` modes
- #316: Fix `rep_count` in `STM_thread` so that negative and positive
tests repeat equally many times
- #318: avoid repetitive interleaving searches in `STM_domain` and `STM_thread`
- #312: Escape and quote `bytes` printed with `STM`'s `bytes` combinator
- #295: ensure `cleanup` is run in the presence of exceptions in
- `STM_sequential.agree_prop` and `STM_domain.agree_prop_par`
- `Lin_thread.lin_prop` and `Lin_effect.lin_prop`
## 0.1.1
- #263: Cleanup resources after each domain-based `Lin` test
- #281: Escape and quote strings printed with `STM`'s `string` combinator
## 0.1
The initial opam release of `qcheck-lin`, `qcheck-stm`, and
`qcheck-multicoretests-util`.
The `multicoretests` package is not released on opam, as it is of
limited use to OCaml developers.
multicoretests-0.7/LICENSE 0000664 0000000 0000000 00000002514 14743672320 0015465 0 ustar 00root root 0000000 0000000 BSD 2-Clause License
Copyright (c) 2021-2022, Jan Midtgaard, Olivier Nicole, Nicolas Osborne
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
multicoretests-0.7/Makefile 0000664 0000000 0000000 00000000100 14743672320 0016105 0 ustar 00root root 0000000 0000000 all:
dune build
clean:
dune clean
rm -f *~ src/*~ issues/*~
multicoretests-0.7/README.md 0000664 0000000 0000000 00000115513 14743672320 0015743 0 ustar 00root root 0000000 0000000 Multicore tests
===============
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/opam.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-52x.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/macosx-intel-52x.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/macosx-arm64-52x.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-52x-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-52x-debug.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-52x-musl.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-52x-32bit.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-52x-fp.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/mingw-52x.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/mingw-52x-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/cygwin-52x.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-530-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/macosx-intel-530-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/macosx-arm64-530-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-530-trunk-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-530-trunk-debug.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-530-trunk-musl.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-530-trunk-32bit.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-530-trunk-fp.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/mingw-530-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/mingw-530-trunk-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/cygwin-530-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/msvc-530-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/msvc-530-trunk-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-540-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/macosx-intel-540-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/macosx-arm64-540-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-540-trunk-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-540-trunk-debug.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-540-trunk-musl.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-540-trunk-32bit.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/linux-540-trunk-fp.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/mingw-540-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/mingw-540-trunk-bytecode.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/cygwin-540-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/msvc-540-trunk.yml)
[](https://github.com/ocaml-multicore/multicoretests/actions/workflows/msvc-540-trunk-bytecode.yml)
Property-based tests of (parts of) the OCaml multicore compiler and run time.
This project contains
- a randomized test suite of OCaml 5.x, packaged up in `multicoretests.opam`
- two reusable testing libraries:
- `Lin` packaged up in `qcheck-lin.opam` and
- `STM` packaged up in `qcheck-stm.opam`
All of the above build on [QCheck](https://github.com/c-cube/qcheck),
a black-box, property-based testing library in the style of QuickCheck.
The two libraries are [already quite helpful](https://tarides.com/blog/2022-12-22-ocaml-5-multicore-testing-tools)
Installation instructions, and running the tests
================================================
The multicore test suite requires OCaml 5.x:
```
opam update
opam switch create 5.0.0
```
Installing the libraries
------------------------
The two testing libraries are available as packages `qcheck-lin`
and `qcheck-stm` from the opam repository. The full versions require
OCaml 5.x and reduced, non-`Domain` versions are available for OCaml
4.12.x to 4.14.x. They can be installed in the usual way:
```
opam install qcheck-lin
opam install qcheck-stm
```
Bleeding edge users can `pin` and install the latest `main` as follows:
```
opam pin -y https://github.com/ocaml-multicore/multicoretests.git#main
```
To use the `Lin` library in parallel mode on a Dune project, add the
following dependency to your dune rule:
```
(libraries qcheck-lin.domain)
```
Using the `STM` library in sequential mode requires the dependency
`(libraries qcheck-stm.sequential)` and the parallel mode similarly
requires the dependency `(libraries qcheck-stm.domain)`.
Running the test suite
----------------------
We have not released the test suite on the [opam
repository](https://github.com/ocaml/opam-repository) at this point.
The test suite can be built and run from a clone of this repository
with the following commands:
```
opam install . --deps-only --with-test
dune build
dune runtest -j1 --no-buffer --display=quiet
```
Individual tests can be run by invoking `dune exec`. For example:
```
$ dune exec src/atomic/stm_tests.exe -- -v
random seed: 51501376
generated error fail pass / total time test name
[âś“] 1000 0 0 1000 / 1000 0.2s sequential atomic test
[âś“] 1000 0 0 1000 / 1000 180.8s parallel atomic test
================================================================================
success (ran 2 tests)
```
See [src/README.md](src/README.md) for an overview of the current
PBTs of OCaml 5.x.
It is also possible to run the test suite in the CI, by altering
[.github/workflows/common.yml](.github/workflows/common.yml) to target
a particular compiler PR:
```
COMPILER_REF: 'refs/pull/12345/head'
```
or a particular branch of a particular fork:
```
COMPILER_REPO: 'login/ocaml'
COMPILER_REF: 'refs/heads/test-me'
```
Since [ocaml/ocaml#13458](https://github.com/ocaml/ocaml/pull/13458)
the test suite can be triggered on an ocaml/ocaml PR (or on a fork of it)
by adding the `run-multicoretests` label.
A Linearization Tester
======================
The `Lin` module lets a user test an API for *sequential consistency*,
i.e., it performs a sequence of random commands in parallel, records
the results, and checks whether the observed results can be linearized
and reconciled with some sequential execution. The library offers an
embedded, combinator DSL to describe signatures succinctly. As an
example, the required specification to test (a small part of) the
`Hashtbl` module is as follows:
``` ocaml
module HashtblSig =
struct
type t = (char, int) Hashtbl.t
let init () = Hashtbl.create ~random:false 42
let cleanup _ = ()
open Lin
let a,b = char_printable,nat_small
let api =
[ val_ "Hashtbl.add" Hashtbl.add (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.remove" Hashtbl.remove (t @-> a @-> returning unit);
val_ "Hashtbl.find" Hashtbl.find (t @-> a @-> returning_or_exc b);
val_ "Hashtbl.mem" Hashtbl.mem (t @-> a @-> returning bool);
val_ "Hashtbl.length" Hashtbl.length (t @-> returning int); ]
end
module HT = Lin_domain.Make(HashtblSig)
;;
QCheck_base_runner.run_tests_main [
HT.lin_test `Domain ~count:1000 ~name:"Lin Hashtbl test";
]
```
The first line indicates the type of the system under test along with
bindings `init` and `cleanup` for setting it up and tearing it down.
The `api` then contains a list of type signature descriptions using
combinators `unit`, `bool`, `int`, `returning`, `returning_or_exc`,
... in the style of [Ctypes](https://github.com/ocamllabs/ocaml-ctypes).
The functor `Lin_domain.Make` expects a description of the tested
commands and outputs a module with a QCheck test `lin_test` that
performs the linearization test.
The QCheck linearization test iterates a number of test
instances. Each instance consists of a "sequential prefix" of calls to
the above commands, followed by a `spawn` of two parallel `Domain`s
that each call a sequence of operations. `Lin` chooses the individual
operations and arguments arbitrarily and records their results. The
framework then performs a search for a sequential interleaving of the
same calls, and succeeds if it finds one.
Since `Hashtbl`s are not safe for parallelism, if you run
`dune exec doc/example/lin_tests.exe` the output can produce the
following output, where each tested command is annotated with its result:
```
Messages for test Lin Hashtbl test:
Results incompatible with sequential execution
|
|
.------------------------------------.
| |
Hashtbl.add t 'a' 0 : () Hashtbl.add t 'a' 0 : ()
Hashtbl.length t : 1 Hashtbl.length t : 1
```
In this case, the test tells us that there is no sequential
interleaving of these calls which would return `1` from both calls to
`Hashtbl.length`. For example, in the following sequential interleaving
the last call should return `2`:
``` ocaml
Hashtbl.add t 'a' 0;;
let res1 = Hashtbl.length t;;
Hashtbl.add t 'a' 0;;
let res2 = Hashtbl.length t;;
```
See [src/atomic/lin_tests.ml](src/atomic/lin_tests.ml) for
another example of testing the `Atomic` module.
A Parallel State-Machine Testing Library
========================================
`STM` contains a revision of [qcstm](https://github.com/jmid/qcstm)
extended to run parallel state-machine tests akin to [Erlang
QuickCheck, Haskell Hedgehog, ScalaCheck, ...](https://github.com/jmid/pbt-frameworks).
To do so, the `STM` library also performs a sequence of random
operations in parallel and records the results. In contrast to `Lin`,
`STM` then checks whether the observed results are linearizable by
reconciling them with a sequential execution of a `model` description.
The `model` expresses the intended meaning of each tested command. As
such, it requires more of the user compared to `Lin`. The
corresponding code to describe a `Hashtbl` test using `STM` is
given below:
``` ocaml
open QCheck
open STM
(** parallel STM tests of Hashtbl *)
module HashtblModel =
struct
type sut = (char, int) Hashtbl.t
type state = (char * int) list
type cmd =
| Add of char * int
| Remove of char
| Find of char
| Mem of char
| Length [@@deriving show { with_path = false }]
let init_sut () = Hashtbl.create ~random:false 42
let cleanup (_:sut) = ()
let arb_cmd (s:state) =
let char =
if s=[]
then Gen.printable
else Gen.(oneof [oneofl (List.map fst s); printable]) in
let int = Gen.nat in
QCheck.make ~print:show_cmd
(Gen.oneof
[Gen.map2 (fun k v -> Add (k,v)) char int;
Gen.map (fun k -> Remove k) char;
Gen.map (fun k -> Find k) char;
Gen.map (fun k -> Mem k) char;
Gen.return Length; ])
let next_state (c:cmd) (s:state) = match c with
| Add (k,v) -> (k,v)::s
| Remove k -> List.remove_assoc k s
| Find _
| Mem _
| Length -> s
let run (c:cmd) (h:sut) = match c with
| Add (k,v) -> Res (unit, Hashtbl.add h k v)
| Remove k -> Res (unit, Hashtbl.remove h k)
| Find k -> Res (result int exn, protect (Hashtbl.find h) k)
| Mem k -> Res (bool, Hashtbl.mem h k)
| Length -> Res (int, Hashtbl.length h)
let init_state = []
let precond (_:cmd) (_:state) = true
let postcond (c:cmd) (s:state) (res:res) = match c,res with
| Add (_,_), Res ((Unit,_),_)
| Remove _, Res ((Unit,_),_) -> true
| Find k, Res ((Result (Int,Exn),_),r) -> r = (try Ok (List.assoc k s) with Not_found -> Error Not_found)
| Mem k, Res ((Bool,_),r) -> r = List.mem_assoc k s
| Length, Res ((Int,_),r) -> r = List.length s
| _ -> false
end
module HT_seq = STM_sequential.Make(HashtblModel)
module HT_dom = STM_domain.Make(HashtblModel)
;;
QCheck_base_runner.run_tests_main
(let count = 200 in
[HT_seq.agree_test ~count ~name:"Hashtbl test sequential";
HT_dom.agree_test_par ~count ~name:"Hashtbl test parallel"; ])
```
Again this requires a type `sut` for the system under test, and
bindings `init_sut` and `cleanup` for setting it up and tearing it
down. The type `cmd` describes the tested commands.
The type `state = (char * int) list` describes with a pure association
list the internal state of a `Hashtbl`. The `init_state` represents
the empty `Hashtbl` mode and the state transition function
`next_state` describes how it changes across each `cmd`. For
example, `Add (k,v)` appends the key-value pair onto the association
list.
`arb_cmd` is a generator of `cmd`s, taking `state` as a parameter.
This allows for `state`-dependent `cmd` generation, which we use
to increase the chance of producing a `Remove 'c'`, `Find 'c'`, ...
following an `Add 'c'`. Internally `arb_cmd` uses QCheck combinators
`Gen.return`, `Gen.map`, and `Gen.map2` to generate one of
5 different commands.
`run` executes the tested `cmd` over the `sut` and wraps the result up
in a result type `res` offered by `STM`. Combinators `unit`, `bool`,
`int`, ... allow to annotate the result with the expected type.
`postcond` expresses a post-condition by matching the received `res`,
for a `cmd` with the corresponding answer from the `model`. For
example, this compares the Boolean result `r` from `Hashtbl.mem` with
the result from `List.mem_assoc`. Similarly `precond` expresses a
pre-condition.
The module is phrased as functors:
- the functor `STM_sequential.Make` produces a module with a function
`agree_test` to test whether the model agrees with the `sut` across
a sequential run of an arbitrary command sequence and
- the functor `STM_domain.Make` produces a module with a function
`agree_test_par` which tests in parallel by `spawn`ing two domains
with `Domain` similarly to `Lin` and searches for a sequential
interleaving over the model.
When running the above with the command `dune exec doc/example/stm_tests.exe`
one may obtain the following output:
```
Messages for test Hashtbl test parallel:
Results incompatible with linearized model
|
|
.------------------------------------.
| |
(Add ('e', 5268)) : () (Add ('!', 4)) : ()
Length : 1 Length : 1
```
This illustrates how two hashtable `Add` commands may interfere when
executed in parallel, leaving only `1` entry in the resulting
`Hashtbl` - which is not reconcilable with the declarative model
description.
The above examples are available from the [doc/example](doc/example)
directory. The [doc](doc) directory also contains our 2022 OCaml
Workshop paper presenting the project in a bit more detail.
Repeatability Efforts
=====================
Both `Lin` and `STM` perform randomized property-based testing with
QCheck. When rerunning a test to shrink/reduce the test input, QCheck
thus starts from the same `Random` seed to limit non-determinism.
This is however not suffient for multicore programs where CPU
scheduling and garbage collection may hinder reproducibility.
`Lin` and `STM` primarily uses test repetition to increase
reproducibility and it is sufficient that only a single repetition
triggers an issue. Currently repeating a non-deterministic QCheck
property can be done in two different ways:
- a `repeat`-combinator lets you test a property, e.g., 50 times
rather than just 1. (Pro: a failure is found faster, Con: wasted,
repetitive testing when there are no failures)
- [a `QCheck` PR](https://github.com/c-cube/qcheck/pull/212) extends `Test.make` with a `~retries` parameter causing
it to only perform repetition during shrinking. (Pro: each test is
cheaper so we can run more, Con: more tests are required to trigger a race)
Issues
======
Replacing blocking functions by non-blocking ones caused deadlocks (new, fixed, runtime)
----------------------------------------------------------------------------------------
[A recently merged PR](https://github.com/ocaml/ocaml/pull/13227) replacing blocking functions
by unblocking ones caused [a regression in the form of deadlocks in the parallel `Sys` STM test](https://github.com/ocaml/ocaml/issuess/13713).
Unboxed `Dynarray` STM tests segfaults (new, fixed, runtime)
------------------------------------------------------------
Early `STM` tests of [the unboxed `Dynarray` PR](https://github.com/ocaml/ocaml/pull/12885) triggered
[segfaults caused by mixing flat float arrays with boxed arrays](https://github.com/ocaml/ocaml/pull/12885#discussion_r1568976695).
Race condition in backup thread logic (new, fixed, runtime)
-----------------------------------------------------------
An assertion error revealed [a race condition between two atomic updates
underlying the coordination between a spawned domain and its backup thread](https://github.com/ocaml/ocaml/issues/13677).
Marking color problem when adopting orphaned Ephemerons (new, runtime)
----------------------------------------------------------------------
An assertion error during the upstreaming of [a mark-delay improvement](https://github.com/ocaml/ocaml/pull/13580), [revealed
a problem](https://github.com/ocaml/ocaml/pull/13580#issuecomment-2478454501) [with the marking color of orphaned Ephemerons]( https://github.com/ocaml-flambda/flambda-backend/pull/3332).
Parallel usage of `flush` may trigger `Sys_error` exception (new, runtime)
--------------------------------------------------------------------------
The `Out_channel` tests found that [`flush` may raise a `Sys_error`
exception when used in parallel with a `close`](https://github.com/ocaml/ocaml/issues/13586).
Registered bytecode fragments leading to bytecode crashes (new, fixed, runtime)
-------------------------------------------------------------------------------
Both the `Gc` and `Domain.DLS` tests triggered crashes due to bytecode fragments in
callbacks [not being properly unregistered](https://github.com/ocaml/ocaml/pull/13549),
[fixed in a separate PR](https://github.com/ocaml/ocaml/pull/13553).
Out of date `Gc.control` documentation (new, fixed, stdlib)
-----------------------------------------------------------
Tests of the `Gc` module revealed that `Gc.control` records contain
[constant zero fields ignored by `Gc.set`](https://github.com/ocaml/ocaml/pull/13440)
Out of date `Gc.quick_stat` documentation (new, fixed, stdlib)
---------------------------------------------------------------
Tests of the `Gc` module revealed that `Gc.quick_stat` did not
return [a record with 4 zero fields as documented](https://github.com/ocaml/ocaml/pull/13424)
Shared heap assertion failure (known, runtime)
----------------------------------------------
New GC tests offered a simple reproducer for consistently triggering
[a shared heap assertion error](https://github.com/ocaml/ocaml/issues/13090)
Unsafe GC interaction in `Gc.counters` binding (known, fixed, runtime)
----------------------------------------------------------------------
New GC tests spotted an issue with unsafe root registration in
`Gc.counters` in 5.2.0, [already fixed upstream](https://github.com/ocaml/ocaml/pull/13370)
Assertion error `s->running` in backup thread termination (new, fixed, runtime)
-------------------------------------------------------------------------------
Tests of `In_channel` would trigger an occasional race in a debug
assertion, due to a [TOCTOU](https://en.wikipedia.org/wiki/Time-of-check_to_time-of-use)
race for [incoming interrupts during backup thread termination](https://github.com/ocaml/ocaml/issues/13386)
Parallel `Dynlink` tests under Windows could deadlock or crash (known, fixed, flexdll)
--------------------------------------------------------------------------------------
Tests of `Dynlink` on Windows revealed that [the underlying FlexDLL was
unsafe for parallel usage](https://github.com/ocaml/ocaml/issues/13046)
`Sys.rename` regression under MinGW/MSVC (new, fixed, runtime)
--------------------------------------------------------------
Earlier fixes to bring Windows behaviour closer to other platforms introduced
[an unfortunate cornercase regression](https://github.com/ocaml/ocaml/pull/13166)
if attempting to `Sys.rename` a parent directory to an empty child directory
Regression causing a Cygwin configure to fail (new, fixed, configure)
---------------------------------------------------------------------
A configure PR accidentally [introduced a regression causing a flexlink test to
fail for a Cygwin build](https://github.com/ocaml/ocaml/pull/13009)
Crash and hangs on MinGW (new, fixed, runtime)
----------------------------------------------
[We observed crashes and hangs of the `threadomain` test under
MinGW](https://github.com/ocaml/ocaml/issues/12230), which turned out to be due
to unsafe systhread yielding.
Regression on output to closed `Out_channel`s (new, fixed, runtime)
-------------------------------------------------------------------
While revising out `Out_channel` tests we discovered [a regression when
outputting to a closed `Out_channel`](https://github.com/ocaml/ocaml/issues/12898)
Failure to build `dune` with trunk (new, fixed, dune)
-----------------------------------------------------
A change to the OCaml compiler's internals revealed that `dune` was [not
using `CAML_INTERNALS` according to the OCaml manual](https://github.com/ocaml/dune/pull/9733)
Hard abort regression on 'failure to create domains' (new, fixed, runtime)
--------------------------------------------------------------------------
The tests found a regression where a failure to create a domain [would trigger an
abort rather than an exception](https://github.com/ocaml/ocaml/pull/12855)
Assertion failures in `runtime/domain.c` on trunk (new, fixed, runtime)
-----------------------------------------------------------------------
A PR merged to `trunk` [reintroduced off-by-one assertion errors in `caml_reset_young_limit`](
https://github.com/ocaml/ocaml/pull/12824)
Assertion failure triggered in runtime/memprof.c (new, fixed, runtime)
----------------------------------------------------------------------
The `thread_joingraph` test triggered [an assertion boundary case in
`caml_memprof_renew_minor_sample` from `memprof.c`](https://github.com/ocaml/ocaml/pull/12817)
Assertion boundary case in `caml_reset_young_limit` (new, fixed, runtime)
-------------------------------------------------------------------------
The `thread_joingraph` test triggered [an assertion boundary case in
`caml_reset_young_limit` which was too strict](https://github.com/ocaml/ocaml/pull/12742)
Assertion race condition in `install_backup_thread` (new, fixed, runtime)
-------------------------------------------------------------------------
A repro test case [submitted upstream from `multicoretests` to the ocaml
compiler test suite](https://github.com/ocaml/ocaml/pull/11749) and two separate
`multicoretests` all [triggered an race condition in `install_backup_thread`](https://github.com/ocaml/ocaml/pull/12707)
Float register preservation on ppc64 (new, fixed, codegen)
----------------------------------------------------------
The sequential `Float.Array` `STM` test revealed that a float register
was not properly preserved on ppc64, sometimes resulting in
[random `float` values appearing](https://github.com/ocaml/ocaml/pull/12546)
Signal-based overflow on ppc64 crash (new, fixed, codegen)
----------------------------------------------------------
The sequential `STM` tests of `Array`, `Bytes`, and `Float.Array`
would [trigger segfaults on ppc64](https://github.com/ocaml/ocaml/issues/12482)
Frame pointer `Effect` crashes (new, fixed, codegen)
----------------------------------------------------
Negative `Lin` `Effect` tests exercising exceptions for unhandled
`Effect`s triggered a [crash on a frame pointer switch](https://github.com/ocaml/ocaml/pull/12535)
s390x `Effect` crashes (new, fixed, codegen)
--------------------------------------------
Negative `Lin` `Effect` tests exercising exceptions for unhandled
`Effect`s also triggered [a crash on the newly restored s390x backend](https://github.com/ocaml/ocaml/issues/12486)
`Sys.rename` behaves differently on corner cases under MingW (new, fixed, stdlib)
---------------------------------------------------------------------------------
Sequential `STM` tests targeting `Sys.rename` found [two corner cases
where MingW behaves differently](https://github.com/ocaml/ocaml/issues/12073)
`flexdll` contains a race condition in its handling of errors (new, fixed, flexdll)
-----------------------------------------------------------------------------------
Parallel `Lin` tests of the `Dynlink` module found [a race
condition](https://github.com/ocaml/flexdll/pull/112) in accesses to
the global variables storing the last error.
`Buffer.add_string` contained a race condition (new, fixed, stdlib)
-------------------------------------------------------------------
Parallel `STM` tests of the `Buffer` module found a segfault, leading
to the discovery of an [assertion failure](https://github.com/ocaml/ocaml/issues/12103)
revealing a race condition in the `add_string` function
Parallel `Weak` `Hashset` usage may crash the runtime (new, fixed, runtime)
---------------------------------------------------------------------------
Parallel `STM` tests found a combination of `Weak` `Hashset` functions
that [may cause the run-time to `abort` or segfault](https://github.com/ocaml/ocaml/issues/11934)
`Sys.readdir` on MingW disagrees with Linux behavior (new, fixed, stdlib)
-------------------------------------------------------------------------
Sequential `STM` tests of `Sys` showed how `Sys.readdir` of a
non-existing directory on MingW Windows [returns an empty `array`, thus
disagreeing with the Linux and macOS behavior](https://github.com/ocaml/ocaml/issues/11829)
`seek` on a closed `in_channel` may read uninitialized memory (new, fixed, runtime)
-----------------------------------------------------------------------------------
A failure of `Lin` `In_channel` tests revealed that `seek` on a closed
`in_channel` [may read uninitialized memory](https://github.com/ocaml/ocaml/issues/11878)
Parallel usage of `Weak` could produce weird values (new, fixed, runtime)
-------------------------------------------------------------------------
Racing `Weak.set` and `Weak.get` [can in some cases produce strange values](https://github.com/ocaml/ocaml/pull/11749)
Bytecode interpreter can segfault on unhandled `Effect` (new, fixed, runtime)
-----------------------------------------------------------------------------
In seldom cases the tests would [trigger a segfault in the bytecode interpreter when treating an unhandled `Effect`](https://github.com/ocaml/ocaml/issues/11669)
`Ephemeron` can fail assert and abort (new, fixed, runtime)
-----------------------------------------------------------
In some cases (even sequential) [the `Ephemeron` tests can trigger an assertion failure and abort](https://github.com/ocaml/ocaml/issues/11503).
Parallel usage of `Bytes.escaped` is unsafe (new, fixed, stdlib)
----------------------------------------------------------------
The `Bytes` tests triggered a segfault which turned out to be caused by [an unsafe `Bytes.escaped` definition](https://github.com/ocaml/ocaml/issues/11508).
Infinite loop in `caml_scan_stack` on ARM64 (known, fixed, runtime)
-------------------------------------------------------------------
The tests triggered [an apparent infinite loop on ARM64 while amd64 would complete the tests as expected](https://github.com/ocaml/ocaml/issues/11425).
Unsafe `Buffer` module (new, fixed, stdlib)
-------------------------------------------
The tests found that the `Buffer` module implementation is [unsafe under parallel usage](https://github.com/ocaml/ocaml/issues/11279) - initially described in [multicoretests#63](https://github.com/ocaml-multicore/multicoretests/pull/63).
MacOS segfault (new, fixed, runtime)
------------------------------------
The tests found an issue causing [a segfault on MacOS](https://github.com/ocaml/ocaml/issues/11226).
`In_channel` and `Out_channel` unsafety (new, fixed, runtime)
-------------------------------------------------------------
The tests found a problem with `In_channel` and `Out_channel` which
could trigger segfaults under parallel usage. For details see
[issue ocaml-multicore/multicoretests#13](https://github.com/ocaml-multicore/multicoretests/pull/13) and
[this ocaml/ocaml#10960 comment](https://github.com/ocaml/ocaml/issues/10960#issuecomment-1087660763).
Cornercase issue in `Domainslib` (new, fixed, domainslib)
---------------------------------------------------------
The tests found an issue in `Domainslib.parallel_for_reduce` which
[would yield the wrong result for empty arrays](https://github.com/ocaml-multicore/domainslib/pull/67).
As of [domainslib#100](https://github.com/ocaml-multicore/domainslib/pull/100)
the `Domainslib` tests have been moved to the `Domainslib` repo.
Specification of `Lockfree.Ws_deque` (new, fixed, lockfree)
-----------------------------------------------------------
The initial tests of `ws_deque` just applied the parallelism property `agree_prop_par`.
However that is not sufficient, as only the original domain (thread)
is allowed to call `push`, `pop`, ..., while a `spawn`ed domain
should call only `steal`.
A custom, revised property test runs a `cmd` prefix, then
`spawn`s a "stealer domain" with `steal`, ... calls, while the
original domain performs calls across a broder random selection
(`push`, `pop`, ...). As of
[lockfree#43](https://github.com/ocaml-multicore/lockfree/pull/43)
this test has now been moved to the `lockfree` repo.
Here is an example output illustrating how `size` may return `-1` when
used in a "stealer domain". The first line in the `Failure` section lists
the original domain's commands and the second lists the stealer
domains commands (`Steal`,...). The second `Messages` section lists a
rough dump of the corresponding return values: `RSteal (Some 73)` is
the result of `Steal`, ... Here it is clear that the spawned domain
successfully steals 73, and then observes both a `-1` and `0` result from
`size` depending on timing. `Size` should therefore not be considered
threadsafe (none of the
[two](https://www.dre.vanderbilt.edu/~schmidt/PDF/work-stealing-dequeue.pdf)
[papers](https://hal.inria.fr/hal-00802885/document) make any such
promises though):
``` ocaml
$ dune exec src/ws_deque_test.exe
random seed: 55610855
generated error fail pass / total time test name
[âś—] 318 0 1 317 / 10000 2.4s parallel ws_deque test (w/repeat)
--- Failure --------------------------------------------------------------------
Test parallel ws_deque test (w/repeat) failed (8 shrink steps):
Seq.prefix: Parallel procs.:
[] [(Push 73); Pop; Is_empty; Size]
[Steal; Size; Size]
+++ Messages ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Messages for test parallel ws_deque test (w/repeat):
Result observations not explainable by linearized model:
Seq.prefix: Parallel procs.:
[] [RPush; (RPop None); (RIs_empty true); (RSize 0)]
[(RSteal (Some 73)); (RSize -1); (RSize 0)]
================================================================================
failure (1 tests failed, 0 tests errored, ran 1 tests)
```
Segfault in Domainslib (known, fixed, domainslib)
-------------------------------------------------
Testing `Domainslib.Task`s with one dependency and with 2 work pools
found a [segfault in domainslib](https://github.com/ocaml-multicore/domainslib/issues/58).
As of [domainslib#100](https://github.com/ocaml-multicore/domainslib/pull/100)
the `domainslib/task_one_dep.ml` test in question has been moved to
the `Domainslib` repo.
Dead-lock in Domainslib (known, fixed, domainslib)
--------------------------------------------------
A reported deadlock in domainslib motivated the development of these tests:
- https://github.com/ocaml-multicore/domainslib/issues/47
- https://github.com/ocaml-multicore/ocaml-multicore/issues/670
The tests `domainslib/task_one_dep.ml` and
`domainslib/task_more_deps.ml` were run with a timeout to prevent
deadlocking indefinitely. `domainslib/task_one_dep.ml` could trigger one
such deadlock. As of [domainslib#100](https://github.com/ocaml-multicore/domainslib/pull/100)
these tests have been moved to the `Domainslib` repo.
The test exhibits no non-determistic behaviour when repeating the same
tested property from within the QCheck test.
However it fails (due to timeout) on the following test input:
```ocaml
$ dune exec -- src/task_one_dep.exe -v
random seed: 147821373
generated error fail pass / total time test name
[âś—] 25 0 1 24 / 100 36.2s Task.async/await
--- Failure --------------------------------------------------------------------
Test Task.async/await failed (2 shrink steps):
{ num_domains = 3; length = 6;
dependencies = [|None; (Some 0); None; (Some 1); None; None|] }
================================================================================
failure (1 tests failed, 0 tests errored, ran 1 tests)
```
This corresponds to the following program with 3+1 domains and 6 promises.
It loops infinitely with both bytecode/native:
```ocaml
...
open Domainslib
(* a simple work item, from ocaml/testsuite/tests/misc/takc.ml *)
let rec tak x y z =
if x > y then tak (tak (x-1) y z) (tak (y-1) z x) (tak (z-1) x y)
else z
let work () =
for _ = 1 to 200 do
assert (7 = tak 18 12 6);
done
let pool = Task.setup_pool ~num_additional_domains:3 ()
let p0 = Task.async pool work
let p1 = Task.async pool (fun () -> work (); Task.await pool p0)
let p2 = Task.async pool work
let p3 = Task.async pool (fun () -> work (); Task.await pool p1)
let p4 = Task.async pool work
let p5 = Task.async pool work
let () = List.iter (fun p -> Task.await pool p) [p0;p1;p2;p3;p4;p5]
let () = Task.teardown_pool pool
```
---
This project has been created and is maintained by Tarides.
multicoretests-0.7/doc/ 0000775 0000000 0000000 00000000000 14743672320 0015223 5 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/README.md 0000664 0000000 0000000 00000001006 14743672320 0016477 0 ustar 00root root 0000000 0000000 This directory contains:
- [example](example) - the `Lin` and `STM` examples from [../README.md](../README.md)
with small tests of the `Hashtbl` module.
- [paper.md](paper.md) - a paper presenting the project (in Markdown format)
- [paper.pdf](paper.pdf) - the same paper presenting the project (in pdf format)
- [paper-latex](paper-latex) - is the LaTeX source code for above
- [paper-examples](paper-examples) - the `Lin` and `STM` examples from
the paper with (slightly larger) tests of the `Hashtbl` module.
multicoretests-0.7/doc/example/ 0000775 0000000 0000000 00000000000 14743672320 0016656 5 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/example/dune 0000664 0000000 0000000 00000000520 14743672320 0017531 0 ustar 00root root 0000000 0000000 ;; A linearization test of the stdlib Hashtbl library
(executable
(name lin_tests)
(modules lin_tests)
(libraries qcheck-lin.domain))
;; A model-based test of the stdlib Hashtbl library
(executable
(name stm_tests)
(modules stm_tests)
(libraries qcheck-stm.sequential qcheck-stm.domain)
(preprocess (pps ppx_deriving.show)))
multicoretests-0.7/doc/example/dune-project 0000664 0000000 0000000 00000000020 14743672320 0021170 0 ustar 00root root 0000000 0000000 (lang dune 2.9)
multicoretests-0.7/doc/example/dune-workspace 0000664 0000000 0000000 00000000000 14743672320 0021516 0 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/example/lin_tests.ml 0000664 0000000 0000000 00000001641 14743672320 0021216 0 ustar 00root root 0000000 0000000 (* ********************************************************************** *)
(* Tests of thread-unsafe [Hashtbl] *)
(* ********************************************************************** *)
module HashtblSig =
struct
type t = (char, int) Hashtbl.t
let init () = Hashtbl.create ~random:false 42
let cleanup _ = ()
open Lin
let a,b = char_printable,nat_small
let api =
[ val_ "Hashtbl.add" Hashtbl.add (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.remove" Hashtbl.remove (t @-> a @-> returning unit);
val_ "Hashtbl.find" Hashtbl.find (t @-> a @-> returning_or_exc b);
val_ "Hashtbl.mem" Hashtbl.mem (t @-> a @-> returning bool);
val_ "Hashtbl.length" Hashtbl.length (t @-> returning int); ]
end
module HT = Lin_domain.Make(HashtblSig)
;;
QCheck_base_runner.run_tests_main [
HT.lin_test ~count:1000 ~name:"Lin Hashtbl test";
]
multicoretests-0.7/doc/example/stm_tests.ml 0000664 0000000 0000000 00000003760 14743672320 0021243 0 ustar 00root root 0000000 0000000 open QCheck
open STM
(** parallel STM tests of Hashtbl *)
module HashtblModel =
struct
type sut = (char, int) Hashtbl.t
type state = (char * int) list
type cmd =
| Add of char * int
| Remove of char
| Find of char
| Mem of char
| Length [@@deriving show { with_path = false }]
let init_sut () = Hashtbl.create ~random:false 42
let cleanup (_:sut) = ()
let arb_cmd (s:state) =
let char =
if s=[]
then Gen.printable
else Gen.(oneof [oneofl (List.map fst s); printable]) in
let int = Gen.nat in
QCheck.make ~print:show_cmd
(Gen.oneof
[Gen.map2 (fun k v -> Add (k,v)) char int;
Gen.map (fun k -> Remove k) char;
Gen.map (fun k -> Find k) char;
Gen.map (fun k -> Mem k) char;
Gen.return Length; ])
let next_state (c:cmd) (s:state) = match c with
| Add (k,v) -> (k,v)::s
| Remove k -> List.remove_assoc k s
| Find _
| Mem _
| Length -> s
let run (c:cmd) (h:sut) = match c with
| Add (k,v) -> Res (unit, Hashtbl.add h k v)
| Remove k -> Res (unit, Hashtbl.remove h k)
| Find k -> Res (result int exn, protect (Hashtbl.find h) k)
| Mem k -> Res (bool, Hashtbl.mem h k)
| Length -> Res (int, Hashtbl.length h)
let init_state = []
let precond (_:cmd) (_:state) = true
let postcond (c:cmd) (s:state) (res:res) = match c,res with
| Add (_,_), Res ((Unit,_),_)
| Remove _, Res ((Unit,_),_) -> true
| Find k, Res ((Result (Int,Exn),_),r) -> r = (try Ok (List.assoc k s) with Not_found -> Error Not_found)
| Mem k, Res ((Bool,_),r) -> r = List.mem_assoc k s
| Length, Res ((Int,_),r) -> r = List.length s
| _ -> false
end
module HT_seq = STM_sequential.Make(HashtblModel)
module HT_dom = STM_domain.Make(HashtblModel)
;;
QCheck_base_runner.run_tests_main
(let count = 200 in
[HT_seq.agree_test ~count ~name:"Hashtbl test sequential";
HT_dom.agree_test_par ~count ~name:"Hashtbl test parallel"; ])
multicoretests-0.7/doc/lin/ 0000775 0000000 0000000 00000000000 14743672320 0016005 5 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/lin/dune 0000664 0000000 0000000 00000000373 14743672320 0016666 0 ustar 00root root 0000000 0000000 (documentation
(package qcheck-lin)
(mld_files index))
(executable
(name mutable_set)
(modules mutable_set)
(libraries qcheck-lin.domain))
(executable
(name mutable_set_lock)
(modules mutable_set_lock)
(libraries qcheck-lin.domain))
multicoretests-0.7/doc/lin/index.mld 0000664 0000000 0000000 00000027554 14743672320 0017627 0 ustar 00root root 0000000 0000000 {0 qcheck-lin}
{1 Content}
- {!module-Lin} is a base module for specifying sequential consistency tests.
- {!module-Lin_domain} exposes a functor that allows to test a library under
parallel usage (with domains).
- {!module-Lin_thread} exposes a functor that allows to test a library under
concurrent usage (with threads).
- {!module-Lin_effect} exposes a functor that allows to test a library under
concurrent usage (with effects).
{1 Overview: what is [qcheck-lin]?}
[qcheck-lin] is a testing library based on [QCheck] for sequential consistency.
A parallel or concurrent program is said to be sequentially consistent if "the
result of any execution is the same as if the operations of all the processors
were executed in some sequential order, and the operations of each individual
processor appear in this sequence in the order specified by its
program."{{!section-ref}{^ 1}}
According to a library description, [qcheck-lin] generates random programs
using the functionalities of this library and runs them, records the results
and checks whether the observed results are linearizable by reconciling them
with a sequential execution.
[qcheck-lin] offers an embedded domain specific language to easily describe
signatures succinctly. It provides three types of tests:
- a parallel one, generating and running parallel programs with two domains and
testing for sequential consistency,
- another concurrent one, generating and running parallel programs with two
threads and testing for sequential consistency,
- a concurrent one using effects, generating and running parallel programs with
two fibers and testing for sequential consistency.
{1 Example: how to test a library?}
Suppose we want to implement a small mutable set library, our main focus being
to have a constant time [cardinal] operation. We will be using {!Stdlib.Set} for
the content, keeping track of the cardinality when adding and removing
elements.
Of course, we will be using [qcheck-lin] for testing!
Our library reads like that:
{[
module type S = sig
type elt
type t
val empty : unit -> t
val mem : elt -> t -> bool
val add : elt -> t -> unit
val remove : elt -> t -> unit
val cardinal : t -> int
end
module Lib : sig
module Make : functor (Ord : Set.OrderedType) -> S with type elt = Ord.t
end = struct
module Make (Ord : Set.OrderedType) = struct
module S = Set.Make (Ord)
type elt = Ord.t
type t = { mutable content : S.t; mutable cardinal : int }
let empty () = { content = S.empty; cardinal = 0 }
let mem a t = S.mem a t.content
let add a t =
if not (mem a t) then begin
t.content <- S.add a t.content;
t.cardinal <- t.cardinal + 1
end
let remove a t =
if mem a t then (
t.content <- S.remove a t.content;
t.cardinal <- t.cardinal - 1)
let cardinal t = t.cardinal
end
end
]}
{2 Writing a specification}
In order to test it for sequential consistency, [qcheck-lin] needs a
lightweight specification of our library's interface. This specification takes
the form of a module matching the {{!Lin.Spec}[Spec]} signature. Then
[qcheck-lin] does all the heavy lifting for us!
This specification exposes one type and three values:
- type [t] which is the main type of our library, here the mutable set.
- [init] that tells [qcheck-lin] how to create an initial value of type [t].
- [cleanup] that tells [qcheck-lin] how to clean up after the tests (which is
necessary when [t] uses resources that must be released, such as opened
files, network connections, etc.).
- [api] which is a list of the library's functions we want to include in the
tests. These functions are encoded using a custom embedded domain specific
language.
{[
open Lin
module LibInt = Lib.Make (Int)
module Spec : Spec = struct
type t = LibInt.t
let init = LibInt.empty
let cleanup _ = ()
let api =
let int = nat_small in
[
val_ "mem" LibInt.mem (int @-> t @-> returning bool);
val_ "add" LibInt.add (int @-> t @-> returning unit);
val_ "remove" LibInt.remove (int @-> t @-> returning unit);
val_ "cardinal" LibInt.cardinal (t @-> returning int);
]
end
]}
Let's have a closer look at the [api] value. This is where [qcheck-lin] gets
the information about the functions we want to include in our tests. [api] is a
list of values. These values should be constructed using either the
{{!Lin.val_}[val_]} function or the {{!Lin.val_freq}[val_freq]} one.
In the example, we use [val_], but let's first describe [val_freq].
[val_freq] takes four arguments:
- an [int] that is a weight used by [QCheck] for generation. It allows to skew
the distribution of the function in the generated programs.
- its name as a [string] so that it can be printed in the output.
- the function itself, so that it can be called in the tests.
- an encoding of its type which is used to generate arguments to the function.
[val_] is a specialization of [val_freq], giving the same weight to all the
elements.
Note that the domain specific language brings some static guarantees about the
type encoding. If we make a mistake, we will know at compile time.
{2 Running the tests}
Now we are set to run our first [qcheck-lin] tests!
We will be testing our library for parallel usage. The functor
{!module-Lin_domain.Make} takes the [Spec] as argument and exposes two
functions:
- {{!Lin_domain.Make.lin_test}[lin_test]} to build a positive [QCheck] test,
- {{!Lin_domain.Make.neg_lin_test}[neg_lin_test]} to build a negative [QCheck] test.
Here, we expect the test to succeed, so we will use the first one.
{[
module LibDomain = Lin_domain.Make (Spec)
let _ =
QCheck_base_runner.run_tests ~verbose:true
[ LibDomain.lin_test ~count:1000 ~name:"Lin parallel tests" ]
]}
And the test fails...
{[
$ dune exec ./mutable_set.exe
random seed: 429006728
generated error fail pass / total time test name
[âś—] 1 0 1 0 / 1000 1.9s Lin parallel tests
--- Failure --------------------------------------------------------------------
Test Lin parallel tests failed (41 shrink steps):
|
|
.---------------------.
| |
add 0 t add 0 t
cardinal t
+++ Messages ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Messages for test Lin parallel tests:
Results incompatible with sequential execution
|
|
.------------------------------------.
| |
add 0 t : () add 0 t : ()
cardinal t : 2
================================================================================
failure (1 tests failed, 0 tests errored, ran 1 tests)
]}
In the case of a failing test, [qcheck-lin] prints the counterexample
it has found, after some shrinking steps, as is customary in property-based
testing {i Ă la} QuickCheck. The counterexample is a program with a sequential
prefix and two parallel suffixes. The counterexample is given twice: the first
time with just the function calls; and the second time, each call is paired
with its result.
Here, the counterexample is composed of an empty sequential prefix and two
spawned processes. The first one adds 0 to the set, while the second one also
adds 0 to the set and then asks for the cardinality of the said set.
In terms of sequential interleaving, there are only three possibilities. If we
prefix the function call by [Left] and [Right] depending on the process in
which they are, those sequential interleavings are:
1. [Left.add 0 t; Right.add 0 t; Right.cardinal t]
2. [Right.add 0 t; Left.add 0 t; Right.cardinal t]
3. [Right.add 0 t; Right.cardinal t; Left.add 0 t]
None of them can explain the value 2 returned by [cardinal t].
What happened is that both calls to [add] have checked whether [0] was already
an element of the set at the same time (or at least before the other one had a
chance to add it). It was not. So, according to our implementation, both calls
added the element and incremented the [cardinal] field.
One way to make our library safe for parallel usage is to protect the set with
a mutex.
{[
module type S = sig
type elt
type t
val empty : unit -> t
val mem : elt -> t -> bool
val add : elt -> t -> unit
val remove : elt -> t -> unit
val cardinal : t -> int
end
module Lib : sig
module Make : functor (Ord : Set.OrderedType) -> S with type elt = Ord.t
end = struct
module Make (Ord : Set.OrderedType) = struct
module S = Set.Make (Ord)
type elt = Ord.t
type t = { mutable content : S.t; mutable cardinal : int; mutex : Mutex.t }
let empty () = { content = S.empty; cardinal = 0; mutex = Mutex.create () }
let mem_non_lock a t = S.mem a t.content
let mem a t =
Mutex.lock t.mutex;
let b = S.mem a t.content in
Mutex.unlock t.mutex;
b
let add a t =
Mutex.lock t.mutex;
if not (mem_non_lock a t) then being
t.content <- S.add a t.content;
t.cardinal <- t.cardinal + 1
end;
Mutex.unlock t.mutex
let remove a t =
Mutex.lock t.mutex;
if mem_non_lock a t then begin
t.content <- S.remove a t.content;
t.cardinal <- t.cardinal - 1
end;
Mutex.unlock t.mutex
let cardinal t =
Mutex.lock t.mutex;
let c = t.cardinal in
Mutex.unlock t.mutex;
c
end
end
]}
Once this is done, we can use exactly the same specification and tests. The
successful output looks like the following:
{[
$ dune exec ./mutable_set_lock.exe
random seed: 162610433
generated error fail pass / total time test name
[âś“] 1000 0 0 1000 / 1000 48.0s Lin parallel tests
================================================================================
success (ran 1 tests)
]}
{1 [Lin] in a bit more detail}
Underneath the hood [Lin] uses QCheck and OCaml's pseudo-random number
generator from the [Random] module to generate arbitrary [cmd]
sequences and arbitrary input argument data to each call. To recreate
a problematic test run, one therefore needs to generate the same
pseudo-random test case input, by passing the same randomness seed.
By running the [Lin] tests using [QCheck_base_runner.run_tests_main]
from QCheck, it is possible to pass a seed as a command line argument
as follows:
{[
$ dune exec ./mutable_set.exe -- -s 429006728
]}
Despite generating and thus running the same test case input, one may
still experience different behaviours on subsequent reruns of the
resulting test, because of CPU scheduling and other factors. This may
materialize as different counterexamples being printed or as one run
failing the test whereas another run passes it. {!Lin_domain} uses the
{!Util.repeat} combinator to repeat each test case 50 times to
address the issue and help increase reproducibility.
{1 Current limitations}
[Lin] comes with a number of limitations which we plan to address in
future releases. Currently {{!Lin.Spec.api}[Spec.api]} descriptions:
- support only one {{!Lin.Spec.t}[Spec.t]} - namely the one resulting
from {{!Lin.Spec.init}[Spec.init]},
- do not support composing {{!Lin.Spec.t}[Spec.t]} with other type
combinators such as {!Lin.list} and {!Lin.option} - this restriction
is expressed with the {!Lin.noncombinable} type parameter in {!Lin.t},
- do not support using the result of {!Lin.int_bound} as both an
argument type and as a result type - you can use {!Lin.int} for the
latter,
- do not support specifying function values using arrow syntax [t1 @-> t2],
- do not support specifying tuple values using product syntax [t1 * t2].
The later two can however be addressed by writing a custom argument
generator using {!Lin.gen}.
{1:ref References}
1. Lamport, {i How to Make a Multiprocessor Computer That Correctly Executes
Multiprocess Program}, 1979, DOI: 10.1109/TC.1979.1675439
multicoretests-0.7/doc/lin/mutable_set.ml 0000664 0000000 0000000 00000002721 14743672320 0020645 0 ustar 00root root 0000000 0000000 module type S = sig
type elt
type t
val empty : unit -> t
val mem : elt -> t -> bool
val add : elt -> t -> unit
val remove : elt -> t -> unit
val cardinal : t -> int
end
module Lib : sig
module Make : functor (Ord : Set.OrderedType) -> S with type elt = Ord.t
end = struct
module Make (Ord : Set.OrderedType) = struct
module S = Set.Make (Ord)
type elt = Ord.t
type t = { mutable content : S.t; mutable cardinal : int }
let empty () = { content = S.empty; cardinal = 0 }
let mem a t = S.mem a t.content
let add a t =
if not (mem a t) then (
t.content <- S.add a t.content;
t.cardinal <- t.cardinal + 1)
let remove a t =
if mem a t then (
t.content <- S.remove a t.content;
t.cardinal <- t.cardinal - 1)
let cardinal t = t.cardinal
end
end
open Lin
module LibInt = Lib.Make (Int)
module Spec : Spec = struct
type t = LibInt.t
let init = LibInt.empty
let cleanup _ = ()
let api =
let int = nat_small in
[
val_ "mem" LibInt.mem (int @-> t @-> returning bool);
val_ "add" LibInt.add (int @-> t @-> returning unit);
val_ "remove" LibInt.remove (int @-> t @-> returning unit);
val_ "cardinal" LibInt.cardinal (t @-> returning int);
]
end
module LibDomain = Lin_domain.Make (Spec)
let _ =
QCheck_base_runner.run_tests ~verbose:true
[ LibDomain.lin_test ~count:1000 ~name:"Lin parallel tests" ]
multicoretests-0.7/doc/lin/mutable_set_lock.ml 0000664 0000000 0000000 00000003536 14743672320 0021662 0 ustar 00root root 0000000 0000000 module type S = sig
type elt
type t
val empty : unit -> t
val mem : elt -> t -> bool
val add : elt -> t -> unit
val remove : elt -> t -> unit
val cardinal : t -> int
end
module Lib : sig
module Make : functor (Ord : Set.OrderedType) -> S with type elt = Ord.t
end = struct
module Make (Ord : Set.OrderedType) = struct
module S = Set.Make (Ord)
type elt = Ord.t
type t = { mutable content : S.t; mutable cardinal : int; mutex : Mutex.t }
let empty () = { content = S.empty; cardinal = 0; mutex = Mutex.create () }
let mem_non_lock a t = S.mem a t.content
let mem a t =
Mutex.lock t.mutex;
let b = S.mem a t.content in
Mutex.unlock t.mutex;
b
let add a t =
Mutex.lock t.mutex;
if not (mem_non_lock a t) then begin
t.content <- S.add a t.content;
t.cardinal <- t.cardinal + 1
end;
Mutex.unlock t.mutex
let remove a t =
Mutex.lock t.mutex;
if mem_non_lock a t then begin
t.content <- S.remove a t.content;
t.cardinal <- t.cardinal - 1
end;
Mutex.unlock t.mutex
let cardinal t =
Mutex.lock t.mutex;
let c = t.cardinal in
Mutex.unlock t.mutex;
c
end
end
open Lin
module LibInt = Lib.Make (Int)
module Spec : Spec = struct
type t = LibInt.t
let init = LibInt.empty
let cleanup _ = ()
let api =
let int = nat_small in
[
val_ "mem" LibInt.mem (int @-> t @-> returning bool);
val_ "add" LibInt.add (int @-> t @-> returning unit);
val_ "remove" LibInt.remove (int @-> t @-> returning unit);
val_ "cardinal" LibInt.cardinal (t @-> returning int);
]
end
module LibDomain = Lin_domain.Make (Spec)
let _ =
QCheck_base_runner.run_tests ~verbose:true
[ LibDomain.lin_test ~count:1000 ~name:"Lin parallel tests" ]
multicoretests-0.7/doc/paper-examples/ 0000775 0000000 0000000 00000000000 14743672320 0020146 5 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/paper-examples/dune 0000664 0000000 0000000 00000000520 14743672320 0021021 0 ustar 00root root 0000000 0000000 ;; A linearization test of the stdlib Hashtbl library
(executable
(name lin_tests)
(modules lin_tests)
(libraries qcheck-lin.domain))
;; A model-based test of the stdlib Hashtbl library
(executable
(name stm_tests)
(modules stm_tests)
(libraries qcheck-stm.sequential qcheck-stm.domain)
(preprocess (pps ppx_deriving.show)))
multicoretests-0.7/doc/paper-examples/dune-project 0000664 0000000 0000000 00000000020 14743672320 0022460 0 ustar 00root root 0000000 0000000 (lang dune 2.9)
multicoretests-0.7/doc/paper-examples/dune-workspace 0000664 0000000 0000000 00000000000 14743672320 0023006 0 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/paper-examples/lin_tests.ml 0000664 0000000 0000000 00000002124 14743672320 0022503 0 ustar 00root root 0000000 0000000 (* ********************************************************************** *)
(* Tests of thread-unsafe [Hashtbl] *)
(* ********************************************************************** *)
module HashtblSig =
struct
type t = (char, int) Hashtbl.t
let init () = Hashtbl.create ~random:false 42
let cleanup _ = ()
open Lin
let a,b = char_printable,nat_small
let api =
[ val_ "Hashtbl.clear" Hashtbl.clear (t @-> returning unit);
val_ "Hashtbl.add" Hashtbl.add (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.remove" Hashtbl.remove (t @-> a @-> returning unit);
val_ "Hashtbl.find" Hashtbl.find (t @-> a @-> returning_or_exc b);
val_ "Hashtbl.replace" Hashtbl.replace (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.mem" Hashtbl.mem (t @-> a @-> returning bool);
val_ "Hashtbl.length" Hashtbl.length (t @-> returning int);
]
end
module HT = Lin_domain.Make(HashtblSig)
;;
QCheck_base_runner.run_tests_main [
HT.lin_test ~count:1000 ~name:"Lin Hashtbl test";
]
multicoretests-0.7/doc/paper-examples/stm_tests.ml 0000664 0000000 0000000 00000004714 14743672320 0022533 0 ustar 00root root 0000000 0000000 open QCheck
open STM
(** parallel STM tests of Hashtbl *)
module HashtblModel =
struct
type sut = (char, int) Hashtbl.t
type state = (char * int) list
type cmd =
| Clear
| Add of char * int
| Remove of char
| Find of char
| Replace of char * int
| Mem of char
| Length [@@deriving show { with_path = false }]
let init_sut () = Hashtbl.create ~random:false 42
let cleanup (_:sut) = ()
let arb_cmd (s:state) =
let char =
if s=[]
then Gen.printable
else Gen.(oneof [oneofl (List.map fst s); printable]) in
let int = Gen.nat in
QCheck.make ~print:show_cmd
(Gen.oneof
[Gen.return Clear;
Gen.map2 (fun k v -> Add (k,v)) char int;
Gen.map (fun k -> Remove k) char;
Gen.map (fun k -> Find k) char;
Gen.map2 (fun k v -> Replace (k,v)) char int;
Gen.map (fun k -> Mem k) char;
Gen.return Length;
])
let next_state (c:cmd) (s:state) = match c with
| Clear -> []
| Add (k,v) -> (k,v)::s
| Remove k -> List.remove_assoc k s
| Find _ -> s
| Replace (k,v) -> (k,v)::(List.remove_assoc k s)
| Mem _
| Length -> s
let run (c:cmd) (h:sut) = match c with
| Clear -> Res (unit, Hashtbl.clear h)
| Add (k,v) -> Res (unit, Hashtbl.add h k v)
| Remove k -> Res (unit, Hashtbl.remove h k)
| Find k -> Res (result int exn, protect (Hashtbl.find h) k)
| Replace (k,v) -> Res (unit, Hashtbl.replace h k v)
| Mem k -> Res (bool, Hashtbl.mem h k)
| Length -> Res (int, Hashtbl.length h)
let init_state = []
let precond (_:cmd) (_:state) = true
let postcond (c:cmd) (s:state) (res:res) = match c,res with
| Clear, Res ((Unit,_),_)
| Add (_,_), Res ((Unit,_),_)
| Remove _, Res ((Unit,_),_) -> true
| Find k, Res ((Result (Int,Exn),_),r) -> r = (try Ok (List.assoc k s) with Not_found -> Error Not_found)
| Replace (_,_), Res ((Unit,_),_) -> true
| Mem k, Res ((Bool,_),r) -> r = List.mem_assoc k s
| Length, Res ((Int,_),r) -> r = List.length s
| _ -> false
end
module HT_seq = STM_sequential.Make(HashtblModel)
module HT_dom = STM_domain.Make(HashtblModel)
;;
QCheck_base_runner.run_tests_main
(let count = 200 in
[HT_seq.agree_test ~count ~name:"Hashtbl test sequential";
HT_dom.agree_test_par ~count ~name:"Hashtbl test parallel";
])
multicoretests-0.7/doc/paper-latex/ 0000775 0000000 0000000 00000000000 14743672320 0017445 5 ustar 00root root 0000000 0000000 multicoretests-0.7/doc/paper-latex/.latexmkrc 0000664 0000000 0000000 00000000172 14743672320 0021440 0 ustar 00root root 0000000 0000000 # Use lualatex, generate PDF
#$pdf_mode = 4;
# Use xelatex, generate PDF
$pdf_mode = 5;
$postscript_mode = $dvi_mode = 0;
multicoretests-0.7/doc/paper-latex/README.md 0000664 0000000 0000000 00000000154 14743672320 0020724 0 ustar 00root root 0000000 0000000 To compile (requires fonts Libertinus and Fira Mono installed as OpenType
fonts):
```
latexmk paper.tex
```
multicoretests-0.7/doc/paper-latex/biblio.bib 0000664 0000000 0000000 00000014232 14743672320 0021365 0 ustar 00root root 0000000 0000000
@online{AddFailingOut2022,
title = {Add (Failing) \{\vphantom\}{{In}},{{Out}}\vphantom\{\}\_channel Linearization Tests},
OPTdate = {2022-03-10},
url = {https://github.com/jmid/multicoretests/pull/13},
OPTorganization = {{Multicoretests Github repository}}
}
@online{Articheck,
title = {Articheck},
url = {https://github.com/braibant/articheck}
}
@inproceedings{artsTestingTelecomsSoftware2006,
title = {Testing Telecoms Software with Quviq {{QuickCheck}}},
booktitle = {Proceedings of the 2006 {{ACM SIGPLAN Workshop}} on {{Erlang}} ({{Erlang}} 2006)},
author = {Arts, Thomas and Hughes, John and Johansson, Joakim and Wiger, Ulf T.},
date = {2006},
pages = {2--10},
}
@online{AuditStdlibMutable2022,
title = {Audit Stdlib for Mutable State (Comment)},
OPTdate = {2022-04-04},
url = {https://github.com/ocaml/ocaml/issues/10960#issuecomment-1087660763},
OPTorganization = {{OCaml Github repository}}
}
@inproceedings{braibantWelltypedSmartFuzzing2014,
title = {Well-Typed Generic Smart Fuzzing for APIs},
author = {Braibant, Thomas and Protzenko, Jonathan and Scherer, Gabriel},
date = {2014},
url = {https://hal.inria.fr/hal-01094006},
eventtitle = {{{ML Familiy Workshop}} ({{ML}} 2014)}
}
@inproceedings{claessenFindingRaceConditions2009,
title = {Finding {{Race Conditions}} in {{Erlang}} with {{QuickCheck}} and {{PULSE}}},
booktitle = {Proceeding of the 14th {{ACM SIGPLAN}} International Conference on {{Functional}} Programming ({{ICFP}} 2009)},
author = {Claessen, Koen and Pałka, Michał and Smallbone, Nicholas and Hughes, John and Svensson, Hans and Arts, Thomas and Wiger, Ulf},
date = {2009},
pages = {12},
}
@inproceedings{claessenQuickCheckLightweightTool2000,
title = {{{QuickCheck}}: A Lightweight Tool for Random Testing of {{Haskell}} Programs},
shorttitle = {{{QuickCheck}}},
booktitle = {Proceedings of the {{Fifth ACM SIGPLAN International Conference}} on {{Functional Programming}} ({{ICFP}} 2000)},
author = {Claessen, Koen and Hughes, John},
date = {2000},
pages = {268--279},
}
@inproceedings{claessenTestingMonadicCode2002,
title = {Testing Monadic Code with {{QuickCheck}}},
booktitle = {Proceedings of the 2002 {{ACM SIGPLAN Workshop}} on {{Haskell}} ({{Haskell}} 2002)},
author = {Claessen, Koen and Hughes, John},
date = {2002},
pages = {65--77},
}
@online{Crowbar,
title = {Crowbar},
url = {https://github.com/stedolan/crowbar}
}
@online{Ctypes,
title = {Ctypes},
url = {https://github.com/ocamllabs/ocaml-ctypes}
}
@article{YALLOP201882,
title = {A modular foreign function interface},
author = {Jeremy Yallop and David Sheets and Anil Madhavapeddy},
journal = {Science of Computer Programming},
volume = {164},
pages = {82-97},
year = {2018},
url = {https://www.sciencedirect.com/science/article/pii/S0167642317300709},
}
@inproceedings{dolanTestingCrowbar2017,
title = {Testing with {{Crowbar}}},
author = {Dolan, Stephen and Preston, Mindy},
date = {2017},
eventtitle = {{OCaml Users and Developers Workshop}},
}
@online{Hedgehog,
title = {Hedgehog},
url = {https://github.com/hedgehogqa/haskell-hedgehog}
}
@inproceedings{koopmanTestingReactiveSystems2003,
title = {Testing Reactive Systems with {{GAST}}},
booktitle = {Revised {{Selected Papers}} from the {{Fourth Symposium}} on {{Trends}} in {{Functional Programming}} ({{TFP}} 2003)},
author = {Koopman, Pieter W. M. and Plasmeijer, Rinus},
date = {2003},
series = {Trends in {{Functional Programming}}},
volume = {4},
pages = {111--129},
}
@online{ParallelAccessBuffer2022,
title = {Parallel Access to {{Buffer}} Can Trigger Segfaults},
OPTdate = {2022-05-26},
url = {https://github.com/ocaml/ocaml/issues/11279},
OPTorganization = {{OCaml Github repository}}
}
@inproceedings{pottierStrongAutomatedTesting2021,
title = {Strong {{Automated Testing}} of {{OCaml Libraries}}},
author = {Pottier, François},
date = {2021-02},
eventtitle = {Journées {{Francophones}} Des {{Langages Applicatifs}} ({{JFLA}} 2021)},
langid = {english},
}
@online{PropCheck,
title = {{{propCheck}}},
url = {https://github.com/1Jajen1/propCheck}
}
@online{Proper,
title = {Proper},
url = {https://github.com/proper-testing/proper}
}
@InProceedings{PropErTypes@Erlang-11,
author = "Manolis Papadakis and Konstantinos Sagonas",
title = "A {PropEr} Integration of Types and Function Specifications
with Property-Based Testing",
pages = "39--50",
booktitle = "Proceedings of the 2011 ACM SIGPLAN Erlang Workshop",
year = "2011",
}
@online{QCheck,
title = {{{QCheck}}},
url = {https://github.com/c-cube/qcheck}
}
@online{Qcstm,
title = {qcstm},
url = {https://github.com/jmid/qcstm}
}
@InProceedings{Midtgaard:OCaml20,
author = "Jan Midtgaard",
title = "A Simple State-Machine Framework for Property-Based Testing in {OCaml}",
year = 2020,
booktitle = {{OCaml Users and Developers Workshop}},
}
@online{QuviqQuickCheck,
title = {Quviq {{QuickCheck}}},
url = {http://quviq.com/documentation/eqc/index.html}
}
@online{ScalaCheck,
title = {{{ScalaCheck}}},
url = {https://github.com/typelevel/scalacheck}
}
@online{SegfaultMacOSXTrunk2022,
title = {Segfault on {{MacOSX}} with Trunk},
OPTdate = {2022-04-29},
url = {https://github.com/ocaml/ocaml/issues/11226},
OPTorganization = {{OCaml Github repository}}
}
@online{STMCleanup2022,
title = {{{STM}} Clean-Up},
OPTdate = {2022-05-10},
url = {https://github.com/jmid/multicoretests/pull/63},
OPTorganization = {{Multicoretests Github repository}}
}
@inproceedings{osborne:hal-03328646,
TITLE = {{Leveraging Formal Specifications to Generate Fuzzing Suites}},
AUTHOR = {Osborne, Nicolas and Pascutto, Cl{\'e}ment},
URL = {https://hal.inria.fr/hal-03328646},
BOOKTITLE = {{OCaml Users and Developers Workshop}},
YEAR = {2021},
PDF = {https://hal.inria.fr/hal-03328646/file/OCaml_2021.pdf},
HAL_ID = {hal-03328646},
HAL_VERSION = {v1},
}
@inproceedings{padhiyarParafuzzCoverageguidedProperty2021,
title = {Parafuzz: {{Coverage-guided Property Fuzzing}} for {{Multicore OCaml}} Programs},
author = {Padhiyar, Sumit and Kamath, Adharsh and Sivaramakrishnan, KC},
date = {2021},
eventtitle = {{OCaml Users and Developers Workshop}},
}
multicoretests-0.7/doc/paper-latex/macros.tex 0000664 0000000 0000000 00000000125 14743672320 0021451 0 ustar 00root root 0000000 0000000 \providecommand{\tightlist}{%
\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
multicoretests-0.7/doc/paper-latex/paper.tex 0000664 0000000 0000000 00000040416 14743672320 0021303 0 ustar 00root root 0000000 0000000 \documentclass[twocolumn,10pt]{article}
\title{Multicoretests -- Parallel Testing Libraries for OCaml~5.0}
\author{Jan~Midtgaard \and Olivier~Nicole \and Nicolas~Osborne}
\date{Tarides} % Yes, I'm abusing this field
\input{preamble}
\input{macros}
\begin{document}
\maketitle
\section{Introduction}
Parallel and concurrent code is notoriously hard to test because of
the involved non-determinism, yet it is facing OCaml programmers with
the coming OCaml~5.0 multicore release. We present two related testing
libraries to improve upon the situation:
\begin{itemize}
\tightlist
\item \texttt{Lin} -- a library to test for sequential consistency
\item \texttt{STM} -- a state-machine testing library
\end{itemize}
Both libraries build on QCheck~\cite{QCheck}, a black-box, property-based
testing library in the style of
QuickCheck~\cite{claessenQuickCheckLightweightTool2000}.
The two libraries represent different trade-offs between required user effort
and provided guarantees and thereby supplement each other.
In this document we will use OCaml's \texttt{Hashtbl} module as a running
example.
\section{The \texttt{Lin} library}
The \texttt{Lin} library performs a sequence of random operations in
parallel, records the results, and checks whether the observed results
are linearizable by reconciling them with a sequential execution.
%
The library offers an embedded, combinator DSL to describe signatures
succinctly. As an example, the required specification to test (parts
of) the \texttt{Hashtbl} module is given in \cref{code:lin}.
\begin{figure*}[htb!]
\begin{lstlisting}
module HashtblSig =
struct
type t = (char, int) Hashtbl.t
let init () = Hashtbl.create ~random:false 42
let cleanup _ = ()
open Lin
let a,b = char_printable,nat_small
let api =
[ val_ "Hashtbl.clear" Hashtbl.clear (t @-> returning unit);
val_ "Hashtbl.add" Hashtbl.add (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.remove" Hashtbl.remove (t @-> a @-> returning unit);
val_ "Hashtbl.find" Hashtbl.find (t @-> a @-> returning_or_exc b);
val_ "Hashtbl.replace" Hashtbl.replace (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.mem" Hashtbl.mem (t @-> a @-> returning bool);
val_ "Hashtbl.length" Hashtbl.length (t @-> returning int); ]
end
\end{lstlisting}%
\vspace{-5mm}
\caption{Specification of selected \texttt{Hashtbl} functions for testing using
\texttt{Lin}.}\label{code:lin}
\vspace{-.7em}%
\end{figure*}
The first line indicates the type of the system under test (SUT). In the
above case we intend to test \texttt{Hashtbl}s with \texttt{char} keys and \texttt{int}
values. The bindings \texttt{init} and \texttt{cleanup} allow for setting up and
tearing down the SUT. The \texttt{api} then contains a list of type signature
descriptions using combinators in the style of Ctypes~\cite{YALLOP201882}. Different
combinators \texttt{unit}, \texttt{bool}, \texttt{int}, \texttt{list}, \texttt{option}, \texttt{returning},
\lstinline|returning_or_exc|, \dots\@ allow for a concise type signature description.
From the signature description the \texttt{Lin} library will iterate a number of
test instances. Each test instance consists of a ``sequential prefix''
of calls to the specified operations, followed by a \texttt{spawn} of two
parallel \texttt{Domain}s that each call a sequence of operations.
For each test instance \texttt{Lin} chooses the individual operations
arbitrarily and records the result received from each operation. The
framework will then perform a search for a sequential interleaving of
the same calls, and succeed if it finds one. Since \texttt{Hashtbl}s are not
safe for parallelism, the output produces the following:
\begingroup\lstset{language={},basicstyle=\ttfamily\small}
\begin{lstlisting}
Results incompatible with sequential execution
|
Hashtbl.add t '@' 4 : ()
|
.-------------------------.
| |
Hashtbl.add t '.' 3 : () Hashtbl.clear t : ()
Hashtbl.length t : 2
\end{lstlisting}
\endgroup
This describes that in one parallel execution, \texttt{Lin} received the
response \texttt{2} from \texttt{Hashtbl.length}, despite having just executed
\texttt{Hashtbl.clear}. It this case, it is not possible to interleave
\texttt{Hashtbl.add t '.' 3} with these two calls to explain this observed
behaviour.
Underneath the hood, \texttt{Lin} does its best to schedule the two parallel
\texttt{Domain}s on top of each other. It also repeats each test instance, to
increase the chance of triggering an error, and it fails if just one
of the repetitions fail to find a sequential interleaving. Finally,
upon finding an error it reduces the involved operation sequences to a
local minimum, which is what is printed above.
\texttt{Lin} is phrased as an OCaml functor, \lstinline|Lin_domain.Make|. The module
resulting from \lstinline|Lin_domain.Make(HashtblSig)| contains a binding \lstinline|lin_test|
that can perform the above linearization test over \texttt{Domain}s, the
basic unit of parallelism coming in OCaml 5.0. An alternative \texttt{Lin}
mode works over \texttt{Thread} for testing concurrent but non-overlapping
executions. This mode thus mimicks the above functionality by
replacing \texttt{Domain.spawn} and \texttt{Domain.join} with \texttt{Thread.create} and
\texttt{Thread.join}, respectively.
\section{The \texttt{STM} library}
Like \texttt{Lin} the \texttt{STM} library also performs a sequence of random
operations in parallel and records the results. In contrast to \texttt{Lin},
\texttt{STM} then checks whether the observed results are linearizable by
reconciling them with a sequential execution of a \texttt{model} description.
The \texttt{model} expresses the intended meaning of each tested
operation. As such, the required \texttt{STM} user input is longer compared
to that of \texttt{Lin}. The corresponding code to describe a \texttt{Hashtbl}
test using \texttt{STM} is given in \cref{code:stm}.
\begin{figure*}[t]
%\vspace{-10mm}
\hspace*{-5mm}
\hfil
\small
\begin{minipage}[t]{.46\textwidth}
\begin{lstlisting}
module HashtblModel =
struct
type sut = (char, int) Hashtbl.t
type state = (char * int) list
type cmd =
| Clear
| Add of char * int
| Remove of char
| Find of char
| Replace of char * int
| Mem of char
| Length [@@deriving show { with_path = false }]
let init_sut () = Hashtbl.create ~random:false 42
let cleanup (_:sut) = ()
let arb_cmd (s:state) =
let char =
if s = []
then Gen.printable
else Gen.(oneof [oneofl (List.map fst s);
printable]) in
let int = Gen.nat in
QCheck.make ~print:show_cmd
(Gen.oneof
[Gen.return Clear;
Gen.map2 (fun k v -> Add (k,v)) char int;
Gen.map (fun k -> Remove k) char;
Gen.map (fun k -> Find k) char;
Gen.map2 (fun k v -> Replace (k,v)) char int;
Gen.map (fun k -> Mem k) char;
Gen.return Length;
])
\end{lstlisting}
\end{minipage}
\hfil
\begin{minipage}[t]{.49\textwidth}
\begin{lstlisting}
let next_state (c:cmd) (s:state) = match c with
| Clear -> []
| Add (k,v) -> (k,v)::s
| Remove k -> List.remove_assoc k s
| Find _ -> s
| Replace (k,v) -> (k,v)::(List.remove_assoc k s)
| Mem _
| Length -> s
let run (c:cmd) (h:sut) = match c with
| Clear -> Res (unit, Hashtbl.clear h)
| Add (k,v) -> Res (unit, Hashtbl.add h k v)
| Remove k -> Res (unit, Hashtbl.remove h k)
| Find k -> Res (result int exn,
protect (Hashtbl.find h) k)
| Replace (k,v) -> Res (unit, Hashtbl.replace h k v)
| Mem k -> Res (bool, Hashtbl.mem h k)
| Length -> Res (int, Hashtbl.length h)
let init_state = []
let precond (_:cmd) (_:state) = true
let postcond (c:cmd) (s:state) (res:res) =
match c,res with
| Clear, Res ((Unit,_),_)
| Add (_,_), Res ((Unit,_),_)
| Remove _, Res ((Unit,_),_) -> true
| Find k, Res ((Result (Int,Exn),_),r) ->
r = (try Ok (List.assoc k s)
with Not_found -> Error Not_found)
| Replace (_,_), Res ((Unit,_),_) -> true
| Mem k, Res ((Bool,_),r) -> r = List.mem_assoc k s
| Length, Res ((Int,_),r) -> r = List.length s
| _ -> false
end
\end{lstlisting}
\end{minipage}
\hfil%
\vspace{-3mm}
\caption{Description of a \texttt{Hashtbl} test using \texttt{STM}.}%
\label{code:stm}
\vspace{-1.25em}%
\end{figure*}
Again this requires a description of the system under test, \texttt{sut}. In
addition \texttt{STM} requires a type \texttt{cmd} for describing the tested
operations. The hooks \lstinline|init_sut| and \texttt{cleanup} match \texttt{init} and
\texttt{cleanup} from \texttt{Lin}, respectively.
A distinguishing feature is \texttt{type state = (char * int) list}
describing with a pure association list the internal state of a
hashtable. \lstinline|next_state| is a simple state transition function
describing how the \texttt{state} changes across each \texttt{cmd}. For example,
\texttt{Add (k,v)} appends the key-value pair onto the association list.
\lstinline|arb_cmd| is a generator of \texttt{cmd}s, taking \texttt{state} as a parameter.
This allows for \texttt{state}-dependent \texttt{cmd} generation, which we use
to increase the chance of producing a \texttt{Remove 'c'}, \texttt{Find 'c'}, \dots\@
following an \texttt{Add 'c'}. Internally \lstinline|arb_cmd| uses combinators
\texttt{Gen.return}, \texttt{Gen.map}, and \texttt{Gen.map2} from QCheck to generate one of
7 different operations. For example, \texttt{Gen.map (fun k -> Mem k) char}
creates a \texttt{Mem} command with the result obtained from the \texttt{char}
generator. \lstinline|arb_cmd| further uses a derived printer \lstinline|show_cmd| to
be able to print counterexamples.
\texttt{run} executes the tested \texttt{cmd} over the SUT and wraps the result up
in a result type \texttt{res} offered by \texttt{STM}. Combinators \texttt{unit}, \texttt{bool},
\texttt{int}, \dots~allow to annotate the result with the expected type.
\texttt{postcond} then expresses a post-condition by matching the received
\texttt{res}, for a given \texttt{cmd} with the corresponding answer from the
\texttt{model} description. For example, this compares the Boolean result \texttt{r}
from \texttt{Hashtbl.mem} with the result from \lstinline|List.mem_assoc|. Similarly
\texttt{precond} expresses a \texttt{cmd} pre-condition.
\texttt{STM} is also phrased as an OCaml functor. The module resulting from
\texttt{STM\_domain.Make(HashtblModel)} thus includes a binding
\lstinline|agree_test| for running sequential tests comparing the SUT
behaviour to the given model.
Another binding
\lstinline|agree_test_par| instead runs parallel tests that make a similar
comparison over a sequential prefix and two parallel \texttt{Domain}s, this
time also searching for a sequential interleaving of \texttt{cmd}s.
For example, one execution of \lstinline|agree_test_par| produced the following
output. Note how no interleaving of \texttt{Remove} from the first parallel
\texttt{cmd} sequence can make the association list model return \lstinline|-1| from
\texttt{Length}:
\begingroup\lstset{language={}}
\begin{lstlisting}
Results incompatible with linearized model
|
(Add ('1', 5)) : ()
|
.-----------------------.
| |
(Remove '1') : () Clear : ()
Length : -1
\end{lstlisting}
\endgroup
\vspace{-1.6em}
\section{Status}
Both libraries are open source and available for download on GitHub
from \url{https://github.com/jmid/multicoretests}.
As the APIs are still unstable and under development, we have not made
a public release yet. Interested users can nevertheless easily install
the libraries with \texttt{opam}.
During development we have used examples such as \texttt{Hashtbl} to
confirm that the approach indeed works as intended. The behaviour is
continuously confirmed by running GitHub Actions of the latest trunk
compiler. As further testament to the usability of the approach, we
have used the libraries to test parts of OCaml's \texttt{Stdlib}, as well as
the \texttt{Domainslib} and \texttt{lockfree} libraries. In doing so, we have been
able to find and report a number of issues which have either already
been fixed or have fixes underway:
\begin{itemize}
\tightlist
\item \lstinline|In_channel| and \lstinline|Out_channel|
unsafety~\cite{AddFailingOut2022,AuditStdlibMutable2022}
\item MacOSX crash~\cite{SegfaultMacOSXTrunk2022}
\item \texttt{Buffer} unsafety~\cite{STMCleanup2022,ParallelAccessBuffer2022}
\end{itemize}
\vspace{-.8em}
\section{Related Work}
QuickCheck~\cite{claessenQuickCheckLightweightTool2000} originally introduced property-based
testing within functional programming with combinator-based
generators, properties, and test-case reduction. It has since been
ported to over 30 other programming languages, including Quviq
QuickCheck~\cite{QuviqQuickCheck}---a commercial port to Erlang.
Model-based testing was initially suggested as a method for testing
monadic code with Haskell's QuickCheck~\cite{claessenTestingMonadicCode2002}. An explicit
framework was later proposed in the GAST property-based testing
library for Clean~\cite{koopmanTestingReactiveSystems2003}. The commercial Quviq
QuickCheck~\cite{QuviqQuickCheck}
was later extended with a state-machine model framework for testing stateful
systems~\cite{artsTestingTelecomsSoftware2006}.
This approach was extended further to test parallel code for data
races~\cite{claessenFindingRaceConditions2009}.
This general approach for parallel testing has since been adopted in
other ports, such as Erlang's open source Proper~\cite{PropErTypes@Erlang-11}, Haskell
Hedgehog~\cite{Hedgehog}, ScalaCheck~\cite{ScalaCheck}, and Kotlin's
propCheck~\cite{PropCheck}. \texttt{STM} continues this adoption tradition.
qcstm~\cite{Midtgaard:OCaml20} is a previous OCaml adoption, also building on QCheck.
It was missing the ability to perform parallel testing though.
\texttt{STM} seeks to remedy this limitation.
Crowbar~\cite{dolanTestingCrowbar2017} is another QuickCheck-style
testing framework with combinator-based generators. In contrast to
QuickCheck, it utilizes AFL-based coverage guidance to effectively
guide the generated input towards unvisited parts of the SUT. Crowbar
does not come with a state-machine
framework. Monolith~\cite{pottierStrongAutomatedTesting2021} is a
model-based testing framework also building on AFL-based coverage
guidance. In contrast to \texttt{STM}, Monolith's models are oracle
implementations with operations matching the type signatures of the
tested operations. Neither Crowbar nor Monolith come with skeletons to
perform parallel or concurrent testing. Furthermore the AFL-based
coverage-guidance underlying both Crowbar and Monolith works best for
deterministic, sequential code.
ParaFuzz~\cite{padhiyarParafuzzCoverageguidedProperty2021} is another approach to fuzz test multicore OCaml programs.
It simulates parallelism in OCaml through concurrency, enabling scheduling order to be controlled by AFL,
which helps to trigger and find scheduling-dependent bugs. A caveat is that ParaFuzz assumes
data race freedom.
Ortac can extract Monolith-based tests from a formal specification
written in Gospel, a specification language for OCaml~\cite{osborne:hal-03328646}.
Gospel specifications include models, pre-conditions, and
post-conditions close to those of \texttt{STM}. The extracted tests
however inherit Monolith's and AFL's focus on sequential code.
ArtiCheck~\cite{braibantWelltypedSmartFuzzing2014} tests random combinations of OCaml calls from
type signature descriptions, similarly to \texttt{Lin}. Whereas \texttt{Lin} and
\texttt{STM} target impure interfaces, ArtiCheck targets persistent
(pure) interfaces. ArtiCheck furthermore targets sequential rather
than parallel or concurrent tests.
\section{Conclusion}
We have presented two libraries, \texttt{Lin} and \texttt{STM} for testing parallel
and concurrent code for OCaml 5.0. Despite still being under
development, we believe both libraries could be helpful to developers
of OCaml~5.0 programs.
\printbibliography
\end{document}
multicoretests-0.7/doc/paper-latex/preamble.tex 0000664 0000000 0000000 00000002763 14743672320 0021766 0 ustar 00root root 0000000 0000000 \usepackage{fontspec}
\setmainfont{Libertinus Serif}
\setmonofont[Scale=0.8]{Fira Mono}
%\setmonofont[Scale=0.8]{FiraMono-Regular.otf}
\frenchspacing
\usepackage{xcolor}
\colorlet{darkred}{red!70!black}
\colorlet{darkgray}{gray!70!black}
\usepackage[final]{listings}
\lstset{
basicstyle=\ttfamily,
keywordstyle=\color{darkred},
commentstyle=\color{darkgray},
columns=fullflexible,
captionpos=b, % sets the caption-position to bottom
escapeinside={\%*}{*)}, % if you want to add LaTeX within your code
% frame=single, % adds a frame around the code
keepspaces=true, % keeps spaces in text, useful for keeping indentation of code (possibly needs columns=flexible)
language=[Objective]Caml, % the language of the code
% numbers=left, % where to put the line-numbers; possible values are (none, left, right)
% numbersep=5pt, % how far the line-numbers are from the code
numberstyle=\scriptsize\color{darkgray}, % the style that is used for the line-numbers
mathescape=true,
% rulecolor=\color{black}, % if not set, the frame-color may be changed on line-breaks within not-black text (e.g. comments (green here))
}
\usepackage[
backend=biber,
maxbibnames=3,
style=numeric]{biblatex}
\bibliography{biblio.bib}
\usepackage{graphicx}
\usepackage{hyperref}
\hypersetup{hidelinks = true}
\usepackage{cleveref}
\usepackage[top=2cm,left=2cm,right=2cm,bottom=20mm]{geometry}
multicoretests-0.7/doc/paper.md 0000664 0000000 0000000 00000042634 14743672320 0016665 0 ustar 00root root 0000000 0000000 Multicoretests - Parallel Testing Libraries for OCaml 5.0
=========================================================
**Jan Midtgaard, Olivier Nicole, and Nicolas Osborne**, *Tarides*
Introduction
------------
Parallel and concurrent code is notoriously hard to test because of
the involved non-determinism, yet it is facing OCaml programmers with
the coming OCaml 5.0 multicore release. We present two related testing
libraries to improve upon the situation:
- `Lin` - a library to test for sequential consistency
- `STM` - a state-machine testing library
Both libraries build on [QCheck][qcheck], a black-box, property-based
testing library in the style of [QuickCheck](#QuickCheck). The two
libraries represent different trade-offs between required user effort
and provided guarantees and thereby supplement each other.
In this document we will use OCaml's `Hashtbl` module as a running
example.
The `Lin` library
-----------------
The `Lin` library performs a sequence of random operations in
parallel, records the results, and checks whether the observed results
are linearizable by reconciling them with a sequential execution.
The library offers an embedded, combinator DSL to describe signatures
succinctly. As an example, the required specification to test (parts
of) the `Hashtbl` module is as follows:
```ocaml
module HashtblSig =
struct
type t = (char, int) Hashtbl.t
let init () = Hashtbl.create ~random:false 42
let cleanup _ = ()
open Lin
let a,b = char_printable,nat_small
let api =
[ val_ "Hashtbl.clear" Hashtbl.clear (t @-> returning unit);
val_ "Hashtbl.add" Hashtbl.add (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.remove" Hashtbl.remove (t @-> a @-> returning unit);
val_ "Hashtbl.find" Hashtbl.find (t @-> a @-> returning_or_exc b);
val_ "Hashtbl.replace" Hashtbl.replace (t @-> a @-> b @-> returning unit);
val_ "Hashtbl.mem" Hashtbl.mem (t @-> a @-> returning bool);
val_ "Hashtbl.length" Hashtbl.length (t @-> returning int); ]
end
```
The first line indicates the type of the system under test (SUT). In the
above case we intend to test `Hashtbl`s with `char` keys and `int`
values. The bindings `init` and `cleanup` allow for setting up and
tearing down the SUT. The `api` then contains a list of type signature
descriptions using combinators in the style of [Ctypes][ctypes]. Different
combinators `unit`, `bool`, `int`, `list`, `option`, `returning`,
`returning_or_exc`, ... allow for a concise type signature description.
From the above description the `Lin` library will iterate a number of
test instances. Each test instance consists of a "sequential prefix"
of calls to the above operations, followed by a `spawn` of two
parallel `Domain`s that each call a sequence of operations.
For each test instance `Lin` chooses the individual operations
arbitrarily and records the result received from each operation. The
framework will then perform a search for a sequential interleaving of
the same calls, and succeed if it finds one. Since `Hashtbl`s are not
safe for parallelism, the output produces the following:
```
Results incompatible with sequential execution
|
Hashtbl.add t '@' 4 : ()
|
.------------------------------------.
| |
Hashtbl.add t '.' 3 : () Hashtbl.clear t : ()
Hashtbl.length t : 2
```
This describes that in one parallel execution, `Lin` received the
response `2` from `Hashtbl.length`, despite having just executed
`Hashtbl.clear`. It this case, it is not possible to interleave
`Hashtbl.add t '.' 3` with these two calls to explain this observed
behaviour.
Underneath the hood, `Lin` does its best to schedule the two parallel
`Domain`s on top of each other. It also repeats each test instance, to
increase the chance of triggering an error, and it fails if just one
of the repetitions fail to find a sequential interleaving. Finally,
upon finding an error it reduces the involved operation sequences to a
local minimum, which is what is printed above.
`Lin` is phrased as an OCaml functor, `Lin_domain.Make`. The module
resulting from `Lin_domain.Make(HashtblSig)` contains a binding `lin_test`
that can perform the above linearization test over `Domain`s, the
basic unit of parallelism coming in OCaml 5.0. An alternative `Lin`
mode works over `Thread` for testing concurrent but non-overlapping
executions. This mode thus mimicks the above functionality by
replacing `Domain.spawn` and `Domain.join` with `Thread.create` and
`Thread.join`, respectively.
The `STM` library
-----------------
Like `Lin` the `STM` library also performs a sequence of random
operations in parallel and records the results. In contrast to `Lin`,
`STM` then checks whether the observed results are linearizable by
reconciling them with a sequential execution of a `model` description.
The `model` expresses the intended meaning of each tested
operation. As such, the required `STM` user input is longer compared
to that of `Lin`. The corresponding code to describe a `Hashtbl`
test using `STM` is given below:
```ocaml
module HashtblModel =
struct
type sut = (char, int) Hashtbl.t
type state = (char * int) list
type cmd =
| Clear
| Add of char * int
| Remove of char
| Find of char
| Replace of char * int
| Mem of char
| Length [@@deriving show { with_path = false }]
let init_sut () = Hashtbl.create ~random:false 42
let cleanup (_:sut) = ()
let arb_cmd (s:state) =
let char =
if s = []
then Gen.printable
else Gen.(oneof [oneofl (List.map fst s);
printable]) in
let int = Gen.nat in
QCheck.make ~print:show_cmd
(Gen.oneof
[Gen.return Clear;
Gen.map2 (fun k v -> Add (k,v)) char int;
Gen.map (fun k -> Remove k) char;
Gen.map (fun k -> Find k) char;
Gen.map2 (fun k v -> Replace (k,v)) char int;
Gen.map (fun k -> Mem k) char;
Gen.return Length;
])
let next_state (c:cmd) (s:state) = match c with
| Clear -> []
| Add (k,v) -> (k,v)::s
| Remove k -> List.remove_assoc k s
| Find _ -> s
| Replace (k,v) -> (k,v)::(List.remove_assoc k s)
| Mem _
| Length -> s
let run (c:cmd) (h:sut) = match c with
| Clear -> Res (unit, Hashtbl.clear h)
| Add (k,v) -> Res (unit, Hashtbl.add h k v)
| Remove k -> Res (unit, Hashtbl.remove h k)
| Find k -> Res (result int exn,
protect (Hashtbl.find h) k)
| Replace (k,v) -> Res (unit, Hashtbl.replace h k v)
| Mem k -> Res (bool, Hashtbl.mem h k)
| Length -> Res (int, Hashtbl.length h)
let init_state = []
let precond (_:cmd) (_:state) = true
let postcond (c:cmd) (s:state) (res:res) =
match c,res with
| Clear, Res ((Unit,_),_)
| Add (_,_), Res ((Unit,_),_)
| Remove _, Res ((Unit,_),_) -> true
| Find k, Res ((Result (Int,Exn),_),r) ->
r = (try Ok (List.assoc k s)
with Not_found -> Error Not_found)
| Replace (_,_), Res ((Unit,_),_) -> true
| Mem k, Res ((Bool,_),r) -> r = List.mem_assoc k s
| Length, Res ((Int,_),r) -> r = List.length s
| _ -> false
```
Again this requires a description of the system under test, `sut`. In
addition `STM` requires a type `cmd` for describing the tested
operations. The hooks `init_sut` and `cleanup` match `init` and
`cleanup` from `Lin`, respectively.
A distinguishing feature is `type state = (char * int) list`
describing with a pure association list the internal state of a
`Hashtbl`. `next_state` is a simple state transition function
describing how the `state` changes across each `cmd`. For example,
`Add (k,v)` appends the key-value pair onto the association list.
`arb_cmd` is a generator of `cmd`s, taking `state` as a parameter.
This allows for `state`-dependent `cmd` generation, which we use
to increase the chance of producing a `Remove 'c'`, `Find 'c'`, ...
following an `Add 'c'`. Internally `arb_cmd` uses combinators
`Gen.return`, `Gen.map`, and `Gen.map2` from QCheck to generate one of
7 different operations. For example, `Gen.map (fun k -> Mem k) char`
creates a `Mem` command with the result obtained from the `char`
generator. `arb_cmd` further uses a derived printer `show_cmd` to
be able to print a counterexample.
`run` executes the tested `cmd` over the SUT and wraps the result up
in a result type `res` offered by `STM`. Combinators `unit`, `bool`,
`int`, ... allow to annotate the result with the expected type.
`postcond` then expresses a post-condition by matching the received
`res`, for a given `cmd` with the corresponding answer from the
`model` description. For example, this compares the Boolean result `r`
from `Hashtbl.mem` with the result from `List.mem_assoc`. Similarly
`precond` expresses a `cmd` pre-condition.
`STM` is also phrased as an OCaml functor. The module resulting from
`STM_domain.Make(HashtblModel)` thus includes a binding
`agree_test` for running sequential tests comparing the SUT
behaviour to the given model.
Another binding `agree_test_par` instead runs parallel tests that make a similar
comparison over a sequential prefix and two parallel `Domain`s, this
time also searching for a sequential interleaving of `cmd`s.
For example, one execution of `agree_test_par` produced the following
output. Note how no interleaving of `Remove` from the first parallel
`cmd` sequence can make the association list model return `-1` from
`Length`:
```
Results incompatible with linearized model
|
(Add ('1', 5)) : ()
|
.-----------------------.
| |
(Remove '1') : () Clear : ()
Length : -1
```
Status
------
Both libraries are open source and available for download on GitHub
from https://github.com/jmid/multicoretests
As the APIs are still unstable and under development, we have not made
a public release yet. Interested users can nevertheless easily install
the libraries with `opam`.
During development we have used examples such as `Hashtbl` to
confirm that the approach indeed works as intended. The behaviour is
continuously confirmed by running GitHub Actions of the latest trunk
compiler. As further testament to the usability of the approach, we
have used the libraries to test parts of OCaml's `Stdlib`, as well as
the `Domainslib` and `lockfree` libraries. In doing so, we have been
able to find and report a number of issues which have either already
been fixed or have fixes underway:
- `In_channel` and `Out_channel` unsafety https://github.com/jmid/multicoretests/pull/13 https://github.com/ocaml/ocaml/issues/10960#issuecomment-1087660763
- MacOSX crash - https://github.com/ocaml/ocaml/issues/11226
- `Buffer` unsafety - https://github.com/jmid/multicoretests/pull/63 https://github.com/ocaml/ocaml/issues/11279
Related work
------------
- [QuickCheck](#QuickCheck) originally introduced property-based
testing within functional programming with combinator-based
generators, properties, and test-case reduction. It has since been
ported to over 30 other programming languages, including [Quviq
QuickCheck][Quviq QuickCheck] - a commercial port to Erlang.
- Model-based testing was initially suggested as a method for [testing
monadic code with Haskell's QuickCheck](#Haskell-model). An explicit
framework was later proposed in [the GAST property-based testing
library for Clean](#Gast-Clean). The commercial [Quviq QuickCheck][Quviq QuickCheck]
was later extended with a [state-machine model framework for testing stateful systems](#Erlang-eqc_commands).
This approach was extended further to [test parallel code for data races](#Erlang-eqc_par_statem).
This general approach for parallel testing has since been adopted in
other ports, such as Erlang's open source [Proper][Proper], Haskell
[Hedgehog][hedgehog], [ScalaCheck][scalacheck], and Kotlin's
[propCheck][propcheck]. `STM` continues this adoption tradition.
[qcstm][qcstm] is a previous OCaml adoption, also building on QCheck.
It was missing the ability to perform parallel testing though.
`STM` seeks to remedy this limitation.
- [Crowbar][crowbar] is another QuickCheck-style
testing framework with combinator-based generators. In contrast to
QuickCheck, it utilizes AFL-based coverage guidance to effectively
guide the generated input towards unvisited parts of the SUT. Crowbar
does not come with a state-machine framework.
[Monolith](#Monolith) is a
model-based testing framework also building on AFL-based coverage
guidance. In contrast to `STM`, Monolith's models are oracle
implementations with operations matching the type signatures of the
tested operations. Neither Crowbar nor Monolith come with skeletons to
perform parallel or concurrent testing. Furthermore the AFL-based
coverage-guidance underlying both Crowbar and Monolith works best for
deterministic, sequential code.
- [ParaFuzz][parafuzz] is another approach to fuzz test multicore OCaml programs.
It simulates parallelism in OCaml through concurrency, enabling
scheduling order to be controlled by AFL, which helps to trigger and
find scheduling-dependent bugs. A caveat is that ParaFuzz assumes data race freedom.
- [Ortac][ortac] can extract Monolith-based tests from a formal specification
written in Gospel, a specification language for OCaml.
Gospel specifications include models, pre-conditions, and
post-conditions close to those of `STM`. The extracted tests
however inherit Monolith's and AFL's focus on sequential code.
- [ArtiCheck][articheck] tests random combinations of OCaml calls from
type signature descriptions, similarly to `Lin`. Whereas `Lin` and
`STM` target impure interfaces, ArtiCheck targets both persistent
(pure) interfaces. ArtiCheck furthermore targets sequential rather
than parallel or concurrent tests.
Conclusion
----------
We have presented two libraries, `Lin` and `STM` for testing parallel
and concurrent code for OCaml 5.0. Despite still being under
development, we believe both libraries could be helpful to developers
of OCaml 5.0 programs.
References
----------
[qcheck]: https://github.com/c-cube/qcheck
##### QuickCheck
Koen Claessen and John Hughes, QuickCheck: A Lightweight Tool for Random Testing of Haskell Programs, ICFP 2000
https://www.cs.tufts.edu/~nr/cs257/archive/john-hughes/quick.pdf
##### ctypes
Jeremy Yallop, David Sheets, and Anil Madhavapeddy, A modular foreign function interface, Science of Computer Programming, vol.164, 2018
https://anil.recoil.org/papers/2018-socp-modular-ffi.pdf
https://github.com/ocamllabs/ocaml-ctypes
[Quviq QuickCheck]: http://quviq.com/documentation/eqc/index.html
##### Haskell-model
Koen Claessen and John Hughes, Testing Monadic Code with QuickCheck, Haskell 2002
https://www.cs.tufts.edu/~nr/cs257/archive/koen-claessen/quickmonad.ps
##### Gast-Clean
Pieter Koopman and Rinus Plasmeijer, Testing reactive systems with GAST, TFP 2003,
https://www.mbsd.cs.ru.nl/publications/papers/2004/koop2004-TestReactiveGAST.pdf
http://www.cs.ru.nl/~pieter/gentest/gentest.html
##### Erlang-eqc_commands
Thomas Arts, John Hughes, Joakim Johansson, and Ulf Wiger, Testing Telecoms Software with Quviq QuickCheck, Erlang 2006,
http://www.quviq.com/wp-content/uploads/2014/08/erlang001-arts.pdf
##### Erlang-eqc_par_statem
Koen Claessen et al., Finding Race Conditions in Erlang with QuickCheck and PULSE, ICFP 2009
https://smallbone.se/papers/finding-race-conditions.pdf
##### Proper
Manolis Papadakis and Konstantinos Sagonas, A PropEr Integration of Types and Function Specifications with Property-Based Testing, Erlang 2011
https://proper-testing.github.io/papers/proper_types.pdf
https://github.com/proper-testing/proper
[hedgehog]: https://github.com/hedgehogqa/haskell-hedgehog
[scalacheck]: https://github.com/typelevel/scalacheck
[propcheck]: https://github.com/1Jajen1/propCheck
##### qcstm
Jan Midtgaard, A Simple State-Machine Framework for Property-Based Testing in OCaml, OCaml Users and Developers Workshop 2020
https://janmidtgaard.dk/papers/Midtgaard%3AOCaml20.pdf
https://github.com/jmid/qcstm
##### crowbar
Stephen Dolan and Mindy Preston, Testing with Crowbar, OCaml Users and Developers Workshop 2017
https://67.207.157.55/meetings/ocaml/2017/extended-abstract__2017__stephen-dolan_mindy-preston__testing-with-crowbar.pdf
https://github.com/stedolan/crowbar
##### Mmonolith
François Pottier, Strong Automated Testing of OCaml Libraries, JPFL 2021,
http://cambium.inria.fr/~fpottier/publis/pottier-monolith-2021.pdf
https://gitlab.inria.fr/fpottier/monolith
##### parafuzz
Sumit Padhiyar, Adharsh Kamath, and KC Sivaramakrishnan, Parafuzz: Coverage-guided Property Fuzzing for Multicore OCaml Programs, OCaml Users and Developers Workshop 2021
https://github.com/ocaml-multicore/parafuzz
##### ortac
Nicolas Osborne and Clément Pascutto, Leveraging Formal Specifications to Generate Fuzzing Suites, OCaml Users and Developers Workshop 2021
https://hal.inria.fr/hal-03328646/file/OCaml_2021.pdf
https://github.com/ocaml-gospel/ortac
##### articheck
Thomas Braibant, Jonathan Protzenko, and Gabriel Scherer, Well-Typed Generic Smart Fuzzing for APIs, ML Familiy Workshop 2014
https://hal.inria.fr/hal-01094006/document
https://github.com/braibant/articheck
multicoretests-0.7/doc/paper.pdf 0000664 0000000 0000000 00000155177 14743672320 0017045 0 ustar 00root root 0000000 0000000 %PDF-1.5
%äđíř
16 0 obj
<>
stream
xÚµ\Io,1nľçWř´˘}Čm€wrhŰí9ĺ0˙˙-$E-U]í™ŕ˝F»«´R—Ź”Ţţń¦Ţdţ§Ţ‚Î˙ĺŰ×˙ć_˙™?úýŹ·żüyű÷˙Po*ś}űóófĽP>˝Ý´Îř·?ß˙ý.Ą~H鼔ö;»ü}Ď›˙–ůc>b.b=Ľ*ß¶ýíěÇMŰô.ý·Ď9™Ë•ËčR}—&ë§6_J©RĘ}xý>¶V;ýÉźJý
OMoŻ~ß±&öj
Ç‘icx`ßĺŤĘ5eîG©Ź˙ůó_Ť>J$çtˇŹMB†Lž(bŇHžOhçŰ(«Łp0ö%ę?B’«Í+™„
e
[[e\ć囍\˙ŚĐ¨Ť‘‚§Ć¶E5'Lé®ŘććUyШ”nޢڻ{L20ʱ4É1xbĎĹx#ÜvęLČÓÖĐÄUpČ,üŚ,6ń# ™ĆxjŘZ߆¸ÓE "=d› …—٧u:ëN©VÂ^Đ˝6„„Śüą}/oiVÚTmś Ś@čkÜžŘΉ»łĽ–ź«`%-ŢEÜÍčł“V(”¦ţáëš·˘7¤Ş
Nx§şMź}Ö‰ŻUyM@ĄĂt˝Őô×–öy~®íYFBxF¦.®U9ĘĄş$ «:{AÇš«x.ş‰ô¶÷H"ýţą3!”
ż#7Ż›ukţř˘żGnš‹îs’(PŘ%¶Iüžć“Ŕą´”ŐbČVdČŞ=s‘0-mş^+ĎHsy[VMo`mďPËa¸˝Z9ŁA$¦IŹ}ayóŐ×fłžJ%áëđó#km„°NÝű¨â#a}˝j”1¤aĘ&ş—†ÚZ(k…6v’NÖ"ś¤©ÖáhiG·bw|˝Y|ő=ą
w2dń
صf*µhíRGkÜę?[ă¨E´™*ÚF‘ża‰ý“JAئJuu¨ßuŽb«k˘Ą!.krĹźićS¦jôÁ&ÝÚĘ۶†I‘71ۨZ“FÓ…IŽüg܆ ߏ˘‘S$fň¸ňĂ€AęľśÓ«Ć„úťŰśµĐNÇćWmXy?W–q˛×ől„ţćnëľµ™.6˝E´?ş5Sj¤}źÄ#Í
ě‘j˝;tTű…őV¨uâąßń]ČRĎ.5/ülr•ě%˛÷©öéťxôYČ1/c¬>!cm=łu›oEÝ0§˘U\٬ŕľČˇU§r!ëL÷&¤T|¦Ë,tícŁHŤÉ¤·«_í<‰śĐĺ>Bpźa›ěG†™'5âÖOwY&eęIľă”;|°5ăiď:›Htť™ÔP
|7¶•
·/a=2˙«„öÔ9±8-ĽkuP‹A=Ă…şuĽě2Í™,VŃť0Ž›ÉFč˘QÜ)¶ŃŚ
ě* g·Đ<(ËTÜT»K3t2ËďŢ
‘đd«Ĺ±;‡Â#14 ‹µí€ŇÝú>ĚÚ承»Î‡‹{F6Đßܬu¸CśĐ0©ŇšB…Đ6Ď~-»îS7›áÔ)|Og FOÝ]÷źö:¸)Ó4ŤŘŻś[¤şb•Ôřŕ©AŮ–Zr+–#w†Ŕ¶ćP9u3…!onü]řS7ŘbŤ5Çă°´.~š-@ôž‹@aŞĐ!? ·e–ˇ
‘Á6ć˝űi«•
ž™ÖNHÚď
.t€š6"?T5Î,ŤhżNµSC}ľ±đ¸ă«6ßÖŽ±Ü»aâˇX˛ßśŘ¬°uä`‚
;0ˇ>•i”‘´Á[ůý#ďYyZ9ÔUĆŠ mWV/#ŢtE‡†HŘ"ŠcĆđ÷x`†=/ŠjB…ě’WóŹqjŁfÜZŇgcó`őĄ?«c…óq®ýÜr1Ą'S”‹Sś‡::gúÚ™ć€|Ź <˝wőJ΢ćëŰę˘#Ú•0˝5ŢË
#l#›A¤•÷lL ŚXK#"CóśK’§EĐŇ#z™ň›9:fÁ·_Ăn¬Ošâ™p…Ć«h8ÜoyUfΫh9¶Ďń2Çű®b/Úý§măťpŇ˙rŰlĘv›ŻJ˛Öí¨ôÖnu¶Uď•řĹ0ocŕ}ťPi±öżş„iŞPe–ÎÚąI"HfŞ~btggÖa]·Ń]č¦jęjcý`ĎĂRú}ŇdśsŕZĹě°>q|ő{7ĆlĐkczůłą«ÜvJ#áA„°Řŕz·Ü^¸8v-şô\ł<"ĄŇĽX;®f˛Ýčň†Ářüŕâ·÷n5ôĆŔo&?¸Če?K V”öx`Ađj5ŐżÎ]ĄĽh†/‚i:×~î}Ą"e$łTČp¤î¸á‰h˘–Nčä7–N%Í˝»(Ú“™8pL¸ľ'/Č o®DĄŃxbjU„eăMAžu+äe«öŇTEpEůyŐŐ!Ş´ţŇvG!4.€ž!ë`•%“ŮV„kç‘+{Z1Z‘˘ž+đŮFwc¬Ćü‹Ôą<%ˇ¤á©dRI(ÝĄŤüŢ´Y &Ś-âGMĆó&oÚf´˘î§‚ľB-çí:ť¬ÄÚ¤;›}|ĄÉ:s´Ô?á÷mŃQřd^jÚ‚ö°n–ťdjz0Ť&=ďĆľĐMţŔúÖ˛
ĘßIať%®Śń€$˘)ű"Ú…>Ćl¤…,Q.WU幪ĘU™SUeÓMÝ „Ł:u$GÝ+r´uů‚=—!*„xI׆§‚yix›wţ[ĘƤą6ʨ^ Ýŕŕ ݧ4–倰xoĹVV‹
úĂŔxJŕŢŮč‘ŐöܨgÁăhv#¤ý†ř'Z¦R'Aů®áĆ™4Ë—0řO7ćŔVYŕ§ąfÉUiľgĘ[ą¦%p_°Gʉ6·Ą†(™Á,9†ˇáÝ €Šj\Ňj#,ůo?ń9JŔ¤—ť
§m“}™lpSŇČYŁ˝ňČvĽdZIę嚤]zŠdo†ĺ•6ôͶĘwćdcOzY˛CÖŽFűU®á8|:pÚű•[˝Ľ@hű˝$Rh732’˦-W@ŢĐg<Ô t˛+˛™^{:os({ľÍ7¶ś*†‹ň6Đł^(j€Á:'™/W|p»@<ĂŃć!`ö–yl8JĐJÄvÎĐäaćG –î5ú!łł×@ł=ą¨1aN4›jĎŤb¨¤}óÖOŘ"Ë˝”’ĹŇôâb¬zĘ`ŮqnşŞ)ĹáŤö¤őϢƞK8
ć}•ňa§…}ÜřŔđô4<-ÖĐ ŹPŕÁ!Ř;P”í1ţRů·
¬˘Ö38çĚ”´&WĹAŻĆĹGĄg§/öµh?Đ