pax_global_header00006660000000000000000000000064151460736770014532gustar00rootroot0000000000000052 comment=e4dcda0551f59e3a9e397aaabd620d25b71fc95a apache-buildstream-27ae392/000077500000000000000000000000001514607367700156225ustar00rootroot00000000000000apache-buildstream-27ae392/.asf.yaml000066400000000000000000000025541514607367700173430ustar00rootroot00000000000000# # Avoid diluting the dev mailing list with meta stuff, # redirect it all to the commit list. # notifications: commits: commits@buildstream.apache.org issues: commits@buildstream.apache.org pullrequests: commits@buildstream.apache.org # # Configure github # github: description: "BuildStream, the software integration tool" homepage: https://buildstream.build/ # Main features features: # Enable wiki for documentation wiki: true # Enable issue management issues: true # Enable projects for project management boards projects: true # Buttons enabled_merge_buttons: # Disable squash button: squash: false # enable merge button: merge: true # disable rebase button: rebase: false # Close branches when pull requests are merged del_branch_on_merge: true # Enable pages publishing ghp_branch: gh-pages ghp_path: / # Testing basic protection of multiple branches, # this needs to be tested on the master branch. # protected_branches: # Test 1 # tristan/test-protected-branch-1: required_status_checks: # strict means "Require branches to be up to date before merging". strict: true # Test 2 # tristan/test-protected-branch-2: required_status_checks: # strict means "Require branches to be up to date before merging". strict: true apache-buildstream-27ae392/.coveragerc000066400000000000000000000020031514607367700177360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # [run] concurrency = multiprocessing plugins = Cython.Coverage omit = # Omit some internals */buildstream/_profile.py */buildstream/__main__.py */buildstream/_version.py # Omit generated code */buildstream/_protos/* */.eggs/* # Omit .tox directory */.tox/* # Omit a dynamically generated Cython file */stringsource [report] show_missing = True precision = 2 [paths] source = src/buildstream/ buildstream/ */site-packages/buildstream/ apache-buildstream-27ae392/.gitattributes000066400000000000000000000000511514607367700205110ustar00rootroot00000000000000src/buildstream/_version.py export-subst apache-buildstream-27ae392/.github/000077500000000000000000000000001514607367700171625ustar00rootroot00000000000000apache-buildstream-27ae392/.github/CODEOWNERS000066400000000000000000000003641514607367700205600ustar00rootroot00000000000000# Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in # the repo, unless a later match takes precedence. # * @gtristan @juergbi @BenjaminSchubert @cs-shadow @abderrahim apache-buildstream-27ae392/.github/common.env000066400000000000000000000003721514607367700211660ustar00rootroot00000000000000# Shared common variables CI_IMAGE_VERSION=master-2310077904 CI_TOXENV_MAIN=py310,py311,py312,py313,py314 CI_TOXENV_PLUGINS=py310-plugins,py311-plugins,py312-plugins,py313-plugins,py314-plugins CI_TOXENV_ALL="${CI_TOXENV_MAIN},${CI_TOXENV_PLUGINS}" apache-buildstream-27ae392/.github/compose/000077500000000000000000000000001514607367700206275ustar00rootroot00000000000000apache-buildstream-27ae392/.github/compose/buildbarn-config/000077500000000000000000000000001514607367700240345ustar00rootroot00000000000000apache-buildstream-27ae392/.github/compose/buildbarn-config/asset.jsonnet000066400000000000000000000021631514607367700265570ustar00rootroot00000000000000{ contentAddressableStorage: { grpc: { address: 'localhost:7982', }, }, fetcher: { // We should never be fetching anything which is not already returned by the caching fetcher. 'error': { code: 5, message: "Asset Not Found", } }, assetCache: { blobAccess: { 'local': { keyLocationMapOnBlockDevice: { file: { path: '/storage/key_location_map', sizeBytes: 1024 * 1024, }, }, keyLocationMapMaximumGetAttempts: 8, keyLocationMapMaximumPutAttempts: 32, oldBlocks: 8, currentBlocks: 24, newBlocks: 1, blocksOnBlockDevice: { source: { file: { path: '/storage/blocks', sizeBytes: 100 * 1024 * 1024, }, }, spareBlocks: 3, }, }, }, }, grpcServers: [{ listenAddresses: [':7981'], authenticationPolicy: { allow: {} }, }], allowUpdatesForInstances: [''], maximumMessageSizeBytes: 16 * 1024 * 1024, fetchAuthorizer: { allow: {} }, pushAuthorizer: { allow: {} }, } apache-buildstream-27ae392/.github/compose/buildbarn-config/storage.jsonnet000066400000000000000000000020161514607367700271010ustar00rootroot00000000000000{ contentAddressableStorage: { backend: { 'local': { keyLocationMapOnBlockDevice: { file: { path: '/cas/key_location_map', sizeBytes: 16 * 1024 * 1024, }, }, keyLocationMapMaximumGetAttempts: 16, keyLocationMapMaximumPutAttempts: 64, oldBlocks: 8, currentBlocks: 24, newBlocks: 3, blocksOnBlockDevice: { source: { file: { path: '/cas/blocks', sizeBytes: 10 * 1024 * 1024 * 1024, }, }, spareBlocks: 3, }, }, }, getAuthorizer: { allow: {} }, putAuthorizer: { allow: {} }, findMissingAuthorizer: { allow: {} }, }, global: { diagnosticsHttpServer: { httpServers: [{ listenAddresses: [':6981'], authenticationPolicy: { allow: {} }, }], } }, grpcServers: [{ listenAddresses: [':7982'], authenticationPolicy: { allow: {} }, }], maximumMessageSizeBytes: 16 * 1024 * 1024, } apache-buildstream-27ae392/.github/compose/ci.buildbarn.yml000066400000000000000000000022051514607367700237050ustar00rootroot00000000000000## # Buildbarn Compose manifest for BuildStream. # # Spins-up a unnamed and unauthenticated cache server: # - STORAGE at http://localhost:7982 # - INDEX at: http://localhost:7981 # # BuildStream configuration snippet: # # artifacts: # - url: https://localhost:7981 # type: index # push: true # - url: https://localhost:7982 # type: storage # push: true # # Basic usage: # - docker-compose -f ci.buildbarn.yml up # - docker-compose -f ci.buildbarn.yml down services: bb-asset: image: ghcr.io/buildbarn/bb-remote-asset:20241031T230517Z-4926e8e command: /config/asset.jsonnet restart: unless-stopped ports: - "7981:7981" volumes: - type: volume source: assets target: /storage - type: bind source: ./buildbarn-config/ target: /config bb-storage: image: ghcr.io/buildbarn/bb-storage:20241121T154059Z-f5a181e command: /config/storage.jsonnet restart: unless-stopped ports: - "7982:7982" volumes: - type: volume source: cas target: /cas - type: bind source: ./buildbarn-config/ target: /config volumes: assets: cas: apache-buildstream-27ae392/.github/compose/ci.buildgrid.yml000066400000000000000000000037321514607367700237160ustar00rootroot00000000000000## # BuildGrid Compose manifest for BuildStream. # # Spins-up a unnamed and unauthenticated grid: # - Controller + CAS + AC at http://localhost:50051 # # BuildStream configuration snippet: # # remote-execution: # execution-service: # url: http://localhost:50051 # action-cache-service: # url: http://localhost:50051 # storage-service: # url: http://localhost:50051 # # Basic usage: # - docker-compose -f ci.buildgrid.yml up # - docker-compose -f ci.buildgrid.yml down # version: "3.2" services: database: image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid-postgres:nightly environment: POSTGRES_USER: bgd POSTGRES_PASSWORD: insecure POSTGRES_DB: bgd volumes: - type: volume source: db target: /var/lib/postgresql networks: - grid ports: - "5432:5432" healthcheck: test: ["CMD", "pg_isready", "-U", "bgd"] interval: 1s timeout: 5s retries: 10 controller: image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid:nightly command: [ "bgd", "server", "start", "-v", "/etc/buildgrid/default.yml"] ports: - 50051:50051 networks: - grid depends_on: database: condition: service_healthy bot: image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildbox:nightly command: [ "sh", "-c", "sleep 15 && ( buildbox-casd --cas-remote=http://controller:50051 /var/lib/buildgrid/cache & buildbox-worker --request-timeout=30 --bots-remote=http://controller:50051 --cas-remote=unix:/var/lib/buildgrid/cache/casd.sock --buildbox-run=buildbox-run-bubblewrap --platform OSFamily=linux --platform ISA=x86-64 --verbose )"] privileged: true volumes: - type: volume source: cache target: /var/lib/buildgrid/cache depends_on: - controller networks: - grid networks: grid: driver: bridge volumes: cache: db: apache-buildstream-27ae392/.github/compose/ci.docker-compose.yml000066400000000000000000000075741514607367700246730ustar00rootroot00000000000000version: '3.4' x-tests-template: &tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:42-${CI_IMAGE_VERSION:-latest} command: tox -vvvvv -- --color=yes --integration environment: TOXENV: ${CI_TOXENV_ALL} # Enable privileges to run the sandbox # privileged: true devices: - /dev/fuse:/dev/fuse # Mount the local directory and set the working directory # to run the tests from. # volumes: - ../..:/home/testuser/buildstream working_dir: /home/testuser/buildstream services: debian-12: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:12-${CI_IMAGE_VERSION:-latest} debian-13: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-debian:13-${CI_IMAGE_VERSION:-latest} fedora-42: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:42-${CI_IMAGE_VERSION:-latest} fedora-43: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:43-${CI_IMAGE_VERSION:-latest} ubuntu-22.04: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-ubuntu:22.04-${CI_IMAGE_VERSION:-latest} # Ensure that tests also pass in the absence of a sandboxing tool fedora-missing-deps: <<: *tests-template image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:minimal-${CI_IMAGE_VERSION:-latest} # Test the master version of external plugins plugins-master: <<: *tests-template environment: BST_PLUGINS_EXPERIMENTAL_VERSION: master buildgrid: <<: *tests-template command: tox -vvvvv -- --color=yes --remote-execution environment: TOXENV: ${CI_TOXENV_MAIN} REMOTE_EXECUTION_SERVICE: http://localhost:50051 # We need to use host networking mode in order to be able to # properly resolve services exposed by adjacent containers. # network_mode: host buildbarn: <<: *tests-template command: tox -vvvvv -- --color=yes --remote-cache environment: TOXENV: ${CI_TOXENV_MAIN} ARTIFACT_INDEX_SERVICE: http://localhost:7981 ARTIFACT_STORAGE_SERVICE: http://localhost:7982 # We need to use host networking mode in order to be able to # properly resolve services exposed by adjacent containers. # network_mode: host docs: <<: *tests-template command: tox -e docs environment: BST_FORCE_SESSION_REBUILD: 1 lint: <<: *tests-template command: tox -e lint,format-check mypy: <<: *tests-template command: tox -e mypy # Test that each BuildStream+BuildBox wheel package can install and run. # on the PyPA official 'manylinux' images that define the base ABI for # Python binary packages. wheels-manylinux_2_28-cp310: <<: *tests-template image: quay.io/pypa/manylinux_2_28_x86_64 command: .github/wheel-helpers/test-wheel-manylinux.sh cp310 /opt/python/cp310-cp310/bin/python3 wheels-manylinux_2_28-cp311: <<: *tests-template image: quay.io/pypa/manylinux_2_28_x86_64 command: .github/wheel-helpers/test-wheel-manylinux.sh cp311 /opt/python/cp311-cp311/bin/python3 wheels-manylinux_2_28-cp312: <<: *tests-template image: quay.io/pypa/manylinux_2_28_x86_64 command: .github/wheel-helpers/test-wheel-manylinux.sh cp312 /opt/python/cp312-cp312/bin/python3 wheels-manylinux_2_28-cp313: <<: *tests-template image: quay.io/pypa/manylinux_2_28_x86_64 command: .github/wheel-helpers/test-wheel-manylinux.sh cp313 /opt/python/cp313-cp313/bin/python3 wheels-manylinux_2_28-cp314: <<: *tests-template image: quay.io/pypa/manylinux_2_28_x86_64 command: .github/wheel-helpers/test-wheel-manylinux.sh cp314-cp314 /opt/python/cp314-cp314/bin/python3 apache-buildstream-27ae392/.github/run-ci.sh000077500000000000000000000060011514607367700207130ustar00rootroot00000000000000#!/bin/bash topdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" function usage () { echo "Usage: " echo " run-ci.sh [OPTIONS] [TEST NAME [TEST NAME...]]" echo echo "Runs the CI tests locally using docker" echo echo "The test names are based on the names of tests in the CI yaml files" echo echo "If no test names are specified, all tests will be run" echo echo "Options:" echo echo " -h --help Display this help message and exit" echo " -s --service Run service tests instead of regular tests" echo " " exit 1; } arg_service=false while : ; do case "$1" in -h|--help) usage; shift ;; -s|--service) arg_service=true shift ;; *) break ;; esac done test_names="${@}" # We need to give ownership to the docker image user `testuser`, # chances are high that this will be the same UID as the primary # user on this host # user_uid="$(id -u)" user_gid="$(id -g)" if [ "${user_uid}" -ne "1000" ] || [ "${user_gid}" -ne "1000" ]; then sudo chown -R 1000:1000 "${topdir}/.." fi # runTest() # # $1 = test name # function runTest() { test_name=$1 # Run docker-compose from it's directory, because it will use # relative paths cd "${topdir}/compose" docker compose \ --env-file ${topdir}/common.env \ --file ${topdir}/compose/ci.docker-compose.yml \ run "${test_name}" return $? } # runServiceTest() # # $1 = test name # function runServiceTest() { local test_name=$1 # Run docker-compose from it's directory, because it will use # relative paths cd "${topdir}/compose" docker compose \ --env-file "${topdir}/common.env" \ --file "${topdir}/compose/ci.${test_name}.yml" \ up --detach --renew-anon-volumes --remove-orphans docker compose \ --env-file "${topdir}/common.env" \ --file "${topdir}/compose/ci.docker-compose.yml" run ${test_name} test_exit_status=$? docker compose \ --env-file "${topdir}/common.env" \ --file "${topdir}/compose/ci.${test_name}.yml" stop docker compose \ --env-file "${topdir}/common.env" \ --file "${topdir}/compose/ci.${test_name}.yml" logs docker compose \ --env-file "${topdir}/common.env" \ --file "${topdir}/compose/ci.${test_name}.yml" down --volumes return $test_exit_status } if [ -z "${test_names}" ]; then for test_name in mypy debian-12 debian-13 fedora-42 fedora-43 fedora-missing-deps ubuntu-22.04; do if ! runTest "${test_name}"; then echo "Tests failed" exit 1 fi done for test_name in buildgrid buildbarn; do if ! runServiceTest "${test_name}"; then echo "Tests failed" exit 1 fi done else if $arg_service; then for test_name in ${test_names}; do if ! runServiceTest "${test_name}"; then echo "Tests failed" exit 1 fi done else for test_name in ${test_names}; do if ! runTest "${test_name}"; then echo "Tests failed" exit 1 fi done fi fi apache-buildstream-27ae392/.github/wheel-helpers/000077500000000000000000000000001514607367700217265ustar00rootroot00000000000000apache-buildstream-27ae392/.github/wheel-helpers/fetch-latest-buildbox-release.sh000077500000000000000000000012621514607367700300750ustar00rootroot00000000000000#!/bin/bash # Download latest release binaries of BuildBox. These are statically linked # binaries produced by the buildbox-integration GitLab project, which we # bundle into BuildStream wheel packages. set -eux # # For now we only support building wheels for linux x86_64 linked against glibc # tarball="buildbox-x86_64-linux-gnu.tgz" curl -L -O "https://gitlab.com/buildgrid/buildbox/buildbox-integration/-/releases/permalink/latest/downloads/${tarball}" mkdir -p src/buildstream/subprojects/buildbox tar --extract --file "./${tarball}" --directory src/buildstream/subprojects/buildbox cd src/buildstream/subprojects/buildbox rm buildbox-run mv buildbox-run-bubblewrap buildbox-run apache-buildstream-27ae392/.github/wheel-helpers/test-wheel-manylinux.sh000077500000000000000000000007601514607367700263730ustar00rootroot00000000000000#!/bin/sh # Script to test that a generated BuildStream+BuildBox wheel package is # functional in the PyPA "manylinux" container images. # # The test is run via `run-ci.sh` which in turn uses `docker-compose` to # execute this script. set -eux COMPATIBILITY_TAGS=$1 PYTHON=$2 dnf install -y bubblewrap "$PYTHON" -m venv /tmp/venv /tmp/venv/bin/pip3 install ./wheelhouse/buildstream-*-$COMPATIBILITY_TAGS-*.whl buildstream-plugins cd doc/examples/autotools /tmp/venv/bin/bst build hello.bst apache-buildstream-27ae392/.github/workflows/000077500000000000000000000000001514607367700212175ustar00rootroot00000000000000apache-buildstream-27ae392/.github/workflows/ci.yml000066400000000000000000000114131514607367700223350ustar00rootroot00000000000000name: PR Checks # Pre-merge CI to run on push and pull_request events, even if this seems # redundant, we avoid concurrency with the below configuration. # on: pull_request: workflow_dispatch: # Use the concurrency feature to ensure we don't run redundant workflows # concurrency: group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} cancel-in-progress: true # Left to-do: # - coverage # - publishing docs to gh-pages # - persistent artifact cache # - overnight jobs # - wsl tasks (TODO: Check if GitHub's Windows runners allow WSL) # # New opportunities: # - run tests on mac (GitHub provides MacOS runners) # - standardize WSL tasks by using GitHub-provided runners jobs: tests: runs-on: ubuntu-24.04 continue-on-error: ${{ matrix.allow-failure || false }} strategy: fail-fast: false matrix: # The names here should map to a valid service defined in # "../compose/ci.docker-compose.yml" test-name: - debian-12 - debian-13 - fedora-42 - fedora-43 - fedora-missing-deps - ubuntu-22.04 - lint - mypy include: - test-name: plugins-master allow-failure: true steps: - name: Disable AppArmor restriction for bubblewrap run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Check out repository uses: actions/checkout@v2 # BuildStream requires tags to be able to find its version. with: fetch-depth: 0 - name: Run tests with Docker Compose run: | ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }} # Matrix of tests which run against remote services which we bring up adjacently service-tests: runs-on: ubuntu-24.04 strategy: matrix: # The names here map to valid services defined in "../compose/ci.docker-compose.yml", # and they also map to corresponding filenames of services which are expected # to be run in the background test-name: - buildbarn - buildgrid steps: - name: Disable AppArmor restriction for bubblewrap run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Check out repository uses: actions/checkout@v2 # BuildStream requires tags to be able to find its version. with: fetch-depth: 0 - name: Bring up the RE cluster run: | ${GITHUB_WORKSPACE}/.github/run-ci.sh --service ${{ matrix.test-name }} build_docs: runs-on: ubuntu-24.04 steps: - name: Disable AppArmor restriction for bubblewrap run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Check out repository uses: actions/checkout@v2 # BuildStream requires tags to be able to find its version. with: fetch-depth: 0 - name: Give `testuser` ownership of the source directory run: sudo chown -R 1000:1000 ${GITHUB_WORKSPACE} - name: Build documentation using Docker Compose run: | docker compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml \ run \ docs - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: docs path: doc/build/html build_wheels: name: Build Python wheel packages on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-24.04] steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Fetch latest BuildBox release run: ${GITHUB_WORKSPACE}/.github/wheel-helpers/fetch-latest-buildbox-release.sh - name: Build wheels run: pipx run cibuildwheel==v3.1.4 - uses: actions/upload-artifact@v4 with: name: wheels path: ./wheelhouse/*.whl test_wheels: name: "Test Python packages: ${{ matrix.test-name }}" needs: [build_wheels] runs-on: ubuntu-24.04 strategy: matrix: # The names here should map to a valid service defined in # "../compose/ci.docker-compose.yml" test-name: - wheels-manylinux_2_28-cp310 - wheels-manylinux_2_28-cp311 - wheels-manylinux_2_28-cp312 - wheels-manylinux_2_28-cp313 - wheels-manylinux_2_28-cp314 steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - uses: actions/download-artifact@v4 with: name: wheels path: ./wheelhouse - name: Run tests with Docker Compose run: | ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }} apache-buildstream-27ae392/.github/workflows/merge.yml000066400000000000000000000051471514607367700230500ustar00rootroot00000000000000name: Merge actions on: push: branches: - master jobs: build_docs: name: "Build documentation tarball" runs-on: ubuntu-24.04 steps: - name: Disable AppArmor restriction for bubblewrap run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Checkout code uses: actions/checkout@v2 # BuildStream requires tags to be able to find its version. with: fetch-depth: 0 - name: Give `testuser` ownership of the source directory run: sudo chown -R 1000:1000 ${GITHUB_WORKSPACE} - name: Build documentation using Docker Compose run: | docker compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml \ run \ docs # Restore permissions to the current user sudo chown -R ${USER} ${GITHUB_WORKSPACE} # Include a tarball in the published docs, allowing for # easy re-publishing of master docs on docs.buildstream.build tar -C doc/build/html -zcf docs.tgz . - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: docs path: | doc/build/html docs.tgz publish_docs: needs: build_docs runs-on: ubuntu-24.04 steps: - name: Download artifact uses: actions/download-artifact@v4 with: name: docs path: docs - name: Checkout code uses: actions/checkout@v2 with: ref: gh-pages path: pages fetch-depth: 0 - name: Update repo run: | # First reset the branch state to the initial commit, this ensures that # we do not pollute the repository with the history of every docs package # we've ever published (history of docs packages for major releases is # also stored as GitHub release assets) # cd pages/ git reset --hard GH_PAGES_FIRST_COMMIT # Copy the docs asset over and overwrite the orphan gh-pages branch, ensure # that we disable GitHub's jekyll by creating the .nojekyll file, otherwise # it will interfere with the rendering of the site. # cp -a ../docs/doc/build/html/* . cp -a ../docs/docs.tgz . touch .nojekyll git add . git config --local user.email "merge-ci@ponyland" git config --local user.name "Github Actions Nightly Job" git commit -m "Update repo for docs build $GITHUB_RUN_NUMBER" git push --force "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY.git" gh-pages apache-buildstream-27ae392/.github/workflows/release.yml000066400000000000000000000074641514607367700233750ustar00rootroot00000000000000name: Release actions on: push: tags: - '*.*.*' jobs: build_docs: name: "Build documentation tarball" runs-on: ubuntu-24.04 steps: - name: Disable AppArmor restriction for bubblewrap run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Checkout code uses: actions/checkout@v2 # BuildStream requires tags to be able to find its version. with: fetch-depth: 0 - name: Give `testuser` ownership of the source directory run: sudo chown -R 1000:1000 ${GITHUB_WORKSPACE} - name: Build documentation using Docker Compose run: | docker compose \ --env-file ${GITHUB_WORKSPACE}/.github/common.env \ --file ${GITHUB_WORKSPACE}/.github/compose/ci.docker-compose.yml \ run \ docs # Restore permissions to the current user sudo chown -R ${USER} ${GITHUB_WORKSPACE} tar -C doc/build/html -zcf docs.tgz . - uses: actions/upload-artifact@v4 with: name: docs path: docs.tgz build_sdist: name: "Build Python source distribution tarball" runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Build sdist run: pipx run build --sdist - uses: actions/upload-artifact@v4 with: name: sdist path: dist/*.tar.gz build_wheels: name: Build Python wheel packages on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-24.04] steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Fetch latest BuildBox release run: ${GITHUB_WORKSPACE}/.github/wheel-helpers/fetch-latest-buildbox-release.sh - name: Build wheels run: pipx run cibuildwheel==v3.1.4 - uses: actions/upload-artifact@v4 with: name: wheels path: ./wheelhouse/*.whl test_wheels: name: "Test Python packages: ${{ matrix.test-name }}" needs: [build_wheels] runs-on: ubuntu-24.04 strategy: matrix: # The names here should map to a valid service defined in # "../compose/ci.docker-compose.yml" test-name: - wheels-manylinux_2_28-cp310 - wheels-manylinux_2_28-cp311 - wheels-manylinux_2_28-cp312 - wheels-manylinux_2_28-cp313 - wheels-manylinux_2_28-cp314 steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - uses: actions/download-artifact@v4 with: name: wheels path: ./wheelhouse - name: Run tests with Docker Compose run: | ${GITHUB_WORKSPACE}/.github/run-ci.sh ${{ matrix.test-name }} upload_github_release: name: Upload GitHub release assets needs: [build_docs, build_sdist, build_wheels, test_wheels] runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - uses: actions/download-artifact@v4 with: name: docs - name: Upload release assets run: | tag_name="${GITHUB_REF##*/}" gh release create "$tag_name" "docs.tgz" --notes "$tag_name" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} upload_pypi_release: name: Upload PyPI release assets needs: [build_docs, build_sdist, build_wheels, test_wheels] runs-on: ubuntu-24.04 steps: - uses: actions/download-artifact@v4 with: name: sdist path: dist - uses: actions/download-artifact@v4 with: name: wheels path: dist - name: Upload to PyPI run: | pipx run twine upload --repository pypi --username __token__ --password "${{ secrets.PYPI_TOKEN }}" dist/* apache-buildstream-27ae392/.gitignore000066400000000000000000000014331514607367700176130ustar00rootroot00000000000000# Compiled python modules. __pycache__ # Generated C files src/buildstream/**/*.c src/buildstream/**/*.so # Cython report files when using annotations src/buildstream/**/*.html # Build output directory build # Setuptools distribution folder. /dist/ # Pip build metadata pip-wheel-metadata/ # Python egg metadata, regenerated from source files by setuptools. *.egg-info .eggs # Some testing related things integration-cache/ tmp .coverage .coverage-reports/ .coverage.* .cache .pytest_cache/ *.bst/ .tox/ # Generated version file src/buildstream/__version__.py # Autogenerated doc doc/source/badges/ doc/source/sessions/ doc/source/elements/ doc/source/sources/ doc/source/modules.rst doc/source/buildstream.rst doc/source/buildstream.*.rst doc/build/ versioneer.pyc # mypy .mypy_cache apache-buildstream-27ae392/.pylintrc000066400000000000000000000417731514607367700175030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # [MAIN] # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code extension-pkg-whitelist= buildstream.node, buildstream._loader.loadelement, buildstream._loader.types, buildstream._types, buildstream._utils, buildstream._variables, buildstream._yaml, ujson # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS,doc # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. ignore-patterns=.*_pb2.py,.*_pb2_grpc.py,.*.pyi # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Use multiple processes to speed up Pylint. jobs=1 # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= # Pickle collected data for later comparisons. persistent=yes # Specify a configuration file. #rcfile= # Add paths to the list of the source roots. Supports globbing patterns. The # source root is an absolute path or a path relative to the current working # directory used to determine a package namespace for modules located under the # source root. source-roots=src # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED confidence= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" # We have three groups of disabled messages: # # 1) Messages that are of no use to us # This is either because we don't follow the convention # (missing-docstring and protected-access come to mind), or because # it's not very useful in CI (too-many-arguments, for example) # # 2) Messages that we would like to enable at some point # We introduced linting quite late into the project, so there are # some issues that just grew out of control. Resolving these would # be nice, but too much work atm. # # 3) Messages related to code formatting # Since we use Black to format code automatically, there's no need for # pylint to also check for those things. # disable=, ##################################### # Messages that are of no use to us # ##################################### consider-using-f-string, fixme, missing-docstring, no-else-return, protected-access, too-few-public-methods, too-many-arguments, too-many-boolean-expressions, too-many-branches, too-many-instance-attributes, too-many-lines, too-many-locals, too-many-nested-blocks, too-many-positional-arguments, too-many-public-methods, too-many-statements, too-many-return-statements, too-many-ancestors, ####################################################### # Messages that we would like to enable at some point # ####################################################### # We have many circular imports that need breaking import-outside-toplevel, duplicate-code, # Some invalid names are alright, we should configure pylint # to accept them, and curb the others invalid-name, unused-argument, # This is good to get context on exceptions, we should enable that # at some point raise-missing-from, # We can probably enable this soon, it is a bit experimental # for the moment and current releases of pylint (August 2021) raise # a lot of false positives. unused-private-member, ################################################## # Formatting-related messages, enforced by Black # ################################################## line-too-long, superfluous-parens, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. enable=c-extension-no-member [REPORTS] # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details #msg-template= # Set the output format. Available formats are text, parseable, colorized, json # and msvs (visual studio).You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. output-format=colorized # Tells whether to display a full report or only the messages reports=no # Activate the evaluation score. score=yes [REFACTORING] # Maximum number of nested blocks for function / method body max-nested-blocks=5 # Complete name of functions that never returns. When checking for # inconsistent-return-statements if a never returning function is called then # it will be considered as an explicit return statement and no message will be # printed. never-returning-functions=optparse.Values,sys.exit [TYPECHECK] # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that # produce valid context managers. contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. generated-members=__enter__ # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # This flag controls whether pylint should warn about no-member and similar # checks whenever an opaque object is returned when inferring. The inference # can return multiple potential results while evaluating a Python object, but # some branches might not be evaluated, which results in partial inference. In # that case, it might be useful to still emit no-member and other checks for # the rest of the inferred objects. ignore-on-opaque-inference=yes # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes=optparse.Values,thread._local,_thread._local,contextlib.closing,gi.repository.GLib.GError,pathlib.PurePath # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. ignored-modules=pkg_resources,gi.repository,grpc,buildstream._protos.*,google.protobuf.* # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. missing-member-hint=yes # The minimum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance=1 # The total number of similar names that should be taken in consideration when # showing a hint for a missing member. missing-member-max-choices=1 [BASIC] # Naming style matching correct argument names argument-naming-style=snake_case # Regular expression matching correct argument names. Overrides argument- # naming-style #argument-rgx= # Naming style matching correct attribute names attr-naming-style=snake_case # Regular expression matching correct attribute names. Overrides attr-naming- # style #attr-rgx= # Bad variable names which should always be refused, separated by a comma bad-names=foo, bar, baz, toto, tutu, tata # Naming style matching correct class attribute names class-attribute-naming-style=any # Regular expression matching correct class attribute names. Overrides class- # attribute-naming-style #class-attribute-rgx= # Naming style matching correct class names class-naming-style=PascalCase # Regular expression matching correct class names. Overrides class-naming-style #class-rgx= # Naming style matching correct constant names const-naming-style=UPPER_CASE # Regular expression matching correct constant names. Overrides const-naming- # style #const-rgx= # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 # Naming style matching correct function names function-naming-style=snake_case # Regular expression matching correct function names. Overrides function- # naming-style #function-rgx= # Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_,e,f # Include a hint for the correct naming format with invalid-name include-naming-hint=no # Naming style matching correct inline iteration names inlinevar-naming-style=any # Regular expression matching correct inline iteration names. Overrides # inlinevar-naming-style #inlinevar-rgx= # Naming style matching correct method names method-naming-style=snake_case # Regular expression matching correct method names. Overrides method-naming- # style #method-rgx= # Naming style matching correct module names module-naming-style=snake_case # Regular expression matching correct module names. Overrides module-naming- # style #module-rgx= # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=^_ # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. property-classes=abc.abstractproperty # Naming style matching correct variable names variable-naming-style=snake_case # Regular expression matching correct variable names. Overrides variable- # naming-style #variable-rgx= [VARIABLES] # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables=yes # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. callbacks=cb_, _cb # A regular expression matching the name of dummy variables (i.e. expectedly # not used). dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.*|^ignored_|^unused_ # Tells whether we should check for unused import in __init__ files. init-import=no # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,past.builtins,future.builtins [LOGGING] # Logging modules to check that the string format arguments are in logging # function parameter format logging-modules=logging [SPELLING] # Limits count of emitted suggestions for spelling mistakes max-spelling-suggestions=4 # Spelling dictionary name. Available dictionaries: none. To make it working # install python-enchant package. spelling-dict= # List of comma separated words that should not be checked. spelling-ignore-words= # A path to a file that contains private dictionary; one word per line. spelling-private-dict-file= # Tells whether to store unknown words to indicated private dictionary in # --spelling-private-dict-file option instead of raising a message. spelling-store-unknown-words=no [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME, XXX, TODO [SIMILARITIES] # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes # Ignore imports when computing similarities. ignore-imports=no # Minimum lines number of a similarity. min-similarity-lines=4 [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Maximum number of characters on a single line. max-line-length=119 # Maximum number of lines in a module max-module-lines=1000 # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no [IMPORTS] # Allow wildcard imports from modules that define __all__. allow-wildcard-with-all=no # Analyse import fallback blocks. This can be used to support both Python 2 and # 3 compatible code, which means that the block might have code that exists # only in one or another interpreter, leading to false positives when analysed. analyse-fallback-blocks=no # Deprecated modules which should not be used, separated by a comma deprecated-modules=optparse,tkinter.tix # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= # Force import order to recognize a module as part of the standard # compatibility libraries. known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant [DESIGN] # Maximum number of arguments for function / method max-args=5 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Maximum number of boolean expressions in a if statement max-bool-expr=5 # Maximum number of branch for function / method body max-branches=12 # Maximum number of locals for function / method body max-locals=15 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of public methods for a class (see R0904). max-public-methods=20 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of statements in function / method body max-statements=50 # Minimum number of public methods for a class (see R0903). min-public-methods=2 [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, setUp # List of member names, which should be excluded from the protected access # warning. exclude-protected=_asdict, _fields, _replace, _source, _make # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=builtins.Exception apache-buildstream-27ae392/COMMITTERS.rst000066400000000000000000000035521514607367700200470ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _committers: Committers ========== Full commit access ------------------- List of people with full commit access, i.e. blanket commit access to the BuildStream codebase. Note that this is not a full list of all contributors. +-----------------------------------+-----------------------------------+ | Full Name | GitHub User | +===================================+===================================+ | Tristan Van Berkom | gtristan | +-----------------------------------+-----------------------------------+ | Jürg Billeter | juergbi | +-----------------------------------+-----------------------------------+ | Chandan Singh | cs-shadow | +-----------------------------------+-----------------------------------+ | Benjamin Schubert | BenjaminSchubert | +-----------------------------------+-----------------------------------+ | Abderrahim Kitouni | abderrahim | +-----------------------------------+-----------------------------------+ | Sander Striker | sstriker | +-----------------------------------+-----------------------------------+ apache-buildstream-27ae392/CONTRIBUTING.rst000066400000000000000000000243311514607367700202660ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Contributing ============ Some tips and guidelines for developers hacking on BuildStream .. _contributing_filing_issues: Filing issues ------------- If you are experiencing an issue with BuildStream, or would like to submit a patch to fix an issue, then you should first search the list of `open issues `_ to see if the issue is already filed, and `open an issue `_ if no issue already exists. .. _contributing_fixing_bugs: Fixing bugs ----------- Before fixing a bug, it is preferred that an :ref:`issue be filed ` first in order to better document the defect, however this need not be followed to the letter for minor fixes. Patches which fix bugs should always come with a regression test. .. _contributing_adding_features: Adding new features ------------------- Feature additions should be proposed on the `mailing list `_ before being considered for inclusion. To save time and avoid any frustration, we strongly recommend proposing your new feature in advance of commencing work. Once consensus has been reached on the mailing list, then the proposing party should :ref:`file an issue ` to track the work. New features must be well documented and tested in our test suite. It is expected that the individual submitting the work take ownership of their feature within BuildStream for a reasonable timeframe of at least one release cycle after their work has landed on the master branch. This is to say that the submitter is expected to address and fix any side effects, bugs or regressions which may have fell through the cracks in the review process, giving us a reasonable timeframe for identifying these. .. _contributing_submitting_patches: Submitting patches ------------------ Submitting patches works in the regular GitHub workflow of submitting pull requests. Branch names ~~~~~~~~~~~~ If you are an apache member with access to the main repository, and are submitting a pull request for a branch within the main repository, then please be careful to use an identifiable branch name. Branch names for pull requests should be prefixed with the submitter's name or nickname, followed by a forward slash, and then a descriptive name. e.g.:: username/fix-that-bug This allows us to more easily identify which branch does what and belongs to whom, especially so that we can effectively cleanup stale branches in the upstream repository over time. Pull requests ~~~~~~~~~~~~~ Once you have created a local branch, you can push it to the upstream BuildStream repository using the command line:: git push origin username/fix-that-bug GitHub will respond to this with a message and a link to allow you to create a new merge request. You can also `create a pull request using the GitHub UI `_. You may open pull requests for the branches you create before you are ready to have them reviewed and considered for inclusion if you like. Until your merge request is ready for review, the pull request title must be prefixed with the ``WIP:`` identifier. Consider marking a pull request as WIP again if you are taking a while to address a review point. This signals that the next action is on you, and it won't appear in a reviewer's search for non-WIP merge requests to review. Organized commits ~~~~~~~~~~~~~~~~~ Submitted branches must not contain a history of the work done in the feature branch. For example, if you had to change your approach, or have a later commit which fixes something in a previous commit on your branch, we do not want to include the history of how you came up with your patch in the upstream master branch. Please use git's interactive rebase feature in order to compose a clean patch series suitable for submission upstream. Every commit in series should pass the test suite, this is very important for tracking down regressions and performing git bisections in the future. We prefer that documentation changes be submitted in separate commits from the code changes which they document, and newly added test cases are also preferred in separate commits. If a commit in your branch modifies behavior such that a test must also be changed to match the new behavior, then the tests should be updated with the same commit, so that every commit passes its own tests. These principles apply whenever a branch is non-WIP. So for example, don't push 'fixup!' commits when addressing review comments, instead amend the commits directly before pushing. Commit messages ~~~~~~~~~~~~~~~ Commit messages must be formatted with a brief summary line, followed by an empty line and then a free form detailed description of the change. The summary line must start with what changed, followed by a colon and a very brief description of the change. If the commit fixes an issue, or is related to an issue; then the issue number must be referenced in the commit message. **Example**:: element.py: Added the frobnicator so that foos are properly frobbed. The new frobnicator frobnicates foos all the way throughout the element. Elements that are not properly frobnicated raise an error to inform the user of invalid frobnication rules. Fixes #123 Note that the 'why' of a change is as important as the 'what'. When reviewing this, folks can suggest better alternatives when they know the 'why'. Perhaps there are other ways to avoid an error when things are not frobnicated. When folks modify this code, there may be uncertainty around whether the foos should always be frobnicated. The comments, the commit message, and issue #123 should shed some light on that. In the case that you have a commit which necessarily modifies multiple components, then the summary line should still mention generally what changed (if possible), followed by a colon and a brief summary. In this case the free form detailed description of the change should contain a bullet list describing what was changed in each component separately. **Example**:: artifact cache: Fixed automatic expiry in the local cache o _artifactcache/artifactcache.py: Updated the API contract of ArtifactCache.remove() so that something detailed is explained here. o _artifactcache/cascache.py: Adhere to the new API contract dictated by the abstract ArtifactCache class. o tests/artifactcache/expiry.py: Modified test expectations to match the new behavior. This is a part of #123 Committer access ---------------- Committers in the BuildStream project are those folks to whom the right to directly commit changes to our version controlled resources has been granted. While every contribution is valued regardless of its source, not every person who contributes code to the project will earn commit access. The `COMMITTERS`_ file lists all committers. .. _COMMITTERS: https://github.com/apache/buildstream/blob/master/COMMITTERS.rst How commit access is granted ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ After someone has successfully contributed a few non-trivial patches, some full committer, usually whoever has reviewed and applied the most patches from that contributor, proposes them for commit access. This proposal is sent only to the other full committers - the ensuing discussion is private, so that everyone can feel comfortable speaking their minds. Assuming there are no objections, the contributor is granted commit access. The decision is made by consensus; there are no formal rules governing the procedure, though generally if someone strongly objects the access is not offered, or is offered on a provisional basis. This of course relies on contributors being responsive and showing willingness to address any problems that may arise after landing patches. However, the primary criterion for commit access is good judgement. You do not have to be a technical wizard or demonstrate deep knowledge of the entire codebase to become a committer. You just need to know what you don't know. Non-code contributions are just as valuable in the path to commit access. If your patches adhere to the guidelines in this file, adhere to all the usual unquantifiable rules of coding (code should be readable, robust, maintainable, etc.), and respect the Hippocratic Principle of "first, do no harm", then you will probably get commit access pretty quickly. The size, complexity, and quantity of your patches do not matter as much as the degree of care you show in avoiding bugs and minimizing unnecessary impact on the rest of the code. Many full committers are people who have not made major code contributions, but rather lots of small, clean fixes, each of which was an unambiguous improvement to the code. (Of course, this does not mean the project needs a bunch of very trivial patches whose only purpose is to gain commit access; knowing what's worth a patch post and what's not is part of showing good judgement.) Windows CI ---------- The infrastructure for running the CI against Windows is different from the usual runners, due to a combination of licensing technicalities and differing containerisation support. The scripts used to generate a CI runner can be found at `https://gitlab.com/BuildStream/windows-startup-script`. The `wsl` branch can be used to generate a runner for WSL, and the `win32` branch can be used to generate a native-windows runner. Further information ------------------- .. toctree:: :maxdepth: 1 hacking/coding_guidelines.rst hacking/using_the_testsuite.rst hacking/writing_documentation.rst hacking/writing_plugins.rst hacking/measuring_performance.rst hacking/making_releases.rst hacking/grpc_protocols.rst hacking/managing_data_files.rst hacking/updating_python_deps.rst hacking/ui.rst apache-buildstream-27ae392/LICENSE000066400000000000000000000342721514607367700166370ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. =============================================================================== Apache BuildStream Subcomponents: The Apache BuildStream project contains subcomponents or borrowed code with separate copyright notices and license terms. Your use of the source code for the these subcomponents is subject to the terms and conditions of the following licenses. ------------------------------------------------------------------------------- doc/bst2html.py (MIT license) ------------------------------------------------------------------------------- Copyright (c) 2013 German M. Bravo (Kronuz) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- src/buildstream/_frontend/complete.py (BSD license) ------------------------------------------------------------------------------- Copyright (c) 2014 by Armin Ronacher. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ------------------------------------------------------------------------------- versioneer.py / src/_version.py (Public Domain) ------------------------------------------------------------------------------- To make Versioneer easier to embed, all its code is dedicated to the public domain. The `_version.py` that it creates is also in the public domain. Specifically, both are released under the Creative Commons "Public Domain Dedication" license (CC0-1.0), as described in https://creativecommons.org/publicdomain/zero/1.0/ . apache-buildstream-27ae392/MANIFEST.in000066400000000000000000000034361514607367700173660ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Basic toplevel package includes include COMMITTERS.rst include CONTRIBUTING.rst include LICENSE include NOTICE include NEWS include README.rst # Cython files recursive-include src/buildstream *.pyx recursive-include src/buildstream *.pxd # Data files required by BuildStream's generic source tests graft src/buildstream/_testing/_sourcetests/project # Documentation package includes include doc/Makefile include doc/badges.py include doc/bst2html.py include doc/source/conf.py include doc/source/plugin.rsttemplate recursive-include doc/source *.rst recursive-include doc/source *.py recursive-include doc/source *.in recursive-include doc/source *.html recursive-include doc/source *.odg recursive-include doc/source *.svg recursive-include doc/examples * recursive-include doc/sessions *.run # Tests graft tests include tox.ini include .coveragerc include .pylintrc # Protocol Buffers recursive-include src/buildstream/_protos *.proto # Requirements files recursive-include requirements *.in recursive-include requirements *.txt # Versioneer include versioneer.py # setuptools.build_meta don't include setup.py by default. Add it include setup.py # bundled binaries should only be in the bdist packages recursive-exclude src/buildstream/subprojects * apache-buildstream-27ae392/NEWS000066400000000000000000001150441514607367700163260ustar00rootroot00000000000000================= buildstream 2.7.0 ================= o Support defining source provenance attributes in project.conf (#2099) o Allow multi-source plugins to provide source provenance info per source (#2098) o Don't require runtime deps when building projects with `--deps none` (#2102) o Add `--artifact-remote` and `--ignore-project-artifact-remotes` options to `bst artifact show` (#2088) o Fix build of projects with multiple link elements with the same target (#2097) o Fix filesystem write access in `bst shell` with a cached build tree (#2093) o Artifact cache fixes to support buildgrid-asset as cache server (#2091) o Fix pip plugins installed in editable mode (#2084) o Drop support for Python 3.9 (#2085) ================= buildstream 2.6.0 ================= o Add the `remote-apis-socket` sandbox configuration to offer build tools in the sandbox (e.g., recc or bazel) access to the Remote Execution API for caching or (nested) remote execution (#2014, #2031, #2038) o Add the `digest-environment` dependency configuration for use by Remote Execution API clients running in the sandbox (#2036) o Additional source provenance in SourceInfo (#2009) o Allow showing the element kind in `bst show` (#2058) o Accept pre-releases for plugins loaded from the `pip` origin (#2019) o BuildStream now requires BuildBox 1.2.6 or later o Support Python 3.13.4+ and 3.14 (#2039, #2064, #2067) o Support protobuf 6+ (#2043) and Click 8.2+ (#2065) ================= buildstream 2.5.0 ================= o Fix directory mtime to support latest buildbox-casd (#2001) o Fix test cases to work with latest buildbox (#1999) o Support gRPC retry and timeout configuration for all remote services (#1987) o Fix our setup.py for surprise update of setuptools (#2007) o Add Source.collect_source_info() and SourceFetcher.get_source_info() methods, support showing introspected source provenance information in `bst show`, make it possible for absence of implementations of these new methods be fatal, and export some related utilities to help plugins support such (#1997, #2008, #2016, #2018) o Some minor fixes to documentation (#1992) o Support showing artifact content digests in `bst show` (#1994) o Better support for parsing boolean configuration that is exported from project options (#2011, #2006) o Better error handling in remote execution if services do not support required sandbox options (#2013) o Fix pushing artifacts with remote caches (#2015) o Fix build warnings stemming from using outdated grpcio tools (#2017) o Fix additional jobs being scheduled after the user has asked to quit (#2005, #1787) ================= buildstream 2.4.1 ================= o Update default min-version for bst init to 2.4 (#1978) o Use subprocess umask support (#1754) o Don't use `preexec_fn` on Python 3.11+ (#1980) o contrib/bst-graph: Escape names when using as node_id (#1979) o _loader: Use the sort algorithm from Python 3.12 for dependencies (#1993) ================= buildstream 2.4.0 ================= o Add new user configuration for the cache (#1950) o docs: fix link to docker install instructions (#1952) o Fix console warnings in newer python versions (#1958) o Update github CI workflow actions (#1957) o Fix `bst artifact show --long` for artifacts with symlinks (#1959) o Fix race condition in public data handling (#1955) o Cleanup deadcode (#1954) o Make remote execution directory fetching more robust (#1962) o Disable logging for frequent cache query jobs (#1963) o Improve performance when using `storage-service` (#1966) o Fixed issues with nested junctions (#1891, #1969, #1975) o Drop support for Python 3.8 and add support for Python 3.13 (#1971) o Load time optimization for commands which don't require cache queries (#1965) o Address deprecation warnings from deprecated pkg_resources (#1948) ================= buildstream 2.3.0 ================= o Proxy remote asset and remote execution requests via buildbox-casd (#1922, #1926) o Optimize bst show when no cached state is needed (#1923) o Support %{description} in bst show commands o Support HTTP Bearer authentication for remote API servers (#1925) o Drop support for Python 3.7 (#1933, #1934, #1936) o downloadablefilesource: Add 10 minutes timeout (#1928) o tar.py: Always check member paths (#1932) o Ensure blobs are available locally when using storage-service (#1938) o Support loading source mirror plugins from pip and junction origins (#1931) o Restore terminal after exiting a bst shell (#1786) o Fail early if buildbox-run is present but non-functional (#1746) ================= buildstream 2.2.1 ================= o Fix parsing of keepalive time (#1916) o Fix alias mapping when using source mirror plugins (#1918) o Fix typing of SourceFetcher.fetch() and Source.translate_url(): the `alias_override` parameter should be considered opaque. o Update documentation regarding the buildbox merge (#1919) ================= buildstream 2.2.0 ================= o Support Python 3.12 (#1881, #1884) o Add suffix parameter to `Source.translate_url()` (#1899) o Error out if a YAML mapping has duplicate keys (#1877) o New configuration option to set the grpc keepalive time (#1879) o Add SourceMirror plugins to allow more flexibility when defining mirrors (#1903) o Use system-installed buildbox components over bundled ones if available (#1902) o Support bearer authentication in DownloadableFileSource (#1911) o Support mapping aliases from subprojects to parent project to ease mirroring (#1910) ================= buildstream 2.1.0 ================= o Support new architecture name "la64v100" (#1811) o Fix race condition when downloading sources (#1831) o Properly handle partial download errors (#1846) o Fix regressed ETag handling to prevent redundant file downloads (#1834) o Fix parsing remotes specified on the command line (#1850) o Fix bug when tracking ref-less sources (like local sources) when using project.refs (#1858) o Follow semantic versioning more strictly (#1854) o New `bst build --retry-failed` feature (#1849) o Fix parsing override-project-caches configuration (#1861) o Fix crashes when instructed to only download from mirrors, and some sources have no mirrors defined (#1862) o Drop unnecessary dependency on dateutil (#1867) o Fix severe hangs when can occur when running out of disk space specifically when attempting to open and write to a new log file (#1865) ================= buildstream 2.0.1 ================= o Stop including generated C code in releases (#1777) o Support completions for Zsh shells (#558) o Support Python 3.11 (#1782, #1806, #1815, #1759) o Fix git submodule tests to work with recent git (#1816) o Enable coverage collection in tests (#1817) o Optimize file copies where possible (#1793) o Some fixes for downloadable file sources (#1826, #1814) o Various cleanups of deprecated code and documentation ================= buildstream 2.0.0 ================= No changes since 1.95.6, releasing voted upon 2.0.0 ================== buildstream 1.95.6 ================== o Adding missing license headers to some files in the _testing module ================== buildstream 1.95.5 ================== o Fixed remaining license header issues (in sources and also docs) o Documenting how to prepare BuildStream for new python versions o Remove generated GPL files from autotools test tarballs o Update package metadata urls and author ================== buildstream 1.95.4 ================== o Fixed license headers in source files o Attribute third party licenses etc properly in the LICENCE and NOTICE files and ensure they are included in source distributions o Removed obsolete update_committers.py script in contrib o Require at least the first cython version which properly supported python 3.10 o Update copyright year and attribution in generated documentation ================== buildstream 1.95.3 ================== o Some fixes to the CI (#1736, #1737) o Fix errors passing certs/keys to casd (#1748) o Update installation instructions for 2.0 (#1741) o Fix issues using `storage-service` (#1750) o Revert to previous mirror ordering behavior when tracking (#1765) ================== buildstream 1.95.2 ================== o Fix stack traces in tab completion of `bst artifact` commands (#1718) o Document new link to obtain latest static buildbox binaries (#1721) o Allow `bst shell --build` directly on artifacts without needing a project (#1711) o Refactor out some unused methods (#781) o Continue pushing build results if builds fail when --on-error=quit (#534) o Fix issue which sometimes caused messages to get lost in the master log (#1728) o Fix remaining ruamel.yaml deprecation warnings (#1731) o Only print the last failure for retried failed elements (#517) o Enhance BuildElement documentation (#1734) o Automatically publish binary wheels which contain static BuildBox binaries when releasing (#1712) ================== buildstream 1.95.1 ================== o Fixed issue loading refs from project.refs files in subprojects (#1655) o Fixed `bst artifact checkout` on machines which do not support the artifact's build architecture (#1352) o Rebuild protos to support newer grpc versions (#1665) o Support the RISC-V machine architecture name (#1577) o Record date and buildstream version in all build logs (#184) o Updated internals to be compliant with REAPI v2.2 (#1676) o Avoid adding unnecessary data to CAS (#1677, #1678) o Disable unneeded gRPC fork support which sometimes causes crashes (#1679) o Added runtime version checks against buildbox-casd (#1189) o Resolve variables in include files in the context of their project (#1485) o Retry failed builds in non-strict mode (#1460) o Optimize staging of sources (#1680) o Fixed `bst artifact delete` in non-strict mode (#1461) o Allow conditional statements in toplevel of project.conf (#1683) o Fix crashes related to cross-project includes / plugin loading (#1686) o Fix addressing of subproject elements on command line (#1455) o Ensure CAS does not get corrupted on systems without buildbox-fuse (#1700) o Optimization when importing files to/from CAS (#1697, #1698) o Print errors in advance of the interactive prompt (#1394) o Fixed Source.set_ref() code to behave as documented (#1685) o Updated installation instructions (#1705, #1710) o Support interactive build shells with all elements (#1263) o Reduce information encoded in cache keys for elements which do not use the sandbox (#1687) ================== buildstream 1.95.0 ================== Core ---- o BuildStream now requires Python >= 3.7 and also supports Python 3.10. API --- o BREAKING CHANGE: Removed SandboxFlags from public API, replaced this with a `root_read_only` boolean parameter to relevant Sandbox APIs. Format ------ o BREAKING CHANGE: Element names must now bare the `.bst` suffix and not contain invalid characters, as was already documented. Now the warning is replaced by an unconditional error. o User configuration now has the ability to override source mirrors defined by project configuration. o User configuration now has fetch source and track source configurations to control what mirrors/URIs can be accessed to obtain sources. CLI --- o BREAKING CHANGE: Removed `--deps plan` from all commands o The `--pull` option is removed from the `bst shell` and `bst artifact checkout` commands, which will now unconditionally try to pull required artifacts. o BREAKING CHANGE: Now the artifact commands only accept element names if specified with a `.bst` suffix (including wildcard expressions), and otherwise assumes the command means to specify an artifact name. ================== buildstream 1.93.6 ================== Format ------ o The `script` element no longer has a `layout` configuration directly, and now exposes a new `location` dependency configuration instead. o The BuildElement now has a new `location` dependency configuration, allowing BuildElement plugins to also stage dependencies into custom locations in the sandbox. o BREAKING CHANGE: Stack element dependencies are now hard required to be both build and runtime dependencies. o BREAKING CHANGE: The old and deprecated "remote-execution" configuration style is no longer supported, one must now specify the "execution-service" block and cannot specify a "url" in the "remote-execution" dictionary anymore. o BREAKING CHANGE: Full redesign of remote cache configurations, junction configurations related to artifact pulling/pushing policies have been removed, users will need to view the new documentation and update cache configuration accordingly. Core ---- o BuildStream now also supports Python 3.9. o BREAKING CHANGE: Changed API signatures to remove Scope type from the API - Element.dependencies() - Element.stage_dependency_artifacts() - Element.search() Elements now can only ever see dependencies in their build scope. * BREAKING CHANGE: Changed ScriptElement.layout_add() API to take Element instances in place of Element names CLI --- o BREAKING CHANGE: `bst shell --use-buildtree` is now a boolean flag. `--use-buildtree=ask` and `--use-buildtree=try` are no longer supported. o BREAKING CHANGE: `--remote` options removed, replaced by `--artifact-remote` or `--source-remote` o BREAKING CHANGE: All old obsolete/deprecated commands removed, including: - bst fetch (now bst source fetch) - bst track (now bst source track) - bst checkout (now bst artifact checkout) - bst pull (now bst artifact pull) - bst push (now bst artifact push) ================== buildstream 1.93.5 ================== Format ------ o BREAKING CHANGE: Some project.conf keys are no longer allowed to be specified outside of the project.conf file (they cannot be specified in an include file), these include: - name - element-path - min-version - plugins CLI --- o Add `bst source push` subcommand. This command pushes element sources to a remote source cache. o `bst source track` now also accepts "build" and "run" as inputs. Artifacts --------- o BREAKING CHANGE: Use Remote Asset API for remote artifact and source caches. Existing artifact servers are not compatible and need to be updated to the latest version which will then allow them to be repopulated with new artifacts. Plugins ------- o tar: filter out 'dev' nodes from the tar archives when extracting them o `buildstream.testing` module now requires pytest >= 6.0.1. ================== buildstream 1.93.4 ================== Core ---- o The bwrap sandboxing backend and the SafeHardlinks FUSE filesystem have been dropped. buildbox-run is now the only sandboxing backend. CLI --- o `bst shell --build` will now automatically fetch missing sources. o `bst build --deps` now also accepts "build" as an input. o `bst source fetch --deps` now also accepts "build" and "run" as inputs. o `bst artifact pull --deps` now also accepts "build" and "run" as inputs. o `bst artifact push --deps` now also accepts "build" and "run" as inputs. o Full paths can now be used to address elements across multiple junction boundaries Format ------ o BREAKING CHANGE: Now deprecation warnings are suppressed using the `allow-deprecated` configuration with the plugin origins in project.conf, instead of on the source/element overrides section (See issue #1291) o BREAKING CHANGE: The `target` option of junctions has been completely removed, Use `link` elements instead. o Variables from an element can now be used in source configurations o Full paths can now be used to address elements across multiple junction boundaries o A new "junction" plugin origin allows loading of plugins across junction boundaries o Support buildstream2.conf as a user configuration, allowing parallel installation of bst 1 & bst 2 Plugins ------- o Cache keys will change for all elements that have defined the `command-subdir` variable. This is the result of fixing a bug where this variable was not included in the cache key correctly. o The `pip` element has been removed. Please use the one from bst-plugins-experimental o Introduced new `link` element which can be used as a symbolic link to other elements or junctions, in the local project or in subprojects. o The DownloadableFileSource is now public API API --- o `Element.node_subst_vars` and `Element.node_subst_sequence_vars` are now deprecated and will get removed in the next version. All config entries are now resolved so there is no need to use them anymore. ================== buildstream 1.93.3 ================== o BREAKING CHANGE: Removed BST_REQUIRED_VERSION_[MAJOR/MINOR] from plugin base class, this is now replaced with BST_MIN_VERSION which is now a hard requirement. o BREAKING CHANGE: Removed BST_FORMAT_VERSION from plugin base class, this is no longer supported. o buildbox-run is now the default sandbox backend on all platforms ================== buildstream 1.93.2 ================== Core ---- o BuildStream now requires Python >= 3.6. o BREAKING CHANGE: Conditional directives `(?)` from files included through junctions are evaluated with the options defined in the sub project the file comes from. o BREAKING CHANGE: "format-version" is removed and replaced with "min-version", which is now required to be specified in project.conf. o Incremental workspace builds are supported again. CLI --- o BREAKING CHANGE: `bst shell --sysroot` has been removed. This is no longer needed now that we support caching buildtrees in CAS. API --- o The `Directory` API has been extended to cover the use cases of more element plugins. ================== buildstream 1.93.1 ================== API --- o `ErrorDomain` and `LoadErrorReason` are now available in the public `buildstream.exceptions` module for use in test suites of external plugins. Plugins ------- o The 'git' source will now only fetch a single revision when its ref is set to an exact tag in git-describe format (...-0-g...). o When a 'git' source has a ref in git-describe format, only the commit ID is used when calculating the element's cache key. This means you can change between ref formats without having to rebuild the element. ================== buildstream 1.93.0 ================== CLI --- o BREAKING CHANGE: `bst show` will now output `junction` instead of `buildable` for the state of junction elements, as they can't be built. API --- o BREAKING CHANGE: The entry point groups used for element and source plugins are now separate. Elements must be defined in `buildstream.plugins.elements` group, and sources in `buildstream.plugins.sources`. o External plugins can now register a `buildstream.tests.source_plugins` entrypoint. The entry point can have an arbitrary name, but its value should point to a module containing a `register_sources()` method. This method should call `register_repo_kind` for all sources you want to have tested in BuildStream. Plugins authors that do this and believe BuildStream should be testing that part of their plugins should open an issue on BuildStream. o BREAKING CHANGE: `Consistency` has been removed and `Source.get_consistency()` has been replaced by `Source.is_resolved()` and `Source.is_cached()`. `Source.is_resolved()` has a default of `self.get_ref() is not None`, so if the only thing your plugin requires for being resolved is to have a valid ref, you don't need to do anything there. `Source.is_cached()` is there to replace a `Consistency.CACHED` implementation and will need to be implemented by plugin authors. Core ---- o BREAKING CHANGE: Once a source has been put in the internal BuildStream cache, its content will not get checked for validity unless a workspace is opened on it. If you see a warning that was not fatal as you fetch your source and want to re-test it to make sure it's gone without changing its ref (most plugins would handle that correctly), you will need to delete the internal source cache first. Plugins ------- o BREAKING CHANGE: deb plugin have been moved to the bst-plugins-experimental repository. Miscellaneous ------------- o By default the update rate of builstream console output (e.g. messages & status updates) when executing a scheduler driven task is restricted to once per second, known as the tick, with messages being batched in the intervening periods if necessary. This restriction can be lifted with `throttle-ui-updates: False` in user configuration. Logging behaviour remains unaffected by this configuration. ================== buildstream 1.91.3 ================== CLI --- o BREAKING CHANGE: Commands no longer accept any options related to tracking. Please use `bst source track` separately prior to running commands, if you need similar functionality. The full list of removed options is as follows: * `bst build --track` * `bst build --track-all` * `bst build --track-except` * `bst build --track-cross-junctions` / `bst build -J` * `bst build --track-save` * `bst source fetch --track` * `bst source fetch --track-cross-junctions` / `bst source fetch -J` * `bst workspace open --track` * `bst workspace reset --track` Plugins ------- o The 'git' source will now fetch submodules recursively when its 'checkout-submodules' option is enabled. API --- o Sources may force early staging to cache by advertising `BST_KEY_REQUIRES_STAGE`. Sources setting this are staged to the cache to generate unique keys. `WorkspaceSource` and `LocalSource` set this. Core ---- o BuildStream now supports Python 3.8. Note that due to a bug in the 3.8.0 series, BuildStream requires Python >= 3.8.1 when running on Python 3.8. o BuildStream now also supports the following machine architectures: * powerpc * powerpc64 * powerpc64le * sun4v ================== buildstream 1.91.2 ================== o Various bug fixes. ================== buildstream 1.91.1 ================== API --- o BREAKING CHANGE: 'Element.node_subst_member' has been removed. Please use 'Element.node_subst_vars' instead. o BREAKING CHANGE: 'Element.node_subst_list' has been removed. Please use 'Element.node_subst_sequence_vars' instead. o BREAKING CHANGE: Workspace incremental builds have been temporarily disabled o A new 'Node.strip_node_info()' is available and allows getting the underlying data structure for the given node. o BuildStream now requires Buildbox-casd 0.0.3 at minimum ================== buildstream 1.91.0 ================== This release is a development snapshot, without any compatibility guarantees. (The list of changes is non-exhaustive) CLI --- o BREAKING CHANGE: `bst track` and `bst fetch` commands are now obsolete. Their functionality is provided by `bst source track` and `bst source fetch` respectively. o BREAKING CHANGE: The top level commands `checkout`, `push` and `pull` have been moved to the `bst artifact` subcommand group and are now obsolete. For example, you must now use `bst artifact pull hello.bst`. The behaviour of `checkout` has changed. The previously mandatory LOCATION argument should now be specified with the `--directory` option. In addition to this, `--tar` is no longer a flag, it is a mutually incompatible option to `--directory`. For example, `bst artifact checkout foo.bst --tar foo.tar.gz`. o BREAKING CHANGE: `bst workspace open` now supports the creation of multiple elements and allows the user to set a default location for their creation. This has meant that the new CLI is no longer backwards compatible with BuildStream 1.2. o BREAKING CHANGE: The `bst build` command no longer accepts the `--all` option. It now accepts `--deps` with a choice of 'plan' (default) and 'all', for equivalent behaviour. o BREAKING CHANGE: `bst init` no longer uses the `--directory` or `-C` option. Instead, it (optionally) takes a directory as an argument. o BREAKING CHANGE: The `bst source-bundle` command has been removed. The functionality it provided has been replaced by the `--include-build-scripts` option of the `bst source checkout` command. To produce a tarball containing an element's sources and generated build scripts you can do the command `bst source checkout --include-build-scripts --tar foo.tar foo.bst`. A `--compression` option is also supported when using `--tar` which supports xz, gz and bz2 compression. o BREAKING CHANGE: The 'auto-init' functionality has been removed. This would offer to create a project in the event that bst was run against a directory without a project, to be friendly to new users. It has been replaced with an error message and a hint instead, to avoid bothering folks that just made a mistake. o BREAKING CHANGE: The unconditional 'Are you sure?' prompts have been removed. These would always ask you if you were sure when running 'bst workspace close --remove-dir' or 'bst workspace reset'. They got in the way too often. o Added `bst artifact show` subcommand which shows the cached status of an artifact. If project/user remotes are available, they are checked for the target elements (and their deps, if specified). Artifacts available in remotes are displayed as "available". o Added `bst artifact list-contents` subcommand which can display the names of files in artifacts in your artifact cache, either by element name or by direct artifact reference. --long option can be used to display more information; file type and size. o Added `bst artifact delete` subcommand. This command removes artifacts from the local cache. Multiple element names and artifact refs can be specified as arguments. o Added `bst artifact log` subcommand for viewing build logs. o Added new `bst source checkout` command to checkout sources of an element. o Running commands without elements specified will now attempt to use the default targets defined in the project configuration. If no default target is defined, all elements in the project will be used. o bst 'build' now has '--remote, -r' option, inline with bst 'push' & 'pull'. Providing a remote will limit build's pull/push remote actions to the given remote specifically, ignoring those defined via user or project configuration. o `bst shell --sysroot` now takes any directory that contains a sysroot, instead of just a specially-formatted build-root with a `root` and `scratch` subdirectory. o Creating a build shell through the interactive mode or `bst shell --build` will now use the cached build tree if available locally. It is now easier to debug local build failures. Plugins ------- o BREAKING CHANGE: cmake, make, makemaker, meson, modulebuild and qmake plugins have been moved to the bst-plugins-experimental repository. o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V' environment variables. There is already a 'make' element with the same variables. Note that this is a breaking change, it will require users to make changes to their .bst files if they are expecting these environment variables to be set. o BREAKING CHANGE: The `git` plugin does not create a local `.git` repository by default. If `git describe` is required to work, the plugin has now a tag tracking feature instead. This can be enabled by setting 'track-tags'. o Added new `pip` source plugin for downloading python packages using pip, based on requirements files from previous sources. API --- o BREAKING CHANGE: The yaml API has been rewritten entirely. When accessing configuration from YAML, please use the new `Node` classes exposed in the `buildstream` package. See the documentation for how to use it. This change has also removed the need of a YAML cache as it proved to no longer be of benefit. o There is a new sandbox API for command batching. It is used for build, script, and compose elements. o Source plugins may now request access access to previous during track and fetch by setting `BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or `BST_REQUIRES_PREVIOUS_SOURCES_FETCH` attributes. Configuration Format -------------------- o Elements may now specify 'build-depends' and 'runtime-depends' fields to avoid having to specify the dependency type for every entry in 'depends'. o Junction elements may now specify another junction as their target, using the `target` configuration option. o Elements may now specify cross-junction dependencies as simple strings using the format '{junction-name}:{element-name}'. Core ---- o BREAKING CHANGE: Reverted the default behaviour of junctions. Subproject elements will no longer interact with the parent project's remote (by default). To enable this behaviour, a new "cache-junction-elements" boolean can be optionally declared as part of your junction element's configuration. Additionally, a new "ignore-junction-remotes" option has also been introduced. This allows you to completely ignore subproject remotes when using the parent project. o BREAKING CHANGE: Symlinks are no longer resolved during staging and absolute symlinks are now preserved instead of being converted to relative symlinks. o BREAKING CHANGE: Overlap whitelists now require absolute paths. This allows use of variables such as %{prefix} and matches the documentation. o BREAKING CHANGE: Default strip-commands have been removed as they are too specific. Recommendation if you are building in Linux is to use the ones being used in freedesktop-sdk project, for example o All elements should now be suffixed with `.bst`. Attempting to use an element that does not have the `.bst` extension, will result in a warning. o Failed builds are included in the cache as well. `bst artifact checkout` will provide anything in `%{install-root}`. A build including cached fails will cause any dependant elements to not be scheduled and fail during artifact assembly, and display the retry prompt during an interactive session. o Added Basic support for the BuildBox sandbox. The sand box will only be used if the environment variable BST_FORCE_SANDBOX is set to `buildbox`. This is the first step in transitioning to only using BuildBox for local sandboxing. Artifacts --------- o BREAKING CHANGE: Artifact as a Proto. The caching of BuildStream artifacts has changed from a reference based impelementation. Existing artifacts and artifact servers are not compatible, as such remote artifact servers need to be updated to the latest version which will then allow them to be repopulated with new artifacts. o BREAKING CHANGE: The project name of dependencies have been included when calculating the cache key. This was required to keep inline with the artifact proto. Additionally, for strict mode, the name of dependencies have also been included in the calculation of the cache key. This is again to keep inline with the proto, but also to maintain symmetry in the behaviour between the strict and non-strict modes. o Due to the element `build tree` being cached in the respective artifact their size in some cases has significantly increased. In *most* cases the build trees are not utilised when building targets, as such by default bst 'pull' & 'build' will not fetch build trees from remotes. This behaviour can be overridden with the cli main option '--pull-buildtrees', or the user configuration cache group option 'pull-buildtrees = True'. The override will also add the build tree to already cached artifacts. When attempting to populate an artifactcache server with cached artifacts, only 'complete' elements can be pushed. If the element is expected to have a populated build tree then it must be cached before pushing. o Artifacts can now be cached explicitly with an empty `build tree` when built. Element types without a build-root were already cached with an empty build tree directory, this can now be extended to all or successful artifacts to save on cache overheads. The cli main option '--cache-buildtrees' or the user configuration cache group option 'cache-buildtrees' can be set as 'always', 'auto' or 'never', with the default being 'auto'. Note, as the cache-key for the artifact is independent of the cached build tree input it will remain unaltered, however the availbility of the build tree content may differ. Workspaces ---------- o Due to enabling the use of relative workspaces, "Legacy" workspaces may need to be closed and remade before the changes will affect them. Downgrading after using this feature may result in workspaces not functioning correctly o Opening a workspace now creates a .bstproject.yaml file that allows BuildStream commands to be run from a workspace that is not inside a project. o Specifying an element is now optional for most commands when BuildStream is run from inside a workspace. See CLI reference for more details. o Added Documentation on how to create out of source builds. This includes the new the `conf-root` variable to make the process easier. And there has been a bug fix to workspaces so they can be build in workspaces too. Miscellaneous ------------- o Generate Docker images from built artifacts using `contrib/bst-docker-import` script. ================= buildstream 1.1.5 ================= o Added a `--tar` option to `bst checkout` which allows a tarball to be created from the artifact contents. o Fetching and tracking will consult mirrors defined in project config, and the preferred mirror to fetch from can be defined in the command line or user config. o Added new `remote` source plugin for downloading file blobs o Added support for the new include '(@)' directive in project.conf and .bst files ================= buildstream 1.1.4 ================= o `bst workspace` commands and `bst track` will substitute their source elements when performing those operations, e.g. performing `bst track` on a filter element will track the sources on the element that it depends on (if it has sources). o Added new simple `make` element o Switch to Remote Execution CAS-based artifact cache on all platforms. Artifact servers need to be migrated. o BuildStream now requires python version >= 3.5 o BuildStream will now automatically clean up old artifacts when it runs out of space. The exact behavior is configurable in the user's buildstream.conf. ================= buildstream 1.1.3 ================= o Added new `bst init` command to initialize a new project. o Cross junction tracking is now disabled by default for projects which can support this by using project.refs ref-storage New options have been added to explicitly enable cross-junction tracking. o Failed jobs are now summarised at the end of a build. Use `--verbose` and `--no-verbose` to adjust the amount of detail given. o BuildElements' `configure-commands` are only run once for workspaces now, which allows for incremental builds. Appropriate API for plugins is also exposed through `Element.prepare`. o The `cmake` plugin now supports building with ninja with the newly added `generator` configuration option. o `bst workspace close` and `bst workspace reset` now support multiple elements. All elements can be specified using `--all`. o The elements whose cache keys had to be determined during the build are summarised at the end of the build. o Fixed versioning introspection to be dynamic, many users use a developer install mode so they can update with git, now the version information is always up to date in logs. This causes a minor API break: The --version output now only outputs the version. ================= buildstream 1.1.2 ================= o New ref-storage option allows one to store source refs, such as git shas, in one central project.refs file instead of inline with the source declarations. o Deprecated `--track-save` optionality in `bst build`, this does not make sense to support now that we have project.refs. o Added the `sandbox` configuration option which can be used in `project.conf` and elements, to control the user ID and group ID used in build sandboxes. o Added new `deb` source implementation, for staging of downloaded deb package files. ================= buildstream 1.1.1 ================= o New project configuration controlling how the sandbox behaves when `bst shell` is used; allowing projects to provide a more functional shell environment. o The `bst shell` command now has a `--mount` option allowing users to mount files and directories into the sandbox for testing purposes. o Log lines are now configurable with the new "message-format" user configuration, allowing one to express optional fields such as microsecond precision and wallclock time. o Newly added filter element o Git source plugin now allows disabling of submodule checkouts o In the same way we allow overriding element configurations by their 'kind' in project.conf, we now support the same for source plugin configurations. o Tar and zip sources now automatically recall an `etag` from the http headers, optimizing tracking of tarballs significantly (issue #62) ================= buildstream 1.1.0 ================= o Multiple artifact caches are now supported in project and user configuration with a priority order (issue #85) o Add junction support for subprojects o Changes towards incremental builds in workspaces o `bst shell --build` now creates true build sandbox o Many bug fixes ================= buildstream 1.0.0 ================= First stable release of BuildStream BuildStream 1.0.0 is all about API stability - for the past months we have been reviewing our various API surfaces, implementing strategies for revisioning of our interfaces and cleaning up. Long term stability is very important for build reproducibility over time, and this release is the first promise we are making on any API surfaces. Stable API surfaces include: o The command line interface o The YAML user configuration file format o The YAML project `.bst` file format o The core Python module imported by external plugins apache-buildstream-27ae392/NOTICE000066400000000000000000000026041514607367700165300ustar00rootroot00000000000000Apache BuildStream Copyright 2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). Software deriving from third parties: * doc/bst2html.py This file is largely derived from https://github.com/Kronuz/ansi2html.git which is under the MIT license and as such is acceptable to redistribute under the Apache License. * src/buildstream/_frontend/complete.py This file is forked from the click library which is under BSD license and as such is acceptable to redistribute under the Apache License. * versioneer.py / src/_version.py These files are distributed under public domain and originate from the versioneer project at: https://github.com/python-versioneer/python-versioneer/ * src/buildstream/_protos/google This directory contains software developed by google: https://github.com/googleapis/googleapis/ * src/buildstream/_protos/build/bazel This directory contains software developed by the Bazel authors: https://github.com/bazelbuild/remote-apis/ * src/buildstream/_protos/build/buildgrid This directory contains software developed by the BuildGrid authors: https://gitlab.com/BuildGrid/buildgrid * src/buildstream/_loader/listsort.c This file is derived from Python and licensed under the Python Software Foundation License Version 2. apache-buildstream-27ae392/README.rst000066400000000000000000000125071514607367700173160ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. About ----- .. image:: https://docs.buildstream.build/master/_static/release.svg :target: https://docs.buildstream.build/master/_static/release.html .. image:: https://github.com/apache/buildstream/actions/workflows/merge.yml/badge.svg :alt: GitHub Workflow Status :target: https://github.com/apache/buildstream/actions/workflows/merge.yml .. image:: https://img.shields.io/pypi/v/BuildStream.svg :target: https://pypi.org/project/BuildStream What is BuildStream? ==================== `BuildStream `_ is a powerful software integration tool that allows developers to automate the integration of software components including operating systems, and to streamline the software development and production process. Some key capabilities of BuildStream include: * Defining software stacks in a declarative format: BuildStream allows users to define the steps required to build and integrate software components, including fetching source code and building dependencies. * Integrating with version control systems: BuildStream can be configured to fetch source code from popular source code management solutions such as GitLab, GitHub, BitBucket as well as a range of non-git technologies. * Supporting a wide range of build technologies: BuildStream supports a wide range of technologies, including key programming languages like C, C++, Python, Rust and Java, as well as many build tools including Make, CMake, Meson, distutils, pip and others. * Ability to create outputs in a range of formats: e.g. debian packages, flatpak runtimes, sysroots, system images, for multiple platforms and chipsets. * Flexible architecture: BuildStream is designed to be flexible and extensible, allowing users to customize their build and integration processes to meet their specific needs and tooling. * Enabling fast and reliable software delivery: By extensibly use of sandboxing techniques and by its capability to distribute the build, BuildStream helps teams deliver high-quality software faster. Why should I use BuildStream? ============================= BuildStream offers the following advantages: * **Declarative build instructions/definitions** BuildStream provides a flexible and extensible framework for the modelling of software build pipelines in a declarative YAML format, which allows you to manipulate filesystem data in a controlled, reproducible sandboxed environment. * **Support for developer and integrator workflows** BuildStream provides traceability and reproducibility for integrators handling stacks of hundreds/thousands of components, as well as workspace features and shortcuts to minimise cycle-time for developers. * **Fast and predictable** BuildStream can cache previous builds and track changes to source file content and build/config commands. BuildStream only rebuilds the things that have changed. * **Extensible** You can extend BuildStream to support your favourite build-system. * **Bootstrap toolchains and bootable systems** BuildStream can create full systems and complete toolchains from scratch, for a range of ISAs including x86_32, x86_64, ARMv7, ARMv8, MIPS. How do I use BuildStream? ========================= Please refer to the `documentation `_ for information about installing BuildStream, and about the BuildStream YAML format and plugin options. How does BuildStream work? ========================== BuildStream operates on a set of YAML files (.bst files), as follows: * Loads the YAML files which describe the target(s) and all dependencies. * Evaluates the version information and build instructions to calculate a build graph for the target(s) and all dependencies and unique cache-keys for each element. * Retrieves previously built elements (artifacts) from a local/remote cache, or builds the elements in a sandboxed environment using the instructions declared in the .bst files. * Transforms/configures and/or deploys the resulting target(s) based on the instructions declared in the .bst files. How can I get started? ====================== To get started, first `install BuildStream by following the installation guide `_ and then follow our tutorial in the `user guide `_. Running BuildStream on macOS or Windows may work under Docker Desktop, WSL, or Podman Desktop, but it is not officially supported, and CI runs tests only on Linux. We also recommend exploring some existing BuildStream projects: * https://gitlab.gnome.org/GNOME/gnome-build-meta/ * https://gitlab.com/freedesktop-sdk/freedesktop-sdk * https://gitlab.com/baserock/definitions If you have any questions please ask on our `#buildstream `_ channel in `irc.gnome.org `_ apache-buildstream-27ae392/contrib/000077500000000000000000000000001514607367700172625ustar00rootroot00000000000000apache-buildstream-27ae392/contrib/bst-docker-import000077500000000000000000000052721514607367700225630ustar00rootroot00000000000000#!/bin/bash # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Chandan Singh # This is a helper script to generate Docker images using checkouts of # BuildStream elements. usage() { cat <&2 exit 1 } bst_cmd=bst docker_import_cmd=(docker import) docker_image_tag= while getopts c:m:t:h arg do case $arg in c) bst_cmd="$OPTARG" ;; m) docker_import_cmd+=('-m' "$OPTARG") ;; t) docker_image_tag="$OPTARG" ;; h) usage 0 ;; \?) usage 1 esac done shift $((OPTIND-1)) if [[ "$#" != 1 ]]; then echo "$0: No element specified" >&2 usage 1 fi element="$1" # Dump to a temporary file in the current directory. # NOTE: We use current directory to try to ensure compatibility with scripts # like bst-here, assuming that the current working directory is mounted # inside the container. checkout_tar="bst-checkout-$(basename "$element")-$RANDOM.tar" echo "INFO: Checking out $element ..." >&2 $bst_cmd artifact checkout "$element" --tar "$checkout_tar" || die "Failed to checkout $element" echo "INFO: Successfully checked out $element" >&2 echo "INFO: Importing Docker image ..." >&2 "${docker_import_cmd[@]}" "$checkout_tar" "$docker_image_tag" || die "Failed to import Docker image from tarball" echo "INFO: Successfully import Docker image $docker_image_tag" >&2 echo "INFO: Cleaning up ..." >&2 rm "$checkout_tar" || die "Failed to remove $checkout_tar" echo "INFO: Clean up finished" >&2 apache-buildstream-27ae392/contrib/bst-graph000077500000000000000000000107121514607367700211000ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Chandan Singh # '''Print dependency graph of given element(s) in DOT format. This script must be run from the same directory where you would normally run `bst` commands. When `--format` option is specified, the output will also be rendered in the given format. A file with name `bst-graph.{format}` will be created in the same directory. To use this option, you must have the `graphviz` command line tool installed. ''' import argparse import subprocess import re import urllib.parse from graphviz import Digraph from ruamel.yaml import YAML def parse_args(): '''Handle parsing of command line arguments. Returns: A argparse.Namespace object ''' parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( 'element', nargs='*', help='Name of the element' ) parser.add_argument( '--format', help='Redner the graph in given format (`pdf`, `png`, `svg` etc)' ) parser.add_argument( '--view', action='store_true', help='Open the rendered graph with the default application' ) return parser.parse_args() def unique_node_name(s): '''Generate unique node name for `s`. Graphviz node names cannot contain colons or backslashes so we use url-encoding to generate the unique node name. (A cryptographic hash could be used instead but that would make the graphviz file less readable.) Args: s: element name Returns: A string containing the unique node name ''' return urllib.parse.quote_plus(s) def parse_graph(lines): '''Return nodes and edges of the parsed grpah. Args: lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS' Returns: Tuple of format (nodes,build_deps,runtime_deps) Each member of build_deps and runtime_deps is also a tuple. ''' parser = YAML(typ="safe") nodes = set() build_deps = set() runtime_deps = set() for line in lines: line = line.strip() if not line: continue # It is safe to split on '|' as it is not a valid character for # element names. name, build_dep, runtime_dep = line.split('|') build_dep = parser.load(build_dep) runtime_dep = parser.load(runtime_dep) nodes.add(name) [build_deps.add((name, dep)) for dep in build_dep if dep] [runtime_deps.add((name, dep)) for dep in runtime_dep if dep] return nodes, build_deps, runtime_deps def generate_graph(nodes, build_deps, runtime_deps): '''Generate graph from given nodes and edges. Args: nodes: set of nodes build_deps: set of tuples of build depdencies runtime_deps: set of tuples of runtime depdencies Returns: A graphviz.Digraph object ''' graph = Digraph() for name in nodes: graph.node(unique_node_name(name), label=name) for source, target in build_deps: graph.edge(unique_node_name(source), unique_node_name(target), label='build-dep') for source, target in runtime_deps: graph.edge(unique_node_name(source), unique_node_name(target), label='runtime-dep') return graph def main(): args = parse_args() cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}||'] if 'element' in args: cmd += args.element graph_lines = subprocess.check_output(cmd, universal_newlines=True) # NOTE: We generate nodes and edges before giving them to graphviz as # the library does not de-deuplicate them. nodes, build_deps, runtime_deps = parse_graph(re.split(r"\|\|", graph_lines)) graph = generate_graph(nodes, build_deps, runtime_deps) print(graph.source) if args.format: graph.render(cleanup=True, filename='bst-graph', format=args.format, view=args.view) if __name__ == '__main__': main() apache-buildstream-27ae392/contrib/bst-here000077500000000000000000000070601514607367700207240ustar00rootroot00000000000000#!/bin/bash # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Charles Bailey # Sam Thursfield # This is a helper script for using BuildStream via Docker. See # docs/source/install.rst for documentation. usage() { cat </dev/null done BST_HERE_PS1="\[\033[01;34m\]\w\[\033[00m\]> " if [ "$#" -eq 0 ]; then command="/bin/bash -i" else command="/usr/local/bin/bst $@" fi if "$update" == true then docker pull "$bst_here_image" fi # FIXME: We run with --privileged to allow bwrap to mount system # directories, but this is overkill. We should add the correct # --cap-add calls, or seccomp settings, but we are not sure # what those are yet. # # Old settings: # --cap-add SYS_ADMIN # --security-opt seccomp=unconfined # exec docker run --rm -i${is_tty:+ -t} \ --privileged \ --env PS1="$BST_HERE_PS1" \ --device /dev/fuse \ --volume buildstream-cache:/root/.cache/buildstream \ --volume buildstream-config:/root/.config \ --volume "$PWD":/src \ $extra_volumes_opt \ --workdir /src \ "$bst_here_image" \ $command apache-buildstream-27ae392/doc/000077500000000000000000000000001514607367700163675ustar00rootroot00000000000000apache-buildstream-27ae392/doc/Makefile000066400000000000000000000130251514607367700200300ustar00rootroot00000000000000# Makefile for Sphinx documentation # # Note, due to a problem with python2/python3 parallel # installability of sphinx (https://github.com/sphinx-doc/sphinx/issues/4375) # we dont use the standard `sphinx-build` and `sphinx-apidoc` entry points. # # The following technique works as long as sphinx is installed for python3, # regardless of the entry point which might be in /usr/bin or PATH. # # Since Sphinx 2.0 is planned to be Python 3-only, this workaround should not # be needed once Spinx 2.0 is released, and we upgrade to it # SPHINXOPTS = SPHINXBUILD = python3 -m sphinx SPHINXAPIDOC = python3 -m sphinx.ext.apidoc PAPER = BUILDDIR = build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -W -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source # Set BST_FORCE_SESSION_REBUILD to force rebuild the docs BST2HTML = $(CURDIR)/bst2html.py BST2HTMLOPTS = ifneq ($(strip $(BST_FORCE_SESSION_REBUILD)),) BST2HTMLOPTS = --force endif # Help Python find buildstream and its plugins PYTHONPATH=$(CURDIR)/..:$(CURDIR)/../src/buildstream/plugins .PHONY: all clean templates templates-clean sessions sessions-prep sessions-clean badges badges-clean html devhelp # Canned recipe for generating plugin api skeletons # $1 = the plugin directory # $2 = the output docs directory # # Explanation: # # Sphinx does not have any option for skipping documentation, # we dont want to document plugin code because nobody uses that # but we do want to use module-level docstrings in plugins in # order to explain how plugins work. # # For this purpose, we replace sphinx-apidoc with a simple # makefile rule which generates a template slightly differently # from how sphinx does it, allowing us to get what we want # from plugin documentation. # define plugin-doc-skeleton @for file in $$(find ${1} -name "*.py" ! -name "_*.py"); do \ base=$$(basename $$file); \ module=${2}.$${base%.py}; \ modname=$${base%.py}; \ echo -n "Generating source/${2}/$${modname}.rst... "; \ sed -e "s|@@MODULE@@|$${module}|g" \ source/plugin.rsttemplate > \ source/${2}/$${modname}.rst.tmp && \ mv source/${2}/$${modname}.rst.tmp source/${2}/$${modname}.rst || exit 1; \ echo "Done."; \ done endef all: html devhelp clean: templates-clean sessions-clean badges-clean rm -rf build ############################################################ # Plugin doc templates # ############################################################ # Generate rst templates for the docs using a mix of sphinx-apidoc and # our 'plugin-doc-skeleton' routine for plugin pages. templates: mkdir -p source/elements mkdir -p source/sources $(SPHINXAPIDOC) --force --separate --module-first --no-headings --no-toc -o source $(CURDIR)/../src/buildstream *_pb2*.py $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream/plugins/elements,elements) $(call plugin-doc-skeleton,$(CURDIR)/../src/buildstream/plugins/sources,sources) templates-clean: rm -rf source/elements rm -rf source/sources ############################################################ # Session captures # ############################################################ # Stage the stored sessions into the place where they will # be used in the build. # # This is separated so that the git tree does not become # dirty as a result of a documentation build process - which # messes up the docs version number and the version number # printed in some command line output. # sessions-prep: mkdir -p source/sessions cp source/sessions-stored/*.html source/sessions # bst2html is called unconditionally for every session file in SESSION_FILES. # # By default, this will generate the html fragments of colorized BuildStream terminal # output only if the output html files don't yet exist. # # Specify BST_FORCE_SESSION_REBUILD=1 to force rebuild all session html files. # SESSION_FILES=$(wildcard sessions/*.run) $(SESSION_FILES): sessions-prep %.run: PYTHONPATH=$(PYTHONPATH) $(BST2HTML) $(BST2HTMLOPTS) $@ sessions: $(SESSION_FILES) sessions-clean: rm -rf source/sessions ############################################################ # Generate release badges and their redirecting html pages # ############################################################ badges-clean: rm -rf source/badges badges: mkdir -p source/badges $(CURDIR)/badges.py > source/badges/release.svg $(CURDIR)/badges.py --redirect > source/badges/release.html ############################################################ # Main sphinx build # ############################################################ # Targets which generate docs with sphinx build # # html devhelp: templates sessions badges @echo "Building $@..." PYTHONPATH=$(PYTHONPATH) \ $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \ $(wildcard source/*.rst) \ $(wildcard source/tutorial/*.rst) \ $(wildcard source/developing/*.rst) \ $(wildcard source/handling-files/*.rst) \ $(wildcard source/junctions/*.rst) \ $(wildcard source/examples/*.rst) \ $(wildcard source/elements/*.rst) \ $(wildcard source/sources/*.rst) \ $(wildcard source/hacking/*.rst) @echo @echo "Build of $@ finished, output: $(CURDIR)/$(BUILDDIR)/$@" # Makefile for Sphinx documentation # apache-buildstream-27ae392/doc/badges.py000077500000000000000000000105141514607367700201720ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # import click import subprocess import re # The badge template is modeled after the gitlab badge svgs # BADGE_TEMPLATE = """ {badge_name} {badge_name} {version} {version} """ # The redirect template is for a static html page hosted in the # latest docs which always redirects you to the latest release on github. # REDIRECT_TEMPLATE = """ Latest {badge_name}

""" URL_FORMAT = 'https://github.com/apache/buildstream/releases/tag/{version}' RELEASE_COLOR = '#0040FF' VERSION_TAG_MATCH = r'([0-9]*)\.([0-9]*)\.([0-9]*)' # Parse a release tag and return a three tuple # of the major, minor and micro version. # # Tags which do not follow the release tag format # will just be returned as (0, 0, 0) # def parse_tag(tag): match = re.search(VERSION_TAG_MATCH, tag) if match: major = match.group(1) minor = match.group(2) micro = match.group(3) return (int(major), int(minor), int(micro)) return (0, 0, 0) # Call out to git and guess the latest version, # this will just return (0, 0, 0) in case of any error. # def guess_version(): try: tags_output = subprocess.check_output(['git', 'tag']) except subprocess.CalledProcessError: return (0, 0, 0) # Parse the `git tag` output into a list of integer tuples tags_output = tags_output.decode('UTF-8') all_tags = tags_output.splitlines() all_versions = [parse_tag(tag) for tag in all_tags] # Make sure they are sorted, and take the last one sorted_versions = sorted(all_versions) latest_version = sorted_versions[-1] return latest_version @click.command(short_help="Generate the version badges") @click.option('--redirect', is_flag=True, default=False, help="Whether to generate the redirect html file") def generate_badges(redirect): """Generate the version badge svg files """ major, minor, micro = guess_version() badge_name = 'release' color = RELEASE_COLOR version = '{major}.{minor}.{micro}'.format(major=major, minor=minor, micro=micro) url_target = URL_FORMAT.format(version=version) if redirect: template = REDIRECT_TEMPLATE else: template = BADGE_TEMPLATE output = template.format(badge_name=badge_name, version=version, color=color, url_target=url_target) click.echo(output, nl=False) return 0 if __name__ == '__main__': generate_badges() apache-buildstream-27ae392/doc/bst2html.py000077500000000000000000000417661514607367700205210ustar00rootroot00000000000000#!/usr/bin/env python3 # # Copyright (c) 2013 German M. Bravo (Kronuz) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # # This file is substantially based on German's work, obtained at: # https://github.com/Kronuz/ansi2html.git # import os import sys import re import shlex import subprocess from contextlib import contextmanager from tempfile import TemporaryDirectory import click from buildstream import _yaml from buildstream import utils from buildstream._exceptions import BstError _ANSI2HTML_STYLES = {} ANSI2HTML_CODES_RE = re.compile('(?:\033\\[(\d+(?:;\d+)*)?([cnRhlABCDfsurgKJipm]))') ANSI2HTML_PALETTE = { # See http://ethanschoonover.com/solarized 'solarized': ['#073642', '#D30102', '#859900', '#B58900', '#268BD2', '#D33682', '#2AA198', '#EEE8D5', '#002B36', '#CB4B16', '#586E75', '#657B83', '#839496', '#6C71C4', '#93A1A1', '#FDF6E3'], # Above mapped onto the xterm 256 color palette 'solarized-xterm': ['#262626', '#AF0000', '#5F8700', '#AF8700', '#0087FF', '#AF005F', '#00AFAF', '#E4E4E4', '#1C1C1C', '#D75F00', '#585858', '#626262', '#808080', '#5F5FAF', '#8A8A8A', '#FFFFD7'], # Gnome default: 'tango': ['#000000', '#CC0000', '#4E9A06', '#C4A000', '#3465A4', '#75507B', '#06989A', '#D3D7CF', '#555753', '#EF2929', '#8AE234', '#FCE94F', '#729FCF', '#AD7FA8', '#34E2E2', '#EEEEEC'], # xterm: 'xterm': ['#000000', '#CD0000', '#00CD00', '#CDCD00', '#0000EE', '#CD00CD', '#00CDCD', '#E5E5E5', '#7F7F7F', '#FF0000', '#00FF00', '#FFFF00', '#5C5CFF', '#FF00FF', '#00FFFF', '#FFFFFF'], 'console': ['#000000', '#AA0000', '#00AA00', '#AA5500', '#0000AA', '#AA00AA', '#00AAAA', '#AAAAAA', '#555555', '#FF5555', '#55FF55', '#FFFF55', '#5555FF', '#FF55FF', '#55FFFF', '#FFFFFF'], } def _ansi2html_get_styles(palette): if palette not in _ANSI2HTML_STYLES: p = ANSI2HTML_PALETTE.get(palette, ANSI2HTML_PALETTE['console']) regular_style = { '1': '', # bold '2': 'opacity:0.5', '4': 'text-decoration:underline', '5': 'font-weight:bold', '7': '', '8': 'display:none', } bold_style = regular_style.copy() for i in range(8): regular_style['3%s' % i] = 'color:%s' % p[i] regular_style['4%s' % i] = 'background-color:%s' % p[i] bold_style['3%s' % i] = 'color:%s' % p[i + 8] bold_style['4%s' % i] = 'background-color:%s' % p[i + 8] # The default xterm 256 colour p: indexed_style = {} for i in range(16): indexed_style['%s' % i] = p[i] for rr in range(6): for gg in range(6): for bb in range(6): i = 16 + rr * 36 + gg * 6 + bb r = (rr * 40 + 55) if rr else 0 g = (gg * 40 + 55) if gg else 0 b = (bb * 40 + 55) if bb else 0 indexed_style['%s' % i] = ''.join('%02X' % c if 0 <= c <= 255 else None for c in (r, g, b)) for g in range(24): i = g + 232 L = g * 10 + 8 indexed_style['%s' % i] = ''.join('%02X' % c if 0 <= c <= 255 else None for c in (L, L, L)) _ANSI2HTML_STYLES[palette] = (regular_style, bold_style, indexed_style) return _ANSI2HTML_STYLES[palette] def ansi2html(text, palette='solarized'): def _ansi2html(m): if m.group(2) != 'm': return '' import sys state = None sub = '' cs = m.group(1) cs = cs.strip() if cs else '' for c in cs.split(';'): c = c.strip().lstrip('0') or '0' if c == '0': while stack: sub += '' stack.pop() elif c in ('38', '48'): extra = [c] state = 'extra' elif state == 'extra': if c == '5': state = 'idx' elif c == '2': state = 'r' elif state: if state == 'idx': extra.append(c) state = None # 256 colors color = indexed_style.get(c) # TODO: convert index to RGB! if color is not None: sub += '' % ('color' if extra[0] == '38' else 'background-color', color) stack.append(extra) elif state in ('r', 'g', 'b'): extra.append(c) if state == 'r': state = 'g' elif state == 'g': state = 'b' else: state = None try: color = '#' + ''.join( '%02X' % c if 0 <= c <= 255 else None for x in extra for c in [int(x)] ) except (ValueError, TypeError): pass else: sub += ''.format( 'color' if extra[0] == '38' else 'background-color', color) stack.append(extra) else: if '1' in stack: style = bold_style.get(c) else: style = regular_style.get(c) if style is not None: sub += '' % style # Still needs to be added to the stack even if style is empty # (so it can check '1' in stack above, for example) stack.append(c) return sub stack = [] regular_style, bold_style, indexed_style = _ansi2html_get_styles(palette) sub = ANSI2HTML_CODES_RE.sub(_ansi2html, text) while stack: sub += '' stack.pop() return sub # workdir() # # Sets up a new temp directory with a config file # # Args: # work_directory (str): The directory where to create a tempdir first # source_cache (str): The directory of a source cache to share with, or None # # Yields: # The buildstream.conf full path # @contextmanager def workdir(source_cache=None): with TemporaryDirectory(prefix='run-bst-', dir=os.getcwd()) as tempdir: if not source_cache: source_cache = os.path.join(tempdir, 'sources') bst_config_file = os.path.join(tempdir, 'buildstream.conf') config = { 'cachedir': tempdir, 'sourcedir': source_cache, 'logdir': os.path.join(tempdir, 'logs'), } _yaml.roundtrip_dump(config, bst_config_file) yield (tempdir, bst_config_file, source_cache) # run_bst_command() # # Runs a command # # Args: # config_file (str): The path to the config file to use # directory (str): The project directory # command (str): A command string # # Returns: # (str): The colorized combined stdout/stderr of BuildStream # def run_bst_command(config_file, directory, command): click.echo("Running bst command in directory '{}': bst {}".format(directory, command), err=True) argv = ['python3', '-m', 'buildstream', '--colors', '--config', config_file] + shlex.split(command) try: out = subprocess.check_output(argv, cwd=directory, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: click.echo("Command failed:\n{}".format(e.output.decode('utf-8').strip())) sys.exit(1) return out.decode('utf-8').strip() # run_shell_command() # # Runs a command # # Args: # directory (str): The project directory # command (str): A shell command # # Returns: # (str): The combined stdout/stderr of the shell command # def run_shell_command(directory, command): click.echo("Running shell command in directory '{}': {}".format(directory, command), err=True) argv = shlex.split(command) p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = p.communicate() return out.decode('utf-8').strip() # generate_html # # Generate html based on the output # # Args: # output (str): The output of the BuildStream command # directory (str): The project directory # config_file (str): The config file # source_cache (str): The source cache # tempdir (str): The base work directory # palette (str): The rendering color style # command (str): The command # fake_output (bool): Whether the provided output is faked or not # # Returns: # (str): The html formatted output # def generate_html(output, directory, config_file, source_cache, tempdir, palette, command, fake_output): test_base_name = os.path.basename(directory) if fake_output: show_command = command else: show_command = 'bst ' + command # Substitute some things we want normalized for the docs output = re.sub(os.environ.get('HOME'), '/home/user', output) output = re.sub(config_file, '/home/user/.config/buildstream.conf', output) output = re.sub(source_cache, '/home/user/.cache/buildstream/sources', output) output = re.sub(tempdir, '/home/user/.cache/buildstream', output) output = re.sub(directory, '/home/user/{}'.format(test_base_name), output) # Now convert to HTML and add some surrounding sugar output = ansi2html(output, palette=palette) # Finally format it nicely into a
final_output = '\n' + \ '
' + \ '
\n' + \
                   'user@host:' + \
                   '~/{}$ '.format(test_base_name) + \
                   show_command + '\n'

    if output:
        final_output += '\n' + output + '\n'

    final_output += '
\n' return final_output # check_needs_build() # # Checks whether filename, specified relative to basedir, # needs to be built (based on whether it exists). # # Args: # basedir (str): The base directory to check relative of, or None for CWD # filename (str): The basedir relative path to the file # force (bool): Whether force rebuilding of existing things is enabled # # Returns: # (bool): Whether the file needs to be built # def check_needs_build(basedir, filename, force=False): if force: return True if basedir is None: basedir = os.getcwd() filename = os.path.join(basedir, filename) filename = os.path.realpath(filename) if not os.path.exists(filename): return True return False def run_session(description, tempdir, source_cache, palette, config_file, force): desc = _yaml.load(description, shortname=os.path.basename(description)) desc_dir = os.path.dirname(description) # Preflight commands and check if we can skip this session # if not force: needs_build = False commands = desc.get_sequence('commands') for command in commands: output = command.get_str('output', default=None) if output is not None and check_needs_build(desc_dir, output, force=False): needs_build = True break if not needs_build: click.echo("Skipping '{}' as no files need to be built".format(description), err=True) return # FIXME: Workaround a setuptools bug where the symlinks # we store in git dont get carried into a release # tarball. This workaround lets us build docs from # a source distribution tarball. # symlinks = desc.get_mapping('workaround-symlinks', default={}) for symlink, target in symlinks.items(): target = target.as_str() # Resolve real path to where symlink should be symlink = os.path.join(desc_dir, symlink) # Ensure dir exists symlink_dir = os.path.dirname(symlink) os.makedirs(symlink_dir, exist_ok=True) click.echo("Generating symlink at: {} (target: {})".format(symlink, target), err=True) # Generate a symlink try: os.symlink(target, symlink) except FileExistsError: # If the files exist, we're running from a git checkout and # not a source distribution, no need to complain pass remove_files = desc.get_str_list('remove-files', default=[]) for remove_file in remove_files: remove_file = os.path.join(desc_dir, remove_file) remove_file = os.path.realpath(remove_file) if os.path.isdir(remove_file): utils._force_rmtree(remove_file) else: utils.safe_remove(remove_file) # Run commands # commands = desc.get_sequence('commands') for command in commands: # Get the directory where this command should be run directory = command.get_str('directory') directory = os.path.join(desc_dir, directory) directory = os.path.realpath(directory) # Get the command string command_str = command.get_str('command') # Check whether this is a shell command and not a bst command is_shell = command.get_bool('shell', default=False) # Check if there is fake output command_fake_output = command.get_str('fake-output', default=None) # Run the command, or just use the fake output if command_fake_output is None: if is_shell: command_out = run_shell_command(directory, command_str) else: command_out = run_bst_command(config_file, directory, command_str) else: command_out = command_fake_output # Encode and save the output if that was asked for output = command.get_str('output', default=None) if output is not None: # Convert / Generate a nice
converted = generate_html(command_out, directory, config_file, source_cache, tempdir, palette, command_str, command_fake_output is not None) # Save it filename = os.path.join(desc_dir, output) filename = os.path.realpath(filename) output_dir = os.path.dirname(filename) os.makedirs(output_dir, exist_ok=True) with open(filename, 'wb') as f: f.write(converted.encode('utf-8')) click.echo("Saved session at '{}'".format(filename), err=True) @click.command(short_help="Run a bst command and capture stdout/stderr in html") @click.option('--directory', '-C', type=click.Path(file_okay=False, dir_okay=True), help="The project directory where to run the command") @click.option('--force', is_flag=True, default=False, help="Force rebuild, even if the file exists") @click.option('--source-cache', type=click.Path(file_okay=False, dir_okay=True), help="A shared source cache") @click.option('--palette', '-p', default='tango', type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']), help="Selects a palette for the output style") @click.argument('description', type=click.Path(file_okay=True, dir_okay=False, readable=True)) def run_bst(directory, force, source_cache, description, palette): """Run a bst command and capture stdout/stderr in html This command normally takes a description yaml file, see the CONTRIBUTING file for information on its format. """ if not source_cache and os.environ.get('BST_SOURCE_CACHE'): source_cache = os.environ['BST_SOURCE_CACHE'] with workdir(source_cache=source_cache) as (tempdir, config_file, source_cache): run_session(description, tempdir, source_cache, palette, config_file, force) return 0 if __name__ == '__main__': try: run_bst() except BstError as e: click.echo("Error: {}".format(e), err=True) sys.exit(-1) apache-buildstream-27ae392/doc/examples/000077500000000000000000000000001514607367700202055ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/autotools/000077500000000000000000000000001514607367700222365ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/autotools/elements/000077500000000000000000000000001514607367700240525ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/autotools/elements/base.bst000066400000000000000000000001001514607367700254650ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/autotools/elements/base/000077500000000000000000000000001514607367700247645ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/autotools/elements/base/alpine.bst000066400000000000000000000004611514607367700267470ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar # This is a post doctored, trimmed down system image # of the Alpine linux distribution. # url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 apache-buildstream-27ae392/doc/examples/autotools/elements/hello.bst000066400000000000000000000007011514607367700256650ustar00rootroot00000000000000kind: autotools description: | Hello world example from automake variables: # The hello world example lives in the doc/amhello folder. # # Set the %{command-subdir} variable to that location # and just have the autotools element run its commands there. # command-subdir: doc/amhello sources: - kind: tar url: gnu:automake-1.16.tar.gz ref: 80da43bb5665596ee389e6d8b64b4f122ea4b92a685b1dbd813cd1f0e0c2d83f depends: - base.bst apache-buildstream-27ae392/doc/examples/autotools/project.conf000066400000000000000000000006601514607367700245550ustar00rootroot00000000000000# Unique project name name: autotools # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define some aliases for the tarballs we download aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ gnu: http://ftpmirror.gnu.org/gnu/automake/ plugins: - origin: pip package-name: buildstream-plugins elements: - autotools apache-buildstream-27ae392/doc/examples/composition/000077500000000000000000000000001514607367700225505ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/composition/elements/000077500000000000000000000000001514607367700243645ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/composition/elements/base.bst000066400000000000000000000001001514607367700257770ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/composition/elements/base/000077500000000000000000000000001514607367700252765ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/composition/elements/base/alpine.bst000066400000000000000000000013451514607367700272630ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 public: bst: # # Run ldconfig in the libdir before running anything # integration-commands: - ldconfig "%{libdir}" # # Extend the runtime split-rule domain for this element, # such that we capture the runtime linker. # # There are various other things provided by this runtime # such as tooling in /bin and an installation of python # and perl, but we'll overlook these for the sake of # this example. # split-rules: runtime: (>): - "/lib/ld*.so*" apache-buildstream-27ae392/doc/examples/composition/elements/hello.bst000066400000000000000000000005761514607367700262110ustar00rootroot00000000000000kind: manual description: | The hello application # Depend on the hello library depends: - libhello.bst # Stage the files/hello directory for building sources: - kind: local path: files/hello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/composition/elements/libhello.bst000066400000000000000000000005751514607367700266770ustar00rootroot00000000000000kind: manual description: | The libhello library # Depend on the base system depends: - base.bst # Stage the files/libhello directory for building sources: - kind: local path: files/libhello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/composition/elements/runtime-only.bst000066400000000000000000000006551514607367700275460ustar00rootroot00000000000000kind: compose # Dependencies of a compose element cannot be transient, # we can only build-depend on the inputs of a composition. # build-depends: - hello.bst config: # Only include files from the runtime domain # include: - runtime # Don't include any files which do not match any existing # split rule domains. # include-orphans: False # Run integration commands before composition # integrate: True apache-buildstream-27ae392/doc/examples/composition/files/000077500000000000000000000000001514607367700236525ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/composition/files/hello/000077500000000000000000000000001514607367700247555ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/composition/files/hello/Makefile000066400000000000000000000003121514607367700264110ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) $< -o $@ -Wall -lhello apache-buildstream-27ae392/doc/examples/composition/files/hello/hello.c000066400000000000000000000004111514607367700262200ustar00rootroot00000000000000/* * hello.c - Simple hello program */ #include #include int main(int argc, char *argv[]) { const char *person = NULL; if (argc > 1) person = argv[1]; if (person) hello(person); else hello("stranger"); return 0; } apache-buildstream-27ae392/doc/examples/composition/files/libhello/000077500000000000000000000000001514607367700254445ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/composition/files/libhello/Makefile000066400000000000000000000005451514607367700271100ustar00rootroot00000000000000# Sample makefile for hello library # .PHONY: all install all: libhello.so install: install -d ${DESTDIR}${PREFIX}/lib install -d ${DESTDIR}${PREFIX}/include install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib install -m 644 libhello.h ${DESTDIR}${PREFIX}/include %.o: %.c %.h $(CC) -c $< -o $@ -Wall libhello.so: libhello.o $(CC) -shared -o $@ $< apache-buildstream-27ae392/doc/examples/composition/files/libhello/libhello.c000066400000000000000000000002001514607367700273720ustar00rootroot00000000000000/* * libhello.c - The hello library */ #include void hello(const char *person) { printf("Hello %s\n", person); } apache-buildstream-27ae392/doc/examples/composition/files/libhello/libhello.h000066400000000000000000000001671514607367700274130ustar00rootroot00000000000000/* * libhello.h - The hello library */ /* * A function to say hello to @person */ void hello(const char *person); apache-buildstream-27ae392/doc/examples/composition/project.conf000066400000000000000000000004441514607367700250670ustar00rootroot00000000000000# Unique project name name: composition # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ apache-buildstream-27ae392/doc/examples/developing/000077500000000000000000000000001514607367700223415ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/developing/elements/000077500000000000000000000000001514607367700241555ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/developing/elements/base.bst000066400000000000000000000001001514607367700255700ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/developing/elements/base/000077500000000000000000000000001514607367700250675ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/developing/elements/base/alpine.bst000066400000000000000000000004611514607367700270520ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar # This is a post doctored, trimmed down system image # of the Alpine linux distribution. # url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 apache-buildstream-27ae392/doc/examples/developing/elements/hello.bst000066400000000000000000000005601514607367700257730ustar00rootroot00000000000000kind: manual description: | Building manually # Depend on the base system depends: - base.bst # Stage the files/src directory for building sources: - kind: local path: files/src # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/developing/files/000077500000000000000000000000001514607367700234435ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/developing/files/src/000077500000000000000000000000001514607367700242325ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/developing/files/src/Makefile000066400000000000000000000003021514607367700256650ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) -Wall -o $@ $< apache-buildstream-27ae392/doc/examples/developing/files/src/hello.c000066400000000000000000000002171514607367700255010ustar00rootroot00000000000000/* * hello.c - Simple hello world program */ #include int main(int argc, char *argv[]) { printf("Hello World\n"); return 0; } apache-buildstream-27ae392/doc/examples/developing/project.conf000066400000000000000000000004441514607367700246600ustar00rootroot00000000000000# Unique project name name: developing # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ apache-buildstream-27ae392/doc/examples/developing/update.patch000066400000000000000000000003661514607367700246510ustar00rootroot00000000000000-- hello.c 2018-06-25 14:48:32.077568920 +0100 +++ hello.c 2018-06-25 14:49:23.025553785 +0100 @@ -5,6 +5,6 @@ int main(int argc, char *argv[]) { - printf("Hello World\n"); + printf("Hello World\nWe can use workspaces!\n"); return 0; } apache-buildstream-27ae392/doc/examples/directives/000077500000000000000000000000001514607367700223465ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/directives/elements/000077500000000000000000000000001514607367700241625ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/directives/elements/base.bst000066400000000000000000000001001514607367700255750ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/directives/elements/base/000077500000000000000000000000001514607367700250745ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/directives/elements/base/alpine.bst000066400000000000000000000004611514607367700270570ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar # This is a post doctored, trimmed down system image # of the Alpine linux distribution. # url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 apache-buildstream-27ae392/doc/examples/directives/elements/hello.bst000066400000000000000000000011211514607367700257720ustar00rootroot00000000000000kind: manual description: | A hello world program with a custom greeting message # Depend on the base system depends: - base.bst # Stage the files/src directory for building sources: - kind: local path: files/src # This include file defines the %{greeting} variable used below variables: (@): include/greeting.bst # Now configure the commands to run config: # This time we inform the Makefile of which greeting we want build-commands: - make PREFIX="%{prefix}" GREETING="%{greeting}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/directives/files/000077500000000000000000000000001514607367700234505ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/directives/files/src/000077500000000000000000000000001514607367700242375ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/directives/files/src/Makefile000066400000000000000000000003471514607367700257030ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) -DGREETING_MESSAGE="\"${GREETING}\"" -Wall -o $@ $< apache-buildstream-27ae392/doc/examples/directives/files/src/hello.c000066400000000000000000000002201514607367700255000ustar00rootroot00000000000000/* * hello.c - Simple hello world program */ #include int main(int argc, char *argv[]) { printf(GREETING_MESSAGE); return 0; } apache-buildstream-27ae392/doc/examples/directives/include/000077500000000000000000000000001514607367700237715ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/directives/include/greeting.bst000066400000000000000000000004021514607367700263030ustar00rootroot00000000000000# We define the greeting message here conditionally (?): - flavor == "normal": greeting: | Hello world ! - flavor == "somber": greeting: | Hey world. - flavor == "excited": greeting: | Howdy there, and what a world it is ! apache-buildstream-27ae392/doc/examples/directives/project.conf000066400000000000000000000007671514607367700246750ustar00rootroot00000000000000# Unique project name name: directives # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ # Define an option for this project # options: flavor: type: enum description: Flavor of the greeting in the hello world program values: - normal - somber - excited default: normal apache-buildstream-27ae392/doc/examples/filtering/000077500000000000000000000000001514607367700221705ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/filtering/elements/000077500000000000000000000000001514607367700240045ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/filtering/elements/base.bst000066400000000000000000000001001514607367700254170ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/filtering/elements/base/000077500000000000000000000000001514607367700247165ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/filtering/elements/base/alpine.bst000066400000000000000000000005151514607367700267010ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 # # Run ldconfig in the libdir before running anything # public: bst: integration-commands: - ldconfig "%{libdir}" apache-buildstream-27ae392/doc/examples/filtering/elements/hello.bst000066400000000000000000000010001514607367700256100ustar00rootroot00000000000000kind: manual description: | The hello application # Depend on the hello library, or the filtered version # (?): - use_filter == True: depends: - libhello-filtered.bst - use_filter == False: depends: - libhello.bst # Stage the files/hello directory for building sources: - kind: local path: files/hello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/filtering/elements/libhello-filtered.bst000066400000000000000000000004451514607367700301070ustar00rootroot00000000000000kind: filter description: | A filtered version of libhello which excludes the defaults # Specify the build dependency to filter build-depends: - libhello.bst # Propagate runtime dependencies runtime-depends: - base.bst # Now configure the commands to run config: exclude: - defaults apache-buildstream-27ae392/doc/examples/filtering/elements/libhello.bst000066400000000000000000000011171514607367700263100ustar00rootroot00000000000000kind: manual description: | The libhello library # Depend on the base system depends: - base.bst # Stage the files/libhello directory for building sources: - kind: local path: files/libhello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install public: bst: # Define a split domain which captures the defaults # which this library installs into %{datadir} # split-rules: defaults: - "%{datadir}/libhello/default-person.txt" apache-buildstream-27ae392/doc/examples/filtering/files/000077500000000000000000000000001514607367700232725ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/filtering/files/hello/000077500000000000000000000000001514607367700243755ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/filtering/files/hello/Makefile000066400000000000000000000007721514607367700260430ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c extra_flags=""; \ if [ -f "${PREFIX}/share/libhello/default-person.txt" ]; then \ extra_flags=-DDEFAULT_PERSON="\"$$(cat ${PREFIX}/share/libhello/default-person.txt)\""; \ fi; \ $(CC) $< -o $@ $${extra_flags} -Wall -lhello apache-buildstream-27ae392/doc/examples/filtering/files/hello/hello.c000066400000000000000000000005151514607367700256450ustar00rootroot00000000000000/* * hello.c - Simple hello program */ #include #include int main(int argc, char *argv[]) { const char *person = NULL; if (argc > 1) person = argv[1]; if (person) hello(person); else { #ifdef DEFAULT_PERSON hello(DEFAULT_PERSON); #else hello("stranger"); #endif } return 0; } apache-buildstream-27ae392/doc/examples/filtering/files/libhello/000077500000000000000000000000001514607367700250645ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/filtering/files/libhello/Makefile000066400000000000000000000007321514607367700265260ustar00rootroot00000000000000# Sample makefile for hello library # .PHONY: all install all: libhello.so install: install -d ${DESTDIR}${PREFIX}/lib install -d ${DESTDIR}${PREFIX}/include install -d ${DESTDIR}${PREFIX}/share/libhello install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib install -m 644 libhello.h ${DESTDIR}${PREFIX}/include install -m 644 default-person.txt ${DESTDIR}${PREFIX}/share/libhello %.o: %.c %.h $(CC) -c $< -o $@ -Wall libhello.so: libhello.o $(CC) -shared -o $@ $< apache-buildstream-27ae392/doc/examples/filtering/files/libhello/default-person.txt000066400000000000000000000000071514607367700305520ustar00rootroot00000000000000Sophia apache-buildstream-27ae392/doc/examples/filtering/files/libhello/libhello.c000066400000000000000000000002001514607367700270120ustar00rootroot00000000000000/* * libhello.c - The hello library */ #include void hello(const char *person) { printf("Hello %s\n", person); } apache-buildstream-27ae392/doc/examples/filtering/files/libhello/libhello.h000066400000000000000000000001671514607367700270330ustar00rootroot00000000000000/* * libhello.h - The hello library */ /* * A function to say hello to @person */ void hello(const char *person); apache-buildstream-27ae392/doc/examples/filtering/project.conf000066400000000000000000000007311514607367700245060ustar00rootroot00000000000000# Unique project name name: filtering # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ # Use an option to decide if we should use the filter # options: use_filter: type: bool description: Whether to use a filter on the libhello.bst element default: False apache-buildstream-27ae392/doc/examples/first-project/000077500000000000000000000000001514607367700230005ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/first-project/elements/000077500000000000000000000000001514607367700246145ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/first-project/elements/hello.bst000066400000000000000000000003441514607367700264320ustar00rootroot00000000000000kind: import # Use a local source to stage our file sources: - kind: local path: hello.world # Configure the import element config: # Place the content staged by sources at the # root of the output artifact target: / apache-buildstream-27ae392/doc/examples/first-project/hello.world000066400000000000000000000000001514607367700251420ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/first-project/project.conf000066400000000000000000000002341514607367700253140ustar00rootroot00000000000000# Unique project name name: first-project # Required BuildStream version min-version: 2.3 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/doc/examples/integration-commands/000077500000000000000000000000001514607367700243275ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/integration-commands/elements/000077500000000000000000000000001514607367700261435ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/integration-commands/elements/base.bst000066400000000000000000000001001514607367700275560ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/integration-commands/elements/base/000077500000000000000000000000001514607367700270555ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/integration-commands/elements/base/alpine.bst000066400000000000000000000005151514607367700310400ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 # # Run ldconfig in the libdir before running anything # public: bst: integration-commands: - ldconfig "%{libdir}" apache-buildstream-27ae392/doc/examples/integration-commands/elements/hello.bst000066400000000000000000000005761514607367700277700ustar00rootroot00000000000000kind: manual description: | The hello application # Depend on the hello library depends: - libhello.bst # Stage the files/hello directory for building sources: - kind: local path: files/hello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/integration-commands/elements/libhello.bst000066400000000000000000000005751514607367700304560ustar00rootroot00000000000000kind: manual description: | The libhello library # Depend on the base system depends: - base.bst # Stage the files/libhello directory for building sources: - kind: local path: files/libhello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/integration-commands/files/000077500000000000000000000000001514607367700254315ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/integration-commands/files/hello/000077500000000000000000000000001514607367700265345ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/integration-commands/files/hello/Makefile000066400000000000000000000003121514607367700301700ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) $< -o $@ -Wall -lhello apache-buildstream-27ae392/doc/examples/integration-commands/files/hello/hello.c000066400000000000000000000004111514607367700277770ustar00rootroot00000000000000/* * hello.c - Simple hello program */ #include #include int main(int argc, char *argv[]) { const char *person = NULL; if (argc > 1) person = argv[1]; if (person) hello(person); else hello("stranger"); return 0; } apache-buildstream-27ae392/doc/examples/integration-commands/files/libhello/000077500000000000000000000000001514607367700272235ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/integration-commands/files/libhello/Makefile000066400000000000000000000005451514607367700306670ustar00rootroot00000000000000# Sample makefile for hello library # .PHONY: all install all: libhello.so install: install -d ${DESTDIR}${PREFIX}/lib install -d ${DESTDIR}${PREFIX}/include install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib install -m 644 libhello.h ${DESTDIR}${PREFIX}/include %.o: %.c %.h $(CC) -c $< -o $@ -Wall libhello.so: libhello.o $(CC) -shared -o $@ $< apache-buildstream-27ae392/doc/examples/integration-commands/files/libhello/libhello.c000066400000000000000000000002001514607367700311510ustar00rootroot00000000000000/* * libhello.c - The hello library */ #include void hello(const char *person) { printf("Hello %s\n", person); } apache-buildstream-27ae392/doc/examples/integration-commands/files/libhello/libhello.h000066400000000000000000000001671514607367700311720ustar00rootroot00000000000000/* * libhello.h - The hello library */ /* * A function to say hello to @person */ void hello(const char *person); apache-buildstream-27ae392/doc/examples/integration-commands/project.conf000066400000000000000000000004551514607367700266500ustar00rootroot00000000000000# Unique project name name: integration-commands # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ apache-buildstream-27ae392/doc/examples/junction-includes/000077500000000000000000000000001514607367700236425ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junction-includes/elements/000077500000000000000000000000001514607367700254565ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junction-includes/elements/hello.bst000066400000000000000000000010311514607367700272660ustar00rootroot00000000000000kind: autotools description: | Hello world example from automake variables: # The special paths.bst from our subproject is used to # define the paths of some elements in this project. # (@): subproject-junction.bst:include/paths.bst # The hello world example lives in the doc/amhello folder. command-subdir: doc/amhello sources: - kind: tar url: gnu:automake-1.16.tar.gz ref: 80da43bb5665596ee389e6d8b64b4f122ea4b92a685b1dbd813cd1f0e0c2d83f depends: - filename: base.bst junction: subproject-junction.bst apache-buildstream-27ae392/doc/examples/junction-includes/elements/subproject-junction.bst000066400000000000000000000004411514607367700321760ustar00rootroot00000000000000kind: junction config: # Configure the options for subproject # # If our project is funky, then it requires # a blue subproject, otherwise we use a red one. # options: color: red (?): - funky == True: color: blue sources: - kind: local path: subproject apache-buildstream-27ae392/doc/examples/junction-includes/project.conf000066400000000000000000000005211514607367700261550ustar00rootroot00000000000000name: project min-version: 2.0 element-path: elements aliases: gnu: http://ftpmirror.gnu.org/gnu/automake/ plugins: - origin: pip package-name: buildstream-plugins elements: - autotools # Define some options for this project # options: funky: type: bool description: Whether this project is funky default: False apache-buildstream-27ae392/doc/examples/junction-includes/subproject/000077500000000000000000000000001514607367700260225ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junction-includes/subproject/elements/000077500000000000000000000000001514607367700276365ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junction-includes/subproject/elements/base.bst000066400000000000000000000001001514607367700312510ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/junction-includes/subproject/elements/base/000077500000000000000000000000001514607367700305505ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junction-includes/subproject/elements/base/alpine.bst000066400000000000000000000004611514607367700325330ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar # This is a post doctored, trimmed down system image # of the Alpine linux distribution. # url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 apache-buildstream-27ae392/doc/examples/junction-includes/subproject/include/000077500000000000000000000000001514607367700274455ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junction-includes/subproject/include/paths.bst000066400000000000000000000002361514607367700312770ustar00rootroot00000000000000# When this project color is blue, including this # file causes installations to be relocated to /opt # prefix: /usr (?): - color == "blue": prefix: /opt apache-buildstream-27ae392/doc/examples/junction-includes/subproject/project.conf000066400000000000000000000005071514607367700303410ustar00rootroot00000000000000name: subproject min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ # Define some options for this project # options: color: type: enum description: The color of this runtime values: - red - green - blue default: blue apache-buildstream-27ae392/doc/examples/junctions/000077500000000000000000000000001514607367700222215ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junctions/autotools/000077500000000000000000000000001514607367700242525ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junctions/autotools/elements/000077500000000000000000000000001514607367700260665ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junctions/autotools/elements/base.bst000066400000000000000000000001001514607367700275010ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/junctions/autotools/elements/base/000077500000000000000000000000001514607367700270005ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junctions/autotools/elements/base/alpine.bst000066400000000000000000000004611514607367700307630ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar # This is a post doctored, trimmed down system image # of the Alpine linux distribution. # url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 apache-buildstream-27ae392/doc/examples/junctions/autotools/elements/hello.bst000066400000000000000000000007011514607367700277010ustar00rootroot00000000000000kind: autotools description: | Hello world example from automake variables: # The hello world example lives in the doc/amhello folder. # # Set the %{command-subdir} variable to that location # and just have the autotools element run its commands there. # command-subdir: doc/amhello sources: - kind: tar url: gnu:automake-1.16.tar.gz ref: 80da43bb5665596ee389e6d8b64b4f122ea4b92a685b1dbd813cd1f0e0c2d83f depends: - base.bst apache-buildstream-27ae392/doc/examples/junctions/autotools/project.conf000066400000000000000000000007321514607367700265710ustar00rootroot00000000000000# Unique project name name: autotools # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define some aliases for the tarballs we download aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ gnu: https://ftpmirror.gnu.org/gnu/automake/ # Declare the plugins we're going to use plugins: - origin: pip package-name: buildstream-plugins elements: - autotools apache-buildstream-27ae392/doc/examples/junctions/elements/000077500000000000000000000000001514607367700240355ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junctions/elements/callHello.bst000066400000000000000000000002031514607367700264410ustar00rootroot00000000000000kind: import sources: - kind: local path: files/callHello.sh depends: - filename: hello.bst junction: hello-junction.bst apache-buildstream-27ae392/doc/examples/junctions/elements/hello-junction.bst000066400000000000000000000000701514607367700274760ustar00rootroot00000000000000kind: junction sources: - kind: local path: autotools apache-buildstream-27ae392/doc/examples/junctions/files/000077500000000000000000000000001514607367700233235ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/junctions/files/callHello.sh000066400000000000000000000000461514607367700255560ustar00rootroot00000000000000#!/bin/sh echo "Calling hello:" hello apache-buildstream-27ae392/doc/examples/junctions/project.conf000066400000000000000000000002401514607367700245320ustar00rootroot00000000000000# Unique project name name: junctions # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/doc/examples/overlaps/000077500000000000000000000000001514607367700220405ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/overlaps/elements/000077500000000000000000000000001514607367700236545ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/overlaps/elements/base.bst000066400000000000000000000001001514607367700252670ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/overlaps/elements/base/000077500000000000000000000000001514607367700245665ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/overlaps/elements/base/alpine.bst000066400000000000000000000013451514607367700265530ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 public: bst: # # Run ldconfig in the libdir before running anything # integration-commands: - ldconfig "%{libdir}" # # Extend the runtime split-rule domain for this element, # such that we capture the runtime linker. # # There are various other things provided by this runtime # such as tooling in /bin and an installation of python # and perl, but we'll overlook these for the sake of # this example. # split-rules: runtime: (>): - "/lib/ld*.so*" apache-buildstream-27ae392/doc/examples/overlaps/elements/hello.bst000066400000000000000000000005761514607367700255010ustar00rootroot00000000000000kind: manual description: | The hello application # Depend on the hello library depends: - libhello.bst # Stage the files/hello directory for building sources: - kind: local path: files/hello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/overlaps/elements/libhello.bst000066400000000000000000000005751514607367700261670ustar00rootroot00000000000000kind: manual description: | The libhello library # Depend on the base system depends: - base.bst # Stage the files/libhello directory for building sources: - kind: local path: files/libhello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/overlaps/elements/runtime-only.bst000066400000000000000000000006551514607367700270360ustar00rootroot00000000000000kind: compose # Dependencies of a compose element cannot be transient, # we can only build-depend on the inputs of a composition. # build-depends: - hello.bst config: # Only include files from the runtime domain # include: - runtime # Don't include any files which do not match any existing # split rule domains. # include-orphans: False # Run integration commands before composition # integrate: True apache-buildstream-27ae392/doc/examples/overlaps/files/000077500000000000000000000000001514607367700231425ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/overlaps/files/hello/000077500000000000000000000000001514607367700242455ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/overlaps/files/hello/Makefile000066400000000000000000000004541514607367700257100ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -d ${DESTDIR}${PREFIX}/share/doc install -m 755 hello ${DESTDIR}${PREFIX}/bin install -m 644 hello.txt ${DESTDIR}${PREFIX}/share/doc hello: hello.c $(CC) $< -o $@ -Wall -lhello apache-buildstream-27ae392/doc/examples/overlaps/files/hello/hello.c000066400000000000000000000004111514607367700255100ustar00rootroot00000000000000/* * hello.c - Simple hello program */ #include #include int main(int argc, char *argv[]) { const char *person = NULL; if (argc > 1) person = argv[1]; if (person) hello(person); else hello("stranger"); return 0; } apache-buildstream-27ae392/doc/examples/overlaps/files/hello/hello.txt000066400000000000000000000000631514607367700261100ustar00rootroot00000000000000This is the documentation about the hello program. apache-buildstream-27ae392/doc/examples/overlaps/files/libhello/000077500000000000000000000000001514607367700247345ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/overlaps/files/libhello/Makefile000066400000000000000000000007071514607367700264000ustar00rootroot00000000000000# Sample makefile for hello library # .PHONY: all install all: libhello.so install: install -d ${DESTDIR}${PREFIX}/lib install -d ${DESTDIR}${PREFIX}/include install -d ${DESTDIR}${PREFIX}/share/doc install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib install -m 644 libhello.h ${DESTDIR}${PREFIX}/include install -m 644 hello.txt ${DESTDIR}${PREFIX}/share/doc %.o: %.c %.h $(CC) -c $< -o $@ -Wall libhello.so: libhello.o $(CC) -shared -o $@ $< apache-buildstream-27ae392/doc/examples/overlaps/files/libhello/hello.txt000066400000000000000000000000631514607367700265770ustar00rootroot00000000000000This is the documentation about the hello library. apache-buildstream-27ae392/doc/examples/overlaps/files/libhello/libhello.c000066400000000000000000000002001514607367700266620ustar00rootroot00000000000000/* * libhello.c - The hello library */ #include void hello(const char *person) { printf("Hello %s\n", person); } apache-buildstream-27ae392/doc/examples/overlaps/files/libhello/libhello.h000066400000000000000000000001671514607367700267030ustar00rootroot00000000000000/* * libhello.h - The hello library */ /* * A function to say hello to @person */ void hello(const char *person); apache-buildstream-27ae392/doc/examples/overlaps/project.conf000066400000000000000000000004411514607367700243540ustar00rootroot00000000000000# Unique project name name: overlaps # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ apache-buildstream-27ae392/doc/examples/running-commands/000077500000000000000000000000001514607367700234645ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/running-commands/elements/000077500000000000000000000000001514607367700253005ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/running-commands/elements/base.bst000066400000000000000000000001001514607367700267130ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/running-commands/elements/base/000077500000000000000000000000001514607367700262125ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/running-commands/elements/base/alpine.bst000066400000000000000000000004611514607367700301750ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar # This is a post doctored, trimmed down system image # of the Alpine linux distribution. # url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 apache-buildstream-27ae392/doc/examples/running-commands/elements/hello.bst000066400000000000000000000005601514607367700271160ustar00rootroot00000000000000kind: manual description: | Building manually # Depend on the base system depends: - base.bst # Stage the files/src directory for building sources: - kind: local path: files/src # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/running-commands/files/000077500000000000000000000000001514607367700245665ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/running-commands/files/src/000077500000000000000000000000001514607367700253555ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/running-commands/files/src/Makefile000066400000000000000000000003021514607367700270100ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) -Wall -o $@ $< apache-buildstream-27ae392/doc/examples/running-commands/files/src/hello.c000066400000000000000000000002171514607367700266240ustar00rootroot00000000000000/* * hello.c - Simple hello world program */ #include int main(int argc, char *argv[]) { printf("Hello World\n"); return 0; } apache-buildstream-27ae392/doc/examples/running-commands/project.conf000066400000000000000000000004511514607367700260010ustar00rootroot00000000000000# Unique project name name: running-commands # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ apache-buildstream-27ae392/doc/examples/strict-mode/000077500000000000000000000000001514607367700224375ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/strict-mode/elements/000077500000000000000000000000001514607367700242535ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/strict-mode/elements/base.bst000066400000000000000000000001001514607367700256660ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/alpine.bst apache-buildstream-27ae392/doc/examples/strict-mode/elements/base/000077500000000000000000000000001514607367700251655ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/strict-mode/elements/base/alpine.bst000066400000000000000000000005151514607367700271500ustar00rootroot00000000000000kind: import description: | Alpine Linux base runtime sources: - kind: tar url: alpine:integration-tests-base.v1.x86_64.tar.xz ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 # # Run ldconfig in the libdir before running anything # public: bst: integration-commands: - ldconfig "%{libdir}" apache-buildstream-27ae392/doc/examples/strict-mode/elements/hello-dynamic.bst000066400000000000000000000006711514607367700275160ustar00rootroot00000000000000kind: manual description: | The dynamically linked hello application # Depend on the hello library depends: - libhello.bst # Stage the files/hello directory for building sources: - kind: local path: files/hello # Now configure the commands to run config: build-commands: - make -f Makefile.dynamic PREFIX="%{prefix}" install-commands: - make -f Makefile.dynamic -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/strict-mode/elements/hello-static.bst000066400000000000000000000007501514607367700273570ustar00rootroot00000000000000kind: manual description: | The statically linked hello application # Depend on the hello library with the strict option # depends: - filename: libhello.bst strict: true # Stage the files/hello directory for building sources: - kind: local path: files/hello # Now configure the commands to run config: build-commands: - make -f Makefile.static PREFIX="%{prefix}" install-commands: - make -f Makefile.static -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/strict-mode/elements/libhello.bst000066400000000000000000000005751514607367700265660ustar00rootroot00000000000000kind: manual description: | The libhello library # Depend on the base system depends: - base.bst # Stage the files/libhello directory for building sources: - kind: local path: files/libhello # Now configure the commands to run config: build-commands: - make PREFIX="%{prefix}" install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install apache-buildstream-27ae392/doc/examples/strict-mode/files/000077500000000000000000000000001514607367700235415ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/strict-mode/files/hello/000077500000000000000000000000001514607367700246445ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/strict-mode/files/hello/Makefile.dynamic000066400000000000000000000003161514607367700277270ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: all install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) -Wall -o $@ $< -lhello apache-buildstream-27ae392/doc/examples/strict-mode/files/hello/Makefile.static000066400000000000000000000003321514607367700275700ustar00rootroot00000000000000# Sample makefile for hello.c # .PHONY: all install all: hello install: all install -d ${DESTDIR}${PREFIX}/bin install -m 755 hello ${DESTDIR}${PREFIX}/bin hello: hello.c $(CC) -Wall -o $@ $< /usr/lib/libhello.a apache-buildstream-27ae392/doc/examples/strict-mode/files/hello/hello.c000066400000000000000000000004111514607367700261070ustar00rootroot00000000000000/* * hello.c - Simple hello program */ #include #include int main(int argc, char *argv[]) { const char *person = NULL; if (argc > 1) person = argv[1]; if (person) hello(person); else hello("stranger"); return 0; } apache-buildstream-27ae392/doc/examples/strict-mode/files/libhello/000077500000000000000000000000001514607367700253335ustar00rootroot00000000000000apache-buildstream-27ae392/doc/examples/strict-mode/files/libhello/Makefile000066400000000000000000000007201514607367700267720ustar00rootroot00000000000000# Sample makefile for hello library # .PHONY: all install all: libhello.so libhello.a install: all install -d ${DESTDIR}${PREFIX}/lib install -d ${DESTDIR}${PREFIX}/include install -m 644 libhello.so ${DESTDIR}${PREFIX}/lib install -m 644 libhello.a ${DESTDIR}${PREFIX}/lib install -m 644 libhello.h ${DESTDIR}${PREFIX}/include %.o: %.c %.h $(CC) -c $< -o $@ -Wall libhello.a: libhello.o $(AR) rcs $@ $^ libhello.so: libhello.o $(CC) -shared -o $@ $< apache-buildstream-27ae392/doc/examples/strict-mode/files/libhello/libhello.c000066400000000000000000000002071514607367700272700ustar00rootroot00000000000000/* * libhello.c - The hello library */ #include void hello(const char *person) { printf("Good morning %s\n", person); } apache-buildstream-27ae392/doc/examples/strict-mode/files/libhello/libhello.h000066400000000000000000000001671514607367700273020ustar00rootroot00000000000000/* * libhello.h - The hello library */ /* * A function to say hello to @person */ void hello(const char *person); apache-buildstream-27ae392/doc/examples/strict-mode/project.conf000066400000000000000000000004441514607367700247560ustar00rootroot00000000000000# Unique project name name: strict-mode # Minimum required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Define an alias for our alpine tarball aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ apache-buildstream-27ae392/doc/examples/strict-mode/update.patch000066400000000000000000000002371514607367700247440ustar00rootroot00000000000000--- libhello.c +++ libhello.c @@ -5,5 +5,5 @@ void hello(const char *person) { - printf("Hello %s\n", person); + printf("Good morning %s\n", person); } apache-buildstream-27ae392/doc/sessions/000077500000000000000000000000001514607367700202355ustar00rootroot00000000000000apache-buildstream-27ae392/doc/sessions/autotools.run000066400000000000000000000011021514607367700230060ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/autotools command: source fetch hello.bst # Capture a `bst show` of the variables - directory: ../examples/autotools output: ../source/sessions/autotools-show-variables.html command: show --deps none --format "%{vars}" hello.bst # Capture a `bst build` - directory: ../examples/autotools output: ../source/sessions/autotools-build.html command: build hello.bst # Capture a shell output - directory: ../examples/autotools output: ../source/sessions/autotools-shell.html command: shell hello.bst -- hello apache-buildstream-27ae392/doc/sessions/composition.run000066400000000000000000000011101514607367700233170ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/composition command: source fetch hello.bst # Capture the build output - directory: ../examples/composition output: ../source/sessions/composition-build.html command: build runtime-only.bst # List the contents - directory: ../examples/composition output: ../source/sessions/composition-list-contents.html command: artifact list-contents runtime-only.bst # Run hello world - directory: ../examples/composition output: ../source/sessions/composition-shell.html command: shell runtime-only.bst -- hello audience apache-buildstream-27ae392/doc/sessions/developing.run000066400000000000000000000035771514607367700231330ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/developing/ command: source fetch hello.bst # Capture workspace open output - directory: ../examples/developing/ output: ../source/sessions/developing-workspace-open.html command: workspace open --directory workspace_hello hello.bst # Catpure output from workspace list - directory: ../examples/developing/ output: ../source/sessions/developing-workspace-list.html command: workspace list # Apply a patch in the workspace - directory: ../examples/developing/ shell: True command: patch workspace_hello/hello.c update.patch # Rebuild - directory: ../examples/developing/ output: ../source/sessions/developing-build-after-changes.html command: build hello.bst # Rebuild, from the workspace - directory: ../examples/developing/workspace_hello output: ../source/sessions/developing-build-after-changes-workspace.html command: build # Capture shell output with changes - directory: ../examples/developing/ output: ../source/sessions/developing-shell-after-changes.html command: shell hello.bst -- hello # Soft Reset - directory: ../examples/developing/ output: ../source/sessions/developing-soft-reset.html command: workspace reset --soft hello.bst # Close workspace - directory: ../examples/developing/ output: ../source/sessions/developing-close-workspace.html command: workspace close hello.bst # Reopen workspace - directory: ../examples/developing/ output: ../source/sessions/developing-reopen-workspace.html command: workspace open --no-checkout --directory workspace_hello hello.bst # Reset workspace - directory: ../examples/developing/ output: ../source/sessions/developing-reset-workspace.html command: workspace reset hello.bst # Discard workspace - directory: ../examples/developing/ output: ../source/sessions/developing-discard-workspace.html command: workspace close --remove-dir hello.bst apache-buildstream-27ae392/doc/sessions/directives.run000066400000000000000000000021241514607367700231230ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/directives command: source fetch hello.bst # Capture a build output - directory: ../examples/directives output: ../source/sessions/directives-build-normal.html command: build hello.bst # Capture a build output - directory: ../examples/directives output: ../source/sessions/directives-build-somber.html command: --option flavor somber build hello.bst # Capture a build output - directory: ../examples/directives output: ../source/sessions/directives-build-excited.html command: --option flavor excited build hello.bst # Capture a shell output - directory: ../examples/directives output: ../source/sessions/directives-shell-normal.html command: shell hello.bst -- hello # Capture a shell output - directory: ../examples/directives output: ../source/sessions/directives-shell-somber.html command: --option flavor somber shell hello.bst -- hello # Capture a shell output - directory: ../examples/directives output: ../source/sessions/directives-shell-excited.html command: --option flavor excited shell hello.bst -- hello apache-buildstream-27ae392/doc/sessions/filtering.run000066400000000000000000000020651514607367700227510ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/filtering command: source fetch hello.bst # Capture a build output - directory: ../examples/filtering command: --option use_filter False build hello.bst # Capture a build output - directory: ../examples/filtering command: --option use_filter True build hello.bst # Capture list-contents output - directory: ../examples/filtering output: ../source/sessions/filtering-list-contents-libhello.html command: artifact list-contents libhello.bst # Capture list-contents output - directory: ../examples/filtering output: ../source/sessions/filtering-list-contents-libhello-filtered.html command: artifact list-contents libhello-filtered.bst # Capture shell output - directory: ../examples/filtering output: ../source/sessions/filtering-shell-without-filter.html command: --option use_filter False shell hello.bst -- hello # Capture shell output - directory: ../examples/filtering output: ../source/sessions/filtering-shell-with-filter.html command: --option use_filter True shell hello.bst -- hello apache-buildstream-27ae392/doc/sessions/first-project.run000066400000000000000000000021541514607367700235600ustar00rootroot00000000000000# Re-create project.conf using `bst init` remove-files: - ../examples/first-project/project.conf - ../examples/first-project/here commands: # Use bst init to create the project.conf - directory: ../examples/first-project output: ../source/sessions/first-project-init.html command: init --project-name first-project # Use bst init to create the project.conf - directory: ../examples/first-project output: ../source/sessions/first-project-touch.html command: touch hello.world fake-output: '' # Capture a build output - directory: ../examples/first-project output: ../source/sessions/first-project-build.html command: build hello.bst # Capture a show output - directory: ../examples/first-project output: ../source/sessions/first-project-show.html command: show hello.bst # Checkout the output - directory: ../examples/first-project output: ../source/sessions/first-project-checkout.html command: artifact checkout --directory here hello.bst # Checkout the output - directory: ../examples/first-project output: ../source/sessions/first-project-ls.html command: ls ./here fake-output: hello.world apache-buildstream-27ae392/doc/sessions/integration-commands.run000066400000000000000000000006771514607367700251170ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/integration-commands command: source fetch hello.bst # Capture a build output - directory: ../examples/integration-commands output: ../source/sessions/integration-commands-build.html command: build hello.bst # Capture a shell output - directory: ../examples/integration-commands output: ../source/sessions/integration-commands-shell.html command: shell hello.bst -- hello pony apache-buildstream-27ae392/doc/sessions/junction-includes.run000066400000000000000000000015011514607367700244150ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/junction-includes command: source fetch hello.bst # Build hello.bst normally - directory: ../examples/junction-includes output: ../source/sessions/junction-includes-build-normal.html command: build hello.bst # Build hello.bst funky - directory: ../examples/junction-includes output: ../source/sessions/junction-includes-build-funky.html command: --option funky True build hello.bst # Run hello.bst in a shell - directory: ../examples/junction-includes output: ../source/sessions/junction-includes-shell-normal.html command: shell hello.bst -- /usr/bin/hello # Run hello.bst in a shell - directory: ../examples/junction-includes output: ../source/sessions/junction-includes-shell-funky.html command: --option funky True shell hello.bst -- /opt/bin/hello apache-buildstream-27ae392/doc/sessions/junctions.run000066400000000000000000000017201514607367700227770ustar00rootroot00000000000000 commands: # Build callHello.bst - directory: ../examples/junctions output: ../source/sessions/junctions-build.html command: build callHello.bst # Run callHello.sh in a shell - directory: ../examples/junctions output: ../source/sessions/junctions-shell.html command: shell callHello.bst -- /bin/sh callHello.sh # Open a workspace on the junction: - directory: ../examples/junctions output: ../source/sessions/junctions-workspace-open-subproject.html command: workspace open --directory workspace_subproject hello-junction.bst # Open a crossJunction workspace: - directory: ../examples/junctions output: ../source/sessions/junctions-workspace-open.html command: workspace open --directory workspace_hello hello-junction.bst:hello.bst # Remove the workspaces - directory: ../examples/junctions command: workspace close --remove-dir hello-junction.bst - directory: ../examples/junctions command: workspace close --remove-dir hello-junction.bst:hello.bst apache-buildstream-27ae392/doc/sessions/overlaps.run000066400000000000000000000003771514607367700226250ustar00rootroot00000000000000 commands: # Build everything up to `hello.bst` - directory: ../examples/overlaps command: build hello.bst # Capture the build output - directory: ../examples/overlaps output: ../source/sessions/overlaps-build.html command: build runtime-only.bst apache-buildstream-27ae392/doc/sessions/running-commands.run000066400000000000000000000013431514607367700242430ustar00rootroot00000000000000 commands: # Make it fetch first - directory: ../examples/running-commands command: source fetch hello.bst # Capture a show output - directory: ../examples/running-commands output: ../source/sessions/running-commands-show-before.html command: show hello.bst # Capture a build output - directory: ../examples/running-commands output: ../source/sessions/running-commands-build.html command: build hello.bst # Capture another show output - directory: ../examples/running-commands output: ../source/sessions/running-commands-show-after.html command: show hello.bst # Capture a shell output - directory: ../examples/running-commands output: ../source/sessions/running-commands-shell.html command: shell hello.bst -- hello apache-buildstream-27ae392/doc/sessions/strict-mode.run000066400000000000000000000042101514607367700232120ustar00rootroot00000000000000 commands: # Build both static and dynamic versions - directory: ../examples/strict-mode command: build hello-static.bst hello-dynamic.bst # Show all of the elements - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-show-initial.html command: show hello-static.bst hello-dynamic.bst # Capture workspace open output - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-workspace-open.html command: workspace open --directory workspace_libhello libhello.bst # Apply a patch in the workspace - directory: ../examples/strict-mode shell: True command: patch workspace_libhello/libhello.c update.patch # Show hello-dynamic.bst with a modified library - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-show-dynamic-strict.html command: show hello-dynamic.bst # Show hello-dynamic.bst with a modified library, without strict - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-show-dynamic-no-strict.html command: --no-strict show hello-dynamic.bst # Show hello-static.bst with a modified library, without strict - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-show-static-no-strict.html command: --no-strict show hello-static.bst # Build hello-dynamic.bst without strict - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-build-dynamic-no-strict.html command: --no-strict build hello-dynamic.bst # Run hello-dynamic.bst without strict - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-run-dynamic-no-strict.html command: --no-strict shell hello-dynamic.bst -- hello # Build hello-static.bst without strict - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-build-static-no-strict.html command: --no-strict build hello-static.bst # Run hello-static.bst without strict - directory: ../examples/strict-mode output: ../source/sessions/strict-mode-run-static-no-strict.html command: --no-strict shell hello-static.bst -- hello # Discard workspace - directory: ../examples/strict-mode command: workspace close --remove-dir libhello.bst apache-buildstream-27ae392/doc/source/000077500000000000000000000000001514607367700176675ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/CONTRIBUTING.rst000077700000000000000000000000001514607367700254122../../CONTRIBUTING.rstustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/additional_docker.rst000066400000000000000000000045721514607367700240700ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _bst_and_docker: BuildStream and Docker ====================== BuildStream integrates with Docker in multiple ways. Here are some ways in which these integrations work. Run BuildStream inside Docker ----------------------------- Refer to the `BuildStream inside Docker `_ documentation for instructions on how to run BuildStream as a Docker container. Generate Docker images ---------------------- The `bst-docker-import script `_ can be used to generate a Docker image from built artifacts. You can download it and make it executable like this: .. code:: bash mkdir -p ~/.local/bin curl --get https://raw.githubusercontent.com/apache/buildstream/master/contrib/bst-docker-import > ~/.local/bin/bst-docker-import chmod +x ~/.local/bin/bst-docker-import Check if ``~/.local/bin`` appears in your PATH environment variable -- if it doesn't, you should `edit your ~/.profile so that it does `_. Once the script is available in your PATH and assuming you have Docker installed, you can start using the ``bst-docker-import`` script. Here is a minimal example to generate an image called ``bst-hello`` from an element called ``hello.bst`` assuming it is already built: .. code:: bash bst-docker-import -t bst-hello hello.bst This script can also be used if you are running BuildStream inside Docker. In this case, you will need to supply the command that you are using to run BuildStream using the ``-c`` option. If you are using the `bst-here wrapper script `_, you can achieve the same results as the above example like this: .. code:: bash bst-docker-import -c bst-here -t bst-hello hello.bst apache-buildstream-27ae392/doc/source/arch_cachekeys.rst000066400000000000000000000126111514607367700233560ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _cachekeys: Cache keys ========== Cache keys for artifacts are generated from the inputs of the build process for the purpose of reusing artifacts in a well-defined, predictable way. Structure --------- Cache keys are SHA256 hash values generated from a UTF-8 JSON document that includes: * Environment (e.g., project configuration and variables). * Element configuration (details depend on element kind, ``Element.get_unique_key()``). * Sources (``Source.get_unique_key()``). * Dependencies (depending on cache key type, see below). * Public data. Cache key types --------------- There are two types of cache keys in BuildStream, ``strong`` and ``weak``. The purpose of a ``strong`` cache key is to capture the state of as many aspects as possible that can have an influence on the build output. The aim is that builds will be fully reproducible as long as the cache key doesn't change, with suitable module build systems that don't embed timestamps, for example. A ``strong`` cache key includes the strong cache key of each build dependency (and their runtime dependencies) of the element as changes in build dependencies (or their runtime dependencies) can result in build differences in reverse dependencies. This means that whenever the strong cache key of a dependency changes, the strong cache key of its reverse dependencies will change as well. A ``weak`` cache key has an almost identical structure, however, it includes only the names of build dependencies, not their cache keys or their runtime dependencies. A weak cache key will thus still change when the element itself or the environment changes but it will not change when a dependency is updated. For elements without build dependencies the ``strong`` cache key is identical to the ``weak`` cache key. Note that dependencies which are not required at build time do not affect either kind of key. Strict build plan ----------------- This is the default build plan that exclusively uses ``strong`` cache keys for the core functionality. An element's cache key can be calculated when the cache keys of the element's build dependencies (and their runtime dependencies) have been calculated and either tracking is not enabled or it has already completed for this element, i.e., the ``ref`` is available. This means that with tracking disabled the cache keys of all elements could be calculated right at the start of a build session. While BuildStream only uses ``strong`` cache keys with the strict build plan for the actual staging and build process, it will still calculate ``weak`` cache keys for each element. This allows BuildStream to store the artifact in the cache with both keys, reducing rebuilds when switching between strict and non-strict build plans. If the artifact cache already contains an artifact with the same ``weak`` cache key, it's replaced. Thus, non-strict builds always use the latest artifact available for a given ``weak`` cache key. Non-strict build plan --------------------- The non-strict build plan disables the time-consuming automatic rebuild of reverse dependencies at the cost of dropping the reproducibility benefits. It uses the ``weak`` cache keys for the core staging and build process. I.e., if an artifact is available with the calculated ``weak`` cache key, it will be reused for staging instead of being rebuilt. ``weak`` cache keys can be calculated early in the build session. After tracking, similar to when ``strong`` cache keys can be calculated with a strict build plan. Similar to how strict build plans also calculate ``weak`` cache keys, non-strict build plans also calculate ``strong`` cache keys. However, this is slightly more complex. To calculate the ``strong`` cache key of an element, BuildStream requires the ``strong`` cache keys of the build dependencies (and their runtime dependencies). The build dependencies of an element may have been updated since the artifact was built. With the non-strict build plan the artifact will still be reused. However, this means that we cannot use a ``strong`` cache key calculated purely based on the element definitions. We need a cache key that matches the environment at the time the artifact was built, not the current definitions. The only way to get the correct ``strong`` cache key is by retrieving it from the metadata stored in the artifact. As artifacts may need to be pulled from a remote artifact cache, the ``strong`` cache key is not readily available early in the build session. However, it can always be retrieved when an element is about to be built, as the dependencies are guaranteed to be in the local artifact cache at that point. ``Element._get_cache_key_from_artifact()`` extracts the ``strong`` cache key from an artifact in the local cache. ``Element._get_cache_key_for_build()`` calculates the ``strong`` cache key that is used for a particular build job. This is used for the embedded metadata and also as key to store the artifact in the cache. apache-buildstream-27ae392/doc/source/arch_caches.rst000066400000000000000000000065051514607367700226520ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _caches: Caches ====== BuildStream uses local caches to avoid repeating work, and can have remote caches configured to allow the results of work to be shared between multiple users. There are caches for both elements and sources that map keys to relevant metadata and point to data in CAS. Content Addressable Storage (CAS) --------------------------------- The majority of data is stored in Content Addressable Storage or CAS, which indexes stored files by the SHA256 hash of their contents. This allows for a flat file structure as well as any repeated data to be shared across a CAS. In order to store directory structures BuildStream's CAS uses `protocol buffers`_ for storing directory and file information as defined in Google's `REAPI`_. The data itself is stored in CAS which is defined by the `remote execution protocol`_, and BuildStream also uses the `remote asset protocol`_ in order to address stored content using symbolic labels, such as :ref:`artifact names ` for artifacts. Artifact caches --------------- Artifacts store build results of an element which is then referred to by its cache key (described in :ref:`cachekeys`). The artifacts information is then stored in a protocol buffer, defined in ``artifact.proto``, which includes metadata such as the digest of the files root; strong and weak keys; and log files digests. The digests point to locations in the CAS of relavant files and directories, allowing BuildStream to query remote CAS servers for this information. Source caches ------------- Sources are cached by running the :mod:`Source.stage ` method and capturing the directory output of this into the CAS, which then use the sources key to refer to this. The source key will be calculated with the plugins defined :mod:`Plugin.get_unique_key ` and, depending on whether the source requires previous sources to be staged (e.g. the patch plugin), the unique key of all sources listed before it in an element. Source caches are simpler than artifacts, as they just need to map a source key to a directory digest, with no additional metadata. .. note:: Not all plugins use the same result as the staged output for workspaces. As a result when initialising a workspace, BuildStream may require fetching the original source if it only has the source in the source cache. .. _protocol buffers: https://developers.google.com/protocol-buffers/docs/overview .. _grpc: https://grpc.io .. _REAPI: https://github.com/bazelbuild/remote-apis .. _remote execution protocol: https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/remote_execution.proto .. _remote asset protocol: https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/asset/v1/remote_asset.proto apache-buildstream-27ae392/doc/source/arch_data_model.rst000066400000000000000000000150161514607367700235120ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Data model ========== This section details the data model on which the BuildStream core operates. This includes an overview of the project data model which is BuildStream's main input, the user preferences, and local state. Project ------- The ``Project`` object is the main component of a given BuildStream *project*, and is responsible for loading and validating the :ref:`project.conf `, and providing this loaded *project data* in a convenient way to the BuildStream core. Conceptually, the *project* is a container for the :mod:`Elements `, which are declared within a user's project, and as such acts as a factory for instantiating elements at load time. Element ------- :mod:`Elements ` are the main processing unit in a pipeline. These are the loaded representation of the ``.bst`` files loaded from the :ref:`project's element path `. The *Element* is an abstract base class which cannot do anything on its own, its concrete class is defined by *plugins* which are either included in the BuildStream :ref:`core set of plugins ` or loaded from external sources :ref:`defined by the project `. The responsibilities of an element include: * Loading the element's configuration from the core provided dictionary. * Providing a unique key for any element specific configuration which might affect the output produced by the element. * Configuring the sandbox. * Staging the data into the sandbox, which might include Sources and the outputs of previous elements. * Assembling the output *artifact*. Element data structure ~~~~~~~~~~~~~~~~~~~~~~ The properties of an element are a composition of what the BuildStream core understands, the configurations exposed by the Element plugin, and free form data which allows annotations and configurations which can be read back by reverse dependencies during processing, as illustrated here: .. image:: images/arch-datamodel-element.svg :align: center Element composition ~~~~~~~~~~~~~~~~~~~ The element is composed of configurations which are sourced from various entry points using the low level YAML utilities. This composition takes place after :ref:`includes ` and :ref:`conditional ` directives are processed, while :ref:`list composition ` directives are processed as a result of this composition. Here is a diagram showing which sources take precedence in the composition process which results in the final element configuration being resolved: .. image:: images/arch-datamodel-element-composition.svg :align: center Note that not all *BuildStream Core Data* is understood by the *Element*, but a great deal of configurations understood by the *Element* is also understood by the core and has default configurations built into BuildStream and configurable with the project configuration. These include values such as *variables*, *environment*, *sandbox*, etc. As shown above, composition is performed in two stages for each element. First we compose everything below the line, this happens just once per 'kind' of element - the result is re-used. Secondly, we compose the element declaration on top. Source ------ :mod:`Sources ` are the abstract objects which are responsible for obtaining remote source code or data to import into the build environment, and ensuring that it is done in a bit-for-bit reproducible way without any contamination of the host or build environment. This is to say that: * User configuration on the host, or filesystem outside of BuildStream designated directories, must never be modified as a side-effect of running BuildStream. * When the Source uses host tools, host side configurations must never result in deviations of what is staged to a build directory. The Source must behave exactly the same way regardless of host side configurations. The responsibilities of a source include: * Loading the source's configuration from the core provided dictionary. * Providing a unique key for any source specific configuration which might affect the staged source. * Implement discovery of new versions of the source upstream (referred to as *"tracking"*). * Staging the unpacked source to a given directory. * Preparing workspaces. Source data structure ~~~~~~~~~~~~~~~~~~~~~ Similar to the *Element*, the properties of a source are a composition of what the BuildStream core understands and the configurations exposed by the Source plugin: .. image:: images/arch-datamodel-source.svg :align: center .. note:: In .bst files, the BuildStream core configurations and Source specific configurations share the same dictionary. Strictly speaking this is limiting, but provides a measure of convenience as .bst files are a bit less wordy to express. Source composition ~~~~~~~~~~~~~~~~~~ Source composition is much simpler than Element composition, because defaults cannot be specified at the project level, excepting for Source type specific value overrides. .. image:: images/arch-datamodel-source-composition.svg :align: center Context ------- The Context object is a very centric part of the BuildStream data model, and is not a part of the Project data described above but rather is where we load and store all of the user preferences. User preferences are sourced from various locations, but usually have a default, an option in the user configuration file, and an option to override it on the command line. .. image:: images/arch-datamodel-context.svg :align: center Asides from being a focal point for loading and storing all user configuration, the Context object also plays a central role in the logging framework. Workspaces ---------- The Workspaces object is yet another kind of state. Unlike the Context and the Project data model, the Workspaces object loads, saves and stores in memory the local state regarding a user's active and open workspaces. These are stored in the local state ``.bst/`` subdirectory of users projects. apache-buildstream-27ae392/doc/source/arch_dependency_model.rst000066400000000000000000000067521514607367700247260ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Dependency model ================ Elements in the data model are related by their *dependencies*. In BuildStream, there are two types of relationship that an Element may have with a *dependency*, :ref:`build and runtime dependencies `. More often than not, an element will require its dependency both to *build* and also at *runtime*. Consider a simple build scenario where you want to build an application, which requires a service be present in order to function properly at *runtime*, a compiler that need only be present at *build time*, and a runtime environment or base system which must be required all the time: .. image:: images/arch-dependency-model.svg Note that in BuildStream we are only concerned with element level granularity in our dependency model, and there is no way to depend on only a part of an element's output *artifact*. Instead we can employ :mod:`compose ` and :mod:`filter ` elements in conjunction with :ref:`split rules ` to achieve sub artifact granularity at build and deploy time. When developing BuildStream, it is important to understand the distinction between dependency types and element :class:`Scope `, which acts as a selector of which elements to consider in the dependency graph of a given element when performing recursive activities. Scope ~~~~~ * **Scope.ALL** In the :func:`Scope.ALL ` scope, all elements are considered. This is used in some cases to forcefully fetch, pull or build all dependencies of a given element, even when not all of them are needed. This scope simply includes all of the dependencies, including the element itself. * **Scope.RUN** In the :func:`Scope.RUN ` scope, only elements which are required to run are considered, including the element itself. Note that these are transitive - the service also requires the base runtime. This is used when for example, launching a ``bst shell`` environment for the purpose of running, or in any case we need to consider which elements are required to run. .. image:: images/arch-dependency-model-runtime.svg :align: center * **Scope.BUILD** In the :func:`Scope.BUILD ` scope, only elements which are required to build are considered, *excluding* the element we intend to build. .. image:: images/arch-dependency-model-build.svg :align: center Note that build type dependencies are not transitive, which is why the *Bootstrap* element is not selected when pulling in the *Compiler* to build the *Application*. Further, note that we still follow the *Compiler* dependency on the *Base Runtime*, this is because when we depend on an element for the purpose of *building*, we expect that element to *run* and as such we include all of the *runtime dependencies* of *build dependencies* when selecting the *Scope.BUILD* elements. apache-buildstream-27ae392/doc/source/arch_overview.rst000066400000000000000000000013751514607367700232720ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Overview of modules =================== Below is a basic overview of the modules, what they are for, and generally what their stacking order is internally in BuildStream. .. image:: images/arch-overview.svg apache-buildstream-27ae392/doc/source/arch_program_flow.rst000066400000000000000000000015741514607367700241230ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Overview of program flow ======================== Here is a little chart to show the approximate highlevel program flow of BuildStream. This is in no way a complete flow chart of BuildStream, but should provide some highlevel insight into how the program operates in general. .. image:: images/arch-program-flow.svg :align: center apache-buildstream-27ae392/doc/source/arch_remote_execution.rst000066400000000000000000000073331514607367700250020ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Remote execution ================ Remote execution allows builds to take place on different machines from the machine *bst* is run on, allowing faster builds, shared resources and different build capabilities, for example different machine architectures. Sandbox extension ~~~~~~~~~~~~~~~~~ The previous section :ref:`sandboxing` describes the two forms of local sandbox: the *bubblewrap* sandbox and the less common *chroot* sandbox (which is used on non-Linux POSIX environments). Remote execution uses a third type, the *remote sandbox*, which functions similarly to the local sandbox types, but is responsible for causing the build to occur on a remote system. Remote sandboxes should produce the same result as local sandboxes. Artifact caches and other storage ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream can transmit the results of local builds to remote artifact caches and retrieve them later. The same mechanism is used for remote execution. The filesystem staged before building is stored in a local content-addressable store, which may be the same as the local artifact cache. The command to execute is also stored as an object in local CAS. Both the initial source filesystem and command are transmitted to remote storage specific to the remote execution service, and after the build is complete, the filesystem after build is retrieved from remote storage to the local CAS. The remote execution service uses the same communication protocol as artifact caches, and may use the same internal storage, but may not implement the Remote Asset API used by BuildStream to store full artifacts. .. image:: images/arch-remote-execution.svg :align: center After sending the prerequisite file system and command to the remote execution cache, BuildStream uses the *Remote Execution API (REAPI)* [1]_ to signal to the build server that it should perform a build. How the build service does this is not BuildStream's concern, but typically there will be a worker that will be assigned the work, retrieve the inputs of the build from CAS, carry out the build, and store the results in CAS. The remote execution service is entitled to return a cached result, if the filesystem and command have already been executed. BuildStream will continue to poll the remote execution server until the build is completed or lost. If it's completed (successfully or otherwise) the resulting objects (typically the finished file system and logs of stdout and stderr) will be pulled to the local cache. BuildStream will retry jobs that are lost by the remote build server or which complete with certain error types. *sandboxremote.py* contains all the communication with the remote execution API. After a successful build, BuildStream will push the completed artifact to the remote artifact servers as if it had built it locally. Use of sandboxes outside builds ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Because staging requires *integration-commands* to be run, a sandbox is necessary for the *bst artifact checkout* command to run. A sandbox is also required for the *bst shell* command. Because the REAPI does not provide any mechanism for interactive use, *bst shell* will always use a local sandbox. .. [1] See https://github.com/bazelbuild/remote-apis. apache-buildstream-27ae392/doc/source/arch_sandboxing.rst000066400000000000000000000167301514607367700235610ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _sandboxing: Sandboxing ========== Introduction ------------ BuildStream assembles each element in a *sandbox*. The sandbox is a container environment which serves two purposes: giving BuildStream control over all build aspects in order to ensure reproducibility of build results, and providing safety guarantees for the host system that BuildStream is running on. The exact implementation of the sandbox varies depending on which platform you are running BuildStream. See below for backend-specific details. There are several factors that affect the build output and must therefore be under BuildStream's control: * Filesystem contents and metadata * The user and permissions model * Network access * Device access Each of these is detailed below. For safety reasons, BuildStream also controls the following things: * Access to files outside of the sandbox directory * Access to certain kernel-specific syscalls Creating a sandbox can require special priviliges. This is a safety concern too because bugs in the `bst` program can cause damage to a host if the program is running with extra privileges. The exact priviliges that are required depend on your platform and backend. Element plugins can run arbitary commands within the sandbox using the :mod:`sandbox API `. What elements can and can't do in the sandbox --------------------------------------------- This section specifies how BuildStream sandboxes are intended to work. A specific sandbox provider may not necessarily be able to achieve all of the requirements listed below so be sure to read the "platform notes" section as well. Filesystem access ~~~~~~~~~~~~~~~~~ The filesystem inside sandboxes should be read-only during element assembly, except for certain directories which element plugins can mark as being read/write. Most elements plugins derive from :mod:`BuildElement `, which marks ``%{build-root}`` and ``%{install-root}`` as read/write. When running integration commands or `bst shell`, the sandbox should have a fully read-write filesystem. The changes made here do not need to persist beyond the lifetime of that sandbox, and **must not** affect the contents of artifacts stored in the cache. Certain top level directories should be treated specially in all sandboxes: * The ``/dev`` directory should contain device nodes, which are described in a separate section. * The ``/proc`` directory should have a UNIX 'procfs' style filesystem mounted. It should not expose any information about processes running outside of the sandbox. * The ``/tmp`` directory should be writable. Filesystem metadata ~~~~~~~~~~~~~~~~~~~ The writable areas inside a BuildStream sandbox are limited in what metadata can be written and stored. * All files must be owned by UID 0 and GID 0 * No files may have the setuid or setgid bits set * Extended file attributes (xattrs) cannot be written to or read. * Hardlinks to other files can be created, but the information about which files are hardlinked to each other will not be stored in the artifact that is created from the sandbox. These restrictions are due to technical limitations. In future we hope to support a wider range of filesystem metadata operations. See `issue #38 `_ for more details. User and permissions model ~~~~~~~~~~~~~~~~~~~~~~~~~~ All commands inside the sandbox run with user ID 0 and group ID 0. It should not be possible to become any other user ID. Network access ~~~~~~~~~~~~~~ Builds should not be able to access the network at all from the sandbox. All remote resources needed to build an element must be specified in the element's ``sources`` list so that BuildStream is able to see when they have changed. A sandbox opened by `bst shell` should allow network access. Device access ~~~~~~~~~~~~~ Builds should not be able to access any hardware devices at all. A few standard UNIX device files are needed, the whitelist is: * ``/dev/full`` * ``/dev/null`` * ``/dev/urandom`` * ``/dev/random`` * ``/dev/zero`` It may seem odd that we have sources of randomness in the sandbox, but a lot of tools do expect them to exist. We take the view that it's up to integrators to ensure that elements do not deliberately include randomness in their output. A sandbox opened by `bst shell` can make any devices available. There needs to be a console device so that it can be used interactively. Platform notes -------------- BuildStream delegates sandboxing for local builds to the ``buildbox-run`` command. ``buildbox-run`` provides a platform-independent interface to execute commands in a sandbox based on parts of the Remote Execution API. Linux ~~~~~ The recommended ``buildbox-run`` implementation for Linux is ``buildbox-run-bubblewrap``, in combination with ``buildbox-fuse``. These implementations use the following isolation and sandboxing primitives: * bind mounts * FUSE * Mount namespaces * Network namespaces * PID (process ID) namespaces * User namespaces (if available) * seccomp We access all of these features through a sandboxing tool named `Bubblewrap `_. User namespaces are not enabled by default in all Linux distributions. BuildStream still runs on such systems but can't build projects that set ``build-uid`` or ``build-gid`` in the ``sandbox`` configuration. The Linux platform can operate as a standard user, if unprivileged user namespace support is available. If user namespace support is not available you have the option of installing bubblewrap as a setuid binary to avoid needing to run the entire ``bst`` process as the ``root`` user. FUSE is used to provide access to directories and files stored in CAS without having to copy or hardlink the complete input tree into a regular filesystem directory structure for each build job. Some of the operations on filesystem metadata listed above are not prohibited by the sandbox, but will instead be silently dropped when an artifact is created. For more details see `issue #38 `_. Some details of the host machine are currently leaked by this platform backend. For more details, see `issue #262 `_. Other POSIX systems ~~~~~~~~~~~~~~~~~~~ On other POSIX systems ``buildbox-run-userchroot`` may be used for sandboxing. `userchroot `_ allows regular users to invoke processes in a chroot environment. ``buildbox-run-userchroot`` stages the input tree for each build job using hardlinks to avoid more expensive file copies. To avoid cache corruption it is vital that hardlinked files cannot be overwritten. Due to this it's required to run ``buildbox-casd`` as a separate user, which owns the files in the local cache. Network access is not blocked in the chroot. However since there is unlikely to be a correct `/etc/resolv.conf` file, any network access that depends on name resolution will most likely fail anyway. apache-buildstream-27ae392/doc/source/arch_scheduler.rst000066400000000000000000000113751514607367700234030ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Scheduler ========= The *Scheduler* is what is responsible for running the main event loop and dispatching *Jobs* to complete tasks on behalf of *Queues*. Jobs ~~~~ The basic functionality of multiprocessing tasks is implemented by the base Job class, which is derived in a few ways but for now we'll only talk about the ElementJob type since that is the most centric. The Job class has the following responsibilities: * Starting the given job as a subprocess. * Offering an abstract method for subclasses to handle the outcome of a job when it completes. * Forcefully terminating its subprocess. * Suspending and resuming its subprocess. * Declaring the types of resources it will require, and which resources it will require exclusively. Below is a rough outline of the interactions between the main process and job specific child process: .. image:: images/arch-scheduler-job.svg :align: center Resources ~~~~~~~~~ To understand how we manage load balancing in the scheduler it is important to understand *resources*. For the scheduler, *resources* are domains which a Job can request which represent physical resources, such as the CPU or some network bandwidth, or the local artifact cache. This is used by the Scheduler when consuming Jobs from Queues and deciding how many jobs can be run at a given time. Queues ~~~~~~ The various Queue implementations in the Scheduler can be combined such that parallelism is maximized. For example one can *Track* and *Build* inside the same session, in this way one does not need to wait for a tracking session to complete in order to start building. The input elements to the scheduler are expected to be sorted in depth first order whenever the order is important, again allowing maximum parallelism at build time. .. image:: images/arch-scheduler-queues.svg :align: center The Queue implementations are: * **Track** The tracking queue must always come first if it is used in the given session. This is because the Source *"ref"*, and consequently the cache key of any elements which have been requested for tracking, cannot be known until tracking is complete. * **Pull** The pull queue tries to obtain a built artifact from a remote artifact server, it should be placed in advance of the fetch queue if used, since we can safely avoid fetching if fetching is not imperative, and we already have a cached artifact. * **Fetch** The fetch queue attempts to download source code to build the given element, this activity is sometimes skipped if the artifact is already present, or if the exact source code is already present. * **Build** The build queue attempts to build the element if its artifact is not locally present. * **Push** The push queue attempts to push the resulting artifact to a remote artifact server. Queue internals ~~~~~~~~~~~~~~~ Internally, the queue has an input queue and an output queue. .. image:: images/arch-scheduler-queue-ports.svg :align: center When elements are on the input queue they get queried for their *QueueStatus* in order to determine which elements should be processed or moved from the input queue to the output queue. When elements are on the output queue, they are ready to be consumed by the scheduler and moved on to the next queue; however each queue holds on to the result status of every element which passed through for later reference and reports to the user. Scheduler ~~~~~~~~~ The scheduler itself has the main responsibility of popping off jobs from the existing queues it was given, and running the jobs as long as elements remain to be processed. A huge simplification of this can be visualized as follows: .. image:: images/arch-scheduler-run.svg :align: center This is implemented by iterating over the Queues given to the scheduler and processing any *"Ready"* elements so long as there are sufficient resource tokens available for the ready elements to run, and by moving the *"Done"* elements onto the subsequent queues in the list of queues. .. note:: When looking for *"Ready"* elements in the queues, we iterate over the queue list in *reverse order*. This is important to allow elements to get as far as they can in the queue list as fast as possible, and helps to prevent resource starvation. apache-buildstream-27ae392/doc/source/conf.py000066400000000000000000000235301514607367700211710ustar00rootroot00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # BuildStream documentation build configuration file, created by # sphinx-quickstart on Mon Nov 7 21:03:37 2016. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os import sys from buildstream import __version__ sys.path.insert(0, os.path.abspath("..")) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ["sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx_click.ext"] # Add any paths that contain templates here, relative to this directory. templates_path = [".templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = "index" # General information about the project. project = "BuildStream" copyright = "2017-2022, The Apache Software Foundation" # pylint: disable=redefined-builtin author = "The Apache Software Foundation" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = __version__ # The full version, including alpha/beta/rc tags. release = __version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # # today = '' # # Else, today_fmt is used as the format for a strftime call. # # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all # documents. # # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = False # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. modindex_common_prefix = ["buildstream."] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. # # html_title = 'BuildStream v0.1' # A shorter title for the navigation bar. Default is the same as html_title. # # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # # html_logo = None # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["badges", "images"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. # # html_extra_path = [] # If not None, a 'Last updated on:' timestamp is inserted at every page # bottom, using the given strftime format. # The empty string is equivalent to '%b %d, %Y'. # # html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # # html_additional_pages = {} # If false, no module index is generated. # # html_domain_indices = True # If false, no index is generated. # # html_use_index = True # If true, the index is split into individual pages for each letter. # # html_split_index = False # If true, links to the reST sources are added to the pages. # # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' # # html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # 'ja' uses this config value. # 'zh' user can custom change `jieba` dictionary path. # # html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. # # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. htmlhelp_basename = "BuildStreamdoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, "BuildStream.tex", "BuildStream Documentation", "The BuildStream Contributors", "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # # latex_use_parts = False # If true, show page references after internal links. # # latex_show_pagerefs = False # If true, show URL addresses after external links. # # latex_show_urls = False # Documents to append as an appendix to all manuals. # # latex_appendices = [] # It false, will not define \strong, \code, itleref, \crossref ... but only # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added # packages. # # latex_keep_old_macro_names = True # If false, no module index is generated. # # latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [(master_doc, "buildstream", "BuildStream Documentation", [author], 1)] # If true, show URL addresses after external links. # # man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ( master_doc, "BuildStream", "BuildStream Documentation", author, "BuildStream", "One line description of project.", "Miscellaneous", ), ] # Documents to append as an appendix to all manuals. # # texinfo_appendices = [] # If false, no module index is generated. # # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. # # texinfo_no_detailmenu = False autodoc_member_order = "bysource" apache-buildstream-27ae392/doc/source/core_additional.rst000066400000000000000000000012111514607367700235340ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Additional writings =================== .. toctree:: :maxdepth: 2 additional_docker apache-buildstream-27ae392/doc/source/core_format.rst000066400000000000000000000015311514607367700227210ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Project format ============== This section details how to use the BuildStream YAML format to create your own project or modify existing projects. .. toctree:: :maxdepth: 2 :caption: Project format format_intro format_project format_declaring format_public format_project_refs apache-buildstream-27ae392/doc/source/core_framework.rst000066400000000000000000000022751514607367700234340ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _core_framework: Plugin API reference ==================== The core public APIs are of interest to anyone who wishes to implement custom :mod:`Element ` or :mod:`Source ` plugins, and can also be useful for working on BuildStream itself. .. toctree:: :maxdepth: 1 buildstream.types buildstream.node buildstream.plugin buildstream.source buildstream.sourcemirror buildstream.downloadablefilesource buildstream.element buildstream.buildelement buildstream.scriptelement buildstream.sandbox.sandbox buildstream.storage.directory buildstream.exceptions buildstream.utils apache-buildstream-27ae392/doc/source/core_plugins.rst000066400000000000000000000042401514607367700231120ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _plugins: Plugin specific documentation ============================= Plugins provide their own individual plugin specific YAML configurations, The element ``.bst`` files can specify plugin specific configuration in the :ref:`config section `, while sources declared on a given element specify their plugin specific configuration directly :ref:`in their source declarations `. .. _plugins_elements: Elements -------- .. toctree:: :maxdepth: 1 elements/stack elements/import elements/compose elements/script elements/link elements/junction elements/filter elements/manual .. _plugins_sources: Sources ------- All source plugins can be staged into an arbitrary directory within the build sandbox with the ``directory`` option. See :ref:`Source class built-in functionality ` for more information. .. toctree:: :maxdepth: 1 sources/local sources/remote sources/tar .. _plugins_external: External plugins ---------------- External plugins need to be :ref:`loading through junctions `, or alternatively installed separately in the python environment where you are running BuildStream and loaded using the :ref:`pip method `. Here is a list of BuildStream plugin projects known to us at this time: * `buildstream-plugins `_ * `buildstream-plugins-community (formerly known as bst-plugins-experimental) `_ * `bst-plugins-container `_ apache-buildstream-27ae392/doc/source/developing/000077500000000000000000000000001514607367700220235ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/developing/strict-mode.rst000066400000000000000000000223221514607367700250100ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _developing_strict_mode: Strict mode =========== In this section, we will cover the usage of :ref:`strict vs non-strict ` build plans in conjunction with :ref:`workspaces `, and how this can help to improve your edit/compile/test cycles. .. note:: This example is distributed with BuildStream in the `doc/examples/strict-mode `_ subdirectory. Overview -------- When working with BuildStream to create integrations, it is typical that you have a lot of components to build, and you frequently need to modify a component at various levels of the stack. When developing one or more applications, you might want to open a workspace and fix a bug in an application, or you might need to open a workspace on a low level shared library to fix the behavior of one or more misbehaving applications. By default, BuildStream will always choose to be deterministic in order to produce the most correct build results as possible. As such, modifying a low level library will result in rebuilding all of it's reverse dependencies, but this can be very time consuming and inconvenient for your edit/compile/test cycles. This is when enabling :ref:`non-strict build plans ` can be helpful. To illustrate the facets of how this works, this example will present a project consisting of an application which is linked both statically and dynamically linked to a common library. Project structure ----------------- This project is mostly based on the :ref:`integration commands ` example, as such we will ignore large parts of this project and only focus on the elements which are of specific interest. To illustrate the relationship of these two applications and the library, let's briefly take a look at the underlying Makefiles which are used in this project, starting with the library and followed by both Makefiles used to build the application. ``files/libhello/Makefile`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/strict-mode/files/libhello/Makefile :language: Makefile ``files/hello/Makefile.dynamic`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/strict-mode/files/hello/Makefile.dynamic :language: Makefile ``files/hello/Makefile.static`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/strict-mode/files/hello/Makefile.static :language: Makefile As we can see, we have a library that is distributed both as the dynamic library ``libhello.so`` and also as the static archive ``libhello.a``. Now let's take a look at the two separate elements which build the application, first the dynamically linked version and then the static one. ``elements/hello-dynamic.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/strict-mode/elements/hello-dynamic.bst :language: yaml Nothing very special to observe about this hello program, just a :mod:`manual ` element quite similar to the one we've already seen in the :ref:`running commands ` example. ``elements/hello-static.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/strict-mode/elements/hello-static.bst :language: yaml Almost the same as the dynamic element, except here we have declared the dependency to the ``libhello.bst`` element differently: this time we have enabled the ``strict`` option in the :ref:`dependency declaration `. The side effect of setting this option is that ``hello-static.bst`` will be rebuilt any time that ``libhello.bst`` has changed, even when :ref:`non-strict build plans ` have been enabled. .. tip:: Some element plugins are designed to consume the content of their dependencies entirely, and output an artifact without any transient runtime dependencies, an example of this is the :mod:`compose ` element. In cases such as :mod:`compose `, it is not necessary to explicitly annotate their dependencies as ``strict``. It is only helpful to set the ``strict`` attribute on a :ref:`dependency declaration ` in the case that the specific dependency relationship causes data to be consumed verbatim, as is the case with static linking. Using the project ----------------- For the sake of brevity, let's assume that you've already built all of the elements of this project, and that you want to make some changes to the ``libhello.bst`` element, and test how it might effect the hello program. Everything is already built ~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. raw:: html :file: ../sessions/strict-mode-show-initial.html Open a workspace and modify libhello.c ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Now let's open up a workspace on the hello library .. raw:: html :file: ../sessions/strict-mode-workspace-open.html And go ahead and make a modification like this: .. literalinclude:: ../../examples/strict-mode/update.patch :language: diff Observing ``hello-dynamic.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Let's take a look at the :ref:`bst show ` output for the dynamically linked ``hello-dynamic.bst`` element. .. raw:: html :file: ../sessions/strict-mode-show-dynamic-strict.html As one might expect, the ``libhello.bst`` element is ready to be built after having been modified, and the ``hello-dynamic.bst`` element is waiting for ``libhello.bst`` to be built before it can build. Now let's take a look at the same elements if we pass the ``--no-strict`` option to ``bst``: .. raw:: html :file: ../sessions/strict-mode-show-dynamic-no-strict.html Note that this time, the ``libhello.bst`` still needs to be built, but the ``hello-dymamic.bst`` element is showing up as ``cached``. .. tip:: The :ref:`bst show ` output will show some cache keys dimmed out in the case that they are not entirely deterministic. Here we can see that ``hello-dynamic.bst`` is dimmed out because it will not be rebuilt against the changed ``libhello.bst`` element, and it also has a different cache key because of this. Observing ``hello-static.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Now let's observe the ``hello-static.bst`` element with strict mode disabled: .. raw:: html :file: ../sessions/strict-mode-show-static-no-strict.html Note that in this case the ``hello-strict.bst`` is going to be rebuilt even in strict mode. This is because we annotated the declaration of the ``libhello.bst`` dependency with the ``strict`` attribute. We did this because ``hello-strict.bst`` consumes the input of ``libhello.bst`` verbatim, by way of statically linking to it, instead of merely being affected by the content of ``libhello.bst`` at runtime, as would be the case of static linking. Building and running ``hello-dynamic.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Now let's build ``hello-dynamic.bst`` with strict mode disabled. .. raw:: html :file: ../sessions/strict-mode-build-dynamic-no-strict.html Note that the :ref:`bst build ` command completed without having to build ``hello-dynamic.bst`` at all. And now we can also run ``hello-dynamic.bst`` .. raw:: html :file: ../sessions/strict-mode-run-dynamic-no-strict.html When running ``hello-dynamic.bst`` with no-strict mode, we are actually reusing the old build of ``hello-dynamic.bst`` staged against the new build of the modified ``libhello.bst`` element. Building and running ``hello-static.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Finally, if we build ``hello-static.bst`` with strict mode disabled, we can see that it will be rebuilt regardless of strict mode being enabled. .. raw:: html :file: ../sessions/strict-mode-build-static-no-strict.html This is of course because we declared its dependency on ``libhello.bst`` as a ``strict`` dependency. And by the same virtue, we can see that when we run the example it has properly relinked against the changed static archive, and has the updated text in the greeting: .. raw:: html :file: ../sessions/strict-mode-run-static-no-strict.html Summary ------- In this chapter we've explored how to use :ref:`non-strict build plans ` in order to avoid rebuilding reverse dependencies of a lower level element you might be working with in a :ref:`workspace `, consequently improving your edit/compile/test experience. We've also explained how to ensure your project still works properly with non-strict build plans when some elements perform static linking (or other operations which consume data from their dependencies verbatim), by annotating :ref:`dependency declarations ` as ``strict``. apache-buildstream-27ae392/doc/source/developing/workspaces.rst000066400000000000000000000120411514607367700247340ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _developing_workspaces: Workspaces ========== In this section we will cover the use of BuildStream's workspaces feature when devloping a BuildStream project. .. note:: This example is distributed with BuildStream in the `doc/examples/developing `_ subdirectory. We will start with the project used in the :ref:`running commands ` tutorial. Recall the element hello.bst, which builds the bellow C file: .. literalinclude:: ../../examples/developing/files/src/hello.c :language: c Suppose we now want to alter the functionality of the *hello* command. We can make changes to the source code of Buildstream elements by making use of BuildStream's workspace command. Opening a workspace ------------------- First we need to open a workspace, we can do this by running .. raw:: html :file: ../sessions/developing-workspace-open.html This command has created the workspace_hello directory in which you can see the source for the hello.bst element, i.e. hello.c and the corresponding makefile. You can view existing workspaces using .. raw:: html :file: ../sessions/developing-workspace-list.html Making code changes ------------------- Let's say we want to alter the message printed when the hello command is run. We can open workspace_hello/hello.c and make the following change: .. literalinclude:: ../../examples/developing/update.patch :language: diff Now, rebuild the hello.bst element. .. raw:: html :file: ../sessions/developing-build-after-changes.html Note that if you run the command from inside the workspace, the element name is optional. .. raw:: html :file: ../sessions/developing-build-after-changes-workspace.html Now running the hello command using bst shell: .. raw:: html :file: ../sessions/developing-shell-after-changes.html This gives us the new message we changed in hello.c. From this point we have several options. If the source is under version control we can commit our changes and push them to the remote repository. Incremental builds ------------------ Once you have opened up your workspace, the workspace build directory will be reused for subsequent builds, which should improve your edit/compile/test cycle time when working with an open workspace. In order to optimize incremental builds, BuildStream treats build configure steps separately from the main build steps, and will only call the :func:`Element.prepare() ` method on an element plugin the first time it gets built. This avoids needlessly rebuilding objects due to header files and such being unconditionally recreated by configuration scripts (such as the typical ``./configure`` script which is called for ``autotools`` elements for instance). A caveat of this optimization however is that changes you might make to the configuration scripts will not be taken into account by default on the next incremental build. A forced reconfiguration can also be required in some cases where build scripts automatically detect sources in it's configuration phase, so newly added sources you add might be ignored. In order to force the configuration step to be called again on the next build, you can use :ref:`bst workspace reset --soft `, like so: In these cases, you can perform a hard reset on the workspace using :ref:`bst workspace reset `, like so: .. raw:: html :file: ../sessions/developing-soft-reset.html This will ensure that the next time you run the build, BuildStream will forcefully rerun the :func:`Element.prepare() ` method and cause the configuration step to occur again. Closing your workspace ---------------------- If we want to close the workspace and come back to our changes later, we can .. raw:: html :file: ../sessions/developing-close-workspace.html We can then reopen the workspace later using: .. raw:: html :file: ../sessions/developing-reopen-workspace.html The --no-checkout option tells BuildStream not to check the source out but to instead hard-link to the workspace_hello directory. Alternatively, if we wish to discard the changes we can use .. raw:: html :file: ../sessions/developing-reset-workspace.html This resets the workspace to its original state. To discard the workspace completely we can do: .. raw:: html :file: ../sessions/developing-discard-workspace.html This will close the workspace and completely remove the workspace_hello directory. apache-buildstream-27ae392/doc/source/examples/000077500000000000000000000000001514607367700215055ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/examples/git-mirror.rst000066400000000000000000000113241514607367700243330ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Creating and using a git mirror ''''''''''''''''''''''''''''''' This is an example of how to create a git mirror using git's `git-http-backend `_ and `lighttpd `_. Prerequisites ============= You will need git installed, and git-http-backend must be present. It is assumed that the git-http-backend binary exists at `/usr/lib/git-core/git-http-backend`. You will need `lighttpd` installed, and at the bare minimum has the modules `mod_alias`, `mod_cgi`, and `mod_setenv`. I will be using gnome-modulesets as an example, which can be cloned from `http://gnome7.codethink.co.uk/gnome-modulesets.git`. Starting a git http server ========================== 1. Set up a directory containing mirrors ---------------------------------------- Choose a suitable directory to hold your mirrors, e.g. `/var/www/git`. Place the git repositories you want to use as mirrors in the mirror dir, e.g. ``git clone --mirror http://git.gnome.org/browse/yelp-xsl /var/www/git/yelp-xsl.git``. 2. Configure lighttpd --------------------- Write out a lighttpd.conf as follows: :: server.document-root = "/var/www/git/" server.port = 3000 server.modules = ( "mod_alias", "mod_cgi", "mod_setenv", ) alias.url += ( "/git" => "/usr/lib/git-core/git-http-backend" ) $HTTP["url"] =~ "^/git" { cgi.assign = ("" => "") setenv.add-environment = ( "GIT_PROJECT_ROOT" => "/var/www/git", "GIT_HTTP_EXPORT_ALL" => "" ) } .. note:: If you have your mirrors in another directory, replace /var/www/git/ with that directory. 3. Start lighttpd ----------------- lighttpd can be invoked with the command-line ``lighttpd -D -f lighttpd.conf``. 4. Test that you can fetch from it ---------------------------------- We can then clone the mirrored repo using git via http with ``git clone http://127.0.0.1:3000/git/yelp-xsl``. .. note:: If you have set server.port to something other than the default, you will need to replace the '3000' in the command-line. 5. Configure the project to use the mirror ------------------------------------------ To add this local http server as a mirror, add the following to the project.conf: .. code:: yaml mirrors: - name: local-mirror aliases: git_gnome_org: - http://127.0.0.1:3000/git/ 6. Test that the mirror works ----------------------------- We can make buildstream use the mirror by setting the alias to an invalid URL, e.g. .. code:: yaml aliases: git_gnome_org: https://www.example.com/invalid/url/ Now, if you build an element that uses the source you placed in the mirror (e.g. ``bst build core-deps/yelp-xsl.bst``), you will see that it uses your mirror. .. _lighttpd_git_tar_conf: Bonus: lighttpd conf for git and tar ==================================== For those who have also used the :ref:`tar-mirror tutorial `, a combined lighttpd.conf is below: :: server.document-root = "/var/www/" server.port = 3000 server.modules = ( "mod_alias", "mod_cgi", "mod_setenv", ) alias.url += ( "/git" => "/usr/lib/git-core/git-http-backend" ) $HTTP["url"] =~ "^/git" { cgi.assign = ("" => "") setenv.add-environment = ( "GIT_PROJECT_ROOT" => "/var/www/git", "GIT_HTTP_EXPORT_ALL" => "" ) } else $HTTP["url"] =~ "^/tar" { dir-listing.activate = "enable" } Further reading =============== If this mirror isn't being used exclusively in a secure network, it is strongly recommended you `use SSL `_. This is the bare minimum required to set up a git mirror. A large, public project would prefer to set it up using the `git protocol `_, and a security-conscious project would be configured to use `git over SSH `_. Lighttpd is documented on `its wiki `_. apache-buildstream-27ae392/doc/source/examples/tar-mirror.rst000066400000000000000000000063011514607367700243350ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _using_tar_mirror: Creating and using a tar mirror ''''''''''''''''''''''''''''''' This is an example of how to create a tar mirror using `lighttpd `_. Prerequisites ============= You will need `lighttpd` installed. I will be using gnome-modulesets as an example, which can be cloned from `http://gnome7.codethink.co.uk/gnome-modulesets.git`. Starting a tar server ===================== 1. Set up a directory containing mirrors ---------------------------------------- Choose a suitable directory to hold your mirrored tar files, e.g. `/var/www/tar`. Place the tar files you want to use as mirrors in your mirror dir, e.g. .. code:: mkdir -p /var/www/tar/gettext wget -O /var/www/tar/gettext/gettext-0.19.8.1.tar.xz https://ftp.gnu.org/gnu/gettext/gettext-0.19.8.1.tar.xz 2. Configure lighttpd --------------------- Write out a lighttpd.conf as follows: :: server.document-root = "/var/www/tar/" server.port = 3000 dir-listing.activate = "enable" .. note:: If you have your mirrors in another directory, replace /var/www/tar/ with that directory. .. note:: An example lighttpd.conf that works for both git and tar services is available :ref:`here ` 3. Start lighttpd ----------------- lighttpd can be invoked with the command-line ``lighttpd -D -f lighttpd.conf``. 4. Test that you can fetch from it ---------------------------------- We can then download the mirrored file with ``wget 127.0.0.1:3000/tar/gettext/gettext-0.19.8.1.tar.xz``. .. note:: If you have set server.port to something other than the default, you will need to replace the '3000' in the command-line. 5. Configure the project to use the mirror ------------------------------------------ To add this local http server as a mirror, add the following to the project.conf: .. code:: yaml mirrors: - name: local-mirror aliases: ftp_gnu_org: - http://127.0.0.1:3000/tar/ 6. Test that the mirror works ----------------------------- We can make buildstream use the mirror by setting the alias to an invalid URL, e.g. .. code:: yaml aliases: ftp_gnu_org: https://www.example.com/invalid/url/ Now, if you build an element that uses the source you placed in the mirror (e.g. ``bst build core-deps/gettext.bst``), you will see that it uses your mirror. Further reading =============== If this mirror isn't being used exclusively in a secure network, it is strongly recommended you `use SSL `_. Lighttpd is documented on `its wiki `_. apache-buildstream-27ae392/doc/source/format_declaring.rst000066400000000000000000000565621514607367700237370ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Declaring elements ================== .. _format_basics: Element basics -------------- Here is a rather complete example using the autotools element kind and git source kind: .. code:: yaml # Specify the kind of element this is kind: autotools # Specify some dependencies depends: - element1.bst - element2.bst # Specify the source which should be built sources: - kind: git url: upstream:modulename.git track: master ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6 # Override some variables variables: sysconfdir: "%{prefix}/etc" # Tweak the sandbox shell environment environment: LD_LIBRARY_PATH: /some/custom/path # Specify the configuration of the element config: # Override autotools element default configure-commands configure-commands: - "%{configure} --enable-fancy-feature" # Specify public domain data, visible to other elements. public: bst: integration-commands: - /usr/bin/update-fancy-feature-cache # Specify a user id and group id to use in the build sandbox. sandbox: build-uid: 0 build-gid: 0 For most use cases you would not need to specify this much detail, we've provided details here in order to have a more complete initial example. Let's break down the above and give a brief explanation of what these attributes mean. .. _format_element_names: Element names and paths ~~~~~~~~~~~~~~~~~~~~~~~ An *element name* is the filename of an element relative to the project's :ref:`element path `. Element names are the identifiers used to refer to elements, they are used to specify an element's :ref:`dependencies `, to select elements to build on the :ref:`command line `, and they are arbitrarily used in various element specific configuration surfaces, for example the *target* configuration of the :mod:`link ` element is also an *element name*. Addressing elements ''''''''''''''''''' When addressing elements in a single project, it is sufficient to use the *element name* as a dependency or configuration parameter. When muliple projects are connected through :mod:`junction ` elements, there is a need to address elements which are not in the same project but in a junctioned *subproject*. In the case that you need to address elements across junction boundaries, one must use *element paths*. An *element path* is a path to the element indicating the junction elements leading up to the project, separated by ``:`` symbols, e.g.: ``junction.bst:element.bst``. Elements can be address across multiple junction boundaries with multiple ``:`` separators, e.g.: ``junction.bst:junction.bst:element.bst``. Element naming rules '''''''''''''''''''' When naming the elements, use the following rules: * The name of the file must have the ``.bst`` extension. * All characters in the name must be printable 7-bit ASCII characters. * Following characters are reserved and must not be part of the name: - ``<`` (less than) - ``>`` (greater than) - ``:`` (colon) - ``"`` (double quote) - ``/`` (forward slash) - ``\`` (backslash) - ``|`` (vertical bar) - ``?`` (question mark) - ``*`` (asterisk) Kind ~~~~ .. code:: yaml # Specify the kind of element this is kind: autotools The ``kind`` attribute specifies which plugin will be operating on the element's input to produce its output. Plugins define element types and each of them can be referred to by name with the ``kind`` attribute. To refer to a third party plugin, prefix the plugin with its package, for example: .. code:: yaml kind: buildstream-plugins:dpkg_build .. _format_depends: Depends ~~~~~~~ .. code:: yaml # Specify some dependencies depends: - element1.bst - element2.bst Relationships between elements are specified with the ``depends`` attribute. Elements may depend on other elements by specifying the :ref:`element names ` they depend on here. See :ref:`format_dependencies` for more information on the dependency model. .. _format_build_depends: Build-Depends ~~~~~~~~~~~~~ .. code:: yaml # Specify some build-dependencies build-depends: - element1.bst - element2.bst Build dependencies between elements can be specified with the ``build-depends`` attribute. The above code snippet is equivalent to: .. code:: yaml # Specify some build-dependencies depends: - filename: element1.bst type: build - filename: element2.bst type: build See :ref:`format_dependencies` for more information on the dependency model. .. _format_runtime_depends: Runtime-Depends ~~~~~~~~~~~~~~~ .. code:: yaml # Specify some runtime-dependencies runtime-depends: - element1.bst - element2.bst Runtime dependencies between elements can be specified with the ``runtime-depends`` attribute. The above code snippet is equivalent to: .. code:: yaml # Specify some runtime-dependencies depends: - filename: element1.bst type: runtime - filename: element2.bst type: runtime See :ref:`format_dependencies` for more information on the dependency model. .. _format_sources: Sources ~~~~~~~ .. code:: yaml # Specify the source which should be built sources: - kind: git url: upstream:modulename.git track: master ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6 Here we specify some input for the element, any number of sources may be specified. By default the sources will be staged in the root of the element's build directory in the build sandbox, but sources may specify a ``directory`` attribute to control where the sources will be staged. The ``directory`` attribute may specify a build sandbox relative subdirectory. For example, one might encounter a component which requires a separate data package in order to build itself, in this case the sources might be listed as: .. code:: yaml sources: # Specify the source which should be built - kind: git url: upstream:modulename.git track: master ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6 # Specify the data package we need for build frobnication, # we need it to be unpacked in a src/frobdir - kind: tarball directory: src/frobdir url: data:frobs.tgz ref: 9d4b1147f8cf244b0002ba74bfb0b8dfb3... Like Elements, Source types are plugins which are indicated by the ``kind`` attribute. Asides from the common ``kind`` and ``directory`` attributes which may be applied to all Sources, refer to the Source specific documentation for meaningful attributes for the particular Source. Variables ~~~~~~~~~ .. code:: yaml # Override some variables variables: sysconfdir: "%{prefix}/etc" Variables can be declared or overridden from an element. Variables can also be declared and overridden in the :ref:`projectconf` See :ref:`format_variables` below for a more in depth discussion on variables in BuildStream. .. _format_environment: Environment ~~~~~~~~~~~ .. code:: yaml # Tweak the sandbox shell environment environment: LD_LIBRARY_PATH: /some/custom/path Environment variables can be set to literal values here, these environment variables will be effective in the :mod:`Sandbox ` where build instructions are run for this element. Environment variables can also be declared and overridden in the :ref:`projectconf` .. _format_config: Config ~~~~~~ .. code:: yaml # Specify the configuration of the element config: # Override autotools element default configure-commands configure-commands: - "%{configure} --enable-fancy-feature" Here we configure the element itself. The autotools element provides sane defaults for building sources which use autotools. Element default configurations can be overridden in the ``project.conf`` file and additionally overridden in the declaration of an element. For meaningful documentation on what can be specified in the ``config`` section for a given element ``kind``, refer to the :ref:`element specific documentation `. .. _format_public: Public ~~~~~~ .. code:: yaml # Specify public domain data, visible to other elements. public: bst: integration-commands: - /usr/bin/update-fancy-feature-cache Metadata declared in the ``public`` section of an element is visible to any other element which depends on the declaring element in a given pipeline. BuildStream itself consumes public data from the ``bst`` domain. The ``integration-commands`` demonstrated above for example, describe commands which should be run in an environment where the given element is installed but before anything should be run. An element is allowed to read domain data from any element it depends on, and users may specify additional domains to be understood and processed by their own element plugins. The public data keys which are recognized under the ``bst`` domain can be viewed in detail in the :ref:`builtin public data ` section. .. _format_sandbox: Sandbox ~~~~~~~ Configuration for the build sandbox (other than :ref:`environment variables `) can be placed in the ``sandbox`` configuration. The UID and GID used by the user in the group can be specified, as well as the desired OS and machine architecture. Possible machine architecture follow the same list as specified in the :ref:`architecture option `. .. code:: yaml # Specify a user id and group id to use in the build sandbox. sandbox: build-uid: 1003 build-gid: 1001 BuildStream normally uses uid 0 and gid 0 (root) to perform all builds. However, the behaviour of certain tools depends on user id, behaving differently when run as non-root. To support those builds, you can supply a different uid or gid for the sandbox. Only bwrap-style sandboxes support custom user IDs at the moment, and hence this will only work on Linux host platforms. .. code:: yaml # Specify build OS and architecture sandbox: build-os: AIX build-arch: power-isa-be When building locally, if these don't match the host machine then generally the build will fail. The exception is when the OS is Linux and the architecture specifies an ``x86-32`` build on an ``x86-64`` machine, or ``aarch32`` build on a ``aarch64`` machine, in which case the ``linux32`` command is prepended to the bubblewrap command. When building remotely, the OS and architecture are added to the ``Platform`` field in the ``Command`` uploaded. Whether this actually results in a building the element for the desired OS and architecture is dependent on the server having implemented these options the same as buildstream. .. code:: yaml # Specify UNIX socket path for access to REAPI for (nested) remote execution sandbox: remote-apis-socket: path: /run/reapi.sock action-cache-enable-update: false Setting a path will add a UNIX socket to the sandbox that allows the use of `REAPI `_ clients such as `recc `_. This enables more fine-grained caching of, e.g., individual compile commands to speed up rebuilds of elements with only small changes. This is supported with and without :ref:`remote execution `. With remote execution configured, this additionally enables scaling out of, e.g., compile commands across a cluster of build machines. Action cache updates from sandboxed REAPI clients are disabled by default to protect action cache integrity. However, if a trusted REAPI client doesn't support remote execution, action cache updates can be enabled by setting ``action-cache-enable-update`` to ``true``. If a remote ``action-cache-service`` is configured without ``execution-service`` in the :ref:`remote execution ` section, ``push`` needs to be set to ``true`` to allow action cache updates to be pushed to the server. .. _format_dependencies: Dependencies ------------ The dependency model in BuildStream is simplified by treating software distribution and software building as separate problem spaces. This is to say that one element can only ever depend on another element but never on a subset of the product which another element produces. In this section we'll quickly go over the few features BuildStream offers in its dependency model. Expressing dependencies ~~~~~~~~~~~~~~~~~~~~~~~ Dependencies in BuildStream are parameterizable objects, however as demonstrated in the :ref:`above example `, they can also be expressed as simple strings as a convenience shorthand in most cases, whenever the default dependency attributes are suitable. .. note:: Note the order in which element dependencies are declared in the ``depends``, ``build-depends`` and ``runtime-depends`` lists are not meaningful. Dependency dictionary: .. code:: yaml # Fully specified dependency depends: - filename: foo.bst type: build junction: baseproject.bst strict: false Attributes: * ``filename`` The :ref:`element name ` to depend on, or a list of mutiple element names. Specifying multiple element names in a single dependency will result in multiple dependencies being declared with common properties. For example, one can declare multiple build dependencies with the same junction: .. code:: yaml # Declare three build dependencies from subproject.bst depends: - type: build junction: subproject.bst filename: - element-a.bst - element-b.bst - element-c.bst * ``junction`` This attribute can be used to specify the junction portion of the :ref:`element name ` separately from the project local element name. This should be the *element name* of the :mod:`junction ` element in the local project, possibly followed by other junctions in subprojects leading to the project in which the element you want to depend on resides. In the case that a *junction* is specified, the ``filename`` attribute indicates an element in the *junctioned project*. * ``type`` This attribute is used to express the :ref:`dependency type `. This field is not permitted in the :ref:`build-depends ` or :ref:`runtime-depends ` lists. * ``strict`` This attribute can be used to specify that this element should be rebuilt when the dependency changes, even when :ref:`strict mode ` has been turned off. This is appropriate whenever a dependency's output is consumed verbatim in the output of the depending element, for instance when static linking is in use. * ``config`` This attribute defines the custom :term:`dependency configuration `, which is supported by select :mod:`Element ` implementations. Elements which support :term:`dependency configuration ` do so by implementing the :func:`Element.configure_dependencies() ` abstract method. It is up to each element or abstract element class to document what is supported in their :term:`dependency configuration `. .. attention:: It is illegal to declare :term:`dependency configuration ` on runtime dependencies, since runtime dependencies are not visible to the depending element. Redundant dependency declarations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is permitted to declare dependencies multiple times on the same element in the same element declaration, the result will be an inclusive OR of all configurations you have expressed in the redundant dependencies on the same element. * If a dependency is defined once as a ``build`` dependency and once as a ``runtime`` :ref:`dependency type `, then the resulting dependency type will be ``all`` * If any of the redundantly declared dependencies are specified as ``strict``, then the resulting dependency will be ``strict``. Declaring redundant dependencies on the same element can be interesting when you need to specify multiple :term:`dependency configurations ` for the same element. For example, one might want to stage the same dependency in multiple locations in the build sandbox. Cross-junction dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~ As explained in the :ref:`element name ` section on element addressing, elements can be addressed across junction boundaries using *element paths* such as ``junction.bst:element.bst``. An element at any depth can be specified by specifying multiple junction elements. For example, one can specify a subproject element dependency with the following syntax: .. code:: yaml build-depends: - baseproject.bst:element.bst And one can specify an element residing in a sub-subproject as a dependency like so: .. code:: yaml depends: - baseproject.bst:middleproject.bst:element.bst .. _format_dependencies_types: Dependency types ~~~~~~~~~~~~~~~~ The dependency ``type`` attribute defines what the dependency is required for and is essential to how BuildStream plots a build plan. There are three types which one can specify for a dependency: * ``build`` A ``build`` dependency type states that the given element's product must be staged in order to build the depending element. Depending on an element which has ``build`` dependencies will not implicitly depend on that element's ``build`` dependencies. For convenience, these can be specified under the :ref:`build-depends ` list. * ``runtime`` A ``runtime`` dependency type states that the given element's product must be present for the depending element to function. An element's ``runtime`` dependencies are not available to the element at build time. For convenience, these can be specified under the :ref:`runtime-depends ` list. * ``all`` An ``all`` dependency is the default dependency type. If ``all`` is specified, or if ``type`` is not specified at all, then it is assumed that the dependency is required both at build time and runtime. .. note:: It is assumed that a dependency which is required for building an element must run while building the depending element. This means that ``build`` depending on a given element implies that that element's ``runtime`` dependencies will also be staged for the purpose of building. .. _format_variables: Using variables --------------- Variables in BuildStream are a way to make your build instructions and element configurations more dynamic. Referring to variables ~~~~~~~~~~~~~~~~~~~~~~ Variables are expressed as ``%{...}``, where ``...`` must contain only alphanumeric characters and the separators ``_`` and ``-``. Further, the first letter of ``...`` must be an alphabetic character. .. code:: yaml This is release version %{version} Declaring and overriding variables ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To declare or override a variable, one need only specify a value in the relevant *variables* section: .. code:: yaml variables: hello: Hello World You can refer to another variable while declaring a variable: .. code:: yaml variables: release-text: This is release version %{version} The order in which you declare variables is arbitrary, so long as there is no cyclic dependency and that all referenced variables are declared, the following is fine: .. code:: yaml variables: release-text: This is release version %{version} version: 5.5 .. note:: It should be noted that variable resolution only happens after all :ref:`Element Composition ` has already taken place. This is to say that overriding ``%{version}`` at a higher priority will affect the final result of ``%{release-text}``. **Example:** .. code:: yaml kind: autotools # Declare variable, expect %{version} was already declared variables: release-text: This is release version %{version} config: # Customize the installation install-commands: - | %{make-install} RELEASE_TEXT="%{release-text}" Variables declared by BuildStream ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream declares a set of :ref:`builtin ` variables that may be overridden. In addition, the following read-only variables are also dynamically declared by BuildStream: * ``element-name`` The name of the element being processed (e.g base/alpine.bst). * ``project-name`` The name of project where BuildStream is being used. * ``project-root`` & ``project-root-uri`` The directory where the project is located on the host. This variable is only available when declaring :ref:`source alias values ` or :ref:`source mirror values ` and allows access to files in a project on the build host. * The ``project-root`` variable is a regular absolute path * The ``project-root-uri`` variable is a properly quoted ``file://`` URI .. tip:: Use this variable to declare :ref:`source alias values ` to refer to files which you store as a part of your project, e.g. tarballs which you have committed to you BuildStream project. .. attention:: This feature has been provided for convenience when putting together a project without the use of proper infrastructure. A better long term solution for accessing internal binaries and source code is to setup internal infrastructure in your organization and use the regular ways to access these sources from a well known internal URI. * ``toplevel-root`` & ``toplevel-root-uri`` The directory where the toplevel project is located on the host. This variable is only available when declaring :ref:`source alias values ` or :ref:`source mirror values ` and allows access to files in a project on the build host. * The ``toplevel-root`` variable is a regular absolute path * The ``toplevel-root-uri`` variable is a properly quoted ``file://`` URI .. tip:: Use this variable to declare :ref:`source alias values ` to refer to files which you do not store as a part of your project, e.g. tarballs or git repositories which must be placed in a directory within the toplevel project before running the build. .. attention:: This feature has been provided for convenience when putting together a project without the use of proper infrastructure. A better long term solution for accessing internal binaries and source code is to setup internal infrastructure in your organization and use the regular ways to access these sources from a well known internal URI. * ``max-jobs`` Maximum number of parallel build processes within a given build, support for this is conditional on the element type and the build system used (any element using 'make' can implement this). apache-buildstream-27ae392/doc/source/format_intro.rst000066400000000000000000000255431514607367700231350ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Introduction ============ At the core of BuildStream is a data model of :mod:`Elements ` which are parsed from ``.bst`` files in a project directory and configured from a few different sources. When BuildStream loads your project, various levels of composition occur, allowing configuration on various levels with different priority. This page provides an introduction to the project directory structure, explains the basic *directives* supported inherently throughout the format, and outlines how composition occurs and what configurations are considered in which order. The meaning of the various constructs expressed in the BuildStream format are covered in other sections of the documentation. .. _format_structure: Directory structure ------------------- A BuildStream project is a directory consisting of: * A project configuration file * BuildStream element files * Optional user defined plugins * An optional project.refs file A typical project structure may look like this:: myproject/project.conf myproject/project.refs myproject/elements/element1.bst myproject/elements/element2.bst myproject/elements/... myproject/plugins/customelement.py myproject/plugins/customelement.yaml myproject/plugins/... Except for the project configuration file, the user is allowed to structure their project directory in any way. For documentation on the format of the project configuration file, refer to the :ref:`projectconf` documentation. Simpler projects may choose to place all element definition files at the root of the project directory while more complex projects may decide to put stacks in one directory and other floating elements into other directories, perhaps placing deployment elements in another directory, this is all fine. The important part to remember is that when you declare dependency relationships, a project relative path to the element one depends on must be provided. .. _format_composition: Composition ----------- Below are the various sources of configuration which go into an element or source in the order in which they are applied. Configurations which are applied later have a higher priority and override configurations which precede them. 1. Builtin defaults ~~~~~~~~~~~~~~~~~~~ The :ref:`builtin defaults ` provide a set of builtin default default values for ``project.conf``. The project wide defaults defined in the builtin project configuration, such as the *variables* or *environment* sections, form the base configuration of all elements. 2. Project configuration ~~~~~~~~~~~~~~~~~~~~~~~~ The :ref:`project wide defaults ` specified in your ``project.conf`` are now applied on top of builtin defaults. Defaults such as the :ref:`variables ` or :ref:`environment ` which are specified in your ``project.conf`` override the builtin defaults for elements. Note that :ref:`plugin type specific configuration ` in ``project.conf`` is not applied until later. 3. Plugin defaults ~~~~~~~~~~~~~~~~~~ Elements and Sources are all implemented as plugins. Each Element plugin installs a ``.yaml`` file along side their plugin to define the default *variables*, *environment* and *config*. The *config* is element specific and as such this is the first place where defaults can be set on the *config* section. The *variables* and *environment* specified in the declaring plugin's defaults here override the project configuration defaults for the given element ``kind``. Source plugins do not have a ``.yaml`` file, and do not have *variables* or *environment*. 4. Project configuration overrides ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``project.conf`` now gives you :ref:`another opportunity ` to override configuration on a per plugin basis. Configurations specified in the :ref:`elements ` or :ref:`sources ` sections of the ``project.conf`` will override the given plugin's defaults. In this phase, it is possible to override any configurations of a given plugin, including configuration in element specific *config* sections. See also :ref:`project_overrides` 5. Plugin declarations ~~~~~~~~~~~~~~~~~~~~~~~ Finally, after having resolved any :ref:`conditionals ` in the parsing phase of loading element declarations; the configurations specified in a ``.bst`` file have the last word on any configuration in the data model. .. _format_directives: Directives ---------- .. _format_directives_conditional: (?) Conditionals ~~~~~~~~~~~~~~~~ The ``(?)`` directive allows expression of conditional statements which test :ref:`project option ` values. The ``(?)`` directive may appear as a key in any dictionary expressed in YAML, and its value is a list of conditional expressions. Each conditional expression must be a single key dictionary, where the key is the conditional expression itself, and the value is a dictionary to be composited into the parent dictionary containing the ``(?)`` directive if the expression evaluates to a truthy value. **Example:** .. code:: yaml variables: prefix: "/usr" enable-debug: False (?): - relocate == True: prefix: "/opt" - debug == True: enable-debug: True Expressions are evaluated in the specified order, and each time an expression evaluates to a truthy value, its value will be composited to the parent dictionary in advance of processing other elements, allowing for logically overriding previous decisions in the condition list. Nesting of conditional statements is also supported. **Example:** .. code:: yaml variables: enable-logging: False enable-debug: False (?): - logging == True: enable-logging: True (?): - debugging == True: enable-debug: True Conditionals are expressed in a pythonic syntax, the specifics for testing the individually supported option types are described in their :ref:`respective documentation `. Compound conditionals are also allowed. **Example:** .. code:: yaml variables: enable-debug: False (?): - (logging == True and debugging == True): enable-debug: True .. important:: Conditional statements are guaranteed to always be resolved in the context of the project where the conditional statement is *declared*. When :ref:`including a file ` from a subproject, any conditionals expressed in that file will already be resolved in the context of the subproject which the file was included from. .. _format_directives_assertion: (!) Assertions ~~~~~~~~~~~~~~ Assertions allow the project author to abort processing and present a custom error message to the user building their project. This is only useful when used with conditionals, allowing the project author to assert some invalid configurations. **Example:** .. code:: yaml variables: (?): - (logging == False and debugging == True): (!): | Impossible to print any debugging information when logging is disabled. .. _format_directives_list_prepend: (<) List Prepend ~~~~~~~~~~~~~~~~ Indicates that the list should be prepended to the target list, instead of the default behavior which is to replace the target list. **Example:** .. code:: yaml config: configure-commands: # Before configuring, lets make sure we're using # the latest config.sub & config.guess (<): - cp %{datadir}/automake-*/config.{sub,guess} . .. _format_directives_list_append: (>) List Append ~~~~~~~~~~~~~~~ Indicates that the list should be appended to the target list, instead of the default behavior which is to replace the target list. **Example:** .. code:: yaml public: bst: split-rules: devel: # This element also adds some extra stubs which # need to be included in the devel domain (>): - "%{libdir}/*.stub" .. _format_directives_list_overwrite: (=) List Overwrite ~~~~~~~~~~~~~~~~~~ Indicates that the list should be overwritten completely. This exists mostly for completeness, and we recommend using literal lists most of the time instead of list overwrite directives when the intent is to overwrite a list. This has the same behavior as a literal list, except that an error will be triggered in the case that there is no underlying list to overwrite; whereas a literal list will simply create a new list. The added error protection can be useful when intentionally overwriting a list in an element's *public data*, which is mostly free form and not validated. **Example:** .. code:: yaml config: install-commands: # This element's `make install` is broken, replace it. (=): - cp src/program %{bindir} .. _format_directives_include: (@) Include ~~~~~~~~~~~ Indicates that content should be loaded from files. The include directive expects a string, or a list of strings when including multiple files. Each of these strings represent a project relative filename to include. Files can be included from subprojects by prefixing the string with the locally defined :mod:`junction element ` and colon (':'). The include directive can be used in any dictionary declared in the :ref:`project.conf `, in any :ref:`.bst file `, or recursively included in another include file. The including YAML fragment has priority over the files it includes, and overrides any values introduced by the includes. When including multiple files, files are included in the order they are declared in the include list, and each subsequent include file takes priority over the previous one. **Example:** .. code:: yaml environment: (@): junction.bst:includes/environment.bst .. important:: Files included across a junction cannot be used to inform the declaration of a :mod:`junction element `, as this can present a circular dependency. Any :ref:`variables `, :ref:`element overrides `, :ref:`source overrides ` or :ref:`mirrors ` used in the declaration of a junction must be declared in the :ref:`project.conf ` or in included files which are local to the project declaring the junction itself. apache-buildstream-27ae392/doc/source/format_project.rst000066400000000000000000001135761514607367700234540ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _projectconf: Project configuration ===================== The project configuration file should be named ``project.conf`` and be located at the project root. It holds information such as Source aliases relevant for the sources used in the given project as well as overrides for the configuration of element types used in the project. Values specified in the project configuration override any of the default BuildStream project configuration, which is included :ref:`here ` for reference. .. _project_essentials: Essentials ---------- .. _project_format_name: Project name ~~~~~~~~~~~~ The project name is a unique symbol for your project and will be used to distinguish your project from others in user preferences, namespacing of your project's artifacts in shared artifact caches, and in any case where BuildStream needs to distinguish between multiple projects. The first thing to setup in your ``project.conf`` should be the name of your project. .. code:: yaml name: my-project-name The project name may contain alphanumeric characters, dashes and underscores, and may not start with a leading digit. .. attention:: The project name must be specified in the ``project.conf`` and cannot be :ref:`included ` from a separate file. .. _project_min_version: Minimum version ~~~~~~~~~~~~~~~ The BuildStream format is guaranteed to be backwards compatible with any earlier minor point releases, which is to say that BuildStream 1.4 can read projects written for BuildStream 1.0, and that BuildStream 2.2 can read projects written for BuildStream 2.0. Projects are required to specify the minimum version of BuildStream which it requires, this allows project authors to convey a useful error message to their users and peers, in the case that a user needs to get a newer version of BuildStream in order to work with a given project. The project's minimum required BuildStream version must be specified in ``project.conf`` using the ``min-version`` field, e.g.: .. code:: yaml # This project uses features which were added in 2.2 min-version: 2.2 It is recommended that when using new features, always consult this documentation and observe which BuildStream version a feature you are using was added in. If a feature in the BuildStream YAML format is not documented with a specific *Since* version, you can assume that it has been there from the beginning. .. note:: External :mod:`Element ` and :mod:`Source ` plugins also implement their own YAML configuration fragments and as such are revisioned separately from the core format. .. attention:: The ``min-version`` must be specified in the ``project.conf`` and cannot be :ref:`included ` from a separate file. .. _project_element_path: Element path ~~~~~~~~~~~~ To allow the user to structure their project nicely, BuildStream allows the user to specify a project subdirectory where element ``.bst`` files are stored. .. code:: yaml element-path: elements Note that elements are referred to by their relative paths, whenever elements are referred to in a ``.bst`` file or on the command line. .. attention:: The ``element-path`` can only be specified in the ``project.conf`` and cannot be :ref:`included ` from a separate file. .. _project_format_ref_storage: Ref storage ~~~~~~~~~~~ By default, BuildStream expects to read and write source references directly in the :ref:`source declaration `, but this can be inconvenient and prohibitive in some workflows. Alternatively, BuildStream allows source references to be stored centrally in a :ref:`project.refs file ` in the toplevel :ref:`project directory `. This can be controlled with the ``ref-storage`` option, which is allowed to be configured with the following values: * ``inline`` Source references are stored directly in the :ref:`source declaration ` * ``project.refs`` Source references are stored in the ``project.refs`` file, and junction source references are stored in the ``junction.refs`` file. To enable storing of source references in ``project.refs``, add the following to your ``project.conf``: .. code:: yaml ref-storage: project.refs .. attention:: **Storing subproject source references in project.refs** When using the ``project.refs`` file, it is possible to override the references in subprojects by editing the ``project.refs`` file directly or by using :ref:`bst source track --cross-junctions `, this can be practical to try out fresher versions of components which are maintained in a subproject. It should be noted however that overridden subproject source references listed in your ``project.refs`` file will be ignored by projects which use your project as a subproject. .. _configurable_warnings: Configurable Warnings ~~~~~~~~~~~~~~~~~~~~~ Warnings can be configured as fatal using the ``fatal-warnings`` configuration item. When a warning is configured as fatal, where a warning would usually be thrown instead an error will be thrown causing the build to fail. Individual warnings can be configured as fatal by setting ``fatal-warnings`` to a list of warnings. .. code:: yaml fatal-warnings: - overlaps - ref-not-in-track - : BuildStream provides a collection of :class:`Core Warnings ` which may be raised by a variety of plugins. Other configurable warnings are plugin specific and should be noted within their individual documentation. .. _project_source_aliases: Source aliases ~~~~~~~~~~~~~~ In order to abstract the download location of source code and any assets which need to be downloaded, and also as a matter of convenience, BuildStream allows one to create named aliases for URLs which are to be used in the individual ``.bst`` files. .. code:: yaml aliases: foo: git://git.foo.org/ bar: http://bar.com/downloads/ If you want this project's alias definitions to also be used for subprojects, see :ref:`Mapping source aliases of subprojects `. Sandbox options ~~~~~~~~~~~~~~~ Sandbox options for the whole project can be supplied in ``project.conf`` in the same way as in an element. See :ref:`element configuration ` for more detail. .. code:: yaml # Specify a user id and group id to use in the build sandbox. sandbox: build-uid: 1003 build-gid: 1001 .. _project_artifact_cache: Artifact server ~~~~~~~~~~~~~~~ When maintaining a BuildStream project, it can be convenient to downstream users of your project to provide access to a :ref:`cache server ` you maintain. The project can provide *recommended* artifact cache servers through project configuration using the same semantics as one normally uses in the ``servers`` list of the :ref:`cache server user configuration `: .. code:: yaml # # A remote cache from which to download prebuilt artifacts # artifacts: - url: https://foo.com:11001 auth: server-cert: server.crt .. attention:: Unlike user configuration, the filenames provided in the :ref:`auth ` configuration block are relative to the :ref:`project directory `. It is recommended to include public keys such as the ``server-cert`` along with your project so that downstream users can have automatic read access to your project. To provide write access to downstream users, it is recommended that the required private keys such as the ``client-key`` be provided to users out of band, and require that users configure write access separately in their own :ref:`user configuration `. .. _project_source_cache: Source cache server ~~~~~~~~~~~~~~~~~~~ In the same way as artifact cache servers, the project can provide *recommended* source cache servers through project configuration using the same semantics as one normally uses in the ``servers`` list of the :ref:`cache server user configuration `: .. code:: yaml # # A remote cache from which to download prestaged sources # source-caches: - url: https://foo.com:11001 auth: server-cert: server.crt .. attention:: Unlike user configuration, the filenames provided in the :ref:`auth ` configuration block are relative to the :ref:`project directory `. It is recommended to include public keys such as the ``server-cert`` along with your project so that downstream users can have automatic read access to your project. To provide write access to downstream users, it is recommended that the required private keys such as the ``client-key`` be provided to users out of band, and require that users configure write access separately in their own :ref:`user configuration `. .. _project_essentials_mirrors: Mirrors ~~~~~~~ A list of mirrors can be defined that couple a location to a mapping of aliases to a list of URIs, e.g. .. code:: yaml mirrors: - name: middle-earth aliases: foo: - http://www.middle-earth.com/foo/1 - http://www.middle-earth.com/foo/2 bar: - http://www.middle-earth.com/bar/1 - http://www.middle-earth.com/bar/2 - name: oz aliases: foo: - http://www.oz.com/foo bar: - http://www.oz.com/bar The order that the mirrors (and the URIs therein) are consulted is in the order they are defined when fetching, and in reverse-order when tracking. The mirrors can be overridden on a per project basis using :ref:`user configuration `. One can also specify which mirror should be used first in the :ref:`user configuration `, or using the :ref:`--default-mirror ` command-line argument. If you want this project's mirrors to also be used for subprojects, see :ref:`Mapping source aliases of subprojects `. .. _project_plugins: Loading plugins --------------- If your project makes use of any custom :mod:`Element ` or :mod:`Source ` plugins, then the project must inform BuildStream of the plugins it means to make use of and the origin from which they can be loaded. Note that plugins with the same name from different origins are not permitted. .. attention:: The plugins can only be specified in the ``project.conf`` and cannot be :ref:`included ` from a separate file. .. _project_plugins_local: Local plugins ~~~~~~~~~~~~~ Local plugins are expected to be found in a subdirectory of the actual BuildStream project. :mod:`Element ` and :mod:`Source ` plugins should be stored in separate directories to avoid namespace collisions, you can achieve this by specifying a separate *origin* for sources and elements. .. code:: yaml plugins: - origin: local path: plugins/sources # We want to use the `mysource` source plugin located in our # project's `plugins/sources` subdirectory. sources: - mysource There is no strict versioning policy for plugins loaded from the local origin because the plugin is provided with the project data and as such, it is considered to be completely deterministic. Usually your project will be managed by a VCS like git, and any changes to your local plugins may have an impact on your project, such as changes to the artifact cache keys produced by elements which use these plugins. Changes to plugins might provide new YAML configuration options, changes in the semantics of existing configurations or even removal of existing YAML configurations. .. _project_plugins_pip: Pip plugins ~~~~~~~~~~~ Plugins loaded from the ``pip`` origin are expected to be installed separately on the host operating system using python's package management system. .. code:: yaml plugins: - origin: pip # Specify the name of the python package containing # the plugins we want to load. The name one would use # on the `pip install` command line. # package-name: potato # We again must specify specifically which plugins we # want loaded from this origin. # elements: - starch Unlike local plugins, plugins loaded from the ``pip`` origin are loaded from the active *python environment*, and as such you do not usually have full control over the plugins your project uses unless one uses strict :ref:`version constraints `. The official plugin packages maintained by the BuildStream community are guaranteed to be fully API stable. If one chooses to load these plugins from the ``pip`` origin, then it is recommended to use *minimal bound dependency* :ref:`constraints ` when using official plugin packages so as to be sure that you have access to all the features you intend to use in your project. .. _project_plugins_pip_version_constraints: Versioning constraints '''''''''''''''''''''' When loading plugins from the ``pip`` plugin origin, it is possible to specify constraints on the versions of packages you want to load your plugins from. The syntax for specifying versioning constraints is the same format supported by the ``pip`` package manager. .. note:: In order to be certain that versioning constraints work properly, plugin packages should be careful to adhere to `PEP 440, Version Identification and Dependency Specification `_. Here are a couple of examples: **Specifying minimal bound dependencies**: .. code:: yaml plugins: - origin: pip # This project uses the API stable potato project and # requires features from at least version 1.2 # package-name: potato>=1.2 **Specifying exact versions**: .. code:: yaml plugins: - origin: pip # This project requires plugins from the potato # project at exactly version 1.2.3 # package-name: potato==1.2.3 **Specifying version constraints**: .. code:: yaml plugins: - origin: pip # This project requires plugins from the potato # project from version 1.2.3 onward until 1.3. # package-name: potato>=1.2.3,<1.3 .. important:: **Unstable plugin packages** When using unstable plugins loaded from the ``pip`` origin, the installed plugins can sometimes be incompatible with your project. **Use virtual environments** Your operating system's default python environment can only have one version of a given package installed at a time, if you work on multiple BuildStream projects on the same host, they may not agree on which versions of plugins to use. In order to guarantee that you can use a specific version of a plugin, you may need to install BuildStream into a `virtual environment `_ in order to control which python package versions are available when using your project. Follow `these instructions `_ to install BuildStream in a virtual environment. **Possible junction conflicts** If you have multiple projects which are connected through :mod:`junction ` elements, these projects can disagree on which version of a plugin is needed from the ``pip`` origin. Since only one version of a given plugin *package* can be installed at a time in a given *python environment*, you must ensure that all projects connected through :mod:`junction ` elements agree on which versions of API unstable plugin packages to use. .. _project_plugins_junction: Junction plugins ~~~~~~~~~~~~~~~~ Junction plugins are loaded from another project which your project has a :mod:`junction ` declaration for. Plugins are loaded directly from the referenced project, the source and element plugins listed will simply be loaded from the subproject regardless of how they were defined in that project. Plugins loaded from a junction might even come from another junction and be *deeply nested*. .. code:: yaml plugins: - origin: junction # Specify the local junction name declared in your # project as the origin from where to load plugins from. # junction: subproject-junction.bst # Here we want to get the `frobnicate` element # from the subproject and use it in our project. # elements: - frobnicate Plugins loaded across junction boundaries will be loaded in the context of your project, and any default values set in the ``project.conf`` of the junctioned project will be ignored when resolving the defaults provided with element plugins. It is recommended to use :ref:`include directives ` in the case that the referenced plugins from junctioned projects depend on variables defined in the project they come from, in this way you can include variables needed by your plugins into your own ``project.conf``. .. tip:: **Distributing plugins as projects** It is encouraged that people use BuildStream projects to distribute plugins which are intended to be shared among projects, especially when these plugins are not guaranteed to be completely API stable. This can still be done while also distributing your plugins as :ref:`pip packages ` at the same time. This can be achieved by simply creating a repository or tarball which contains only the plugins you want to distribute, along with a ``project.conf`` file declaring these plugins as :ref:`local plugins `. Using plugins which are distributed as local plugins in a BuildStream project ensures that you always have full control over which exact plugin your project is using at all times, without needing to store the plugin as a :ref:`local plugin ` in your own project. .. _project_plugins_deprecation: Suppressing deprecation warnings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Plugins can be deprecated over time, and using deprecated plugins will trigger a warning when loading elements and sources which use deprecated plugin kinds. These deprecation warnings can be suppressed for the entire plugin origin or on a per plugin kind basis. To suppress all deprecation warnings from the origin, set the ``allow-deprecated`` flag for the origin as follows: .. code:: yaml plugins: - origin: local path: plugins/sources # Suppress deprecation warnings for any plugins loaded here allow-deprecated: True sources: - mysource In order to suppress deprecation warnings for a single element or source kind within an origin, you will have to use a dictionary to declare the specific plugin kind and set the ``allow-deprecated`` flag on that dictionary as follows: .. code:: yaml plugins: - origin: pip package-name: potato # Here we use a dictionary to declare the "starch" # element kind, and specify that it is allowed to # be deprecated. # elements: - kind: starch allow-deprecated: True .. _project_options: Options ------- Options are how BuildStream projects can define parameters which can be configured by users invoking BuildStream to build your project. Options are declared in the ``project.conf`` in the main ``options`` dictionary. .. code:: yaml options: debug: type: bool description: Whether to enable debugging default: False Project options can be specified on the command line using :ref:`bst --option ... ` .. note:: The name of the option may contain alphanumeric characters underscores, and may not start with a leading digit. Common properties ~~~~~~~~~~~~~~~~~ All option types accept the following common attributes * ``type`` Indicates the type of option to declare * ``description`` A description of the meaning of the option * ``variable`` Optionally indicate a :ref:`variable ` name to export the option to. A string form of the selected option will be used to set the exported value. If used, this value will override any existing value for the variable declared in ``project.conf``, and will be overridden in the regular :ref:`composition order `. .. note:: The name of the variable to export may contain alphanumeric characters, dashes, underscores, and may not start with a leading digit. .. _project_options_bool: Boolean ~~~~~~~ The ``bool`` option type allows specifying boolean values which can be cased in conditional expressions. **Declaring** .. code:: yaml options: debug: type: bool description: Whether to enable debugging default: False **Evaluating** Boolean options can be tested in expressions with equality tests: .. code:: yaml variables: enable-debug: False (?): - debug == True: enable-debug: True Or simply treated as truthy values: .. code:: yaml variables: enable-debug: False (?): - debug: enable-debug: True **Exporting** When exporting boolean options as variables, a ``True`` option value will be exported as ``1`` and a ``False`` option as ``0`` .. _project_options_enum: Enumeration ~~~~~~~~~~~ The ``enum`` option type allows specifying a string value with a restricted set of possible values. **Declaring** .. code:: yaml options: loglevel: type: enum description: The logging level values: - debug - info - warning default: info **Evaluating** Enumeration options must be tested as strings in conditional expressions: .. code:: yaml variables: enable-debug: False (?): - loglevel == "debug": enable-debug: True **Exporting** When exporting enumeration options as variables, the value is exported as a variable directly, as it is a simple string. .. _project_options_flags: Flags ~~~~~ The ``flags`` option type allows specifying a list of string values with a restricted set of possible values. In contrast with the ``enum`` option type, the *default* value need not be specified and will default to an empty set. **Declaring** .. code:: yaml options: logmask: type: flags description: The logging mask values: - debug - info - warning default: - info **Evaluating** Options of type ``flags`` can be tested in conditional expressions using a pythonic *in* syntax to test if an element is present in a set: .. code:: yaml variables: enable-debug: False (?): - ("debug" in logmask): enable-debug: True **Exporting** When exporting flags options as variables, the value is exported as a comma separated list of selected value strings. .. _project_options_arch: Architecture ~~~~~~~~~~~~ The ``arch`` option type is a special enumeration option which defaults via `uname -m` results to the following list. * aarch32 * aarch64 * aarch64-be * power-isa-be * power-isa-le * sparc-v9 * x86-32 * x86-64 The reason for this, opposed to using just `uname -m`, is that we want an OS-independent list, as well as several results mapping to the same architecture (e.g. i386, i486 etc. are all x86-32). It does not support assigning any default in the project configuration. .. code:: yaml options: machine_arch: type: arch description: The machine architecture values: - aarch32 - aarch64 - x86-32 - x86-64 Architecture options can be tested with the same expressions as other Enumeration options. .. _project_options_os: OS ~~ The ``os`` option type is a special enumeration option, which defaults to the results of `uname -s`. It does not support assigning any default in the project configuration. .. code:: yaml options: machine_os: type: os description: The machine OS values: - Linux - SunOS - Darwin - FreeBSD Os options can be tested with the same expressions as other Enumeration options. .. _project_options_element_mask: Element mask ~~~~~~~~~~~~ The ``element-mask`` option type is a special Flags option which automatically allows only element names as values. .. code:: yaml options: debug_elements: type: element-mask description: The elements to build in debug mode This can be convenient for automatically declaring an option which might apply to any element, and can be tested with the same syntax as other Flag options. .. code:: yaml variables: enable-debug: False (?): - ("element.bst" in debug_elements): enable-debug: True .. _project_junctions: Junctions --------- In this section of ``project.conf``, we can define the relationship a project has with :mod:`junction ` elements in the same project, or even in subprojects. Sometimes when your project has multiple :mod:`junction ` elements, a situation can arise where you have multiple instances of the same project loaded at the same time. In most cases, you will want to reconcile this conflict by ensuring that your projects share the same junction. In order to reconcile conflicts by ensuring nested junctions to the same project are shared, please refer to :ref:`the documentation on nested junctions `. In some exceptional cases, it is entirely intentional and appropriate to use the same project more than once in the same build pipeline. The attributes in the ``junctions`` group here in ``project.conf`` provide some tools you can use to explicitly allow the coexistence of the same project multiple times. Duplicate junctions ~~~~~~~~~~~~~~~~~~~ In the case that you are faced with an error due to subprojects sharing a common sub-subproject, you can use the ``duplicates`` configuration in order to allow the said project to be loaded twice. **Example**: .. code:: yaml junctions: duplicates: # Here we use the packaging tooling completely separately from # the payload that we are packaging, they are never staged to # the same location in a given sandbox, and as such we would # prefer to allow the 'runtime' project to be loaded separately. # # This statement will ensure that loading the 'runtime' project # from these two locations will not produce any errors. # runtime: - payload.bst:runtime.bst - packaging.bst:runtime.bst When considering duplicated projects in the same pipeline, all instances of the said project need to be marked as ``duplicates`` in order to avoid a *conflicting junction error* at load time. .. tip:: The declaration of ``duplicates`` is inherited by any dependant projects which may later decide to depend on your project. If you depend on a project which itself has ``duplicates``, and you need to duplicate it again, then you only need to declare the new duplicate, you do not need to redeclare duplicates redundantly. Internal junctions ~~~~~~~~~~~~~~~~~~ Another way to avoid *conflicting junction errors* when you know that your subproject should not conflict with other instances of the same subproject, is to declare the said subproject as *internal*. **Example**: .. code:: yaml junctions: # Declare this subproject as "internal" because we know # that we only use it for build dependencies, and as such # we know that it cannot collide with elements in dependant # projects. # internal: - special-compiler.bst When compared to *duplicates* above, *internal* projects have the advantage of never producing any *conflicting junction errors* in dependant projects (reverse dependency projects). This approach is preferrable in cases where you know for sure that dependant projects will not be depending directly on elements from your internal subproject. .. attention:: Declaring a junction as *internal* is a promise that dependant projects will not accrue runtime dependencies on elements in your *internal* subproject. .. _project_junctions_source_aliases: Mapping source aliases of subprojects ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :mod:`junction ` elements allow source aliases of subprojects to be mapped to aliases of the parent project. This makes it possible to control the translation of aliases to URLs including mirror configuration across multiple project levels. To ensure that there are mappings for all aliases of all subprojects, you can set the ``disallow-subproject-uris`` flag in the ``junctions`` group here in ``project.conf``. top-level .. code:: yaml junctions: disallow-subproject-uris: True This will raise an error if an alias without a mapping is encountered. This flag is applied recursively across all junctions. It also configures ``unaliased-url`` as a fatal warning in all subprojects to ensure that the current project is in full control over all source URLs. As the fatal warning configuration contributes to the cache key, this flag will affect the cache key of subprojects that haven't already configured ``unaliased-url`` as a fatal warning. .. _project_defaults: Element default configuration ----------------------------- The ``project.conf`` plays a role in defining elements by providing default values and also by overriding values declared by plugins on a plugin wide basis. See the :ref:`composition ` documentation for more detail on how elements are composed. .. _project_defaults_variables: Variables ~~~~~~~~~ The defaults for :ref:`Variables ` used in your project is defined here. .. code:: yaml variables: prefix: "/usr" .. _project_defaults_environment: Environment ~~~~~~~~~~~ The defaults environment for the build sandbox is defined here. .. code:: yaml environment: PATH: /usr/bin:/bin:/usr/sbin:/sbin Additionally, the special ``environment-nocache`` list which specifies which environment variables do not affect build output, and are thus not considered in the calculation of artifact keys can be defined here. .. code:: yaml environment-nocache: - MAXJOBS Note that the ``environment-nocache`` list only exists so that we can control parameters such as ``make -j ${MAXJOBS}``, allowing us to control the number of jobs for a given build without affecting the resulting cache key. .. _project_split_rules: Split rules ~~~~~~~~~~~ The project wide :ref:`split rules ` defaults can be specified here. .. code:: yaml split-rules: devel: - | %{includedir} - | %{includedir}/** - | %{libdir}/lib*.a - | %{libdir}/lib*.la .. _project_overrides: Overriding plugin defaults -------------------------- Base attributes declared by element and source plugins can be overridden on a project wide basis. This section explains how to make project wide statements which augment the configuration of an element or source plugin. .. _project_element_overrides: Element overrides ~~~~~~~~~~~~~~~~~ The elements dictionary can be used to override variables, environments or plugin specific configuration data as shown below. .. code:: yaml elements: # Override default values for all autotools elements autotools: variables: bindir: "%{prefix}/bin" config: configure-commands: ... environment: PKG_CONFIG_PATH=%{libdir}/pkgconfig .. _project_source_overrides: Source overrides ~~~~~~~~~~~~~~~~ The sources dictionary can be used to override source plugin specific configuration data as shown below. .. code:: yaml sources: # Override default values for all git sources git: config: checkout-submodules: False .. _project_shell: Customizing the shell --------------------- Since BuildStream cannot know intimate details about your host or about the nature of the runtime and software that you are building, the shell environment for debugging and testing applications may need some help. The ``shell`` section allows some customization of the shell environment. Interactive shell command ~~~~~~~~~~~~~~~~~~~~~~~~~ By default, BuildStream will use ``sh -i`` when running an interactive shell, unless a specific command is given to the ``bst shell`` command. BuildStream will automatically set a convenient prompt via the ``PS1`` environment variable for interactive shells; which might be overwritten depending on the shell you use in your runtime. If you are using ``bash``, we recommend the following configuration to ensure that the customized prompt is not overwritten: .. code:: yaml shell: # Specify the command to run by default for interactive shells command: [ 'bash', '--noprofile', '--norc', '-i' ] Environment assignments ~~~~~~~~~~~~~~~~~~~~~~~ In order to cooperate with your host environment, a debugging shell sometimes needs to be configured with some extra knowledge inheriting from your host environment. This can be achieved by setting up the shell ``environment`` configuration, which is expressed as a dictionary very similar to the :ref:`default environment `, except that it supports host side environment variable expansion in values. For example, to share your host ``DISPLAY`` and ``DBUS_SESSION_BUS_ADDRESS`` environments with debugging shells for your project, specify the following: .. code:: yaml shell: # Share some environment variables from the host environment environment: DISPLAY: '$DISPLAY' DBUS_SESSION_BUS_ADDRESS: '$DBUS_SESSION_BUS_ADDRESS' Or, a more complex example is how one might share the host pulseaudio server with a ``bst shell`` environment: .. code:: yaml shell: # Set some environment variables explicitly environment: PULSE_SERVER: 'unix:${XDG_RUNTIME_DIR}/pulse/native' Host files ~~~~~~~~~~ It can be useful to share some files on the host with a shell so that it can integrate better with the host environment. The ``host-files`` configuration allows one to specify files and directories on the host to be bind mounted into the sandbox. .. warning:: One should never mount directories where one expects to find data and files which belong to the user, such as ``/home`` on POSIX platforms. This is because the unsuspecting user may corrupt their own files accidentally as a result. Instead users can use the ``--mount`` option of ``bst shell`` to mount data into the shell. The ``host-files`` configuration is an ordered list of *mount specifications*. Members of the list can be *fully specified* as a dictionary, or a simple string can be used if only the defaults are required. The fully specified dictionary has the following members: * ``path`` The path inside the sandbox. This is the only mandatory member of the mount specification. * ``host_path`` The host path to mount at ``path`` in the sandbox. This will default to ``path`` if left unspecified. * ``optional`` Whether the mount should be considered optional. This is ``False`` by default. Here is an example of a *fully specified mount specification*: .. code:: yaml shell: # Mount an arbitrary resolv.conf from the host to # /etc/resolv.conf in the sandbox, and avoid any # warnings if the host resolv.conf doesnt exist. host-files: - host_path: '/usr/local/work/etc/resolv.conf' path: '/etc/resolv.conf' optional: True Here is an example of using *shorthand mount specifications*: .. code:: yaml shell: # Specify a list of files to mount in the sandbox # directory from the host. # # If these do not exist on the host, a warning will # be issued but the shell will still be launched. host-files: - '/etc/passwd' - '/etc/group' - '/etc/resolv.conf' Host side environment variable expansion is also supported: .. code:: yaml shell: # Mount a host side pulseaudio server socket into # the shell environment at the same location. host-files: - '${XDG_RUNTIME_DIR}/pulse/native' .. _project_default_targets: Default targets --------------- When running BuildStream commands from a project directory or subdirectory without specifying any target elements on the command line, the default targets of the project will be used. The default targets can be configured in the ``defaults`` section as follows: .. code:: yaml defaults: # List of default target elements targets: - app.bst If no default targets are configured in ``project.conf``, BuildStream commands will default to all ``.bst`` files in the configured element path. Commands that cannot support junctions as target elements (``bst build``, ``bst artifact push``, and ``bst artifact pull``) ignore junctions in the list of default targets. When running BuildStream commands from a workspace directory (that is not a BuildStream project directory), project default targets are not used and the workspace element will be used as the default target instead. ``bst artifact checkout``, ``bst source checkout``, and ``bst shell`` are currently limited to a single target element and due to this, they currently do not use project default targets. However, they still use the workspace element as default target when run from a workspace directory. .. _project_builtin_defaults: Builtin defaults ---------------- BuildStream defines some default values for convenience, the default values overridden by your project's ``project.conf`` are presented here: .. literalinclude:: ../../src/buildstream/data/projectconfig.yaml :language: yaml apache-buildstream-27ae392/doc/source/format_project_refs.rst000066400000000000000000000063341514607367700244640ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _projectrefs: The project.refs file ===================== If one has elected to store source references in a single ``project.refs`` file, then it will be stored at the toplevel of your project directory adjacent to ``project.conf``. This can be configured in your project using the :ref:`ref-storage configuration ` Sources for :mod:`junction ` elements are stored separately in an adjacent ``junction.refs`` file of the same format. .. _projectrefs_basics: Basic behavior -------------- When a ``project.refs`` file is in use, any source references found in the :ref:`inline source declarations ` are considered invalid and will be ignored, and a warning will be emitted for them. When ``bst source track`` is run for your project, the ``project.refs`` file will be updated instead of the inline source declarations. In the absence of a ``project.refs`` file, ``bst source track`` will create one automatically with the tracking results. An interesting property of ``project.refs`` is that it allows for *cross junction tracking*. This is to say that it is possible to override the *ref* of a given source in a project that your project depends on via a :mod:`junction `, without actually modifying the junctioned project. .. _projectrefs_format: Format ------ The ``project.refs`` uses the same YAML format used throughout BuildStream, and supports the same :ref:`directives ` which apply to ``project.conf`` and element declaration files (i.e. *element.bst* files). The ``project.refs`` file format itself is very simple, it contains a single ``projects`` key at the toplevel, which is a dictionary of :ref:`project names `. Each *project name* is a dictionary of *element names*, and each *element name* holds a list of dictionaries corresponding to the element's :ref:`sources `. **Example** .. code:: yaml # Main toplevel "projects" key projects: # The local project's name is "core" core: # A dictionary of element names base/automake.bst: # A list of sources corresponding to the element # in the same order in which they were declared. # # The values of this list are dictionaries of the # symbolic "ref" portion understood by the given # source plugin implementation. # - ref: af6ba39142220687c500f79b4aa2f181d9b24e4... # The "core" project depends on the "bootstrap" project, # here we are allowed to override the refs for the projects # we depend on through junctions. bootstrap: zlib.bst: - ref: 4ff941449631ace0d4d203e3483be9dbc9da4540... apache-buildstream-27ae392/doc/source/format_public.rst000066400000000000000000000066751514607367700232650ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _public_builtin: Builtin public data =================== Elements can provide public data which can be read by other elements later in the pipeline, the format for exposing public data on a given element is :ref:`described here `. Any element may use public data for whatever purpose it wants, but BuildStream has some built-in expectations of public data, which resides completely in the ``bst`` domain. In this section we will describe the public data in the ``bst`` domain. .. _public_integration: Integration commands -------------------- .. code:: yaml # Specify some integration commands public: bst: integration-commands: - /usr/bin/update-fancy-feature-cache The built-in ``integration-commands`` list indicates that depending elements should run this set of commands before expecting the staged runtime environment to be functional. Typical cases for this include running ``ldconfig`` at the base of a pipeline, or running commands to update various system caches. Integration commands of a given element are automatically run by the :func:`Element.integrate() ` method and are used by various plugins. Notably the :mod:`BuildElement ` derived classes will always integrate the build dependencies after staging and before running any build commands. .. _public_split_rules: Split rules ----------- .. code:: yaml # Specify some split rules public: bst: split-rules: runtime: - | %{bindir}/* - | %{sbindir}/* - | %{libexecdir}/* - | %{libdir}/lib*.so* Split rules indicate how the output of an element can be categorized into *domains*. The ``split-rules`` domains are used by the :func:`Element.stage_artifact() ` method when deciding what domains of an artifact should be staged. The strings listed in each domain are first substituted with the :ref:`variables ` in context of the given element, and then applied as a glob style match, as understood by :func:`utils.glob() ` This is used for creating compositions with the :mod:`compose ` element and can be used by other deployment related elements for the purpose of splitting element artifacts into separate packages. .. _public_overlap_whitelist: Overlap whitelist ----------------- The overlap whitelist indicates which files this element is allowed to overlap over other elements when staged together with other elements. Each item in the overlap whitelist has substitutions applied from :ref:`variables `, and is then applied as a glob-style match (i.e. :func:`utils.glob() `). .. code:: yaml public: bst: overlap-whitelist: - | %{sysconfdir}/* - | /etc/fontcache apache-buildstream-27ae392/doc/source/hacking/000077500000000000000000000000001514607367700212735ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/hacking/coding_guidelines.rst000066400000000000000000001006231514607367700255020ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _coding_guidelines: Coding guidelines ----------------- This section discusses coding style and other guidelines for hacking on BuildStream. This is important to read through for writing any non-trivial patches and especially outlines what people should watch out for when reviewing patches. Much of the rationale behind what is layed out in this section considers good traceability of lines of code with *git blame*, overall sensible modular structure, consistency in how we write code, and long term maintenance in mind. Approximate PEP-8 Style ~~~~~~~~~~~~~~~~~~~~~~~ Python coding style for BuildStream is approximately `pep8 `_. The coding style is automatically enforced by `black `_. Formatting will be checked automatically when running the testsuite on CI. For details on how to format your code locally, see :ref:`formatting code `. .. _contributing_documenting_symbols: Documenting symbols ~~~~~~~~~~~~~~~~~~~ In BuildStream, we maintain what we call a *"Public API Surface"* that is guaranteed to be stable and unchanging across stable releases. The symbols which fall into this special class are documented using Python's standard *docstrings*, while all other internals of BuildStream are documented with comments above the related symbol. When documenting the public API surface which is rendered in the reference manual, we always mention the major version in which the API was introduced, as shown in the examples below. If a public API exists without the *Since* annotation, this is taken to mean that it was available since the first stable major point release (e.g: 2.0). We also always ensure that the **public API** is entirely typed using type annotations inline. The private API *can* be typed inline or in the documentation at the author's discretion. .. note:: Types are checked using `mypy`. You can run it like :command:`tox -e mypy` Here are some examples to get the hang of the format of API documenting comments and docstrings. **Public API Surface method**:: def frobnicate(self, source: Source, *, frobilicious: bool = False) -> Element: """Frobnicates this element with the specified source Args: source: The Source to frobnicate with frobilicious: Optionally specify that frobnication should be performed frobiliciously Returns: The frobnicated version of this Element. *Since: 2.2* """ ... **Internal method**:: # frobnicate(): # # Frobnicates this element with the specified source # # Args: # source: The Source to frobnicate with # frobilicious: Optionally specify that frobnication should be # performed frobiliciously # # Returns: # The frobnicated version of this Element. # def frobnicate(self, source: Source, *, frobilicious: bool = False) -> Element: ... **Public API Surface instance variable**:: def __init__(self, context, element): self.name = self._compute_name(context, element) """The name of this foo *Since: 2.2* """ .. note:: Python does not support docstrings on instance variables, but sphinx does pick them up and includes them in the generated documentation. **Internal instance variable**:: def __init__(self, context, element): self.name = self._compute_name(context, element) # The name of this foo **Internal instance variable (long)**:: def __init__(self, context, element): # This instance variable required a longer explanation, so # it is on a line above the instance variable declaration. self.name = self._compute_name(context, element) **Public API Surface class**:: class Foo(Bar): """The main Foo object in the data model Explanation about Foo. Note that we always document the constructor arguments here, and not beside the __init__ method. Args: context: The invocation Context count: The number to count *Since: 2.2* """ def __init__(self, context: Context, count: int) -> None: ... **Internal class**:: # Foo() # # The main Foo object in the data model # # Args: # context (Context): The invocation Context # count (int): The number to count # class Foo(Bar): ... .. _contributing_class_order: Class structure and ordering ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When creating or modifying an object class in BuildStream, it is important to keep in mind the order in which symbols should appear and keep this consistent. Here is an example to illustrate the expected ordering of symbols on a Python class in BuildStream:: class Foo(Bar): # Public class-wide variables come first, if any. # Private class-wide variables, if any # Now we have the dunder/magic methods, always starting # with the __init__() method. def __init__(self, name): super().__init__() # NOTE: In the instance initializer we declare any instance variables, # always declare the public instance variables (if any) before # the private ones. # # It is preferred to avoid any public instance variables, and # always expose an accessor method for it instead. # # Public instance variables # self.name = name # The name of this foo # # Private instance variables # self._count = 0 # The count of this foo ################################################ # Abstract Methods # ################################################ # NOTE: Abstract methods in BuildStream are allowed to have # default methods. # # Subclasses must NEVER override any method which was # not advertized as an abstract method by the parent class. # frob() # # Implementors should implement this to frob this foo # count times if possible. # # Args: # count (int): The number of times to frob this foo # # Returns: # (int): The number of times this foo was frobbed. # # Raises: # (FooError): Implementors are expected to raise this error # def frob(self, count): # # An abstract method in BuildStream is allowed to have # a default implementation. # self._count = self._do_frobbing(count) return self._count ################################################ # Implementation of abstract methods # ################################################ # NOTE: Implementations of abstract methods defined by # the parent class should NEVER document the API # here redundantly. def frobbish(self): # # Implementation of the "frobbish" abstract method # defined by the parent Bar class. # return True ################################################ # Public Methods # ################################################ # NOTE: Public methods here are the ones which are expected # to be called from outside of this class. # # These, along with any abstract methods, usually # constitute the API surface of this class. # frobnicate() # # Perform the frobnication process on this Foo # # Raises: # (FrobError): In the case that a frobnication error was # encountered # def frobnicate(self): frobnicator.frobnicate(self) # set_count() # # Sets the count of this foo # # Args: # count (int): The new count to set # def set_count(self, count): self._count = count # get_count() # # Accessor for the count value of this foo. # # Returns: # (int): The count of this foo # def get_count(self, count): return self._count ################################################ # Private Methods # ################################################ # NOTE: Private methods are the ones which are internal # implementation details of this class. # # Even though these are private implementation # details, they still MUST have API documenting # comments on them. # _do_frobbing() # # Does the actual frobbing # # Args: # count (int): The number of times to frob this foo # # Returns: # (int): The number of times this foo was frobbed. # def self._do_frobbing(self, count): return count .. _contributing_public_and_private: Public and private symbols ~~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream mostly follows the PEP-8 for defining *public* and *private* symbols for any given class, with some deviations. Please read the `section on inheritance `_ for reference on how the PEP-8 defines public and non-public. * A *public* symbol is any symbol which you expect to be used by clients of your class or module within BuildStream. Public symbols are written without any leading underscores. * A *private* symbol is any symbol which is entirely internal to your class or module within BuildStream. These symbols cannot ever be accessed by external clients or modules. A private symbol must be denoted by a leading underscore. * When a class can have subclasses, then private symbols should be denoted by two leading underscores. For example, the ``Sandbox`` or ``Platform`` classes which have various implementations, or the ``Element`` and ``Source`` classes which plugins derive from. The double leading underscore naming convention invokes Python's name mangling algorithm which helps prevent namespace collisions in the case that subclasses might have a private symbol with the same name. In BuildStream, we have what we call a *"Public API Surface"*, as previously mentioned in :ref:`contributing_documenting_symbols`. In the :ref:`next section ` we will discuss the *"Public API Surface"* and outline the exceptions to the rules discussed here. .. _contributing_public_api_surface: Public API surface ~~~~~~~~~~~~~~~~~~ BuildStream exposes what we call a *"Public API Surface"* which is stable and unchanging. This is for the sake of stability of the interfaces which plugins use, so it can also be referred to as the *"Plugin facing API"*. Any symbols which are a part of the *"Public API Surface*" are never allowed to change once they have landed in a stable release version of BuildStream. As such, we aim to keep the *"Public API Surface"* as small as possible at all times, and never expose any internal details to plugins inadvertently. One problem which arises from this is that we end up having symbols which are *public* according to the :ref:`rules discussed in the previous section `, but must be hidden away from the *"Public API Surface"*. For example, BuildStream internal classes need to invoke methods on the ``Element`` and ``Source`` classes, whereas these methods need to be hidden from the *"Public API Surface"*. This is where BuildStream deviates from the PEP-8 standard for public and private symbol naming. In order to disambiguate between: * Symbols which are publicly accessible details of the ``Element`` class, can be accessed by BuildStream internals, but must remain hidden from the *"Public API Surface"*. * Symbols which are private to the ``Element`` class, and cannot be accessed from outside of the ``Element`` class at all. We denote the former category of symbols with only a single underscore, and the latter category of symbols with a double underscore. We often refer to this distinction as *"API Private"* (the former category) and *"Local Private"* (the latter category). Classes which are a part of the *"Public API Surface"* and require this disambiguation were not discussed in :ref:`the class ordering section `, for these classes, the *"API Private"* symbols always come **before** the *"Local Private"* symbols in the class declaration. Modules which are not a part of the *"Public API Surface"* have their Python files prefixed with a single underscore, and are not imported in BuildStream's the master ``__init__.py`` which is used by plugins. .. note:: The ``utils.py`` module is public and exposes a handful of utility functions, however many of the functions it provides are *"API Private"*. In this case, the *"API Private"* functions are prefixed with a single underscore. Any objects which are a part of the *"Public API Surface"* should be exposed via the toplevel ``__init__.py`` of the ``buildstream`` package. File naming convention ~~~~~~~~~~~~~~~~~~~~~~ With the exception of a few helper objects and data structures, we structure the code in BuildStream such that every filename is named after the object it implements. E.g. The ``Project`` object is implemented in ``_project.py``, the ``Context`` object in ``_context.py``, the base ``Element`` class in ``element.py``, etc. As mentioned in the previous section, objects which are not a part of the :ref:`public, plugin facing API surface ` have their filenames prefixed with a leading underscore (like ``_context.py`` and ``_project.py`` in the examples above). When an object name has multiple words in it, e.g. ``ArtifactCache``, then the resulting file is named all in lower case without any underscore to separate words. In the case of ``ArtifactCache``, the filename implementing this object is found at ``_artifactcache/artifactcache.py``. Imports ~~~~~~~ Module imports inside BuildStream are done with relative ``.`` notation: **Good**:: from ._context import Context **Bad**:: from buildstream._context import Context The exception to the above rule is when authoring plugins, plugins do not reside in the same namespace so they must address buildstream in the imports. An element plugin will derive from Element by importing:: from buildstream import Element When importing utilities specifically, don't import function names from there, instead import the module itself:: from . import utils This makes things clear when reading code that said functions are not defined in the same file but come from utils.py for example. .. _contributing_instance_variables: Instance variables ~~~~~~~~~~~~~~~~~~ It is preferred that all instance state variables be declared as :ref:`private symbols `, however in some cases, especially when the state is immutable for the object's life time (like an ``Element`` name for example), it is acceptable to save some typing by using a publicly accessible instance variable. It is never acceptable to modify the value of an instance variable from outside of the declaring class, even if the variable is *public*. In other words, the class which exposes an instance variable is the only one in control of the value of this variable. * If an instance variable is public and must be modified; then it must be modified using a :ref:`mutator `. * Ideally for better encapsulation, all object state is declared as :ref:`private instance variables ` and can only be accessed by external classes via public :ref:`accessors and mutators `. .. note:: In some cases, we may use small data structures declared as objects for the sake of better readability, where the object class itself has no real supporting code. In these exceptions, it can be acceptable to modify the instance variables of these objects directly, unless they are otherwise documented to be immutable. .. _contributing_accessor_mutator: Accessors and mutators ~~~~~~~~~~~~~~~~~~~~~~ An accessor and mutator, are methods defined on the object class to access (get) or mutate (set) a value owned by the declaring class, respectively. An accessor might derive the returned value from one or more of its components, and a mutator might have side effects, or delegate the mutation to a component. Accessors and mutators are always :ref:`public ` (even if they might have a single leading underscore and are considered :ref:`API Private `), as their purpose is to enforce encapsulation with regards to any accesses to the state which is owned by the declaring class. Accessors and mutators are functions prefixed with ``get_`` and ``set_`` respectively, e.g.:: class Foo(): def __init__(self): # Declare some internal state self._count = 0 # get_count() # # Gets the count of this Foo. # # Returns: # (int): The current count of this Foo # def get_foo(self): return self._count # set_count() # # Sets the count of this Foo. # # Args: # count (int): The new count for this Foo # def set_foo(self, count): self._count = count .. attention:: We are aware that Python offers a facility for accessors and mutators using the ``@property`` decorator instead. Do not use the ``@property`` decorator. The decision to use explicitly defined functions instead of the ``@property`` decorator is rather arbitrary, there is not much technical merit to preferring one technique over the other. However as :ref:`discussed below `, it is of the utmost importance that we do not mix both techniques in the same codebase. .. _contributing_abstract_methods: Abstract methods ~~~~~~~~~~~~~~~~ In BuildStream, an *"Abstract Method"* is a bit of a misnomer and does not match up to how Python defines abstract methods, we need to seek out a new nomenclature to refer to these methods. In Python, an *"Abstract Method"* is a method which **must** be implemented by a subclass, whereas all methods in Python can be overridden. In BuildStream, we use the term *"Abstract Method"*, to refer to a method which **can** be overridden by a subclass, whereas it is **illegal** to override any other method. * Abstract methods are allowed to have default implementations. * Subclasses are not allowed to redefine the calling signature of an abstract method, or redefine the API contract in any way. * Subclasses are not allowed to override any other methods. The key here is that in BuildStream, we consider it unacceptable that a subclass overrides a method of its parent class unless the said parent class has explicitly given permission to subclasses to do so, and outlined the API contract for this purpose. No surprises are allowed. Error handling ~~~~~~~~~~~~~~ In BuildStream, all non recoverable errors are expressed via subclasses of the ``BstError`` exception. This exception is handled deep in the core in a few places, and it is rarely necessary to handle a ``BstError``. Raising exceptions '''''''''''''''''' When writing code in the BuildStream core, ensure that all system calls and third party library calls are wrapped in a ``try:`` block, and raise a descriptive ``BstError`` of the appropriate class explaining what exactly failed. Ensure that the original system call error is formatted into your new exception, and that you use the Python ``from`` semantic to retain the original call trace, example:: try: os.utime(self._refpath(ref)) except FileNotFoundError as e: raise ArtifactError("Attempt to access unavailable artifact: {}".format(e)) from e Enhancing exceptions '''''''''''''''''''' Sometimes the ``BstError`` originates from a lower level component, and the code segment which raised the exception did not have enough context to create a complete, informative summary of the error for the user. In these cases it is necessary to handle the error and raise a new one, e.g.:: try: extracted_artifact = self._artifacts.extract(self, cache_key) except ArtifactError as e: raise ElementError("Failed to extract {} while checking out {}: {}" .format(cache_key, self.name, e)) from e Programming errors '''''''''''''''''' Sometimes you are writing code and have detected an unexpected condition, or a broken invariant for which the code cannot be prepared to handle gracefully. In these cases, do **not** raise any of the ``BstError`` class exceptions. Instead, use the ``assert`` statement, e.g.:: assert utils._is_main_process(), \ "Attempted to save workspace configuration from child process" This will result in a ``BUG`` message with the stack trace included being logged and reported in the frontend. BstError parameters ''''''''''''''''''' When raising ``BstError`` class exceptions, there are some common properties which can be useful to know about: * **message:** The brief human readable error, will be formatted on one line in the frontend. * **detail:** An optional detailed human readable message to accompany the **message** summary of the error. This is often used to recommend the user some course of action, or to provide additional context about the error. * **temporary:** Some errors are allowed to be *temporary*, this attribute is only observed from child processes which fail in a temporary way. This distinction is used to determine whether the task should be *retried* or not. An error is usually only a *temporary* error if the cause of the error was a network timeout. * **reason:** A machine readable identifier for the error. This is used for the purpose of regression testing, such that we check that BuildStream has errored out for the expected reason in a given failure mode. Documenting Exceptions '''''''''''''''''''''' We have already seen :ref:`some examples ` of how exceptions are documented in API documenting comments, but this is worth some additional disambiguation. * Only document the exceptions which are raised directly by the function in question. It is otherwise nearly impossible to keep track of what exceptions *might* be raised indirectly by calling the given function. * For a regular public or private method, your audience is a caller of the function; document the exception in terms of what exception might be raised as a result of calling this method. * For an :ref:`abstract method `, your audience is the implementor of the method in a subclass; document the exception in terms of what exception is prescribed for the implementing class to raise. .. _contributing_always_consistent: Always be consistent ~~~~~~~~~~~~~~~~~~~~ There are various ways to define functions and classes in Python, which has evolved with various features over time. In BuildStream, we may not have leveraged all of the nice features we could have, that is okay, and where it does not break API, we can consider changing it. Even if you know there is a *better* way to do a given thing in Python when compared to the way we do it in BuildStream, *do not do it*. Consistency of how we do things in the codebase is more important than the actual way in which things are done, always. Instead, if you like a certain Python feature and think the BuildStream codebase should use it, then propose your change on the `mailing list `_. Chances are that we will reach agreement to use your preferred approach, and in that case, it will be important to apply the change unilaterally across the entire codebase, such that we continue to have a consistent codebase. Avoid tail calling ~~~~~~~~~~~~~~~~~~ With the exception of tail calling with simple functions from the standard Python library, such as splitting and joining lines of text and encoding/decoding text; always avoid tail calling. **Good**:: # Variables that we will need declared up top context = self._get_context() workspaces = context.get_workspaces() ... # Saving the workspace configuration workspaces.save_config() **Bad**:: # Saving the workspace configuration self._get_context().get_workspaces().save_config() **Acceptable**:: # Decode the raw text loaded from a log file for display, # join them into a single utf-8 string and strip away any # trailing whitespace. return '\n'.join([line.decode('utf-8') for line in lines]).rstrip() When you need to obtain a delegate object via an accessor function, either do it at the beginning of the function, or at the beginning of a code block within the function that will use that object. There are several reasons for this convention: * When observing a stack trace, it is always faster and easier to determine what went wrong when all statements are on separate lines. * We always want individual lines to trace back to their origin as much as possible for the purpose of tracing the history of code with *git blame*. One day, you might need the ``Context`` or ``Workspaces`` object in the same function for another reason, at which point it will be unacceptable to leave the existing line as written, because it will introduce a redundant accessor to the same object, so the line written as:: self._get_context().get_workspaces().save_config() Will have to change at that point, meaning we lose the valuable information of which commit originally introduced this call when running *git blame*. * For similar reasons, we prefer delegate objects be accessed near the beginning of a function or code block so that there is less chance that this statement will have to move in the future, if the same function or code block needs the delegate object for any other reason. Asides from this, code is generally more legible and uniform when variables are declared at the beginning of function blocks. Vertical stacking of modules ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ For the sake of overall comprehensiveness of the BuildStream architecture, it is important that we retain vertical stacking order of the dependencies and knowledge of modules as much as possible, and avoid any cyclic relationships in modules. For instance, the ``Source`` objects are owned by ``Element`` objects in the BuildStream data model, and as such the ``Element`` will delegate some activities to the ``Source`` objects in its possession. The ``Source`` objects should however never call functions on the ``Element`` object, nor should the ``Source`` object itself have any understanding of what an ``Element`` is. If you are implementing a low level utility layer, for example as a part of the ``YAML`` loading code layers, it can be tempting to derive context from the higher levels of the codebase which use these low level utilities, instead of defining properly stand alone APIs for these utilities to work: Never do this. Unfortunately, unlike other languages where include files play a big part in ensuring that it is difficult to make a mess; Python, allows you to just call methods on arbitrary objects passed through a function call without having to import the module which defines those methods - this leads to cyclic dependencies of modules quickly if the developer does not take special care of ensuring this does not happen. Minimize arguments in methods ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When creating an object, or adding a new API method to an existing object, always strive to keep as much context as possible on the object itself rather than expecting callers of the methods to provide everything the method needs every time. If the value or object that is needed in a function call is a constant for the lifetime of the object which exposes the given method, then that value or object should be passed in the constructor instead of via a method call. Minimize API surfaces ~~~~~~~~~~~~~~~~~~~~~ When creating an object, or adding new functionality in any way, try to keep the number of :ref:`public, outward facing ` symbols to a minimum, this is important for both :ref:`internal and public, plugin facing API surfaces `. When anyone visits a file, there are two levels of comprehension: * What do I need to know in order to *use* this object. * What do I need to know in order to *modify* this object. For the former, we want the reader to understand with as little effort as possible, what the public API contract is for a given object and consequently, how it is expected to be used. This is also why we :ref:`order the symbols of a class ` in such a way as to keep all outward facing public API surfaces at the top of the file, so that the reader never needs to dig deep into the bottom of the file to find something they might need to use. For the latter, when it comes to having to modify the file or add functionality, you want to retain as much freedom as possible to modify internals, while being sure that nothing external will be affected by internal modifications. Less client facing API means that you have less surrounding code to modify when your API changes. Further, ensuring that there is minimal outward facing API for any module minimizes the complexity for the developer working on that module, by limiting the considerations needed regarding external side effects of their modifications to the module. When modifying a file, one should not have to understand or think too much about external side effects, when the API surface of the file is well documented and minimal. When adding new API to a given object for a new purpose, consider whether the new API is in any way redundant with other API (should this value now go into the constructor, since we use it more than once? could this value be passed along with another function, and the other function renamed, to better suit the new purposes of this module/object?) and repurpose the outward facing API of an object as a whole every time. Avoid transient state on instances ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ At times, it can be tempting to store transient state that is the result of one operation on an instance, only to be retrieved later via an accessor function elsewhere. As a basic rule of thumb, if the value is transient and just the result of one operation, which needs to be observed directly after by another code segment, then never store it on the instance. BuildStream is complicated in the sense that it is multi processed and it is not always obvious how to pass the transient state around as a return value or a function parameter. Do not fall prey to this obstacle and pollute object instances with transient state. Instead, always refactor the surrounding code so that the value is propagated to the desired end point via a well defined API, either by adding new code paths or changing the design such that the architecture continues to make sense. Refactor the codebase as needed ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Especially when implementing features, always move the BuildStream codebase forward as a whole. Taking a short cut is alright when prototyping, but circumventing existing architecture and design to get a feature implemented without re-designing the surrounding architecture to accommodate the new feature instead, is never acceptable upstream. For example, let's say that you have to implement a feature and you've successfully prototyped it, but it launches a ``Job`` directly from a ``Queue`` implementation to get the feature to work, while the ``Scheduler`` is normally responsible for dispatching ``Jobs`` for the elements on a ``Queue``. This means that you've proven that your feature can work, and now it is time to start working on a patch for upstream. Consider what the scenario is and why you are circumventing the design, and then redesign the ``Scheduler`` and ``Queue`` objects to accommodate for the new feature and condition under which you need to dispatch a ``Job``, or how you can give the ``Queue`` implementation the additional context it needs. apache-buildstream-27ae392/doc/source/hacking/grpc_protocols.rst000066400000000000000000000023631514607367700250700ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _protocol_buffers: Generating protocol buffers --------------------------- BuildStream uses protobuf and gRPC for serialization and communication with artifact cache servers. This requires ``.proto`` files and Python code generated from the ``.proto`` files using protoc. All these files live in the ``src/buildstream/_protos`` directory. The generated files are included in the git repository to avoid depending on grpcio-tools for user installations. Regenerating code ~~~~~~~~~~~~~~~~~ When ``.proto`` files are modified, the corresponding Python code needs to be regenerated:: tox -e build-grpc This installs the correct version of ``grpcio-tools`` and regenerates the protobuf and grpc code. apache-buildstream-27ae392/doc/source/hacking/making_releases.rst000066400000000000000000000210571514607367700251630ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _making_releases: Making releases --------------- This is a checklist of activities which must be observed when creating BuildStream releases, it is important to keep this section up to date whenever the release process changes. Requirements ~~~~~~~~~~~~ There are a couple of requirements and accounts required in order to publish a release. * Ability to send email to ``dev@buildstream.apache.org``. * Shell account at ``master.gnome.org``. * Access to the `BuildStream project on PyPI `_ * An email client which still knows how to send emails in plain text. Pre-release changes ~~~~~~~~~~~~~~~~~~~ Before actually rolling the release, here is a list of changes which might need to be done in preparation of the release. * Ensure that the man pages are up to date The man pages are committed to the repository because we are currently unable to integrate this generation into the setuptools build phase, as outlined in issue #8. If any of the user facing CLI has changed, or if any of the related docstrings have changed, then you should :ref:`regenerate the man pages ` and add/commit the results before wrapping a release. * Ensure the documentation session HTML is up to date The session HTML files are committed to the repository for multiple reasons, one of them being that the documentation must be buildable from within a release build environment so that downstream distribution packagers can easily create the docs package. This is currently only needed for the first stable release in a stable line of releases, after this point the API is frozen and will not change for the remainder of the stable release lifetime, so nothing interesting will have changed in these session files. If regeneration is needed, follow :ref:`the instructions above `. * Ensure the NEWS entry is up to date and ready For a stable release where features have not been added, we should at least add some entries about the issues which have been fixed since the last stable release. For development releases, it is worthwhile going over the existing entries and ensuring all the major feature additions are mentioned and there are no redundancies. * Push pre-release changes Now that any final pre-release changes to generated files or NEWS have been made, push these directly to the upstream repository. Do not sit around waiting for CI or approval, these superficial changes do not affect CI and you are intended to push these changes directly to the upstream repository. Release process ~~~~~~~~~~~~~~~ * Ensure that the latest commit is passing in CI Of course, we do not release software which does not pass it's own tests. * Get the list of contributors The list of contributors for a given list is a list of any contributors who have landed any patches since the last release. An easy way to get this list is to ask git to summarize the authors of commits since the *last release tag*. For example, if we are about to create the ``1.1.1`` release, then we need to observe all of the commits since the ``1.1.0`` release: .. code:: shell git shortlog -s 1.1.0...@ At times, the same contributor might make contributions from different machines which they have setup their author names differently, you can see that some of the authors are actually duplicates, then remove the duplicates. * Start composing the release announcement email The first thing to do when composing the release email is to ensure your mail client has disabled any HTML formatting and will safely use plain text only. Try to make the release announcement consistent with other release announcements as much as possible, an example of the email can be `found here `_. The recipient of the email is ``dev@buildstream.apache.org`` and the title of the email should be of the form: ``BuildStream 1.1.1 released``, without any exclamation point. The format of the email is essentially:: Hi all, This is the personalized message written to you about this release. If this is an unstable release, this should include a warning to this effect and an invitation to users to please help us test this release. This is also a good place to highlight specific bug fixes which users may have been waiting for, or highlight a new feature we want users to try out. What is BuildStream ? ===================== This is a concise blurb which describes BuildStream in a couple of sentences, and is taken from the the README.rst. The easiest thing is to just copy this over from the last release email. ================= buildstream 1.1.1 ================= This section is directly copy pasted from the top of the NEWS file Contributors ============ - This is Where - You Put - The Contributor - Names Which - You Extracted - Using git shortlog -s Where can I get it ? ==================== https://download.gnome.org/sources/BuildStream/1.1/ For more information on the BuildStream project, visit our home page at https://buildstream.build/ * Publish the release tag Now that any pre-release changes are upstream, create and push the signed release tag like so: .. code:: shell git tag -s 1.1.1 git push origin 1.1.1 This will trigger the "Release actions" workflow which also takes care of: * uploading Github release artifacts * uploading Python source and binary packages to PyPI * Upload the release tarball First get yourself into a clean repository state, ensure that you don't have any unfinished work or precious, uncommitted files lying around in your checkout and then run: .. code:: shell git clean -xdff Create the tarball with the following command: .. code:: shell python3 setup.py sdist And upload the resulting tarball to the master GNOME server: .. code:: shell scp dist/BuildStream-1.1.1.tar.gz @master.gnome.org: And finally login to your account at ``master.gnome.org`` and run the install scripts to publish the tarball and update the mirrors: .. code:: shell ftpadmin install BuildStream-1.1.1.tar.gz * Send the release email Now that the release tag is up and the tarball is published, you can send the release email. Post-release activities ~~~~~~~~~~~~~~~~~~~~~~~ Once the release has been published, there are some activities which need to be done to ensure everything is up to date. * Check that the release was successfully uploaded to PyPI. If it is an unstable release, make sure the version on PyPI has the correct "dev0" postfix so to avoid being treated as stable. * Update the topic line in the #buildstream IRC channel if needed The IRC channel usually advertizes the latest stable release in the topic line, now is the right time to update it. * Update the website repository The website wants to link to release announcements, but this cannot be automated because we cannot guess what the link to the release email will be in the mailing list archive. Find the URL to the announcement you just published `in the mailing list archives `_, and use that URL to update the ``anouncements.json`` file in the website repository. Commit and push this change to the the ``anouncements.json`` file to the upstream website repository, and gitlab will take care of automatically updating the website accordingly. * Regenerate BuildStream documentation In order to update the badges we use in various documentation which reflects what is the latest stable releases and the latest development snapshots, we simply need to ensure a pipeline runs for the master branch in the BuildStream repository. You can do this by using the "Run Pipeline" feature on the `pipelines page in the gitlab UI `_. apache-buildstream-27ae392/doc/source/hacking/managing_data_files.rst000066400000000000000000000020331514607367700257570ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _managing_data_files: Managing data files ------------------- When adding data files which need to be discovered at runtime by BuildStream, update setup.py accordingly. When adding data files for the purpose of docs or tests, or anything that is not covered by setup.py, update the MANIFEST.in accordingly. At any time, running the following command to create a source distribution should result in creating a tarball which contains everything we want it to include:: ./setup.py sdist apache-buildstream-27ae392/doc/source/hacking/measuring_performance.rst000066400000000000000000000077431514607367700264130ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _measuring_performance: Measuring performance --------------------- Benchmarking framework ~~~~~~~~~~~~~~~~~~~~~~~ BuildStream has a utility to measure performance which is available from a separate repository at https://gitlab.com/BuildStream/benchmarks. This tool allows you to run a fixed set of workloads with multiple versions of BuildStream. From this you can see whether one version performs better or worse than another which is useful when looking for regressions and when testing potential optimizations. For full documentation on how to use the benchmarking tool see the README in the 'benchmarks' repository. Profiling tools ~~~~~~~~~~~~~~~ When looking for ways to speed up the code you should make use of a profiling tool. Python provides `cProfile `_ which gives you a list of all functions called during execution and how much time was spent in each function. Here is an example of running ``bst --help`` under cProfile: python3 -m cProfile -o bst.cprofile -- $(which bst) --help You can then analyze the results interactively using the 'pstats' module: python3 -m pstats ./bst.cprofile For more detailed documentation of cProfile and 'pstats', see: https://docs.python.org/3/library/profile.html. For a richer and interactive visualisation of the `.cprofile` files, you can try `snakeviz `_. You can install it with `pip install snakeviz`. Here is an example invocation: snakeviz bst.cprofile It will then start a webserver and launch a browser to the relevant page. .. note:: If you want to also profile cython files, you will need to add BST_CYTHON_PROFILE=1 and recompile the cython files. ``pip install`` would take care of that. Profiling specific parts of BuildStream with BST_PROFILE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream can also turn on cProfile for specific parts of execution using BST_PROFILE. BST_PROFILE can be set to a section name, or a list of section names separated by ":". You can also use "all" for getting all profiles at the same time. There is a list of topics in `src/buildstream/_profile.py`. For example, running:: BST_PROFILE=load-pipeline bst build bootstrap-system-x86.bst will produce a profile in the current directory for the time take to call most of `initialized`, for each element. These profile files are in the same cProfile format as those mentioned in the previous section, and can be analysed in the same way. Fixing performance issues ~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream uses `Cython `_ in order to speed up specific parts of the code base. .. note:: When optimizing for performance, please ensure that you optimize the algorithms before jumping into Cython code. Cython will make the code harder to maintain and less accessible to all developers. When adding a new cython file to the codebase, you will need to register it in ``setup.py``. For example, for a module ``buildstream._my_module``, to be written in ``src/buildstream/_my_module.pyx``, you would do:: register_cython_module("buildstream._my_module") In ``setup.py`` and the build tool will automatically use your module. .. note:: Please register cython modules at the same place as the others. When adding a definition class to share cython symbols between modules, please document the implementation file and only expose the required definitions. apache-buildstream-27ae392/doc/source/hacking/ui.rst000066400000000000000000000076701514607367700224540ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Contributing to the UI ~~~~~~~~~~~~~~~~~~~~~~ As we wish to cleanly separate BuildStream's core from the frontend, anything user facing should be defined within the modules contained within the `_frontend `_ directory. BuildStream's frontend includes: * Implementation of the command line interface (cli.py) * Logline/text formatting (widget.py) * The main application state (app.py) which initializes the stream to handle logging and user interactions. * Colour profiles (profile.py) * Rendering of the status bar (status.py) * Autocompletion behaviour (completions.py) The command line interface '''''''''''''''''''''''''' All of BuildStream's commands are defined within the module `cli.py `_ -, the main entry point which implements the CLI. The command line interface is generated with `Click `_ - a third party Python package. Click is easy to use and automatically generates help pages. When working with commands, please adhere to the following checklist: 1. All commands should be defined with a help text 2. The command should be placed within the appropriate sub group - If the command manipulates sources, it should be part of the :ref:`source_subcommands`. - If the command manipulates cached artifacts, it should be part of the :ref:`artifact_subcommands`. - If the command has anything to do with workspaces, it should be part of the :ref:`workspace_subcommands`. 3. If the command is intended to work with artifact refs as well as element names, the command's argument should be "artifacts" as this supports the auto-completion of artifact refs. 4. The supported `--deps` options are: "run", "build", "all", "plan" and "none". These are always of type `click.Choice `_ and should always be specified with a default. If the default is "none", note that we use the string "none", not the Python built-in ``None``. In addition to this, the ``show_default`` flag should be set to ``True``. 5. Commands should use the app and go through the stream (via a similarly named method within Stream) in order to communicate to BuildStream's core. Displaying information '''''''''''''''''''''' Output which we wish to display to the user from the frontend should use the implemented classes in widget.py. This module contains classes which represent widgets for displaying information to the user. To report messages back to the frontend, we use the ``Message()`` object which is available from the ``Context``. Supported message types are defined in `_message.py `_ and various uses of the messenger are defined in `_messenger.py `_ The ``Messenger`` class defines various methods which allow us to report back to the frontend in particular ways. The common methods are: * ``Messenger.message()`` - the central point through which all messages pass * ``Messenger.timed_activity()`` - a context manager for performing and logging timed activities. * ``Messenger.simple_task()`` - a Context manager for creating a task to report progress too. apache-buildstream-27ae392/doc/source/hacking/updating_python_deps.rst000066400000000000000000000121051514607367700262530ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _updating_python_deps: Updating BuildStream's Python dependencies ------------------------------------------ BuildStream's Python dependencies are listed in multiple `requirements files `_ present in the ``requirements`` directory. All ``.txt`` files in this directory are generated from the corresponding ``.in`` file, and each ``.in`` file represents a set of dependencies. For example, ``requirements.in`` contains all runtime dependencies of BuildStream. ``requirements.txt`` is generated from it, and contains pinned versions of all runtime dependencies (including transitive dependencies) of BuildStream. When adding a new dependency to BuildStream, or updating existing dependencies, it is important to update the appropriate requirements file accordingly. After changing the ``.in`` file, run the following to update the matching ``.txt`` file:: make -C requirements Adding support for a new Python release --------------------------------------- When a new stable release of Python 3 appears, we must explicitly declare our support for it in the following places. tox.ini ~~~~~~~ The ``tox.ini`` file defines the environments where the BuildStream test suite runs. Every ``py{3.x,3.y}`` list must be updated to contain the new version number such as ``311`` for CPython 3.11. Use ``tox -e py311-nocover`` to run the test suite with the new version of Python. pyproject.toml ~~~~~~~~~~~~~~ Bump cython version ''''''''''''''''''' New releases of Cython must be depended on with new versions of Python in lock step. When supporting a new Python version, it is important to bump the minimal dependency on Cython to a new enough version which also supports the new version of Python. Wheel details ''''''''''''' We produce binary "wheel" packages for each supported version of Python. The cibuildwheel build tool will build for all released versions of Python so no change is needed in the config. However, if you want to test wheel building with a prerelease version of Python you will need to set ``CIBW_PRERELEASE_PYTHONS=1`` in the cibuildwheel environment. .github/compose/ci.docker-compose.yml ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Each binary package is tested in a container, using the `pypa/manylinux `_ images. You need to add a new docker-compose service here -- copy the latest one and update the version number where it appears. .github/workflows/ci.yml and .github/workflows/release.yml ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There is a separate CI job to run each of the above tests. Update the matrix config for the ``test_wheels`` jobs in ``ci.yml`` and ``release.yml`` to add the new Python version. Removing support for a Python release ------------------------------------- tox.ini ~~~~~~~ You will need to update the ``py{3.x,3.y}`` lists to remove the old version. In the ``envlist`` section, make sure the oldest version still has coverage enabled while the other versions are marked ``-nocover``. pyproject.toml ~~~~~~~~~~~~~~ The cibuildwheel tool will produce wheels for all versions of Python supported upstream.. If we drop support for a version before upstream do, update the ``tool.cibuildwheel.skip`` list to skip all platform tags for that version. The glob ``cp36-*`` would skip all CPython 3.6 builds, for example. .github/compose/ci.docker-compose.yml ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Remove the corresponding service. .github/workflows/ci.yml and .github/workflows/release.yml ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Update the matrix config for the `test_wheels` jobs in `ci.yml` and `release.yml` to remove the old Python version. ABI compatibility for binary Python packages -------------------------------------------- The Python binary packages declare system requirements using `platform compatibility tags `_. For linux-gnu systems we use `manylinux_x_y platform tags `_ to specify a minimum GLIBC version. The platform tag is controlled in ``pyproject.toml`` with the ``tool.cibuildwheel.manylinux-x86_64-image`` key. It must correspond with the version of GLIBC used in `buildbox-integration `_ to produce static buildbox binaries that are included in the package. The ``cibuildwheel`` tool uses `auditwheel `_ to ensure the correct platform tag is declared. apache-buildstream-27ae392/doc/source/hacking/using_the_testsuite.rst000066400000000000000000000250601514607367700261260ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _contributing_testing: Using the test suite -------------------- BuildStream uses `tox `_ as a frontend to run the tests which are implemented using `pytest `_. We use pytest for regression tests and testing out the behavior of newly added components. The elaborate documentation for pytest can be found here: http://doc.pytest.org/en/latest/contents.html Don't get lost in the docs if you don't need to, follow existing examples instead. .. _contributing_build_deps: Installing build dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Some of BuildStream's dependencies have non-python build dependencies. When running tests with ``tox``, you will first need to install these dependencies. Exact steps to install these will depend on your operating system. Commands for installing them for some common distributions are listed below. For Fedora-based systems:: dnf install gcc python3-devel For Debian-based systems:: apt install gcc python3-dev Installing runtime dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To be able to run BuildStream as part of the test suite, BuildStream's runtime dependencies must also be installed. Instructions on how to do so can be found in :ref:`install-dependencies`. If you are not interested in running the integration tests, you can skip the installation of ``buildbox-run``. Running tests ~~~~~~~~~~~~~ To run the tests, simply navigate to the toplevel directory of your BuildStream checkout and run:: tox By default, the test suite will be run against every supported python version found on your host. If you have multiple python versions installed, you may want to run tests against only one version and you can do that using the ``-e`` option when running tox:: tox -e py312 If you would like to test and lint at the same time, or if you do have multiple python versions installed and would like to test against multiple versions, then we recommend using `detox `_, just run it with the same arguments you would give `tox`:: detox -e lint,py311,py312 The output of all failing tests will always be printed in the summary, but if you want to observe the stdout and stderr generated by a passing test, you can pass the ``-s`` option to pytest as such:: tox -- -s .. tip:: The ``-s`` option is `a pytest option `_. Any options specified before the ``--`` separator are consumed by ``tox``, and any options after the ``--`` separator will be passed along to pytest. You can always abort on the first failure by running:: tox -- -x Similarly, you may also be interested in the ``--last-failed`` and ``--failed-first`` options as per the `pytest cache `_ documentation. If you want to run a specific test or a group of tests, you can specify a prefix to match. E.g. if you want to run all of the frontend tests you can do:: tox -- tests/frontend/ Specific tests can be chosen by using the :: delimiter after the test module. If you wanted to run the test_build_track test within frontend/buildtrack.py you could do:: tox -- tests/frontend/buildtrack.py::test_build_track When running only a few tests, you may find the coverage and timing output excessive, there are options to trim them. Note that coverage step will fail. Here is an example:: tox -- --no-cov --durations=1 tests/frontend/buildtrack.py::test_build_track We also have a set of slow integration tests that are disabled by default - you will notice most of them marked with SKIP in the pytest output. To run them, you can use:: tox -- --integration In case BuildStream's dependencies were updated since you last ran the tests, you might see some errors like ``pytest: error: unrecognized arguments: --codestyle``. If this happens, you will need to force ``tox`` to recreate the test environment(s). To do so, you can run ``tox`` with ``-r`` or ``--recreate`` option. .. note:: By default, we do not allow use of site packages in our ``tox`` configuration to enable running the tests in an isolated environment. If you need to enable use of site packages for whatever reason, you can do so by passing the ``--sitepackages`` option to ``tox``. Also, you will not need to install any of the build dependencies mentioned above if you use this approach. .. note:: While using ``tox`` is practical for developers running tests in more predictable execution environments, it is still possible to execute the test suite against a specific installation environment using pytest directly:: pytest If you want to run coverage, you will need need to add ``BST_CYTHON_TRACE=1`` to your environment if you also want coverage on cython files. You could then get coverage by running:: BST_CYTHON_TRACE=1 coverage run pytest Note that you will have to have all dependencies installed already, when running tests directly via ``pytest``. This includes the following: * Cython and Setuptools, as build dependencies * Runtime dependencies and test dependencies are specified in requirements files, present in the ``requirements`` subdirectory. Refer to the ``.in`` files for loose dependencies and ``.txt`` files for fixed version of all dependencies that are known to work. * Additionally, if you are running tests that involve external plugins, you will need to have those installed as well. You can also have a look at our tox configuration in ``tox.ini`` file if you are unsure about dependencies. .. tip:: We also have an environment called 'venv' which takes any arguments you give it and runs them inside the same virtualenv we use for our tests:: tox -e venv -- Any commands after ``--`` will be run a virtualenv managed by tox. Running linters ~~~~~~~~~~~~~~~ Linting is performed separately from testing. In order to run the linting step which consists of running the ``pylint`` tool, run the following:: tox -e lint .. tip:: The project specific pylint configuration is stored in the toplevel buildstream directory in the ``.pylintrc`` file. This configuration can be interesting to use with IDEs and other developer tooling. .. _contributing_formatting_code: Formatting code ~~~~~~~~~~~~~~~ Similar to linting, code formatting is also done via a ``tox`` environment. To format the code using the ``black`` tool, run the following:: tox -e format Observing coverage ~~~~~~~~~~~~~~~~~~ Once you have run the tests using `tox` (or `detox`), some coverage reports will have been left behind. To view the coverage report of the last test run, simply run:: tox -e coverage This will collate any reports from separate python environments that may be under test before displaying the combined coverage. Adding tests ~~~~~~~~~~~~ Tests are found in the tests subdirectory, inside of which there is a separate directory for each *domain* of tests. All tests are collected as:: tests/*/*.py If the new test is not appropriate for the existing test domains, then simply create a new directory for it under the tests subdirectory. Various tests may include data files to test on, there are examples of this in the existing tests. When adding data for a test, create a subdirectory beside your test in which to store data. When creating a test that needs data, use the datafiles extension to decorate your test case (again, examples exist in the existing tests for this), documentation on the datafiles extension can be found here: https://pypi.python.org/pypi/pytest-datafiles. Tests that run a sandbox should be decorated with:: @pytest.mark.integration and use the integration cli helper. You must test your changes in an end-to-end fashion. Consider the first end to be the appropriate user interface, and the other end to be the change you have made. The aim for our tests is to make assertions about how you impact and define the outward user experience. You should be able to exercise all code paths via the user interface, just as one can test the strength of rivets by sailing dozens of ocean liners. Keep in mind that your ocean liners could be sailing properly *because* of a malfunctioning rivet. End-to-end testing will warn you that fixing the rivet will sink the ships. The primary user interface is the cli, so that should be the first target 'end' for testing. Most of the value of BuildStream comes from what you can achieve with the cli. We also have what we call a *"Public API Surface"*, as previously mentioned in :ref:`contributing_documenting_symbols`. You should consider this a secondary target. This is mainly for advanced users to implement their plugins against. Note that both of these targets for testing are guaranteed to continue working in the same way across versions. This means that tests written in terms of them will be robust to large changes to the code. This important property means that BuildStream developers can make large refactorings without needing to rewrite fragile tests. Another user to consider is the BuildStream developer, therefore internal API surfaces are also targets for testing. For example the YAML loading code, and the CasCache. Remember that these surfaces are still just a means to the end of providing value through the cli and the *"Public API Surface"*. It may be impractical to sufficiently examine some changes in an end-to-end fashion. The number of cases to test, and the running time of each test, may be too high. Such typically low-level things, e.g. parsers, may also be tested with unit tests; alongside the mandatory end-to-end tests. It is important to write unit tests that are not fragile, i.e. in such a way that they do not break due to changes unrelated to what they are meant to test. For example, if the test relies on a lot of BuildStream internals, a large refactoring will likely require the test to be rewritten. Pure functions that only rely on the Python Standard Library are excellent candidates for unit testing. Unit tests only make it easier to implement things correctly, end-to-end tests make it easier to implement the right thing. apache-buildstream-27ae392/doc/source/hacking/writing_documentation.rst000066400000000000000000000267451514607367700264570ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _writing_documentation: Writing documentation --------------------- BuildStream starts out as a documented project from day one and uses `sphinx `_ to document itself. This section discusses formatting policies for editing files in the ``doc/source`` directory, and describes the details of how the docs are generated so that you can easily generate and view the docs yourself before submitting patches to the documentation. For details on how API documenting comments and docstrings are formatted, refer to the :ref:`documenting section of the coding guidelines `. Documentation formatting policy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The BuildStream documentation style is as follows: * Titles and headings require two leading empty lines above them. Only the first word in a title should be capitalized. * If there is an ``.. _internal_link:`` anchor, there should be two empty lines above the anchor, followed by one leading empty line. * Within a section, paragraphs should be separated by one empty line. * Notes are defined using: ``.. note::`` blocks, followed by an empty line and then indented (3 spaces) text. * Other kinds of notes can be used throughout the documentation and will be decorated in different ways, these work in the same way as ``.. note::`` does. Feel free to also use ``.. attention::`` or ``.. important::`` to call special attention to a paragraph, ``.. tip::`` to give the reader a special tip on how to use an advanced feature or ``.. warning::`` to warn the user about a potential misuse of the API and explain its consequences. * Code blocks are defined using: ``.. code:: LANGUAGE`` blocks, followed by an empty line and then indented (3 spaces) text. Note that the default language is ``python``. * Cross references should be of the form ``:role:`target```. * Explicit anchors can be declared as ``.. _anchor_name:`` on a line by itself. * To cross reference arbitrary locations with, for example, the anchor ``anchor_name``, always provide some explicit text in the link instead of deriving the text from the target, e.g.: ``:ref:`Link text ```. Note that the "_" prefix is not used when referring to the target. For further information about using the reStructuredText with sphinx, please see the `Sphinx Documentation `_. Building Docs ~~~~~~~~~~~~~ Before you can build the docs, you will end to ensure that you have installed the required :ref:`build dependencies ` as mentioned in the testing section above. To build the documentation, just run the following:: tox -e docs This will give you a ``doc/build/html`` directory with the html docs which you can view in your browser locally to test. .. _contributing_session_html: Regenerating session html ''''''''''''''''''''''''' The documentation build will build the session files if they are missing, or if explicitly asked to rebuild. We revision the generated session html files in order to reduce the burden on documentation contributors. To explicitly rebuild the session snapshot html files, it is recommended that you first set the ``BST_SOURCE_CACHE`` environment variable to your source cache, this will make the docs build reuse already downloaded sources:: export BST_SOURCE_CACHE=~/.cache/buildstream/sources To force rebuild session html while building the doc, simply run `tox` with the ``BST_FORCE_SESSION_REBUILD`` environment variable set, like so:: env BST_FORCE_SESSION_REBUILD=1 tox -e docs .. _contributing_man_pages: Man pages ~~~~~~~~~ Unfortunately it is quite difficult to integrate the man pages build into the ``setup.py``, as such, whenever the frontend command line interface changes, the static man pages should be regenerated and committed with that. To do this, make sure you know the version you are generating the docs for, which should usually be the *next* ``major.minor`` version of BuildStream, and then run the following from the the toplevel directory of BuildStream:: tox -e man -- And commit the result, ensuring that you have added anything in the ``man/`` subdirectory, which will be automatically included in the buildstream distribution. User guide ~~~~~~~~~~ The :ref:`user guide ` is comprised of free form documentation in manually written ``.rst`` files and is split up into a few sections, of main interest are the :ref:`tutorial ` and the :ref:`examples `. The distinction of the two categories of user guides is important to understand too. * **Tutorial** The tutorial is structured as a series of exercises which start with the most basic concepts and build upon the previous chapters in order to arrive at a basic understanding of how to create BuildStream projects. This series of examples should be easy enough to complete in a matter of a few hours for a new user, and should provide just enough insight to get the user started in creating their own projects. Going through the tutorial step by step should also result in the user becoming proficient enough with the reference manual to get by on their own. * **Examples** These exist to demonstrate how to accomplish more advanced tasks which are not always obvious and discoverable. Alternatively, these also demonstrate elegant and recommended ways of accomplishing some tasks which could be done in various ways. Guidelines '''''''''' Here are some general guidelines for adding new free form documentation to the user guide. * **Focus on a single subject** It is important to stay focused on a single subject and avoid getting into tangential material when creating a new entry, so that the articles remain concise and the user is not distracted by unrelated subject material. A single tutorial chapter or example should not introduce any additional subject material than the material being added for the given example. * **Reuse existing sample project elements** To help avoid distracting from the topic at hand, it is always preferable to reuse the same project sample material from other examples and only deviate slightly to demonstrate the new material, than to create completely new projects. This helps us remain focused on a single topic at a time, and reduces the amount of unrelated material the reader needs to learn in order to digest the new example. * **Don't be redundant** When something has already been explained in the tutorial or in another example, it is best to simply refer to the other user guide entry in a new example. Always prefer to link to the tutorial if an explanation exists in the tutorial, rather than linking to another example, where possible. * **Link into the reference manual at every opportunity** The format and plugin API is 100% documented at all times. Whenever discussing anything about the format or plugin API, always do so while providing a link into the more terse reference material. We don't want users to have to search for the material themselves, and we also want the user to become proficient at navigating the reference material over time. * **Use concise terminology** As developers, we tend to come up with code names for features we develop, and then end up documenting a new feature in an example. Never use a code name or shorthand to refer to a feature in the user guide, instead always use fully qualified sentences outlining very explicitly what we are doing in the example, or what the example is for in the case of a title. We need to be considerate that the audience of our user guide is probably a proficient developer or integrator, but has no idea what we might have decided to name a given activity. Structure of an example ''''''''''''''''''''''' The :ref:`tutorial ` and the :ref:`examples ` sections of the documentation contain a series of sample projects, each chapter in the tutorial, or standalone example uses a sample project. Here is the the structure for adding new examples and tutorial chapters. * The example has a ``${name}``. * The example has a project users can copy and use. * This project is added in the directory ``doc/examples/${name}``. * The example has a documentation component. * This is added at ``doc/source/examples/${name}.rst`` * An entry for ``examples/${name}`` is added to the toctree in ``doc/source/using_examples.rst`` * This documentation discusses the project elements declared in the project and may provide some BuildStream command examples. * This documentation links out to the reference manual at every opportunity. .. note:: In the case of a tutorial chapter, the ``.rst`` file is added in at ``doc/source/tutorial/${name}.rst`` and an entry for ``tutorial/${name}`` is added to ``doc/source/using_tutorial.rst``. * The example has a CI test component. * This is an integration test added at ``tests/examples/${name}``. * This test runs BuildStream in the ways described in the example and assert that we get the results which we advertize to users in the said examples. Adding BuildStream command output ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ As a part of building the docs, BuildStream will run itself and extract some html for the colorized output which is produced. If you want to run BuildStream to produce some nice html for your documentation, then you can do so by adding new ``.run`` files to the ``doc/sessions/`` directory. Any files added as ``doc/sessions/${example}.run`` will result in generated file at ``doc/source/sessions/${example}.html``, and these files can be included in the reStructuredText documentation at any time with:: .. raw:: html :file: sessions/${example}.html The ``.run`` file format is just another YAML dictionary which consists of a ``commands`` list, instructing the program what to do command by command. Each *command* is a dictionary, the members of which are listed here: * ``directory``: The input file relative project directory. * ``output``: The input file relative output html file to generate (optional). * ``fake-output``: Don't really run the command, just pretend to and pretend this was the output, an empty string will enable this too. * ``command``: The command to run, without the leading ``bst``. * ``shell``: Specifying ``True`` indicates that ``command`` should be run as a shell command from the project directory, instead of a bst command (optional). When adding a new ``.run`` file, one should normally also commit the new resulting generated ``.html`` file(s) into the ``doc/source/sessions-stored/`` directory at the same time, this ensures that other developers do not need to regenerate them locally in order to build the docs. **Example**: .. code:: yaml commands: # Make it fetch first - directory: ../examples/foo command: source fetch hello.bst # Capture a build output - directory: ../examples/foo output: ../source/sessions/foo-build.html command: build hello.bst apache-buildstream-27ae392/doc/source/hacking/writing_plugins.rst000066400000000000000000000054511514607367700252560ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _writing_plugins: Adding core plugins ------------------- This is a checklist of things which need to be done when adding a new core plugin to BuildStream proper. Update documentation index ~~~~~~~~~~~~~~~~~~~~~~~~~~ The documentation generating scripts will automatically pick up your newly added plugin and generate HTML, but will not add a link to the documentation of your plugin automatically. Whenever adding a new plugin, you must add an entry for it in ``doc/source/core_plugins.rst``. Add tests ~~~~~~~~~ Needless to say, all new feature additions need to be tested. For ``Element`` plugins, these usually need to be added to the integration tests. For ``Source`` plugins, the tests are added in two ways: * For most normal ``Source`` plugins, it is important to add a new ``Repo`` implementation for your plugin in the ``tests/testutils/repo/`` directory and update ``ALL_REPO_KINDS`` in ``tests/testutils/repo/__init__.py``. This will include your new ``Source`` implementation in a series of already existing tests, ensuring it works well under normal operating conditions. * For other source plugins, or in order to test edge cases, such as failure modes, which are not tested under the normal test battery, add new tests in ``tests/sources``. Extend the cachekey test ~~~~~~~~~~~~~~~~~~~~~~~~ For any newly added plugins, it is important to add some new simple elements in ``tests/cachekey/project/elements`` or ``tests/cachekey/project/sources``, and ensure that the newly added elements are depended on by ``tests/cachekey/project/target.bst``. One new element should be added to the cache key test for every configuration value which your plugin understands which can possibly affect the result of your plugin's ``Plugin.get_unique_key()`` implementation. This test ensures that cache keys do not unexpectedly change or become incompatible due to code changes. As such, the cache key test should have full coverage of every YAML configuration which can possibly affect cache key outcome at all times. See the ``src/buildstream/testing/_update_cachekeys.py`` file for instructions on running the updater, you need to run the updater to generate the ``.expected`` files and add the new ``.expected`` files in the same commit which extends the cache key test. apache-buildstream-27ae392/doc/source/handling-files/000077500000000000000000000000001514607367700225535ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/handling-files/composition.rst000066400000000000000000000155511514607367700256570ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _handling_files_composition: Composition =========== In this chapter we will explore how to create *compositions* of multiple input filesystem trees, using the :mod:`compose ` element. .. note:: This example is distributed with BuildStream in the `doc/examples/composition `_ subdirectory. Overview -------- Composing a directory tree based on a set of build dependencies is often one of the important steps you might perform in order to create a single artifact which can be checked out and deployed. In order to use the :mod:`compose ` element, it is important to first understand the concept of :ref:`split rules `, which we will cover in this chapter. Introducing split rules ~~~~~~~~~~~~~~~~~~~~~~~ The :ref:`split rules ` of an element declaration denote which sets of files in the given element's resulting artifact belong to which *domain name*. The *domains* can then be used in various ways, using plugins which understand *split rule domains*. BuildStream's :ref:`default project configuration ` contains a sensible set of default *split rule domains* for the purpose of artifact splitting, they can be overridden in :ref:`your project.conf `, and finally on a per element basis in the :ref:`public data ` of your element declarations. .. note:: Projects are free to add additional *split rule domains* on top of the default domains provided by the default project configuration. There is nothing wrong with defining split rule domains which *overlap*, possibly capturing some of the same files also captured by another *domain*, however you should be aware of this when later using your split rules with a plugin which processes them, like the :mod:`compose ` element described in this chapter. Example of split rule declaration ''''''''''''''''''''''''''''''''' In an element, you might need to define or extend the ``split-rules`` in order to capture files in custom locations in a logical *domain*. Here is an example of how you might use the :ref:`list append directive ` to append an additional rule to your ``split-rules`` list in order to capture additional data files which your application or library might want to include in the *runtime domain*: .. code:: yaml # Add our .dat files to the runtime domain public: bst: split-rules: runtime: (>): - | %{datadir}/foo/*.dat Split rules are absolute paths which denote files within an artifact's root directory. The globbing patterns supported in split rules are defined in the :func:`reference documentation here `. .. important:: Note that because of variable expansion, split rules can often be *resolved differently* for elements which have overridden path related variables, like ``%{prefix}``. This usually means that you do not need to explicitly extend or override split rules on a specific element unless your element installs files to special case locations. Project structure ----------------- In this example we expand on the chapter about :ref:`integration commands `, so we will only discuss the files which are added or changed from that example. ``elements/base/alpine.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/composition/elements/base/alpine.bst :language: yaml Here we have modified the base runtime, so as to specify that for this element, we want to also include the runtime linker into the *runtime domain*. ``elements/runtime-only.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/composition/elements/runtime-only.bst :language: yaml As we can see, this :mod:`compose ` element has been configured to only include files from the *runtime domain*. Using the project ----------------- Now that we've presented how :ref:`split rules ` work and shown how to use them in the context of this example, lets use the :mod:`compose ` element we've created and observe the results. Building the project ~~~~~~~~~~~~~~~~~~~~ .. raw:: html :file: ../sessions/composition-build.html As you can see in the output, this composition has only a few hundred files, but the complete ``alpine.bst`` runtime has several thousand files. List the content ~~~~~~~~~~~~~~~~ At the risk of this being a long list, let's :ref:`list the contents of this artifact ` .. raw:: html :file: ../sessions/composition-list-contents.html Some things to observe here: * The list does include the ``/usr/bin/hello`` program and also the ``/usr/lib/libhello.so`` shared library. These paths are both captured by the default split rules for the *runtime domain*. * The list does not include the ``/usr/include/libhello.h`` header file which was used to compile ``/usr/bin/hello``. The header file is not captured by the *runtime domain* by default. It is however captured by the *devel domain*. * The runtime linker ``/lib/ld-musl-x86_64.so.1``, as this was explicitly added to the *runtime domain* for the ``base/alpine.bst`` element which provides this file. .. tip:: The reader at this time might want to list the content of other elements built from this project, such as the ``hello.bst`` element by itself, or the ``base/alpine.bst`` element. Run the program ~~~~~~~~~~~~~~~ Finally, lets just run the program we built. .. raw:: html :file: ../sessions/composition-shell.html Here we can see that we at least have the required files to run our hello world program, however we would not have if we were missing the runtime linker which we added in ``base/alpine.bst``. Summary ------- In this chapter we've gotten familiar with :ref:`split rules ` annotations, and we've learned enough about the :mod:`compose ` element such that we can start creating our own compositions using *split domains*. We've also used the :ref:`list append directive ` and we are now observing the contents of artifacts using :ref:`bst artifact list-contents `. apache-buildstream-27ae392/doc/source/handling-files/filtering.rst000066400000000000000000000156231514607367700252770ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _handling_files_filtering: Filtering ========= In this chapter we will explore how to *filter* the files in an artifact using the :mod:`filter ` element, such that an element might depend on a subset of the files provided by a filtered element. .. note:: This example is distributed with BuildStream in the `doc/examples/filtering `_ subdirectory. Overview -------- In some cases, it can be useful to depend on a *subset* of the files of an element, without depending on the entire element. One scenario where filtering can be useful, is when you have an element which will build differently depending on what is present in the system where it is building. In an edge case where a module fails to offer configure time options to disable an unwanted feature or behavior in the build, you might use :mod:`filter ` elements to ensure that special header files or pkg-config files are *filtered out* from the system at build time, such that the unwanted behavior cannot be built. In many ways, a :mod:`filter ` element is like a :mod:`compose ` element, except that it operates on a single :ref:`build dependency `, without compositing the filtered element with its :ref:`runtime dependencies `. .. tip:: The :mod:`filter ` element is special in the sense that it acts as a *window* into it's primary :ref:`build dependency `. As such, :ref:`opening a workspace ` on a :mod:`filter ` element will result in opening a workspace on the element which it filters. Any other workspace commands will also be forwarded directly to the filtered element. Project structure ----------------- This example again expands on the example presenting in the chapter about :ref:`integration commands `. In this case we will modify ``libhello.bst`` such that it produces a new file which, if present, will affect the behavior of it's reverse dependency ``hello.bst``. Let's first take a look at how the sources have changed. ``files/hello/Makefile`` ~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/filtering/files/hello/Makefile :language: Makefile Now we have our Makefile discovering the system defined default person to say hello to. ``files/hello/hello.c`` ~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/filtering/files/hello/hello.c :language: c If this program has been given a ``DEFAULT_PERSON``, then it will say hello to that person in the absence of any argument. ``project.conf`` ~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/filtering/project.conf :language: yaml Here, we've added a :ref:`project option ` to decide whether to use the :mod:`filter ` element or not. This is merely for brevity, so that we can demonstrate the behavior of depending on the filtered element without defining two separate versions of the ``hello.bst`` element. ``elements/libhello.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/filtering/elements/libhello.bst :language: yaml We've added some :ref:`split rules ` here to declare a new *split domain* named ``defaults``, and we've added the new ``default-person.txt`` file to this *domain*. ``elements/libhello-filtered.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/filtering/elements/libhello-filtered.bst :language: yaml And we've added a new :mod:`filter ` element to the project which uses the ``exclude`` option of the filter configuration. This is essentially a statement that any files mentioned in the the ``defaults`` *domain* of the ``libhello.bst`` element should be excluded from the resulting artifact. .. important:: Notice that you need to explicitly declare any :ref:`runtime dependencies ` which are required by the resulting artifact of a :mod:`filter ` element, as runtime dependencies of the build dependency are not transient. ``elements/hello.bst`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/filtering/elements/hello.bst :language: yaml Here we've merely added a :ref:`conditional statement ` which allows us to test the ``hello.bst`` element depending on the filtered version of the library, or the unfiltered version. Using the project ----------------- Let's just skip over building the ``hello.bst`` element with the ``use_filter`` option both ``True`` and ``False``, these elements are easily built with :ref:`bst build ` as such: .. code:: shell bst --option use_filter True build hello.bst bst --option use_filter False build hello.bst Observing the artifacts ~~~~~~~~~~~~~~~~~~~~~~~ Let's take a look at the built artifacts. ``libhello.bst`` '''''''''''''''' .. raw:: html :file: ../sessions/filtering-list-contents-libhello.html Here we can see the full content of the ``libhello.bst`` artifact. ``libhello-filtered.bst`` ''''''''''''''''''''''''' .. raw:: html :file: ../sessions/filtering-list-contents-libhello-filtered.html Here we can see that the ``default-person.txt`` file has been filtered out of the ``libhello.bst`` artifact when creating the ``libhello-filtered.bst`` artifact. Running hello.bst ~~~~~~~~~~~~~~~~~ Now if we run the program built by ``hello.bst`` in either build modes, we can observe the expected behavior. Run ``hello.bst`` built directly against ``libhello.bst`` ''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. raw:: html :file: ../sessions/filtering-shell-without-filter.html Here we can see that the hello world program is using the system configured default person to say hello to. Run ``hello.bst`` built against ``libhello-filtered.bst`` ''''''''''''''''''''''''''''''''''''''''''''''''''''''''' .. raw:: html :file: ../sessions/filtering-shell-with-filter.html And now we're reverting to the behavior we have when no system configured default person was installed at build time. Summary ------- In this chapter, we've introduced the :mod:`filter ` element which allows one to filter the output of an element and effectively create a dependency on a subset of an element's files. apache-buildstream-27ae392/doc/source/handling-files/overlaps.rst000066400000000000000000000114371514607367700251460ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _handling_files_overlaps: Overlapping files ================= In this chapter, we will discuss what happens when files from multiple element artifacts conflict with eachother, and what strategies we can use to resolve these situations. .. note:: This example is distributed with BuildStream in the `doc/examples/overlaps `_ subdirectory. Overview -------- This project builds on the previous chapter on :ref:`composition `, and as such we'll only go over what has changed from there, which is not much. Project structure ----------------- In this example we've just extended the ``libhello.bst`` and the ``hello.bst`` elements such that they both install an additional file: ``%{docdir}/hello.txt``. We've updated the following Makefiles: files/libhello/Makefile ~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/overlaps/files/libhello/Makefile :language: Makefile files/hello/Makefile ~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/overlaps/files/hello/Makefile :language: Makefile As you can see, this now presents a conflict of sorts, where multiple elements in the pipeline want to install the *same file*. Using the project ----------------- In this chapter, we're only going to present the warning and then discuss how to mitigate this situation. See what happens when we try to build the ``runtime-only.bst`` :mod:`compose ` element: .. raw:: html :file: ../sessions/overlaps-build.html Notice the warning message about the conflicting file, it is there to inform the user about which files are *overlapping*, and also which elements are being staged in which order. Note also that BuildStream does not discover the overlap until the moment that you build a reverse dependency which will require staging of both artifacts. .. tip:: The ``overlaps`` warning discussed here can be configured to be a :ref:`fatal warning `. This is useful in the case that you want to be strict about avoiding overlapping files in your project. Mitigating overlapping files ---------------------------- Since we recently discussed :ref:`filtering of artifacts `, we should note that it is of course possible to handle this case by having ``hello.bst`` depend on a *filtered* version of ``libhello.bst`` with the offending file excluded. However, working with :mod:`filter ` elements just for the sake of handling a conflicting artifact would be quite inconvenient, so we have other means. Whitelisting overlapping files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream allows explicitly ignoring such errors by adding the files to an :ref:`overlap whitelist `, you could achieve this in the given example by adding the following to the ``hello.bst`` element: .. code:: yaml public: bst: overlap-whitelist: - | %{docdir}/hello.txt .. note:: Note that :func:`glob patterns ` are also supported in the whitelist. Artifact munging ~~~~~~~~~~~~~~~~ Another way around this situation is the *munge* the artifacts at install time such that there is no conflict. This is the easiest approach in the case that you might want to keep the underlying ``%{docdir}/hello.txt`` from ``libhello.bst`` and discard the same file from ``hello.bst``. In this case, we might modify the ``hello.bst`` file so that it's install command contain an ``rm`` statement, as such: .. code:: yaml install-commands: - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install - | # Rid ourselves of the unwanted file at install time rm -f %{install-root}%{docdir}/hello.txt This would cause later builds of ``runtime-only.bst`` to no longer conflict on the given file. Summary ------- In this chapter we've presented a situation where an artifact can *conflict* with another artifact by way of providing the same files. We've presented the :ref:`overlap whitelist ` public data which is the typical solution for silencing the error when the outcome is desired, and also presented a strategy to deal with cases where you want to keep files from the *overlapped* artifact instead. apache-buildstream-27ae392/doc/source/image-sources/000077500000000000000000000000001514607367700224325ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/image-sources/arch-datamodel-context.odg000066400000000000000000000422671514607367700274670ustar00rootroot00000000000000PKótNMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPKótNMs§"„ÇÇThumbnails/thumbnail.png‰PNG  IHDRÕ(Þ|PLTE'$&-"9*"*%**8(5(4''33E==Rv5G5=R=#{#F4"E33Q<(Q<<[D-eK2lP5yZ>²))Ó::×II±QQ­UU»Cq«Ju­Mx¯U}²ee‡nn“rr˜eeº||¥qq¾GGÙSSÜiiÂbbÞvvÅllàuuâ.‚.4‡4(©(6®6=±=DŽDI–IVžVGµG\£\R¹R[½[fˆfm“mr˜ri¥i`¾`{¥{{¸{cÀckÃkxÈxW€³X€´a‡·bˆ¸i»p’¿u–ÁzšÃŠ+Š-Ž3“:‘9%£¦%%­66œL9¢S/¨]6«c;ˆfC’mHšsL†dd’mmšss®CC¡VC±II¶YY¯j@¬jZ¦|R¸wK¥qe¥{{ºee´yjÿ{$ÀbbÄppÿLLÿWWÿggÿww­‚V´‡Y¿Œÿ.ÿ†7Ä‹[˘eÿŒAÿ“LÿšWÿŸ`ÿ£gÿ¬uÿ°|‰‰‰———‚‚®ˆˆµ¬ˆ·ˆ”¿”­¶‰‰§§§···„„ɘ˜ËŸŸØˆˆæ––郡ƅ¢ÈŠ¦Ê¨Ë’¬Íš²Ñ¥¥Ú ·Ô£ºÕ¨¼×©½Ø¸¸Ë¦¦ì²²ï½½ñ…Ç…ŒÐŒ˜Ë˜˜Õ˜£È££Ú£¬Ý¬²Ý²´á´¾ä¾¬ÀÙ°ÃÛ¼ÌáÉ„„ӈ˘˜ÑŽŽÄ®«ß¯¯Ø»³ÿ‰‰ÿ––ÿ·‡ÿ½‘㺺ÿ¦¦ÿ¶¶ÜÁºÿÛÿʨÿÏ°ÿÔ¸ÈÈÈÂÚÂ×××ÏÏêÆÆóÂÐãÉÖæËØçÎÙéÓÓìÓÝêÚÚï××õÂæÂÆèÆËéËÑãÑÓíÓÙíÙÜðÜ×àíÚâíæÈÇÿÊÊÿÛÄþÖÖÿáÎÿæ×çççàçðääøäêòèîôëëúãóãêöêîøîîò÷ýèæþóíóóóòóúôùô÷ùûþôòÿøôþþþnòÅ‚IDATxÚí] |UÅ•§*ÀPÂ÷g…$«îŠ[]ѵq[R·Øbøè1µíjmµv·`·Tººµm­ p¯–•´Z]ý-Ÿ1ÔªÝhÒÄL§Ùíª@Ö«!2‹?@àdöüÏ}÷Â{„;÷æ~Ì̽yó¿3çÌ™9çLw‡>!!!!!!!!!!!!!!!!!!!!§‡÷_~ñ©³ϽüNà?ïºöÎ_¼øÜ‹ÏŸÕßóOßùW_{¾ðñÓw¼™¨ôw}½ÓZptûÛí%íãýXsúG;ÚÉùQâxóºŸ'ë¯y©ã rêæ8ºýèºýu>Ø¿óÀ~W·Óí¯«suû§9G{]ÙÛuûíΕþíÑ£uö×í?zàè}ûŽîpÇv;P·óƒ¥ôŠ„pâÚ—[Ûþ÷š—;L/-sîƒéuÓ·æN+›^—3cë´£¹9uÓw|á@Î*ýtçvnÉÝšó«œÛéë甕þjîÎœ²œí[·—~¡lZnٌҜ²Ûoßž³sZÙÖ„p×Ó‰noÿuí‡%ïØêÔÍu9¹;¦4½îö­;¦¿=}úNºÞ—ûQPvææîÏÝ’³u¿;ú…Üœ[wl9J”•N£Sé´ÒÒ2ÚæØ>wn"xùë'œâÈ¡#³¥¬tëþ­;gÌ}Û••¹e¥Ç(ò”æ›»½´ôö-Û˶—mÍÝ’››îx? 4÷ë½ôíÑ°¯ÕùŒ›}ûvbÛ9cÆÜhľh.\|töüé«É`:Ï=•Ø÷}p ilð©k‡T¿ðåBñ m?ýε=¦ð‹¶YÀ>ó—n½õKm·|òÓ&g¹¦Çð•6»-æÒ/o‹%”n»åæ¶Úá‡=€;ÚàÖ/n‹5Ü\ܸù¶˜øbçHÑÎ?WmÄi%T$¿ß"«ßi!­H——&+|‡RùZßùBúF *§¯2µ/ Ò­/}œ¤RR¬ zŒ°SÚ÷Ó™Ú£KAEÔ‚’i—¾Pô)èÎy8‰°e+.æfÞòomñ6BgVpBxNh©Œ6E¥Ï¤Ÿ{ëI"8tðAà+ßYÀ$|áI*.– QRÓ6˜øÖ|*ò&œ„‡÷ #Ukmæ˜ÁųgÑ>»xÖ¬ÙsŠg®=sݬÁ³¶¥lÛ6föìµÅ3וÆ~œ¥ï«<*)Á 5ý^IuÁÓôM=ëè`EÉ”5ò&:i~/=`^椤Îœ:{LÊe³¨ôÃÄ”ÔYÛúLœ“’².eÌàÙ±Á€ŠÉ¾ÖÝŒ œ¢ë¨¯ÛR¼n]p\Gwˆ¡$:ÄHmsgéŒot*ÂØèe«®•µç €Ò(é{a[”æ•žqÑ9Ô¦Ä7– h¢¨[ ¤(_)‹DKÑ!ÑÊë™H~b#‰ü< Š£\n]Ê,þà‘¿m|ž¸míºØÙ Èñ<4*S±#‚M‹H›É4™ ’2}ô”§¥géΓDéˆ]ÄÁô¬#jçK¦w”è(AáYO'¯ ÌþTjêec>Õ/eΘ‹RǤŽI™yј~©S&–ÆØضaŽ@ü0P|+ž'P5ˆ¢;X&q?Ææ3ƒwDê­ $â„ø¦õ}úìT_/âüÆwÉ­ƒÇôëWx}Jú¤ôQ©4T 諃µ–Ld¨þXîyšº ÎS`­ÄýžDuHn ضv溙ŸJ™3{í¬µÅ³h›]<‡bŠãè Zn²Løm¾Lƒ |<Ž´¨ !pÍÏŸÜFž´±’È®Á"2_qsy£Rà©‹/&Oj³c§Í9áű /l‹#zŒ4øùÛZ}êÒ6¾~pÙ9èオSOX £qéHæ!ÉÇJÑM¸õÖ˜Åá[në” @^Ñ~“It›È„Oh­<_wOþçÒÛZ|êö×?i; ‘{êÒøš#’åH8¢~$c»%nÝ¥_º5Ññ¿–'¾¤›[.ýƒëT”š„I’µ'Áý!Az%™WwOÜÎèÀïœÈ…Y§ßÏ nÚ™?­zàX›Á¨˜ç!zü˜`Û2«Ò4Zɺ€°T”².¨1ÓìÞ€±F3öã —P#ˆ9k@ŒÔîuO»Ý[rêîDImm‹¬‡Olê…”lrYG¨Ô›KÜÆîDÖá’ZÜn,Ù›=yãÆZ\º›7¾UR»iòælŠ?A±½ € ›Üä’©%µYÙ%Y{]m6E}¶¶¤äª½Y›7lÚ0ùðUG²NdeoÚT2ùpÖ‘›6lÜ”=yïä^U6ÒWßU’µ±6ë„;|“Û»7ëÄÆ’É.kÓ†MÙYnò‰¬#YÙµ6\uä¦#7e×îýìd׫8’=µvïÔ©G²³÷N¥ÛÙSÔfg™ê¦Ö–ÔnÜ}•›Z’]²wSvvíg7gælS{ÜÝ‚P»9zYKç쬛NEGt8²Á€€€€€ô¥îtø[ª–YŒ‘̓P°ik`ÍHç”NÏþÅŸµ—ÿåQW\ýV 5Às<.¨ Ò'„Q¸5N i¤°B “IÉ‚ÒœÀŒ)Ý …uÂÓJ°‚”….”“¸JJEñ½ÄÒ ào.ÿÌ1„Ç?}ù“`}è8IOx‘I`Ì@ß ³Ã£¾¡äL\(ÉÃL±RJB¿ˆ ,•/|CË êAÒG °ûΣí©×€g¯xü‰ØÂç®øcgà›)*FS<|{OóüfùYÊ·TjBˆµÆè{šÕ¾  åA]Bó( mAéfÊKè ^™àðé¿~"Öpõã}(Ú0¢¨5nhLņ̃–hêVM»¡²#]að‰¢µEFäS˜FÀÃ2ÐÉiT#,½Aê×€SàÉÓþº @¬4ͽ»p˜[@W`m¶6§ÄÚ›'KÀüYœ\ùø-?ûÚN…Õ¢s¨ú[ªá–õý¨æzDו aÐp‰2j¬ E-Â`𭆛e—Ôž4­¶ç© \Óÿ±Aã4eà”¸k4Y„OŘ' b´¯ˆÞyÐ’‘ ~P“ÊÙ ’’SŒ/(Hö E ¡Îk ?pt¿¸›ëóP±|4PSY#Èy>  ¯+‰ÿI"ëÄ TL'Ži¡+ù–Õgéd•§Ï7ŒžðÄèÑž2:n îeaüv¨4ûЛҬ)'¡IíS7‘.SCJ8Ÿ+ ”¢0‡í*ª ЮbÕ1-Î ˆR€gCÛùå^ÌE 6pî¹À k"ç)ן^è ] J»Ê“5îœ0ðúqkÆO™2áúOŒ›pãø Ö\?~Êø ã¨-Œ¿qÊ„¸v$¬Э¤Á)ô±§ôïc¿7Œxc¿þ7ŒþÄšA}Æ_ß¿ÿcØÛêVDhÀ„Aƒn8ˆ80~ô'Ž×MÿþãÆõŸ0h|ÿ'z3mô§Œ—Ü®pw é²@àÉ €«?Óv©Ÿ<.püxS䪩9²éøñNÊÉ|ò\ð¹?áÛáÊgãàg{V¼w²Éw‡öÐÁ555­nr»‹ö,w‹ÚO§’r&Ú›š¨ÔÇ‹p«O®"¬ðŒk:™<N\yu'M?* ^ùéøG…­^Q]X³rwÑÁUU«V¾·reõºÝ««—,ZѸ|U Aqüg+W6®^¾òõÝ«‹jV¬¨Y½úÑê¢åTSظ‚¢vZ¾ª:y£Âo]}…+¯è,\þ9Û5¾Wt¨È¡À«Vî®nr…Îí®*ªÚ½²¨jUSS¡[u°°f5š–¿¾»puQõžê=…®èõ•*t…Õ{š ‹­X]U³gyM]Ò5ítÁ—”ZU}°Š®kªV½NG\Ò ;…ªƒ¯W!k ]S¬Ò±ª ÆÊkuW6ØvP1ëÊöJ=AMQsá 5EuÓ @8;|(k`'…vÔÚê­y (’ÐÞ¸W”C|×úUªÛ ŒÉOÈÀRÈÒï—¦dÎò€¯cË`ö¡…2•’F3bRÅ1Âr´ÆHªÂü Ò™ˆw–Ð(è¾H%|›)ù_9¡8xfÐÀx±éÐí<ài/°“Æì·Âç†C ž.¦MD¦ `!,`4®PèÖ²j&Âatér 3jÔS)0&v{ÐÀ¬‚í¶hªÁìê‚ê3W[êTr5LJg[2X‰ó=šÕe)-¬¨1%b%Ô èQÉÑng 1ÀîE$&—UÇ£æÝ™ è6g…;ÛSÆñ¾éÉ\ ìtºb?kU@a’>¤$(6È<ÌúBé‰éºö}-92â4 „Ó:\-ôÙ#üPanÓ†1)Vt à ì>4˜–c^˜n”1§†‰±ô§º1Жb_SŠ¯¡OF1´´™AACª»ûqÌö+ ßOÄÄÀÃÀáŒ/m0g =2©àK…yª+ÃyÚð<ôè5.p¥âõŒfRPƒ²hùð…jÐþ¦X™*’ƒFiâšè;ã@¬ÎWॆµªØ%r OÏ$‚qOv›³14ï®\€>&<†E{1ÿ*ñ( Å ÀoøæÝ­Â=ÿÔ:æîï<`ã€õ©ùjTsªÑÔ”oƒjL]}Ô|ÅIȨ/cð±F­ò³P©“Ü‹4Éà_îþÁÒÖaÙ1Køßx%N4<AíI¶C# ¤ÎY°6hR)R Niv¥H‘(!è"óEêûBª$ðÛ»—.‹-,ýÆoã Њ"Ù*O*P¾T«zÂC I€”D=BÉC!)C‹”Ø ‰Yò p¤…Lü`Y¬á‡ÿp%ù“8Ââî\lB‡ £uŠýiÒ•’ûCŠw<ÈýɺæˆL÷,M¾m¬u'Yà›ÉàyI,‹–ðé‘%  ™Øfæ×ìMŸÙ?°²9cGªbÆ$ €QÇ̟Ÿ7¶ùyËæåõ]/‘>ÉC;)´qŒôa´ ç(‰©o @$’a*bÀ,@&»Æ ŠÅntñgxÐ× –rÀžÎjM·~ÄÂÆïáìd—þ@:ƒüle‚ëd°Á!‹†.ºxHß¼E—Ìï;¼oÞð”¼á}Gö~IÞˆKæÇìÁ,OŠ ûŽA!<(ÀëÀá,E(XHpµNò€À/ ÖHLC€ø¡kL,10)µ ¤(p„Ä0oéüGFzdÄ’¼‘‹æzdÞ¨%´ÌõÈüy£æÇ €ñÁÖð[yÐ>u1 sB°5ö ,E¦%’†q.ŸM$<‚ žùBóô>ðÏËÖx‘ } ´vÛ®°fãq<”5@ƒ†‘hƒqR ©È±¬ 1—»àq$¦;#¶ìIžê|4“$À`‡Cå¦:¢à…ÿü´g ë›ÓSl‹\öŒlÇÒDâ‡Å¿óä C=cfèŸÿ>f¾ûPBÐÝ€“÷|7Æòÿàn›H¨×+A ÙŒž=ïžÜ»ßù‡{¢ûi·§m÷|ëÛ”HØk:{s„™üyÀ™ßsx;6œ½n>üþ,‡ÅϨ¼~1?ê"€²'ÌíîÙÌ 4E4õ1ž\ó±É5ý¼À¸ËÀ^*Ÿ .Nq.­šbÌÛkª†ª²Ç.XÐÁÜB¡€€€„P^^Scyä¾Ò5Ò±¢¢!š¡¾²¢ÅWžìM¸òʆòÆ‚Å å õõõ?­wåõH}y¾ñ'å'+*ë)©¡²²žr–W46”ÓMÅÉ^@î‚ÊÅõ••T®ò‚“IekXì Ê+w5PDEeEÅ®†]õ'ËËqAq‹+Ê+*{ •?­¯,¯_\YP¿‹ ·«ÁQµø¸€¡R/.¯‹ëŸ,`ê»Å­šFO`qy¹£âW6T–—7(w»ê‰"”WºŠòJŠ%(~WEîòJ« L*+Ë+ê)ôêÛ­"+ *OK‰¦6„l0    ©4þÇ[‡_ÿ挨×~#~d€›WˆTX+³M½8¬Þ†ž›µÔ 6»„i€ª‘H0ïÞ÷í…1„ûîý~c,°çT¸Ô…Û@áFŠÿÁ2äì(MaqLúÐûQAëÄÂ*Ngª@%J°Ñf}©`Géñ*v‰ÀÞ»ðÕØÂ÷î‹xÂuÁ áð„ky.ý`ùô`¥U.Ÿb…)vLõ„§µ¼ÒòŠ}¬EE šWµÇ¢ë¾/D§z"qðð÷^5Ü÷Z HÅ MBKvPþK®jv&ì¬Â,1æûoÐ ¦1¥Î‹+ã« ŠÈzäÒÇBô,>'÷ý8f¾ÿëXj€T&Xy\±zYÖë®[ºU«:ëÓ«ÍkèÉ;¶#QÁBå‘5Ë5NjèUÁŠÄ(ú*‘ÜûJbˆ+H{þ¹À½¿N<†m„š+…³¥ž=%v†N>÷Í ã‚t¾y0ãÁ. 3¡ »ÃÚ¢˜hQP‡ ì`.¥á6Rìr×· †­¬‚ÔH^è`ÂM·í|í¬ÈÈÏOŸ4,mlþ°ôŒŒŒ´>ë»ô£$ 'š&}6 „Á¬'}Žg»9À’ÉÖ” ò´“×*…uœ×hÒUOòÒìžòé ‰à•–,H¿8-mAzÚ€aécÇèR ÀbuTPúÑÚãy_v­ ûÚ$W·p•O Àç¬ÇÞX5ZZ^¼Üúµl43(TÁv¼Õ“*©5`ÁØôõù“òÌÏH›žŸ?–jÀkq#?0ˆ…©?ßÀBœê0þú=lK,NôáK=#bP©솞ž±:ÀçEë})t ‚=Jˆ6üÔŒüÓÊüJzï ¨i÷§?˜~FFþÅéùéùhéc'}pÅ.H ª•–úI§'žºCWÊEî Li’ À€W_öà€´õÒŒ@ûýc/^?¬Ï¤¬Ç?¾3$5H8{@5–x$Èwèà LiÖ‰‡uX&Œ#ØÁ†ƒ.%ec? UZÅÖoêZÈÖ7l}Ú°û' 66mlzÚú´´ôIiÃ2†½š?lä¡ ò aÆÇb+°‰0(Ã#Qˆ»úìŽþA°Rµ`I(øèŠÕƒÙãŽSšsÒL¶àì‰ ¸zd‰ÊÀ’†Ýç õWN¨·€µEØ¡:ä}v-IÈ‹´@xTãÀ)MÏè sk–l #'܉ÀÒÃÁþIÔ’ITlJÃIÈzºÆR4¼.I¶œRpIceR &¾}Ø`§f Æœ;.°pá9€í(Z²€ÈÙ¶‡ls¤‹×3NÞ¹ üøÛ]ÀÀ,„Íaè -ã\n‹ñ/Èâb$!@„š?ŒÉ4²Òó^\ËmÄ;&øÚ½ z˜7>Ð͇x{˜·‡ÂCß¿÷w]Æp[ ÀgŠæõGüÀZıI!ËHXn NR$û ƒò¨ÇäQ‹€[/!|õͶã)âBÞQÖh™O]qJcW†Å©`û$Ãx\¨·ï³§1ÇÒ‰ €èaɘÀ†„øÒ#^W¼„4§žï2]ù¸«ó&0ü0XAÃ~(¬<xMP¨$<È%/Dæf'°Ò¾ŒX’Ä¥HÝ.¿üe2zÞÿý•Ž§X,»~àöÌeycê½hžR¿ íG÷ø¨`»¾#<óLb²Î0ä¾ö§Äÿ·wœèÚƒV´ÒEÕN&€w®{?áÜùL\ÙÑÿã).âØ‹&–” ˆ+{+ErpÏý]¢Ëÿ\|Í n£óF<Âƅ°çPLÁÃd¾JœÐò¿tÍ;ñU¸Ð„<ÜË q™#.E‚ò¬MÞ•ÈVpâ®ëÞŒ³áù„²ˆXÒñáàxÖ`Ô#˜û9ë‘>¦¾|Ýso&¦øï¿tÝ/Ï®>i!\BǸ_¼îš„„kŸ~ßuËЧÓ&$¸îú¸ <„„„„„„„„„„„„„„„„„„„\XáÿMò CçFIEND®B`‚PKótNMmeta.xml“AÛ …ïý–»W0Î6FŽWê¡êa«Vj*õ90›ÒÚ`Þ¤ÿ¾Ç©½­r¾7ï1 åém¢g0Vjµ‰IŠã×BªÃ&þ±ý”¬ã‡ê]©Ÿž$&4ï[P.iÁÕ‘—*ËÆ£MÜÅtm¥eªnÁ2Ç™î@M6§Y0wNT6ñ/ç:†ÐñxL4Õæ€HQ(œN¨à®ëM(Á408XDR‚&vHxk¨Gê XZ»Ð”ÛjÌ5óZZëKèA66 DÏ0ÎѸžhÛÊæVÇM¸n;ï¹o=­•lo-3°¯RŒ͵Gñ™)ò]¯]Ykàq’³ÉÐ#ïž«JTÃs_ÒBHÕ’NOº°KÔ¦ŒµÊøxÊ@jú’I2È âÌ|H¼Þ£›ïÜÞipØ®vA§ÖǦh&;÷œ>ò´á ®ÑrÕe©¨òÄØÁHï *’×Ó½lkqñ„%ëóÏZ¡w(0Ê›LѪ4¸Œ^Vz,‡ÅšÝËZe9SCæûågižˆóë}¬‹@äÞ4§~*àÄOIL½„Ƽ¼ùh²u;ì˜Ï(äµ{+r‹Ðu •6,0Û†â:½ÙHòV4Å ¶3‘P™÷X ¦bH)»§‰ëÞûGÇŽÞùLò2âϤå_vÍà%»#’iÏ|D¸Vý“â2u÷OˆŒ9™h°d:I6d|má!¿~*p冕åsdûýB~«ÛÓâ™"Ò¶T4{–LR°Doç|"œ/Iüu\¶}ÞW‘ñ“¤t4‹ô\ ü±ìY›êæ#žRpÇ“_©tôßFÈ¿J)6FB<^ÿ&î¯ÝÀ œ(pf‡sâ6„1î…8¸Ž‚ÜxdÁÏÛ îé ÀïQð{½º&'¬,8Ùz;lΘa8 « ê„8¿˜£ø Æ×ÈðÐ!¡)©x]36 Õx¯$)Ö,vÞú³WÀ™F¥bPcâ&¥’â+…KPV|7›_œ“y-~Ê8o)¢«4† ’ŠÅ–òDa ˆ\xø¹ª ’èÍz[i‰©”( ‚e-K¨0¬„kÒؽ¨òXU:âõÚà,+ð†bèPxQo))â¬D@UCÁ[-Ô‚fõk—KO-{ÎÄò„â m½ ¢ ò”ð’¶`Á…@E‰Þ6®VËŽzíi[•ÔƒÊ4Oo^#©dE{Båj@ZÓKöôh^(=ÆI¾ªÈ †h®b¸) &ÿõóÞºÜ_I¾›Ð:\»á1;Ô×›4´‡uC©wkÿiO¼ÁszÿÈ®-×à¾-uÍvwnIÿüÉíìÓ‹…Æ0v`˜KEà)÷øpÑ1aâÖœ%®B–`5Ä™ûhd™²B1ÛN<‹pê>=¦xm\´Ç®|`:R‡&c\·ŒGº¦ k5o-${xA†`e+0ý—ªT,Ýj,H‚-bE £sÆ",…RX7Ñ8M•V`— ÙjmQŒÖ$ѵ-K kЃü_RåÝ÷Ñè·ƒÄF÷Ëÿƒè~ge?¹îøÍÆ¡ìå†Ù[VœSå"ŽÃµß5 ÉÃ6Ƶû¿ÿþ§u8kËçôœŒå'K ´¶¾ü±ôÓ$ŽÆ·?ƒáðY îNIMý¼3?ÿ^GÙ!MÀ!zúÔð–4’6 ¼?„f'C(4~þ¡Ëw‰ÐùÉŠÎæï¡‹“!4{—ø|8>ï4O_ž ¡ù;ÍÓW'Dè}æé08Dç¬Dm‘ëŠÍ+¼6`o¨vV¨°©"ê¢l*ˆþ¨©RúcmRëöÀššùa|¯G5rL7 룘êÆrçBf„w«( e±y¢˜‹œNBÚÚJT El+;‹¤EWk)ªÕÚ ÛßbŸIm‹ž‘΄®äwÝååúÛ;®^Ö=3ÖhÖǵ)÷4®ÀlÒt}lLšå6µ­›õêø£ êgÊé\÷‰»7m—e°E»@ÍýôVm³ÂX»´¡w8Ž´|  6QiMí#at€¡ß:ÔôÄÔ(Ü2ªÜrŒ+ݲŒ¨ÝÒÇaÑþE³bMŒ£îE’¤œÑtÀ)ÅÝO1Ô‰ K Vó둦—X~¡±Ú0Ç‹é<ö¾šö+‘í‹HÞ^Çã¹í±³ð|¤Cè°¶e wöápïÌ„íhëlf2ù±hšTó’pֹ쎕L»ô‹t»ž 9œ>Ø3|Õó Ó¿Ô‘2YôiÑñ Š¿s?q-ü Ruƒoýg8f×práàl¦âßö¤Œ,“ žŒæ”?†”K‘lßRÒÃ׺úBrÉÑZaãZ ú£« )•öÊm÷v'(…CøD¡ûl¿›i´n_ÞÜ£wòNá1 rœ57•‚à¨ÓæVÍåïêa@©•EŒwÞLÖ¯¹ué¢sÑΛtÐåm\ëÕÙ,ýáo!ÞüPKð-=- Å8PKótNM content.xmlí]Ýr㶾ïSp˜¶“¤á?E‘îÚž$\Ùì¦Ýô&“ „,I0 )Yy‰^öýú$=DŠ¢DÙTâu¥YÛppð_P ÷ÍícI+Ì2B“kÙPuY‰O’,®åÞ§¸òíÍÞÐ0$>¾ ¨_Ä8ÉŸ&9ü•€;É®ÊÞk¹`ÉEÉ®ãì*÷¯hŠ“šëªM}%æ*[²|fÄmî?æc™9m‡=ŒŸY·¹†Öc™9-€ÚféXæÇ,RB ¨Ç)ÊÉŽI>^ËËY¦ª¡Õ´1ÎÑXù8m[¤¤ˆ0 ÊQO«)ÃÀr¹aŽ¨ÍÓ±¯Õb´u­0ûKÄFÛ™ Œ7+hóÆ(_è×Õî¡Süº¿ÛÚ‹ÇÎÅi;PùŒ¤£—YR·ù)¥¨œ¡tv!®©ë¶V~nQ¯’¯É1k‘ûÉ}ù â4ÞÐP(xÅM¾q"D6À`jewCœƒC¸¿{ç/qŒ¶Ää8±B’,GÉ™,&Ñh-í€Ñ¢„Œ6NÛsÆañ™ÆpJYÞ((Ÿ`³Áh™ÇÑpã½5é‚Á^RÇÒ œA0QV¯?“;Ùé°az;†)Bý1AÔÎ ]ã4M8SÝ&.¶hrkH‹$(õPˆSÌïB‘`»êŒÐÖVDŸ0d•[#t’ÁQ¹š%í†R%ÎÀÁaizÕâîæ<?ŽŽ;) ÂÝw–ŸeV¾ÏÞ¯ñ>…§|HjÕL­RÇ”o꺦Œc™Ö4„Pß(!ò±`?ÊnÞ”ù©i–ÊÏ\îkùž&ȧ†,Aò¨IÀ%7u,u¸y—²À ¬¢YLÌ’IJr~Hq k‡ç¾#h…j¥w(ÉÌ=Bü¥4ûëaÙøDÉVˆaûG„k–?).cgßYñ„È0£ÉDƒ!ÃQ²qÂßZ8ÀcŸ]?¸lM²ì9²ý ÿŒþQÖg‹fŒH›,Çñ³db”b:é;EÈÿ8,[Ÿö7‘ñ;†ñh­®çJ  EϪ9å%†¯ˆqš°*~wä R£™¬’³Ú@))Z´–Ûg]°ë‚¡tIüº9EŒïaÅ¥dâÅW€X ×ãV,J ™³œàL —åŒ~ÄÊš¼$×U}fø — Êtö3Jÿ-…5›íR@1´¿?$/ä" [§ÏBñªzhŠ|’ÃRô?U-<­Á:²¤ŒüJyÎTPDÚ~.²œ„›]Â_‡¿%‹ TR¸¢âªQŒ®•%&‹%dÔEô†ô*&IÓꩶår«öj¶jºvÕœ¢€_>Pršò³öšç¼6ßÓá0çæ|·‡•Ó×]ÜØZpÈÌóXƒ@­4‡k9¡ –ÛR+Q¯–zkÚ#úT€ºG'žmœŠõ›»Èp<¯ÎÄÞãû°sð~__ÑU×vú¾b¨žùB}Å~U¾R+mG-°y0Oƒeö¼ÅÕ ïœÞâ8o9Í,œ—`Ügš@7±Yðêû³8Í,æ/Ã,ç\Ñ‚Å%·žjîéfA~Æ~¾&ù’9WÁtöá‹×ìãZ~·‰cœÃ¶ð'Sÿékº:`DölÞ1¢NN“¹_”ßö¶– 5$”ª)¯Õ?ã6wÍƶ¸=,cVÃäc~iôñ{{¡¶øgœ··ÝxÚ¼Â@O˜¶WΞ¦­£õç•Uòˆ¾.y=¼!^«-C—æ€KôgäWè7ô4¯ÙÄøE`‘—q¥©¾·sFe%]öm'ÿf)ÂU÷xô{Uó}¥ÚÊ×3zµÜ 7•–ÒÝgéÜ=¨r÷$÷*Ÿgh¶#çÕx/?QتH>¯°ÞdŠÒí¼Âú$ÎtV!ß÷29MœùvKןœ›¶Ú7YŒ¢Ãþ›Ñˆ$U¡ÅËæŠJn1©x¿•a]Ó4 NH ¢B­z»ò·F­ öÝÍ)ýÑ›þáñ1ÃxÅ÷j ³+Þù$õîœSíû?£Ö{èX­;â5•»Øo·|‡Òõ„>Útzÿ©Úêîã}T×}ÿœ?ÕàÜÛ3œ]ñ—àüûçÞæk¬Öð¼Êyq÷OÓÝ{ƱŠç•¿~vq÷OÐÝ{;ïñînÁë,×Å^˜_?ÏoŸ©ŸÞņ³×Êû¼òL¥òAWzn¥|Ä•¦/”û—ZŽéêô ‡'Á»{9ñ$ø˜€#îéŠHVÁÝèÎÛ^á¨ÒÊCE8—ÊNÞŽ$—Ë.…ßÄs-ÿ÷?ÿnðl ÒBUðð/!#ô=Íw_NùÕÒaUÔz{—#önÃkŠŒ‡ž$ keX@Á¬À»þcÏÄ7GÚ¡•ž€ƒù ²”Ÿû|À!e¸áõ!dM†¡š¯!{B„ÜW‰Ðl2„LÕ~•9“!d½J|æÓáóJã´;Bö+ÓÞ„½Î8Íkì‰ š}ZºÕ]ÕÚÚà{—/§é¶Ù.›86©gtvËCš#8ç{W@fs÷-ð(—7 æQF£g;%2mF°î ª‚Rù^IiFÊÇ™¡¸‘«Ý›­ù •8¨v»oaåQœAâ‚ŸvèGeéÉ<Âï?uG;€V‚‡ ¹/˜šƒÁÔ>LÕsì=ÑÔtf­h:WuÇé„SÓœ‹pºø>ÖѵʰmIÌ:ôBÞMúýï ùæ[Ç€štG,}Íâ Y•ë8[÷ÒgÌcn³ë¡a¬#Ã@#¡”¥ØçA5Àó%–üj%ü’´D+,ZC’ HÊÐR>“P²‘ .`Ò‰ !YÊpˆ»R'|zÇ”`Ë7?ð¥K“,ŠòôÄŠžM¦è÷ ;¡)¿-.¨¢…,ÏHU ×' …¢ÿ¡Îs(³²ªG¾yË(¿“Aús›ÅÕÃH¦vºù4ò,ÿ\rÔ×àPPªåTúˆq x—‹àO˜‘¸ß mm½”ÃuÁO+昿‡xŸñò|%‘DÊ ðJ>Ê€ŒäɸšW8!ü)jÜQK–3ÀŒY±Xà,—´BQCd…¿äƒ®—„¿¢/D@ "€mpæT8ŽÄ$€ºòMU·ƒ+vD<Âu>·õ&3•o ï F±”-IÊ]T¬áaøǯïïJæwñ%g<½Bå]øyÁ„äÞÿt‚­ˆêø3DQ‚ ØëP$ÐvÒ¨¶Mø#7TÖ` 0;Z˜êÌëï¨TsÞªLuîš­ ÀSg†uÙPu6TÖvE—MÔeuÙD}J›({0€:â‰fWd'Aí&ÝO犭“í®ôv¦?š½ŸŸŽfƒhÎ'ƒÌjAÛXþ¤£K:º¤£K:ú?JGf'™gŽ­S¦£Þ•‡s¥"gI·¤ÕAÒ:Kb·UËš_²Ô%K]²Ô%K½ú,5Qµ»T¡ŸvbD>¢IᎲ}€¹ƒ€5´ƒX+–ú[UO‹}4xbÖÝæ‰vc{ ³¤0KŠmþW5J<+Ñø#î„¢ÛVËæETpó„п5-þ0»†Ül5¶‰«ÿuŒöÞ% ¥+žçH†bY&ü6,Ó“÷ä8É°,½µ.¢¿Âfáz[ÞAØh#„°yÂÖ(„MÉáBÂŽ®Âú\áëó„u÷…!ܱáz»>Œ°¥†7„°uÂöH¶A¨•b˜†»TC‚ï ¨ZUT-Z§†y“8¬uN kÿ»ÜÍÿPKP™«6Ò žnPKótNM settings.xmlåZ]sâ6}ï¯Èx¶3»įM˜À$aÃ,Ih€m·o¾€YòHrÿ¾WRlâÈv§3ÍC2±¥sŽ®®t?àòËKÀžA**xéVàžð)Ÿ6œÑ°S9w¾4¹“ õ î / €ëŠ­qˆ:Àé\Õ—¯N$y]EU“T]{u_O«oŽ®ÇdË'/Œò§†3Ó:¬»î|>?œ 9uknüv=4” ŠèXp6ÂÍ9›´žà:ÍŠ²½9_ñ*ÚLX.,~T­ž¸Ëÿ×£U@YV.3¶â‰ DÍcö“­§AV3vgÕKUûµ_¼j_mÖ†‹9͵?¬Ý y¹2ÎòO…jŒ¬ )ëÏæ¯Þã$ÍûyÎwªŒZÈP„Îú¥^„ø’rí4+µÚñù¥» ô.ðLt"úIí(?úïÔ׳$ø£ãÓãÓÜð·@§³Dùµ³‹êç¬ø•€„Ê}x› æÉ»ÏA‡“‹,ŠaÞõ·d*-Ñœ¦qŒš%¾JêwÕÊ[ðc!î4µŒ |G ¾mâWð aʽ«œ„CaXŠ×¾Fï“)Ü9¥\•Gb~÷(‡TŠÌô0þ <Ý‘ø°t–¾À3TÎbú,R·„û T‹ÍÉâ-ïÍE[ëZ’éOKß{¸a4 œhè ¶ˆÍÖúe该ÓjÕv5»\elÑj?zdoŸ(8;i£¹pšnæŸó†–>ÞšüWÔô„÷þÛRÜ·¡îEKãõ?.cïÅ•`bWb~gŒÈ㹫¿aÞ”3!ñèXž›0¼ÑÀ7à…wUì wÂ/Á]u-"4ò£ÞÓ^ôOSƒpš˜àjFøŲP(žfÀ¨ªòQÌ“÷àÄòúE«$l@|ÕZnëÿ ï>+9ﶔoÁ+A¤‚Tõµ£Ï–Ûúøí¹Ð;˜îžy¯j{h“õÅ¢ÿ¸ÇòY-d‰×XmÇ)L K?þ…uü(g]§†æ†›¬'µ,Ê‘ô²Ó?…xBŒÍ¢9ááfuözU°'P ³÷f–"‡¿÷4iZaÈ#òšhR|ÜlS _=öRú>bm«?¹jF$¸!a(~=nø¡°O+bWi¬ºˆô•Û©Á,AL¥ä•LcѴņ(¥Ð¬*Î]ß\qÚÊ<;Òk¢f…[Ù·Ô1r:9ÿ°S7ÁüŽ(ªpLo›u„Sßþê3ù›h=ÌQ£„$~µAÀm.2ü)¨£e®D$%;m³j†± SÊMKÉá†û{çç½tߎ˜Ä§Vø¹šÚíÈ-Ñ^ñ‡|fê¿ä-…>Ò¸§£½…W¾†›é¡´…xÂÅ”NÐ&ÞSUê&ÇÞ~tn äåô"côÕ–9RF‚ÛøÖ5¥´»cƒ~_V:fRåÚiHKÑon\¼x ¿flåâ?ñgÛ®ˆÕÞQ‘„ßiM­r@Ëb+v°s‰yD@«^ˆQÓ‘©$álAY—nìa¿E„Q½mö<©d#°ùÀck,D”’d4™˜W0K,2ËÌE™pÚ³'"+Œèh;}oÛ|@žáûòóø~Å„*ã <<åTø1ôþ…ð(ô1nad †„lO K¿ýS«hwç;nÚ·bšPKÞÅb6 W#PKótNMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKótNMŸ.Ä++mimetypePKótNMs§"„ÇÇQThumbnails/thumbnail.pngPKótNMïNÅÇÃ-Nmeta.xmlPKótNMG!Configurations2/floater/PKótNM}!Configurations2/menubar/PKótNM³!Configurations2/toolbar/PKótNMé!Configurations2/progressbar/PKótNM#"Configurations2/statusbar/PKótNM'["Configurations2/accelerator/current.xmlPKótNM²"Configurations2/toolpanel/PKótNMê"Configurations2/popupmenu/PKótNM"#Configurations2/images/Bitmaps/PKótNMð-=- Å8 _#styles.xmlPKótNMP™«6Ò žn Ä,content.xmlPKótNMÞÅb6 W# Ï9settings.xmlPKótNM¿(ÖÞ?META-INF/manifest.xmlPK6k@apache-buildstream-27ae392/doc/source/image-sources/arch-datamodel-element-composition.odg000066400000000000000000000426471514607367700317770ustar00rootroot00000000000000PK£9MMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPK£9MMbéêäThumbnails/thumbnail.png‰PNG  IHDRã] :ÌýPLTE ()/#7)++:,;,=--;;,55H==Q k or6I6>S>"{"C2!D33S>)Q==k;KK8YB,RR=tHeK2mQ5}S(xZfš9i¦BB­GG±XX¶CjœAoªIu­Oy°R{±ee‡ll‘cy•ttšnn½zz£ss¿HHÿYYÿnnÄvvÈggÿvvÿ.ƒ.3ˆ32¬2=°=@Œ@L“LT˜TIµITºTe†el‘ls™sg¨ga¿a|¥|z·ziÂixÈx[‚µg‹ºm½q“¾x˜Âšž‡& ‹-‘7Ž:&’:$¢§%%ª33Z+†^5šE#˜F2ŸQ?‡f4¢S.¦Y3™UEŒfA’mI•qMšzZˆff™ss­EE¡UC³GG¸ZZ wO¤{R°jAµrG§}}³bb¾ssÿ((ÿ::ÁddÂ}}ÿHHÿWWÿggÿww†S††d„j’’m™™r¬U¸‰[¡‰r¬œi¯¡n¤¤{²¥r¹°}ˆY˘e‰‰‰———­‰‰·­‡·‡¯ƒƒ«š‰»ŒŒ¶–‚­­¸¸‰¨¨¨¶¶¶‡‡Æ˜˜Ë’’Ó‡‡ÿ˜˜ÿ‚ Æ…¢È‰¥Ê¨Ë“¬ÌŸµÓ££Ù¤ºÖ¸¸Ñ©©ÿ¹¹â··ÿ‡Ê‡ŒÐŒ˜Ë˜–Ô–©Ë©§Û§±Ï±µÙµ¹â¹®ÂÚ±ÄÛ·ÈÞ¿ÎâÄ‚‚Å—‹Ë˜˜Ë¢˜Ë¹©Ú©¨ÿ††ÿ——â··ÿ©©ÿ´´Ë˘ÓĶÈÈÈÉÖÉÚÎÂØØØÂÊäÇÇÿÄÒäÊÖçÎÙéÒÜéÕÕÿÅçÅÔíÔÛðÛÖàìäËÇá×ÎèÚÕÿÇÇûØØçéççé÷ìõììñöøëêôðíóóóôöúôùôöøúúõôùøöþþþÿÿÿµ¢àDATxÚí]}|”GçÔkOA ­¶Ô´æH {KëU{¥„ãø»±ê]LR¢¥œÚzwÚª§)øF½žÖ#Is‰9 ×>/’™ÆäƒmÁ4)!šzžØ&8v¤VT`È$æûýæÙ$»Ù—çÙÍÃî†îì³ÏûÎ>ó}~oó›™ßÌPoñ4#@€ 2dÈ @€ 2dÈœ~ÿúËÏ¥*ýâõߧ€ /¬\¿aÿÁ„Ó_?wp*iÿ†õ+žHÃ/ÜýÂÔþž MõÑ.¼¼ò'RÀÀÊç§ú×—\x8ÿÁ»_O ÿsÇÔÿ×àn<ÞÀŠRÀë+ÞÀMoÝ9XŸï«»¨êÎ÷ö^<z×ùÞ£u½úÞщ3b|¯µ´½£=ä§'aim»<¾§/hO.3"•_¿ÿ‹[ÏÕ;ÚÛ{tëÑÚ'ŽÕžë{¢¯îhmï±Ú£55½½jtëÅ£çj{Ï×­«UçjGŸ€ƒï뭹ȅñÐðÚåk÷•vø|¾m¾Ž’’ÖÛÔÚŽ¾Ö“%%áë(-mõl-io…Km­%'K|ã@®x9Ù¬´þñXMÐÀÖÚc5[Ï×\<úD_oí¹ZH[ëjàœ:_«.ÖžßÚ××WS3ÚwTm­ƒKçj¿wq2 œXÛ6°¶Ôç+m_[ ¥õµúZ[Õ@‰.=±¶½¥­—¾ÛKÛJ:ຯm‚~×@røÅkÛWSÛWWWSs¬·wjûz¡ µG·ÖÖôÕíSþÚº>8¬;®¦OAáOôÕn=?™ÚJJJÚá¯v ¤´µÃ×Vª”¯´µ½¤­õ„>»vmûÚ’Ö¶ßZ_Üãkoóœx ýÏ%€õo¸’1w®Oú|ñÔá 2øÃÝÁ78ˆëþÕ?Ø{#ƒ#AWO Z÷ŒDÈØ´ÑÎ ûƒÉàïŸ:è/[×ÓútY÷HYO÷i€ ì¥Oœîïééïì|©§Ç:7ØÓßõœpþ¿'Ö¶–tœô<Ña{ï/×'€õg‚ºÖ­ë¼¿³»ìþ®2Xw Ž”©®û?UÖ½î£÷¿ÔUÖU¶®ëc#ƒeª³kÝG×õt–uý”Š˜ÿÛÚ1¼¶­ÄwûIßíöyGÊX×3ØÕYÖSö©®O­ëÿD·ê_wªì¥ÎOÜß]ÖÙUVÖ]V¥ï¹¿¿¬ì¥Á]e=ÎY 4âí%¥ Ó ˆÄ vwŸîéìïQݧÔé®îÁ‘îSý§û{»û{zz{Ô)8wª§»çTw¿óÿmoëPí':ÚO¶·;ÀŸ<†ÝBÛ†âJɤ€á»\ÊغÀ4¦7SReÀ]éÈj0xZÄú87€cæsz0~ !›W\ €ûæΞ=köìÙïŽú{Ø‘)|2ŠÏܪÀ‰©¤TR@Ù¤^„”ê}Nâ×úÁî;&•÷Íýä!›t_8Xàðìû:cçÓyßÜCÑ)À€Ò™&7 B™ipbP˜iÂ>%†¦a2Ç&59U„˜pÖä¦EfObÖžC¶é“³Gì©í#÷ÙgÔ9÷·Q)ÀPœB©Á(á&,JL(!̦â&\pl*Ó”@®\6ÂÈ^ü|î!iÖˆ- ŒÌîtѽ‡£R'\RÆ9å€/=¡?œQ®$a‚*¸oŸ)§(Ü €E¦H‡ïuÀd‘ÃÃdÐà¬C‰®cbº"˜«v@0{ Á¦+ðØš5Y€‡žœ  ò©Ϧ+8§0©ð€°d""0‚Ù¢”\]PQX^^^°p÷ª… Ê'@cpS8˜ * * + +0»Â9(HžQʉ °“*“Hþ˜„¾€=ƒ Z xH“œéS’€À‹šSXU~u°q *+Ë+Ê+«* «**++bÌò‚òŠªBÈ( bOE¡ 0 ð&,&È|Ðv°VC…HýÊDõÇA=¢f€•¡ðŲ̈ÁA-¸&‚˜~Ò±c8CvÅ¢°oàÍj+ǯ-XI-dÀ;‡~9ªOú]¢€r‹P¯}Ï¡ŠàwÆZVF•Ú]9žÑáØZ@JåzŠCT\«`÷5åsʯ™SR°° 0ðNÃ`2ªXPUtAáî…s*®)˜³°|NÅœöv’²c{cëð$•‰ZP±àꊕåWÃÓ¿ J1gAÁž…‡âd\ú¤òêk0³ª……v2„p¾—a…À0Å#0û@îùÑþãÐ8Ñ`š` ÇÀ¯à6S¸#**Uڽ§Ra¯j÷¡ÝÑ]:£ªª=À••U aµgÏn;R*¨@Å‘³^bY¾PvELDd )M"Æ&e‘ $@L ×´@tá区'n ¢¤Ç„:¾u|éP% ¨9y›@"ŒjÂÙ!@?B…(aj2àg‰ ûíëX`¶#SøgSñ1:e!¡6è Ó€?îÚ òܟǪ „ëñ´ˆeÄ)F>þñÎÃQŸ7 çî {á£{b;ðO:ï­Šî@@C@ - @ØKxßöãŽ% ¼Ì8•¸öÃI6åÚ ’;+$½{ÖìÐpV~E"ø!>9~ÐOg‡œù»‘¨!é¥`Þ›`š†0Ð0d&Gì^n@½€1t€ù ²LcOÚùCœù!É‘A98‚›AÉ9ì á X_ŠÜ<ѧóÀßY‹ø­ÔGƒx„¹Ë>Ax­ôúCŠ5 LoÙÄXvô¡Ð3±ø€0(^•®{…GØoˆ#'•rÎùŽïøC(çèýà‚IÊÔ…AÌkg°{†ÂE€ô„À…› *Èäe@¤ÇfÜdäÄf‹«]ÀïwY Dxl@À «\2HÂ1¤MÛ ÐçÜ “t™’Ñ2ļ6êDg-¨i×6(Üjör˜Ñô¤:=)ÀQÿ'|ë €4¤€¿X>\”Ô ™ã‹ueà¤+|âg!§£g…A¡Gú©f]A›÷/dz’‘²Šõ¼þ +Îx?ñLâ"¸^ ú‰ :¡¡¢) l–¡†d†„š9.PÅ‹ú ,ŠÁB”a‚í&š<++}³Òw{å¤?1°FkeŬ¬àÞ?LÐÍ <•¢°‘‘žW;ÍDz‡…%‰,àGe9Me€[BСpÄÝw '×Ô C!蟞 ]ðâlúi»\bᤷø…i*\d5dÀk?ztcHÚT=éøGCŽÊvñG›BX½irFgm(@F¨cðER ÿÊƼ£Ì‰GмvýÆíÛŸŠ•¶=ró.ÝüÈ6»Œ®±e€‰Î"ˆI¥LèʼzÇPÜ &\¤Jx Cï!S—C×o?n›žü¢ØTmŸÑ¶›GcR€ž!‚­DÔ&åÓ®1¢]aÊê<æ‘0á&arâœ}ßqéziOü—ƒŒ¾òJL „@á°QœyÑ{LwÃ&…=°yLq ŽŽC>uðÊW0Ƙá¤t/}.P êž±! ”B%aÂs]4ׇˀKñ±@ yâ‘„C¡9‘Pj ;Õ¶ ‰e¥ËÍ(\†]‘Øsî¤Ì5°oœèw?Þ0-мc<£æã‰i¤Õ&ر8§±Ñãñ,öäxsr{ê‹¢S@x»@9Oƒ§!§~1È”œê‹sšc²@$-Ô6#Ñ·‡LX:÷ê44í»ÁóNOѾzÏâÆ¢wz<7ÄÃrÜ¢j.j¨¿Á“SäÙWäiôܘSÔxCc|À€˜àœJì'M±m”cw"ì ì(}9ìOãñ†æƆÆæ};š=°ÝQŸ €Ü¯onöìhÞ±¯¹¾Uas\vp?h0Mì úQxMÇ` ³i#ú·¢[‚qËü à8^‚˜dŸ%ãt·(­Ñ>•Hèå €xp¦Ž'f BmPŠq'º%„´zèë~!]“+Cav@8 ÞÜì8/Ÿ à¨:¼íe«Uõ“ö5ß|ñ’3-¨ ‰ñÚ'?"S஑'·§mú³}¬­´}ÛSÛŸÚòÅ?:q€½oË6Ìã©@NÖ¢óÐgáóäõ¯9ô Jôv@õßË©äe -eôÓU ”þó?B«Ý4äÈ'øÇMp÷–j\鿃2“Õ›^qêæÚ!b¢ÓÃÃûFâv¡bl 8jpòŽZ†¸³ž&‘({Àì)å†ú£Ö(9BPw) ü±¹;CÃ…“ÎFé×. 8sÐGȉ"ÆÎFi×GèÀ€ƒv¤xšQ€kmƒÎšÆ6Œ¤[Ë“Ê™®=DÜê"ãh|Ï´í “Ìé&xÊúHÝÈÂm’8pCУIq¹(À‘t“Æaâ5˜dº“5€…ÄŽSöL†¹qŒ§À%6—‚qD¸ý»ºl2à2i‡‹šŠ&† T 7˜b<C`r…U¦1ƒ˜Fª´€«,0~—4µÕ ¶`¤ö[Ã!cè1¨5jëFDÛÊ&±u¥¸«l¼ZÉ} °Çt•¬2<äÎ%ÖR ‚¡GÐp¼—‰TQ€£(‰jµBª»ÊÆÝKLå« L½c—‰L·ìWYÀ?á ÄV|"*±o„0M˜Téå°d š¬G•rÛ."—Íp•&œèüÀæ?ø*i`Ê+£=!8®‹l`c€…~c(`xà@húé‹-!Ç/8d'eôÓ¶Sz@ `é„©0– l)è@†Í†¨ˆ[uçtÅMdÀÀò%ËBÒò}pùòàK—ÜãèM~xÉÒÉ9…/]²<6¦\Nö0¼ìƒ-v鞥÷„»¬Âþ î²ËèÙe¶ÕAH„Ÿr¿.0°äǶ´<»$ÜÛö_KÿÆ>£–Û^´© tÓí!”X!Å(ÃæQ%Æ#ŒáF8è$èˆ,sðØ-Kì_Âð'-;›(Hzbz…î¢C§pÂ#&˜Ì` …Z‚&*žw%À°­¼8Aû82åñâ}ôð?ö K{AeAw¢;¦t™fæ¯^³fÕª–Õ;׬Z€p58@q^q~Ë®–]»Zôz•s «;‰BµO(*©®R´à ŽäÁJ(G#g©—«½ àªÜ5ÅWÍœŸ›ëâùù1ˆÍós×äæ½-oþÌÜ«Þ537w¾s H¾¡€ÜUŹÅùÅ«WåçíÌÏ‹‹‚)`~þêÜüÜüÕùy3óóvÚRŸ ­ 8€p#/DîÚõ, ®4²Iw @žmY³w×ØÁp§h0èŒv­Æݺøðbºù' A`ÿâ¼Õ«‹sçç箉‹4cùkòróòV¯ÎËŸ™_Âó‹cÎÁ×2þü¼53gææ嘹Ó@èW——»«¸xÕμâ¼üUk€b cÉ€âÜÕkòWïÊËÏÏÇìò§‹ˆCFצd ¦VÄa Úhh@¦!üniB„·ßöcêÉ—Kîi±¥Ý{nsàYz§ƒjåmo¤ ,¹Óžsï\æ€eK£#9vêÎ¥*Ýd€RÏ.]¶ÔJË–.ÇÕ‡>4¶‡ø.·^"u¡xq¹þÉDFÖbÓ™-ÿ±W8ùÉáé žM©{™ný„“ž=n†ÿL3 Æž=ÃnPI:R€ßQ %ÆlǽÇÕ[qzQ€\\¸ü“¦± pÔ2tS€Rg¼F¯X Ô¡’šÒ’Íi'.¼ÅeJ2 ¤]?Aàª)<)`ºZ‚®É€·¼ W´lH1üê›}&$ýË?†ÿÓ7ß ¯1ˆ+F¼ò¹ï<;=ýµÏ½yåj¡‡ž~Æ6ýë—£S£“‡‰àX,¤7Ñx,¬ô‘gzÆAúÌh40Ü ÎÿÁ1÷ ÔÀ9’°Ÿ „{12%û¹aätŽ‘²BcÓ§–^ýRBŒ³ÇÉžˆÞŽ¥%ñF ûûØ»çÍîí8+ŠAp <¥“¦V$À8èÙ®0ÔÇ8W' Ó]±˜zÌ+N€áp†( aâ8X‘>ðJ0ÙYÐôØ3ÙMMM?Øø˜#ì“°íœZBïɪ¾6;;+ë×fmÌ àÒ«B(à¦gžÉÚòž¯dÍÛ˜=/{c,€hb¶8:j·]bùØ8 t¯äÂTi-š²ÂeÀ¥ˆ– ±"œ•H…3¢jâ9CGÅ2NŸªgÆà d„¡¸—Y³ÒŠš,‚ÿAõ£8l™à¡ÈZçB5½ë£âÄwP,œ„é9Qp\Š|œJÏe ô  A˜2àéyÙÕóª³æÍËξi^vSö¼ÇnÊþÓì'çm™7/²´’©u=…D€¼q8-î€æGu×Ѐ ˆÿ%8U^AcoÓš²žÌ®†¢g5Ý4/+뱬¬¦ka›½å¦¬›"S€ƒñåŽ[ÏÒBl×Þ´å±-M[²¯},ûÑ-MÙÕÙÕOg7=ºÉƺ²´¦wüI$CèRÂ`‰|úç$½í€¨)ŒÆF¼Têip6(«7»5ÔS¡v %PIŒŒJ´€ä8¯(ÊÇ´¡€+CcÖ¼d%^=U¶5ÔÕÐœ5‘2¨(P-2±6dÒ £†$^ª±âi#†úŽ}ù¿öÏÑeŒQÁ™Æ0tˆ¡gHÃHp8m‘5Y\–” ¬!hˆ M±Gè͇¾ôÕ¯~MB6:YÇÿðå0'Y̆Ÿ»(Õ>Á³¿š”öîÛ l}QÙ€ïYdÆWD«@Au”ØÈ`ˆtö ;™lm‚t=˜àœpÝýÛÔóa2üšz­©‘Æ¡Úe2%HúõzÍž„Ç]% Ƹ’ÒÀ¨§:îñ‘Š8¥G8~4ƒC fb½ ÄO–f çHsÜGˆY‘®°2Dôܹ¦¶ŒÑðGïŽxIFõœ©DOœj"¤{Aºõb ¿¹»’0ÓÊ §?övü„),Ãv"äóÔ´ì#4Î:¸” }ÃRY3,Z_˜6#cX“¤Ã9=_¢ %ƒi9ÇÈØ{ô¢U‡¶0É«ùž)a ‡Ã›8öá¡è£ÆGÔ¾iÛÂÓ²@ :¯ƒ€âÓ!N  {èÖîœO[ ÃC¾(¨Xs' väiLNú‚CHv­ž "Ø DÀèÄO¡Ž„’¬#=æ“pª0"Æym€°QBÒZ\p•MÒÜ DŒ&7FÖ“ƒ\ñô® $D‘¢Éq£]M¹òëP? /ˆÄ‰E´9a¦e‘H,@){”aÇ0Ðch1C]Àx”)ÓOœIp6¨Q ‹1ÁaçŠF5É„Á¢ƒ–%#²N…­¬ W@ôN;ŒjB0‰°¦‹ H<¦¨ ‰Á4vÀlbc¥š^ýþÃ_IßþFèñ·¾>F]À*kHýV†½l¨ ¦‡ˆ ^}ð[ÿþ¿1ÓwþB”$P«¥8;†ºÃÐ×DØ$qj —Ñ8f?Å™!)ªwxzPÀÐg¿{Ä6ýÛÃþ0~×^(¶o1²‰Žó‡–/3Qb,…Þn fÄþ1z– FÁmí©•o~þˆƒôÙˆ-Cðb™k…7 ¶ƒjÏÄþ º¼cT‰8=žÕQ†™£h4#’¤ ¼úpBŒ¹6Áè“þ¡a)qV)ýB‰Ñá¡ Ž´óS{F1þ‘µŽp¦ˆ`·hŠÛ Ügª±ðjÈÖS2}) JãèX„«!Ê…Lc å#0fæ· N…ÂX'Â!èðj†„ ‰: Ý)`ó­{÷ÁÏ‘Í‹?²wópóã_ß Göâ7{‘)âcéèr&ÃY€MO pÍò­E——’Hjå€;LW¯°ZfÒ‰ó&€œ¦°þõÉÌüùoC_\î_¶ßø¶º|,paE2)à… a>àÏááˆé㛿uQ]>øɧ“ €EXx닯ÆHÿ§Wá4ê" ¬øC2PŸ>èJÆî±À먤0°bÀŒ]cÿŠ_&µßÄ]:»ÿ²@$ü6¤ <¿r Ù(ÿ¦®xýþiQþˆ(ÿþ SŽÀϺÁÿû¸Ìå €RW>Ÿzð¿<å§H5üüÝûyf éoœ™ZúÝs+÷'!RÅŒ¨W†_~`}*ÓýJ¥€·HÊ @€ 2dÈ @€ Þ‚éÿ5þGò~r†IEND®B`‚PK£9MMmeta.xmlSM›0¼÷W w¯`,`¥ö´«­ÔTê-"¶7u 6²Í’ý÷RÒDUŽ~oæͼçǦÞ…±R«à@(¦¹T‡|ß>…ðX~ÊõÛ›d‚rͺF(6ÂUÁ@U–N©tFQ]Yi©ªa©cT·BͺDS/4EŽµT¿ ðÓ¹–BØ÷}Ô“H›ÄY–AŸ¡œqmgjâ ŠZŒ âÃ;:¼×Ôˆ]Zj°C¶r~(÷ÕXr–µ´ÖgÓ#m€·#”Àé=£m#ë{GlÈtÓšûúb¦•’ͽeFì•ëƒá¼¾µ”Á3ÃÔ+W…ïRôŸÁÅü¿ÛìÜíé ‡ƒr¾²qeî—ÂŒðÞÂAP”1›£“-N(I)!Qš¦˜dd½Îá ÆTå ”0•Ó¦|–{#^½ \E8J"üð,UwÜýؤ»4 €]kô/ÁĨA_:YóŸTþVÌ9£WæRŠ Ei„É Å$^ ´69\ºá‹…¼3Þmùu›l^VäÛ©þUþ’Æ>X-l¹þ} OØó_µãQZ'Yàãz?65N§\°ÌáÅäá­ß^þPKʆ¿+PK£9MMConfigurations2/floater/PK£9MMConfigurations2/menubar/PK£9MMConfigurations2/toolbar/PK£9MMConfigurations2/progressbar/PK£9MMConfigurations2/statusbar/PK£9MM'Configurations2/accelerator/current.xmlPKPK£9MMConfigurations2/toolpanel/PK£9MMConfigurations2/popupmenu/PK£9MMConfigurations2/images/Bitmaps/PK£9MM styles.xmlÝ[moã¸þÞ_aèpýFK²»›Ú(p¹Ý»~=Ðes—ŠŽãý÷±ÿ¯¿¤3¤^([räÄIö² lVœ!9óÌ 9#åéÝ3Up™Ýxá8ðF,‹d̳Õ÷ë/ɵ÷ÃíŸ>È$á[Ä2Ú¤,Ó¤Ð;ÁŠLΊ…%Þx•-$-x±ÈhÊŠ…Ž2gY5iár/ÌVvÄ,6tºavgkö ‡NFÞÖ\º¾³avgÇŠn‡NF^ÀԞȡ“ AI"™æTó=)ϾÜxk­ó…ïo·Ûñv:–jå‡óùÜ7ÔZà¨æË7J®8ò™`¸Yá‡ãЯxS¦éPù×)Û¤K¦CC5=°j®X, .úå°…Ü9-ÿº_ ö®ûUÌÑšªÁ~f˜Û®2‡»Ê4vç¦T¯{ì{íßÑüs÷SãW*ºò¶ ŠÏ«i¹ÝùRÊZTœ`ƒÝˆ; ‚™oŸîíQö­âš)‡=:ÊQՈ˴ 4à }à ì]¾Ö;åb°ÖÀÛã$4メGÞWU~¯†¾b¹Tº$žta—I2Ö:ý)©ëJÅq'+ˆ3õ!}@ð’{ζßy­Óà¸#Ì÷Á¤ÖǦ&7÷>òÔá ®ÑjUe‰Üd±µƒ=äLq$Qa¦-Z+¸Öò K–石BëPàLT™¢V©s)IZžA€È|áÌne­¢˜ê.óýòoiODÈùå>ÎE`âÝV§~"áÄOhÄHÌ"QÜ~°ÙºÙgòÆû‰Cê7Ê>Ñ  RjÅ ³»ñþLsYüeÏz£ÖÒÈOV,­!ð•LiÖâȹŽ 5ÞSÅoøÃEƒ%“A²!ãk x„g®Øò¢xŽl?²Ïô?›ãötx†ˆ´+4KŸ%“’<6Û>R!–4úÒ/Û!ï«ÈøQ1ÖšCz®~_˜–ãö®~ûÓ!\&Ô¦FæÿVÈ¿*%·VBÌã“7^0 F“`4 ì8$¤»Æ qp= ¾âÆ= ~Ú¥p!~›¿•«rÌ‹\ÐÙcõI†Óp4&—3”ÀøI ­äPžkWœ“2;SšC¥…” FRH¸\7]^ÐY¹mÂ…¨)W“yAzKäb K™Ûkt& >7BåTQ³Yk+CƒЖENQ3iY©È×´rÊ|“EzcÒ‘Yœ–§9žÓ–å#KÅ(”(Z[W¼ÛAEDRÃòB½ly:Ïb†,7Í*(ˆ)K* Vƒ×Uæ†B¿Z5;êu í¦`ê³Xn‰Ù¼DR« k ÕwÒ†^ð¯@ŸÌrmÆÍVº‚!–™®ZÉýt°.[Íö³mÃkWÜﱂ=<²kÍÕ¹oM]óýkÒ?~öû´b¡2Œf ¤˜ªØ;=\œ„ëHÁcÏ ¡-1lƒ(õ,›à@V(éê‰ã N=¤G ïc•‹¶À•LGj×dŒëZƒþH7tm ¡FÖRñ¯¯‰¬|¦ÿ¼)4OvÆsc{‡@Ì (áä…qK©5VO]4ÁmØ'(¾Z;k€5M…ÇãÀ$p8L“‡6m⮓Xé~à_ðÓÕwÉrÙÂ2“åF¦G–ˆã&Ç›GK"XÌßxÿûïïµÃ9‹8>gæ¤<#‚.RÛúðûÒO•8*ßþ„“q)…7$5µóÎìÂ8þ1MOÀaò Ì©A–,‘ŠU ¼?„¦gC(´~þîš¡ëw‰ÐÅÙšŒgï¡Ë³!4}—ø\Ÿwš§¯Ï†ÐìæéùzŸy: ÎÑÅ+Q;ä²bóû ¯£Øª*l«ˆ²„(« ¢=j«”öX] ”æ=°fv~ß›Qƒ7 Ëú(¦¦«†Ü™T)Í*ZAYlß«e2cƒv¶’"Ö•C2¢ëµ’›Õš”]'w‹C&½Ë÷xz:¦’ßw——k¾ï¹Vx]ö@ìX¥Yת@<ÐxfS¶ëãbR-·-m]­WÆÓP?(§3Ó'êìÞÔ]–Îþñ%÷ÓûÈÕ }½ÜŠÞàØÓò±€ºD¤%µ„Ò†vèXGzS¥pÍÜ«rÍѯtÍÒ£vMïW¼‡ÅøKó5µŽzIŠ Î’§”÷]>ÙÅP&‚&)8ͯGš^rù™EzË5/¶-ÞÙûªÚ¯T՟ノÇsÛcãð¢§C˜?°ú}‘ÞÙÕñÞ™ ÛÞÖÙÔfòSÑ´©æ%á,sÙ=/¸qéév=As8}°gø2ª?æ¶i"e°èÿ4¢ã;fÿŵð'HÕ b¼U:´ßêØeF¸Æ(“#œ=‚©øwÒ F–MOÆsÊCÊ¥Œwo)éñk]y!¹„ƒäd­°qžpÖûâõ´ë»RÛuí6ÿÉjÚkò·«ä uCûÚ_SáàZ¼ibâȳÙIŽŒ*…o¡SïK«Àe¯|l"{jŽÉé昼øY\ß–‚æm\ËIEóù·eƒ!AnwÙoŒÃžAMè°ïÓzÖèÓ““5£ñ[_9A_¢9›ôÁ9 ‡ctZ`T½Iªz;Úm’ª¯±”">ºJ+ß,t³Ž×äÑsM-6ê5âõ°ø>K‰Ýõ}I缞¯RÛV)ú&㺬χÚhÒ•ü½/ËGüÚ ?‰HE¨p^1u²-+ýë.ð:xŽ¸^Ï1±7l °óªFÍ~%páÕøêÚ¯[Ñã«™¡”B*ÈHÅ›ßlÁÏùå.HŽ*ýz癲ü#‚ ÔÀ¸ƒ^ˆÌ•ÂÏÕK{-¥Šñh|´ô{mVRZhã—»æãSP ‡ðBóì~«iµ®¿è< 7òá±-¢Ÿ5GURá(fWÃåïëaA)•EŒ÷>œ6º5 £qÒÆ…/N‰c\çËÞ=,ýîßÆ»ý?PKË4< Í7PK£9MM content.xmlí]Í’Û¸¾ç)XÚ$µ» )‚¿âÄ3®ì:{²Wì¤6¹¤ ”°¦†¤FÒžö rL¥*y9?Ià@Q”(4;sª,‰D7ÐøºÑ 4@úÅËõ"RîIšQߎ¦û, ñìvô—÷ß©“ÑË»_¼`aH}r0¹ q®ú,Îá[î8»)JoGË4¾a8£ÙMŒ$»Éý–¸âº‘©oD[Å,ßD½Ù±Ì“uÞ—™Ó6xñ´Ë‚XæR¼êËÌiT™=d}™×Y¤† P_$8§;R¬#¸Íó<¹W«•¶25–ÎÆÈó¼±(­ökºd™F‚*ðÇ$"¼±lŒ44®h$Ç}åã´²Hñr1%iohpŽ[ZMR’ t—f¿Šdž†}ÝÏz[×ý¬fŽÓÞv&ˆ›¦býMÅ dÞÎçúŒß@¡øxózkWé¢o[œ¶•ŸÒ¤w7 j™Ÿ1V‹ÊŠÁ.Ä5tÝ×õê ù*¥9I%rÿ ¹#¿Fœ-öth *¹ç&_"DÖÁ`Œ‹âš8 :«þþÍëwþœ,ð–˜'Viœå8Þ"“-hÔ[ @Ûa´8¦½MÓ¶†NÊ¡q{œ’„¥y­ °€VŒ£y¾ˆº]/­Hgiì%qÌ1¸3p&ê=%«/FètØ0½Ã®þ‹ ’cÁA¤9MíNÀT·+Õ±5dË8(ôPHÖ I)/‘`»iÔ k+bgTYÆc©†F¢$ª‹LпpÄ_Yï§H òÂäuËlÚ6g–) œï/ç=ºÅ,&å î}ɼO²TÑøß…8T!˜¾ÞTs–ðú¨Nº?eyÎ's{Š"æ¼ÀpwKR:›KEÜh$EÒªqºVùÜ-ÀipH™ôrAðkņâ¯,a öi]ѵ«—9K錇ÜJ3?,³œ†›¾ T|À©³”­Ô9)€q”ê]и¾‹ Í1ŒRU¼ ìˆ§ÈÚÑà1ó:æÐÛÒ€9k5 Ötã´Znð{‘çéðw±àû°Œð~^Ë×µ‰å´ ižñD ß~4Ãßcº.™CkLt{ºëç)Œ]wœkÏããåÉŽóiŽ÷)˜ÅDGÞ5h=S|zfñDèä)˜w¥õˆ¾¸·àþb0‹ÓÌÂ{ fá8¦y=oñ¤ƒÈ5 ¤_9£pšøâï ¹½Û,$O©ÏS¿OAY¬È²Ýg•A@g$†~Ö5£­™æäÄ>^)Mrµ>ššëœØÅOméoh¦îÒÇ·-C….a!2t Ø êê€$m$­¹ej’w¤’Ý'|?ç[†všŒÇ3b׿eD²øWl·•ÕyLÕ<¢¬I¾U† 't§•8Ó Ê\ÐuuÞÊœ…}· _Z+b%£?‚ì“$¯„„Ë8 iDcR¸¹:eSt¯âSqFqÜàÞ–ñ£ Y¥ýAl­¨ÏÔx™Ï¸®Æ[뼘'¬²¯+lkõq6²bíw]aÛsⳡ};hOýž”Û噀ÂôíÏ£ï•ûSn_öîÏû–zxå£^«HHKέëCÓ¡"ûÄú^áD),·óÈCŽ¶b\•s¼)‹‚N÷[´%Š©iYÚlTªµ"Ø_wí¾;j¯Ë»ë¯ID ýµÕ2¾¾Úâù®A[¬­ÖL÷˜¶NŸvœ„ïîdä$ø:˜ÏŦ5ïoÉÂc–ü˜–ÜZlôÕVuœfÐÖåµ%žÝˆhVÖØîåðy¦~zr1„¬gꧽ "ô<ý4O4^"ûÓrÔRq9×w>HQLY°©/Ê'&î^ˆ´n¢Hss~ê'6õV©xô¢<å$\¨Õã7!^F|É%ø‚§]x£Þ(mlþ­J"¼á"Ã[æÅ!‡5‚ˆ£¡É¤:õ°A|òÜúÆÚàF]—ÈÒ®ÓÒžj-Z)¥¿Ì-5›ãd¯´F§´F·´Õ¡]3u«’¨>½jj–S˹Ý°ÍZîm·$±EC$žó§MuF?§± ü±ÉoT£+ºb G/?+5Ñ4e©tˆ§>x#W{ÕÍÂçsèsJrž0 *–Š“ºò6Á¦IPdˆS‚³ÛÑËÐT^†ü³á_}(q“þX¬‘Õ”ø9Žgõ™¡ hHI ÌÈ3 Í°'¶a¹¶9©)Õ$âqë7P± 0|ÿ^†åµøö”¿‰û8¯á7ªú¾üæD¯ëßÁ€8 WþXÙ3ùç²ÈHÃ$¬O;³t±Œ0¼£ñ!´ËðK]ù:£ñ—/CøñeBÇh¢õÕáJŒÝJ@Zåk9æØuLç0³¹ËÌOÓüæeh(‡ù¬]¾œ%=Øì]6qDG=Îèì2‚zpº{{@f›ìëàQ.¯CÌ£Œ¨e;22#Xw•~©ø­&,£Åû€$5J³ðŽUd‘ËR[D]«ÜèbÉOVu”ãuQ^ŒäÚk¶üOU ûÐRð0…Nîó§f§?µŽúS[Ó §åN5Ãm:SwÇ— WºmtÊÖ•g-§²fåvar·ËߣÑÝ7KïÀëá…ò-L”W8Çe<æ\UlNÐŒ1º{»œFÔW^A,§ñ6Æ;]oÐ3ÆY:±êÄÐlorD'†fN Y)†æêhˆpgn{4Dµ!ª QíSŠjv§uÄ;/n(ÈNƒÊH›WWs®—ŠyP^Ĺb—]+9(º½ r{@%/´ ÍahCCÂÐ熌F:žÇz˜o½`úCñÎCD~„‹“׊En'”“‹…íFÒO׌! ÁhFC0ú¬‚‘ÕFÇ“€t®ŒFoSÆŸâU¾eqHgËëÆ£I'šžŒ¦Ó@Ó¹šò ÓÕ\Ã"Õ©†H5DªÏ(RÙ ßz¹­«é[uóz˦ëæï¼Nk»ºl$²5dºC$"щ†HôE"ÔˆDŽÍ= oEöµ–L7šÒ€œŠXCE,݇"Ò»Ñ~¬$ë/Ó{²=thjŽ´”D[\êC‡œ¢~ÓFܘ¸Fýn¤âÝKBû²²ŠÛ³hÉmâÁÖÞø[–jrCº)Es ÿ7&¨Ø\(]õL¤ª‰tøD†eŽÚÏT ÀjŸ‰¬ý9F2ÂõfâæÉN¤YÝ›§alõÂn\,ـߪ©»ð‰ ÃÛ²ĦþÄ@6 W)‘Ȧ òDó\³ cë4Œí–Î[ÿt±ll•é†ÒçjwAéœ%êi®.JWiKŸ`Ûð‰LKßïL ‡øŒS}èxtês¬Ï¹ò±>k»Jj¤ó®zîîÚç\ÍjíÉ™ÃriX. Ë¥a¹ô-—*]4¯®å[/¸\zÄó=÷—¼”^/(­<9”»ûKΦ†05„©!L=û0u!º»‰d ›H=à ßD·ÂÍn†póüÃÍv*_©°yuÙHÄ7Kœai¢¹íM$GÎsŠÖ&RgòÝmÜ>š1žH7;2Æ·ØDr>ÑM¤&~h»ˆl ÜÊÉ;]{§!ìöBXääÇÊÉ£/Z9þÌÇÞ¤¼­ÙÛ7°´³òž¦[—wÚ‡Òò ÍþúIî­I P¨`N”ŒÃ±ÂgÛ§óp¿Jõ@=\ÿV!kŸ$9pà\¡9¯*f¹"ÞÔ ž–Êt#ªKJgã7šÏ/„Êa²ŒJPNŠµs]ÑÅ_–ƒ4´õÌ|@XšÓØSE`ÿŽ+ˆ’âÐÛrº­û=9=ŒÍÒ\´kk–&í‹ßÆKrja/hjßn‰«>¤` JN¡ÇÂL°RÙ" ábFïI¬|üé¿h|üé܆hÌÿ/™œb¨£Ã àø2ÕܽŸsQað`>QX1pŠþ”]üøÓ <ÂY¦|üÀJ‰ W©’’eFζón:°…ØÇ€&-‚ùŠí5-ÈjflutM"0=Ùæƒg˜[†€¹]ÂvJ"¶j×jèiSq—kÇ™ôŠ0~K¼ükÜx5X]Ìü%o^¯—Ã÷ÝÿPKÄnä sŠPK£9MM settings.xmlÝZ]S"9}ß_auÍVÍ<`+:ÎH S 2RƒÊ Ìîì[è¾@ÖtÒ•¤EþýÞpèFLwom­Zv'çœ$7÷ .¾{_¿\ˆñ˜P EDÀuEÖ8Dàt®j‹×u/‘¼&ˆ¢ªÆIª¦ƒšˆ¯¦ÕÖG×,ÙâÉ3£ü±îMµŽk¾?›Íg'‡BNüãóósß¾] %(„"Ú Þp}Î:m ø˜NöEYŒ^Ÿ/„xm&,f…WŽNýÅÿ«Ñ*¢l_.3¶ˆ(FÍ#öÓ^N£}aÌØ­U/TíÖ~þ¢}yXk&Rõ+{X™Aãb¹9‹?ª!26r°|l4Ö=¤¬=Q˜½X—6ïç9ß©2Д@"öV/õ<Æ—”k¯Q©žV/ümœ7awa¬ÓÁOÎs£ÿNC=Mƒ¯VÏNŽrÃßLSåŸU?~Ú¿‘¸ByÏnrÁ,ý°ì´79ßG1Ì:á†L¥%Z‚×0vá¸Ñ_% ;j¹ð#!î5´L |[ ¾¹Å/àc”#zGõ9‰°¯}…Þ#¸%rB¹*ÄüîR™lÓýè/t[âÃÒYzïP9‹é±DÝ2PM6#ó׬7™Ý­+I&}¼i,ûxÜ-àšÑˆr¢¡'ØÜn[覡KuK½^Gms•qDËóè’9ƽM|¢àì´…"äÜkø{ÿ|®»i顇Ôä¿¢¦+‚G_—â¿u'šÝÿ¨Œ¼—‚‰m‰ùý!a KŒ7ÆWÃ0¼(§BâÕq¼7}`èÑ 4à…w”5…[–°u%ÜäKFƒÇ<ëëfá<4–àrJøÄ¢N(ž¦ÏhªòAÌÒÏàÔÑý⮤€uµù³ËÿiÚ~wÚýÙ=Û¼D*ÈT\ýäxªÿ€ïО ½ùß.áùwªv‡6IŸýÇÏ’h!Kä¸ÂZÛf0,püøÖñ£œut”Í Í57IOfU”#90è%$§ Ýó”»GðHy¸^[g½^–ëéèý3‹‰´Áï-šf³ùP¼"š5[TãBƆ݌îÁ»÷XÙ꾚 ~Lʇ_OZïÞ'(ìÃ’ØWk."ÃC%Fnj0G)y)"ÓU4=±J)4§²™ká+ÜÄ\5-\Kè¨Æ$aåôuzVâ½®£„mQT9™ÝLk 9¢aüÅ–ò·Öº˜¹&)©ýò€€»bbø3P‡}ÇŠHJ¶šiKÔ=ÊÆL(7&g„k׿HIH¬ð{5q;‘¢ƒâ/ùÔ5ø’7ÚH/áNv–cùÚp¦³ÒâS:A‹eÔ®ë;»Ô¹I0À—Ó¡´èmªw¥Nù n¬×5¥4Á-‡A¿+«9m”A¹û4 ¥è7oánÆUŽ áPp¶àŠXí-•Hò­†Õ27t¬Á¬%˜«HÌ#"’Z ;¶‘z˜ŽL$‰§ý$ŠÊrºÖÂ~K£zsÛóô«,,ÈF`ó)8ÆÖ.™‹$#Éh01«`ö.Xb–™‹2å¶ïŸˆ,1ú “Ítö­Íô>y‚ï‹éïù%ªŒ;ÜðB–Sø[èÝ} 7àabÜÂÈ ŠÙŽö¶êÜV×þÖ!ü¬¯Ê4þPK75Ùl#PK£9MMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPK£9MMŸ.Ä++mimetypePK£9MMbéêäQThumbnails/thumbnail.pngPK£9MMʆ¿+$meta.xmlPK£9MM!Configurations2/floater/PK£9MMO!Configurations2/menubar/PK£9MM…!Configurations2/toolbar/PK£9MM»!Configurations2/progressbar/PK£9MMõ!Configurations2/statusbar/PK£9MM'-"Configurations2/accelerator/current.xmlPK£9MM„"Configurations2/toolpanel/PK£9MM¼"Configurations2/popupmenu/PK£9MMô"Configurations2/images/Bitmaps/PK£9MMË4< Í7 1#styles.xmlPK£9MMÄnä sŠ ¥,content.xmlPK£9MM75Ùl# Â:settings.xmlPK£9MM¿(ÖÞ@META-INF/manifest.xmlPK6[Aapache-buildstream-27ae392/doc/source/image-sources/arch-datamodel-element.odg000066400000000000000000000337561514607367700274370ustar00rootroot00000000000000PK*ñ›õ‰Ýî×[þý°Ñ??ÿ÷ÙãÓ\'ˆ]ñPb…²,ñ€±᧺…Ÿ>~ÝhÕjµzØÚ†Î×k“ÿ®·¾®õšmL ;¸X€§þ§Ú|këÜLÿã‡S]c¦=L¸‡„Ç÷ô f§»‡ïï½¢Áv}P¯·Zó:<ëA=€ßoøúI½КÔƶ^û äâÜ|ûÝé®`´†A¯‰vðcŒ9åM|y2äóþ)Rxnþ¸xåüÃ={eéWüýÛ+¸‹kÀ_¯à.¾{Û¸YCSG"ŽB&ápÀ¡XÅ•’ˆ_´eÑÞ2m ìŒåŠ¹<Çv Š… Hñˆ“`cÎÀĺß'›_¤Éøò Ö`$qâˆ3OL,Ö+Æ"0Á@_QÁÐY’°yœs›`˜®¬€6Â\`8ö4…öÐÚãyæôÁÄ·~«¿mÁª˜ûá ¿=úA#lMü‰?÷À¿#…JÂ\yÔ2O{'¶DBG= îa† d5ô:Š$ fgÐo~cî7­†ß„ÜüÔÆÖëJ€3¡~­9:0O M4ÌqÝÓ–LØãFON}®Ö˜W[å/`N‚q6ý9tuÜA? ú óæÄ/IžKJEÖ…³´ŸyæWØãr³Hœÿ‚Nñ¿æ¯`.f¸v³ I¤˜vã’Y&©Tš2)4¢Ç¨áÊb¡˜BÖ0½" ´ȤÌq~Z­$` ‚ÐŒxèM; د…Á¤5nÚ‰«iÎO«Oúƒ¾mÙV€â2±ýIý ÃÚ`<OÆÎó[á|<>‰xDcÀºB áÄ ‰1DbB‰\I@‰!„²°VHQK%ÃV!Á  XD® sК_7~cÒj‚VÐÚû¶ÑhøN´ÂfÓ¯;&høà›„M8¥áo×æxŽ‰­ùÍfì]@U£Q÷¡&j€Æv×lœ¤(ÇŠK,ŒZHAA§qC™@ÌÅrŠ’R]·ŒJ¬AéaЂ°Á’#É86øü0¨9Ƈöç¿U›´B à¿Iß=íþvØ‚0h¿6ða €£Ò êacÞ²ùV0öà ¬ƒX[kl÷›p‰qÝÑ©­÷ýæëqÀ52„óÓYá9´€Š¶ò)×Éç›H ´hå«<³'{g©³£{>™CšØÖdry]¬` Ma ÎÃ@@Œ¹$–K)…€a z_SN¤e`í58© øÌå$¢ØZJ)1 àQÊÎ@Øhþ©±]ßڪ׵Öö ¼|~~2³àPîq°{8§˜p h¡¬$B¤&$ï ë¬!,ÁLcÆñᜀ^ÐÔY?p!G•g#ÁZ³Ößn¸‘]Û ·Â`¾ àÀxž`Â&¹âBjjM¾p”ª$Þ\*%p W !§¹¨´£„Vê,’‰îcjX\ ŽçG6ïÄÙþãHÈ0ÕZ+m´6ÊÐ+Ós×[¿Væ:?Ùò·ûµÖåK@x'aï$(†qe¸'ߨ}`íy8s¦Ù /]tÜ^Ï»@’0‰ÄU¥zÍ@ºŸ¾ çóÁuå€Ë3„ZóÈyªÆ'û3ë ^Z­`ÒÁ ¸?öûuÐ õFËÜŸ¼ 0nƒnØ€AËú“­¾?™[¿Ùª[ÿí‘€0„asÞ´Á8 ìxƒ&XÆo,Òº^0iN‚É`¶&Íq2‹ò^–b«¹ÎsÙ¡®e££v»ý¬4w´ŸÆ’Q›ss@=tS¢ÐŸ×ëýúxr…ðA¦¥V—–†±Tl©ýž]ï´sËí[ËdÓÉ¥d'K.çÞ[.e–—†©\z5ùÞîÒÎ9h a³êßï7qp…ða.—{/r{w5–Ë-Ûd;³Þ^[ŦkËÉÕÌr,÷þúr,›Y¥;Ãõtúý—³Hý~8 ¶šÝIç²Ù·ÝÙ]?&³»ë»P2Ì%?È®g²d&•®ïdÛN,5L¦SíTîbœ¡1pÿ|ÎÇ“ñÜEHÀ1 |('ƒñ¸a µ!þx>˜‡¿ž=:·LwOH¹´ÛfxÎ!ÐŒ ¡šï×A4Aø'Ðÿ~cû~3^„âXEõ‰ßÛ¨¹Ýt³‚Û÷ÃzðuÍŸ_°\ ‚f-¨û[ƒþˆ¿å7ÃƤ6®ùM[«¶ÖÀ6ð-¨OêÍæü‚9à­²~½´ä8ô’Eú­ÜÂ&óøÅóÞ" @L©”F*-4¶œbíêÒh¥”Ô˜in”° ¡ C]ˆii­ Zˆ—Ò -ÝÔ©RÚ¼aà&DV<Œ)l)åØÅË .&"raƒ ÂTPBWLbŠeBiF]|„y+˜3h·‚0¶ŠBß0 С èºD6Ø2Ì(VœrjP¥E€ˆ¢Ì0L‚ RR2j¥§eB0h£Ü<)Œ¿ipB2Hžöš†œ—ÒÓƒ…r€`òWóØ/‹~Qµ?èOkŒ~’1¿¸Â©®ï®åbÉÕT®‹d&HJÊ'T1.‰\±ŒhÆ…"ØQG”ˆ5JC~i¬œ+b,EÑ1±RQa9piBBv‘"~*ríáz2•ÊÒÉäAgaà‚A ‚‰Ûº(…CI€Ý:déBA.øåcˆ#xÔšiê‚ä.b.& B Ò…8ÁÈŘ\té(‚~ª!¶§Îpjsé÷§»‹’ x¸ÿ(D”Ú-û‘.:žPÒi;Py ÄP¥´Ðn ´“ÌZ"€ )£FÌ©²P¦©¤Rƒ¥ô"ÈÐYIpš;øm»úâ(âå–à“ð°`—‰ýoËäÚ“¹¥¬Mgvw2é]ð1R»Év2»–}˜Î¤†±tò wàŒ;m¸‰–kGh0¼+Ñfñ¬ÛµõäÃvƦ†™X6•ìÄÖ³ÙL'ýp)¶üáZ2 ng.¹6<¸@ `¨oŽa@? ²»Åna·@æJ$ ×Þî·V׳mI@$ ÉäRéöÎÎzîb%ÀóÁ<ÅÁŠ D@^žðÀT_™3îEîÒׯäÞ³U~$Z'=m¯„\ÚMMm'¹=p]×¥l6õ¸Ó¾|KàºÎ´sôAê ;}[%Àv¦ö`¸€»¸Ö3B‡íÁRz˜±ééжíÁtšÞ±Óá4mN!3… ìíp§™Fé¬ÍØ¡=€¿4t h:„32ÎzN§QÔî^w #ȶS;Ë);]ZÎì¤b™µä­µäêêÚ4›®w2õT*µ”M®¾û~f5k¿—NwÒ99™öêòZúV;f×bp‘lûV;³¾”ûb5•Í¼;H¾¾ä‹°Ù³½Ñì{ùêq…ËÏòeØ^‚ÀàßYk/eìãµ?ì´×o¥—ÚË;ËëËË©Ìõ´³œ\^N?L®ÞZN-¥VÛ±è¡>´“;Ë«tÊ®wbÚìÁÃáÒÃ¥å[éÝÎò­ä­‡«¹[¹Õäð¶_OnÇKÝÛŸVªårÞV«…?­ä?½{X,olÄ÷6ËÅÊÇ›ùýK™=h?q ~é ¼ž”Nƒ1ñÔ§Þí¸ìSŽ=x} ød¿ºyçn¹PÍÇ{vs¯PÚŒÇ㕽oÇã‡ÅÃbuããÛùêÇ¥7{Vø¤Tµ³Âf¥ÐëU÷Ê0*¶Ð­V{£b¥ZíÏâ¥Y©RÝ,VßX^%³ê^µzô·힤½ãÂ(·÷û•€+Tƒ×B® š™½°Ÿß߯Þ.oŽªûû¥ÙæÂ$¦´ †2­¬@ÈJ£¥²ÊD¡I­¹TÆ2+­Y‘Zst`$kµ6Ò(l­+Ñôìlîm”‹ÅâæèQ¥8;£}q H`NHôª¦–ÂEȽ3éVʃwL)ÂQ÷.¡Ñ˜°È{ ƈ3äae œO1¶îÍBÌÏ@q¶±WÙïîÏ*£òloQàÞó0.3è¥{W IB˜ð ÿ  ¬ ÏàY)ˆBŠP+9"TSBñå®^†±<³Ù™ Ëß‘ˆonV»ÅÙþb8Öîs DZœh.HÙ§„Ô ±õÜöÅ£ÓP(KÀá%@M¸à [¢V€ ]x—p3ã‚?ÜPÁ„VŒ#KJ•’RQ÷BIÂy†4՜ɠ-u¡e <ÂÅÙ¨l–f…ýJ¡0Þ£Ø7°Y¹˜P&À|Ð,Æ k 9é¢çV"!¹'°¤Ú„Êb d8ulüð¦X‚Ô0À@á>ÊãâãJZ&%w/ŽI$„¡\ ¥7ØÅz€•à–íÖ0 PI3.JD¤†¼{±DŠ°3ÐëºvÓö{£Qw¶ßÛíÛøÖŽF³Q¯×ëÚî¨;G½ˆ8/Ú|."O˜fú2HÐ¥n%_Ø¿Ýý(^,æ+ÅB9XÜ,T ¥r7ß-äã}ôQ¥Tº[ÈÃÏaïü µ:¢>eŽ—)}´‰²/ hižæåñr¢—†µz}¾v|X(WóÝüæÝ|Ù–îtó³øþÝêe`ˆÍr¾ì<òx© ûÍ;£ÏÀ_¢ÅMö$wRN™å±G åYÂ86v…"m2Έ2J¡Ð}DA;S ,I‘ð$˜q ü@‘@PJál}.˜»—ËGQ‰¢á1­yqïLbŠ8b³+`î¹+)€e‚YàA†ÝZ ÈsOa0&à€–„3"h5|]~øó¯º?³àÌõ¬çåjŠzQ©«t?Gã?:Ê°GÙ (OUG¶ º®n`œýèXå¨iïhé«è 6¼ mƒ#žµÐ\ƒ7 ¬PD(6Òrþ7\F…P %xB‚KÀÁv/Þqh&¥–ÑN‰×Àþ׳(@Ì ÍnùN1_Ìç+ñÂ'½½îíц-—a0Ä«ñÊÞ~¼œ/Ý.n”÷»›¥ýB¼Ðý¤¿íÎÝVøÏYÞ?­ÄËwöó•|>oóùÃR¯Z‰ãÕ;…Ã;w @àßœ$æe™ÂÅZŸ‹í_ŸÝÍ#p‡7ó¥8t,wT(ê6lá°²YînTJùêh¿)mt £ÃJe¯p·Ø+•ò¥èÌ ¸ÀÇ .ù;Å8ÐH~#T¹aË[)ò•j©²¯T?>ÿgôïWÀð•vOXÈèa l9H‚{äZc 7LI}!¯Õ½äkr_½ø1ÁQ¯bGGvM€{xXœÍÍf{wm· NÃlVr5#éÑ Òá“aívotx8‹ÎÁIv£ÚΚÎ\ýîÉÇ«0`ß`š_'À/dþ-³ý_#¾€Ojž/½Î'5êGiøÇ…}TõÏWüiÝßú¬îß¾üö.û£ºß|wïÞÕ>~{ò—¥ß‚Ï*ß|Zû€n¸à€n¸à€nxKÓÿ)ÆÓ•ÐIEND®B`‚PK*NщFÚ1ÉóûÛüÊu½i<%8†F‹IDðÌŽï55²KK;ÜÖÎ7å¾Ëœe-­õÕô˜65À[Oâ8ÅÓy¦m+›{G6äºíÍC³êi­d{o™‘}åúh„hn}Êà™â¡ëµ«Ã §·h5ÿm~}íe ƒ— jž²ñ;ªÂ 7ཅƒ TIL²Ä!!»„0³$hœt“åiodLUŽ ÀÔN›êI |ö2x‘(ÈÓTýyÿ=Ûî·i°öÑ?;Lâ6~xßËF„ä¢ò·b!8[›KwdË(e ‰Þe%4£yglrBºaÅBÑï¶ú²#“Oúõ"ð Xçñß¼[mÿ¡/ች.«§Ò:ÉׇñUÃäôÊ•ˆ"\xÕy|kÛ«?PK[ÉuCÂ+PK*_{¥òel·Ûñv6r„‹Å"ÐÔZà¨æËKÉ5W”SܬÂqXÞ”*2T>äuEÊÊtEå`hˆ"VÍ%-€ÔE¿¶;§å_wëÁÞu·î9Ú9ØÏ4sÛUfñpW™ÅîÜ”¨M}¯‚[ ê·?7~%Ó¡{!o ªH²|°š†Û/„¨EÅ &ص¸ÓÉd˜g‡{{”}+™¢ÒaŽ²G„G5â"í øÂ8|z‡._ë2>Xkàíq’±ÁÐ#ï«J¿WË@Ò\HU’ Oº°Ë´N•òþ”T˺–qÜÉ âÌH¼þ£Ûo¼ÖipÜ{Ž SëCS4“›{N'òÔá ®Ñr]e‰(³ØØÁHïs*’×Ó–­\kqñˆ%«óÏY¡u(0Êm¦¨Uê\F?-|–A€ˆ|éÌne­¢˜©.óýök€4ODÈùÕ>ÎE`êÝØS?pâ'$¢~L#^ܼ7Ùº™gòÚ»‰Dè •Z˜¥x£Öl$ùkšbÛ©ˆ©ÌZ,9S¤¿„ÝÓØ Žïý3ƒcG?úH²bÚ!Äw$Å÷{Œfð‘’ÝÉ´g> \­þYqºûžÆgDFŠ”œM4X2$2¾´p€G—_?¸bËŠâ)²ýƒ~"”Çíéð iW(š>I&)X¬·} œ¯Hô¹_¶CÞ‘ñƒ¤´4‡ôT ‚¾ìY›êæ=žRpÇ“Ÿ©é¿?J)¶FB<^÷×Þd4M'£ÙÄŒÃ9qÂ÷CÜL'_Õcš’W•šµÒr-I¾a‘gy«g?‡“„JÅ ²Ãõ %Åg W.à2ÿÍl~yAæž2aœ×”wÓEAÜ&b¹…¥|‘›k{&||n„ʉ$z³ÖVš„‘OJ%Šœ`1Éb* +áù†X´ó2‹T©ãL¯ Ö`iŽ÷C‡r‡ú+I ”D ?‹”¥à]*0séµÇ¥¯V-²,¦xqÂòV¯‚‚è28!¼ 5Xp PE^ ûÕªÙQ¯mË‚úPÆbëëÍ+$•,iK¨¾ HkzÁ¾}:Ï•ã$[—d C4Ó\E”“ÿþñ`]n$ÛO# ®myÌÄÕ&–öec)Õn–ð÷_÷Ä{3§÷ìZsuî[S7lçšôÏ_¼Æ>­X°†qÃÀ\(÷6{§‡‹Ž /°Žà,öÜÚ²k¸I”zF–I +”õÄñ§Ò#Š÷?ë¢-på#Ó‘Ú5ãºÖ ?Ò5]›@¨ù!Ù×RV¶Ó* Å’öÁœÄØNò!fP”pzÂ8„•P «µ.§‰Ò ì$[oŠ1À†Äº¢dqŒà úu ªüû6m⮓hu¿šàðÝEâ¬h'×=rÙ8›Ü0û«’sªF†ˆãpÙöÌ£!ùØ<¸öþ÷ßÿÔç,âøœž“²Ìçd”ÚÖ— @З~lâ°¾ý,øq—®÷†¤¦vÞ™_|«£ì˜¦'à0}úÔðW4’ZÞB³³!?sÍψÐÕ›DèâlMÇó7‰ÐåÙš½I|ÞŸ7š§¯Î†ÐüæéÅz›y:œœ ¢‹¿V¢vÈUÅô^G °7T;kTØTU Q Ú ¢=jª”öX] Tú=°¡f~8™|«G5rL7 냘êvrgB¦„7«( e±ù/„´³•(ŠXWvI‹®6R”ëA·½Å!“Úå{<= ]Éï»Ëóu•÷\+¼ªz fÌjÖÆÕˆ—`6iº>.&v¹mek»^TAýìC9é>Qg÷¦î²t6F¨¸ß µ+ô5)-½Á±§åcu‰:H+j ¤# íбVë &«pÍÜ«rÍѯtÍÒ£vMïW¼‡EûMó 1ŽzI’rF“§w]>ÙÅP%‚&)8ͯš^bõ‰FjË/¦ãÜÙû²íW"ë×üƒŽÇSÛcãð¢§C6Ñÿ::`u£üHïìÝñÞ™ ÛÞÖÙÌdòSÑ4©æ9á¬rÙ+˜végév=Bs8}°gø<ª?䦩#e°èÿÒ¢ÿ9ü‰âãïLàO\ ƒTÍ Æ›Õ!fEÎÉÎw—á£LŒpö¦âßî¤gŒ,“ 政†”+ï^SÒã׺êBr ÉÉZaã¨k·ùOVÓ\“¿^%O¨Ú×~[L „ƒ)þª‰aˆ#Ïç'92ª¾†N½ZMÌQöÂÇæð(2‘¡æ˜žnŽé³ŸÅõmiÒ|ײA’DÑbñuÙ`H›]öÛã°§AP:ìû¸1úì¤ÃdCIüÚWŽAÐWhΧ}pNÃá£WIU¯R»Mbû+ÁãÓ¡{¦´òÕB7ïè01A=Ñ”’¢”/¯‡Å÷YJì®÷K:çõ¼•âØÖ¦tHèeÆTUŸµÑ´+{¯ÜUø¶¾‚ù–`q^Sê:(d[Vú÷íÄëà9âz=ÇÄÞ°9€ÂÎÈŽêý*à¦áøbáŽÛVôb|y¡1­d” €’5_¤ÁoHÂ\ŒMúô6Îeõ•&¨qqœ˜k‰oÇWæZ ãÉø`1ôš¬"¤¤PÚ-wÍK• áG ͳûŽ¥Ñwë¦7òá1¢Ÿ5?•‚á¨ò&WÍìëa@©”EŒ÷^Öï¹5ù¢ñÑƃm>h’ˆ6®óÆê–A÷—ÿnþPKÿŽ+ <8PK*¿×§ÚíÍÞ2Ï£™»ÌIC%ºÃ¢^HGbžQ¯µ”Gs†ó‡DÌgÎbRó*÷\•]É&è,®˜«Ò yHº KÞš,^tY1W¥]Ž×]…%/€Z÷XWáèÔÃ'tÇŠ‡€F÷×Ú2Iâ¹a¬×ëÁÚ0îæl63µ4Ø)ù┊Ëu 9˜0Ìi¼!IpWû$oÕ¤( „w†'xÏ«1'X`º20»)ªÊÔâkåwŽ®•ß³³Ä¼sœ)æz¨Øn÷P±Ýªlˆ“e‹§Æ ª?î¶qÅîcIÞT§qçifÜUyÆXiªÈ»2×GFö¹Â½>Ⱦæ4!¼Âîdwpà”ˆ³° 4à3 àÐÉJ†|¹ˆ$¢EÀ22rÉ,ÜVÕ_>Ü}r–$Ä[fzœY§‘Hp´EF„4èìàm ZÑΡ y÷–—ÁЊøØà$f<)äu/0ŠUb´L =…IjÁês×mdslÒ$}EÉú­Væl'0Uª?&¢˜ªµà €94$O™N T·…‹ûemõX¹™2ÉCL8•$(±yMCÕ[;Ae^+jEŠ’ È\å”Õ0¦‡,‹çézÍãáC7ur‘2×ÛÕ¸“°!ì¤)>ÿÕ4]–|(jùH•VÇÒnŠ¾&ËcÂ(/xÐßèvˆî'7o³úT^FÙgi÷µöEØa¦† x,°$7EC5iIÒ}Á\!›…Ì%<ª±Ä4q á{ô¸šqxì; …V¹}‘°Œxc&Þì0fO´l…9U±ĸrú½âÒuô÷ˆ g!îÍ4Péu²M2>µq€GS\Ÿ œXS!αíù ÿ==ìÏ O“6"!áY6qF]5zƒ`ûvÛöyŸÄÆ÷œÐ*¤s-0Ú²g~§ “-†£+=eZUköº±Y–Û™ß@é1ö+ÓÝõùž¨Ïq¼¤Nq9Æ\Þêz&$›/sW+ôæ"z •‡ð„¡ g÷D_SW¶äÃÁpl:‹´ Út~O8¨…ÖËaǻР5Ó=ÈF.`pëô§~r ‹±C˜Êð»ú•Üþü¢¬u09¬/§?3YHuPêÝO©H¨·Ùe\ÉÉ9[¶B{Er.é/Ýçl­/ õ—Pf= zlÒ¨¼:Œg¶œE~=ŸÜt0µ¦ùå»rOAOX,gmŽw®/X’Ȇ½/‘ëj—Â³á ’ŒÀJT ë2!¢PËbäZ‹XD´jÔžªŸŠÏ Þ#AÐê4µÁ'ë‘ ØÏ”«i›åã^.°š$L9k–Û¦Üm5—™¬T\UöË srˆì…aþ^œWÍ¿à¸{¡tÚ¸*—>bØÑyÞ:r':+“Q´:{¡ÞT?µòÚÖ‹”¢ ú3ÐÍaœbJ?ÜèЈèù¦j¾à¶Š¥ ÷ç8ª‹o‰r+! 9¹;úã¯}ϳá§ýòÆ®ú´ ´ÑFZ èô1x~ÞË=rÚ9:?œ ¨ÛÂ’—}Ù5´päªÕû­ëWýáóLóô´7„FÏ4OÏzDèyæiyßDã¯+QWÈy¯m´žÌÌ ænÊùÌ›·êˆ†<ˆ™ÖÈzsùÙÔŠã^›òŒ˜:Ë™Ÿ áê§^œçõpÈ»!Åà¤ìÐÅÇd_‘:ÙYúªCûXP¼‘žÁ KåÍÇÊŸçî˜ ¦WjÉËkÛaöôª¸(w•#{T|ÞÈϦ)Œåṡ$Zʧu\Ý',$ ß(ùØÉ Ô ÑYæd˜ÿ-¦O9g¼r6³‡*—®|2šDÜ´xØØÉ”Æ6§Ëïvk~wè–gÆünOÍz·\OöY¹ºzŽ}sØOD™P|îáþlŽäídñ@ôm)Œq§žN–Oo£¿¨WH•«/À{Áì"£Ëó ¨Ö‘ŸÜ?XˆÞý«”CÁL ›^HV{/„Ë‹ yÓTÜ—„“~ò¦ÿAûY°¢U{‡e›+2ª#¿Ô*ΆéfZÊÕ”!¾DT7˜9z¬™/¥÷_ý&MƒE£ï~‘EÏ¥üW#Û5º'úk#wð‹H?ø)º_Qß-‹ÙSnC¬:ÞA¶Ï SPùê)ìzœóAìnü,]nõa¨ŽÒXvoºlÔã,Úè†[q¤{èÛmøú®Ú.£ÜÿÜî –{£FmçÔhùŸÓnþPKê;79 zMPK*_ñ*ÚLX,Ì ¯W«ÇþâÿÕhQ¶/—[ D£æ1ûÁV„Óh_3vkÕ U»µŸ½j_nÖš‹Ô½ÖÊVnÐ:_gñ§B5DÆG–Ʀ‡”g ³WïñÒæý8çUÆm d(boõRÏc|I¹öZ•£ÚIýÜßzxu:zý47øï4ÔÓ4ôzý¸vœþèdšª¾vR«îm›JDâ å!¼@¸É³ôͲsÐßä|Å0ë†2•–è ^ËøEÍÍ¿JvÕÒðc!îµ´L |G ¾iâWðG”#zW 8‰‡Â°¯}…Þ'¸%rB¹*ÄüîQ™˜é~üº#ñaé,}g¨œÅôY¢n¨6›‘ù[Þ›‹ÌZëJ’ÉOËÞw¸f4¢œhè 6·fëáÝ6t©×ÒçjÕu5Û\elÑr?zdŽqoŸ(89¾@rîµü½N›nZúxCjò_QÓÁ„oKñ߆ºm×ÿ¸Œ ¼—‚‰m‰ùý!a KŒ7æ®þŠax3PN…Ä£ãxnÀðFƒÐ€ ÜUÖnEX‚)ºêJ$häKFƒ§!¼èëfá<4–àrJøÄ¢N(žfÀhªòAÌÒ÷À14VIÙ{Õ:nëÿ?íþ\rÚ}äžm^ "dª¯ÕqÜÕÀwhÏ…ÞÁüo—ð|È;U»C›¤ÏŠþã‹gI´%r\a­m3˜ –8¾ÿ ëø^Î:ºÊf†æš›¤'³*Ê‘ô’Ó?…ˆîyJˆÝ#x¤<\¯­³^/Ëõô ôþ™ÅƒDÚà÷žM;ŽÙ|¤@^MŠšTãB‡ÆF½ŒîÁ‡XÙêO¾š ~LʇŸ.>|LPا%±¯4Ö\D†‡JŒÝÔ`Ž &RòRD¦«hzbC”RhNe3×7Wœµ²ÀôŠ¨iáVµÀ#I˜F9ž}¸G©ëh aGU6f7Í:BŽiõ™ü-´f¨IJ ¿Ü à®—þ ÔÑÀ1U"’’­¦Ùuòð&”›†’3Â5wÎÏ{é¾0IH¬ðs5qÛ‘¢ƒâùÔ5ÒÉÛ }¤Ÿð@';Ë®|í6ÓA¹â S:Á žÊ¨Q×9vv£s“` /§iÑ;TïJ‘òÜØ[×p”Òì¶ý®¬&´eRåÚiHKÑon\¼x ¿f\å> g›®ˆÕÞQ‰„ßjL-s@ÇZË:X‚¹ŠÄ<""©E¯c»¨éÈD’x:H¢¨¬K×zØo aToš=O_Ê‚ìb6Ÿvcl푹H2’Œ³ fé‚%f™¹(SNûþ‰Èc:ÙLgßÛ4gø¶ø0þž_2¡Ê8Ãd9¾…ÞÝŸpÅ!Æ-ŒŒÑ¢˜íˆaï«Âmío}áÁÏúJLëoPKàA“=T#PK*DŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPK*o@pC~T)CCZCZC[[DLLfQQmZZyKdKQlQ[z[Se}ffLllQuuWffflllsss{{{™Ÿ ((¨<<¬ ÿÿÿ##ÿ**ÿ33ÿ::ÿ4e¤:i¦>l¨EE¯KK´WW»Jož@n©Gq¦Er«Ht­Nx¯U~²eeˆnn’ss™hh»{{¥CCÿJJÿSSÿ[[ÿeeÁxxÆeeÿllÿrrÿ{{ÿ-‚-3„35‰5=Ž=B’BO•ONšNQ–QVšVZ¡Zc…cm’mt›tfªfj¥ji«i{¦{u´u{³{W€³[‚µa‡·fŠºi»m½q“¿t•Ày™ÂŽp=“sHšyWÿ ÿÿÿ$$ÿ++ÿ44ÿ==ÿBBÿJJÿRRÿ\\ÿccÿkkÿssÿzzŸˆUˆˆf’’m››t£\¥‘^ b«›h²˜¯¡n¥¥{³¨uƒƒƒ‹‹‹“““›››¬‡‡µ­ƒ´ƒˆ¶ˆ‹º‹˜¶˜°žŒ¬¬¸¡Š¹§•¹¹Š£££«««³³³»»»††Ç€ŸÆ˜˜Ë™™Õ‚‚ÿ‹‹ÿ’’ÿ››ÿ‚ Æ‡¤Éˆ¤É’¬Í—°Ð›³Ñ¬¬×¡³Ê ·Ô£¹Õ¬¾Õ©½Ø££ÿªªÿ»»ã³³ÿººÿŽÄŽ“Ó–É–˜Ë˜¡Æ¡ªÌª¸Ô¸²ÄÜ·ÉÞºÊÞ»ËàÀ¬—Á½ŠÅ²ŸÉ·¥É½±ÿƒƒÿ‹‹ÿ””ÿ™™ÿ££ÿ««ÿ´´ÿ¼¼Ë˘ÏÀ°Ø˾ÃÃÃËËËÃÛÃÓÓÓÛÛÛÇÉçÃÃÿËËÿÄÒäÉÖçÌØèÒÒìÓÞëÓÓÿÜÜÿÑãÑÛéÛÚãîÝåðäÜÓÿÄÄÿËËÿÓÓÿÚÚèàÙãããîéäëëëãåóããÿâéñéîõëëþîò÷ÿååÿëëôðìóóóóóþõùõ÷ùûþóóþþþå zûÙIDATxÚí œUÇ]Çc ÁOò‰$ä%°¼äqÏÉ£ LjÕ˜åeÃR?õSP‰ÖÖOjmBÄOÕ8PE™G?5¶µºH ˜Vkm!„€Î8SÐZ¨­!8N©[p¶ÙùøŸswÙ]²›Ý»ì^ØåÎ2÷œ3çq™ï™™ÿož÷ªx…»«jjjjjjjjjjjjjjjjjjjjjj:»7¿ô¡_pàÿ>úí!à?sÿ_üÍÀ»ÏøÞo(ø̽>H_ý½ÜÿåËÀGïÿûÁûò¿¼ïÛ—;€¿¿÷›ƒùíüzËå à;÷}fP¿ýûþÙå à‘G:´ž=ÛZÞ;Üz>ß8ÏëéÎÖ³œk¿¹‹ûæýß¿œ|çÞÎytãë?u®5žm=xzuk빆C kž8ÐÚðÆÙV,âÜzî\kkŠè'×>™®Œp]ÚÄÖÇÖ=q(ÂnëÆ#¹}èß.g_¾¿óÑÚC§×¼þä‘#k^[sdÍéÇ×=zêñWÞ8}èãmx퉵kO=±fý««×¬YÿXŒGžˆñµ³kÖ=~nýškÖ>ÑO¯Žo¬Žk×=öÊÇ7l\óøkí}ñÅ! uõºµkÏ=vîÀ†ô·áàÙG[Oo\}øȈmÃã ?Ü°níS^[sîQHá†Wâ§ý•Ç=_]Î>µáÑ×V·Ø°úÕö§~é7† €Ó>õØëðÚ¯ƒ¿õ>ùÊÆÓ ®?ðT\ÿÔ¡ƒ^ûÔ“G6n<ò$O¾±ñàÁõ¯¯>ûñx`Ýá 1>µñµ7®?ôè¡5‡ ˆ´x`È8µ~}Ãk­l8½ñ솳§¾rvcÃÁÖS ^§7<õÆ¡§ohh}åÔ‘CÅÛ?ØÚ §7ÆsŽÄxp}ÃáøFë7œÚpðà«CÀ฀¡àLsO—4Ÿétp²iO76u¾üdóвpñìݱ¢z&6'WI¾eÉ¢…K â-ñLËÉØSøñ%-EÀÐ<³éÄ™—æ,|¨yfó²es7-Z<ûø“q'ÄfOó¢Å³ÎÌZ<»qç»êg6Ï^2«iæÉ]Kšf×7.œÝ8sÑ ›³pññ»—Ì<ãÌ3-3O.[4{÷œ{^˜3§~¨dãõ³ŽÏ9vbÖñ{âÂÝ3ÏìYOì‚퉋w§$¾Bïn©oܳhÏ¢¦%qæKwÇ¥;—¾_XzÏÎÙ'ÚµhwóÝ»ÇY'ãÉ™±eÖÉeKg7-njzháÒ!`é®söÜÓT_ß4祻÷ÌŽM _Z¼kΞ¥-ñ…%Ç—ìiZtlÖž9qÎKËÀ5.Þý®—f›yìžc:s÷»NÎ>¾pgã’¥gf«…'ê—왽çîc³šîÞsOãp¢~iS<¾tgË™¥»–ƒ,ݸôXl\º'¦½¦ØòBý‰c»b}ˮ㻎Ÿ©¯¿§¥~g}ËÒ––e»v-Û]ßrrics㉓/@vyè¡]-ÍËwîÞÕظ¬©qšÁÆ…÷4]Ù: g‹YB55555ÃÀ·*p?èzü6y~Ï›/2¾jþõý×][»îÆ—{`ËM âèCðѧÝòaôÊ—v‚<A.¦kŠËBLWa¯\ùÎ<¸þîÚŸø Ï÷ùï ÿÒµ¿Ö+€,ÅGcÂtNËBN‘Rs¨PTfŠÒö3#¡È䨴ÍåÅÈs•ëŒæ<óœ`5¨¾uãû÷ïß×w·ÿþ_¼ö_{àòô™;‹878d>BdÞ‹¨ãRW"yÎ' {k”¶<‹– ™„¹<ùÃ…ˆ *€½×ïÝ_¡Û÷s¿Ð €kEb®™ Fp‹mÉ"›i*¥%[Îln§*K¬ ,×)DR¸“ÂýŒCQ*!È4wƒ àW>°¯RûùçzËNÈÙ\E´µZB‡C-—Ã^Ô\{®Rq¨S°H[c ØÈRt­ÐVzmœŠ†«Á-~ñÇÿ¯^W¡¤}„DzÚfð—úàúAÓ¡ê:à~þÂè}aÕ@½ñîH"®Mw‰RÀó#—OÞ¿|ùþûWlY±jå”}+—W À»$„À¸»dð,¹õ‘CFwÈ dØoKÉðÃÖâb ÚÀ¥ËÒ-·]3x) €—ÓÇÞ‘ãV-ŸË{Øò¶P[H`üÁjÂí.gΓj¦€-«Vœ²bÅä‘{G¬˜´rÜó+F._YY ä4vŒfLp²È°ðY²o’IŸ% à²è-¥YÚRårPYÃ+ÏB€”ÂXæ)È ¾J¡‹,^nÏ7Œ[ñ©qã¶,7zÕ”}“nX5iÜ–ŠøL#Ðz„S.rlˆô-ufrHöX"k°…¢õ‡ ¡Š/ J€3*AC…ÌNdZIdÀ_2+”ß¾J A%2 îœD°:Jaƒ€’ÁH­ãžJ:)¼VxØŠ”@*½m˵1^i¥p "Ý ¦€¿ëF }ª“<ܲ·r+0Æ!VÙõ7tÄÄäžÿÂ'÷ï]µwòŠO~rÿ'·¤½åSV=¿jÿ§žßRP}­ÀÈ}+Æ·|ôòÉ“S öpÊèÑS&­9eÒ¤I#·œ/+z »$ZÝV£µ¶JHV=t0zÄwïþÉSFŽ\9rï¤q#'­„Èï…͸}ôp/RØùÜ{<1Á¦ŒÙ²é':}Ú§@+€THÿ¨Àc°ÕpÛTð€8ç«`’ø¡É£W½rܸÉà†I#÷‚‡è¯}ÃràÐa ùº^tÍ,X~]’ˆ—Bô%ž½C:w© äJ°=H‹BnA$©N­#ƒé@PΕ‰!¥‡T%¤ê~{µï¾½mEðÞÎÆ —€¬&šl %¯•D¬fà<éšÈeF9»&3\²™[GJHF!Ä’ÂDKf‘)áLGC K’‡ÁOU‡ß@®°È!’Æáÿ.à%ºÌ0}fË°ÎÀÖI¢EDÈûçÈE“ËâÖÜå>5dx€cJˆv—N¼Å½Ü‡`¥rÉ„{aá3DmÀ[a’€=iHoTÁ¾ÖÅV¶7"ð‰"*olÔ¦†+epºê: —ð#n5²—È þÑTÞ"ô#ðŸ×}°Òø?|Ý{ Emþ–·U™:Òu'Ñà<íÄí`ˆ_ûÁ.•¡Þ €ýpCï#:Ù€ eÒø" í di•ò¾±²5d}¯´!Urg„ RÉ@@¤ÓÞþ¯¸æúëŸÿAï\NmDŠŠTÑuP«C:™x¨ØƒcÞk)”$‘+ͨ£P[ÌtfsÅ…PXR¦ÁtP°‘ÕèûÏ/néÖýQ·¡_üV_ºÆÀ®At"Tˆ½ãžD /aÌSŸ‰cð^K¡ð® ~’Ì=œÊV˜e†0Éœ&ÕÐS¬>×¥6ˆ(Ã!gÈ碶 cA¶©±óÌGE 9ÒˆÕç<€FÔ ˜‚d@9†{•ºTœÖßê7(ëÎwúr»Ÿ+SXQA€ê@ª›ÚýRý Õàj¿pGê'„“Î…¾w 4 Î÷À1ƒ=g€ä®\¾ˆ¿¶W,€"m›ÞvøÆ•ÀÙ€€€€áà~úåÑ!ôÕ;ŠÊÇ·Ýû¿~ c¿óƒbÏ_|áCCo_Òz-¼*eÑÏÈÏ{WÊ¡‚6#œ÷¥øP[!-žabÙCÕ<ž!"ÏfÈó¾xháìu~¿ü`7Þ‚o{h‡ŸQŠm-?Ípm¾ýá*Ò6_~(»¨ð“©Ò‘¼+WLz÷Ö¦eʃŠmyÄ‚?¿ßþàÏ=бßéAE§H_â»wýßÍ—\ €Z!XPPm¦çªT3ÛÏ$Qu|™â|ÃÒPç8í¢u&(/S·r‚kÒ`80678U ²OÄž)H%ª:þï¿¿ÚûÓ?þ§î‚ÿûÿúXZ劶uÔðB 6¦ÜÔn”¶Úze ק†W¯­‹*¦€4h6ýÁU®\& >€ºmÌm¸1c~¿?fPñËÕ þþ˜ßýÊÑ ÜWþpÌo'ð/·}ìh…î÷Æ|µW!ò.&, “x‹q÷oÛêlY¨€¼íhÅîcë €DNcÀö¦?:¤‰Jù_Ç4@*º"0—È&¹äáì¾÷êŠ9øt9dš Í`øÄÇ*ð{?Ú åºÊy&KŒi¨0#B‚§ÄÅ€%èŸsA 0ÉC’€ ê ,Ç&M)ãéAðÛ¿Õ·öY.¸4Äç2Ã9”ùL!•Œæ*MŸ1”¤1ÑL])hJtÑW ÒÉc„qºÝ¡4}ÎSš™ËÀm½ÈÒ Ã| $uQ8"f_â¤ö•‹¸Ì\ÉHê+–˜SÂYšKȉL·ÁÍ¥ê¸fêø£ÛŽÂ¿éÏn+¶·TÀZÈìP €ÜmUêÖÉôC~w6M‰ñÚCV©÷µœï]*šJˆr¦ÚÝË ¬«>€­#ž¹é雦N›ºéæ[ž¹eÚøzö‚+þàÖadß à›¶>3þÙñ¦˜ºé Ry Ò ¾ÏŒ? )`Ú¦›Æ?só´i7 D *Cë¹î4V Ãa*奰­ð_¼ÿÌWŠ2`ÛÑ‹`™NHˆqàÌi‘Š9ñGVî9T<“Y¡@¬a<\ZçݦžNT ³„I"@.`¢·:”¨HcÅ •XqR4 ƒÄ\ùKàé.±Þ´õèÖM À¡ˆÁúk°|%Jeæe¹?š¶ÉܧVn Yp"¿„nyú™›§oÛôô´éGÇoš6}Ó3SÇO}vꦭS§õ@Èhæs%)A)²åj~>§˜b1÷ˆ´é‹<&èf(oÚ4õæ©·ŒxöG¯Ù6uÚøMSošöŽé·\§ Òþ|ÿXh¯ ÁƇÔæ;ŠÀKYÞ¼uÛMÛ®¹yúôMÛ®S§Ýž¾uÓ5Ïö?îb5Ü@Ø6mÚÖ£ÓŸ> Y`ÚÑéP@6Ø:uÓ¶‹ÉCNôîúÀv4ÙóY<ØP–f@š.kZ+AµçÔ¨’”TÆ™G¹–‚hf•*ËjØ š¤QàHƒ@ó• Ô-ÖBX!:7™Ä®äªàýð£È|æ )Å8e©Žœ…R sÌ èBFYy‚yµ N“Ø…ÎCÅì?è#sŠiVŒ ÏT€`Ì ˜ÜÑ-X5ÿÕŸFÑ©8´Ùÿò˜™Ð>væüaù¨<<¦ÚÍâ2æw*j?ú‡c>1œšÅcüêmcÆté¹õÖ[!üm)üÖôW8ØI#·þɆ€Øú?_ûçnÜ×¾V­ø×áþ§5Æa GÓÝÿ)3ÀÕÚ\ÉBš0à®`å)3áŠà®ø)3ÖmÍ=e¦6Nð¢0Ðø&-ŠÀÚJRÒ¡íþL»vëHÌPOPß³9e‘F¿YîY)N´£ÄÁ‘H:Àq`iÁ”(á:‡1Š’Ì9ñŽR?tP%ö™E.+z<¸D©LÔ3¥¨ÀÊ(,gœ¨kr‡”¤©ÁØgŽ¨\26tÀ;¥Êç)!¤åð¨ &­—IŒ×¹Ìu)ZW ’cÊ9¶§Á +–Lýê1­ŸÃôà"—Ö PAœrÏÒH‰¨r$s‚EȤùópVq„x0‹Uê&|"PZ„/Œ†p¨™Á€T£ª<Á #oYŽ£¼Ûb-ëPœ‚&§™ÆŠ†ÌbÄ L-3Þœ+ŸºHsª ÂPÔáÀ B2-§„4ÕåÊE[颌—°mHp‡%çH”ÃY‚æ6›×$ŠF#r‘90iæHæ<œ–)JTÈ™Ê) s4©ºyÊÏÀäƒ T‘Ê’çB0["‚9.HRÄÆc•F“ÞÚÌ8•i)L‘b"(^ÓÊRŽb4áÑI=J,hâ$ \ Ÿ$3H£4¤³®€–¿íÎ}v{·ÁÍ}B„åU^Vµ¿¾~×í¹»þ¶O) Øj/+ÛO;n¿ëÓÛ+pŸ~÷íï본`UðN=ãîrðÝ;ß¹½B÷žÛÿ£¯ò"žÜ··øN•åD^z;îüt¥¶¿ó'úÀä$sA¹/@¹hJ$™{~EªÄ(—2 ‚€ DÆ6ÍKötƒ¯€Ÿ­8@¸«OÔr-gìyZF R„Áœç &,Q2 ¥4D΃ÌyÆÓì1D-q¤ªà=ýðÞ;û )â‚«çN˜8qÔ‚«æÕm5î¨ùE–ø\ñ±cøe·–7oŸ;wû‚‰s7ÏÛçMìRÃ,°£ã¨Z…`{°`^‘à´çû‰ÛŸæVà]‹€«ç×MØu‘¸ BÂkÆJÑÜdZÛÌ•4·Ñ`‡SU°ãÎ>dþ]÷¢OMb¥9@žqX1!6åyŸ&•Tó(œci‘cÜ1!” åƒ4ŽÄžoId_¿ãÝݾå·ðÞÛ?;ŒÌ`üôí?Û‘øÛk;º· ÅÑûîxONZvÜqûÜ]wöäÒ™»î¸ãö÷µô€ëm¤q(÷ƒëóÝáÊÆK1c$Æï~}Gïî³í;ÿñÝ7c$ÓïG™4)\GmƒÓ>Z}~aœ 5N«åØ’L¨t´N‚I-î³Ö…Š†êxçhîÍžN\ð ¹p‘p,Ë¿²"á”QJ‹_›ŒH ®áCæšÈÌsÈ}.ÀBÉÁ&¦™T¢¢Ÿ_ª€Ø7ÁËÌŸÅO¯‘4lÐgQCŠQõ'9εŸ „Ü;‚˜–áÜ U!—ù ²A&’ ŒðîÌ0•¥i´&@#0®˜Lkê0Â3ECÉ2BÐä2äPeZèòJ>M6(ò½1>­- y߃·ªücRƪ"Ìø´¦Ž×Ö¤ÂÂùbqÑôƒí&MZqCÀÛ8ïç?0d\f: ª Øž±“ÖíFïöw™^<8øºû ¦ßX—V4>h9^AñeÔ|’è÷¦0öIxåÒ6ƒÓaÀüÃীÏüY7Ù½”Œ!7iM=®Ë|aÿ±ÌÁ* ´h (¬XÂEžrB$xÀüÛ½ÿ;Ø>úb7 €‹äËé*†lžì¿Nö¿}Aݤ“ÒÏí‚2ST,©7ÀÞ|à¯Àý_zk˜þ"’G›qM’ý‡:0CŸÍe2ý]Z\R¤µIHëìRÆT ‚·?ß÷7þ/~¸‘œÖÑLm›65†¨´vfZh/•…®h1^—WJŠ!­»kSþ‡Óz S@ŒùРÆÿë?õínB±Õs½øþýÄ4ðŸú«x©]/â÷~ó¾/šøñ¿Œ—=øoÞûÈçÿý;î¾ü™?ôÍ8ÄoüùGüñv÷=øÈ‹oÆ¡ ¹ÿýþ»–x™¸«âîjjjjjjjjjjjjjjjjjjjjjj® ÷ÿ œ$`ü2˜ÎIEND®B`‚PKda]Mmeta.xml“M›0†ïýˆîìÁ„€EX©‡ª‡­Z©Y©·ˆØÞÔ-ØÈ6KúïËGH¡U9zü̼¯Ç3ùã¹®¼Wa¬ÔjçCˆ}O(¦¹T§ÿ¼ÿ¤þcñ.×//’ Ê5kk¡\P Wz}ª²tºÚù­QT—VZªÊZXêÕPs ]Òtš"çJª_;ÿ‡s E¨ëº°#¡6'Y–¡ñvF9»rMkª‘â ‰J Ahf‡÷šØ¥¥ÆÛß–nlÊ}5–9ËZZë«é!mjÀh=Â8FÓy¦m-«{6`ºnzÍcµêi©d}o™}ãúd8¯n}Jï™ ¾ë¥+ƒW)º÷þj þÿÚìúÚË,/ò‹yʆï(òñS˜£· E„! @öS’PBÂ$I€dd»ÍÑŒ©ÊI(aJ§Mñ$F|eÐ&„0ááIªö|øž&‡$öÀ¡1ú§`®ñÇVV<€‹Êߊ9gte.ÊöPŠÓ ÅÛÍ6&9š±É‘àÒõ+ðÖŒn‹¯{ø}Þào7À:ýf•°À?ø%>Á×mµÃXZ'™7ÆõqxV?:­r}뉊­zní{ñPK•Ù Å-PKda]MConfigurations2/floater/PKda]MConfigurations2/menubar/PKda]MConfigurations2/toolbar/PKda]MConfigurations2/progressbar/PKda]MConfigurations2/statusbar/PKda]M'Configurations2/accelerator/current.xmlPKPKda]MConfigurations2/toolpanel/PKda]MConfigurations2/popupmenu/PKda]MConfigurations2/images/Bitmaps/PKda]M styles.xmlÝ[msÛ¸þÞ_¡áÍõÌÉŽ¥Æ¹i{“™Îœ¯æ®_o ”€„,+â>öÿõ—tà (‘2eËvÎÉLbÀî³/À.é÷?ÜgbrÇTÉe~ã…7ay,ž¯n¼_ùH®½>üé½LS³E"ãMÆrMJ½¬œÀä¼\Xâ·QùBÒ’—‹œf¬\èx! –ד.÷ÂleGÌbc§fw¶f÷zìdäíÌ¥Ëñ;fwv¢èvìdäLÝé©;ù¾$•$–YA5ß“â^ðüË·ÖºXøþv»½ØN/¤Zùá|>÷ µ8nøŠ†+‰}&nVúáEè×¼Ót¬|È는o²%S£¡¡šXµP¬PýrÜBÝ­F{×ÝjæxMÕh?3Ì]W™&ã]eš¸s3ª×ö½öohþ¹ý©õ+•Ý y;PÅŠ£Õ´Üî|)e#*N°ÁnÄ‚`æÛg‡{{”}«¸fÊa²ÇTÄ â2ë øB8»C—oôθ­5ð8 Íùhè‘÷ÀU‚?¨á¥¯X!•nIÇ']Ø%jRÆZgb8e µf]©$éeq¦>¤^rÇÙö;¯sw„ùž#˜ÔúÐÃäæÞ£ÂÀGž&|Á5ÚƒB­š£,•›<±v°²û‚)Ž$*Ì´Eg×ZB>bÉêüsVè œ‰:S4*õ.#%ÉJÂsY,œÙ¬U–SÝg¾_þí#à‰9¿ÚǹDÞ‡úÔO%œø)IX,Êïm¶n†'ö…¼ñ~âúò“O4‡ƒ”Z³Bàìn¼?ÓB–Ù㳃ޤ³4ò“ËAk|%3šw8 ®cHwTqãþxÑ`Ét”lÈøÒÂá+·¼,Ÿ"Ûì3ýÏæ¸=ž1"íJͲ'ɤ$OÌv“Tˆ%¿ ËvÈû"2~TŒ €æž*?¦Õ¸½«xé.ê Só+ä_•’[+!æñ¿Éû/˜“(˜L; é6„1AB\GÁWÜx`ÁO» .„ ÀoQð[µº!'¼,Ý‘=¶Éa8 'ó j…¸¼š¡øÆ×ÈÐJRj%¿€ò´\»â|ä9CI~„q–ôŠ‚,“½*I‹u=Cê2„Äè>Áò^ÎÂù»ïÛ•5ͱ2ƒ‡$,¥QUNµE+gX)Z¬yìÕ¼Õ3) ³3¥9TZˆ@¥`,…„ËõwÓÙÕ%UÛ¦\ˆ†ò.š§1¤·T.¶°‘…½Fç’às+TA5›u¶2$<ÝhYµá “–•ŠbMk§,6y¬7&™µÁiyVà9méP~0²TŒB‰R¢µuMÁ»TD$“ ,/ÑËŽ§óKä–˜Í+$µÚ°ŽPC mè%ÿ ôhVh3&h¾ÚÐ ±Ü Äp5Ð Lþ맃u Üâh¾Ÿm[\»æ±;TW›Ô´¯ëšRíVþþóážxìþ]®Þ}êšïïÜþñ³×Ú§ µaÜÀ°0c %T%Þéáâ$XG žxnmy‚aÄ™÷`dÙ²BI×L¼ˆpê!=fx«]´Ã®|d:Rû&c\7 Gº¡ kS5²–Š•xM„`å+0ýçM©yº3>XÐÛ;bE £KÆ!,¥ÖX=õÑKµQ`Ÿ øjíP¬Ö41O gÀáT2Mî»ht‰»^b­ûu€Á LWG@Þ%G|ÈePü ËL–!˜žX"Ž›o-‰`1ãýï¿¿7ç,âøœ™“ñœºJcë+PÀJ?uâ¨}ûXNÆ¥Þ˜ÔÔÍ;³KsàøÇ4=‡è 8˜Sƒ,Y*«Ax{MφPhýüÍ!4;#B×o¡Ë³!]ÌÞ$BWgChú&ñyw>|Þhž¾>B³7š§çgDèmæé08D—¬Dí«ŠÍ*¼Ž6`o¨vV¨°­"ª¢¬+ˆî¨­RºcMR ›öÀšÙùa|oF rÜ4,냘š®rçReT´«he±}¯–ËœBÚÙJn4ŠØTvɈ®×JnVkRuÜ-™ô®ØãèL˜J~ß]ž¯ù¾çZáuÕ±cµf]\ëñ@ã ˜MÙ®‹I½Ü¶²u½^LCýL œÎMŸ¨·{ÓtYzûÇTÜï#×+ õrkz‹ã@ËÇêMVÔ.HGº  céQLµÂ ó Ê Ç°Ò Ë€Ú }Xñã_,+ÖÔ:êA$)&8K{œRÞõùdC•Ú¤à4¿hzÉågë-×p¼Ø¶xoï«n¿RÕ|ŽC:Om]„—²Àüéé€5ïŽôÎÞïÙ°lMm&?M›jžÎ*—Ýñ’—~–n×#4‡Ó{†Ï£úC¾aû—&RF‹þO#:¾³Aññg.ñ_\ ‚Tí Æ[­C÷­Ž]f‚kLr9ÁÙ˜Šÿw'=cdÙñh0§ü1¤\Êd÷š’¿ÖU’+8HNÖ ç)gƒï ^N»¡+µ}Q×m󟬦½&»JžP7t¯ýu15®Å«&†1Ž<›äȨRø: ¾´ ìQöÂÇæø(²‘±æˆN7Gôìgqs[ Ú·q¤iÏçß– ƹÝe¿=p4B}×#°FŸžt˜¬M^ûÊ1 ú ÍY4gŽÇè´À¨1z•Tõz uÛ$u_c)Er:tÏ”V¾Yèf=&®!Èã§"š1ZnÔKÄëañ}–»ïû’Þy_¥8¶­S:$ôMÎuUŸµQÔ—ü½/«GüÚ ?‰IM¨q^1u²+ýë6ðzxŽ¸ÞÀ1±7l °÷ªGÍ~pad[õpÛ‰¾˜GÓJFÅ©xû‹-ø5¿¢ÜÅÈÑdØAo“â QV¿bDpwðÀ ±¹Røµze®¥T žŒƒþ É*BFKmÜr×~{ Já¾RhŸÝO5­ÖÍôVÞ1<¶R³Æà§JŠ#UÀäj¸ü}=,(•²ˆñÞwÓæ;·6_´>Úz°óÁ)qŒë|Ø»‡¥ßÿËxþPKÄo”; Ì7PKda]M content.xmlíÛŽã¶õ½_!8m‘¤Õ]ò­;3h²P`7 :Û"íKÀ‘(›YIT)i<ÎSþ EöçöKzHJ2eY¶µ²Ó~Ãœm¦õ‡H’š öØ–Éqšp¦ºI\lÑäÖˆ–i(õ ˆ2Ì¡XÍ[3¨ÚŠé#¦¬ò±2C+Iב«ÙÒÎi(Õ“ –fs…ºóXò0l:î¤4Œ¶gÜ XAž»Å.{x÷'“Ãtžò!©U+)¥Ž3º®ëÇr³ˆ ¾Ñ#`=ÄAœ_¿’ù©Öä3çûjô†@²âÕnQ Q’H ®¹¾ýe4ÿÝži­©9¾¾À)B£ J[)H÷ˆa}æpÖ`Êhoñc3ò°O(¸|Eòü)¼½Æ?¢¿”ûõ©à ai8yOŒ’P,§}ƒâøïûyëâ~¿a÷M=•³ÏM«qT”ç²@ó4þ+~¶ø 3»Y¬â³ªÔõ -”ívI¬Cº`([’ ÎãÍ’xÐ%Ïò!b᨞·"Ñ3q˜çBryÁè{¬¯HÈk?Ë°|;±pÞ dï1ƒi¡ÆÜ`¸¾¿Yw7<"1¯b 5úg‘øª 4C)`+Ö¯ª?aH_RF~¢<8ë(& ˆ¡?–yA¢õ6â=ßG°AK¤l\aqÕè FWú“ÅBw„â '$mF'ÆÌ÷8ÃÕxµÏ°íY5œ¡÷©zA3¾AÛß¿£EÁ‹À G8“m“Ë× nlŠì³ç<Ö ¤&Íáj”Ò+:l ”ÿ–„!‰:ÇíÐýèö®€jsµ¬ñزÎã ³YÌfÿ[÷Œ©7î¾mÌ÷y¾÷Ñ §9ð¯>Ÿètç÷ÿ9xËxìºçò– xÖÞâ=OooôîG+R,iYpœÎ>ñõû¸Ý®“PµýàX?üž®ö‘çOZFttæzfZœœY‹­à¢|9R‡t>5I1þkxÂáèèÐÀ¿N>u­N?±Î3¦îq;œ}b;t ך·EÛz‰—W%MrµVÄ븜ü¼O³¢f^ –iˆY õŠ »MC'·WÓé('(mQo`ü„+Æ:\ˆzû‘¯ºój¼SV~LoÈc§HzV¡ªj‡o§S=:"ˆ¤|VÙ¿ë$1>ÛhP ”¯I§ê;„i;²mq_lƤó(26õÔ>¯ WUÙpGã°×íåZ*P”h´½¨2k°{î&lôÌÞÀûçoPÄ ÃµÕÉÙCµÅuÅkŒ‹¶>¢¶:•Î!mŸîŽ’ïv,Žä£éüöÚÕèÃ?ÿÑhX™Dѳ áÍ_ŒîÒ4®cÙ˜íwœÚ’n¡)½]'°ÅZeÎ;ì4„T`V¶T°o[‰ç‹¾ËÜ·Ó#äàô|‰2ÜHÜLlÎ|Z]ßÕ­ù'tà- y‚Õ\±›9“úT«>íÔƒ¼©1<ß­Ÿ×üÙžp/¨Ì+3+Fqºäkõ¦ü¶ÀZðûù_Q˜ÆÒ,ͱÇVõ³Þ>aŒ2å@«9„RÁõ1V˜¡b ;f¸–% xÑ‚M›É»ë6‚|AÇ0ʯF7‘«ÝD|ûð]–ë óûÏÐ{ê JÍùYBC̀؞¹ŽáøSßñ&¾;­‘dâs=oaâ ˆá{ø¾‰¦Úñ{¦ýUŒKἿíZRßW¿9Ò›æï© °¹@ÿ¦}[Û þ{){lÅü¢æ¶eI#î#s½MðKKû2'éç7üñyFL{j}ñÅþIœíI€[íK×»ædìŽ÷»ÛÄü€ð77‘£í§ó¶é š ó·ÉÄ©£~˜p¼M(Ï8PNvlºkƒ©f=l$´;¶#%£‚u‡q•äßzFs"?ȈÀá¨ÍWü†7눭ÂøÖtH$)ùaq=H¸ôdâwÇŸ FЊñˆÁ&wES§7šº£©oXÎx;šBï;iÓÉV,¡t³è}¨#k•^U.œ:ìBÒM»ðwöèú«’Äá-D=”h_CJÖ^£UyŽSÕ9/{Â2¿¥% 0¬FdQVï÷ú—2·vhnô00ѹ½ªñªÆ1üY'Ñm©Æ1ÆS5ÏÁ³¸zÉs›<çnvtÉm—ÜvÉmŸRnóz¨/>b9'À; k o?+¶ž*ñ¹MFzƒÉ|t®Tä÷Jr¬JÒkIrX–š¤£HÒ7lwrÉR—,uÉR—,õ”¥ìVl=üªkhlõÚ±U\W>i–úŽQþáíF qþè$EÓ&¢l—ǽRœì•_ƒ’ÝWŸÀ{°ù¹ÉLÉÝöF,ÑÍåøµ˜NœæB»¼ /”¯êJ/â’›,¤ƒ¹ñ«ñ º£ ªÈŽ\ÔÿÖ5Ÿ3eé3×Öݵ-øi;ž;êæ=W€§fL³-ÐJÀüªÉ.ÙNŽ•m-Ihþ7öÆ%éc[¤ é¶$Yatxå¼ò¤i/ŸÓƒîâ{Û[|Ú*Þ26<µ¤k8=•¯€†¾¦‰ˆf8ÔŠ%Ö"ÂòB+l·X¢BCZUòÑþ^{œj~þ÷{’†~þFrMþ¯‚ ˜Â8ñ;osºð´i®ß-9«¹ ™5H|·r?r‡~þ€c”çÿ¯7d-†M1á2ÇÏwkßÒϵ?È(º’{Ê«…ò¨#× êÓ2Fï!† 4- aߦÆ^‡˜=É!¦‡€œâÏú=Â1,ûœ.!ƒ¬¥; â¦^IYX‘b-w8¦«zmW}äé“mõÊ~v²Ü­öEÃw/§T—¾èÒ]ú¢ÿ§¾¨6ÒöÓiû"ˆ­öøloïdIðÉuEµPú»"(¬I_W䶆vEž2¸¯+šp¦,}jó®Èñ'¼+²­Ùî® {»"³ÊW å– ÷_ÌÖí³çß_ÿPKÌô…§ ‡XPKda]M settings.xmlÝZ]sÚ8}ß_‘ñtgÚâ¦Ma ¡aJ6@w»o¾€6²ä‘äþý^ È°‰ãÍC2±¥sŽ¤«û_ŸvôRQÁNõøÄ9î ŸòYú•/Î×æ/b:¥Ô}áEp]Q 5QG8«úêuÉ$¯ ¢¨ªs€ªk¯.Bà›iõíÑuK¶zòÄ(h8s­Ãºë.‹ãÅÇc!gnµV«¹öífh(A!ÑVp:Âí9Û´žàS:K‹²½=_ñ,ÚLX-Ì ?=99sWÿoF«€²´\flÅAˆš'ìÅ^Nƒ´0fìÞªWªk¯=k_Ö–‰œ:Í=lÌ y±ÞœÕŸ Õ9Z?6RÖ),ž­Ç‰›÷rΪ̴$‘ÍK½ ñ%åÚiVªŸjî>Λ°û0Õñàgçs£ÿN}=ƒ¯~ú\;É  t6•_­žžWÓâWV(÷á ü].XÄ–ƒö&—iâçïÈTZ¢%8Mc©•¾Äý&©ßSëýØŸÁ€p§©eyà»RðÝ-~Ÿ¦2¢÷Ô“p$ KñÚ7è2ƒ"g”«òHÌï>åHQÀ6ÝMþOw%>,e 𕳘‹Ô5á>Õb ²|Ízs‘Ùݺ”d6ěƒ'»\1PN4 [ÚmëãÝ2tñnï$£×ë©}®2Žh}}²Ä¸·‹O|>k£¹tšnêŸ/lZè!5ù¯¨é ïü×¥¸¯CÝŠ–F÷?)ãoEG0±/1¿±ßG d‰ñÆøêï†wå\H¼:ïÍz4ð x¡À=eMáFø%lEO]Š7¹Ã¨÷0‚'}åÓÄ œ‡Ætæ„Ïà^¬ê„âi†Œú  ïÅ"þ Î2º_Ü•˜°®6vù?M»3&›©ÓîÓìÙfG© YýéyÆSýü€ö\è]Ìÿ χ|Puvh“ôYÑÜbñ,‰²DŽK¬µm“ÀRÇÏa?ËYGOÙ¬ÑÐ\q“ô$VE9’ƒ^Brú§Á ±)‚GÌÃíÚ:éõº\ @§o̬DÒ¿·thZaÈ–cò’hR|ÔlS Sß÷ºïÞce«?¸jN$¸!a(~ýØ~÷>BaÖÄ®ÒXsé+1ɦsI0‘’˜®¢é‰PJ¡9•Í\ _±—MÌ%QóµøµÀ”DL£œ¡ŽÏJ²G¯«`~WUN&7ÓºBN¨ï¶¥ü­µ>f®QLj¿> àY 12ü ¨ãaÆŠHJöšikÔecf”›FSf„+׿H‰O¬ð{5Ëv"×D{Å_òyÖ à;HÞRh#ƒˆ{::XŽåkÙÎJ[ˆ\Lémâ=”Q»nsìRç&Á_N‡Ò¢w©>”:å'¸¶^×p”Ò·ý¶¬æ´eQåîÓˆ–¢ßx\t¼…»™¬rñïø‚³ÝWÄjo€¨H˜ï5¬Ö¹aÆÌX„¹ŠÄ<" ±ÅpÆ6ÒÓ‘™$á|AYN×ZØoaTïn{ž~•…ÙÃl>ÇØÚ'K%$M&ÌÞ‹Ì2sQÆÜöô‰Èc:ÚMgßÚL’Gø±úþŽw˜PeÜᡇ²œÂßBî[d‡>Æ-ŒŒÁ‚ˆao«Îmuíî}ÂMúªLóoPK:%5ßl#PKda]MMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKda]MŸ.Ä++mimetypePKda]Mp7Q,QThumbnails/thumbnail.pngPKda]M•Ù Å-¢meta.xmlPKda]MConfigurations2/floater/PKda]MÓConfigurations2/menubar/PKda]M Configurations2/toolbar/PKda]M? Configurations2/progressbar/PKda]My Configurations2/statusbar/PKda]M'± Configurations2/accelerator/current.xmlPKda]M!Configurations2/toolpanel/PKda]M@!Configurations2/popupmenu/PKda]Mx!Configurations2/images/Bitmaps/PKda]MÄo”; Ì7 µ!styles.xmlPKda]MÌô…§ ‡X (+content.xmlPKda]M:%5ßl# 7settings.xmlPKda]M¿(ÖÞI<META-INF/manifest.xmlPK6¡=apache-buildstream-27ae392/doc/source/image-sources/arch-datamodel-source.odg000066400000000000000000000254401514607367700272750ustar00rootroot00000000000000PKz>ØJJÚSSÜ\\Ýbbßffàkkàssâ||ä`†·f‹ºj»n‘¾v–Àw˜Âx™Âÿjjÿqqÿ{{ƒƒƒŒŒŒ“““›››£££«««³³³»»»ƒƒåŠŠæŽŽè””蛛ꄡDŽ¢ÈŠ¦ÊŒ¨Ë‘¬Í¦»Ö¢¢ë««í²²î¶¶ð½½ñ¶ÈÞ¿Îâÿƒƒÿ‹‹ÿ““ÿššÿ££ÿ¬¬ÿ³³ÿ»»ÄÄÄËËËÓÓÓÛÛÛÄÄòËËôÃÒãÈÖæÎÚéÑÜêÓÓõÚÚ÷ÝÝøÙâîÿÂÂÿËËÿÓÓÿÛÛãããëëëããøâéòììúìðöÿââÿëëóóóôôüõøúÿóóþþþÿÿÿñšÁü>IDATxÚ횉›Ú¸ÀÓÍp“f7Íb[²Áê1µ1Ûáð!i#\ÀÇzÁïÿÿ[úÌLš™Ýo;K&tÆ°lIôóÓ;ôé¦ïß/ß͜ɖ«ÝÒ›]¹ï–0ƒéüÝt{59€¿ÿã{„ W2YÉûJ¦dù Õ?äËŸ.ï×€ÕäÊY/¯¦ÞÔMÁ™yÓéÚu`7n&ž³™¤0™®§“Ù ©À_ÿuÈøU•Õ—©Ò¢ä2©úoîiß­õ×ÕÝò°ñÃësô‡x˜¼yî^·n„“ð?ŽIõÅ·†7~妨÷åx•ùä”TÙÿùÈ}ÈUâkj49ç*áTªU²ªõó>QÃ~Ø÷ÑÜ*UNA!:Þ@£r…_ÔÚ窦(ÊQ`hµ³>‡X[ØV*U ¥ ô% È$”)¨JîS¢f2':YÍ÷UBÕ T’Ë(!* 8º!y¨5¾HåÉÀYm¼õ,uæ)"Àosµ=íÈsÔñòj–çxÄ"Û7dûVdMßLéBž]÷±_Ó’eXÀÌ×K]Ïi¼ãîpÎÊ TÕ‰`š‚—nSwµÁïüqçwb 87/°»5é·wǾۿ·»' `;Y/wÎf™®—Îzƒ§Ëݦ 4—«tƒÓÃËåÓ „oënõ{oÎÚqRg=ÙëÁÜuÝ™»^¥KLGæéÓÕ€ålçl×8xÏK½µ³Mwshž·çz3ÇMa;_º»Õ©§@a•w®ížHÄâËŠ‚¸)­¨9ž.JOcÿûèYÂd6Œ†¶.ؘÍúb`\˜¦­ÛɈYÆè›A©ÇL¯KÉ° ë¡'O)FO’J©[ÅïtBºÔÓëaÏ”"©Ww^I’=–zL,b®_°§À”cñ`dÕ’0l(u3‘#ËÛ¸°%ÓÝ0X)EV\ÔÝa©Ãhh=¥t8Gãæ½ÿÜȾnÌôñøvÝøº'D›šÓòÆÀÖ(PB|sR7––ÅõµÅ-‹¦å 0mqa0ÆQš¦.Y=ûUÑ1;=]ù²ˆDGHL!ˆž‹Nù!£+lêHÝ¿Š¥‚dI½^·ˆ’!ârÐ,è•ÇЪbVWaVפ„Íúˆ† O^r“6Õøü:]ä„báãE`§ÊÇ> ›Är 2ÿ·çJ¿nJ¨÷Ÿf Xקõ~Ô5|yYú†ú(S@£¥t¿( }O%T)Q2•hA d2—)%¡B°ä2W¹ â;‘Ùé¼@FTN¸ªøÅ"Ò Õ8%™â%ð5Å—9Ör *Õ4™SU-'TµRPa8×TË!êR¦†0î¦ÛƒH Œ­ŒŽ)Ù½¨gé«[ŸÄ ÜVÚêöu¨æ·»TŸ›ª_]¯rú`P…e™Ì(^ ´óÀ¤.I–):¦UÂI„ÚÏZòý¸«›g<öz@T—õ¸E‘”8âb‘ŒË¨¬ãbQœf €ÂCøHH šŸû>W³@SÑøT$ôiNªÇÐH¼÷ð ¾ 6YdÆÖØ"nT¬Ú.ˆ1TfÄ„u¥Í,ˆÐ-$ 3jr’ÄŽÍZàMbˆðXØ j«´’²¿Ð€<Û¯kRÐ@S‚ bN Uy†&?ô¹æ+¹jÖWså14 –’áø"i‎©ÅEÂj+éb0ì°¨ÐuSÒmŒØxˆAºé¶iw¬ÂŠ/zfýj1Ž/F¶Y ¥¨¸ˆ†B1a×m5ÓX+¨çâê0=k^ÚDVÜ´^[÷ê/PKØ+´Ã,PKz_{¥òel·Ûñv6r„‹Å"ÐÔZà¨æËKÉ5W”SܬÂqXÞ”*2T>äuEÊÊtEå`hˆ"VÍ%-€ÔE¿¶;§å_wëÁÞu·î9Ú9ØÏ4sÛUfñpW™ÅîÜ”¨M}¯‚[ ê·?7~%Ó¡{!o ªH²|°š†Û/„¨EÅ &ص¸ÓÉd˜g‡{{”}+™¢ÒaŽ²G„G5â"í øÂ8|z‡._ë2>Xkàíq’±ÁÐ#ï«J¿WË@Ò\HU’ Oº°Ë´N•òþ”T˺–qÜÉ âÌH¼þ£Ûo¼ÖipÜ{Ž SëCS4“›{N'òÔá ®Ñr]e‰(³ØØÁHïs*’×Ó–­\kqñˆ%«óÏY¡u(0Êm¦¨Uê\F?-|–A€ˆ|éÌne­¢˜©.óýök€4ODÈùÕ>ÎE`êÝØS?pâ'$¢~L#^ܼ7Ùº™gòÚ»‰Dè •Z˜¥x£Öl$ùkšbÛ©ˆ©ÌZ,9S¤¿„ÝÓØ Žïý3ƒcG?úH²bÚ!Äw$Å÷{Œfð‘’ÝÉ´g> \­þYqºûžÆgDFŠ”œM4X2$2¾´p€G—_?¸bËŠâ)²ýƒ~"”Çíéð iW(š>I&)X¬·} œ¯Hô¹_¶CÞ‘ñƒ¤´4‡ôT ‚¾ìY›êæ=žRpÇ“Ÿ©é¿?J)¶FB<^÷×Þd4M'£ÙÄŒÃ9qÂ÷CÜL'_Õcš’W•šµÒr-I¾a‘gy«g?‡“„JÅ ²Ãõ %Åg W.à2ÿÍl~yAæž2aœ×”wÓEAÜ&b¹…¥|‘›k{&||n„ʉ$z³ÖVš„‘OJ%Šœ`1Éb* +áù†X´ó2‹T©ãL¯ Ö`iŽ÷C‡r‡ú+I ”D ?‹”¥à]*0séµÇ¥¯V-²,¦xqÂòV¯‚‚è28!¼ 5Xp PE^ ûÕªÙQ¯mË‚úPÆbëëÍ+$•,iK¨¾ HkzÁ¾}:Ï•ã$[—d C4Ó\E”“ÿþñ`]n$ÛO# ®myÌÄÕ&–öec)Õn–ð÷_÷Ä{3§÷ìZsuî[S7lçšôÏ_¼Æ>­X°†qÃÀ\(÷6{§‡‹Ž /°Žà,öÜÚ²k¸I”zF–I +”õÄñ§Ò#Š÷?ë¢-på#Ó‘Ú5ãºÖ ?Ò5]›@¨ù!Ù×RV¶Ó* Å’öÁœÄØNò!fP”pzÂ8„•P «µ.§‰Ò ì$[oŠ1À†Äº¢dqŒà úu ªüû6m⮓hu¿šàðÝEâ¬h'×=rÙ8›Ü0û«’sªF†ˆãpÙöÌ£!ùØ<¸öþ÷ßÿÔç,âøœž“²Ìçd”ÚÖ— @З~lâ°¾ý,øq—®÷†¤¦vÞ™_|«£ì˜¦'à0}úÔðW4’ZÞB³³!?sÍψÐÕ›DèâlMÇó7‰ÐåÙš½I|ÞŸ7š§¯Î†ÐüæéÅz›y:œœ ¢‹¿V¢vÈUÅô^G °7T;kTØTU Q Ú ¢=jª”öX] Tú=°¡f~8™|«G5rL7 냘êvrgB¦„7«( e±ù/„´³•(ŠXWvI‹®6R”ëA·½Å!“Úå{<= ]Éï»Ëóu•÷\+¼ªz fÌjÖÆÕˆ—`6iº>.&v¹mek»^TAýìC9é>Qg÷¦î²t6F¨¸ß µ+ô5)-½Á±§åcu‰:H+j ¤# íбVë &«pÍÜ«rÍѯtÍÒ£vMïW¼‡EûMó 1ŽzI’rF“§w]>ÙÅP%‚&)8ͯš^bõ‰FjË/¦ãÜÙû²íW"ë×üƒŽÇSÛcãð¢§C6Ñÿ::`u£üHïìÝñÞ™ ÛÞÖÙÌdòSÑ4©æ9á¬rÙ+˜végév=Bs8}°gø<ª?䦩#e°èÿÒ¢ÿ9ü‰âãïLàO\ ƒTÍ Æ›Õ!fEÎÉÎw—á£LŒpö¦âßî¤gŒ,“ 政†”+ï^SÒã׺êBr ÉÉZaã¨k·ùOVÓ\“¿^%O¨Ú×~[L „ƒ)þª‰aˆ#Ïç'92ª¾†N½ZMÌQöÂÇæð(2‘¡æ˜žnŽé³ŸÅõmiÒ|ײA’DÑbñuÙ`H›]öÛã°§AP:ìû¸1úì¤ÃdCIüÚWŽAÐWhΧ}pNÃá£WIU¯R»Mbû+ÁãÓ¡{¦´òÕB7ïè01A=Ñ”’¢”/¯‡Å÷YJì®÷K:çõ¼•âØÖ¦tHèeÆTUŸµÑ´+{¯ÜUø¶¾‚ù–`q^Sê:(d[Vú÷íÄëà9âz=ÇÄÞ°9€ÂÎÈŽêý*à¦áøbáŽÛVôÅxq© •Œ’0B²æ‹4øíI˜‹‘£I¿ƒÞÆùA¢¬¾Òäã5.îàbs-ñíøÊ\+!c<,ƒ^“U„”J»å®y©”Â!üH¡yvß±4ZãnÝôFÞ!<¦Rô³Fà§Rð#UÀ䪹‚}= (•²ˆñÞ Áú=·&_4>Úx°ÍMÑÆuÞXÝÃ2èþòßÍÿPKyÄœ  <8PKz¢(Z"ÿ¹Û¶cÞ7±ñ#ø´éZ Ì®ìYŒ£LPÙbø†ÒS¥Uõ³aoÚÕd…Å ”‘¢Um¹Ç¢+v$ºb(]¿N“÷°êÂÈ…dó è¥ÞBÄH¡ò`&æ 9.}ÆƆ²%·FÖÄöi´éì3P ­ÿžÃL9 j§‡$’\DáÖé›P} M‘O,ÅúS1"ˬkÊÈoTÖLEd¥í׌ î_ä:ü=[L “—t±btc¬1Y­¡¢†(â@ é"&I5êŽìéX\ŒëðF®=-†SÈdž ©\ =9_R!doÞBŠp($Á™RX>}I’ÁV €SÑàÜ&jy8Üé M°^Ò‰–úÔÜ[òžñw§ÓÔŸŒç¯ÃÄýÀdò*P~9ÊR‹„%çmq×’ûíÛ*gUŠëŠÀ~¹ÀrM>–]ï+Ì? óºù7œ÷(”.›WeÍWL;¾Î[gCîBgå2ŠÖd/ÕÛêÓ(¤]]H):'¿ݶRQŠ)ýpK‡YDlO‹ ·W, ¸GIS|O” "¼-ÈýÑŸ|¥è‡¡ Ÿô«[¸&è^h¢­´ Pï5x~>Ê=r ú5 º>œ8HÐÁRT}Ù4tpªÕ÷½›"E/iô×\ª$·Ë6ùEºbPòýýs”\oîŸi¿Ýþ*L§×@Ú!|)¢Ge£/¢Sõ¹8⫱Emó¾í éãl«:©‚Úœt€èεWôný—zë¨Úöõ–ìô|(o%”Å(úªÒVÃñMû‡w|·þKTèoîøé…~ ê«vå7tÕôí3ªó†Õ¹iFm×~ÃŒ:¼£8Õ¾Ž:â!]Ô®ü†òÞÔC7vÅUP_ ä|ðò¾Tg”"‹o¾›±õ:C„_pTl…eEXh9QŽCZÓóËœdÈC)wúïÿýO~MIÍJF>L‹Ð(Õ£êiþ<­Óo× O±§] K,!ȸìÀ’€nŒ}ÑÀ2|èÉñD=.3O­ô88WàÀSùc‰CÊp ÂûCÈ !{ä¼K„Æ"ä½K„&ƒ!äŒÆï¡é`¹ïŸÙpø¼Ó<í †Ðøæéù€½ÏPKzr´zl4Ö=MY{&0õ/iÞÏs¾i Ѐ†<òÖ/Õ"Ò/ S^£tV>?¿òwÞÞƒ‰JD?©^dÿ`5KB¯VO+§™áoLg‰ê+Ÿ+åê¡ø¥E%Â0¼Þæ‚yòfÙ9ÚßÄâÅ0ïâ-™R í ^ÃøEÅÍ_Á]¹²Çü˜s ˆy %bÈßœm›ø|‚¨tDïÊCÑ–üµ¯Ñûh wHL “Å‘˜ß= •"3=Œÿ‚@u„~X8KŸë3TÌbú4–·ˆa ²Içhñ–÷f"³Öºh:Ð'¦o»ÜP†ô9]X³õôÕ4t‰×ÒY¹ìºš]®"¶hµ=´ÐqoIø|ÚÒ"ÄÂkøÿ\ÔÝ´ôõ ©ÐEMO€ß–â¿ uÏ›J_ÿã"6ðž·9廳;ûcLAoÌ]ýM‡áí@9ãBÇs3ªo4ÀÆZا±/•®¹ÀÇ’ÝÔèA H‰6MWÑôĆZJ®9•Í\ß\qÚÊ7Òk$g¹[;j Š©Òr*9ûpR7áp‡çU6¦7Í:\Œ ÆÀ^}&{ ­§3Ô8!…_m0W‡‹  êhà˜*!AÐNÓl…z@yØ‚)a¦¡äŒpÃðÞùY/Ý·&ÂDƒå~®¦n;r‹Tÿ!Ÿ¹Fúo XSjéÇ,PñÞ²+[»ÍtPZœ?éÅNÐBÁS5ê&ÇÞntfÈ‹éDZôQûR¤ì·öÖ5…4»-‡A¿/ª m†„B±v’Bô›W_¼¹_3®r8€0gt;Àå±Ú;@20b;©UèXkY‹u®"t¢Ä¢×±]Ô×éÈT h6ˆÃ°¨K×zØo1¢Dm›=K_Ê‚èêl>íÖ±µ‡DŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKz Configurations2/floater/PKz [content.xmlPKzl¨f½f6–fÅfÌl ÍlÂoÏpÏsÐtÐc)Ãz"Ò+ÔUM¸Kg‹Eg’Wd–]{Ÿ@n©Im¥Cr¬Iu­O{¯Nz°Pz°[{¶gIƒsHžyE¬mysb…q~…x}€xUÊxYÊ\ƒ¶^ˆ¹s‰|„|…Š|Š“cŒ½j»k¾q“¿s•ÀvÆx˜Â˜||ÌÎÏÐÍ<<Ò!!Ô,,Õ11×88Ø>>ÑGJØ@@ÚIIÐTYÝVVÝ[[ßccàffáooâqqä{{€,Ô„2Õˆ:׊=Ø“LÚ–QÛ˜UÜœ[Ý—`ÍœcÕ `Þ yÇ£eà¦láªrâ°}ä„„„€†Š‚‰„„‹‹‹…‘™ˆ‘•Œ•šŽ˜“““™Ÿššš‡— ‹›¤ŽŸ¨’œ¢• ¦•¢©™¥¬›¨¯•¦°—©³œª³›®¸ž±» ™§¤¤¤ªªª ®¶¤³¼³³³¼¼¼‚ Æ‰¤É¨ÌŠ­Ñ‘«Í²Ôž¶Óš»Ù«…Ï·•×´‚幋ç»è¾“飶Á¦¹Å¨·ÀªºÄ¬½È²¹Þ­ÀË¢ÁÝ­ÀÙ±Â̲ÆѵÁÛµÉÔ·ÌظËÖ¹ÎÙ¼ÑÝ®Ëã²Ïæ¼Ìà·Óè¾ÔáºÕêΔ”傂牉钒ëë  í««ï³³ðµµð»»ÄêÆ¡ëÌ«íѲîÒ´ðÕ»ðÃÃÃËËËÏÌÒÓÓÓÜÜÜÀÏãÀÕâÃÙåÆÜéÈÞìÆßðÛÄòÞÊóÐØáÒÝêÉáîÎæôÛäîòÀÀôÍÍôÐÐ÷ÜÜøßßãÙìâÐõéÛ÷ãããêêêâéòìáøëïõëðöøààúììðæúñéúòòòôóúüòòùöýþþþÿÿÿt\JIDATxÚípgÇiÅ¡0B'‚½é¢FñW­ü3H‹Ø*ƒ§VíÔÒNÛ©Öj—¸!Çfe.\lÄ]²¯Ôjm¡I)Á€¡lmkhy_ß59Úê]‹ížSšööoß÷ݻܯ½Ë»!KraŸÛìí¾· û~îy¿ïó¾ûì2ùÀmF „B!€@ „B!€@ „B!€@€[6Ëß°uˆåñ9&¹ BìéÀRMÃ"˜Ð5‚¦ã › Ë¡ F$‹ˆƒå,Æ]€ L?žn ª!-«Z†jVPC:¦jVAºbš2Ô‘l;:Ö!-§ݶ5ÅÒ Tt[!Žm Á Q-ÀŽ‰µ,ýUÆt`šfŒl`ÖL›] nV‰m`‹Š´@Z¦u¡ôf*Þ9È2ºYR>½˜:ë ÉY„oéÄ µ¦%¼öÐRz}ö^ &¢iŽ  «X× nX†Ù&ÐLCqÃÔhç Y²NÑP‘¦Ørm@ ˜¤e§ Ê¶jº¥@û,Ôe¡©UËR²aª¦ªC U1t«D©Ý©È&0 IöÔ 9Tih˜X7 ©B`ÚÔô¬Áˆ¥ÓŒ ÖBº&©ñçP”öBfÈdªˆ íì‰:Aÿu%˲ËRÐt ÉÐ5_覩ˆùÿ…îö¥øÂÒ€Q?‡#Œ†ÐB€íÒeÚjKšª-x¹?R [²½Qøú7¿só“O>ûÖ4@hø(iˆExÊ×AVuVoº`LèâŸþéYj›nØø½'ÞªkÄŒ)ÐöÒ„,0vðÖ37m¸ùõú`Æ4«L‹ú:±8`dÓ†[ÎÔ':˜ÌVˆ P/XÚ ŒlúÖÉz`KæDE‚O]ÿtý Qè/*nåqÀÉ 'ë€+F×fyÄ[EEprÃH0õÊ"¹¼©WÓ$UÌ =qC}@^S’&¦0ænÜTWÏI-5)X”/Ÿû`4¿é.ì¯@øµ`A@°"zäó÷ˆyãSu@|0óü]¢G>½±~`É™xp®?Y7 SÀ݇Þòýº ã <àµõ€DÄ×ÇY?R'âáÎÏÔ :Ah€óĦ: XÁxÀs7Ö Ù Fþ½±NøÐ@_ଯYÉ Fœë_¯„`™<àÌÈÈ ™òb|@‡'^žYÏlÙ©°úkAhÀM ÀŸ§¾H‘H, 8óõë7N} pp4âãÒ¦¯8`ýú禖CW²¬¢–—eÉ84àÍSåöæw¿}ªÒNO&,EcB&>=àÚÆ«*íÓŸô(\Òôê¤À’)˜ø…T‰øЀS×ËÚ±Ýÿš$v _ë±tŇ|muƇ=ö™Ó“IÕjëá8æCV)©aj°°}ß¾Á ¯N •^ÖB,Û!–ûqÉe õáW¦Šë74gþŽ-‹·­Y³hËû·Ï_صêg«¶O1¹:B–IK׌ ,¹ìWÍ<5àÊ—Šë·oA*=káìm3ÌÚ6gÖÂUï¾dV  ÌÊÿM€â+}Ôhµ1†¬öÔJéz©ihØ7î¢î¹kfÏ[5wÕÏ/]573%= ¦ j€-áJ£¶f¯gqSfÒØEº‡<…°€2°#Qä ¥óU<:Ìßú™ëÓŸýlçàðPæà”ñ€S?ˆ\÷çÀr¡FG³v¶rùíJ 0JIØèjHììÚÙßÙÙÓÙ›J6lï‘®îì;ÜÙÑÕ‘lq5`¤)Ñ" 0)0]Xö`Yo}CE yÙeï++àWG?Á´¶%[Ûí‰Í}-›;ú.o÷4gº·&[3=;7'úšÛ—÷Ç9€OÅmœ÷1€&È2‹<€)?“@Úp[–l‘ˆl-á*HîLm“’‰Î¶D:ÞÕÓÑ–éèØÞ–énO&ñíí‰nWþIaË2:T‰5P+”³FXjÍ’bŒ º€b:9?ˆjÔê0ömAÈò›N–xô©Ã©Ú[–Ï]÷ŒlÚä …x(,Bv-ð „<ô5'»;»:úÕ»A¢ëç’¼ºAÖ@ÄS~iÓÈ„p\G$èè’û[±xw¦ÆXÀw–õ8dcV•@ˆh È3¿éV±wø yèŒ'w%’‰t%€B7hK$p@)ük‚7@Øâkëó3ÎM‰ p…$F"vEŽc5SM°§é˜úßÿÕbÿ$°åâM1¨éüÅÞsK‰ñ"]R|M‰­mÚ}ÿîû Æövç¬äj«OG(fÀ`±ÐÚ†‡)²G!ô9)z`—‡}éË»ví©(ýuÉ`Ø’ ÙÈL?߯‹£÷>(p¿6à€œXwO4  ;vúË${2€Ì p×D ¾Bß šêàHÙ h `Â5ÀQqÔ±ñB5(¸ýe¡~  @ “ñÕûÒ€Ûÿ+ÔJµ)L`IRK…ŽÂj€„ ü<Ä—Ü6qç(?ƒr?püÎÉ t+ˆŸnÀP¬™ÆEæœüt~<àÞ¿‰ç'€Ÿ¯ÀÜzz’ØŠ"Ådel‰SUðíeK™‰y¶ó†jŠ•5Psè¼Ô ¬þ+þÝO„¾%~ë ûã1ÿÊŠ~go§^.T`jŠ\nŠ“äJSŒ ·<áïw­n*}5}öcMeE«¯Ýãuµûøòw<[›¡Vœg´ò$«Î–W@dل¦UÜí ½¢±_]µûÈ‘ÇË^ÌJKøÊœŸÛ<ÀŽ©@ä õ˜î€&cËò~Ì•×ã¯PEø«D~à*ÑÉþÇVž-ÿå‡ïð bºåyž•'._Äž÷Q”iT|5:¸kw:Ÿ(Ù[;Xö»/.÷¬?‰ Ÿ!”ýÈýa–é…-LÈV<Œîšt'Ãaþ¡'vݬ¬¤éñ\Ö_ÃÞ½û»R½½™E öÚ±oïÁZÖû‹]¹$ȵÿ(ýÕ——{O… ¥$IŸªÅÏŽ=¸È*¾N?^°§=±=a,–ùøµoÀƒÄ.„»Ï€¬œŠµÊû€ÁïÙöÞ‹ÎÞqñŃóæÏY³eּŠhÁŒ†y—ÌñðÂòªÍŠ.ͺgÄN•®Yª]µRb9ì!W´–nò#´Üj˜cá?U<€EΣ…\!šÜü×Á™kæ]tѶY[f5 Ï;ÐpÍ¥3çv5Ìœ¹xË%×Ì›³8`ˆÿÎÛ¹héžÕ¦Âr`çÓ4ÝïŠK *ÎÉlžêÆ]Ÿ¯ÙÎoºMbÔË ¯Éx¢K†7ÊzÁ2½½©CG‡¤eŽ¦‡úû¥Ú?8|.©‚¼²‚‡ˆgZ~ïéê£s« zùSÊ(fO«7€˜e›æšãäí&šŽTH_ºhqSspöáeKÿJ@}ôÎp±ÔNٶǥK÷C<ë—zpe»úÂS£1v)ÛzµË1Y#F»j)‰‚•ä•ëiõ¥û ôw÷öý¡?ÝךÊìØv4Ý7Ôº­Ïðè]tpÇo½â‡ÿ!oŽœ"U *EÈžÈk², ÌŸÝÆÙ‡ó½ùx=Àäí ±48®{t“oñ…¿P-p@ƒßJ÷q· tÄÛ6_ØÚÒÙ½yk²ýr)±£9Ó7§úã‰ÍmnF躰aâ‡>ÿÅËƸE){@•)ž§ؾ«RãÖŽë*ÏùCü:È•qŸ †þ­¾‚sÐÖoMtÇ›“í--‰žæÎÄöÖL‚H'âmñ͹&ðè>@=`ÌHž5Ü#ݳ¢'ŠYÅï ØW•ohœà ¶ BXPÄü¾]ÛØD9ä g†O g†ZÒéaºNÓu*5‚Õ¼Úíå´ç1X ^ ªyÊÆë¨ )TsŸ-t†¼ŠÞ„…ïℸ'Q†ó,OÒŠ4e1]&"®2š+¿T;NË LÎË¥!^y•Ì—‚”¬\æ$£åRÅKQÐÜNZºÜK½•Ñ‹®™ÝÙŠÞ©±“‘·3—¬Æ﬙ÝÙ‰$Û±“‘0u§§bìä»’û©ðc‘D±=)î8Ë¿^y¥Šel·Ûév>rD———¦6Ç _QI®¹’8 œâfeM£ÀòfT‘±ò!¯+R^e+*GCC9°j!i , .ú帅Ü9ÿº]ö®ÛõÌñ†ÈÑ~¦™»®2OÆ»ÊÊ7ˆ‹¬4à‹àðé-º|£wÆøh­wÀIHÎFC¼®*üA ÏI !UH:>éÂ.³&elTƇSR-ëZ&I/+ˆ3 }@ðú·Œn¿ó:§ÁqG¸ÜsZš¢™ÜÜ{tBÈÓ„/¸F{PÈus”¥¢Êc ½+¨dH"\O[vVp­ÅÅ#–¬Ï?g…ΡÀ(·™¢Q©w!ü¬ôY"Š¥3»“µÊr®úÌ÷ËÏÒ|<!ç×û8™wmOýTÀ‰Ÿ’˜ú yyýÑdëfxb¾£WÞ R¿V~ò™ä`R-+ÎîÊû3)Dù—=>3èM:K#¿¿¦9h /EFòGÁT ©ñ–H¦}#/,™Ž’ _C¸Ð/ä·ê8fÏ‘Ê]©hö$™¤`‰Þnò‰p¾"ñ×aÙy_EÆO’ÒÐÒS%†B¡7÷áë˜ràÀ–_©œèß•Rl„˜+ÿ&pNfádšqú›ƸáàfÞãÆ ~Þepé~Ÿ…¿×«krÂÊ‚“¿Ç6’ ŠæÑä2œµBœ/P ü€ñ 2Ü·FHhJ*^´ïµ$ņŞ孿û$(*ƒ‚7)•_)œ \Àñ»ùâüŒ,jñSÆyCù0»LcˆÒT,·°”/ sÌ…ß[¡ "‰Þ¬³•&a~óI¥DY¬QXB…a%¼Øk÷¢ÊcUé×kƒ_°¬ÀãÆÐáMý•¤nÚ%ª,¯(p±÷3‘Àò\újÕq&–'Ïc¬šô*(ˆ®®RÂKÚ€§€*Š½mX­†õ:ж*©eF"¶¾Þ¼FRÉŠv„ʇ€´¦—ìè³E¡ô'ùº"k¢¹ˆá„SLþëçƒu}¸Œ|?¡µ<¸¶å1;Ô×›XÚýÆRêÝ,áï?î‰×1NïصáêÝ·¡nØþÎ éŸ?y­}:±` ㆹT®2ñN&^`ÁYâ¹!´e –aœyF–É! +T&ÍÄé §ÒcŠ× ë¢på#Ó‘Ú7ãºÑ`8Ò5]›@¨ù!Ù½ÀÛ+[ƒé¿T¥béNû`AìRø3(J4;CaÂJ(…E@ÓTiö ’­7Å`C]¨°$Áp}Èÿ%Uþ].q×K´º_„ø¼@7'8+»Éuχ\65 7Ìþªâœª‰!â8Üá<óÕ|¬I¯¼ÿý÷?Ã9‹8>§çd,÷9Y¥±õ9( ¥›8¬o Âá³Ü“šºygqö½Ž²cšž€Ãì 8èSÃ_Q¨“©áý!46„"ãçï¡Å3"tñ.:{6„fÓÅ»DèüÙš¿K|><>ï4O_<B‹wš§/Ÿ¡÷™§£ðÙ :ûc%j‡\WlÁPáu´±{Cµ³F…MQ—õ ­ º£¦JéŽ5J=¬ÛjæGaø½ÕÈ1Ý@0¬bªWÈ ™Þ®¢$”ÅæñP.r: ig+Q)±©ì’]m¤¨Ö¿~žënqȤvÅÏ@gBWòûîòr=ä=׊.ꈳšuqµâƘMš®‹‰]n[ÛÚ®WÇUP?ûPNçºOÔÛ½iº,½-ÚjîÇ·jí CíRKoqhù@]¢ÒšÚEÂéC·t¬é;ŠÉ*Ü0ªÜp +Ý° ¨ÝЇ`ÑþE³bCŒ£D’¤œÑ´Ç)ÅmŸOö1Ô‰ M Nó릗X}¡±Ú2Ç‹é<÷ö¾lû•Èæ­ÿ ãñÔöØ4:è…ú_O¬iÙé}8Þ;3a;Ø:››L~*š&Õ¼$œu.»e%Ó.ý"Ý®Gh§ö _Fõ‡|Ãô/u¤Œý_Zt|@ƒâãg.ð'®…Ÿ U;ˆñfuè>Ã1ËLpI.&8{SñwwÒ F–IÆsÊCÊ•Hvo)éñk]}!9‡ƒäd­°qž2:ø âõ´ºR›uÝ6ÿÉjškò·«ä uC÷Úo‹©‘p0Åß41ŒqäÅâ$GF•¢·Ðið¡UhŽ²W>6ÇG‘¹ˆŒ5ÇìtsÌ^ü,nnKaû4®cƒ4ãËËoËc‚Üì²ß˜F ‚†ÐcßÇõŒÑç'&J’·¾rŒ‚¾Fs1‚sÇè´À°½Iªz;ºmÛ×X žœÝ ¥•oºEO‡‰)òø©ˆf””•|x=,¾Ÿ¥Äî{¿¤wÞÀ[)ŽmmJ‡„^åLÕõùXÍúr@°÷ò_ýß6Ã7»cß,ÎkêC]…lÇJÿ¾ ½ž#®7pLì ›(ê=€ì¨Þ¯.:ŸžuÆÛVô4ÒÔj%`„díßgàKé’0#G“a½IŠƒDYÿ¥Œ 4¸¸ƒNˆ ̵ė®ks­„Lðd|° MV2R*í–»öõNP ‡ð‘BûÝ}9Óhݼ½y@oåÃc: å0k ~*?ÂQçL®š+Ø×ÀR+‹ｚ¬ßskóE룭Û|Ð&m\çÝÙ=,ƒþ¿)»þ?PKš- ñý“6PK[°KM content.xmlí][sã¶~ï¯à(m'I‡"Á;ݵ=ÙfvúàítâmgÛ— M‚³$Á€”måOô±ÿ¯¿¤àEàM¢,zãxå["ppøƒsÈäÍõcK÷˜æI/h©.$œú$ˆÒÕåâÞÉÎâúêwoHF>¾ˆ¿IpZÈ>I ø”€;Í/ÊÞËņ¦ÄË£ü"õœ_þÉpZs]ˆÔ|®²%/¶ñdvN,rø±˜ÊÌh[¼ÞÝô™9±ÈPïa*3£PEöLe~Ìc9$€z’yEÔ‘â1ŽÒO—‹uQdŠòðð°|З„®亮Â{ý†.ÛИS¾‚cÌ&Ë´DJM›à›*£EJ7ɦ“¡ñ ¯§ÕŒâHàv™aNHäiÙ×ýj²uݯF`ö×lgœ¸m*z0ÝTô@äM¼b=¢_GyüÏû›]Ñdê\Œ¶•O£lòm–Ô"?!¤•1”‹‹«©ª¡”×õÃ^ò˜ äþ^rß‹ýq’ tH ß3“o"aД²»!΃ѡ?¾¿¹õ×8ñvÄÑab9JóÂKwÈäIOÖÐŽ­—F“MÑö–eÆ0Š¸©PœZ4 §˜Ek0ZI<îÂXoMº¢A0H âè ¸3p&ò}„¾Z´¢Ó~Ãt;†É]ý!N$Æ‚½ HUMãNÀTw‹®šØ’M”z(ĦëòbÎvÑAÔVLž0d…ZA*Âqí¹š[†9ÉÁaÁ’ìBànÇ<š^õäî'ìQ±&›‚ià„…ËÅí6Ip9ýšúãwTµÇ† ÓnÙP+8O Á/L‹Î‹Ò¢ï ¹ÏY‹“µè¾(-Z–ªúþY‹Çj©çûW ´†å¾ÄHû÷ÞÞ ˜ÇÕÜ(½iÕs3äj°|Ìv€˜··7"Î[îp¹“ÊVåiÒ—<¼¯MÞ5`hâ»nyô ÜR³Bhcƒ\."°¹Ýòj¨e/¼´æ;ùÊ«zǘÙRŒGÙ›þj€ééíK§‘î2|¢& ÛÛ-8Õl*þNcÿŽ±W¹þF,¼ºÑŽ…;]k,y°¯1DÖ;µ^Í÷D+¬ª®çµÂ^•òÂUü+è³W< ¢gUã‡^ d£-ž3ð17i€i¥¸¦)­¡f|¨RŠ;ó‘rÔš`xìccÌÀø Ÿaº*ziÁTUôZ³AÞ]G!:ÂüT¸z1{¸>áö|Ìœv;<ø3šm/9¤‡/ÈF{yÐiÞõl£ûõÀŸ£¼±ý¨ Zˆ1¾Çq5ñÝ&Žq!•¬¼ÿ¢¼,»dö¼äåâÿýO£;aAƒœ‡Õ¾±w=MÝ^žï_µÜBM~»Màk69Û`H(Á9ee%Ýஉæ8.ûîô´pÈ3öTÆ Å5¯!}6„ÐR{•3"ä¼J„ÌÙÒ–Æ«DÈš !ýUâcχÏ+õÓÎl¯ÔO»3"ô:ý4;Gš "ó·å¨…î*×VFŸ³­:îH°m.ªj¯ÞðÝ-öXm¹ÏUææì-êÇø¶Í2·: Ë Lùó¸rýtvèmbVL•fZû#ðt›³®Ö;†á=±·e/ƒÁÙåÉ]¥céjZ}šWsiKU³êÆGþ¸jˆ¶üÚ`G€•`|Þ;òX™YVƒ(HS™€‰¤ýþÐÿvÅ”û` ?*k,}—eP&•ÅW¯ÈUzbãÔ:ËàKGe‡X%¤¿ÉA‹r¾ö²AµQõƒ jKÛ5º B®k ZK×t ÙCyÇ°¼¥’§köR ¯0aÇÙ[ÎÀ^§yK`UR% Yjõ·¶ˆRB…ÃÐæSì®@Û™W¬á†).üuF"X5|kÐiïlÛåî,Å^~¹¸ué:4àׄßú(¸Øf˜½®e»L±_xéª9{MH…¦À¬ïî¨!ã¯Þ½‡Ál¸õð{:Ò ÿt¥ñöøŽjt>VŸŒè¦ùîpÄ@ü·ô·Ú~ñÏ›ÒÄ„õ6OÑšlby‘…²u~¯JßæQúõu_¾Î"9ê7ßìDëÒJßêÈÒÛÒ­ýÌz—™(ÿé:Ô¤ý|F—¯ Ù6³ËÆO©åÃŒV—±<ŸÀiÞ!½ŸÍºÁƒ\QÏvJdDF°î ®Qù]ÎH•ïÚ!H¸hõæìÔ!N죰ž°ü(C”’ {¸`¤ß{,ûËÕ«Ô~²çsêÑiŠBú¨5ù›bÈõ2i_=—oŸŒñI[\ ±è¹bqR r5t'p²œŒ¥mèsÅ s9Ç‘s9Ç‘ÏG´V9\èœæçªs ŽÜbzõâp yòÀúâê0¢ˆâ@ò ‰nÒ"JF&9=P™£:²D™-™u¤C]éБ»´u½£#ãÃÎ1ìÃÎ1ìåÇ0[ôzË?ÎñOósÅ0¨•þB’,Š19ˆ™í vÇwŸ3ŽYûâØ3¦öR7Ìs°:«s°:«ßX°2ZÁÊxfÿ8c°zëåXúa_A4WÀò☇«üÉñŠ¤)xYB‡ôdêÉV†àÁý ½¯þ“!¨‹;bÊrìb4~x‹v¥°¶«…kÉØK}ܼÄmܲyoØ¢€€ÃŸ«,5Ø‘kB£H¬–Ó½·@@IcBÉHÕI•‘ih’¥iåä°gG{QútÖg‹!Vic;kg>¬!!3µ½X#X¦ÝÛ8l4¶~l$,ÁFªjpU“a]~áxa­Z¬Ï9kw>¬úõÑk}iîN®Öm mçrŽ„Z‚Ú8 µÓK:u¯;º>„ªƉÎè*ÚŽ€ûé"¤mgÂ!u1DµyÕwˆš Ÿd1™<i’‰ ¹–-•ŽÂ1^æ(Ü“wV§,ø…-‡^·évm§•îñPú Sp¾ ˆd²É5N@•ÀƒajŽ ~Lè€\~’Ê1¤Ž¢©‹©H‡¶¯öÕÍÈФ"<…ÞIE¬óóç2ï\æ˼×]æM9˜9ÅÎUËíϤïqXŽ?‹qÊ9ÚÇ÷7RÅ8—RŒ¨ì RoE®±´÷øîåTxí£Ì¡dB_ªŽÑÍ„­ã“‰Á ÏÚ%:R&²é@2!;6Ôq¦Å?yÖ ŽÍ“3÷ô”â¹ëDî9o8ç ç¼áœ7|qyî&«Uؾz6ç:cNñ–Ü —ÍŸJÜâ8”þJò"JWÒþCÓ—“:tPGå†êlD{š¶|ìVóÁÔalËíR ¤€’Í„’‘kTX&’‘¬Yª-îPXª3i‡âóalï ­ó3WÄé²59AÓGQÜí±GPVE”AD–¹¹¾e2]Û„ÄÍäŸ#Ûj®mͱ«6ã–pgs½ÚÀ4žlÍã8µ)Z3Ù¦Qî`"ÙTCÑ’Ëíwwä¨CµJ5ìÃZ©R½•ð>kâoª)­÷Ø”‘ÿÅÜÕÿPK‹d„ £nPK[°KM settings.xmlÝZ]sÚ8}ß_‘ñtgÚb IÛ0$¡aJ6@w»o¾€6²ä‘äþý^ È°‰ãÍC2±¥sŽ®®t?àüëSÀŽA**xÓ©W#àžð)Ÿ5ñ¨[ùâ|mýr.¦SêAÃ^×Zãu„Ó¹j¬^7Hò† Šª'¨†ö"¾™ÖØÝ°d«'OŒò‡¦3×:l¸îb±8^|<ræÖÎÎÎ\ûv34” Šh+8áöœmZOð)¥EYÞž/„xm&¬f…׫Õwõÿf´ (KËeÆV<„¨yÂ^ØŠp¤…1c÷V½RuXûÙ³öõfm¹HÝimüaã­óµqV*TC`|ähýØhl:HÙx¤°xö'nÞË9?¨2hK #:›—zâKʵӪԪõOçî>ЛÀû0Õ±èõ“³Üà¿S_ÏãÐkŸ«§'¹á¯Îæ±êkõúi--~% a…ržÀßå‚EüfÙ9èor™F1,zþŽL¥%z‚Ó2~‘ZéKÜo’ú=µ¶ÇüD„;--#Èß•‚ïšø|J˜ÊˆÞSCN‘0,Åkß È nˆœQ®Ê#1¿û”C"Efº›üžîJ|X:Ë@à*g1©kÂ}ªÍdùš÷æ"³Öº”d6Ä“Æ’·'»\1PN4 [Z³õñnºØké´Zͺš}®2¶h½}²Ä¸·‹O|:é ¹tZnêŸ/ÍlZxCjò_QÓÞø¯Kq_‡ºm×ÿ¤Œ ¼‚‰}‰ùý>b KŒ7æ®þŽax7PÎ…Ä£“ñÜ á¾/¸§¬+Ü¿SôÔ¥ˆÐÈŒz#xÒW>M Âyh,ÁÅœðÜ‹UP<ÍQÔä½XÄïAÆTÐX%fìU›q[ÿÿi÷iÉiw={¶y!ˆT¬¾þ9ã®þ~H{ô.懄çC>¨:;´Iú¬è?n±x–D Y"Ç%ÖÚ6ƒI`)€ãç¿°ŽŸå¬£§lÖhh®¸Iz«¢ÉA/!9ýSˆàŽÇ„ØÁ#æávmôz]®ÇP Ó7fV"iƒß[:4í0d˱yI4)>jv¨Æ…ŽŒ+Œïû ݃wï±²Õ\5'Ü0”¿~ì¼{¡°kbWi¬¹ˆô•˜dSƒ9‚$˜HÉ ˜®¢é‰PJ¡9•Í\_]qÒʼl¤—DÍ ·²ŸQ LIÄ4Êêøì#{”º &àwEQecrÓ¬+ä„ú>ðgŸÉßBëc†Ťðë žÕá"߀:fL•ˆ¤d¯i¶FMQv`F¹i(eF¸âþÁùy/Ý×&ñ)‚~®fÙväšh¯øC>Ï鿃äm…>2ˆ¸§£ƒeW¾v›é t„xÀÅ”NÐ!ÞC5ê6ÇÁntn äåt"-z—êC)R~‚k{ëŽRšÝ–àߖՄ¶ #Ê \;h)úÍ‹oá×LV9‚ø÷@|ÁÙn€+bµ7@T$aÌ÷Së0c­e,Â\EbØ¢7c»h€éÈL’p>Œ‚ ¬K×zØoaTïš=O_Ê‚ìa6Ÿvclí“¥ˆ’Œ‹ fé‚Ef™¹(cN{úDd1í¦³omšÉ#üX}Ç/˜Peœá¡‡²œßBîOd‡>Æ-ŒŒÁ‚ˆao«Âmíî}áÁMúJLëoPKL~—UT#PK[°KMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPK[°KMŸ.Ä++mimetypePK[°KMGìäzzQThumbnails/thumbnail.pngPK[°KMöÀå\Å.meta.xmlPK[°KMüConfigurations2/menubar/PK[°KM2Configurations2/floater/PK[°KMhConfigurations2/toolbar/PK[°KMžConfigurations2/progressbar/PK[°KM'ØConfigurations2/accelerator/current.xmlPK[°KM/Configurations2/statusbar/PK[°KMgConfigurations2/toolpanel/PK[°KMŸConfigurations2/popupmenu/PK[°KM×Configurations2/images/Bitmaps/PK[°KMš- ñý“6 styles.xmlPK[°KM‹d„ £n Icontent.xmlPK[°KML~—UT# ,settings.xmlPK[°KM¿(ÖÞH1META-INF/manifest.xmlPK6 2apache-buildstream-27ae392/doc/source/image-sources/arch-dependency-model-runtime.odg000066400000000000000000000334531514607367700307450ustar00rootroot00000000000000PKdŽKMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPKdŽKM£þ¬WÓÓThumbnails/thumbnail.png‰PNG  IHDRžê·$ÓëPLTE#')%*,(,/).1-25/4806939<6=@8?C;BF=DH@GLAHMCKPHQVLU[OX^PY_R\bV`fYdj_jpfffmmmbntcn}nexlnqdpvfryiu}kxtnyrrrsvxw{}zzz%;»ÌÏÐÑ##Ò,,Ô33Õ;;×==Ø6f¥=l¨»L1/p›€@ €@ €@ 0u}hÍ0ñ›Ð´¡ŸÚd ÈŠ¢XÇØ°¬bë:2LãôŒ°n ÝÔ#ªýY嬚*ž,DE¢&…×y, ˆƒ2i¡©ó(P‚â‘,BÁÔòqš  rXÔI€´ÛàU‘0ÐDyM€S°©C^Ò°¨KPD¤šªL?ÈD2˜7 ÉÆ$L¤ˆ j@ÃHŠaj ²g()RtSUU©d·ct¨$È( ” Ç=#l=«|rP$:†nHµÖ É ŽÐ4, º‰€¡ÓŸè”Y*ê¤`ˆ¢ÉH–D(ɪÌC²*Q‘yÓ”Q¢¨r’Ä+dœyÀ‘ˬúŽ³82È’&@|¿%@¨ˆ† ê'+‚" —% ?â{ºæ`’ÁŽü¼ú½‰@4e(H/+ËX’¡" ¨²¬J:÷$UÒ$“±€ŒƒP&ÝCi ì\–L±Ù5O'HF~C§¿µg_ŠÕê^ŸH£ÀxE6kŽ3Ø0¡Œ—­9q¡p¨ÉÐyÔ^&{/0TV´5îÚö›ÖòÊÎÚä¶e«šÝѺmW@÷äðÖO<~à _¼tÆÊ9—\oõáÂN°i0ä°Ÿ'™ü0pãêw‹NÐP8­ücþÍãÀ,=óž=`R1-¤•€«þœ[rÇ3ÅG‚ZÈ(17ß”WôLkû“5ïç—ó¥àå+òÊ>©¹­ý'ª_Ëß"§”€«Þq+ݸ€ÚBkZp5][ðÝo^½ c íù„Ë¿Uýé>8|=ë/O=ÀúK–€|XÀ}¬¿„\é8<ߘ­düìqsü`ÊRɸâ-? r¥``®é‡Ì*£D¼|½é‡L•€ÕOú£E)×öG˜/W¼ã4®Dxðž`V•€wç›þ(ÀŒè%``à•o Ž¿ Ãà°aLtƒóæR;<þ @¡*bá ¯€'iûWûáD L|pÅܹ_ôÅ„«ª¸‰ïÌÃ:€Ç8 ª O8çÛoÌ/+B:MÏsκ!ðF^éÝâ¼]³0?Ííêeùe5{>ñ¦!f´Ð˜.›Œ ÀÛ?:À˜øubcÄ/w©Œ†äˆzž¼¿ì5æ3ý©§ë<(@äTÂÆùðzMN#»Òzè~1À?—yP ¶ãü²1Dˆcp$ç<cycbvíÊÚÖµµ·»üW„­VjC3€Qƒô1Y`Ç·š:†³…cðR€*gϼlÊ£•—7ξ¤kάŠâÊÚÍRÑrpžü%W•+k+ʻʭ­øÒc¥ÃV€æl ÝëÖž§­§B˜ ^ºÝ²*@‹à<8<¾{ùùàô}™D·6{Ñš²_†l!“´ªh¤}€³d{@lÀéò‰ €úDC¢¹sgs¢+ÛÑomÙ±5Õ¶=±>£€èï뺮åÌøUv) w«"8”V>À–û#‘ B Ò~€1ñŠV™ÕÎ4ÈÑ ²¾9²isgý¦–ø¦úæ†è¦ÄïRñúøî €ïGÜì+_È) ‡|¡*kÏh›åú°BÝ JFòf9l—‘0/yS±ÄˆŒð3zã-‰hK<‹ÆvGãm[:š¶´¥š¢ñø¨] ×ösjˆ:Aº=ˆÍÁò„€¼ÛìÚñb„¨p:wÉp‰H÷oñìó}Ìq‚pÔPÐî(,ç€!È#=ínÅpÎ1 uÓIØÍÛgʲÆÈŒÐ8îÜ$÷8 >Úº9º5Ž5o®ß¾½ig曳…p¿k äÒ|@éÙ!BÖ(`HÒ¹€"îq@C¬e3×oˆÅws/„hXî!¶º€æÜɪåT )ð|õ¬zXu?èmiÛÝ’hk'^ìÚÝöâΞllNÐy,àˆ÷‰Âœo´,ZÄð¾@õï‡Ä ÛY/PÚÉ/‡ÃÎ8ÀëõÃ"ðç×`…oƃäöÐ~YpêÓð€ɬ¿·lÃ>b¬ÉÕ }]½ÁÓ)1>"çšD&Wã²d/*>€NG{vÿÞý{÷îÝ·‡Lû©‘ûúk±×þŽ¼öíÝ÷ô‘O¼…®MåÝJ•¬Ó!óH¼F½4ñì³6çéâ(K7¼¹Aï¸QO?=û°÷3SľÐGORð`¼`"ÙWH1'¸¼õÏ šè ð–Eà€ˆ&ºLû @ï©_ú¥ž©iôC uæN¿À3õ¢˜<Ì"¿À–$…8á‚vÍ|R7~uôÀ=õGŒ»ÖWLÉš?ìT…±›ú²œsò2 xQ€Ì8¾…õó à‹ýQÇâEü ñ|$Ìó£çcÜÃè?úö"jŒG:“y>Äñ‚? àée†Qöùç7xš¶ÿNƤ:Àt”£Yù”Ðtã!}}x/ã?½›øã#l.@®ò’NèшUULç—¾Á¸Á-^Äê11k–l¤*¤úÀ³…™§ïfÝâ‹ÿËÚÿXOö©!—GFxª¬ÈåÏ…#œK±œ'‹¥§Ü¶ùêž[ÖÔ­Yã|»î{äí–¬¢ûßs€ë ËŸ[ŸP~%eà€Áq d61”Kàô5.ýýò}'Ž?~"ûÍZ:ŠN¬[x–UZX,5”Â’7"‡s.¿hÎëT¹_ ¼…%ÏçmóÈò“Œ×^«É€à*€°äV%·*rÀ€ˆ[¢¦VøÚ„”;D}°äï¹ÛñqzÙ§uo²u#Ä|õ¤À­ÇS 9»t~¢”Æ4QÍZ"Z€ˆ­ „ñPÖNóƒ¥grJÒÏé™YqèPk÷oûZ[R³?WÑzì±ý,-kN6w¹PÃî~ñÎd:hUUµj‡q¦rÅY;œfh@¯KC€*½> ¬” òžÎÙ³¯Nå)g–æh ý4¹î‹fm«˜vùŒ]Ó¦u—WÎ\ÛXV>§bÚŒ9•S+Ê/ýŒ+F…ŽÍdv •PH³‹è{¦ï¢âh ¦[C¤íª•™­"ŠÅzPÑÀÝ ¯:Ð=}UùÔ‹¶•5^\Ñ[~´¢¶|úeÍÓ/ž-«-Ÿ1Ç`8^,ø¬Vª€áËÇ´íÖ¾²ö’ A¦ó¡;);&gN¯ãB 0ÍÁ¯yè#kí®>ÍP÷y+]pôÛE†ФIZš] ,h¹+»~šZ¬°MÎQ©74GEé2Kà‘ºÀ3 ÍÿÅé.ÐÔ¾¥!–ˆ®‹o­ßkYÅR±¦èºþ­MMÑz»lxu;UÀ¨‘|Æ ¦kJ*jgSYÝUXÃ(`w¤!O7äÕa÷—þ¬¬rÔ»hééa'Ø›êíëMõÔ÷÷ööÐu²Lõ÷õö¤ú’=½½Ã>EjÃàt¬ÓøŒ!pÉKÆ9î⇜r ƒië!.¯£¿Ð£´Þ¤O6—™ÀxcI> ß­‘ÎQ¦÷lД=+‰#{©Ž¦Ûr¦ÖR/%šñ¶xG<7³2/]¶„éÛJhà‚††A«“¡k[Vç·,L·ªÐÐÏŠ ­É ¸FU@V(ŸÊÕªìзè{ÿ<›|¶:òFZ¦k„þÁÏë3|d‚YµTõ?(öèÎÓI€î¸Nûç<Ûg‰·ö½’¿1…]x÷eà-õ›÷Š³Ãåµêi,µáÄsëò0U°£3S(°.á¦÷ðÚ{¯ÿPK~»-Ã-PKdŽKMConfigurations2/menubar/PKdŽKMConfigurations2/floater/PKdŽKMConfigurations2/toolbar/PKdŽKMConfigurations2/progressbar/PKdŽKM'Configurations2/accelerator/current.xmlPKPKdŽKMConfigurations2/statusbar/PKdŽKMConfigurations2/toolpanel/PKdŽKMConfigurations2/popupmenu/PKdŽKMConfigurations2/images/Bitmaps/PKdŽKM styles.xmlÝ[Ûnã8}߯04˜}“%ÙN:ñv2Ø X`2»˜ž™×-Q6»)Q ¨8ÎOìãþß~ÉV‘¢DÙ’#çÚ“n ŽYE²êÔ…¬’òñ‡»ŒOn©,™È¯¼hzšÇ"aùúÊûõ—Oþ…÷ÃõŸ>Š4e1]&"®2š+¿T;NË LÎË¥!^y•Ì—‚”¬\æ$£åRÅKQÐÜNZºÜK½•Ñ‹®™ÝÙŠÞ©±“‘·3—¬Æ﬙ÝÙ‰$Û±“‘0u§§bìä»’û©ðc‘D±=)î8Ë¿^y¥Šel·Ûév>rD———¦6Ç _QI®¹’8 œâfeM£ÀòfT‘±ò!¯+R^e+*GCC9°j!i , .ú帅Ü9ÿº]ö®ÛõÌñ†ÈÑ~¦™»®2OÆ»ÊÊ7ˆ‹¬4à‹àðé-º|£wÆøh­wÀIHÎFC¼®*üA ÏI !UH:>éÂ.³&elTƇSR-ëZ&I/+ˆ3 }@ðú·Œn¿ó:§ÁqG¸ÜsZš¢™ÜÜ{tBÈÓ„/¸F{PÈus”¥¢Êc ½+¨dH"\O[vVp­ÅÅ#–¬Ï?g…ΡÀ(·™¢Q©w!ü¬ôY"Š¥3»“µÊr®úÌ÷ËÏÒ|<!ç×û8™wmOýTÀ‰Ÿ’˜ú yyýÑdëfxb¾£WÞ R¿V~ò™ä`R-+ÎîÊû3)Dù—=>3èM:K#¿¿¦9h /EFòGÁT ©ñ–H¦}#/,™Ž’ _C¸Ð/ä·ê8fÏ‘Ê]©hö$™¤`‰Þnò‰p¾"ñ×aÙy_EÆO’ÒÐÒS%†B¡7÷áë˜ràÀ–_©œèß•Rl„˜+ÿ&pNfádšqú›ƸáàfÞãÆ ~Þepé~Ÿ…¿×«krÂÊ‚“¿Ç6’ ŠæÑä2œµBœ/P ü€ñ 2Ü·FHhJ*^´ïµ$ņŞ孿û$(*ƒ‚7)•_)œ \Àñ»ùâüŒ,jñSÆyCù0»LcˆÒT,·°”/ sÌ…ß[¡ "‰Þ¬³•&a~óI¥DY¬QXB…a%¼Øk÷¢ÊcUé×kƒ_°¬ÀãÆÐáMý•¤nÚ%ª,¯(p±÷3‘Àò\újÕq&–'Ïc¬šô*(ˆ®®RÂKÚ€§€*Š½mX­†õ:ж*©eF"¶¾Þ¼FRÉŠv„ʇ€´¦—ìè³E¡ô'ùº"k¢¹ˆá„SLþëçƒu}¸Œ|?¡µ<¸¶å1;Ô×›XÚýÆRêÝ,áï?î‰×1NïصáêÝ·¡nØþÎ éŸ?y­}:±` ㆹT®2ñN&^`ÁYâ¹!´e –aœyF–É! +T&ÍÄé §ÒcŠ× ë¢på#Ó‘Ú7ãºÑ`8Ò5]›@¨ù!Ù½ÀÛ+[ƒé¿T¥béNû`AìRø3(J4;CaÂJ(…E@ÓTiö ’­7Å`C]¨°$Áp}Èÿ%Uþ].q×K´º_„ø¼@7'8+»Éuχ\65 7Ìþªâœª‰!â8Üá<óÕ|¬I¯¼ÿý÷?Ã9‹8>§çd,÷9Y¥±õ9( ¥›8¬o Âá³Ü“šºygqö½Ž²cšž€Ãì 8èSÃ_Q¨“©áý!46„"ãçï¡Å3"tñ.:{6„fÓÅ»DèüÙš¿K|><>ï4O_<B‹wš§/Ÿ¡÷™§£ðÙ :ûc%j‡\WlÁPáu´±{Cµ³F…MQ—õ ­ º£¦JéŽ5J=¬ÛjæGaø½ÕÈ1Ý@0¬bªWÈ ™Þ®¢$”ÅæñP.r: ig+Q)±©ì’]m¤¨Ö¿~žënqȤvÅÏ@gBWòûîòr=ä=׊.ꈳšuqµâƘMš®‹‰]n[ÛÚ®WÇUP?ûPNçºOÔÛ½iº,½-ÚjîÇ·jí CíRKoqhù@]¢ÒšÚEÂéC·t¬é;ŠÉ*Ü0ªÜp +Ý° ¨ÝЇ`ÑþE³bCŒ£D’¤œÑ´Ç)ÅmŸOö1Ô‰ M Nó릗X}¡±Ú2Ç‹é<÷ö¾lû•Èæ­ÿ ãñÔöØ4:è…ú_O¬iÙé}8Þ;3a;Ø:››L~*š&Õ¼$œu.»e%Ó.ý"Ý®Gh§ö _Fõ‡|Ãô/u¤Œý_Zt|@ƒâãg.ð'®…Ÿ U;ˆñfuè>Ã1ËLpI.&8{SñwwÒ F–IÆsÊCÊ•Hvo)éñk]}!9‡ƒäd­°qž2:ø âõ´ºR›uÝ6ÿÉjškò·«ä uC÷Úo‹©‘p0Åß41ŒqäÅâ$GF•¢·Ðið¡UhŽ²W>6ÇG‘¹ˆŒ5ÇìtsÌ^ü,nnKaû4®cƒ4ãËËoËc‚Üì²ß˜F ‚†ÐcßÇõŒÑç'&J’·¾rŒ‚¾Fs1‚sÇè´À°½Iªz;ºmÛ×X žœÝ ¥•oºEO‡‰)òø©ˆf””•|x=,¾Ÿ¥Äî{¿¤wÞÀ[)ŽmmJ‡„^åLÕõùXÍúr@°÷ò_ýß6Ã7»cß,ÎkêC]…lÇJÿ¾ ½ž#®7pLì ›(ê=€ì¨Þ¯.:ŸžuÆÛVô4ÒÔj%`„díßgàKé’0#G“a½IŠƒDYÿ¥Œ 4¸¸ƒNˆ ̵ė®ks­„Lðd|° MV2R*í–»öõNP ‡ð‘BûÝ}9Óhݼ½y@oåÃc: å0k ~*?ÂQçL®š+Ø×ÀR+‹ｚ¬ßskóE룭Û|Ð&m\çÝÙ=,ƒþ¿)»þ?PKš- ñý“6PKdŽKM content.xmlí]Ýr㶾ïSp”¶“¤C‘à?Õµ=I3™^x;õ¶³íM†&A‰Y’`@ʲò½ìûõIzþ¤H‰¶)WõÊ;¶Dààð;ç “w7I,=`šG$½š¡¹:“pê“ J—W³¿}üQvf7׿yGÂ0òñ" þ:Ái!û$-àSî4_”½W³5MÄË£|‘z Î…¿ Nk®…H½às•-y±G³sb‘»ÀÅXfFÛâõîÇÏ̉Eî€z›±ÌŒ@ÙC2–ù1åêIæQGŠÇ8J?_ÍVE‘-e³ÙÌ7úœÐ¥‚\×Uxo#°ßÐeksªÀWpŒÙd¹‚æH©i\xcåc´¢Hé:¹Çt44^áíi5£8¸]f˜ãyZöõ°m]˘ý•GGÛ'n›ŠŒ7=y¯X è×QÞC'ÿóþvgW4;£mAåÓ(}›%µÈOiDe åbçâjªj(åµ@½9H¾¡Q©@î$÷½Øo'Ih@‡ ñ3ùf1 òM)»â<úÓûÛ;…oG'–£4/¼t‡LžDñh-í€Ñzi4ÚíÞÒ¡Ì7Š3B‹FAáø ³h F«"‰‡]ë­I—4zIA]wÎD~ˆðæ«Y+:6L·c˜ÜÕcáDb,8È€T…Ñ4îLu¸è²‰­!Y§A©‡@ü˜a±./æl‹Ö¢¶bòŒ!«x,ŒÐ RŽkÏÕÜRï0„ÈI† –d »óhò8n8¶HIvGì8,?Ïõ¢Ï>~PXŸÌB>µj&!ÕÑf×u^Sú±\iBÈoäÐó±`?ίߕñ©i–Êk&÷Õì6‚`Çá•î¼¼‘š–æöjö{/#ù;teãLj Íèå%Npu”$^ڢȢ‡`ðàш[Ÿ2^42%#| á~À?{_ÆL !R¾Í œ¼H&J¢€O'ýèÅñ½ç–mŸöUdü‘b<šÐõR ”¡¥Pµ{ë‚°xáË|œfð¿-yƒ 5“UrVÙ°œyKáv÷Y—tuI½lùusæQVð ¹db‘4ðh0«Ç­Xä ܦE„s‰ óQòR’â̲‚iL }ýJå?³’:Œâ¸¦mÒÿT=Ìç€\ˆØtà¨e/Ž–àu’¢ð‡d‘D©¼ÂÑrU0gdÛ–ŸpØ(ᢄM°¼S«&ÙÊ;K<úSòÛ…nš] ˆøýý-ÔüÛ¡ÙEmEhô+aa Æíçu^Dáv,¼œŠ¨¼¤dÓÀzq¾¾:w ~Ý^I­Í‘Vµf^À b¹ #Gf§ýžË6{ºbò)4»ÛCëÙË®'¨^ÿ2–„ª= ã²"N½"4Ã>Ï%a¾ºî{–†ÅNcÿ9S³Ðmã<ÍÂzºYûŸ±_l¢bEÖSÁtöÁ§ï?Ã>®fwÛ$Ádo?iêOßQÐÕ#2L»eD-=ÆŸ™í³Òâ³Wù®EçÌ´xY‹ÏÑ¢{ ´ÿÃ@kXîyZ¤^ìâR©vÌâ¯{{7`×r£ó¦EÔ:ÌÍ€«±ò1Û~¼{{#â¼å÷Pý;ª¢Už'}ÉÃûÚä]û…&¾ë–G¿Âí 5+„66ÈÕ,“Û­®†ZöòÈKk±“/¼ªwˆ™!Åøq½é¯¯‘½-‹§i¤[?S#…ÝÛH8sólÅéêM°‰Þ¾F¥¬wötø¼Q?íL†ñFý´;!BoÓO³3š‰ 2ÿ¿µÐ]åÚÊàC¬UÇ= ¶ÍEõ´êõ;¾»ÅžY-÷¹ÊÜœ]£Yýøܶ9ßâ½VMy)ØU®}½uÌŠ©rÃŒBëþüé×æ©uxÆÎ8xOìmÙ›VðAÖEy*VéÁ˜»šVŸ”ÕGHÚ\Õ¬ºñ‘?f‡¢-¿6ØñZ%Ÿ÷ž§îæ±8¤:Pƒ¿ˆµˆ@ö(¨—IûêT¾uª¥Í®…Xtªd¼(¹:‚“8YNÆÜ6ô©bÐ%Ž\âÈ%Ž\âÈ+Å­GŽ—:/óÆ‘;L bì!ÏX‡ Œ(¢8¼B¢e9uª@eêÈud¶tdÕ‘•¥sDGîÜÖõŽŽŒK »Ä°K »Ä°óa–èõ–<žã¿Ì?NÃŒÙõŸH’E1¦1³Äî×QH§Œc¯½égÏuü«K°º«K°:ÿ`Õ*¸ŒV°2Nì',¸¾÷r,}8TMUuyqÌÃUþìxEÒ¼,¡}z²õd÷+Cðàþš>Tÿ±ŽGÔÅ1e9öN1?¾E»RXÛÕµdì•9n^â6nÙ¼Œ×lQ@ÀáOU–ìÈ5¡Q$VËéÀÀÞ[  ¤1¡d¤jŽ¤ÊÈ44ÉÒ´ò rØCÏ{QútÖg‹!Vic;k{:¬!!3µƒX#X¦ÝÛx:بlý8Ø.HX‚TÕà0ª&úüÂñìÃZµXŸóB¬é°vê—3k¬õ¹¹;»fX·)´Ëy"ÔZÔÆq¨˜^ÒA¨ÝÑõ>T%è0ÎÒ¶#à¾CEºiÛ™pH]gQmDõ¢&È'YL&OGšd"CB®eK¥£pŒ—9 ÷ÅÆ;¡£è¸ jC„ºíKŽ@­O5ÚAÈ°6J¬‘lªˆ¡hÉ¥£pœ2sî99eKôÁ[næºít êh­¤pŸnÖ½>Ù"™lr ÃU%ˆ|à4ÇÌzÀ„¨›F y¤ôuÁlmáÖIGûêÐ2ô#Y§¬w²>ëò(Ì¥¢¾TÔ—ŠúmWÔ§v S•ÍFsN)ý€CÀrø±——ìö~z+…QŒs)Å8€"º õ®ï KOJ_XLO˜K´Oûr }®:F·è°žžKôÓÖ.—ÐA2—Mr Ù±¡d6-þÉ“ÝýêØ<v_žQ uÓãG¿cv‹Å›{I.‰Ã%q¸$_\â°+Êj¶¯Næ\'L*¾'¤7èeÓçw8¥?“¼ˆÒ¥tø€ú|r‡ê¨ÜÑAÏÞó±[ÍGs‡1{>(ÙL(¹¦Y…e"Éš¥Úrï…¥:£v(^c{i ¹"ÆH­ÑÚðÎZïn=€²*¢ "²ÔÍõ-“èÚ&dn&ÿØÂtmëø¦R¥Ká6ÖÄßUSZo²)ÿ·ëÿPKeŽùl nPKdŽKM settings.xmlÝZ]sÚ8}ß_‘ñtgÚb IÛ0$¡aJ6@w»o¾€6²ä‘äþý^ È°‰ãÍC2±¥sŽ®®t?àüëSÀŽA**xÓ©W#àžð)Ÿ5ñ¨[ùâ|mýr.¦SêAÃ^×Zãu„Ó¹j¬^7Hò† Šª'¨†ö"¾™ÖØÝ°d«'OŒò‡¦3×:l¸îb±8^|<ræÖÎÎÎ\ûv34” Šh+8áöœmZOð)¥EYÞž/„xm&¬f…׫Õwõÿf´ (KËeÆV<„¨yÂ^ØŠp¤…1c÷V½RuXûÙ³öõfm¹HÝimüaã­óµqV*TC`|ähýØhl:HÙx¤°xö'nÞË9?¨2hK #:›—zâKʵӪԪõOçî>ЛÀû0Õ±èõ“³Üà¿S_ÏãÐkŸ«§'¹á¯Îæ±êkõúi--~% a…ržÀßå‚EüfÙ9èor™F1,zþŽL¥%z‚Ó2~‘ZéKÜo’ú=µ¶ÇüD„;--#Èß•‚ïšø|J˜ÊˆÞSCN‘0,Åkß È nˆœQ®Ê#1¿û”C"Efº›üžîJ|X:Ë@à*g1©kÂ}ªÍdùš÷æ"³Öº”d6Ä“Æ’·'»\1PN4 [Z³õñnºØké´Zͺš}®2¶h½}²Ä¸·‹O|:é ¹tZnêŸ/ÍlZxCjò_QÓÞø¯Kq_‡ºm×ÿ¤Œ ¼‚‰}‰ùý>b KŒ7æ®þŽax7PÎ…Ä£“ñÜ á¾/¸§¬+Ü¿SôÔ¥ˆÐÈŒz#xÒW>M Âyh,ÁÅœðÜ‹UP<ÍQÔä½XÄïAÆTÐX%fìU›q[ÿÿi÷iÉiw={¶y!ˆT¬¾þ9ã®þ~H{ô.懄çC>¨:;´Iú¬è?n±x–D Y"Ç%ÖÚ6ƒI`)€ãç¿°ŽŸå¬£§lÖhh®¸Iz«¢ÉA/!9ýSˆàŽÇ„ØÁ#æávmôz]®ÇP Ó7fV"iƒß[:4í0d˱yI4)>jv¨Æ…ŽŒ+Œïû ݃wï±²Õ\5'Ü0”¿~ì¼{¡°kbWi¬¹ˆô•˜dSƒ9‚$˜HÉ ˜®¢é‰PJ¡9•Í\_]qÒʼl¤—DÍ ·²ŸQ LIÄ4Êêøì#{”º &àwEQecrÓ¬+ä„ú>ðgŸÉßBëc†Ťðë žÕá"߀:fL•ˆ¤d¯i¶FMQv`F¹i(eF¸âþÁùy/Ý×&ñ)‚~®fÙväšh¯øC>Ï鿃äm…>2ˆ¸§£ƒeW¾v›é t„xÀÅ”NÐ!ÞC5ê6ÇÁntn äåt"-z—êC)R~‚k{ëŽRšÝ–àߖՄ¶ #Ê \;h)úÍ‹oá×LV9‚ø÷@|ÁÙn€+bµ7@T$aÌ÷Së0c­e,Â\EbØ¢7c»h€éÈL’p>Œ‚ ¬K×zØoaTïš=O_Ê‚ìa6Ÿvclí“¥ˆ’Œ‹ fé‚Ef™¹(cN{úDd1í¦³omšÉ#üX}Ç/˜Peœá¡‡²œßBîOd‡>Æ-ŒŒÁ‚ˆao«Âmíî}áÁMúJLëoPKL~—UT#PKdŽKMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKdŽKMŸ.Ä++mimetypePKdŽKM£þ¬WÓÓQThumbnails/thumbnail.pngPKdŽKM~»-Ã-Zmeta.xmlPKdŽKMSConfigurations2/menubar/PKdŽKM‰Configurations2/floater/PKdŽKM¿Configurations2/toolbar/PKdŽKMõConfigurations2/progressbar/PKdŽKM'/Configurations2/accelerator/current.xmlPKdŽKM†Configurations2/statusbar/PKdŽKM¾Configurations2/toolpanel/PKdŽKMöConfigurations2/popupmenu/PKdŽKM.Configurations2/images/Bitmaps/PKdŽKMš- ñý“6 kstyles.xmlPKdŽKMeŽùl n  content.xmlPKdŽKML~—UT# E,settings.xmlPKdŽKM¿(ÖÞ‡1META-INF/manifest.xmlPK6ß2apache-buildstream-27ae392/doc/source/image-sources/arch-dependency-model.odg000066400000000000000000000330321514607367700272550ustar00rootroot00000000000000PKOKMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPKOKM;ô¯ýññThumbnails/thumbnail.png‰PNG  IHDRyáöéPLTE5;?8>B=DH@GLAHMEMRGPUNW\OX^TTTPY^YYYS\bU`eWbhZekaaannnbmtepwfryiu|kxttt|||ÌÌÏÐ&&Ó1<É55Õ;:×.G¹D ÒgÌuÐ~(Óu:Ìn{‚q~†@@ØLLÚUUÜrXÝccßlpÅhuÒudÆvxÅggàiiàyeàssâ||ä[ƒµuƒ‹x†Žy‡|‹”e€¾d‹ºkŽ¼l½q“¿m”Ár–ÁušÄz™ÃzŸÈ~£Ê>Z¿.ZKf´FT­SdÌÏÔ**Õ22Ö99Ø??ØCCÚIIÜSSÞ[[ÓOaÞbbàeeàjjâttä{{.Õˆ:Ö‹kŠ„r•µ{‹µ‘AØ’KÚšWÝŸ`Þ{ä¡cß°eÂ¥jà«uâ±}䂉‹‹‹„“ƒ’›‰’–‹“™Ž˜žšŸ™™™¦€‹±Šš£Žž¨’£•²• ¦”¡©™¥¬›¨¯”¦°œª²›®¸Ÿ±¼£££©©© ®¶¤³»³§¼²²²½½½…“ÊŠ’ß„„剉æ蓓蜛ꂠDž£È…¨ÍŠ¦ÊŽªÍŠ­ÑŽ±Ó”¦ÆªÍ™®Í–¯Ð”±Ò—¸×š²ÑšºÙ©“鵅彑袵À¥¹Ä¨·À«»Ä¬¾È¢¸Õ¨½Ø¥¥ì¬¬í¬µä²±ì¹¹ð­ÀˤÃÞ­ÀÚ±Â̳ÆгÅܶÈÓ¶ÉܸËÖºÍÛ¼ÑܨÆà¬Éâ³Ïæ»Ëà¶Òè¾Ôà¸Óé½ØëÄ…’æ……é””ê››ë¢¢í««ï³³ðººÁšêÁ±ÁÛ«Ðʧìϱïвï×¼ñÃÃÃÉÉÉÔÔÔÜÜÜÀÏâÃÃòËËôÂÑãÃÙæÄÜêÈÖæËÛêÝÆóÒÝêÔÔõÚÚöÜÜøÉàíÎæôÖàìÚâîÞæðòÆÆõÓÓ÷ÚÚøÝÝàÍôæÖöåååìììääùâéñíâøéîôììúíñöïóøøããúëëòêúòòòôôûöøúüóóùöüþþþ`Ê ¬IDATxÚíœ XçÀ[Ÿ§ãÙS§Ïc±6í\ÝØ[æÓîaÚ¬)t(u_©VÔɇ>íƒv T¥ÛB–¤¶fk—4µ0¦Å%àÕ~ \“ÃŒdˤ]Õä¤æjƒJk±-a³÷BwIî.äK´÷Oî½ÜÝûÞ{÷Ëÿÿ¿î½[¼_q¹…Ààp8€Ààp8€ÀˆQ<9™L’u>~9íküIfVÄŸ&Õ•¤È:Š3IWægqÊèìC6ÏŒ82'€#g–c‡ãôÿg Þõ¿÷é·|Òx< V³p®ŒOF†Ü¿÷“ßy?Ùñ~ÙŽ÷Ê^,{ñÅEχ&zè‹„8Ë +'>  ò†ðÂíÏ—-Zt{ÙïºëÝ0Úl‰×Ur4à³0–ô颲Ow¼××ß+{÷Ó?¿»ã½0O[òLà_h4!fŒ»Øh]IÙ ô©>jL(ÏÐÐÐ[sZøÏñ›S…â6ñü*À¸´Ù¯¿v'“$€§nyú’ŒŒ7ÿâPUÎO¤S,ЦÜRuÛÜ–T_ž4äç®ÛØã¾™xöÜ[{ús²pâvö0ŽanLj|'øL`b¸û “û¦кXé™q18ѳ¦û&ðö½ç"/I¥€;7ï¦P—î™I1¸Š”tSÁMàÜâËäb°žÀ?^'ÅžÈí»áxî¢l?ÒÅ ›Òº°nìFðæž *šjhõfOµx„Ü^õrP{¤?7†/26áö^¸8­]îä¸?´Mz•,¥ƒ”Íȹc0‘W`îí1õä» »]¦>“„æBSO¿ÉÜ×kN,€·3X"”2ï1ÅÀe*(,(,,,˜È7™óÁºÛš ]ùÉ~Ž À%æãîuq0 RÊéÍ 6{½1j€wýØìϤ{cÔ¯©÷F |#f på%ÀÅä¸ÿó˜5À››׸’ óx7Å€»Ÿ"¹"QþXâ°º€4`“;.†MY#…ê@œ\nmMo=£¸ú zûÆ`¢‚‰D›Àç< Ê5 W$=~!þº]Ip‚+x¼%1û€ "Q7þ’R z¾yÏå˜}ÀØãk&n6²œ²=ðe‡í|$pžz’SO=uêeו٠@õX=£df×€ŽÌzÉ´ÍVMXº{ðìÙ4àl6Î*™Wg)€RoVë[!ÔK?ÚƦǦc· þˆ–r–ú³7€ùUå‚¥‚r¸¦üa€ýl@€ ´b`áRàáÔrA* À©Y `/ @ÍBpióB¾°FP΀^0~M9È‚;ù|¡-€3“u±s“=žÖéæÉP"5 ¼0k@?¨œ‘MÀ/âà3¡ª«[^·¢N©T.?ýæieíeír°ªÍ¡²¶nÏå¸ðoᇴ¸Çpfsb8bÃF[$`ÕsÑo®×¸œÃàçÑ‚ŸZ\ ö :‡v¦à6÷>é×£d$$ $“WÈ%˜VRm÷øŦ<ªüìWÓ{@ MôK¢$ }…LN¤—UÊä.%¸D.ƒ ØÀñ¿þ³×åŽÁ|x†¼E²‰Œ­_krýÁÀ€êt¤SÀZD+pÖ:ýþx!H~óïéßÝ"ц?& Õa DfPÀ°6(t˜…±Ýí`VÅJã¿Ìùk6¸g“t¯ïñ†p>à“0¯é'ù”Ö“&p­?×5{¸×º‚K5tàï×émh¤¥ÀDP=ˆC®EƒÞÆ º×™cpÆOyÝ!¥@õn’Vkwʤy¤à €ÂTúlµ>U?Y{âbôÒŸ‰€þþ¦6=ÈkCÛ ¨A¯×£¨}F5Aju3AuxØšàXþ xÎQ$Ç[ቀM~klìd«4°¶XŸ ¢ \yÁÅÈœYF‘Å<Þ=­q0±ÖÿãËÌή€œè2¾'¦v¾šÍÞT5œð}(çèêZ½Ÿ½ËœÝxé—ãeæ©á»«Ç÷©öªT`!B•êÁúV{ë&ǵúû'Sø©|)pŽIyŸŽ»6j+ÞˆŸ,¤}ŠAcC¯°e”þXAßl(sݱŒ€kã,঺I¸ÐÚ@2Œm¤ðŸkí¸Y2ôÓ÷Z[4qЀR&y®?€ÞÚC#%qЀ­LL}×@!}£d¼(P ýdÈc¸Ú»0Ct289@háÈÈøø wÜ:Bì™Íœz4%BÑãá>‘)ºbp`·P¸`Þœš[‘š;Ä©w¤SRç#u?xê¬5´ðz@Kª`éBþ)wRæ·}mÎü)M‘Ð Fç#W¡ÐÈß%Êjp¾¸½\XS^^.<'—‡­ââÉܬYc-Â`i‹k©©€”Âr"}s`êì¨/úHiIhâ”´ eUâ*±pH‹ðoýñLD[ ¾r4 À;Z¼Yn&­8¢yƒßÙ´ë¶pÿÑáƒaSíšúÕ™hÇ|–סĀ¼EbŽ!Š¡v… ,X8^ï5ÍÖ­Pë{k/‘^¡°#Nƒ5ê (¦Wo‡¬Ýàïwó~T¼µd$ì%4†1´YnT (bGt:Á,9| ‡%•¿ÎGI* )®VBÕ$•Uƒ ŽÑÖ{¿ïs‚R)òŒdw;Aò ) Q¡SØ*Ä b×·y<ÞÝK–Üwß}Ä,ßòh‡dr¨Ú)©”·HåÕÚR¹I Ÿt¢ízÖÙqÑk :°2 T׎‚Åg†°§LÛ2øÁd)€8µ†g0‚µ6Øn‡ízûk(`*­ïœ6 €ÐW!D -"P(`|·K컟U€¿N~h„›AÖ„2wØÔFéu‘¿BcªÀõ0ñ¹¾vR`p¼ªŠ€áTDNPÉq Ɉ/=Ol¶K y5"‘K ÏhPæÃRp†ÊêCòÈ_¡1UÀa¢É’jhg…^À›õQú˜ÝN€ÊšÝÛ1»Op»sb`—ÓÖ˜Ãiúx€>sÊÜa' Îâ´ÛÉöãÜϬG¦bq#Hk«È} €þYWg|ÿ‡›¼¬:ê#¨¬¢Ïû [[ áÆk´±Õ#VÞ„ãÈrÌìöÏçìó²ðª^>È&ÙL¨c%Mª#¾°þ1–?îÀj"꺼§‡Î½ž½ûüMZYûsÂ=ƒ¦?à³wÚ˜ÅÁ| _ÚˆHáÓYg™y¯§Í¹£m†oÏbîmr€Ààp8€Ààp8€Ààp¾2òv±µ2‡SÇIEND®B`‚PKOKMmeta.xmlSM›0¼÷W w¯`l °R=mÕJM¥Þ"b{S·`#Û,鿯1!…nTEâ‚ßÌ›y_åÓ¹kƒW®P²(ŠAÀ%ULÈS¾í?†9xªß•êåEPN˜¢CÇ¥ ;n›ÀQ¥!s¨ƒ–D5F"›Žb)Q=— …¬ÑÄ Í/çVÈ_øamO Ç1“HéDEQ@] Œ^qý [bò–O ¢Á;9¼×Ô„][ê57.ÚXß”ûr¬9ë\J©«é‰67À[ÇqœÂùA›N´÷*Nت®wšÇvÓÓFŠîÞ4öë“f¬½5ç9®ëmÂWÁÇ÷`³ÿ¯¶¸V{YƒÕâaP/[6£.ýP¨æÞ[èyc”‡(Úㄸ%Ñe»,I³¬„7%£dKÅû““$Žp^Ùcú¸+á›U9Ö@ÈísÕ_ö8ÿ´C_/oâ[ýM[nêìôåyÆž¸äŽ¬tý,Žšö•Ã]„¢4BÏBçÃ÷<;di°z­~rj!Š»øáà Z¢‹Ìߌ³ÂõVÍ´”Æ øwuœ2¸Å¤­Î¬K¸é<¼uíõPKf ä¤À+PKOKMConfigurations2/floater/PKOKMConfigurations2/menubar/PKOKMConfigurations2/toolbar/PKOKMConfigurations2/progressbar/PKOKMConfigurations2/statusbar/PKOKM'Configurations2/accelerator/current.xmlPKPKOKMConfigurations2/toolpanel/PKOKMConfigurations2/popupmenu/PKOKMConfigurations2/images/Bitmaps/PKOKM styles.xmlÝ[Ûnã8}߯04˜}“u±“Ž½ ö‚˜Ì.¦gæu@K”ÍnJ(*Žóû¸ÿ·_²U¤.”-9râ$=éâ˜U$«N]È*)¸OùäŽÊ‚‰ìÚ ¦¾3¡Y$b–­¯_ùä^9?Üüé£HÑe,¢2¥™r µã´˜Àä¬XâµSÊl)HÁŠeFRZ,U´9ÍêIK›{©·2#z±±Ó5³=[Ñ{5v2òvæ’Õø5³=;–d;v2ò¦öôDŒ|_p7n$Òœ(¶'Å=gÙ×kg£T¾ô¼ív;ÝΦB®½`±XxšÚ5|y)¹æŠ#rŠ›^0 ¼š7¥ŠŒ•ym‘²2]Q9¢ÈUsI `uÑ/Ç-dÏéø×Ýz´wÝ­`Ž6DŽö3ÍÜu•Y<ÞUf±=7%j3`ß+ïˆúÇí­_Étì^ÈÛ*’,­¦á¶ç !Qq‚ v-nèûsÏ|·¸·GÙ·’)*-öè({DxÔ .Ò>Ѐ/ð€Ã¥wèòÞ)㣵Þ'! =ò¸ªDð5¼ð$Í…T Éø¤ »„MÊب”§ ¤Ö¬kǽ¬ Î̃ôÁëÞ1ºýÎéœÇa±ç:µ>6E3Ù¹÷è„À÷§ _pö ëæ(KD™ÅÆ@zŸSÉD¸ž¶ì¬`[‹‹',YÖ CQ^gŠF¥Þe„pÓÂeˆÈ—ÖìNÖ*Š™ê3ß/?{HsñD„œ_íc]Bç¦>õ'~B"êÆ4âÅÍG“­›á‰ùŽB^;?2HýZùÉg’A€AJ­Y!pv×ΟI.Š¿ìñ™AgÒYùÝ5Í@k|)R’u8r¦"HwD2íÞxÑ`Éd”lÈøÂýƒ~!¿•Ç1³xFˆTì EÓgÉ$‹õv“O„ó‰¾ËvÈû*2~’”€f‘ž+7 Õ¸¹ß|Ä”¶üJåDÿn„ü«”bk$Ä\ù7qíøú“™oÆ!èoãn€ƒ›ÐÀü¼KáÒ üú¿W«krÌŠœ“»Ç6’ fÁdᇭ—s”?`|ƒ ­bš’W@ Z…÷Z’|Ã"§æ­¾»9$(*ƒ‚7)”_)œ \Àñ»Ùüò‚Ì+ñÆyCù.’¢4Ë-,åŠÜÜ3áâ÷V¨œH¢7ël¥I˜ß\R*QäkSaX Ï7¤¶{^f‘*uÈëµÁ/Xšãqcèp‹¦îJR7íU5¯(p±wSÃò\ºjÕq&–ÅÏc¬šô*(ˆ®® ڀ§€*ò½mX­†õ:ж,¨ eF,¶®Þ¼BRÉ’v„ʇ€´¦ìèá2©}“1® †#]Sе „š»’=¼í@°²5˜þKY(–ì´æ$Æ.… 1ƒ¢á cVB),úhœ&J+°Ol½±(ÆëB…Å1€5èBþ/¨rï»ht‰»^b­û•ÿÁ ts‚³¢›\÷|ÈfãPÃpÃì®JΩš"ŽÃÎ1_ ÉÅšôÚùßÿÓ8œµˆåszNÊ2—“P[_‚ÞPú©GíÛŸÁ‚pø¬wƤ¦nÞ™_|¯£ì˜¦'à>}j¸+ u2­AxÍΆP`üüÝ!4?#BWï¡‹³!Nçï¡Ë³!4{—ø|8>ï4O_ ¡ù;ÍÓ‹3"ô>ótàŸ ¢‹?V¢¶ÈUÅæ ^G °7T;kTØTU Q ÖDwÔT)ݱ¦@©†u{`CÍüÀ÷¿×£9¦†õQLuã ¹3!SÂÛU”„²Ø<ÊDFG!mm%J…"6•EÒ¢«åzãVÏsí-™Ô.ßãèLèJ~ß]^®‡¼çZÁUÕ1cµf]\ëñ@ãÌ&M×ÇƤ^n[Ùº^¯Š?ª ~v¡œÎtŸ¨·{ÓtYz[´TÜOoÕÖ+ µKkz‹ã@ËÇjuVÔ.HGº  cMßQLµÂ ó Ê Ç°Ò Ë€Ú }Xñí_4Í7Ä8êA$IÊMzœRÜõùdC•Ú¤`5¿iz‰Õ©-Sp¼˜Îsoï«n¿Ù¼Uât<žÛ›2_ÿëé€5-û#½³Ç{g&l[g3“ÉOEÓ¤š—„³Êew¬`Ú¥_¤ÛõÍáôÁžá˨þ˜o˜þ¥Ž”Ñ¢ÿK‹ŽhP|üÌþĵð¤j1ÞjºÏpÌ2\c’‰ ΞÀTüÝžô‚‘eÄ“qÀœòÇr%âÝ[JzüZW]H.á 9Y+lœ'Œ>ƒx=톮ÔæA]·Í²šæšüí*yBÝнö×ÅÔH8˜âošÆ8ò|~’#£JÁ[è4øÐÊ7GÙ+›ã£È\DÆš#<Ýá‹ŸÅÍmÉoŸÆul$Q´X|[6äf—ýöÀ4h4„û>­G`Œ>;é0ÙP¿õ•côšópÎ0ÑiQcô&©êí@ê¶Iê¾ÆJðøtè^(­|³ÐÍ{:LLAGÏE4¥¤(åkÄëañ}–»ïý’Þyo¥X¶­S:$ô2cªªÏÇÚ(ìËÞÞËÕW|Û ßìŽÜšP㼦.ÔuPÈv¬ôï[ßéá9âzÇÄÞ°9€‚Þ¨ÕûUÀ…Áôba·­èi ¨U2JÀÉÚ¿ÏÀ—Ò%a6F–&Ãz牲úKhp±œ˜k‰/]WæZ ãÉøh1è š¬"¤¤PÚ-wíë á#…ö»ýr¦Ñºy{ó€ÞÊ;†Çt@ŠaÖüT ~„£Ê˜\5—·¯‡¥R1Þ{5Y¿çÖæ‹ÖG[®óA›D´q­wg÷°ôúÿ¦ìæÿPK[¦þ“6PKOKM content.xmlí]Û’ÛÆ}ÏW èÄe;3¸¯µ«²âRåa•rYJJÉ‹ HX €{ñOä1ÿ—/IÏà6 Ü•ÕªÚ%9Ý=hœnôef¸zõú~“(·„å1M/h©/’4ŒÓÕåâ¯ÞªîâõÕï^Ñ(ŠrÒ`»!i¡4-àUé4¿(©—‹-K/¨ŸÇùEêoH~Q4#i-u!s_ˆk•#yñŒ̲tAœ·#ëߌ¿²`–¥Cæß漪,ѱÂ÷y¢FPßd~ïhqŸÄé§Ëź(² M»»»[ÞKÊVòЀiÀ¡’[îòÍCÄÈ°V’æ<œúã»ë÷Ášlü–9>άÆi^øi‹L¾‰“ÑVÞ§õÓx´+pÞ½G‡qgDÜÒÉ(+Eã“\7­‹M2Â8µf]±0ìeu Âõ6&w_-:Ùé°cz;Ž)Bý1Á$炃H×8ONÀUÛÄÅVMnè6 K;”’ûŒ°˜“üDˆ]tf­•ÐGLYåci†N’ŠIRG®æ–z§¡TÝäàÈðÀÒìB’îæ<¶¹7Hiíθ°‚<7Š>øð³Æi*OùÔª+I¥^\ÕuMÇr­ˆ ¾Q#? jH‚$¿zUæ§fX)?s½/×1$;¯òÞO!J@©YáÑ|¸\|íg4ÿ~‡¯\(©9¿º")¡ŽÑŸv8²¸ Üú,Þ§W ¦ŒFéÆ?‡r?’_ý¿mc&ñŒP)È ²y’NŒÆ¡¸œòÖO’?ø4¬Û>ïgÑñ-#d4‰ôT ´¡G¡÷·åù"PÅ<Í3"~wô 3Ô\¬Ò³ª†ÕÌ_I·»/ºb{¢+ægë8¨‡3Ÿñ†D|PK!žICŸ…‹zÞJDÍ ŒVÄ$È壟ˆz‡¼¾Ò—º…€…ë5ûDL u\ËaXÖ.d¶~z'<+'êூˆ8‘UQxD­}uMYüåáNõ“xQé×m^ÄÑÃ.ã-×:hÙ61$ARqqC¨+FïÔ5‰Wk†‘Ÿä@èÅ&N›Q}éš6W¯¯´ÆK„«ÑÌyã§4ãìÈÚ¿¡EÁ«ªRB"q ììRX}õ’Ä=K²ö!Óãó˜^€VÚþr‘Ò”,do¨-¦‹’-kÞ}ãFâßX›íXR> c~"ÎýD`Óyž„9Ûþܶ7óyÚÞ:ÝöôæWwq±¦Û‚›à‘N0ƒàþq¹xÿ°ÙJ‘_°þË luÀ‰LËé8Q' ¶ÏÌŠö³²bHYn¶âh+:ÏÊŠ¶=?‹±¢û"²i§ü¤™ÄÜŠþ‡ÿm~µ–Ž×“_¥c»Ÿ1¿VÕþÚAcœÜÄûûŽ°k_ÀÏ|öR÷ÐDø!ýEªúyì§4¾O•û’:¢½Æ÷Q×}ö´Gúú`?¯ú{ÉÄ^8Ùý<#¿ü°߸Z‹“uŒÛ4$,‰SRfÂÐZè®Ê74 Gß[)TSûE;·Þ#ÜÐ…øxlöÂé1lÚ~†ÿ[ôÁÕ •ÅwÛ:âl(–•IEì^óé(—“7äáék‡½ôr‚"ý N*Oz:~CÒ…g/µŒuÓzñdvÓIì°—>ÇÚ¡^Šœí0‰öê€ña{¶ÃévGR“8¯fìß@ ™!!·$©.|³MR(%‘C`\”K’ÊÏ ^.þóï5¶“&‘,(dx3™ø7@ia»lôg‰ÚGÞC“ûþa·XC°Íy[žòuÁYyIÁ¶d×EL«\O8t§'à€Ÿ€Cžñ“7$¢ŒÔ ¼<„ŒÉBKü"2'DÈ}‘Y“!„—æ‹DÈž !ãEâãL‡Ï Óîd™/4N{"ô2ã4ß}™"ëË Ô¹ªµµÁ³«ᆆ͇êêÕ+±pɪ–K˜emÎ?£zL\W:íZí.åa⌫ZŸxŽüm›)ÁlsÐCÍ×~Fö'g_›=¤ÎîØO5%ñø÷¬à…n‹rÿ°Ùr<³>/Ð6ÄN=vÏ7ù\Ë­??p²mòí³Ê»2­Ò“¤kþÝ”P]ÊO <þ%‹7¦Ñ]Áš¿òw}÷1c”IûcÍž–L®wźÄÌ/ÖpÃŒÁ:£1Ø]¬û¸Íäû ]†rå˜?¿\¼Ž åudÂ?vÍñ~ˆO•‘ ðÓU³·¡aÅ„°ÑÞQC&¾õ&sàÖ?ÂÏëÈU®Å«§ü]Œ—€\Ã{T£ó±zåL×Í{W â?”¿Ô®Aþ¹-OãK5ÛÀ”m¶‰ÏŸƒ…vHí ü^W¾Ëãô›×¼ù&‹5äêß~{x¼; h«|g ÛÐÛ° »Â|ñ¯#¬–3wå š³vÅÄÆ¥z\ÐÞ,·IGH:½w@sûnð¨”7 æQA´ç;%2² xw˜T¨|¯f4Ëo`£8' Só;þµÖAZ¦1xžˆz¯Bžˆ7[¾ß<@÷ïKzùôò¨Þsj‚4+Å#7Ù@ñ`5Ä÷‡.bÐ=ëǤûé\±µÕè†Þס¶J·²Š¸ŽÃ„Ó}ú´¸ú!ˈžÜBU¶ã¼uæËjÈÚ‹i-^#sñ¤äat'p²%œÌ%2ñT9hÎ#s™óÈœG>SÁ<‚Ï'Ì#ï »…Ž§?‡°ã`{ a 6ÒuSÀ¨[ëòÀ³kÝæ4÷‰X;Óaí.qùÂk!ë.nCΉPã>¨ÍãP»pyƒR·†k}¨*@0Ÿ¤Ý@ L„eH»Áœ¥áxCâi 5ZH-PP±¸R€ž°b!SAží(e¤ð¼—G ï9yïN¨°6d¬»ÁäÖÆ4X£k„@C¶Q‚TKGF[-C…7–y¨ðž–§Û©cl 6ø±Ù ËîÒ±ìn¬pÍ&š8˜C±]S݉ ñ±8W6ø&‘ÆÕâø*«¹´¼[˜ÙKÃ’ 3d£!©2CK[0ÌÇUڮ떹´M¹îYmžà¹žà/§öƒ©=&˜Z^O0móDMm³M'îsÅÕ5YA枸Ã5Ú³ŸOk”'™êâ7bQøGÂÿÌ5Iƒ‡‰ïלJÑ8µ¸ª–Q·¿5•ª/p»öþ¯ýMö½2ÖÏgÙ‰×܆Ùmp°‰v‹n³³ÂYðNÑm˜ŠÅÖ=Qt“ÝŸiEh£:°xÎÒƒa±0ØKÇét»¦±ÔÚ,½#yi¡ZzŸ<ÞÖœ»º¹«{ ]ÝÁmͳЩzºö°(Ôx`9ü݃§ì`~|w­DqBr%%$$!غ>z³&ÊÁãªO,-',&ºGwûŠ ¼ôÚõÝzçÇ:½˜èÝÑ´ÛbÂàUƒ(&TË…bBu¬Ø–-^ņÃ~u±áMPRª(Žxý˜{„—¸]ð˜ë†¹n˜ë†¹nø¿©Úž¬6a÷ÓÙ‚ë„5ÅJ ƒ~6})ñž$‘ògšqºR~F«R]Ô‘ØTwwÖ!NØuw:ÃGK‡1»îTÃÕy– E…m!©ØÖµw…ÂÖÝç´ÖÛ.Ú­Ï: #P¶GhÃgzW{œ”ueP‘Wnn`[Dϱ p³ÄëÀ)ϱ"Ѫòc%ýU >$þÞ…ÖùkÚÀþ|õ_PKwèË> =zPKOKM settings.xmlÝZ]sâ6}ï¯Èx¶3»Ä@²Û ØfIBl»}öÔÈ’G’Cø÷½&eÁ&ÄNóLl霣«+ݸüò°“'Š Þtj§Uç¸'|ÊçMg2îU>;_Z?]ŠÙŒzÐð…ÀuEÖ8Dàt®ë×M'’¼!ˆ¢ªÁIª¡½†o¦5¶G7,ÙúÉ3£ü±é,´®»\.O—g§BÎÝÚÅÅ…kßn††BmG¸=g›Ö|FçÇ¢¬GoÏB¼ˆ6Ö ³ÂëÕê¹»þ3Z”ËeÆV<„¨yÊ~°á48ÆŒÝ[õZÕaí/ÚãÍÚr‘ºÓÚøÃÆ Z—±qÖ*TC`|ä$~l46¤l7³iâ ©ÉEÍ@xà¿.Å}êN´5^ÿÓ26ðNtûó;ûCÄ@–oÌ]ýÃðn \‰G'ã¹Ã |^(p_YW¸~ ¦è«+¡‘»ŒzcxÖ×>M Âyh,AwAøĺN(žfĨjòA,“÷ c*h¬’°öªÍŸÿOÓî%§Ýgٳͮ RAªúZý—Œ†ÿü€ö\è=Ìÿ χ|Puvh“ôYÑÜañ,‰²DŽ+¬µm“ÂRÇ÷aßËYG_Ù¬ÑÐ\s“ô¤VE9’ƒ^Brú§Á=O±G„‡ÛµuÚë¸\O @ߘY?ˆ¤ ~oéдí& äѤø¨Ù¡:6®0y¤tÞ½ÇÊVpÕ‚HpCÂP>ü|Öy÷>BabbWi¬¹ˆôO•˜fSƒ9‚$˜HÉ®LWÑôÄÆ(¥ÐœÊf®¯®8me^6Ò+¢…[ÙϨf$båŒtrö‘=J]Sð{¢¨²1½iÖrJ}ø‹Ïäo¡ 0CRøxƒ€gu¸Èð§ NFS%")ÙkšÅ¨G”‡˜SnJ™®¹p~ÞK÷õ€I|Š`…Ÿ«y¶¹!Ú+þ/²Fú¯ y[¡ #îéè`Ù•¯Ýf:(!q1¥tˆ÷XFºÍq°›y9H‹Þ£úPŠ”ŸàÆÞº†£”f·å0èwe5¡-Ø2(×NcZŠ~sãâÅ[ø5“UŽ þ_p¶àŠXí-I˜ð½ÆTœf¬µ¬ƒE˜«HÌ#’Xôfl 1™K.FQ”uéZû-"Œê]³çéKYX}ŒÀæÓnŒ­²QJ’ÑbbYÁ,]°È,3eÂi?>‰1F £Ýtö­Móy‚oëãïy— UÆyx Ë)ð-ôáþD6àIècÜÂÈŒ!Ùö¶*ÜVÑîÞÜ´¯Ä´þPK„|­ T#PKOKMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKOKMŸ.Ä++mimetypePKOKM;ô¯ýññQThumbnails/thumbnail.pngPKOKMf ä¤À+xmeta.xmlPKOKMnConfigurations2/floater/PKOKM¤Configurations2/menubar/PKOKMÚConfigurations2/toolbar/PKOKMConfigurations2/progressbar/PKOKMJConfigurations2/statusbar/PKOKM'‚Configurations2/accelerator/current.xmlPKOKMÙConfigurations2/toolpanel/PKOKMConfigurations2/popupmenu/PKOKMIConfigurations2/images/Bitmaps/PKOKM[¦þ“6 †styles.xmlPKOKMwèË> =z ¼content.xmlPKOKM„|­ T# 3+settings.xmlPKOKM¿(ÖÞv0META-INF/manifest.xmlPK6Î1apache-buildstream-27ae392/doc/source/image-sources/arch-overview.odg000066400000000000000000000505221514607367700257120ustar00rootroot00000000000000PK\ QŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPK\ Qmeta.xml“KÛ …÷ý­ÍÃŽ#Û#uQu1U+5#u9À¤´6X€Çé¿/~¥N'ª²¾sÏárÉÏM¼ c¥VÀPLs©NxÞ wà±|—ë—ÉåšuP.l„«/U–NG茢º²ÒRU5ÂRǨn…Z$tMÓÑhÚ9×Rý*ÀçZ aß÷QGÚœ β Ž§ ÊÙ…k;SgPÔbp°G.ìðÞP»ŽÔaýiåƦÜWc­Y×ÒZ_B²©ct‚P§õBÛFÖ÷:lÈtÓzÏc}ÕÓJÉæÞ2û&õÉp^ßzŸ9†¾ë•«ÂW)ú÷àj þÛìrÛy VƒG@¹LÙðe>> 3bÌzCQ„w!F!Êö„P”Q’D›­O„Èåð†"çŒÎRâu^÷8¦ˆÐÍ6Jpã4Iqlr\:ÿBÞ™±Vùu?áÝg’}›]Þ×Bö›Õ–8ýŸ÷'ø$”ðjmÊ'y4âËxy˜F8ÚDäáIªî|ø¾Ki¬€CkôOÁÄèáC'k’Ùäo½©þå³Úa*­“,÷õqÐûÉé”+À.°ÌáUëá­ï^þPKAwÊNÆ,PK\ Q settings.xmlÝZ]sÚ8}ß_‘ñtgÚb’¦i` ¡aJ6@w»o¾mdÉ#É!üû½2&KÁ&ÄF;;›‡dbKç]]é~Àå—ç=TTð†sr\uŽ€{§_ñ"ÚLX.,~Z­ž¹ËÿW£UHÙ¾\flÅa„š'ì'[NÃ}aÌØ­U/UíÖ^{ÑžnÖš‹œ:Í•?¬Ü y™gù§B5„ÆGŽÒÇFcÃAÊú…ù‹÷8Yó~žó*c€–2‘³z©¾¤\;Í“Oçµê¥» ô&ð>Lu&z­ú©4øïÔ׳Lô‹êÉEiø Á,[ýYíóÞò+!‰*”ûð þ&̳7+™ƒþ&û(†yÏß©´DOpšÆ/NŠY⫤~O¥öØ€ŸÁ€p§©e eà»RðM¿€O SÑ{jÈI4†åðÚWèÀ-‘åʉùݧr)`¦ûÉ_àé®Ä‡ÖYÏÅ X¬n÷¨›“ÅkÞ[Š,±Ö•$ÁOËßžâpÍhH9Ñ0l‘˜­tËÐe_ÙÕ‚7vOmsÙØ¢t?údqoŸ(8?k£¹pšîÞ?bZxCjò_QÓÞ#ø¯Kq_‡º-×ÿÄÆÞ‰Ž`b[bygˆH‹ñÆÜÕß0 oÊ™xt ž›!0¼ÑÀ7àî©Än…oÁ=u%b4r‡QïqÏúÚ§¹A¸ MBЙÀƒXÖ ‡§2ꃀ|óì=8+xý¢U26 ¹jËgÆÿÓ´»f9í>/žmv‘ òÕŸ~.høÀwh/…ÞÅüo—ðrÈ;U‡6I_"ú;,ž%ÑBZä¸ÂZ;É`rXÀñã_XÇ;ëè©$k44×Ü$=¹UQ‰äÀ [HNÿ"¼ç!öš[Üà ¾ ƒ÷O §LÌûoñöH•ñp½Ï{ö²(Ðûw–b™DÚ·´ƒZQÄcòŠhrøݦ:2~7~èç´*Þ½ŸH¬ÝL¶KöÁU3"ÁÃeÀ¯ÛïÞÇ(ðC*ÀUé+1)¦ I0{“šV¦iÄPÒA¹$]~uåy+óŠ‘^5³fm¿ &˜’˜i”5Ô9©OáyNÀïŠCÕ¬ù»®êûÀ_|§|ÿ®éqœQ?¤¼¨ãņ?u<,˜§IÉVÇ.EÝ£6mC@¹éfF¸æþÎùe/á×£5ñ)‚Y;_A±¹!Ú³wègEÓŽo yK¡Ï bîéxg X®÷gÚ9m!q1Ö ÚÄ{´Q0¯sìl—&Á@o§-š w©¶”¯¥7É-l8¬tÞƒ~g«#ž0Œ(»vQ+úÍ Œ±µë¦¨,Aü ¾à,7}/±ê[ *–0æ[ݲ4G,X&Žc#1¿If%^°‡5À4%$š ã0´uù&žö[LÕ›f/Ó,K`Aö02›à1æöÉBÄ9ÉG · fñ‚Åf™¥(3Nýþ JŠ1 î§át¼™ïÌê 3áßø‰Á<Á÷å7îy‡ eãÎzxðít7èÝÍ™bÀãÈÇ8ivqaÄvÄÌ·u’ªÞÝú¶‡›÷} æßPK›¥º+Q$PK\ Q content.xmlí]Ý–Û8r¾ÏSàh“ÏÆ¢ð¿3ngæ¬s쉳íÙÌæf›„ÔS¤–¤úg®ö r™›äåöI‚R¤D©É¶Ô†ÏLw“H°P¨¯ ¨*|ûú~¢[š¤A½š`E y±D‹W“Ÿ?ý8µ'¯/ÿáÛx>ã•ë¬¢ùýYEóŶK7»é_{öòÞoø*Yö}Ô­‘ÊK‚UïÏÌk‹íã8®º òÉλKTUŸå×Bí»Õï’ £‰PÝÛYÝsC¯¢x¼l#«‡g¬Æ”ÞËW“‘v4 ³¼¸ªœúþåÃû+ï†.ÝMå`åi¥™m(“.ƒ°÷(°ºLëFAoV€º[S'f褸1Kè*N²j€æýA€½…T4ºÉ–a·ƒÒ²ê"ñýÖª¬;ÚŒ‰3&L¦·½ûݤ†N»Ói0&õûšðJ"ìl€ÕÔ©Ä cÕ p%‹ [çñ:òóqÈ HïW4  È y³‹ÚÄÑ ã#Yà±ð„H4,%WõI­‰ãé2eŒÌ&l¼ºZ×1/YÞ÷{LÒØŸ7ŸØX^šjY?|úã ʦù ÔŠ7 ª™\–zM.ÇÒYucÎô›éÜõèÔ§^˜^~›ãSuå×ÐïW“÷;N^tåFLJ0)«²©ùðjòÏî*NÿµQ/¿9AµGCýé‚FŒLÔ%ñÒj5VAæ10¸u“€s߬×Ø#ç½úŸ¢s?ÅY¼›bU§èÎ÷ôW÷OëÝêôèRúftù¨>%qàóסÝ0¼v½ÏÝ}Û®û$}ü1¡´ƒhBÑc{0ëš™Å}wÅ_Þ”?§š²üg­¿þ W/+úY(çÓ•»>w»é"ÙjºHÜÕMà•·Wnö¿˜æØ}7ñ'ås‹&Ó“j4Éš"è{_†ºéÍD¼5…;ŒšADÿBÔ¿|Ï®¨Ÿ“»¨pø #ªŠj`oY+òâ0fªøï<þ¯xîÒM>Ó„u“©©›Æša@c±îöòy†Õ³çü_Q¯\/ÈiÔ*î€ftq§7qüƒ|Ÿºa°`bø×ušó‡fÅ[ ‹·©¶ êÓ¢ õt‘ÄwÓ,n˜ôŸ»aÊJçñÅ2ˆª»¦bX&t¸¸_|6K/n¯\LÝi¯à±Ñ¸gè‘-E!gP@¬fI’¿¾*ÊGòÆõYEâW œ¶‹íÈ°]G´•sTþO÷²n7#4FBUtNŠ¾Xç‹wÌ™1&†7§ÖÜø²Ó@Ul½eŠYÍŽ§˜Œ½~fÜÞoÐÃÂtXrY 9%šº¢¼=%,6SÈiN ó+™†s]¬Ãé_ÿJ½ì.Ènâu?rr´JÓMÃÕ˜7¯&WË%͘B Þ›„ñðŽÉ¥VmrÕÆ£ÕOŒ»í'—l­\nš—,ôÇóç$õ¢hOªÀŽ4¶ÎÆØÒ[-ÆV,ÒäˆS·¶°*5­Ø3¬2Ú„Žîœ¦ÌÁO±Âótš£j˘Ä>*#-@ÈQöÚTˆz¢¶–k1#¾¦¨>ÑÁÿJctH˜‘–cNŽ0cÅk#ÚäG«â_¹MŽXZ‘Ö×—ÂF¬o 6Í´ÏÍüzúå ç«o8fûšsªö×S,ú|qX% 9p¡›Œ´&qZtÑbJ˜‘ ö¯PZhŠ…ɶ´0KÕNSZssŒ8R 7œ­2’Í~Z„ab”Xf${Vî5õØk2U?ѽ&òU˜óD1­C%ÉWáz )¦~ ]¤}|>ö1Wn-8ËÇ›üß–ÜÇ;ä~yû®Ð-¢8Yºa7àñ nY¡ëéÀh~0ZÞPU)ÞÑ{´?m)G@áÉ× ³Ÿ¶®}$9|7ï:ýÞ“»ÎUíMksÃðæýi³¥öe—öÙi09¢Ña3ó@>-¤ò–¢7>î}»yp¯dÛÇ…‹­-íò8 ¤Ÿ¶Ô»ñæÊÈç¼gÓ– û¸q'7O{iñÌÉßã‰X!¤·4,^}½Cš¡¼î³NNò˼h yN_Mþþ?ÿ] ‘ða xX ÝkVR-@šùâÐnIP²ÂUæ&WKÆQ%Ö)¬”F°ÇkÌ%kÚäÝÈ×çv}ét  Cº‚ôe×t'´$Âó£6…°Bž%…ô)d?K ƒQˆ(ú³¤9…´gIk8úÑéý”¡H°\ƒÏOG¹{Ÿ—ç³d~»Ü( D‘Zt|ž°l¯¤S¼î—¡ºbæQÚ¢UË|j…´Ç6:e(íu|_ÊÄqÅ~rù‡áp´]þ O.Lb8îÉ/Ð*–È·*‰²yÓlC‘ž¤uIßK$SÑŒ&°’»‰$" )Dî“$aB„„ 'z§4D-¼œ&õ«Q„ãPðÁÊ?¸A„~ˆ@¤~äêü / ”ÕÃ7íxòˆW͆xŒ6¹|û)üH<äÅË%c»tˆE‡Œ…„Æ.$Üð©ñÙ˶bo)†‚±#0‚£Øvƒ°„I “&%Lž Lš»`ò™pµ]2ÐÒÆAÃ7«UxÅyk™›Qô‚Ý:a@ŒÛ¸að,ÚÞPô÷¿ýï“Öîòïû¿—(g…ñbD x¹Ni’ã¥ëÁg¦GCfELœÇICX a¶3„Þ:¹-Âcî1ð‡Y =†ížüÈ¢¼œ4xƒT ÕE§ 슳°¨˜å·7ë˜Ü}¢˜ þ¦:ÉʦÝ0Dƒ7;: ˆ¦b„b Ll•ÿ5ÙÆ[VÆk™"XÏêÄë©Øô´öN0MQu{¶‰UÅ°k‹¶d(c$IœiU™X\.Õ ¥~"õ©ŸHýd<3ÞU|¨ž¼gN“}rɾrA3fõe:]ºâظN|u0 Y¢—D/‰^½N½ö[×” ˜Óë}çŽb)ö9ƒ(DHŒ'M0 bÄ$ˆI{œÄÞòˆ®ë¦G?ܘ\æáb=gUBâ˜Ä1‰cÇ$Ž +Aű0NÐ*‰çAHG€²âÉ#"™Ó9µøˆr¸êW;â#ˆboÆcã“cš¢OÎV€S>,ÁbÞ"]»¦cŒ"‚¦³½Q ±¨9‡2¼B¢¤DI‰’'ƒ’ ¯ÀLº["¸_Alk;Dð8‰'ú•qhîzàGóæã»±` «”Û^šBH‹~áè¢/•©X–H78aEZh{$öHìy؃»C§û˜h­"Ô²ìÝ"Ô1†7Ñ"¸²®§?ƒ#©Gó`±N8ï½Doß¿Cn²XóÑ ²[¹è–èlE¿· ¤)/³Ð%BJ„”)ò¬r¯ }L|·¦Ø¤%n£.I!ˆKQ*E©¥R”ž³(í66·dà–8ps#3¹ Õ7ñÀ¹ ÕÌaS‰°òI g例a‚û»ëSv¬ › ØuããXk£{d»c¾÷¬©èV‹i6ÌHÇi ãðay8àiŽ«–+,©Æs¿XŒïK”‘þ¹ aÅn¢%ÄhòÆb%¶È"vÚ3t”xFÜázp@£©¨Â/†F,l“¼Æ†1ˆ°¸]ö­Ö¨Õn‹az¥WaâÍFå2ÜÑd=DôJEkxŠ‘®:ì'Öu½%Ò‘@ñø0GÜ7ZÛÕ-'`ýêЬwöa»º5Ik)FÿM]MÁ¦ØzjôÞÐeº´EZš>¯ ]lZÌptS³4Û1ŒŠ ¥Î+u^©óžœÎûd€¨+6Æ" bS!š#""T!5D,ds+ ê€h´¢±uÖC„Mè•ŠCGiØa?Fâv@$Ùˆ_p]WL[o 5¬£Y±Žj*Ùžï 5Eä†ãmtû@÷YõÂŽÖ¤,Q1D4'$Ï®+׼䚗ĉÿç‚ÿÝ+#ݾ·}Ö¼ð¶4“ z]‚:V]‚Îà+^¡›Á°æÒˆ9¹|sÍD±ëeÈ Ý4í¢&w¿¾ øºÊA­ Š tE=&©½ƒZ3™ŸM•ÁWâH·Oœ³—ßLEÛ¶üu†ØNálqËŸéFûZ“ËO7tãIl2Yº_ØjOiùåÑÍèÚM^¢ü\Èè…`¨ÜŒÑ|ñ’gör£‡Mb¯Û {@LÜðÇÃ’™‚>Ý)bÿy!u£©NͨæI¼ä=P×¾[¡Ÿ¯"w“íèݧ‘‘n‡ý™ÅÔØ­5t]ÑuQ 0×u2M1ˆL"•2©”I¥ìY(e¤{Å Ïv•©¶¸Å噆!jjZCˆrO¹a7¬€Ìëpð\W¬üEñìbÉæ ›NMS¸ûk|žh¶O3× *ò’ƒÒpã& Ÿ#|Æ!z-õö0öùè¯kº~„÷Å#З֒ÙSÁ2˜„z õêŸÔžÍr‡èÿ-¾â«U“ÌM?waùÀ˜¡Z>rA˜„1 cÏÆʤ²C„cÿöÒ˜@¶1ÊFC³/°‹mE'Ï$žI<“xö,ðŒt‡Ó¹KÝjfµX¢öà;ã<ÖbŒåWx0ßf×ÝÒéÿ´×]CNþ+Ýôö]ÓxxpÊ}¹‰Z|òƒ„ûú=¼D É£f¬Î²:KØ©‘ïf.bȲö²uÂÁ Ä€]ôC±6 /¸â/`-R†Yð¨p—½š@·V¶ß°5 ¾¸€eŠP©HE@*ÏExÌúìcEèÀj@C/Ñ÷tE#ŸF^GBÏGë [92!Hã Ýnå¬;šs¿™®†¹­œ9¤¶=®¤æO¾âÃJÓ‰ÌcH“'?²iÒ[¢ee‘@(W!¸7Wî'Æ`‡·¬Ù¢sUh1×èÎ}€ýä°Í„§çíGpíë#Ù?,7¹5W4á8ѶQ±•ac§ó1I)#늩p ³U}ܯpå‹ÜüVqÄI~wx7ˆ‰9tMÃøŽ·™7Ò ^¯Sp¨äçwq¥É2…±a³dÍ„&S½ÏÜ›!ah<½ža™¸qúþƒÐy`&iŽ¡©8¶^DQÜiªB°ÜÑ—£Ô¥ÆxNã^9û˜Ð4CQÛRN8:$©ÕH.¯AÊ MŠR)J¥(•¢ôŒDi·E±38m£«–£_¿Ú%^1ÞŠ©nºH[bþ»\¼êxpÃüßW0>CŸÚ1l®¦´05Ó8S³\ó®ç†há°ÌŠ½8ò9Ó1# ¢è¨ dLJY`4›0«K˜•0+aVÂì׳v f÷¯·Â,Q°Ñ×5¬&ÌjCÃìŸÜ$p¯‡?k8 ýc ¯.CÐ0d¢¶È•w[v½Zšs¦(]_§Y­9#V— ú ZpÎ\ÆOÑbÓö|áòã&ü~%üJø•ðû5ÁoEÆå#í\¬•ùHEº iþ9 B6eÇÓK´G¯mæ”U›Rû½âl¦ìUYLE%µ…y]1tSª,Re‘*‹TY¾&•Å©‰×ý^@¶àðÓ¥¯@Bõºp5ñà¶Z¬¦ò>¾C¹غÄQîg¾*Ð:™Æâ¤tù‚r4‚q/Á¨nÝp]´¤€àõ'y²R×)Ö D ‡ëB¬mà»eqí£äì׺ƒÚNQ øÕZŽbMSˆ±ÉŸ'é×…U"€xÙ½Z–b³‚2!±%Üs«›\ÄXe]Aš¯‡dÄxj M5ØOL­%± ú¤Ͷ迕Ãÿ©h9ŸÉMžüò „Á ÅÈOVo#¸]»-R—´ …Ó1Z--4£½½RÖTé#ì˜#¹ið¿Z„•™Pv~c‚MoŽýªæQ›º"dïÜ;*X=„øX˜&ë ŸPß êT³MDˆ­N16þWõ™²eøI¨?d¢tBÄ©Olá<ËB‰Uˆà-ÙJ}ã°9Q¹tî–O„ ƒíèêmGztÜ3=ú^“­ÛÒ=Ö÷ÔP,S<¤|Ok+vµ¥/ 3i˜IÃLfça˜}ùm9¢Z,†æ(D®qIQ*E©¥g%J\ãÚ+C{ì¼×7^E~) ÖˆRe5|¬çPËYà›¯S´ R†¡Q¸ª–¶^–ûqù¶[•ˆ¶ç°Àµ¤îÀ;c'°^ì&šZM%šJ4•h*Ñ´g¼†ã4ÑtðŒöi°€íNO+çŠÂk%Ï°t#wÁä(ßïñ©ËçóCe~þéÝ/(ÿ.ÄY‚•œ/”2ÃTB©„R ¥J%”o˜>”JÃô,д–éWÇŠÅ %šJ4•h*ÑT¢i4%X«ËP›¸Œ•ÿgœ|f¥Þ GXü†+š øš§;ô…O¿Ii0kô·Ê›Àô®ü°_çÁb<*S¨¥°Ý6ºç.¡ï¹KèÕú:}H3ºMëÀê#´M!Ûyþ°˜jã>÷¾¬…$A¨.u¤Ú!Õ©v< µw;òö91§UˆZ–½Oˆ:ƒo/WY…ó¤³y’‚ášYÀ§©ˆh“Ëï!9ñHNûú¹>Y4†¥j-› Ñ62•uj!Jp§3£’åÛáz%wüíèf4†!DcX¬“›Ð1ˆÓ§iØ™òŒ¶Ø Ò36cûXóJt™Çõ£¨†Í²ÈÌã·o®Ðe#ÞqÒYÅ äëúVˆ4Š%ªKT—¨~V¨>ÎZ|‡m$;j¢ÃÅoÝtÜœ€ Ì”aÖßßOhšB’@tÅwA‰”£«uŸ¥tô±=¦¦7ÌÞæ¹=š3l*£âÜžFâ(|@¥~ʦ\~@ õË}ëÞÆ EïŠ3cr‡¼0¾ƒ³a±>-·;øÙ/ãùòI-³¶Ë^wŽÔM¤n"u©›|õºIßDŒ±êBTÃöºÉ[—‘ú4Us·j,Wù~gË—hRŽ¹u™øŸòzðyk¥ÓQíþ§ÁVq1\ðdþc‰¬Y%²Jdí¬:®ËP;C#ë?_ý€fèÊÓ?¸ ÄQ}>]g<~ø,ƒOÄi»Öü‚Õœ(p—E7Å· z¿¢I@!‹ 8ïñ³dŸ0º5pùŒÞmEuô&ÚÊóI%ÜJ¸•p+á¶Ü2j ´üà¼ÜÝìG7¹š• 7Nv7¹{çE×Jƒ(e²6 \~ž@þiÜ€-¾ ­¸C]ú’¡2-.<4÷‘¯úâT¾Ñö·ƒ¼š‚m{ñÞ¨§u¶j‘f¤\Ü—À+W¯Þ³^ü¸Ým¼}ð¡z]„Ö|©@„ªƒäó¦Xf=íEä²—ÕÊñ‡Ø_‡ý=xlbCbKx”ð(áQÂcOx„ƒ¼­†~{µX=]`¼f¢€ñmè¦y6°dñÓb½x¹d½‡ðøânÆ¡›Áë:}D£Û ‰#°G€ÑY ø;žÏ^À-öâÍëØŠco o›zùŽðåÿPK™Í[AÏJwPK\ QConfigurations2/toolpanel/PK\ QConfigurations2/menubar/PK\ QConfigurations2/popupmenu/PK\ QConfigurations2/floater/PK\ QConfigurations2/toolbar/PK\ QConfigurations2/images/Bitmaps/PK\ QConfigurations2/progressbar/PK\ Q'Configurations2/accelerator/current.xmlPKPK\ QConfigurations2/statusbar/PK\ QþðVŒŒThumbnails/thumbnail.png‰PNG  IHDRy ~)ÆPLTEOjOPjPTqTXvX\{\6f¤>m¨@n©lyi~‘p}…c„cf‰fiim‘mp–ptœtw wp®xz£z~©~s±uv³zyµ{|¸}_†·t‚Šx†Žq‡›u‹Ÿ|‹“v¡y¤}”©{µ}µŒ~¸…}œÄ€—€˜„”‡– ˆ—¡Š›¤Žž¨„œ±“ž¥¬…²…€·Žƒ½„€¹‹ˆ¶ˆ‹º‹†»”†º™ˆ½–Š¾› ªˆ ¶‹¤¹¨¾¿ ’¢¬™¦­›¨¯”¦°–¨³›ª³œ®¹ž°» ®¶¡¯¸¤³¼­±·¨¶¿´´´¼¼¼Œ§Ê‘«Á‘«Í–°Æ—°ÐžµÓœºØ£¶À¦¹Ä¨¶ÀªºÄ¬¾È¥ºÖ¤¿ÛŒÃŒ’Ã’–É–—˘˜Ë˜À£Á¦•Ã­—Ä°šÆµœÈµÈº¢Ë¾¤ÌÁ®À˦ÁܯÄЪÄݪÐɱÂͳÆѱÇÞµÈÔ¶ËÙ¸ËÖºÎÙ´ÖÕ¶×ؽÑÝ»ÚÞ¨Æà¬Éá²Ìã¹ÎãµÐæ¶Òè½Ñã¹Ôé¾Üá½ØëÄÄÄÂÆÊÃÈÎËËËÅÊÒÇÎØÈÎÕÉÐ×ËÑØÓÓÓÛÛÛÁÔãÇ×èÃÙåÅÜëÉÖæÉ×èÊÛêÅßðÐÙãÓÝëÄàèÉáîÇàðÎæôÖàìÙâîÞæðãããëëëàçñãêòéîôìðöôôôóõùöøúþþþÿÿÿêÂ÷CIDATxÚí]sÛÖ}ÏíÚmÅ]ÛKwj›¹Wû¤Ljj9¶.7É-ò•Ö2o# ´–㤷îBË6R$óÊÞ- H( ’‰H‚ÈÿrßïHü2g«ðxÀß÷ëûã}ßÓ[µ×Þzƒüù òi#Û¾­-âÿéé|3Ù>- ÛÁß:lѲ*• ìý|©=ˆr®PäŠUìî.ÈöÿçºmwD¶_s«:>®oþÔÕÞ¶pš÷gstƒË6~³îUÜ~×xÍAóbîï~úñÇ?ÿøç?ûøƒŸðÉ?~ðÓ?ùÙÜkÜÏæBh^1ÝðÞÔÈѱk£ü{c£ÑèØÜÔܲ{gk¯žvÕl†ä,Ë'gGSóóɤJͦØ;ÓÓKž9#˜f¹žP–MZ–ØÝr¢Ë饔îÞÙu‘K^àBî(f9ü »OJ4iÎ=×MŽ°ëømez¦c„Ƥÿå ßÂxäúøädäú$ÆoŠæýC/ò¹3?x{ØAþƒ‡æsï ;æÜùpä>h>.y/DÞ‰ŒOxi¦Ÿ19>~aüúøb9X·{ +¥ÃòøøðøõÈ/ÍÿBób<2¹y4ÐÜ®üð·Ï¡ ·Ö°Å/<ÈÞp#¼UA°* ¬Q^ÄÝr~ññòd°U}~ñâå‹—.]¤üÞp[Õ³__ÂÓKõðë§nnÓ\©Ü-T”B¾p·Ï ù…< …¢ò›J#UÇž$˜eôЪP@µÝP³‹Íx“èÚaßÈá4•J¡h‹J÷œ(•¢¢(8­ùi~zãæ­›·?ºu“n7¾¨£|~ Ná2\ƒÃÍÏwz¢¹YmYÅbE©(JQXÑ‚ü+Ú>ä µçSnEºìöaŸ=ìRÃi¦•ªf5ª­ašÛ–ÝTêíyr|bbZîíÅÜVõÙ£GÆ'ƯO^Ÿ8?1á"‡õž¨RÊ] jT­€µ,¯(ù¼µP(( w …¥y"2|nèÜð™ááÅfOÈ“‘ááóÃΞë±=ÛB¡V'¹RiE~4q!9a"2ùÈG3æFäÜ…‰‰EoO܇ÙùB¡{¥ Ü-(ùB¾¸ñ» üü5ÌAž˜ŒD~5ñîD+òu(ø¬‡ÞÞ3¸ß¶AQƒ]݉jO¸±@°eù‘ÿD{ÏGŸ6*ÒÃ/<È­\2œf«hUjКà`c¼¢ØŽ[Ñò4«,rÃKþðOn£úã•Ëž«ï_ºò…+Ó  ôV K±0³•…J2;¯ä}Ù½ÒgG²kôÀ«l_æc[²+m‰Ž²ý!?©…óg ëU(/*PÏ”hV ¶0%ï{ðÛ\²þ GС™Ö%Ëvªu…š° [~«®®8›Œ~Vq[UWˆsbòúË ]²M&Á~z̼R˜‚Û›Lb+˜áütTë kÅÀ_]¸Íà” AV)û;n·ºFæÙÑj5Dv°I¶-{±?¾Õ÷·~wö°7ÈÈv¡øºh®Øß²ÝqLÀQwzkÌþ1‰ÖÝíÎ6tÃj”SÓP¡G¤²JXݨØÀÛm|²bÛA–õί@Ó}¥FG¬hY FP´ºcºZhhͯ´¸äÁ¾7ìÁîÚ+Ý/éq£ñÿËop .çê“U}K÷þméÏUjæ\­ŸãFc¿Æöä¨IÃAÖI^ìÅム½J­‹¨º¹ëS|_¶J{Òòš®ï´ zjó½Ô.šœ›¿6ÿÉÔ½åfwûvrdìG£¿¬#Wk5£ãÞÈȵ÷ØȶKE–SK)AJ ôæJ˜ÅÕ ;³n¼Ìr©7ávµùª\ÛM9ÔÊÜfàèDsç 6_î ïh;zÙ±N·©7­4ººÆ˪”‘$I—äL“fO%M–xY—TQÍŒ, ˜T…+*Z±×[iΰ eÊ=Ѭ ‚P2E†‡“KÁÇ0Ò,wâ’õVE¡mãVQr¯Tº÷$P› •óàk ýU¢·M`Fª&§jÐejÜ.;£¾m2IÍòV3²½’‡“îå¬ š¢Pû=@ð1¥ÐÈm†ˆq&Fnr[34ÇÉ ;ÍÄ¢”)¬{zÛÛ¯Øv5Ä*Õ´öÙ¾ëõr6´ÖÖ6ºÝ°ç-,@É°I¡¹ô$ ܶ ãZ Õ1o-@+.ÐÔûz‡Ö^Î^š-j·pƒæbÙVÍMæh Ð.­þÀ“èÕml[[Úak' RQ±Â{O¨VhßT ëÚi÷ݸ÷rcÛ ›óçHÖ†a”6á¢sö`‡~“5¼ÊÓõW»évj€K m½‚ã7–}"ž`÷É1ìÓáI=j± ¶ƒ]©y¡ Á9«úõÔc¿Ûú–^uÉåÕ¿ÍÀ¶O5ëOŸæÿSÈ–5ànžäÂ@¤?«`õ‚ìmVv}@6À~ëµûØ]lÂ'ò-ëöíž-dBÎå–bŸÄß"vò vô•’ý=A9y!‡‡Ó ¹Š^•_gŸ‡l¹ÃAÓ\}Ò*ñËljLh—ÏŒÜ. ì=2%ŸÌ‡¦ŸÐŦS³¿¼÷“Ñä赫£WGSW—fgçÇ®ª äã¾-ë½! Ëèœ*ë’$K’¤É;2«/§%¾aÅüúh°¹ÝûÈ;"Û¦YÍè$MõG/ËÚš®I¦&kŽ£ªªëT=X=šU‘†0¨zܳû ÃrL†IHhù 'ò wZ¹­1¢$Šœ$q¼ r¢ AÐB'1š ¥:͹ ŠM‘öËN%&u¿-t¤B Óò"ëÀr‚¤áˆ"T4cÒ3kIê‰fGûDô«X ‡dÐ=¨6‹ùÖ– q‰a ÇNäbé̳Áˆg$W: ²‡9^hžö§ ûŒ­Xhá±lôÃ(úX(4y[ÓÊzYZS5m[S5±¬m[7ÊeM+‡ÕmǦäÑqIòð£X ¡j :&µu šzæB¥àÚ1|­ê{;Á¤X&:6!3þ.4´n[]‚ü*“+ôמ»åv‘ª’…PZ@u›:¤!µô¡¡þCx»@]#óJù¾åÊ•KWÞ§"—a»äÙ ¼ÿ40·ÝRF§,Ð'm4ÒX¨U‚Ѓž"jÃXÒytäj–óEôÏò:ky£MOÀ šOØž/¹/Ÿ¼‰ ½óÎpäÝHäüðùÈÙó×é–"?|ORG^<y÷ÝÀMœŸ¸p.2þλã‘wëÈ7O£ß¾ÔÌÓ³‘so¿}.2ôƒóg‡Î Mz\~O£÷ô O ùââ$Ý'ïC'|tÈõ> îINr¦ùôÖ›7?ºyó„›ÎÆÜ î,ÞøhÍ|>`™DíË—9 ·Uyų©ï<’}7š’ä—zÀv ¼Ýn[sŽw¹Q«­çÚfBy7'ä仧ìNâiQ–Ã3ïÕ´`zäG×®ÜñŽëºÅ¹k6FÄW:vS'M—õ5]÷Zo×!š}oäÚÈÕ«³¿œêä²:š;ŒeËÔdIYqR÷"8U.d7B¬ Ó¡jÝÔ½#ë]MÙjˈå².—5Ø[’o­Ôjac± RÄ"‚PÁøiÎdYÕ„Œ¦Ê@ôsw€ãE ÄôìÈÈÕ‘÷¦Ú>u%,·eI͈ª ¢« JF==Ь3žˆÃeb ©A³CqŠM./ I^XKfø4¿œ’––ay9%Èì § ÜÎp4€ôˆAmiU¼(©¢Ïk€vrrjvlnvê“ŸðŸDïFçF“S©¹±¹;£#©Ùé)gáׇ!¹Íó<Ç ´³t]ö•3H³£µ!wŽö]–I9#¨ %ì<Ë`†7Ú+ýs/²þJ ÝÒÿïyû0µ[…q£D‚y?žˆ‰ >qœùKGÓ$)˜áÚÍÑW­þÞî‡ÿ®‹ñÁA.—±1 ç8svÎ¥2MëE>”_ìzh¤>VWWTσú3ôʨ‘ñÃô%éBs/w¨k—Iz×1VCäíêË}Ïv4`kE@n¯d›ni/¶ô¯åÁÒܽn;gI×åe—Kª= ÿñé½ýÕÊþKx-˜fÇ®66öÞÈȵў@þû,>xØØüÌ4ߧלëøwù‹ä®ü9\Ò8¹H§X=|ôð¦ù!UêÎ ½=äÌòú}ˆv#(‡èïœqAý·i^O¸<9~4×6ƒhÞeYVàÓRš$ž×zC^œ‚©R3¹ø Ž<~æìÐÛÃC‘3oM„ ;4º”QeIG ¢&õHsäÌ™3CgÎF†þò¯æÅÅñ ã.„æ¶C3O¢ B2ÐçÇ$ÊôH³wž(E¾Bçv9s~x8âΘ ¯aš¡ eôBÓÔr8Í “üêÒeÜáÇ;8óê2^lܹøÔ|n‡I€¬>“@ÏA³'dsÇh—Ýrm'̾-›zF¡ŒŸÁAD·‘0äÃœ_ü(w`ôÚ‹½@©w‘94[2¢®J<éešÕNήv£¹M†ÕKáìêÀØØV··µÆF]à üÑçûÇac7+® èþÞ)ÅWÙ\0ÍMiŽO¥ææ“ÝîÐg"Jß™‡k>ƒ·gÑ’åù¥djÖ³Ž—¾XηùÔÜ™õLÞ2|ÈÂì|2•ZöXù«Þ¹PÒ|úÞüTªó¬¯4oI®›Á«|åªÁs¡6{ ¹‹EÏ©äRÙûpÙ/öo–3¥¹<“ˆ’û±4ÉDI4›agÖÚvÝ 2\4*Fqx›0 ™¦ÎÍaðjTža˜™ÛêíÛd& §FcîUäz9M]Åh§YÇ5Â@ßuÜE4Cª{甼jN¹Œ¦AÇ@hÖg}ukä”33Cb±(#’ˆ“aOøF9«H'£2\BH° Rn–³Js!Ÿ!Ìm3ò/Fåܤ5ØŒ.Q#¦³Œú&‡fMßF½^ÛA]W/KZf£Q°ð”jèÎì&üX—怑2Úž©ÔXN`ÒeJi^=¨#ßgÒ Ôª  ‰·ì¶ª¦b)ÆH’ \ï_ß îÃäUskk7³,àÔ5N“ø4ÏóòŽ¹e®â2‡Ù­-skÃŽ‰+¹áH‹¹cÂù TÁž¼€»xËÔ1ì»&¦Ù2_„®rX‚ðÍ_¾ùËŸ¿ùŸß|?ùæÏÛxÍ5îb´´WÚÛƒã÷{ÎýƒR=ì•6¿,5ÃË0]Ò¬žÁ·÷ûá¾fp5„Üö^‰æƒƒÓ|p¬’‡XcKÇ}Œˆ¶¼:ØTñï»ò*…(“ÔŸíì·zð¯v CëFˆé"Ùy¶šËå:ù­ærÏ©{këžuÇãK!6“ ä=ZÐ]¸nzntJê|‹–Bé¨v⺠½eçQ2Ö–æÙ¥7 ¹ö 4×7ký8nx‘Wª=#wn“êþ/¾ôXéê¢uÙL×u½ãXhîß~«Ö^‰fdÙ?üÈ!$š‰33BnÇðÀ„“±x†!\&™ƒ¦q(2Jfbšÿù¿è[ÞîЬ×9®3Ÿ9¯âˆ$¸€iÇ!EàÔ’ ©eÉjÇ]zˆj_ÚSÎ< ž8†MHåÛ:Ã$–!‹1Ǥ9‰ëÃã#\»Yo.–Êñ¢(p‚& ’À³ 9ú˜rœè‚êåÒG £ ³><#‰Ï2Ô{U\Ÿ‰d¾Ä¡[+÷LÌtêOTÃèacÅä'uG žxO¾t=„²|òOFóCõûhqýîpÛæõbë­ÖC/¶Þîf©7ÿCà ò_òÉþDç-pi§ï¶'éjqí ·¾²ýOîšhyµO½œÿy‹.rj¹ QÒU«¾çvÏîzÕ±ýúBŽê¤jk‚‹vd1º?_ Ìmh­§SBFh¢=†M!ÅgÒ2/ùì¢GMçO’aÓ|šW{ô¢öXÅéhttT÷/‡ÚTtÊÑEÇæçFç|F>á3:5–ºšœK7}¥ûñ¡)·¬%h5‚Z=ì‘掫XšT7‚¾Z ñ¡ÑÈýÈ„#é(!IœF¢ñã o˜„á†p hG!ždb \ðqÕ¥-„ }Ô6XP?„XŒ# ÎõÒ: Y/—Ëüí” ½Üðʯ(͆w1‰¡kÎ-Is¸4ë˜Â¤ï 8×2¡3øôÇoæâlœêd&Æ\‡æèN .‰ÌL\bn'’@Q"Mˆh 6¨…p&žˆA†üÏÔ’ÑìÁ{Ç%ÒØAó綄‹åm¹A3uàt ¹°‹õLÑ4Z·wQã0…¦nët½)\¥ ¢Á¾R͹¦ãàËÓ‚¯Uíx­ aóåª)2ˆåw[ )"°Çj9Å9¢}ÌMðTMÚ<îŽü˜Þïêá«/ktGÆÆæÆÆö¦±±½¾a[5ð¸n¨®YÍ9ÝܨÃÝ6JŽþPÂÇ×Õmu{]ÛÜ^‡·@T‡K/«oÆèZi–»ój?`fT×îz;<^eÔ+ˆ>â“üRz™õóš¬]âyöNz)•Ng’I>µ”žM±ì»‹ZóÑs:J6;ŸœÍÌ/GSw–æçÓIg8mõÛÞ8Æè|jvvìÚ½ÑÓÀö\Íšswp«äìü©LrþÎÜtrŽM~éÔz¿-/E“s™ÔüعٹùùÙd硲¾8F ïJ­¿AºžÿÇKsàˆ¥Æ?ø錔ÛÉ™hœãâÀ†ø8O¢Ðÿ’è‚IÜEæ‰1Ñû,‰Þ'D†T©(IÎà¨"#ÍiB€1l*Ê „L3p/F…›ÐÜÎеÿègk¦ $`·\õqGC×˪3œ‡zeCÅÜ.íÖ% ÃÔàIdgñÉÜ6Drx|;|q<ÆÆfˆA˹šÕèh$%|‘4|«!¦¹Í’˜w,=|f¸xœŽ_fºç¶·†ágjŽÃdYu)ë嬩ºÃ ÜvTi§Ѻ½iʺ(eI×%àÎr+\PÅ©îA4¯‡-ˆŸ5Ë„ùgFdã‰d‚ĸ4ù#­aß®r8C'ñÈ3B" |+A(¥mn”>í¦šÝêº">5)_›åÆs¹îKJóäÉA;¯òTòÝÇÕà”’ˆ¶7ñÏp"°m¸¼êˆÞ4èp!ÊÌhwÌcÎè=g•+ˆ8é6ö‚úíÓæÕ€röh¼ ¥×®U¼ ªmת®×¡óCýGë+P½±V¼A~ƒüyÐáò;WLðÇöIEND®B`‚PK\ Q styles.xmlí]“Û¶ñ½¿B£LúÆ£HIw'ÕçLSO:‰Žt¦Oˆ%Ø$Á¡ÓÿDûÿúKº‹”H%߇£Ä™±#ìØo,+½úî.KG·T”Œç7ãàb2Ñ<â1Ë×7ã_~þÁ»÷úO¯x’°ˆ.cm3šK¯”÷)-G09/—x3ÞŠ|ÉIÉÊeN2Z.e´äÍí¤¥‹½T[éµØÐé Ù-é:qsÉjøÎ Ù ²:qA¦îô„|W¦^½ˆg‘lŠ»”åŸnÆ)‹¥ïïv»‹Ýô‚‹µ, _A+‚£ ¯ØŠTaÅ‘OSŠ›•~pø7£’ ¥q]’òm¶¢b°hˆ$Z--ØE»¶;§a_·ëÁÖu»îs´!b°)䦩Lãá¦2ݹ‘›ý^ûo¨þzûcmW"ºâ6D V fSc»ó9ç©8A;»"7œLf¾þì`ïzÑw‚I*ô¨="iTIœgmB¼À Þ¢ÉW|g,Ì5àv ÉÙ`Ñ#î© ~'‡s_Ђ Y $ta—° ™¥Ý!¡u-â¸È™ú>Ày½[Fwߌ§A¿!,ö A…Ö‡¦($7ööN&>âTî ¦Qb]e ßæ±Öƒ ½+¨`"©š¶l¬àj+å',iÎ?g…Æ¡Àhj#EÅRë2œ{Y鱄Kgv#j•åT¶©ïç÷>Â<<!æ›}œD ¿¶§~ÂáÄOHD½˜Fiùú•ŽÖÕðHF"oÆ?2ýŠùÑ’ƒƒAHµ¨à8÷7ã?“‚—ÙÃÓƒãQciÄ÷Ö4®ÁñÏHÞÀ(˜Œ 4ÞÁ”møÃIƒ%“A´!âs÷ŽKÞ/± ã9ÈyC?’mû rpTÞ—’f_D“à,VÛ~ iº"ѧnÚqŸ…Æ¥Bs@_Jßå™f\§ç¯_a„NbAc¤>2YšÒø×pòë÷邪Ŭ,Rrï¹(#nòtÈ()Õd:p¦ò½o®ÂE%‡P˜Îçáå¢1#W^2 ’ &“oIí’¯q÷édbV\Ęe†?ˆÕ¿ƒò>^]„Ì^­¢Ëy ³Axu†/Ìì{÷±Zƒf4I.ã˸…Ñh}aFÿMÓ”ïúxm` a7¦óE »«ÙÕõdöäìfD|¢Âeö¯B ýQ0ÕúžßÁ„ÑdNFÓ‰‡œámc©àà&œ|îYðÃ}w68(@³z‹ü*´QA0 F‹IX1¿œ!øŒo¡¦¤”‚‚`EÊMS™9EJÞÀx§ÙætÔ€E I;ƒË2q»½”æk¼?‹«oë•%ɱb¡«ˆÓ„lSSÜ°Øo°Ábâ±Å5Ÿ½’/*$£¥’€a° {³Ë9±“€5DÊ„/w°”Ç }Ó͹‡Ÿk¢ "ˆÚ¬±•aîæ‘­äeAS®QIZlˆ=DŠmÉ­JgÔÚpÈ°¬ÀTZÃѼ• ä@PÛ²qü°<¦x¡À²BÅÝTy(!iI+‰@z’ãE‰çS7í:ÀÒ¶¤Þvä;OmnÄ%Å–6ˆêJè@œ ^²Ïg…Tc)øÝ–¬aã; D¢KzýåÃÁºܦH¾ŸÕ8¸¶ÅÑ;9šM,ìóÆBÌnð·w‡{â}2¥wìZaµî[A7lç ôwãZ? ƒ·Šq­_‹½%&"ïŽ×Ã:ƒ¤á‰q+.%V1Ú`)M¤b` Øzã@´6$V•Çu‚£=ÈK*½»¦4šÀûV åýz‚ÿ¨êj ÁÕë±!-¥·4ÕÈÞj gµi Ž«@®>j‡Eµ›ñÿþûŸÊàœE›Ss2–{)Y¤Òõ%0àw…8¬m Âñ·âéxHhjÆÙ\*~§GÈ!ü9¨£Á[Ñ„ j…p~š>š„mçg'¡Ù#Jèú,%44 …³³”Ðå£Ihz–ò¹z<ùœiœ¾~4 ÍÎ4N/QB秃ɣ‰hþÛ ÔØÜØü®‹Woõö†ÛÎÖ·s…0ƒöÑÕ·”æXuA1ê<°¡z¾®zÁ¨’Sú LU©±s.2’Ö«H×bý¾óœ’´³ßJ$±ºÙ9 EºÜ¾]o³ë79LŸæF©#|u´¦ óÆ ˆó@E‹Å×¥ƒ!q¶õ ]]/jëBMýžÖ,¤•>=*òÚÊq×{¡lö–ÒYÑ=‘Ë|µ¢›µ´Q1  •¨¿÷åló¿À‡?y`¹[Sî=|+ øçÛɸ§G¾¸7¬½Z(†Y”qCKoô· ë@PÛhmÁÎwx=G¹ÎoìÉÒoÿ ²×ÿPK÷o ÒÅ ÂLPK\ QMETA-INF/manifest.xml­SKjÃ0ÝçF{KmVEÄɢФPå±3 „>!¹}e'.%%¦ÙÍ0£÷™‡6»“5ÕBDG {å/¬Ò®Eêö¹ÿ¨ßØn»ÚXEØALr*ªòŽâµmX$Š%) Q&-jÎ(ÉŸûrdºv3kv6NnèåÔ¹L­JeûB'‡‘2Òuj3„‘i»ªn:4P—õp¾ è²1µWéÐ0qW×íТªÓÙCÔ÷õ(H©åã øÜ:ïƒòÔ‘‰%:,$Å‹‘;¼©xÃxh„”J´ñéÀÚQ¼>÷ÝQ‡}ããZ(­Á@i]:‡ð7çÿ¸LBJNEMQIQVJSXNX]PY^S]bV`fWbhZdj]hn^jpmmmbmsdpvfryhu|kx4e¤:i¦>l¨@n©Eq«Kv®Nx¯S|±mzp}…rˆ\ƒµuƒ‹x†Žy‡|Š“dŠ¹i»o’¾q“¿u–Áw˜Â{›ÃƒƒƒŒŒŒ€—ƒ“›“““›››‡— Šš£ŽŸ¨ ©‘£¬”¦°–¨²™«µ›­¸ž±»£££««« ³¾³³³»»»€ŸÆ‚ Æ†£É§Ç‰¥É¨Ë®Ñ“­Î—°Ð›³Ñ›¸ÖŸ½Ú ¯Â£¶Á¥¹Ä¨»Æ©½È£ºÖ¥¾Ù©½Ø­ÁͦÂÝ«ÄÝ°ÅѲÅÝ´ÉÕµÊÚ¼ÇÖ¾É׸ÍÛ»ÑݧÅàªÆà­Éâ³Ìâ¹Îã´Ðæ¶Òè¼Ñä¼Õé¾ØëÃÃÃËËËÓÓÓÖÙÝÛÛÛÁÏãÂÔåÂØæÄÝìÉÖçÈ×èËØçËØèÆßðÒÝêÈàíÇàðÎæôÕàíÙâîÞæðßèñãããâæìæêîëëëàçñãêòéîôìñöïóøóóóñôøõøúþþþÿÿÿRSub÷IDATxÚí]wÚV–Ï̤uOÒu½98x:NCZRl°t2{²iw×6ÓqŒa„GTÒªBvd¼²©f^°£#ýÕûž>@` „>°0<ñ!½÷ôñ~º÷¾û®î»º£Ìxº3`À€9sæ̘0`À€9sL©^³IüM]j‡°N¬rM°IÕ› jwEµ øøŽ;ã¸ò&‡þ8.Gä9î&¨hÍå-nIGö€·‚ðäql9šÛŒFã‹K ‰{ÅåÔš7Àµµéâßåö’ßosjVå¯ÿÚ’ƒ €Î;”rÏ÷ ›;;Dy{g_§€ÃÕz‹çyW(ØÊ€jèdÀû+jIj ˆ‚ìÊIímЗJ½ŽþÐo½uS8¡É ÔIÑ@‘I”|Ón¬çëKmö%V[PâYå|è«oNad&›Å0p«pš´«^Y`z[¾È€é`¼Ë½\]Ì8c]¯~úüKy¦¸\}5k Š"€(ÂÿÿûÚfÐn2(O-„Û¢žgT×öaúŠ´\F/ÒŽ$v‹DµUgô<æú‘Dó‘ÌEL‘ZõWã$¸Àf7z Í£ÕA ìPc¦Òâ3>¶TC+Ó‰€4ö\ãó}‘Wo;ŽÛïàEf1†ŸR$In࢘ÁàÅP N‘IŠXPˆ>àdEÇ0RÂáP-“Éà ûÃ1œdHÃ0I?zÆÖ \Ê€©@–åÏ×Uøòîn³TÞÉ…BÁL ´Ñ€³u~Þp¤‘üÛ‘ø£Äb|­Ü}@!ÞRhóW&hh n–ól(Ø3Á L 2on=J•ª‘^¿î®V ß\eCÀñÐâ/^ù  †€Öð©¡¿­®^x‡€‘ñ:÷ªðPÀè`1€Û¥8ˆá€ÐS€“`AKåöÊ'SãoHÐŽö+2#™†Án\eÝ$ ž®ÚÎPbH§È ³á‘€Ù¢1h4vF³“ ˜KÀ`5}58€µdÃnãKÖñó&KÁàG7(9t"‰,a )ÂÛŠ(Fah’¦‚ }åwöÉ¡¤W'‚g•L6Koù6«ÀM ˆxWÕœ4…(@dh‘QàW‚kŒ(HAIC—€ ˜¨pÀñçð6klÂ\¾wT­Ó ¼«Âv3FHÚ_€ÿÐiáY¶›3„…€Žj#¢ßçÐÃ?}¾î´ý}Hî_TÓö× ãyPµNFjÙ®Bårl®VG“€|d<!ü [™#U#‘•óá.2²Â{u‘¹"êÆRÿÏî*A|°Ü]6]1áEô9Iåù2»»ó}³YfË9V›0¡Þ¿?¿`5'©BŠm–÷šÜÞ&Ç6M¬¿¬ Â~šeË{,—ÛaË»ìV~3WB3&ŽQ; ?š¡ÐÔ]šºvÿÍxv“ÛJ}úi"H¤­Å£I}Ê r’[ðB£€ýH<Y‹§ÒÑýÅ”vöŠ6eæ… l'cñåÔýí­øý­hl±\ŒÝ#zŽ’íѤv Ž-6 ib»Ìmíc¶Â`DQb©{ÿSÅ2Á–óûpÎ@ /1;&79}HÙÌ—öž¢úœ¡—ÿÖé´ßë©Å_K?êñ{ñÝååXÞƒŸG–׶`ÞÏ«+ü»ããwæºõS!•ŽÅ¢ËÉd<–Ê N`2Àp•mÿbWC÷#kàŸ?>¼ºì¦qX óÒÂ^¦ÞNLzû½ø _Ú„ô'Nº×åQc°fœ”]¯÷ò´uõ·ë&×q'NFÖæ:Aè~eFžG¶B˜0zk©G³'¿y²L팮íÍ( ¦ÿky Lóœ¡ú@ûÃã*û§ëÃC&¯ú½È·”`ì‡&äJÚËax"Ç7Ž-™¡<\íir¯QþYó¹Àñ¹"wÆiÊõ•oTW  §1Ó[Óó<°€Õž"ÎH(”D(")Q–œõ]Ñh0¹I¬ÝM,=†jêR4ÿ$žêMš’_ PYÝ/´‹ê£ âP£€2D)‹ð¼@Ba¥%@[Ù_/~ª_úd´Ú½¼"“UpÃ3Oé§ÙìFØÒ€:àx¨žæ¹3á &þŒ;³™5¦ÏE‡ª,ÇqggV:_û][ap’dh £24Ná¬ÂîãY>Rƒ{CÆB±¼á`Ж­±«;e†{’|¶•H¦†ÙXcNaµX-Â_mc`¸x5ÚAÀ$Ïê‰Ô”(ì—rù¼ñø ÍcŠÁÿªÖªÕ£*ü­©¦°#¸ {ûÆ}™ËQˆ/d÷Ý ï OV#±¦uÙƒÊ=Š¼¸Èôwƒ^Ž.Ü€¶Ÿxˆ | 7c–ÖÆNwf¶]…Ë0°Àuð¬^dy†A8å‡YþOÏÇ¡6AÉW àщfkÿ¥cß~Ø©žAºÁà6í·E £Y!O[É€~!¨mÑ ÿR­Ö®·äÚYÏeG0†æFÇr¤7¬Ô`ÁNR °¬„Q4†ãj¢8©P0WÄa )JœÊ oš|áˆNLÏš,hÀôägVo¤yXë­¿÷dÕE>W ¡ ÿÐWDa@ ¹}¡2¤30´i.‰ÊˆjÐlGƒ4£>ùÉé¥ýøؽ%¶æÉ,®3Túa3áÚvQ„c0I¡U/4u® Ph”×EÖ-šÆ_Œ–í¯.µg»‘G‹‰»ê@ÐlûýòBmdiñîÒÞ½B÷éŒÖü—ïÑœ¡-N ~K‹jD uʌױMa$NaEÑ “H“8NÈ40œ¢©,,¢qÏ’.ef°iò±m§Ö1=Ú9\YWoãnI(éñj¦ç>+Ÿ£&9¡PÐ×+­¯¬"ž->ÿýÒVòQ4¾´»»ó>e&€nÐÆxl6 HÇ¿jŠ9Ï_Œè+ôÝ:?i<žˆ/,.Ä7ÕÇK z‡z‰#½”‘½óþÂGÉ»±ß'>‚Ð㫇õ#Û¨nž… MŠð¶#G0õµXŠú~+H"E2 ƒr 0`˜Cš10ø÷•—êÍ‘Gw–”‰ŒºÎÊ5º’¥\­i.…§pxR(ƒ¥î¬1¯2À¢iä GIÆŠbh‰Fxb‰Bf 6˜BQ"^–ƒ®E?ÒNLyº/+èºÖ]6zr^FÁšQ9c@ezþzBŽn*¯W`z3°Tª¼OŠ4¶óŸä²V5h#ìr³ìi7p²¿l8·–{µˆ"™%1 vùØ$0t½†õÓ¿þ'ŸNÎáU 3·¨¨ýý$ÐhÀ¦ýèiéx÷ßw› {€ÓZ/œþf¯èþ"œ JÐbp\K¨ýæÅ0ΕosUk¸ÍgÌ7 Jac ï¢ª’¿ï¹Cc‰g62xözT¨ý„Æ&8´–ĈpÜéÐì „BÈìßX?P¦@ô§¯W¾ ‡ “Ýç—7Å7'®ÌW_wÂÔ¹9&)X[ŸRb’½ÀäeÀ¡<@‘ã¸R‘ã¤OHˆ~îúϯV{^Ñšëù_,î­%ÿÀûE^œ¤¤€Û¯(¿>ì\c³‹©êÝäTCøõ¨pfŸ0 w1çïÜd@ãæ4Áà“dç×ëGõF¨š‡ä\«5joë—7!'Þþ“«×íMŒ&€ŸÓç§ïþ»"#)–Q1Å¡A3 Z» š}ÕÂHÀGÝ&¸=øQrú¤£«·‰\E¸UøLmÚð=’”4ebÀÿÀÊý´z“Úû—ö4²€8¾\©ŒxËÆm뻨À2”\ Í&±·Û$š…2—*ñe®LplˆPü§íÀ#+B.¾¶¼»œŠ~¶™L>_ŠE>KEÃ@ ÁÕEZe³½Aúχ€ í•!¦‹\ÀתÝåMµÚÛhO#µœ–¾4¼EwüÕóø[€âÅP: iv¹cÝÅ‹hV$rPe€êʨâÈQnk_¦›gT‡Ùÿó eý½"ô5‰ÈïZÍSO´MÛ꣋® eB莿€‡‘& @kÖ˜SÀŒPZÿúýLðáÁg‘Î/ºéÕaoýbV8à¸SË¥6#£Á"¡ ƒ¹°Y„&@2™ˆ§òŸ¥S3@Õxßp1¿M»ûDaF) ´FÑ @Æ-2ÈQׂ¶ØÀ%D¾%d¶TáéKsæ̘0`À€9sæÌhúyC¤r"6TLIEND®B`‚PKÇ`]Mmeta.xml“Qœ …ßû+ ÝWÔq”¨›ô¡éÃ6mÒÙ¤oØ)­‚Ü™þû"ŽSíNšy¾sÏár)Ï]¼rm„’À—T1!xÞ} sðX¿+ÕË‹ œ0E‡ŽKvÜ6“JC¦£ ZÕaˆl:nˆ¥Dõ\⤉7švέ¿*ðÃÚž@x:¢S)}„¸( èOg”Ñ+׺õ£·|t0GÎì˜ðÞP#»ŒÔknÜic}SÔ,k)¥®¡GÙÔ=F(…Óz¦M'Ú{G6¤ªëç¡]õ´‘¢»·ÌȾI}ÔŒµ·ÅeN ëzc›ðUðÓ{°ƒÿ߶¸Þö2‹Á‹A=OÙøué…jî³…Î×1ÂyˆQˆã]ŒÞœD(M²¢@YZŠ’Q²’ÆÅ.Æeo¢ Ú&Û ¢e!¾Øü­89\ÿ«ÓXA¿¯c7<ƒ´È3뮺oýøúPKŸq4Å/PKÇ`]MConfigurations2/floater/PKÇ`]MConfigurations2/menubar/PKÇ`]MConfigurations2/statusbar/PKÇ`]MConfigurations2/popupmenu/PKÇ`]MConfigurations2/images/Bitmaps/PKÇ`]M'Configurations2/accelerator/current.xmlPKPKÇ`]MConfigurations2/toolpanel/PKÇ`]MConfigurations2/progressbar/PKÇ`]MConfigurations2/toolbar/PKÇ`]M styles.xmlÝ[_oã6¿Oa¨hßI¶“}›w-8 énÛ{-h‰²¹K‰EÅñ~‰>öûõ“Ü ©?”-9rþn³ lVœræÇ™!g¤¼ÿþ.“[¦ .³+/< ¼ Ë"ól}åýúËré}ý·÷2IxÄ–±ŒÊ”ešz'X1á¬XZâ•Wªl)iÁ‹eFSV,u´”9Ëj¡¥Ë½4KÙ3ÙXqÃìJkv§Ç #oG–®Æ¯l˜]éXÑíXaäL]ñDŽ¾+I$‰dšSÍ÷´¸<û|åm´Î—¾¿Ýn϶³3©Ö~¸X,|CmŽ¾¼TÂpÅ‘ÏÃÅ ?< ýš7ešŽÕy]•²2]15ªéÁ®æŠÀæ¢_Ž›È•éø×íz´wÝ®`Ž6Tö3ÃÜu•Y<ÞUf±+›R½ØßKÿˆæŸ›ŸZ¿Réص·U¤x>ÚLËíÊK)UQÀ»Qwsß>;ÜÛ£ì[Å5S{t”=¢"j—ihÀúÀAØ-º|cwÊÅh«wÀIhÆGC¼®ªüA Ï}År©tH2>éÂ*Ó&elt*†SRkÖµŠã^VPgæCú€à%·œm¿ñ:§ÁqGXì9‚I­÷‰&7÷yšð×h µnŽ²D–Yl÷ÁÈîr¦8’¨0bËÎ în ù€)«óÏ™¡s(p&êLјÔ;”$-Ï @d¾t¤;Y«(fºoû~ù¯4‚'"äüjç"0õ®ëS?‘pâ'4b$f‘(®ßÛlÝ Oì3*yåýÄ!õã'i)µf…ÀÙ]yßÑ\ß㳃ޤ35ò“5ËÀj|%Sšu8r®#H·TqãþxÕ`Êd”nÈøÊýÈ>Ñÿ•Ç1sxF¨Tì ÍÒGé¤$Ír“Tˆ>ëvÈû":~PŒ €æ«? Õ¸½_¿Ç”¶úÌÔÄüß*ù¥äÖjˆ¹òŸòîÊ &ÁdLf‡ ¿ aL7Óà .<0áÇ] —.0à·ið[5»!ǼÈÝ‘=¶Éa8 '‹`Ú*q~1G-ðŒo¡Õ¤ÐJ~ãi±qÕùÀ3†šüã,îUY&zuíW,Òµ„ÔEÉÇ}$‚ek¼…‹w߶3kšaÉa‰YBKQU'õŽVΰV4ßðÈ«y«g’CödJs¨fÊÀH Øofó‹s:¯–M¸ åÝt‘DB¹ÜÂTDæöªšI‚Ï­R9UÔ,ÖYÊ0ùZjYä­á1“–•Š|Ck§ÌË,Ò¥ÉGfnpZžæxZ:\ñY)F¡ (p·uMÁûT$•1L/Ñ«Ž§ó,fxYÀ’ÎÌ‚Š˜Ò/¡¢` Xpô¨2/0†ÍjØÑ®kË‚¨b¹%fñ I­JÖQj(YÒ†^ð/@ŸÎsmÆÍÖ%]ÃËÌ@ǯV°å¿~<˜—ÀM‰fûÙ¶åÁ¹k»BqµHMû²©)Õj5ᇟ×Ä»¢`w÷¬Úpõ®ÛP7|å†ô¯Ÿ½v:±PoŒf ¤˜ªØ;=\œ„óHÁcÏ ¡-1lƒ(õî,›à@W(›Á³)ŠÒ#†wžÚE; àÊGÄ‘Ú'ŒqÝX0醂®M!ÔÈF*þEâU ‚•¯aë?•…æÉÎø`Ncl¡ˆT%œž£2a%µÆ ¥&X¢ûÅׇb7`CcSEñ8Æp NÓ䮋F—¸ë%Ö¶_ø¼ÀtNä]rć\6–°ÌdU ÁôÄqÜäxóhI æ+ïÏ?~oΙÄñ9#“òŒºJ³×`€?”~êÄQûöGØA8WRxcRS7ïÌÏÍã³ô¦ÀÁœdÅ ˆg5o¡Ù“!Z?sÍŸ¡Ë7‰Ðù“!4=›¿I„.ž ¡Ù›ÄçÝÓáóFóôå“!4£yzñ„½Í<OÑù_+Q;äªbó‡ ¯£Xª5l«ˆª„¨ë ¢;j«”îXS Tæ=°aV> ‚oͨAŽ›‚e½SÓUCîLª”Šv­ ,¶ï®2™±QH;KÉR£ŠMeçŒêz£d¹Þªëä.qȤwùÏ@gÂTòûîò| î=× /«ˆ«-ëâZˆ—°mÊv}\Lêé¶Õ^×óUñÇ4ÔÏÊéÌô‰z»7M—¥·|@Åýð>r=ÃP/·¦·8´|, .ÑiEí"á€t„¡Û:Ö‘ÅTÜ0šÜp Ý° ˜ÝЇ `1þÅÒ|C­£D’b‚³¤Ç)åmŸOö1T‰ M Nó랦—\}b‘Þr Ç‹m‹÷ö¾êö+UÍ'/ä ãñØöØYx>Ð! ÌŸžXó>áHïìÝñÞ™ ÛÁÖÙÌfòSÑ´©æ9á¬rÙ-/¸qégév=Àr8}°gø<¦ßç¶i"e´êÿ6ªã;TfÿŹð'hÕb¼Õ6tßêØi&8Ç$“”ž€(þßzÆȲ âÁ8`Nùkh¹’ñî55=~­«.$pœl6ÎÎßA¼œuCWjû¢®Ûæ?ÙL{Mþz<¡nè^ûëbj$\‹WM cy>?ɑѤð5l|iØ£ì…ÍñQd/"c·czúvLŸý,nnKAû6®³IE‹Å×µc‚Ü®²ß8  ¡gÖ#°›>;é0Ù0¿ö•côšóéœÓpv¦}9Àßû2±zįÍð³óˆÔ„ç5#P×A!ÛÙ¥ÿÜ^Ï×8&ö†íö@õ¨Y¯nvyÖ®;Ñ3ðtó.¤RQqÀE*Þþî~0¯(w!r öÏ›8?È“Õoñœ Å<ðAì_®~^íÖJªÆ{kApÇ*BJ m¼r×~z Fá¾QhŸÝ/5­ÕÍ÷œôVß1<¶R ³Fà¦JŠ#UÀÜj¸ü};,(•±ˆñÞgÓæ3·6]´.Ú:°ó½)q6×ù®wK¿ÿ÷Ý®ÿPK^´W&/ /7PKÇ`]M content.xmlí]m“ã6rþž_Ò%©»Ë‘"@‚'ÞÝr|ë*WÆWï^ÊÉ—+ŽDihS¢Ž¤æÅ"óÿòKÒE à‹DI”V3ƒuY#/„@?Ýîæ7ž1zÓ,J–ïFØ´F(\N’i´œ¿ýõË÷}xÿß$³Y4 o¦Éd½—¹1I–9üEÐz™Ý¥ïFëty“Y”Ý,ƒE˜Ýä“›d.ËV7ríñ¬âN–?ǽ›‹Êrë<|Êû6æu•¶Á]ÿ'‹Êrëi<ömÌëQå泤oã§,6f P}± ò¨6Š§8ZþúntŸç«›ñøññÑ|´Í$±ïûcQZ xRÕ[­ÓXÔšNÆaò‡eclâqYwæAßññºò–ëÅ]˜ö&MY]¥aUàçò…Ù¯#¹²¾æ½W×üƒÌ“û í½ÎDeu©ØÓþKÅžÊmA~ß1¿lü#Šo·ë*]ô}¯«j’F«Þ?³¨-·O’¤*oPlv1\bYθ¸–j?î¬þ˜Fy˜JÕ';«O‚xRQþn¤ Óî…éצ`õûšˆJ2ìl€­1¯S±Xª[àJç¶Î’õrZÌCAÀði¦/ bÑìFéAž­89¢Ë K=( …qɹªŸÔÚM’‹ 2lØdu#µV1/]<õëŽoÒd:«÷XcX“,³ó¶õðå§1/38ä¨mž$‰:dô¾”k >–«3oŒY0 i8‰³÷ßøTÝFÅ5÷»Ñm`'È‹>Kà"eUØšÏïFÿ¬’ì_kõŠ›#¤tÍëóp „V—&‹`©ÔXEùÀà!H#±úÆý‡]ÎzW¼Äàþþüçz7ͤ:=†”=gy¸8iLiMÅãÐ÷Aß“_»ÇÖ¬{‘1~Ÿ†aѤ¢SG0îÚ ›ûÁ:O8^L ÑOµGħ2Þé WÛŒs# «`.ýÜfÓyÚh:OƒÕ}4)o¯‚”+$âÂ(q$étTö»ib¬€„i…™ \–§É¯¡ñM¹|e™Å È\é¯a Ý‚·­aSZ¯ÈÖ^>‹bŽÊqrðï&³Ð›ÑM çH0êÀ¸OÒè·„³;#ˆ£9p¥_ÖYÍžëø¨'Ûj‹@0ÜÔâaÌÓäѸ£ù=0ÃYgP:KnѲºk™Ôöøð6÷7£¶M\Þ]S®øy²âÕ1­Ý¿KòœKU-Eq8 ^½$-Ÿ^ñ•%Íö®©'ç™zA´bîß–É2É«¡œ1Kü“沬ۜܙø×wΚ³â³ÃèbN—äî—p’?Fù}²ÎùàÜ-„²—ÎÛæÝèóó4GàI#Öß¾Ma ïØ[õ”½¥ÌGª_Ùêv.°º§Av?’oüÀD´ 9Íÿ WáttàŒ»âßy¥²—’U0‰r õO_—u k·ðNX–Ľ ÷ÜÌå}0…1Ëä9`áÑÆV Ó9t«¹WÅ0Þ>oœazZLt-§Ut 2³æž½0žu¤(èºÒűBÄáÿ½%!Â1‹5wvL]RB†ÀÖUAêÑ‹éC*>“uEó3ðfúŒ¶òŠñKãoĶ㠋Û!„9¸£wäWÚ‘Ô´ü«3-üÒŒøLæ¨7©]ù¾Û\À ØÅ!sÿâ-B6öüÉG˜„®ë'Ž,g²Ÿ|%È…½Ó H•3YÞ$ßk=_$¦Q4y›“[”>ùWŠzäÆóÃîãeÏ=0Úåâì,Ñeþ5îŠO©‡¹Ä\Â̾]K¾ø•gOÕÜ ÌÉ]RxrG¿Û°!ȃ=ãs*úaDÚ»Ó¤ZÑF”©Õ…Wýcg«\ñûâw ‹‚eGw+ŽÃ§¢´?‰ÚäQ$:7!Ê[›íz—ÄÓÞä)•¥íMêµ4®ÊEóþäm(lGnÓN³Êy7nCùºò T?¹W—&òZ¼E€01ZbEOr¡€ØMiÙå±k°£÷ª¼»ÿc—iCã¼$è9Ɔþ÷Uïõ³¬†ªõ¢÷<jË7@wÈßÞù7|óXø8‘Lè‡<÷D¹õ«­®òÌäD ;Ž‡ÞSò¥1#œ £ƒ¥TQq½œ†i-Ã’Ål½T^ø¥¡¸ì£@Ž#tCS;t¾Ø%ÖPÀN[bïZ©ÜÐÃúRùõòý†nw"o³ãmÖYy[{璘=‘±"Ž²M ±oñH®‡a¼yðÝ:ŽÃ…ü> qT\EOQðnôÿû?ÕäHHS$ÚpãeÜAÉö4»0,îf¥å"ø ÝççüÄ’댛—ÜeAÔÜ,ƒ<]‡õ5àЕa×/=€ä:d+yÎ’4,‰ðú(dF!l’WI!g@ ±WI!:…ˆé¼J ¹ƒQÈ~•ôñ†£Ï+åÓl0 9¯”OûRèuòin²ˆDôe1j©x#k;S[l î’ésuÁ…Ñrþþa5ä™, ûa!›ók\9Y?Wž"ÆÆ›!ËÃT¤À0Ê„(³`smIT˜¬3‡‘Ý«°Ù‘HQù,(ÞüÿißDSP<¦e5õ*žy†6ø“¬óÂE¥ d•‹·äÓJ¼òæ¯C·•ž9Fc›sÍÂ[7?!\Þó¬VSc&<ÌæY4àé™þ-n,d!‚]kóY&JÓ$•\5*÷ ¹¸tÐP WA~´HÃ|r¿J"XB9cUçÍ Ïj…Â’›†Aönôaf£3þ§ð³›?¯Bžþf½œi8Ƀå¼ò Y$Óh…)4¶·¿¨"ÃJ¤rû:óà§ÿ ÿ˜1t+þúè¿Äý‚ ·ð—Ôùyó—Wº­¾3Ñs"þ7úK¹j¿¯‹<>ÒbœUžFIºXÇß"£ñ®¸Þà-ôÇ,Zþþà ¾ü~1³þð‡Ýz'0ZôG»öØsmwwc»Þ˜»»üˇA»Û9õvy²êь֛ cC·Þ°ðØéÑÒký…@èÝÍXÛÜÛÊïæÞ†¸±v ÊÈ auOã *¾«$‹ŠÜPQ8ëÊ¥Ù#O8NKþ-—¥°ŸBãɉkîúÔQ<åÅîå ¿ç”2?Ý |–Âlã­¤“·ÚG2Pæ°Ý ”2Á@·½KžJ~º[y¤d¶ÂËfùq=JJ°†J •*_ TÒN6ïîåå(«N=…•cž AeæÌñED(ÿÈ]+Ñ·«U lSLí©ð¸K³u;‰æµMb­5ÍÖ¸a²f Xç(š­«’¸AíJÄàŠª ] ‡7 ”º¶á J|‹Ù-Š*åΙõÔ­ˆ`+"±vU…õ(¶Vaµ «qYãòkÄåž*¬£ðÖ»ê*l o=“ û)r>«6? ÒÌÄ>‘AšÉZ>i•fDÔð¥$Lªù™(·eóóV‘šn«Û[¨g0ÄøÃ=ßçàE„m„-#܆õPB¾ػʂÜ/Cö{\×É5Úk´×h¯Ñþ­¡=U˜+=’¹ªhßÆ\‡‡ûo{΂Iþ]ó€>¥á,æùvàǶiùöNä¯ÑHUÚ ß©x´ðt ð؆ „¿ÇØÇmˆÎE|ý§Ì‚0TZX¾éiÔÖ¨­Q[£ö›BmOAí :µ}•¹ºxhÔþa ÓÄÑo!ºMæóh9¿zÀvM,ÑE6S=Åx_ì’v­€í*·k®bM(÷¤›reK‚x†ˆŽe®ïr{¼CáÓ·ÚÜÊ\¸ßÇ©ìBæøŽâ‹WžÎW&¾b/!"³í´†°N Ó6 ûÆ[ Û†ˆ>,ËÀ˜Ù£ØÀžç„RG|k¡4”QeôtáÊë$øq–ö>¡5R¼ªQËUZ®Òr•–«^¼\Å:™§"W•TU¯VZ1ÝÃ[ýÁ­!Ÿ¢ñ7¡³Ós¹ð[Ã÷V®Vð5i Ò@ô¨§‚_§vyê1|ƒ»žéþK²ÐèSšðwñ]½’¯?ø Ô~ªhù6·+ë ¢N§]^ÖÛ·ab–t·C×ÄŒa‹?ß`ØG–A<Û° lùð§ýHÞAç>‘Þy(ƒ§á]û†w ïoÞk™2Žå®Í@1ïÜð.ð?; ®ÞÓNz°ŒÚ%²sg<¯?°WK»i^¶+Î-c¾ì‚×ù> R@>¹çæe¿ý”Þâ”Þ來†‡ÕXF¼?˜SÓ%Uê˜þÖr/ò8XöuMêHÇL:ï«K Äò™é»6ómâø®ŽÄÓÒˆ–F®XÙ‰¤ØJ=ÓR¡Ô-ß:V")¯¡Óz¦oUz5:º@.oÆÕVâ²_Ö‘…öÌ€òõ½šUˆâ‘¡£%±K I<›™¿Û Êj Þ‰×.#ø ÉÒ2vLì+:›»UsÙ?FÂÜ}OöÚ›b€`º§)ëjZ|j©FK5ZªÑR–jvÚþ±Iˆâ‘w¨mËrM­‘ ©ÝDÐ×êã‡eÃA—“†! ŒahĵZMˆ¸@š$,q2zöÐÒÒOëåÆ:…¾ýôÃKu†àö.QsT—§—ÔÙë™Ü!>Æá"\æÙYÎJ†¶ðxŠ(øÜ4ñ¨’ЭaãñvðàFž¡òÔ«Q¯øذצáZõ†k ×o®ÕŒ-øØ”-MçÅ:{® ³$~Ñ¿‡ÏçAës©v júõ$B¶šI€WéÎ"TIV=Óa)̓Á Lù˜çp ó|‘Hw&—M$„Õ̸Gjác’hÜ׸¯q_ãþÛÃ}5-0/A¿žøE:¡)úœù¥ßasL.!9ï@«P#‘ë´ŠÝÙ ÚE€®ôRŠAlÃ0 Ùàžçl`8 ")°êÜhû¦†Q@9Ýð´( E- hQൈ¸¯, †ÒãccéÛÏ`ý¡eï’%pÂõ$GŸùd¬ã0Eÿ±×áÕ›D&"E pM§%YV‚’ˆíAw2vW†®l:j¾"Ëà:"a#>÷\àw_³-âáCˆÝ´VíÚö;62s?Z._jzJ*ÇtpmñêȇkM´ S´˜òrÄ”½ì¾û•sLymŸšn¡ºìf÷œ×Ù=1%¤…Kî3¨ð{fz†´tbÞ|— Néö³˜Óö~†-*e›™F…ºÏ¿ŸR#ÔÅÕ©…⻤üTxû4 ÉrÚűËGßÊR Þ+xòx×Ú»”Hçoº’éˆeZ[Q@¤GT—)ê+´)¡Yó¡sÄ„1q\Nj˜U˜g`Ö&Œaƈy—2Ö5Zš ã] ¢Ž.MÈ&êDKDÚX£¥ -½)è¸s›£X¨z4ÓÊB‡7ÇÜ&st-¯þ8†ÿzŠ¤Æ¦½UùRó:j"ãvZ­/•(Ú´¾¸_–±]Ê)A -‚…˜åˆø&¬,­n˜ÜÊÒË óRU×WAb湊9‹‹Gv ¾Ó™|[!Ý#Bx—ˆ=@ÃóläR¸Âüm;©¢˜"×=!ÈwGÙì7“Ó³[¢l¨+«@Ï—í¤|! ÃépÓ“gpG;4’ý»³›LÝïD=šLª9™Ø&öä”÷\jáJƒ’éþ&Ùt½Êì˜\î;bƒi½µ·Õï?í£õRÓ#v]ëµOÖz¿]­PÔa?¥_19Á$:ðEë¾_O÷… ê2\×}j ®Ô}ÙˆÚš±‘ȧµW T@bÃ0¹ Cãñ™<¤ÀSb9ðѵ–c£SÒZàïد {£ öHM—ÊopLß–¥;~^!Z «Y³ ¼¿…~¸¿›mÈlG?îÕõ#Ȗ߇(äH‚â$Yý jVi2 ³¬(Û„¡õ2b~ëÝ!ZYNáèÿó{Ä1Jf¢ÉßÅ1çŸ8RUÒ8/ `)¨3A"X¥S¡™W/“33»ÁA˜ÌAl¸C½:q»eòîàîv"¹BÃœ‚M°C¸½ÍáŸaî5[%NæAñuKœÄ“_!_ˆRø<’Ô.°@xê~U€j”Tc«Ë]5¥ é©a•ôMŸ)‡¾éj«¤œ¶JÚ*©­’¯Ú*)1XÕ‡ô1YJo ë²YÖÀ¾`°ÃÛ,?ë,Ü'×^PÕW#žÉþˆçª>’%$Ï1uNVõ?¥Éb•£¿f]tÓ*ù×SÉùËú¨êbXJ([š×QÃ=ÓqœNº;ìP~]¶¾ü¿šI¼´m ÃDØçC³ ˜ × u|Û’'¹@y¯Gò¤½{²;Á²ƒ kÛ}ŽµÛ%HË–%HÛôká2ž µ©H-@¾²;á*AªyHŸ<t›«^’ ‰»‡ÁÚçˆG\/. A’¾sª‹du¹‹˜v›8ÞðÂb¾r\ä™DÕ†+ W®^\õ ™#jø<é>ßL¥ÞðqŽ«Ä q;lzz@«ÏaŽ¾TÇCß•ÇC×~:T³`lèõÕóžIÅc‹×¡ÝFT¹¿÷åóDVZ-Â.‚Au¹C’K1ˆÏ:’}Öë ùbn.HGjV¢ÒdKRžô·np:psZIÚ•“@öýra˜âhÛ<â>qùùË=_|k#·ÇD²‚±ˆ_ó4Î5å4ŽÓWŽ5æ8î/Àß8’»¦cKþ~ :,{tD@q-×_ÿÐRÊ&}¢²ûZ*TÄÚ椅x-Äk!þÍ ñjŽAÒ'Ç`û©¥"Å‹SKVã°ZŠ—Ž¤T)¾<ˆTŽ¤W1áC3 Ó2b»ó_ñͦn›r e/»&ýR5ãB·ÙÄët½2f÷}¡o×Ûz$&@,Æóáz®ˆ-²}ʉîRÃff®[==]Œì>‘Õ—sÀ—Î:6ö5ö¼…Ø®Iº°ìö4‚]$•¬#¨È}-ø Ûe 2˜y°Š}ìÁ߶5ìc†¼œõ¢9š” ÌIšl³Ò`»`ewÛGZS4ÙrnFWägfE*& Þð]X¸¾XÅx|k3„0ž°Ñ·N]¬Ã¾½ÂöÔL °|-*Ö—ÝÁž M»-OÕ²ì)ØNÇ2–(îÁ ¬eÏm¹ˆ £edƒâ[ÁáN/—wÁVÆì¡^“!Ï€˜¬mÚV mÚVð:l=MVÔ.O}MF“½ž'%Ãç0Ëø„^/AúüRß²È ¦Ä€{Úv­ñHã‘Æ£·†Gª»¤=Ôk›šìõ ›ãŠ>>EÇ'Õ¸”öeoˆçÂcÂR-*ð‰Šhíê·­º]ýïa3ÄF‰ˆk9–0 ð÷7 Ç¡åÓº3v*ΗÆ/Wb2¡Ô|áEp]Q 5Q8«Úòu݉$¯ ¢¨ªq€ªi¯&BàëiµÍѵ˜lùä…QþTwfZ‡5×Ïç‡ó“C!§îÑå奿] %(„":œpsÎ&­'ø„N³¢,GoÎB¼Š6– ‹…W«ŸÝåÿëÑ* ,+—[ñD¢æ1ûÉV„Ó +Œ»³ê¥ªýÚ/_µ¯6kÃEŽÆÚÖnиZgù§B5ÆGVƺƒ”µg óWïq’æý<ç;UÆM d(BgýR/B|I¹vgçWî.λ°{0ÑIàGgÕã£Üè¿S_Ï’àONONsÃßÎRä_VϳâWV(÷áüm.˜'oV<ýM.²(†y×ß’©´DOpÆ/, ýUR¿«VöØ‚ Á€p§¡eyà;Rðm¿‚OS–è]5à$ ÃR¼ö5zŸLáŽÈ)åª<ó»G9¤R`¦‡ñ_àéŽÄ‡¥³ôž¡rÓg‘º%Üg šlNoyo.²ØZ×’LxÒXúöØ{À £åDC_°El¶>ÐMC—x-V«¶«Ùå*c‹VûÑ# Œ{ÛøDÁÙçŠ §áfþ¹¨Ûiéã ©ÉEMOxOà¿-Å}ê^45^ÿã26ð^´»ó;ûcÄ@–oÌ]ý Ãðv œ ‰GÇòÜ €á¾/¸«bW¸~ ¦èªk¡‘ÛŒzOCxÑ7>M Âyhb‚öŒð)<ŠeP<Í€QT䣘'ïÁgËë­’°ñUk¹­ÿÿ´û¬ä´ÛÒ8&l "¤ª?:>·ÜÕÀ÷hÏ…ÞÁüoŸð|È{UÛC›¤/ýÇ=Ï’h!Kä¸ÆZ;Î`RX àøñ/¬ãG9ëèª8k447Ü$=©UQŽäÀ —œþ)DðÀBlÍ 7‹ë´×«z=y€½3³|É8ú½§EÓ C¶)×D“âÃf‹j\èÐøÂè±—Ò>øðK[ýÉU3"Á CùðëIëÃÇ…}Z»JcÑE¤¨ÄØN & ’`&%Û"0mEÓ¢”B“ª8u-|Åž˜k¢f…kñ-µÀ„DL£œNNKìÃ×M0¿#Šª'Ó»i!ÇÔ÷¿úRþÞZS×(!·_mp[GŒ  êh`™CIÉN7m…š¡nlÁ”rÓi²F¸áþÞùy/ã·#)ñ)‚~®¦v;rK´Wü!ŸÙ¦ß@ò¦BéGÜÓÑÞz,_δVZB<ábJ'hï©Œâu“co›:7 ørZ”1z‡jËÜ)#Ám|ëŽRºà1‡A¿/«;3 )ƒrí4¤¥è77.^¼…_3¶rñø‚³íWÄjHˆït¬V¹¡e;X„¹ŠÄ<" ‰Õ°e©éÈT’p6ˆ‚ ¬K7ö°ß"¨Þ6{ž†U ²‹Ø| Ž±µG"JI2LÌ+˜½ ™eæ¢L8íÙ‘Æt´Î¾·›> Ïð}ù)ýo3¡Ê8Ãd9• ½¿qa< }Œ[ƒ!!ÛÃÒoÿÔêÚÝù&„›ö]™ÆßPK†.òm#PKÇ`]MMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKÇ`]MŸ.Ä++mimetypePKÇ`]Maƒ€ããQThumbnails/thumbnail.pngPKÇ`]MŸq4Å/jmeta.xmlPKÇ`]MeConfigurations2/floater/PKÇ`]M›Configurations2/menubar/PKÇ`]MÑConfigurations2/statusbar/PKÇ`]M Configurations2/popupmenu/PKÇ`]MAConfigurations2/images/Bitmaps/PKÇ`]M'~Configurations2/accelerator/current.xmlPKÇ`]MÕConfigurations2/toolpanel/PKÇ`]M Configurations2/progressbar/PKÇ`]MGConfigurations2/toolbar/PKÇ`]M^´W&/ /7 }styles.xmlPKÇ`]Mi’o¸W‰" ä content.xmlPKÇ`]M†.òm# t9settings.xmlPKÇ`]M¿(ÖÞ³>META-INF/manifest.xmlPK6 @apache-buildstream-27ae392/doc/source/image-sources/arch-remote-execution.odg000066400000000000000000000423761514607367700273500ustar00rootroot00000000000000PKD\tMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPKD\tMmeta.xml“ÍŽ›0…÷} ÄÌÖø‚ŒÔEÕÅT­ÔŒÔ]DlOÆ-ØÈ6CúöRÒ‰ª,mçžãëëâñØ6Á›0VjU†8Ba Ó\ªC>o?MøX}(ôË‹d‚rÍúV(Záê`”*Kç£2좺¶ÒRU·ÂRǨî„Z$tMSo4ï©~•á«s…p†hˆ#mçyýé‚rvæºÞ4žâ ŠFLâÃ…Þjbב:#ìxZ;ß”Ûj¬5ëZZësèI67ÀG'%p^/´mes«ãĦÛnôÜ7=­•lo-3±ïR ç͵G3Çpìzíjð&Åp^ŒÁÿo›Ÿo{ƒÕà‘°Z¦lzŽªðÂŒðÙÀh(*‚ð`p¼Å9%}È"”§)Š³dSÀ+Š‚3º’b@Ðc'åQš§ &e\°ÙUpéÆxo|­êÛgŸÈ—øûÉåq)d¿Y#l…“ðÓþ „£Z›êIîøê/ÓEiDê»›t—&Á ØuFÿÌA„Ztÿ±— ädó·âìpþ®všKë$ ü¾ÞOÆÙé•+C’…°*àEóáµ_ýPK#)ñ‘Ç.PKD\tM settings.xmlÝZ]sâ6}ï¯Èx¶3»Ääc›Àv€„†Y’ÐÛnß„}5²ä‘äþ}¯ ¤,؄ȨÓi’Á–Î9ººÒýW_^"vô RQÁkÞÉqÙ;ˆòIÍÚ¥KïKý§+1Óª¡’¸.)Ї¨#œÎUuñºæ%’WQTU9‰@UuP1ðÕ´êúèjJ¶xòÂ(ªyS­ãªïÏf³ãÙÙ±ÿ¤R©øéÛÕÐX‚B(¢SÁû®ÏY§ Óɾ(‹Ñë󅯢̈́ÅÂRá§åò¹¿ø¼­"Êöå2cKˆbÔÐ C—y-}.—mW³Íåb‹–ûÑ%s {›øDÁ/çM!ç^Ýßûç²f§¥‡7¤&ÿ5]£!¨ÈG1ËÞƒsËë­’±éUk¹­ÿû¬ÛÒÔ{gÝûd³%ˆT¯þôÂrSÿߥ½zÓ¿]‹!ïTmmr¾Tô÷X:K¢…tÈq•všÀä°€ãû¿°ŽïnÖÑQiÒhhn¸Éyr‹¢¹Aw›þ)DôÀ3"ìbGƒ" 4Aðá䘉Y&$˜¿Ÿ+ãázŸ÷zÙÈ @ïßZ|I¬ÜL®›HöÉWS"Á ÃeÀÏgÍøi)ÀW+1²S…i‰$˜»É–ˆLÓtá(é i\š,;[y`'ꚨ©3M¡¥&“„i”Õ×Ù‰‘}½‰F¶Å¡*Úü~^[È C௾U¼»×Åä9ɨ.–ÜÖ1߃:ì[¦qDR²ÕÏ[¢îQ¹6aB¹éuY#Üðpçü¢—ôÛÁœ„Áœ¯‰ÝÎܸ;ôSÛ¬ä+HÞPè3½„:ÙY!ë šfOSˆ'\Œs‚& ž\”Óë;ç…I0pÓ4MÑÛTïJçŠܦ·°ápÒ—O9 ú½«~yÊ0  ÜÚi@è770^Äή‚„@BÁYnz_`Õw@T"aÈ·ziËÒ²>L-ÁFb~‘ÌBݲíÒÃ4e"I<í'QäêòM=í·„0ª7Í^¤•–‚ì`d6ÿŸÇ˜Û%s‘ä$u,ìJ˜å –˜e¢Ì8õû'(KŒ>èd3Í}oŸ¿OžáÛâë¼Å„rq–ûH7M‰zwOÅx‡¿0BFˆb¶#–½¯šO«që+~Þ—xêPKÜ\ »$PKD\tM content.xmlí]érã¸þŸ§`i³©™Í’â-ÒÛµgåðlmÔ&¶`”¸C IÉö¾D~æýò$i€‡@ñuy,/§Ê–tƒF£¿nļ»~ a…“Ô'ÑåD‘䉀#‡¸~4¿œüýî{Ñš\_ýîñ<ßÁ.q–!Ž2Ñ!QŸpGéE^{9Y&ÑA©Ÿ^D(ÄéEæ\G%×O}Áž•—¤ÙS0˜óÜ~̆2SÚ/ºþdFÌs» zÊLiA©<»G†2?¦èÐz£Ìßâ1𣗓E–ÅÓéÃÃô I$™OÛ¶§¬¶Ø©èâe0*×™âÓ‡¥SER¦%mˆ34T>JË‹-Ã{œ V ÊPcTã§@Ý¥†9¬!ž§f_«ù`ëZÍ;Ôì,P2ØÎqÝT4w¸©h.Ï¢lÑ1¾Öô=T²_ïoÖv•„CŸEikªr?ÜÍœšç'„T¢R†|²3qUYÖ§ùwŽú¡—ü!ñ3œpäN/¹ƒ§Ò8 Û”tÊ(D¼¢&_M"ªˆ´ƒAæÕqêv6ýÓû›[gC´&ö·‹~”f(Zk& ý`ð(m‡Ñ¢Èl ”¶1uj 7¦ ŽI’Uä xŠZéh‘…A· £µ%ékŽNRâz›-n8,'Mµ¬Íî>LiH!@­xꨓ«2®ÉýX:­ <ˆoD9Xt±¤Wïr|ªŠ…ü;•ûrrãØ1õ ·(/ R’ÂÔ|ºœüÅ$ýÓ]^8jMSzqŽ#P¸º„„(ªQÄ~æ¬Pâ3ë› šôÉF ŸC¸oñ/èË~q4DJŸÒ ‡É”ße¾GApœÝ²5iŸEÆïŒ;”ÆUR‚²ð3á[¼BšO·Ym>‹~~ ɇ囿þM¸ýFø€çË%Rv1MÚi—[)ÊÑ2#{‘µSùö»Ö77Vª‡}(2 1FsN5MÖyÒ`'(^øNY£„&w싘3ѨÄE‰;)Û-XÄ\2N2§ž—T@”.&|‘HKÀ2ýÿ¬Ê? ß°›EAðà»4À•%ÙPœ°Vå€@ñ™Éþí†(ùˆbì5³f”™§€¨£½Þóƒ jÛcÿŠ#ÇÏ@5òçE ÅÐ $ñ%œDøsÀ_–iæ{O›„+ªgMú²à‚Šµ8Oȃ¸Àþ|Ðå¡ …Z\„~T•Z’Ì”QÝÐ$ÃÖŠâ¹4M3Óþ)ÆFù=É2·TØËh…:Û¬Iò§WUù@. óº¡FÍZŸÕ©Ï`u‰p«áÈì7ì%m·l „,éL;ôX;M{¦Ì)æ…ãá™g|ÚY K¶l4§"ÙŠòŒÓ`‡±×ÏÌÚ‡ ZsXfênz1ÎL/kÖŽ–`îÞcrÿ v²?[eFÅÚÚu~Î_N¾J`JQ”uHä îw -na[uÏQó5•n3¨Ô8;wÙ™æn†cÐã‚$ðÓ:Áòz‘<ÿî°b¶DWŠ$ÛêsFWPõ£‘U¾ÌM¿ìb)ö3ºæþŒ:‡ÊKÜ@_?¹‹PäqÆŒ3f˜©ì‘•÷…âO?¹Î>³#7åD ÜÞ]NIà»Ã×.,ãó†˜AÚŽj8$—ÝêAnÕ2ô³r«‡¤w¿ESÙk5Ðvf¶Z‹~^ ¼Gj|:kiuÁ÷êL?‘!©XF œ~îe5Ko7$Ó<+C:dq`4¤ÁëÕ¸£ç_py†-³32½cú0URõÙYYß‹8ƒ“£å?6òf ±Ç‚ù©¡.!†í¶V;×UÃ|C eÚ—R|ÓCA;ˆßHbyñOøÜF&¹›Ú¶f»{j-çauuòÍ w(b‡RÿW莢ÆWF¹œø0᩵–ÅŽ'Aå«6D”ú(*[â+™yµe“\}ÞjIÐÞ6=àÇÎÖ«úîö+ö„áãÜH•÷œEšqÚÉÐHÖ^¸Qrhm,gS­uÕ€ÒÚá*jd(§RÑËèn#ŽÞ«»§µÚF”v>VÛôG½®°éˆk©M :çH¡OZå…eÔS/Í«zYTÅD±ë”A–ó¥É]VÅ~äBIÞÂJcèÄ.ø…ª«ßÙ©DJ‘$DÁº•,AQš¿9R¬L³v–)M6"zü‹ñJÊ’eEÂØÉ2£’WÁ:WÅz”-²œ/ʘD“({Š7hس;N[·xÌe³¯ùy¶)– Ê„‰R“s3²)Û+R?œÁxˆ=Eì­9^7•Üådo?_<ÐÁó!R]†–©…€ R9–Ö³ÂMé £ºeè aJÆa¼@ù`5†8Á½–‘!«¶i#(,tm­»øÞ,g8:@½ËS{““çF¶Bö—4÷ãvUÐh/MGË0òµ›ÞFxáí7ow t£MOÆÕ c¯Ý5}›¢Ûe—IÛ«ßMCßI}Ìûê¦åûéæ”`{×ÀÚd|¾\ÿ®±Ž7HÌ#GŒüÍc$»f½i[›WlëvMàŠ~ß/k!¯¤å0g'ù×¼J¤7\Nþ÷ßÿTók„›Œ‡î[èjª=WsH˜[Î÷Û %·O!xàA“µ>Mô<É™öõt=¨è!é;±÷œ.•ðú4¤MCŠ¤¾J éGÔõ*5dMCª¤¿J ™GÓö*õ3;ž~^©Ÿ¶Ž¦!ý•úiûˆz~šîFIEÆy9j®ºˆµ§·œ÷Ä}ª¾×™\½cK×ôR“|;Íéw¥:,ùTˆc÷¢Sš‹QB±¼gÈCË€&ÆŒÀY¦ ÍcÜlˆÝ’R«ºû±¬ нÝ> çÍ_áKºi—2×Wy˜šY>²è-»ƒ~¢ßM%,ä›K,ÆIJüi¾Ë£‚YÜ6yn]²+·Nv£bdñÜ¢!Éš¾…Ý\?žfò쀚¶…}ÖÉ®I†Zp3«‰Ëfp´ ·á¹â“gÉã ×º}M YU1åâw9ò~’„;†Z-EðÕåIÈzeŒ2º2‘àÌY0ñE:P²U5Þ$xªä{ FéåäÚÓ„kO‡~Ê{kò5„^ÆY®“¡h^z ‰ë{>N€Yɪ$+²®êºªÚZAQ©$f×A¾‡†g †ŸàçÚ³„öi ÿdå¹rnào¥ÔÔOÅ'%º©þ¶ƒBú/á‡Òñ¿—ùò 7ó¼ê•4Èç—¢þ öVe“á÷²ðEêGo®=øãMìOK~û¶¿u³VøBSLm:ƒiÕϬm2Óݬ?^{ªÐϧoòe$Àfl²±}2q;£¹É˜o› àœµöÝÏfµup+—Ý!æVF¥a;¹fxF°n7(rþ7ø‘ÔÏUáúý„¢6} ma·+¾.¹…ÅGðÒ—tÿ±£=æõùL¦èÖîÊ < Á½:Ù$j'h[ÄGÙYš©$¶¤¨V töþN!{î=y,k\ð‚¨åJ„Q³þN™\}À!É°ðÝ#v–l`Sœ¬¨  §Œ%èÇ¥–ÖOž®U4~µN­é[µ¦J†­ljM‘4ͨÁ¯)óZƒøJÏáwÄ ƒ´uFÜqgÄs½ӃìÖ\”ûni¤õo;;WUUû+»°ðxõ7|§€À‡zÈÉÁpì GZ4:µhnU½l¡ Þy–W´ºÞkMH†`ß(’1“g¶©©æÌÖí·£ Š:Þƒ'@þ¤AËo»ôÙ¯:mr/ÓÅÞÈM¢ ƒ$mê2;Õ5kWGÎ2Ywa<*4ŒQì*öyRhœ£kW鬑4{mf*GRˆFßsd“…Ú¶R+^§¨ló®˜wîš\å yb-L—÷:H(ØðÈ… Mš‘©Ì)|L1­ëp›ùÍ:õim5?H« ³9Se…_…0$MágjLo¦ê<~÷Õ9U¤huª¨òû}St6³ZBE{Vóf†¦:üJeÙ\¨Ø¢{‹RŒ*¢®û/AiWÓŠŽ¡èŠŽ¡è‹Eé©›6mwz莠  TIÖù@“TËä#€ÚšÄO+ÑN%%½†XãÅ,¥X*5 uÝ‘>9Ï%$e];FH§ ùä1[¦þL1‚:ÆcŒ0ÆcŒp†1Ây šYñ§^g9zœ¢Iö:,i §êŸ*žÒ$‹[’h‹§ê¯*ž*ºö‰ã©¾%>¥ûdÌák|¦¤¬#úö5>SÊ—– ÑjK|Z­xëŸÎö-ñ™T(ÝÔ(ð+†¦ˆª.Ð?øìZôSVý”î=㪪oÝOÖ[âUUæ·?g’\;ƒcBýl§u?EÙ¿‚Þ‚f…7+™G+?!ûOf¿†€/éÛ£¯Ù+Ý›ÇÊö ŽõåG”Ö¨tÕ®¾­Šú°éã8´E/úÛÐV@æûkjk~Ô½W4 ?R¥õ9¶î%TÓ¨ë+?ØpXzô K‡Bú¿Êá¤]9‡4ÿÆA)˸¤ø©C÷cÎ5æ\cÎ5æ\/:çÚêþ»#`EæÏ ”ó¥þmWd0T½¬ê¨À!®û´`(%Ëè¸P„._ ^€æðÁaõ#PŒ@1Å«Šîã=4…X…^ŠA'v[€B­eõ…€æÚÔA;,¸:×ÜsŒlôá£}øèÃÏÛ‡w)¬ûpµæ÷/; ö7øq¢}v¾]5ƒ}Ãv¿ô·ÿ•Ñ™Î|tæ£3ÑÎü$/¶ž ß|©°|½:Ïtì ¿¦.ú O¶­Ñs4¼uF êŒPg«mïÀÔòUÒµ£ïx4Ò•ãŸ^³F×ŸØ !o„¼òFÈ;CÈë?{Ò}¨¨'Aa‡´ÐÌÚ©"Sš)õSÚ5­Ò3f~¥¤vÔD¯ó§GøÇ*rƒ+ì8j¢( ¡`‚T+˲3Sm9\³T°lyûá’gÓc™Âí©ÇôeÁ³Wâaêš(4ç.)¢Eìú¡iír¢ªš8K ¶sF|^ýPKò:„B›PKD\tMConfigurations2/menubar/PKD\tMConfigurations2/toolpanel/PKD\tMConfigurations2/popupmenu/PKD\tMConfigurations2/progressbar/PKD\tM'Configurations2/accelerator/current.xmlPKPKD\tMConfigurations2/images/Bitmaps/PKD\tMConfigurations2/toolbar/PKD\tMConfigurations2/floater/PKD\tMConfigurations2/statusbar/PKD\tMÌ;)´´Thumbnails/thumbnail.png‰PNG  IHDR±M“0úPLTE%%%999$3E8?HQR=EFHIS[VMDYVFVWVS_bXaK[kW\r]YcgWfu^tca\[aiMcgWgtZt{Xjjjbmvgydjsxpoqsxgxxy;j§@n©Gt¬My°S|²gx‡c{–u|„ ´w¬|¶|&¦y$º~u‚^kglˆrwˆhvr}m{“v¹‚(½ˆ4¾„?¿’7ÀŠ7‘X´m€l‚–z„ŠzŠ–}œƒ}‘›jˆ®b‹»h‘¿uŒ¡{’¥`¸­m”ÁwœÆ}¢ÊLÊGÇšXÆZɘVË¢jÊ“vΗcΧdЪnÓ°qÔ±shƒ{x’l~€‹hˆ|„“m…™w’ƒu‘šo’q‹¡|”¥|¡¢®{¥±}…††„”„›ƒ‡“š‘†”‰ƒ”ŽŠœ…›“‹’’’‘•›“™š›–‘œœœˆ˜¥†ž´’œ¢’ªœž£žŸ¨Š¥ƒŠ¬‘Ž±””«…™³Šš¹•Œ ­Š£·˜£©—©µ”¾¡›±¼£–Š£™ Ÿ §¡›¦¹‰£¼’´¥š§§§¥¬³¬³®§³º³«¤¹²«¼¼¼ŸÆˆ¥ÉŠ­Ñ”­Éš´Å”´Ô§¹Æ¥»Öµ¼Ã…Ñ›ŸÃŸƒÑ ˆÙºšÖ£”Û¹©Å™±Â‹²Ëš»Óœ©Ì¤¥×¥§Ü´°Ï¡»Å¿¹Ù¨”ÞÄàɬĄ́Ä׶ÃɵÈÙ¼ÒÛ«Èá¯ÐçµÎä·Óç¦ãέåÒ»æÌ·èÕǹ­Å½µÑ½¨Á֚ɺÁÞ¬ÃݸÔȺÁà­Æâ´ÃÃÃÂÆÊÁÌÇÃÊÍÉÅÃËÉÃÊÊËÆÍÓÏÐÏÇÔÚÓÌÇ×ÒÌÔÓÓÐ×ÞÖØÔÕÛÞÛÖÐÜÚÕÙÙÙÀÏâÆØçÃßðÓÜåÄçËÃìÝÓéÉÒìÒËäëÎæôÎðäÚâêÚèóÖòèæÙÌäÜÔëãÚãðÙñéÛúòÞçççäëóèóéëôöòíå÷ñæôñëøðåùôìüøíôôôóöùðúöõùûùöóüùôþþþÿÿÿÐDÿuIDATxÚí]}Xšç¹7íÙÚ³lmºžSššÌžmvD›®K‚ñ˜xLÚ¦výÈ4ÒKsõHHmÕ‚‰%à ”à×Ë‹­"fqÎ$5´($a[ÊŽo"¸f8¬^u<Ù‘j¢Ë…èÉzA0Ï?çýÄÓ Êͼ¼¢Ïïýýîû~¾ |Ë-& @€(Q¢Dˆ @€(Q¢Dˆ À»cµÃ`}DïÖoïFÛlº[»uŠ6I{¿x²Z­8èÝq©ÜÙû¹š{ƒóuÁ0óS]ü>÷2mÄ¢Î$Aâ²Âé*ˆ~‹TE«bF.7³¥‘Zrƒù'‰Ztpº®Ï6,8|Y4I·’úzˆvƒ9Y<Á о¨;\+ªÕ^­íß«Û§/ì3ëDjÑ}•^.þì°Zt½E.Sgµýö[Úaó uÀ:Ù?`3kûçfû'ÍÀ|ÅlÕ™û&§'Í“Ó€uPk±ëÌ@;F@ÿÄŠE7gÖÙçtfÛ‹íÊÐM ¡u:`Ö‹52°j\Äœc Zè¡Àj …ÀY 2¬# «…a6£¥L >P)bˆ@ÇâuV‹=ÐÚþžøk¥ˆ‹ÃóȽ¼§MöÀ¨mC Å‚û¨óg¦ñ× ¯ú«8q†I–pÀŠý÷ãé[2wdîX` »m4¬Ç*3Qs«ÌÌLOÃjûÌá €¶€™§M¾mG&Þþ-~ª[ÖãqÄÆ` á¸ÉŸmQ üH0ú«nNÇÞ‚Â< l9ï€ÌZvù­>÷ÄÂðÀ‡pkÌX*ó‚³aCÎkÞ|Ü@‡çjøUçІ1t. m–Òè\f*“žJãe0½ È· ™M‰‰´nfÚF:Íè@\¸K¿ò£[ kh‡$´Ôb&wã<]Œ ~½ŒÙ²¶ŽvÞ KK€äbÀ˜Â ènRŠn‰¾KÔE0ï¦.O¥P(|¥DÁ À.,´  8ÎxZ„H༫ec Å!#q½½@Ð@!mÁž•Fwtá. ÄÕ-ÙHán£Çsã%i¼x“—˜0u“è;ãé~bF “tΑL„»y%~§ž“–F)bVt¥.d@17¾1^Ä­ØJ‘Ó„ž²EJÀZ#‘HÆ$¨”Ÿ©$Ç›wÏ ”HTÝr¥Š/W6é=ö8ïÛº3`·#$Ƙ+ðŽz<þÑUÞy€ÃˆøÇ¥ÝQ5CjSÉQcj ýœéÐyoÀdÊã›R+)ü4ŽÉX¢rc@8w†NLü¹TB‰/éU¤ÅÓÏUžóö©Ý4nZš<­Ž¾-ÃÄmr÷a<`-œ÷•D®âëUJ=ß8Ækr1ÀQÏ'ïêîRè"•IÄ‹¤¾@ú9¿ýÝÝ(vø­6>劥a €½}hÉô•åâ L`&Aå»ŧ!ì%°§@ƒÏ7aJoƪü 5'Ø#" €éô§ÓQÛ’îeO5õÍOùªMß°e„»œÿ»Áp•©:l0¨ÐV ø¡kÐ~Xa8LT¨æË3ó+Çà˜Úÿ¯’Ì@꿶(Ü%€'+5ñK x Â_<´2@µ1 0ÇbÀº˜€ (Œ ÆbÀÎ8§†!É-¤l-è‰L Ìç0Ò€MÔb hó¹¹ú@ðÄE÷À€ÈÀ=0þ¶3 ì;C@ Iý3àf4˜á/ú@ ˜`€7€¿éúÑ4HÚ1FþÈܰІþœzpà¯(b­§¸ ÆÐÃ` -ü‘4ñÊ€z¨¬lÆ÷„½>+â>@ ûàŸÙ4œ°ÔÏ‚ÊÁ°—€ug°L~î+î™FkÏÞR¿?©Q`€Ñ3@\6¾°ò–@hG=„v–Q±CBL@ PøƒÅE3hmýöq`àµE¤!4 €ƒï,Ü 0U°—‡1`²`¯(R%€ÙÙ@CåÉwmU……Ý^x;Tx¬ˆæzÜÒOöûyŸYé]J 0ÈîŽ:ˆVJ')_ô-‘sî °X0ö¿;æî%0 jý$šƒæ„s] éï@ã=ªt4+eŒHgE³Ð: IíuÒJ»¸GÔPÿ¬èlf2OAÂ9ØêèãÔ‡F³Ñ%`#¹1ŠùN¬ˆŒÜî0Á(¦0V<à·2œg† ˜1 \æ–In € V ø­$E„PŠÐaD„P˜1 D²¦›ÿ¨á?Ž?Þàç†Ö¶Ø–__½ñÚbvòö2H`îøù-é¶å–Àï7g½º˜½¸ùZ(À—ùÚ6´øZ'ë½r*s¹—Dýþ™3Hï¢ÖúÜW¡èÊ4eë'|¿1"~™Dû{{_|=„ À“߆ÆÅ/?féW}¿ñºðR+ÑB ‚2õ(qЫ‘u:ðÒÛúJ¨%ÐØ8àºO, -h‰­Þ”K&ç¤ÈMÊÝž”‚ç±êß¼j  ˜ð·\ÒU¹ãêò:Á—ÚâúcÍŒM:ñDÎÚê_'“ó¨ùlr @R<$@R™ŒL*㘢¬Ùd0T\9oU—=sH¤1Àq‘1 Ô 5¿;R}êt{uù©£å.9 Hï«¡Àì&øó¦ï?YºŠY²2¾qŒòð­än|˜ûÃ5k˜+‰Mt„~›àdqJ@ã"B`ô^rãÒè ­ I9·¿*Ž³jç#+n¼öÐw²Ÿ,ݸ*ûû?–×åb€¹èÁ‡fÑ÷hº˜Ás(ú{8}¤½ÝãDh% #À`PªFU(á ¥Ñd0¨ÆŒ¨ *£Òh$ ïh*z0&&æÁ Ç¬#ÁY=ú®1+â~Òæl)’“\M.'oJNI9•ò(59iÊvWX ˜|o ó<±#ãñ:µ´Çí¦öH}ßtæàL‹µöç­ˆ3 >‘sâ5?ø¹&¹z{2™œ—|t“K­Ë<šAaRê×(\û#\Öú¸•¤Ð ‰9yˆ¶Þ# ö¶³Û;ÛËOTí`O².v,›p^ìïeÿÅø½Œ‡¿cün©7˜4û %÷ìˆÃ5Ζj°KN$˜93¤å‹ûóc;ŸX¹mÕÎï¿SúðN“{†à/Õg ¡yâæÛ‘öN¤ýí#²ŽN ‚ôº2¡ÖeJ„\Ns}˜ÿ3ŽŒ>SáIáYœø·s…s`V§›ÁÑ$N'ýA2;†m‡í`x}=}kz~nñ¥Vwwÿ6«<éÑvÞöÕ¬E‡ µˆ^ Sao P;ñ¾êÛ[¦î‡ú¤=ûAå»°VØ'­“¾/­“Š÷ùé5ª¾‘X ÷‹³g¾Ö „Úcõýâ¾2TØÑTä¿RØkÉÉÏû0)v;âÌ°ÊJxG“ÿ½ò.ÔB”?q„ ÆåBž°ª¬PÜmÔª!C æ2¦‰%eµ~¦‘y€¾*03xÛ … ¬ ‚zʸ4†;óÉ  éD°"“É5ðʬPJ÷è5h¼Ûáƒc:cæŠEܯ–Þ‚û'ûÿ&Uºá¾qÝçjé•ÿÓ‹ûê! À}_ˆû¬:1dûZ¨Õ èz>‡EWµ'ŸGÜR~ëu“ÀsŸ†Z£ *?×ÜqŠx<—àgHÈ æîâ#ö&œ™yáE§ØqŸGdĘiÞ=s槯…~H¬9]åŸøó{ļ6 Í:[msOœÍÏ<Ò×Órˆ×~ü n?Æ fÿþŒ—Ýõõ¸a¢y½/{ ½a…¸'œ¿5\7$ÖOŠ¥BHø™X_É€®ˆ¿dßí¢Ž]ðX-„ÅÒ˯ ÷CPý,\ÿ õÀý•Hëªz¡ûÂ,ûmܬ·f˜¹íi!wM†Ùl3öûmôÏaÏ36ôf·ÙÑ'ü»ÍÿÒ{°ôšv¹ò,OPöOáEu±*L•õPɸùøÄ!1÷¦Äó/Õ áʺž‚+ðƒºJ3ôAågXZYý :蟺o#K°)ñ¸N7'žÒMˆ»u_÷÷k†áñ~Ý@ü¥è†ÅW‡ÄRõ¨w 0®° gu£jéÀt¿N79(Õ ka} éuÃò`sNw´lð)wËœ öÍi§—ô© ö`¸ý‹“YÏýã·Ïü^Nþ+{þ±-›ÿñ ^N>ó¿hù£ï¡óصAe\Ì×ûsõæ{¸*ÃX: ÞÝñ^û{šh°Qd½XÁЗ2¼ øÃÙÉëK‘€ßÝÁža`‹ŸK|o°€a¨ î!é4 ‰ l-l†‚ó_=#[|ÌÜ™I¼øÚR$`½»m¸gA2¬‚™·ä••uòJÁ[RÅ1®` r´,8 ØÛ<ÛÚÎ>í™%¸0ø$þ´Aå1 âøC°óÁK`X8%@]ß yH×#ÖŽëÄÂañ8·?&:ÉN#w¬­fu°°:òOIJYíלùmp HM|ˆ†ãfÊîâKyF®<ž×Í3:OàD,¶àøà NŽ)âêäŸÅæP·“s“;ÉI¬•!îBhûEpÔÏà"@šÉxˆÄüH•V´†Öâž!îX扑EœàõV7kM>’’—_SžŸ_òëäÜ|DãÞcè RÄxæ pé¼C%<:ŸÇ­àƒ‘Àr~œž{"¤™0"ŠzÝf’.¹ÆÎñ‘\JðôþzÄ~g†–7 \|Á]ø‘¬Sã6†ÖÛÛ¡qö¦Û^XBoÐïêéÁýàúoðvç——WwPO°Y姒XGóËÙù2d?»šZÎ>Z}¤šÚAÅcƒìâR:Cn¹<%ç—'á ‡ÝIÓ¡ûÉ€«Ã(^Ɔ —0™ÿî‡ÏV¯ÊMIÞ$K®Že¯>B&WA’Äæ5¿ƒÊÞÄfUç³ÊQ÷—4¶œZý36›…yDœ K“€ãƒu3„L>]ž!/)Ñ»èüø4.]5–Jç)›2¨¾Ñ(p-‹ÈuÔÙ½ÍÚžD~duJùöäØœØ)Éy«Y¬¤”ܵoˆg½ÿvIPaÌWV¤(”¦’ÒJz6½Y™V²†yn,µD´Ñ˜ðýO„°8ŸÏÚ_Í^»=?ï@y;›…º¿S¬dj2‹Ê¢¦äW;æƒì y8Á1f7GÄçsä"¾ˆ/çvIärŽí(wêK¿1 ¸§Â/#Îå#È¥S'|¤÷‰„çÓô"ó^áå_.) NPåããÄГJ£cŠì›H„Üœ ý¥{ç>â6 Y°¼ùøÚR$p¼Ñÿøüª9£¿!¡û%`y¾ Ùóª& -Y½­YšÖ—em{dm/·Éö´É^nÓdµbåÕW/½ø‚=È!1¢78½¾y‘,…Á¸¥|£QË…¾º}ò"ø-Ÿ€‹h9yû:V®ßÆ øä"V.~ ®Y‚ttåG7dîÞµ{7~Çmv¸kþŮݙ þþÉ‘û@ÈG…]Ÿ!b»ÐÒŒÞÙ¸ß7¼?NpYæœCò¶« ¡·Æ¢4ollh$/>ò@%pñõ7<í?½^¿ñUhpH`|Czã"¶ëéÝË.ë¿Øœµˆíùé+·C/[B÷7æ}BµA±ÜØLDz«Ìwn\ñ yñ¥P2€X›ð‘óãÁÆ1q@µÞ¶¼øäyÄ5Þå,Äc¥8†Àó!Àc^€Âmé–(DF%•?&R(&¥\¡TˆT $ü9’{µWZ&4hkg²^ ¡ˆ©± Žî°ˆÖD_C™ [)pjCjA…©¤‹^D—— Õ·îy¬Ê£²ÙTB•½Me±‹CðÇP.•µ3À±+ÖÀú»„ð›§Fï2HÀÉ€´?Wtï,¡óK·§R„Mi¨+Ü(Ï T8…€3ÀZo ½´ƒ^p6;ÑÑÎîEÊ/¹^‡t¥(ä6"d2+Syt OÎc2™0/m+³~=ƒ»SäbÀLýã+\†Œ_ó i`±eŒ•›³?emJN‹¼ýßR’rócSXHï²IÀéõ]>§Å”Nö ¬Óª½¶ÄÕ«Ïjëïå6‚ïCøI+ÿÞÔNÎËý0™LîøÞögÉä¤òw7Q5˶TÖ{j¬»ÉÏÔ˜^ó¸ŸtòžlEÌëÌ?wJa×Ô°;N·×Yí¬òò£5ìÎV‡3Z x~Ï i£„©drxãÂÅÒ悇:ýc¤ìDÛÙ"P»Ì®,ˆßÄjzx%öÕ§®MSÄÔ/±ö‡xåœüX®(àÉúÎ’lzs*—îQ`Aâç¾…äW&ð– wâöNÜLª&FXQƒÔ"«¦væF:d²YÒY#ë”u8²¡Öe€g"$âó¸ü´"Žè¼ÇŽÊ`6Nþ*~]¼ý_³³•—k97Ê^7ô~ " ùí|HÞ”üäê3²'ر«-§nJNb‘sŽ."¤>@;ï'<úÂFaׂ.q0£Âo™n$â‰Á›eЪ©Ä%e Î=Cäý±žM&'­>-Ë{66ùY$¶œLÎ''Ÿ ò¤Ðïò’À˜Ê+8÷NÃñ4¨?˜M*+ nû6RÁÒ@³@”ìç/¡èìDõ€ OH»cÛDkÈ·Î: ’ÓÅäð <SÄáq[ètŽ’ÙT¡9fÆü²­³mŽ0H,¿„+_p»ä˜ü@B¼qRë΀¢ÒRz ?‘O¯H=»ê|Z]xh[ZZ SE @¨ÌmÏû0Ž…Û2Èúâ{³w‹ÁôL9_d¤ëù]"_)á‹ä NS“H.Éø ÚP}åæëYÎ0è>Õƒ¸o#Á^~#”ƒ¢¸‹¾ú´Ñ+œpÔµ;|ûs8T\{î 2ßð³ yõ¹ë¡ ƒj<7'4ª±s»üÌŒÙCµo|ú\Ö™E¬mÏæ»ô @1qùÆ6 ¶- èxDŸ0KpœjôÝN3i8t^àú뛳×ï~E’ŸDÈèw¿¸–pù¾I¿@÷ò!æus Ü°Y¦ ٠̬L›±íÑh±šíV3¸UuVG^‡+´˜ëAØ€ò\kžÞ÷ù­l Aa!Ó€8Û¢#YÑbÙ;h& ‚BÀbà…´$POù²È @$ðm±(Q¢Dˆ @€(Q¢Dˆ @€oý?YLD³gIEND®B`‚PKD\tM styles.xmlÝ[ÛrÛ8}߯`ijö ¦¨›%mì©Ý¤R{‰³[ã™}‚HPB,´lÿÄ>îÿí—l7@R DÊ”ãKÆI*¶Ð@÷A£nRï~ºM„wÃTÎez1Ά¥¡Œxº¾üúËG2ütù‡w2ŽyÈ–‘ ‹„¥šäúN°ÜƒÎi¾´Â‹A¡Ò¥¤9Ï—)MX¾ÔáRf,­:-]ôÒLe[Ì`}»°Û[³[Ý·3b}éªÿÌìöŽÝöíŒXàÔí˾osAbIB™dTó=-nO¿^ 6ZgKßßn·gÛñ™Tk?X,¾‘Ö ‡5.+”0¨(ô™`8Yîg_a¦i_ý몔ÉŠ©ÞÔPMV5S,˜‹~Ùo ·OÿnÖ½½ëfÝAs¸¡ª·ŸpÓUÆQWGnß„êMÇúÎý+šÿ®>íüJ%}çBlƒªPñ¬·™íö—RÖªb»Ùº£ápâÛÏz{¾U\3åÀãðŠ°f\&m¤.ðAØ º|mwÂEo«Ûá$4彩Gì«*$¿Ó©¯X&•® ‰û]˜eT‡ŒNDwÈ@i]«(j…‚:cÂl^rÃÙö‡Aã48î‹=G0¡õ¡.äÆÞ£‚¡˜zû‚kì µ®²Xid×ÁÈn3¦8Š¨0Ý–ÜÕòC–çŸ3BãPàLT‘¢6©u)I’žÂ‘ÙÒé݈Zy>ÖmË÷ËÏ>ÊžˆóËyœ‹ÀhpYú±„?¦!# E~ùÎFëºÙ³ŸQÉ‹Á'¡ßï]Ó6„Ô çîbðGšÉüO{8Û8ðC#ž¬Y VÃÆW2¡i‘qBh¼¡Šßðû«CƽtCàK(÷}¡ÿ.Žsæ`z¨”ßåš%ߤ“’<2Óy©+~íÖíû":~TŒu戞SƒOrõ÷ÝД®¡O·[í_„ŸÏRK»,ïÿþïú½÷3[‚ªN-»:<™¶~WX)ÛmnqùÃ7\~ÔW¦<ó»5èÏJÉ­ÕÏ¿ÈÛ‹ÁÐz£¡7Úv W´ `ãf4¼Ç‰ øÛhø[(ÓÞÀc„Ï3AïˆòjÄÁìA0¼`:ohŒ½`²D̽ù‚Lgç!ðËÄ`¡×„̧^0ž‘Ù°s2™yÁ, Á9þðp€é؛̎éÿW*bT¿¶ã‹u«ˆô¬%shü›6I¼¿š"“ åt3Ýé‘k%¿ÂÒÑ|ã*ó‘§ uøí,jÕ!^C^&€Š…µÞRçCîG"XºÆ«p°8ÿq7²¦)&Ÿ¦±Þ ‹i!Ê<µòÇÒÍ׊f*lù™dpŽ2¥9äµhi`(…„Tæ‡ñd6¥“rÚ˜ QKÎG‹8„Ã$–Ë- Edf“–Tü¼S*£ŠšÉSÄZæEkxĤ…R‘mhµ¥²" uaN&36l9ždx+²rHöY)F!!ý9òélFžF Á(Îf2ý˜ŠœÕŒÀM ˜“YŽ»µ[÷ŽÊ˜TäŒ@ÊÉ-1“—tiU°†R]g3Ðiä9¿ùh’iÓ&hº.èšXjB¸miëúëõÁ¸.Æ4Ý?\w»ÂØJËI*Ùý¦’”³U‚÷ŸçÄÔ@°Ûf­Q­óÖÒ ßŸ¹ýíó`·> ‡¯Æõ~K3î–ˆªhpúžpv=Œ#î>Ùò÷æ0LnÃ@WÈ’ëŽg#ìz(^q+mÀ•tGi[gܼµÝÛÙHе)ì'²‘ŠßK¼yÃŽäkXú/E®y|g|0£VÌìT%MQG°’ZcBÚ&ŠûÅ×Gb`C#“4ó( à48?s¦Ém“¦ð®UXÙ>â_ðS(\ÉraòiaÁdUÁ´g…Øn¹ùhEë#ƒÿý÷?µÃ9ƒ8>gú$<%‚®@R¯õ ð»ÂO8*ß¾†¼¾KVR ú„¦fÜ™LÍ©â³ôFßÀƒ9ÈŠÅR±Š„·ÇÐøÉ ¬Ÿ¿9†&OÈÐüM24}2†Fg“7ÉÐìÉ¿I~ΟŽŸ7§çOÆÐäÆéÅ2ô6ãt0|2Š¦¿¯@íˆËŒÍïJ¼ŽV`nÈvÖh°Í"Ê¢l¬2ˆf«ÍRšmu‚R6›òÀ†ÙþÁpø£i5ÌqS@°Ð95…?D§R%TìFÑ Òbû¨2•)ëÅ´3•,4ªXgvŽÈ¨®7Jë )KKî‡ }—ía:*&“ßw—ç{ž±çZÁ¼¬Ø¶Ê²&¯U‚x`q˦lÕÇå¤n[®u5^¹ÿ˜†ü™@:š:Qkõ¦®²´>.8  D?þ±A5BW¹¹’ïxì(ùXB]¡Ù¤¥´É„CÒ@³tìD/Pep î4¹Ft]C:Ì®å݆w@Œ±$ÛPë¨;I1ÁYÜâ”ò¦Í'Ûe اøõ@ÑK®¾°Po¹†ãÅ>;h­}U5Vªê7œÈAÅã[ËcgÁ´£B64Z*`õ##µ³óãµ3»m;KgcÉOeÓ†šç¤³Œe7<çÆ¥Ÿ¥ÚõËáôÁšáó˜þoØú¥Ù)½Uÿ§QÌ úø3•ø?Ž…?A«]#î·Ê†æ£;Œ‡cx©ô°·]ñw·Ó3î, ÍƔ߇–+ݽ¦¦Ç¯uå…dÉÉVaá<æ¬óÄËY×u¥¶Oãšeþ“Í´×äï×Èò†æµ¿J¦zÒÁµxÕÀÐÇ‘'““M ^æ·VC{”½ð±ÙÙ‹Hßå¾£g?‹ëÛÒp÷4®±q†‹Å÷µ}6¹e¿®F`}|Òa²a4zí+G/êK6'£.:GAŽNÛG¯ª^¤f™¤ªk¬¤ˆN§î™ÂÊwKݤ¥ÂÄ5lòð[MÍ õûõ0ù~’»íý’Ö~o¥8k[…tèEÊu™Ÿ÷]£Q[ ð÷^ž,?â+eø-ƒT‚Šç5#×A"ÛX¥] -˜#®×qLì5Û(h=€ªV3_I\0;›6ÚëRtp61Kb¤â»ï á$å.GŽ%Ýze²üÖÁj^ÜÆ'ÄæZáÊåZIáÉø`2èw.Y)Hh®[Þí^£° )ì>»ïcZ«ë·6ä;}û`l$à§JŠ#ˆ2`p5(ßKJi,r¼÷š¼yÏm/v>ºó`ç­Râ,®óîñ—~û÷/ÿPKPß”Ÿ 9PKD\tMMETA-INF/manifest.xml­SKjÃ0ÝçF{KmVEÄɢФPå±3 „>!¹}e'.%%¦ÙÍ0£÷™‡6»“5ÕBDG {å/¬Ò®Eêö¹ÿ¨ßØn»ÚXEØALr*ªòŽâµmX$Š%) Q&-jÎ(ÉŸûrdºv3kv6NnèåÔ¹L­JeûB'‡‘2Òuj3„‘i»ªn:4P—õp¾ è²1µWéÐ0qW×íТªÓÙCÔ÷õ(H©åã øÜ:ïƒòÔ‘‰%:,$Å‹‘;¼©xÃxh„”J´ñéÀÚQ¼>÷ÝQ‡}ããZ(­Á@i]:‡ð7çÿ¸Ll¨BoªDq«Iu­Nx¯T}²mzp}…\ƒµr€‡u‚Šx†Žy‡{Š’d‰¹t•Àw˜Â}œÄ„„„‹‹‹€˜ƒ’›’’’›››‡– Š›¤Žž¨ ©‘¢¬•¦°–¨²™«µœ®¸ž°»¤¤¤««« ³¾³³³»»»€ŸÆ…¤Êˆ¥ÉŒ¨Ì”­Îœ´Ò£¶Á¥¹Ä¨»Çª¾É¥ºÖ¥¾Ú©½×­ÁͦÂݯÄЬÂÛ±ÅÒ²ÅÝ´ÉÕ¶ÊÙ¸ÍÚ»ÑÞ§Åà¨Æà­Èá³ÌãºÎãµÑæ¼Òâ¹Ôé½ØëÃÃÃËËËÁÎÝÓÓÓÛÛÛÃÔåÇÖèÂØåÃÜëÉÖæÌØçÌØèÆßðÒÜêÉàîÆàðÎæôÖàìÚâîÝåðãããìììàçðâéòèíôíñöóóóñôøöøúþþþÿÿÿvÛ¶¦‹IDATxÚí]sÚFîõ®é$î'™\Í´êÈ£DØ€ÚI & çŒïŒ-g•]„ `*P¤ÿÿÞ lƒ‘`W?0×óX>Þ·oß·«ÝÕÎê¦/n°Ý`»Áöÿ„Í2V›uƒ-H2–Å)BDW +£aYÅD':6ŒŒªëX7TCçµQ rdD°›ÍÊ©9„t‚Tɺ ðp«q°Šˆ oœØ,ÈGuàÿ©€Q%ê•Ó?åĦ:1ŒÑ]×áI§w÷•û†~q€~ñ6Ä xt´»3þÒù³ß3:Ôð®¶oâÂM?­w}Ó‘wb\T‰üðgœÓJ £5†îK?uÎ?¶<« Ný±ý{.§9¨oÀ?s$¥Š¸ƒhC‚=m¹„¶Ìyz|Ï SÞ"·[¯~™Þ¼˜Øé8sëÞ™.ŒºOÒûiÿ-C\`°Ûœ|^åtN<}Íã§sôó©S…W÷zúéçyØšA±MØ­YïVY5ÒTúûøâ8x(‡ç›n÷­žÓìZêARšÀÖõÀµ´—ýqïÞÍöѵîÞý´sòÁ£µDéQòÁÛ¢ð`óö_Ääƒv›ä´©iû%/NÚíq¿î S l×|'5ZuMƒÜvksiËÖâ¸0§¼æÔÇL¾ò6W£ž~Èéç£Ó“ÓQ:9©ÔNkãק5maŸÃdz}B¿<º»/iv4Õêã‚ýù2ÙÇto8Þqx8…9‘ŽG¹‡öð³‘Fº(†v¸XߎA#µ#ÒH¡°…н,Ø´0Ø4.lm^l­8eêLú-zlfçÕïQp:8{Ý‹šÓÞÝõ_¢Ð½ƒûë#çôׇ œ2”·ã{v$~ªÔŽk÷ßj—¯•—ÝÞ×&2šÊÇ Êi=t?Ò8Ã9}>vxméÁOßE§Cøú¢uHZH'Ÿmî<ëfËO¥‚”/JÙlx´³NmïÅúÎŽ$ŠB™_¿Ñ|*RYxú\ÊwÓi3ŸN§ i)¼ÝŠRñùN:]<”vÖ¤T±ÀÏ©«ßò•R¥X¬4ªU±\©>¯4´eXtïx8uí/lßÉT~'¹‘~šÚ’Ú.Ä {Ÿ—ñÖ½{%­[Ò"ò-hï•töÑ7Ÿ“ uˆ õ#ÒWèéc“«îU&¿þzâµÒF#1ÈŠøõ[ˆó§±ë·ö ÛSî1­ëáÔ¸®v §‹íf~x©…Àf8 MÌ€mxýûØœNŸùË)ñÏÂiý¾›3dþÖÒ¦«u?»Õ'ëÞMîô¸±M>uH»YÌŠ›îÛH‡H‰½D2]MV¥t"]N¤Å%c›£ßºÏ‹•çR¶œ®KRáPÌ—‚ô“Çd·î~9_,¥Ë•ýb¥,VJ«Á鸼 Å[ÙDr;›M‰›b*í¾÷±¿,l†1¿¿÷Oîpj±Â³Bw%Ê[¯É Ÿ¯Én´Ïáôø¸6º½®¿R&õó5Ù-Bý3¶Öõ`3Œ»EX¿E¨{cµÛÙ‹ïYÙòöpýו-o‡v(lìº×šmömí]»=ûö€[ÏdþcÓ‡^ñ³ÃÞ$ê1¶³L{¶¯Ýì³ í5†v½EÜßr‡·ÒqמeÝ¿Ïá<Ÿ Àúxò m3T-þmgŒé`X‘œ…‘Ž-É9¢#DŠ‘WžÓØÏóÁ*ÉÑ™É:F9œCŽùt¶dOÐù½6ŸÝÜIqtVvÀ‚Ø2°;) é2¾œJíßrŽN¬3twN‹JTâ³,:ÊVwgäYìæ_WøõqYÁû|´˜ûC¼9åÏ'ª>®ÖBN£Ävy”Yo{ÞZ=>NmŸŒÎx9½(•g¾ã֚LJ`œn§OJw¾Ù76×›cl×_µ>ÌŽ›øöª5¿ÑoÜ7¤íDóË””ÚxØx°ùõ”ÝZ9UöGÏZá’Ó×>»SvªÝÆa@Nƒ³¸}!0§ïï¿øDó?×vjôS:·¢6{«ÕŽNúÎâXùÉñÅ—!½žÈÔg]TñTcÀÆc§FPüñ4pŒž÷ Ïôˆç5^ÒZaNy†þ…ätq{a*Ùæsiº—ÓOûw׿[ÝëÏË»½Hü´ç7_£Ã×^˜š÷ñóâù# ~Úñ5oxüÔ_÷NÌшC÷2p“îU„¢(e+Ry§p(•j~¯,=WÜXo{ÄSÃW#m Rqg_ÊSÝlºº'³Ù=)œîm>K‹Y©/KÏ(È’P”RÙ Øíìa‡ÙO)¶’˜”žîç“;BWÌ…½ÂÓìö^xÝKs>Töª‡‡õñ¬°Û÷ë¯F¢öeëRé¶êSºw,{Û¿2©VëÍʾ¤4´ê^¥¨]á4¨îí&1ñ•˜Jj—åí ³ÈO¯jË5!™N¥ž¬‰ÔÚæ­úm»î] É«š¿/D§{k<Ú²Ã2ÆŒ[÷ú&Ó›á£{GjÙ½Ç9Œ¦&×ΖO}ÊÛjh$–þ·Pš<ªñHÁçØÆ¡{™úÿ9[÷ÆÏigfðÔ‰’øýÔé|f†6˜±Lm™P~ –c>×6=‡&~Nƒ'‹ÅOÛáì[ì~¶Uæ4?[<~ú‡äÔZ]? ÞÑÆ㧶ÖYœ×Á©ÙêÙÎâ[ÿCo¶xâé´AlB—Y¢'Áá§í)W´¦ÿ­»Ÿ¯vÞÉtõXcwé´i‘20¹cV¤º“!­:­Øœ÷Kñû©ØÃÔ½1èÞÖr9åÂÖæôÓί?½ŒŠÓóeö ‹®Ï†VI8N¿e˜7Íh·L&#ÿ¸•ÙÊÈ? Õ­-„r(§Ú·vT~JÇýdè¢ó˜Žó1]—’ÛÔ¼W¯{ËóSoÝË4ï#n?õi×»}ŒJ·ÛQ_œ{b<Ô8š¬ºª‘A»(#Ë™ÜañÓaýÊÚ€é=a­¸)|ùL³Éd*%„êt±er™]yk#yw7‡äœŠ¦–8ŸjN¬óófýñÁ”ÝšÅzµQRJÃüAåà°R­9_ïÍ©5ób.§½õ_f×TÌf± ¦¤ít7ì8‡0¾`u<ú´÷KR©TŠÂÎÅ€¤~pNÝ5êG«ÒU3e•.§ÈëÛ3v‹ð¼ M?­¶vÿ¹û#Ú…úWÞÊåþü7Æúíâ×æœÏ: Ou÷êØuUº0ì"Ƙui‰~ý=¤ËÓïØ£o½·ã¬{[Ü},Å¡‘BŒi æ§ôúªŒeŒe}fô±Æ"´ÛÕÛÊ  xCýjÞ¸8 OuÏÒ>_[ÆÁé‡À—©¿7§S&nægs‘eϹkßØ-Pyë¬.6»³ºå­c¯®Ý¸€¶³¨¯µPx\k©Ãzv÷÷–ÍÉ)Wqó¸ÖÏ\9Þk-ñ‰:†öé¨xuEÐYŽ°µÂa›ýk«ÄÈÁëêÆ>?kÚ úŽ¹Ý²…ož§DqS’_?¸ýDô_CÒßO;C'lW®g¤Ôf³›F][ƒ{íMÇÔl'lWy[짶ö{P±òÚ^‹üÔÔzŽ=¶ÏG§oÇ·Ñ“ÆéÛ·oOYל?ÓB]:eN<µÍ¡9 7Hfup¾gš6S<5µNÈ §„\ÜÇOéÕ]¶ãDŒoÌÉÌø^Û„–PgàD‘B®ëÞzGg€˜´VýÝ€5Œî 8!9=ønýM[£%r¹×åay»m¿ìsõ#EÏiõþ½ŸR}Mvë­¯¿Xˆ-~»MèÞþez÷¢w¹3ôѽËãÔ¬5ß6G÷·O?‹ótZÚvŽR#íl+J¥+–¥Z:V¥¬Ðë+]wy£±^bjCI OQHÝÙx”oú^ctyœR»•Ý™lnu4¹ªÚýäŽìàÖ½qpzëÑ7ɃíÛÏ¥ÄÆÄ­ÂbªÉ¹~~J9U>5›õOy­©i‡Z¹Ñì6­î«{—Ëé3!!ü£()©ñ•Î¦ÄÂæSßù§K¼Ÿk·jºšo”ÊåÃn¡RUÊ¥J©Ò]?ý|D¯°T«¸Ïîvô8:á»ÖR,ñôü:KÛÑÃæ½ÖRÜ:$Š>®¸to}ƒqéÞúß–Í)÷µ––ˆm¹œrõ“Mu]Åo7.ÃMõ.Ço7×aLÚtÇ÷2°9ŽÍxs®ÛŠ¤l7Øn°Ý`»Ávƒm2ý+„jãñ×öÑIEND®B`‚PK‡i]Mmeta.xmlSËŽ›0Ý÷+;[06äŒÔEWSµR§Rw±=OÁF¶Ò¿¯mBJš¨ŠÄßsî9÷U>»6zçÚ%+€’D\RÅ„ +üð"ôkÞ”i:V?äuUÊÊtÍÔhh¨¦G»š+V ˜‹~9n"W¦ã_·›ÑÞu»€9ÚR5ÚÏ s×UfñxW™Å®lJõv`¯ü šn~jýJ¥c×BÞT‘âùh3-·+/¥lTEìFÝiÌ}ûìpïN²ï×L9ìÑIöˆŠ¨A\¦} _èa·èòÝ)£­Þ'¡ =ò¹ªBð-\øŠåRéd|Ò…U¦MÊØêT § ¤Ö¬ǽ¬ Î̇ôÁKn9Û}çuNƒÓŽ°ò4á ®ÑjÓe‰,³ØîƒÝåLq$QaÄVÜÝòSVçŸ3CçPàLÔ™¢1©w)IZžA€È|åHw²VQÌtßöýòoiODÈùÕ:ÎE`ê}¨OýD‰ŸÐˆ‘˜E¢øðÞfëfxbŸQÉkï'©ß?ùD30H©5+ÎþÚû3Íeñ—>;èM:S#?Ù° ¬†ÀW2¥Y‡#ç:‚ÔxK7¾áW ¦LF醌/¡Üì3ýOy3‡g„JžÐ,}”NJòØ,7ùH…XÓèË°nǼ/¢ãGÅØhé±øC¡PÛûð‡÷˜ràÀV_˜š˜ÿ[%ÿª”ÜY 1WþMÞ]{Á$˜LƒÉ,°ãô7!Œ âàv|Å…&ü´OáÒü6 ~«f7䘹 {rÀ6Ò gádL[%—sÔÀøZM ­ä0ž[W2sƒÓò4dzÐÒáŠÏÈZ1 e@»­k ÞŸ ê ©Œaz¡ˆ^wä² (°„e&ëR¦'–ˆã&Ç›GK"X0_{ÿûïïÃ9“8>gdRžA×@iöú ð‡ÒO8jßþ;'ãZ oLjêæùÂ8þ)KÏÀaúÌ©AÖ ŠxVƒðöš=B¡õó7‡Ðü ºz“-ž ¡éÅüM"tùdÍÞ$>7š§¯ž ¡ùÍÓË'Dèmæé0x2ˆ¬Dí«ŠÍ*¼N6`m¨v6h°­"ª¢¬+ˆî¨­RºcMR ›öÀ–Yù0¾7£9n–õ^LMW ¹3©R*ÚY´‚²Ø¾»ÊdÆF!í,%K*6•C2ªë­’åfKª®“»Ä1“Þç< SɺËó5¸\+¼ªz v¬¶¬‹k] Y\¶)Ûõq1©§ÛU{]ÏWÅÓP?(§3Ó'êíÞ4]–Þþñ÷ÃûÈõ C½ÜšÞâ8Ðò±€ºD¤µ‹„Ò †nèTGzSmpÃÙÇP%‚6)8ͯ{š^rý™EzÇ5/¶-ÞÛûªÛ¯T5Ÿ¼£ŽÇcÛcáb C˜?=°æ}‰ÞÙ»Ó½3¶ƒ­³™Íäç¢iSÍsÂYå²[^pãÒÏÒíz€åpú`ÏðyL¿Ï7lÿÒDÊhÕÿiTÇw6¨>þÌ$þ‹sáOЪÄx«mè¾Õ±ÓLpŽI&'(=Qü¿+ôŒ‘eăqÀœòÇÐr-ãýkjzúZW]H.á 9Û*lœ'œ ¾ƒx9놮ÔöE]·Í¶™öšüíyFÝнö×ÅÔH8¸¯šÆ8ò|~–#£IákØ4øÒ*°GÙ ›ã£È^DÆnÇôüí˜>ûYÜÜ–‚öm\g’$Š–Ëokƹ]å°=p4BÏþ>¬G`7}vÖa²e4~í+Ç(è+4çÓ!8§áxŒÎ Œ£WIU¯R·MR÷5ÖRÄçC÷Liå›…nÞÓaâ‚L•Ç&ÐF?Ý@“üáÝã&’îqš…$¾ž!EI8öIÆ«ëÙ_î~’Ù»·÷Y.C_ÄßnpœË>‰sø-Aë8»*j¯gÛ4¾"^fW±·ÁÙUî_‘ÇU«+žúŠÝ«(Éò]4º9#æ[çø1Û˜Ò6Úz‹ñwfÄ|ë õÆ6¦´ T¾ù’Œmü˜Eò’€Ô7‰—‡-.£0þr=[çyr5Ÿ?<<(ºBÒÕ¹®;gµ5Ã~M—lÓˆQþG˜Þ,›#Í+Ú Î½±üQZž¥x»Yàt´h¼ÜëÌj’â H`¸T1ÇuÄ·iè×ýj´vݯÄ쯽t´ž1⦪èÁxUѾíÆË×óëÌ?B%ûïãÍ^¯ÒÍØ{QÚ†¨ü4LF³ æÛBjViƒb±3v5U5æÅ5Gýpü! sœräþArß‹üZâdÓ'4 Cs ñ=UùzQAd ´yQ]gÁ`׿|¼¹õ×xãí‰Ã§‰å0Îr/ÞK&Û„ÑèYÚ¥õâp´*PÚÎÒI©2 JÜœ§8!i^OÐr<À]´ZFë| ›0Z[‘®Ò è%vô9˜30&ò}ˆ~7k ÓaÅt[ŠÉLýSM uNijsªº®tUcë’lã ˜‡B€ø1ÁiH«¼ˆ5»jôÀÏVDNè²Äc®‡H…8ª,W=¤Þn‘7(2,X’\q­›˜—nÇuG) –í[ËÏ2=ïÓ‡»ÏsZ'SÈP+ïĹ:Úìmå×v,›×Kðoä¥çc9À~”½ý¡À§ºX*®)ß׳›ÀŽ‰Wºõb°"),ÍÝõ콄dÿÜ¢+ gR£kJ/¯p ‚S—’7(’0÷ î½4dÚ7Ït¹Å%üÌýÿêýÇö°Ì8š,e»,Ç›gñ”’0`·“~ò¢háù_†yëÒ~J1Wõ\æCK¡,÷¶9¡xáˬŸz°ÿü ªoVòYzÃrâ­¸áv›®ÒNÓUê%ëЯŠ/¥ »‹Fdñ+öó‡0_“m¾ £hVÝ l+'`Opš‡8“('pã”|¡œzÙzÆÉ´ÄÆø¯šú×?À ¹—a@½3UQMäoU>‰8Á¿³Ø¿²ß—~Á)ð â¾±nš´1OÙ_OGt=‹IŒËjá@ {tL ²…+°q›°¨–ÄИNÎIBûC¬;®|Aòœ:L=U^æ´B³Û5i¸ZsUTi¸‰<4«Úñ³Jý£ÀKƒC“y`^&~=±þÛK³= k’†¿ bÕ<üºÍòp¹;]ŒŠ./y•’y 1/½(+&sÆu©ª8†UÎ -/¹ÖaíÖt]ÈÜëç™ûÆB.VGÏZTÙ¿¡•Ô˜Ý%ûwÄkN‹mYÇÉÅ8³¥;Îhé†ezÆ ëæzv»ÛlpPCM濤 Ã—Q¨é«±læ߆v»†sœ\¬WañR#‰ç‡9 Eý‡—ÅGÑL­‹¦b:îeb€-´álÚ`*V1äŽ6¸ª ŽÐ†³iOÄíW‡K5îß‚:”%%ÿ/«#,ØvŸŠšq™*‚ÔWåfT{&ÅÐŽ•Ê ›%,•çÓP,#˜O±€<6Nu±ã=4jnõ6UÝš§&é@*Þ}Lw­à±³#rí=‰ó2Û áyfÏxßNˆ|ÒD+ä‰R+Ú°º&9Û;ÍÂ߀w'É;®´Tö²Ð‹êè^„‹Úñ"êÄŸ.¢ö&iSjQÑÔe´“ëY°HmZUüP†˜¤/”rÑ_ÉÌ`Y[uÊÕýVC½×ó4Ð]?|‡š¤¼ÇøÙîDÕ'ZAgå¼v¤~Sìßup…ö6;Ú0Âmà4 c\iy±Û:¾ Q0ÚŽ4¸¿iÃÌÒMÖ|¼l:xö”l„!x†!¸ë òÅ}¼Ö¾Þ a …Q˜•}6ßÑŒ'ˆð=ŽÊ[/¶Q„s©¨¤åÀ䬸,ªdšAv=û¿ÿýŸz¹N¸©dm¨{y ¨©ãÒòðá°)ª”åðÛÝ–x%„mF£ä8€Ø‘Q–ê’§[ÜÖÃdvv~h¤GÈA{†²„žS/𒤸Âë“>™„¢½J JÈy•2'“¦¯RBÖdÒ_¥|ìéäóJí´3™„ŒWj§Ý %ô:í4ÝèžHDæ·e¨¹êÒמf– ìê‹2ÅðílS€&Û…oN¯Qê·«·ÃY®byØ“å8eŠr•¯ºô¶ª {º°ŒÅz_»±Õþ©ª‰¼}:~‘m^ì¥?"vTg¸U:ÎŽTZ» tCQ©Y(µ)¡"­;š¢Ù<‹ü5ãÒQÄs©):z‚I›Á„ÊÙÚKz™Õ™- ¸ ˆá‚*Y©yÕ3Ž:EÎÑjÖÇ¥Šf9õ€àZ¯NÓÙ€öÝhIÝÇkúüF ¯0¡ÙO;Ö€>ˆð#~TI•4d©åÿ•Ž…iJRî´¯>¡ã««s‘feâåkFŠsVeWuêλ»&A±÷•b/»ž½[êÒ»¥?&üTɪù.Á4ÑÂ|9Å~îÅ«úpqC‚pâëûÕbHØCK¡3†þ ü¼[:Ò ûíJÿÉÊ ÜÀߨ’Î/åoJtSÿí°ˆ ñ¿¤Ÿ«ˆÿ{[d¬sëzY@AÀ¾Ùut§ ß´;ˆJ+Uü-'$ ‹§äòµÙM­ÇA…|] ë Ë2 q¸ÙÒÓózﱨ/VomC;6§ªà-jÉø2…AöYW}кŽ1¡¦ntM¨Š¬¦ uÛ0¡†e3º¿ñ‚|z/}|n09ͦf¥ˆÍ«É(Åfè$v5" Dúæé(Å)=ü© ‰žPâ„@4D(Sœm£\!ÂÏN"ù§R–{9~é· © ª/§>z %@J€”)R'½£Þ“ˆfÍ–•I0·¯Â˜~zÒ?^î8ÎåNßzOã*‚’³ÆaœÛ(~ò0©\éáÓ8WwÔçŸÆ½\¢’­˜HçåÚ“’\|¯h‡Ï9{S’ÝÉvS’m`ìÜ)É·œ9oªù.%ôôË”NÙÖlÅûÑ-gJìG ÇJ8Vߎc5UüÚ|¿šèíøUsWį-“+âWaf…™½l3{9ñ«æ*êùâWüælCuûÒ§~¨õa‰‰¾,Ñ9÷Ó ™!’!$ H$ é¤s?°¡S§¢&ý‘b{-eì+Œ¡ÿ5àÉ”^s‹ªù½‰úòÈ-*„Cã÷¨V<Ò“4ÉúßïáíooÔ­e¦ÊàšëŠ®Óíƹ@7±Ç%N Ý«Bº³ç\ҷèZÛOžtù!Îqšn“<\ÀÑ—¾þ›—Æ/ÿÒWÔ|'šè¥pí]A•s!@J„`˜0}óÀt†LSÍ»‚Ô„Úgxë+µúõ0ä!Æ)\y¹ä±Ÿl’ç8øž}V"÷¬—i0ŸõÒIÏÐT>~êOϨ7 +îFf¾ô¦g þËó348»×ü ¨¸ a€Ô,µñí ü-^®4Ç¥ñz]UœÁ·óÔáõX±}ƒO(2EI³ Uõ)]C•5ͱ%Ã@ªl9bõe9–Fë&xìß¹Ýðê©i]ònu³mC@oM³!ítM1…&¼0á… /ìuxaH4¯c,¨êZ]jÙfÄênÛ†ºÓºazñ˜÷`DSõ»TÏè|>E7€„·Û,°•ÞÜ~ø×»Û»O¯ ~Æ-¦âô3ø¦ /”Ñ÷ÿþóÝå2z‡SX>^^ðz÷ÇÏ/—×?‡QÄØü󇛛‹U´£XZÃ95 zjfe7Ža´‚Äþþ]ïW ï›ò^¨±÷BÁ—Á ·×D?!æêà•‚Co›Ôsè{§¤m"IÓ^è#b£ŸUç·MYªõY¾ŽE ñÅ¿±´óq@ˆv´f Ùù:`½+Ú¯qG¾š¼Þ»}ê0qg>ó9€aHFÏP½¾¯²Ô NóZ_e£Ëxr½»[‡™T}þô¤%ûµD–l÷cC?qWÕÓlˆ„±ß|§Llõilx5¬ öbT.­=˜Ï†“î°/÷$”p$y9«ÎC˜òR…<¾géєϜRèß °ä-s¶QDÅÈŽ²!T“|’ìdË)¬èïëg|³àåÔ+«º§{O1ÆålA‹·^í(s‰·òèÇ žÿ¥—ñ| AßjÍ*>|z?0MǼ BWôo²„Áá+ZÆ«}Á‚;®šøàt@  +%‡ßoÿPKžm[Ò•ÆPK‡i]M settings.xmlÝZ]sâ6}ï¯Èx¶3»Ää;a;@†Y’ÐÛnß„}5²ä‘äþ}¯ ¤,ØÄ+ÛNóLléœ#éê~Áõç×€¼€TTðºstXu€{§|ZwFÃNåÒùÜøåZL&Ôƒš/¼(®+ ´Æ!ê§sU[¾®;‘ä5AU5NP5íÕD|=­¶9º“-Ÿ¼2ÊŸëÎLë°æºóùüp~r(äÔ=ºººrã·ë¡¡…PDÇ‚³nÎÙ¤õŸÐiV”åèÍùBˆ7ÑfÂra±ðãjõÔ]þ¿­ʲr™±O!j³öŠpd…1cwV½Tµ_ûÕ›öÕam˜È±ÓXÛÃÚ ׫ÍYþ©P ±‘ƒÕc£±î eí…ÂüÍzœ¤y?ÎùF•Ù€¦2¡³~©!¾¤\;£Ó‹³£kwè§À{0щè'ç¹Á§¾ž%¡Ÿœœå†¿:%«?¿ª^dů$¬PîÃ+øÛ\0O>¬xÚ›\dQ ó®¿%Si‰–à4Œ]XžâIý®ZíÇüX„; -#Èß‘‚ooñø„0e‰ÞUN¡0,Åk_£÷ÉR®Ê#1¿{”C*EÛô8þ <Ý‘ø°t–¾À;TÎbú,Rw„û T“ÍÉâ=ëÍEïÖ$ÓÞ4–~<öpËh@9ÑÐlo[覡KtKgÕªíjv¹Ê8¢ÕyôÈãÞ6>Qp~ÚBrá4ÜÌ?—u;-}ôšüWÔô„÷ þûRÜ÷¡DS£û—q€¢-˜Ø•˜ßØŸ"²Äxc|õW ÃÛr&$^Ë{3† |^(pWŦp/ü¶¢«nD„›ÜfÔ{«¾õijÎC´g„OáI,ë„âiŒú ú ŸÄ<ù N-Ý/îJÂÄ®ÖòXÿÿiw~øýi÷¥}¶ÙD*HUt|ayªÿ€ïÑž ½ƒùß>áù÷ª¶‡6I_,ú,ž%ÑB–ÈqƒµvœÁ¤°Àñý_XÇ÷rÖÑUqÖhhn¹IzR«¢ÉA/!9ýSˆà‘'„Ø,šn×i¯Wõzò:{gfù ’qôû™M3 Ùb¤@ÞMŠ›-ªq¡Cc £§^JûàÃG,mõ'W͈7$ åï'­#öiEì*E‘þ¡c;5˜$H‚™”l‹À´MSlˆR MªâÔµð{vbnˆš®Å·Ô1r:%-±_·ÁüŽ(ªžLï¦u„SßþfKù{k=L]£„Ü~u@Àm 12ü)¨£eE$%;Ý´j†º±SÊM§Éá–û{ççuÆïGRâS+ü^MíNäŽh¯øK>³M¾‚äM…6Ò¸§£½õX¾>œi­´„xÆÅ”NÐ"ÞsÅë&ÇÞ6un ðå´(côÕ–¹SF‚»ØëŽRºà1‡A(«;3 )ƒr÷iHKÑo<.:ÞÂÝŒ­Aü' ¾àl;À±Ú{ *’0â;«UnhY„Åa®"1Hb5lÙGêc:2•$œ ¢ (ËéÆö[DÕÛÛž§aÂìb6ƒclí‘…ˆR’Œó fï‚Ef™¹(n{öDd…1m§³?ÛMø¶ü”þ‘·™PeÜᇲœÊ?†Þ߸°…>Æ-ŒŒÁ‚í‰aéÞ?µºvw¾ á¦}W¦ñ7PKnù„m#PK‡i]MMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPK‡i]MŸ.Ä++mimetypePK‡i]Máž88QThumbnails/thumbnail.pngPK‡i]MÀ³½+¿meta.xmlPK‡i]M²Configurations2/menubar/PK‡i]MèConfigurations2/floater/PK‡i]MConfigurations2/progressbar/PK‡i]MXConfigurations2/toolbar/PK‡i]M'ŽConfigurations2/accelerator/current.xmlPK‡i]MåConfigurations2/statusbar/PK‡i]MConfigurations2/toolpanel/PK‡i]MUConfigurations2/popupmenu/PK‡i]MConfigurations2/images/Bitmaps/PK‡i]M/ž ^1 07 Êstyles.xmlPK‡i]Mžm[Ò•Æ 3 content.xmlPK‡i]Mnù„m# ‡1settings.xmlPK‡i]M¿(ÖÞÅ6META-INF/manifest.xmlPK68apache-buildstream-27ae392/doc/source/image-sources/arch-scheduler-queue-ports.odg000066400000000000000000000374631514607367700303220ustar00rootroot00000000000000PKG9NMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPKG9NMDÞpÑGGThumbnails/thumbnail.png‰PNG  IHDRœqµïóPLTE%(''!,!(6(>//>>/7:F<>h7m,)GG5\E-RR>sFyPcJ1nR6{\=BDUCZCIRWWAAXXAKKdRZe\\{KeKXuX[ek^jpeKKwYYddJyy[bmtgszš ›ž¡##¥44«5f¤7h¥:j¦=l¨L\ŽBB®CC³VV·BaFw›@n©Gs¬Oz°S}±bb„oo”o|ƒq~†pp•aa¿zz¤qq¾ddÁzzÅ,‚,.ƒ43„3AŒAG•GTšTfˆfn“nt›tfªftªty¶yW´\ƒµZ…¸v„Œ~Œ”`†·`‡¸cŠºi»l½q“¾w˜Â~£Êœ ‡% ˆ(0’7’9$¢¦++ª55°;;‚^,\7›G%˜E1…b/‡g6¥X2…cB’mI™wUŽjj•oouu®BB²KK»UUªeU³oE¸wK¨~~»ee±rcÅzz™N†S„„b““n¯ƒW¤]º†X¡ƒdª™f²˜§§}¸¯|ÁˆWÆ_˘eˆˆˆ€˜ƒ’›™™™„„±Š›¤­†¸†¼ƒ«£“¤®—¨³Ÿ±¼¬·Ÿˆ»™¤¬¬¹¹Š¨¨¨¡³¾¶¶¶È€žÆ˜˜Ë››ÖŠ¦ÊŠ­Ñ°ÒªÍšµÔ®®Þ¤¸Ã ¶Ô£¹Õ¨¼×©½Ø½½äʢ Å °Ï³µÓµ¬ÀÌ¢ÁÝ­ÁÚ´ÆÜ»ÑݪÇà°Íä»Ëà¾Ôá»Öễ‚Ӈ˘˜Î§À»ˆÍ½­Ë¾²Ú¥¥Ú·´áµµË˘ÓĵÈÈÈÃÚÃÙÌÀßÕË×××ÀÌãÃÑäÂÙæÄÜìÉÖæËØçÌÙèÒÜêÎáÎÑãÑÊáïÎæôÖàìÚâîÞæðãÅÂâÓËëÜØðÚÚçççåêóîôîëðöóéçôðíóóóðóøóøóõøúùóóúøöþþþ–è$QIDATxÚí\tÅy7ZZÀ´6¦’B*“€ÿ‹Z›œí¾Ðju©iCp©-›à Ø’u†à´¤¡ íË¿¾Ò4”ÞkÉU[ƒk;i²ž¾«u6˜"kÁ²Mâ@'dtXOj°Î‰3·‡6¬wú}³wÒét··'ÝùÞ}§›Ûùv÷·3ß7óÍjç7KøXf5À5À5À5À5À5À5À5À½+ÀYÃÉR’²ê .¶ò“%dspUo]Áõ†Nœ.-ÇÚúë .ôÒi79ª#8kåéw0¸O6À¹ƒ;uª¿(¸>hÖãóv¶ìÞy÷]{vï¾»{þ-;O·ì¼Ë76÷Ƽ½-ó>¸pa˼ én9ÝÒ²ú¸_šõÍ»vïÛ·ûàÞ}{÷ì=¸wçžÓûöõò½CœhôsïLp'üÛωÍϸ‚Û­'¸”²õ(Hÿ²Ÿñ¿£ñ­mV=Á‰XÀMbõÓ-ùÁ$8ãlårü4‡88æß N`À¿àâ„Á…Aß‚Q”MI¿‚ëè Æ:} îð¦´"6é¾— ¦Òm"Õ6àCpŠ.œRt¿ËD•a!Á[Ä2~g%”.$Á +®ÄFk n0·‘"O—¬4}‹²ÅÑuÀÁo,¸9™©)¸½ÑTôpf 7<šé5ã =u’ •t4¨töçpäÀaãnQ‚[õt­ÀY‰xb 71ïíé¸>‹Žvꉎt9ŽÎ ƶ(JøðH^ì‘Žé +J46˜¶jŽ¤úL+™SèÃÃb03<"Ít"RÚÂpÙ‚C&sú=nS6EûÓÕwˆÔ„eeÍd,¨„õá¢5œ“=¬ wô YµõÖ¡%.Y\œÓñP×PíÀ¥:”¸«ºCу#5§+ Ó]£8èž‚‰š€‹¶•½ëòà„Õ¶f ?[S3 >[V–Ñé „Š”$†-Oà`˜ŒöèŸe`@¶õÒ ŵX?60ð¬®+±©eÐ {— v3ú¨—9¨‡fÅ™F´¸EëåÁuE‘Z™®8sUªx?R\ð»—¸Û\Óœõ6]Ü<÷7¬/îšk¯½öšG«Ô¬—~ôߥ¥mîyø6oØд~nsSó܉š{ª(¸kß÷…뮹扙5k¾ìz¿G›ÛµáÈ8âÒ6÷Pº’ç§åOL·Y÷O©BüÖµŸ;³l°ýåjîù©•6\UGqÓm®Íúñ›µ+îp­¹;Ö:jŸû;Wp;>WQ³ŠÛn*6¸æªð¶eÿå¨íºâ©RÀЮø±£ö?÷îp«¸û~TYÍÙ·þ^ Yßåûsúwξ¬´Ì~.§öò¶O¡lÿ)Ÿm/Wl Dৠ/ŒÃß–¹ïÛò3‘w`öÏ:!樬Yë"Þk®®2pf8“°¼<ƒ;¿lÒü²¢Bœ”óI'¡Tã… JhNK5ƽ6+15Æu8•FÕ Â5ªiDJ4#Â`‡2Í$×L¸¶F‰<Œ¨LhLÀ†Ù†Æ(åS)P8 ¥ D A4r"ìºÓkÍÁ% ¥ª&¨F¸. pBFS‰F BU à€Ž€«Ãå Ø”WtîЄ{Ю¶¹™X…”€®!¨J ñ|ž›•G§ƒ«›*êÉf“C#ƒ1ȇ¬]h(HZÚ2ç$6e …Ä pÅ@S6£‰ºPµŒp“{÷V£à9˜-XYs6ŒBã3jå­ÜÈY,&œ‚‚ ž¸¼ÉqbN³³Gä|)O8Þ7Ô(¸€9]p*X›1MUÕh65!*:‡ LUÁÔQ9X6ÓT.§‚ëhšÊ"à2 »hpBS9˜lµ#¨ þ¶KL»”Á?"4Â+í„Ñ‹àü†Šnç³F+àp~´1B¹­¢±£ ƒ×‡€£ÀÕO‚Mw÷ÄÕì&*œå gS¹Qq³š Âær#›°eX·…ig ½ ›Ë¦å‹„iÚÐJ<ƒÙPj‚«@1Ø.ôÜ„Ã`J¸<ÄÙ3Œ18ù9nçÂÌ™)bvìÕΟÄäœÆØÚ™57mpLöBÛ™ c †0æÂ@ˆÁö [f0N``åGN<èeu8áØ‘À`+M}@¯d@W®6ÌI…ÍjhءŸ`-"— â$èD1&Rñ{}CSe(¤qP…ÎzbŒ†`˜Ð0œbÐC¥ ‹¿FÔÇàº_ ã( Ö^à@xuˆ((N81Õà†Ê°‡"![…¨Ç+Ôƒ:Á8HÅ¡  PªòÁBäPÈñ”8t0:-›#Sæ­9èS²rW(è_Yn़fÞjD8æ]§¼ô1¥Æ£R‡€hšpðˆ3?ù „„ЮGOãMŒ6 æĆpòU A<´:Ç¢INL0Dô' Pá`ŒÈ‰ŒPM ÀÄÛ—'gjCÖ¨f¼®A` I±^¨ j‚à„Á„ûµ¡9 ž »báMCÍ‚¾Æ*©9^Ìz 5ìŠ:U>ÉGèù!8Í}vÒa>[Á Ñ8ôxÆë› &Y8çR¥ÕolUUŽSŠ³Qp`˜ ¨š3ÛÅþÕT­z‚?ABÎ4jÑâ¡1d7‚É« N§ù&#cbÆÒUæßè"zHÔMV¥Ë€PÒõÂëeÀ‰þPª>Øâ›Ì²àÄV%žÎœwÑC§:O‘÷熣Êt%0í#;Eºê’ ´U×`« NÉø\°QsïF›k€{W‚ Y>×!ü ÎÛ{Âõ—VÒi¿‚ëŠåSp£Î·«4E,õ•ÍéÏ“ÞU‰¼½™/K›!¸èªŽp)Ù¼ª¾KåcOM¢ƒä?°N%­mQý—µgÅFúã±ÞdBìý©D:‹G“ô$G2ÞêˆOf *|š>šZ;ãzªÚàF™Ì¡‘žCb8‘Ñ… ]Ï$,}`(í„zŠ\²Ø£þô`,¬:cƒVõbœ Âr°$£­­]‰ìG%ÿa Å;ÑáÚz«®.fîúO3¡“µw¸­Rjc™0¾£f¼L=Ár¡¤^v`h×\´ü YœH{k®7\~@ï÷’d*0¼b/½Ä6j(+Še†¶zo±ÚÍ^úÝS0/Ò°#q¥3S\LI5Q5p96™‚ÜpWYpCÅ° '½ZIÆ:“|¬ŽÞd± è —×SÌÕ£`+ËéU ¶¹ìÛºx²@±WÓ­åH£:ŠÙÖœÎÃt9xaå Jñ‚!wpVñÿ4‡½ý#5ðòRe¦”šWê#!Æ^Í“ÿüƒW4±÷VþˆùBžüÇýwÞÞOóÖÊŸûß<Ñþðy{?7¯¯âþâ/ïùLI¹gû«Yu¾ö²5ùògù;WÌÞ•»Ñ‡®zðÀ?)Ÿ}`Bü«¾å\ö ësÛ¿ôŠ‹<¼ýŽúºµ®¤³_pÔ{è5ùþ{PQÍýìžW\åKßsÔKSGHY›%ú쿸{íëUîä_¸ƒû²®_ɺ,¸«\±½öÏUÔ¬ÕwgUÁUXs³ÏkÍ•÷Íob2\IÒ¨§J×Ü·_ûÎøÖÌ›õúE¿xñâ/.þâ¯áÞI™÷•ÉÍÚ>§½¹½yWó†æ íÚÛ×·ÏÉ‚»£ܯ?²äŸ–‚<òí¥\]…š»~ñß^tå¢+¯¼þú\ÎÉB›k_°«iÁ‚ææ¹sš´/¸¸©in©š[úþ«—.y?È’«¯~OÀ-zeñ¬‹-šõ[³^.esíOYÐ|ÁúY f͹°éW/¾ø KÙÜïY²dɯ€,uѹÌÇkâ­æô¼õ;ãf7³š;é§~Îòs?—kÖíÞÀ]î­æÞû}WpWVsoÝóµ’À°¥·ŸtÔÿ|+¸5Gµ‡þÑ܃ߪ,dúÞö‡]ä3gÕ_˜½îŽÒ²æö쵟\õõÇKË_?ðVEÍ ûùÚ¹œþ ë\äãëç^ÿyü­ ƒÍºHcÿy¨9“ 2¾f”å+M^Širƒ¹,xdù›lR¶QÀµR8\w®â r&YQT\&Ï1• èh¸ò—±ˆ¦Èí#4I Â‘J…kŒ¡:Q…©$"B¥„hBŵĚ*„Á¥é_ë½Y¹ —Å…Ó6åªd†‹“®pp¸zž©T —Š&ùTˆi8ICDåbg¤#‚{ò¸¨*×û\ÈŒ«KᔸžS3+®9.83™Am&crÆ È Bþra2ÆMj ø,Ø„&ℛȭÃ0¡¸6ɇàp“›Ùãpi©!U’h”gɉjî­Eéz Zs‘3csªQzmV\†làJ|C›’@ÂÆ_h)a[˜…²• Š3¶eÛwm+— –€„3!á 5¨ü…B<6òPÀ×t z¬9ÉÙ#ÙV3SE@à„¬>Ë‘ ‚шŠ š>êR&I~ÀÚT ŽüpÊ Ï5Á*WŸU‘MÏ¡F„…TCÈê!*‡³4lïàЬÑàÁÄϳiœðJ®©èº˜ª/:ôAÚYMÍhBq€‰ÿ1°Œ#}˜­RJU"ÆT¼†ùpœ‰«‚f5æ2¤iŒYrÏ–;\6ˆ‰Ã¥†,D&É8C6Á]ÆdLÈÆÇRhdÔ6á0ÿ å¹fʼmN×!Ìœõ–èi³”n¹b« `êv©Ïà°Ÿ°¡w3FÎ6Š|)¹T ú gA¶Ð0¡Ï3 ÇC¢7èÎa¿‚}£odõ‰ ý tK£`gÁ¥°w“½ çÌ0²]×fE㫳%å&°¯¥8dhè²ÈAÁƲrµ¼-#—Î3ôGŸ8ú’ ÉYÐj59¾U…žQS#HwRQÍí"¿œ .‹‹¤¹@î5 †$”À Ý+ø`4cÎD¶· }©€'F<˜Cqøª–LL›–Í‘½)™V?Í´±‰.û|†LÆy ™* ©=2HòǧÍ7P+p&‘wÒ­ú-Æ‘CÓ^§2XòÛ¢BúƧ15l4k•j.o–‹ï¤PòSQšÉˆb’ç¹9%˜Ø”Ò¯„æÎaâ¡´Š³9Cö²Hˆ†,=ÀäIsU1!_@h;ŒÿPŒ;8ÒãÄTå0#Ç+!†ê$BÙ3d\Ã)eÍŠÔm4"¹pU-± 9K Ù›@²7•G ÉG ‘¹!ñh8ÏÆH9t‘^rÉUç0ÎaÈ…Ó]˜º"M¨)à RQTbR†,{œðEÊ=d$æHî)Ù÷`h8†JŠ>$êƒ).Lp‘àÏ@žc¤Å\¸'$ •a%T1·†˜Ù)68 ÆæVÙ['b—ÙªÁ\ãò‚3Uì­œå°<,9#Gb[îlM²n*X‘I󌽵ðÕÈ1Ô“–ü²HèŒ3AÇ eÁ`2ÁNÁK¸&©¬Éç‡Î]U™iÁü ú Ð%ð3^!8*xŸ/¡#$mÃD‚á'ù$FE†haJ¯“sXŠD‰ªC Ø4CQ¤@”£àK8ÒÔ×Wøv„èé)hV|ÖsQÇ`×=;:†u˜S($Å3Ò\ra3Ãf6N¿¸SÄ5Ž†ÏÒm0È€rÁ¤cð=§³ ¸´2õ'súÑÓñ"[µQÖ{BSÞ·™Jµ²¿.cW*4õ¤©o%Û¶$ŠJo¢%»3¾•-×Iäå÷NÑÛ›;æžHlY³<€Ã—gÏ¿}{–ð±4À5À5À5À5À5À5À5À5À••ÿd= ÍF§IEND®B`‚PKG9NMmeta.xmlSM›0¼÷W w¯`H°R=mÕJM¥Þ"b{³Þ‚l³¤ÿ¾Æ„ºQ•£ß›y3ïÃÅã¹m‚7®P²8B à’*&ä©?öŸÃ |«3ø·ùµÛË,/Õ|eã:ªÂ/…jî½…NW1ÂYˆQˆ“=ÎÉ&#i¥8Kwùe¼Á(%kjºÇ[‚Ái„2”n„Qgؤʙ°î„¬×¾Võm'_ðîûEã]~M£¿iÃMµù} OØ—Ü‘•®žÄQó¯¾s¸‰p”FøáIÈþ|ø™mÛ4XV¯œZˆQ‹>õ¢a!¾Èü­8)\ÿªÒXAWDZ‚;œ^ÚÄ9€UW“‡·~{õPKû6j|À+PKG9NMConfigurations2/progressbar/PKG9NMConfigurations2/toolbar/PKG9NMConfigurations2/menubar/PKG9NMConfigurations2/floater/PKG9NMConfigurations2/statusbar/PKG9NM'Configurations2/accelerator/current.xmlPKPKG9NMConfigurations2/toolpanel/PKG9NMConfigurations2/popupmenu/PKG9NMConfigurations2/images/Bitmaps/PKG9NM styles.xmlÝ[Ûnã8}߯04˜}St±“N¼ ö‚˜Ì.¦gæu@K”ÍnJ(:Žóû¸ÿ·_²U¤(Q¶äȉ“ô¤ˆcV‘¬:u!«¤|üá>ç“;*+&Šk/: ½ -‘²byíýúË'ÿÒûáæOE–±„ÎS‘¬sZ(¿R[N« L.ª¹!^{kYÌ©X5/HN«¹J梤…4w¹çz+3¢;]3»³½Wc'#og.YŒßY3»³SI6c'#/`êNÏÄØÉ÷÷3á'"/‰b;RÜsV|½öVJ•ó Øl6g›é™Ë ººº 4µ8iøʵäš+MÊ)nVÑYXÞœ*2V>äuE*Öù‚ÊÑÐEö¬ZJZ ¨‹~9n!wNÇ¿î–£½ën9s²"r´Ÿi殫LÓñ®2Mݹ9Q«û^·@Ô?nlýJæc÷BÞT‰dåh5 ·;_шŠL°kqã0œæ»Ã½9Ⱦ‘LQé°'Ù“q‘÷|Q>½C—oôέ5ð8 )Øhè‘wÏU%‚?¨áy i)¤jÉÆ']Ø%nRÆJå|8e Õ².ešö²‚8ÓÒ¯Çèæ;¯sv„«GЩõ±)šÉͽ'Da€+ @D9wfw²VUMUŸù~ù9@š'"äüzç"{7öÔÏœøI¨ŸÒ„W7M¶n†'æ; yíýÈ õkå'ŸI)Õ²Bàl¯½?“RTÙá3ƒÞ¤³4òûKZ€ÖøRä¤èp”L%ïˆdÚ7‚ñ¢Á’Ù(Ùñµ…<¢WmXU=G¶Ð/ä·õa{:\”H±›ÐZ\Ûò˜jˆëM,íae)õn–ð÷Ÿö÷Ä«"§÷ìÚpõîÛPWlwç†ôÏŸ¼Ö>X°†qÃÀ\)W™zLJ‹Ž /°Žà,õÜÚ°Ë–0ɽG#Ë䪦fâYŒS÷é Å+uѸòéH훌qÝh0éš‚®M Ôü•ìAàM ‚•-Áô_Ö•bÙVû`IRì ø3(JŸ£0a!”Â¥Æi¦´»É–+‡b °"©.¢Xšb8ƒ>äÿŠ*ÿ¾‹F—¸í%ZÝ/Cü^ 'œUÝäºãC.‡úŠf±æœª‰!â8Ü/=óÕ|¬—¯½ÿý÷?Ã9‹8>§çä¬ð9Y¥±õ( ¥›8¬o Âá³Ü“šºygvþ½Ž²CšCü ô©á/(ÔðÔ‚ðþšž ¡Èøù»ChvB„.ß%Bç'C(>›½K„.N†Ðô]âóátø¼Ó<}y2„fï4O_¡÷™§£ðdÿ±µC®+¶`¨ð:ØX€½¡ÚY¢Â¦Š¨KˆzÐVÝQS¥tÇš¥Öí5ó£0ü^jä˜n ÖG1Õ+ä.„Ì oWQÊbó誅´³•X+±©ì’]­¤X/W~ý¬ÙÝbŸIm˞΄®äwÝååúÛ;®]Ö=3f5ëâj Ä=×`6iº>.&v¹Mmk»^TAýìC9]è>Qo÷¦é²ô¶h÷¨¹ŸÞªµ+ µK-½Åq åcu‰:Hkj ¤ ÝС¦ï(&«pÃ<¨rÃ1¬tÃ2 vCV|€EûÍË1ŽºI’rF³§w}>ÙÇP'‚6)8ͯGš^bñ…&jÃ/¦óÜÛû²íW"›7^ü½ŽÇsÛcgÑù@‡,Ôÿz:`MËþ@ïìÃáÞ™ ÛÁÖÙÔdòcÑ4©æ%á¬sÙ«˜véév=As8}°gø2ª?榩#e´èÿÒ¢ã? ?q-ü©ÚAŒ7«C÷ŽYf‚kL 1ÁÙ˜Š¿»“^0²L‚x2˜SþR.Dº}KI_ëê É$Gk…óŒÑÁg¯§ÝЕÚ<¨ë¶ùVÓ\“¿]%¨º×~[L„ƒ)þ¦‰aŒ#ÏfG92ª½…Nƒ­Bs”½ò±9>ŠÌEd¬9âãÍ¿øYÜÜ–Âöi\ÇY–$WWß– ƹÙe·=p 4B}ŸÖ#0FŸu˜¬(IßúÊ1 úÍY<gÇè¸À°½Iªz;ºmÛ×XžÝ ¥•oºYO‡‰)ò么æ”Tkùñº_|Ÿ¤Äî{¿¤wÞÀ[)ŽmmJ‡„¾.˜ªëó±6Šûr@°óò_ýß6÷Îß,ÎKêC]…lÇJÿ¾ ½ž®7pLì ›(ê=€ì¨Þ¯.ŠMKÃÛNtž]ÆúaH-£dŒ¬ýÛ|a^æbäh2ì ·i¹—(ë¿âñqwpÏ ±¹”øBxm®…)žŒƒÁ ÉjBN*¥ÝrÛ¾Þ Já>Rh¿»/g­›·7÷è­¼cxL¤fMÀO¥à8ê<€ÉUs»zPjeãW“õ{nm¾h}´õ`›Ú$¢ë¼;»ƒeÐÿ÷n7ÿPK›À# /7PKG9NM content.xmlí]moãÆþÞ_A(m‘¤¡¨å›H÷ìCŠæ€wí5¾¶i¿kr)mLq¾XV>åôcQ ýs÷K:»|)Š%Qw²l‰;3ËÙgggf_H¿xù¸ð¥Å”×#4žŒ$8Ì¥Áìzô×w¯dkôòæW/˜çQ‡\¹ÌI$Hd‡ |J ÄWõz”FÁÃ1¯¼ ñUâ\±…ÔU•ûJÜ++‰“•ßY\0W¥ò˜tæ¼5Y|×ý΂¹*íFxÙU˜ó¨Uqu~Œ}Ùc€ú"Ä ÝÐâѧÁýõhž$á•¢,—ËñR³h¦ Û¶A-vJ¾0|Áå: ñ ¿Y¬ 1R ÞIpWý8oU¥ ]Ü‘¨348Á^ # 4—f·Šª25ûz˜u¶®‡Y ÌÎGíL0×MEs»›ŠæVe8™·ô¯¥¼¢øóæõÚ®¢E×{qÞTNDÃÎÍ̸«òŒ±RU. v¡®:™èJv]á^îd_F4!Q…ÝÙÉî`ß)g‹m R€C&ÜäËAĈ[T%#—̱ÛZõ÷o^ß:s²ÀkfºŸY¦Aœà`L¼ ~ç^Þ£Åíl œ·1t"n ­ˆJDB%ey݃ÜE-1š' ¿Ý…qjÁ:‹\w++¨£)àÎÀ™È”,?Õ¢Ónô7 S¸ú}"‚© v  ‰ÂyJw¦º\ѬŒ­K7ë‡ @ò’ˆrö…ØU­†joùìˆ*óx\©¡¤(ñ ÏU6ik5ŒÉ‹ , ¯*Òõ˜-»UÇ)s½Í7–ÇZ²ÍÞ}§pšÌC>µüN•TGÝyMæÇb¥,ð ¿‘=ìÙ%Žß¼ÈâSY,e×\ïëÑk ÁNÀ+Ý⼑‚†æêzô[²ø÷|YáHªUÍùå pu[à ÆÒÄ`ð€#*¬Oé®TéuÒ3~håÔ#pñ’Æñ)ºý‘üˆÿ–îîÏ O•VqB'é1êŠÛI¯°ïßaç¾]·&ïÑñUDH hÒ©(mÃ4/ÇiÂx,sdQO9~Åßš¾nˆÊ›åz智âY¥¹MÑYÔE8œS§(qÄ'KâB΄x”wq䎊zs9G¢„’X '»'ò’º<÷›Œ'r®äƒÑ=‰ ZÈ1ךalr@ÔÝN÷¨Ï3ŸAŽþ™'~r ±ChÊä7y ÷ŸÐ,ÏYDfÜ9Ëا3ð¡?¦qB½Õ&ão‡³f[PÙ$çâ]#Ï"¶”ç„Îæàº=ìÇ@õØÕ‚e©>6U®o^œ7Ã.šÅ!vù4UNXÈÛ‡6ËïX’ðp É'^ êt“ew/HÜÖ*ý¿ËÔn RÑ—“‰iN&ç±ÛvÛþ¸V1[Ó-fƶŠ.Ó,´ó˜…@-ëüëQÀÒbü§Ò™ï/°·Ó6ºò1Ý: ý†‹mWðéy¸ðÁ2 —C‡‹q faššv>³¼èáfa™Ö93-doMµtõ2­az Nš ûœ±Ã4'q˜YX,ÕÚžOš5©cËÔk¢} –Ïs¦2„õ>ÉàÓŒÁò3 4¹»ÐtÓÀú™<¢G¦žññí=-»8Ó ÕåÍ>§sÄj »û‘8É’&s–&\û 9×£ÛÕbA’ˆ:?¨“¾ŽÀŠw /ݘֆ×ÁáëÒìûL«+ƒß{âñðL«KgKÁ…©¥‚ožš„…ÊЀl·­Ýæ¤åÎGYqµ"П·±h–Cø&íê7üï‘êç‹ÒçU¶áeªÊžñ¾ +> ¤½áöHÔ2A«³‹í¸˜þ º[aR(/ÓÀ%‘O’¹2kÈšWÈÉ8¦8¨I¯iü„‹O3jw+ŠGZZ¾ž{^Kk¬s­ìe˜5Í3+ÛX8u wh²Ís#ßN,­\»·¯±2p”¯èÛ#¬WÔùÏèãÁÓœ!pÄÅyµ9o;vheé㙵=1˜ŸÉö6£QQ´Ìs®;æ»#P&TP·‹ÖÔá’.ÄÀ·÷ücïx-¦4o¼ö“ûœËŸ}l|Þ5WvÔknöTÇË»†;Ú‡Íf”ª¨ÒvRq[Ç—eŠ¦çëgoH7íˆ-¨õ›Þ-µ—ôöúí­†sëÚ[Åz«ÿÞÏIù4Îk¬ÛE£*ƒOˆŸßø.õ}’H‘—ƒŠ£ì2#ÉüÁ˜ëÑûÿ«ìáJ%•~2|…ÄÇw@)ÊÌl‘d·›+,é6ÁÑíjM, Hc¾¢¸l) ÎÜ–’(%›V¢b DÙÕÒpPOÀ!ù×;ⱈ ´”*/ò§“n^ˆ¥þŒR¶è‘åæüŠ}ÊU¹¹)sÊwŸã„Dâá&¹xÔÍéϧ\‚ÁIcÐCŽç8$ÍŠÄCOågmõmAññŠ?ü,M²óbËx2FÓi±^l~cd«Eá£Hß ­¸†^AúXC\*7°PÉU%Áœ?3íÊ3Âø!ˆ•àÿþA=i"©Èœä h±¨²…^n{WÉžmâdmŽHâÌCF¡ëÅ$Ì*+o2¬ê ÙzSDp|=zéiÒKO‡_~Ëãž«ð‡+aö)GÄIp0+wìÌ¥%#[S'ËPõ©¡YÅq¼’P¼4à T<¾‡ß—ž%½Ÿ¶ôQžó¾£©ïóOÎôºün Äý§ô§ÂRÈOi6Ë® Wž÷aÑ"õ1#e—ÚøõDú2¦Áç/=øòyHdM¾øbw%êf% ­ô¥†LM™šš¹[XÛæG~÷ÒS¥Ýrú¦\ÂÂbƦ˜8ç ï47³S$§[[@ﳶ5p¯”Ý¢æ^AÔ° ™ª X·ëç~)û.‡,¦Ù[B€4Õ¨ñ’?>JÜÂgWiŒ-"?Ê6è"åÇSZèø1£g#™;ùíþ§ T}hG·ª¶ºUu¯[UdžÝðªàEm«âUñD­;U[œïœêÚ©jë Žtp¤ƒ#½@GêEÐÈmTkõ ºxYÈÝ©[ôEýêlÎu­Ò{,|m>©ê¨Žf7A“þnÞFÌ!qLÜ|Á9‹ EX ¶¾•²†«c Ò[4ú ­LÚƒ†4Ä !}B1ȮŠûÜεÇô Sÿ|Èh…ϬÂWÚÖÆå‰ZcuˆNCt¢Ó>Áèt^Úcº½§ax¾d¶‚„ª1Ȭ… s/~|ëC<íßØú@õ­Ó¬ ¨õé°ó‡! á艆£KØùЪûÉàt'hpªC^?8ÒÁ‘>!GzêªSaáõ«³9×SþïvWçJø§­èMûfE ˆ†÷!þ ñgˆ?ŸRü1jñ§§-åvçÚcüáKNdžŸVÔ¬VÔöoñÕ$„6±Ñ P¯­&iÙ›]sḻ¡¶gíÁFÝü%%)‘(œpQüó­XÊþ/„¥÷¿üWpÜ%ßÿò? ††„}_bž”̉Tüó1i9§Î\‚`"Ñ@Ph¦‰ô“¨'B&¡ ·wÅ Pú©FØɺA3&¹”Ç<ƒZ4 úqݦüUrR˜ë Áì+ F­`Œù€O}I,q¼àãÇàCäDåÍZ09=;²[­´¯m74¶#xØvÒ£!=Ò£O)=ÒkéQO;ríεÇôèï˜&çŠ?hÒ€jrZ >íxø¦;V7 £R‡©û›†Ø4Ħg›Pû㜵à„jÞuÿ“ž'z׃Ó7ÅtìÛàèÅ‚üÌ[·!ؾ‰ZÖØ+ÞÕI£‡üíéˆÏÐ5­œÖ¯ÇÑ\OáÕMu½à^èÆß;-º¾ÚQYñÌO¹ÁB0ï£É­È]³W «Ì“ìvÐ÷oL¸½¤‚Rªn©£fœªniÕ©ÔA¼@Pµ1j‚ªµªmƒN?:P²,UÖTˆo¦Ž²/›¾O  H§©=6šª=Ù©q(ئ&e`«ªT€­í[ïö¾Äµ=íG'øÏ]{o‡•“!;²Ó!;}ÙiÇ•“¢ ëWgs®ý'§ÒÛr àI䨺UýöØ4­9jÝÖØoV‚yY8­FrÝ’l~ ˆäÈ–ÐT¯Î?fnMA-N¹ì©0ÙÙD Y5ÔŠ§ˆ·Â6­ïM™¬Já®” ©\-ÀÐæk’ièü2©mÉ)Ò9å²SÓ&Ò|Ÿ´‰t«ˆ´}8Ò|* °6²/ÓmÙi†·ùÄð†¹ÀñFíÀêí#k*™š‘}i¦cý¸Ûùµ§k8ò½oçí%$ZïaóíxB“yeûÙ•Àªâì!œ¯øVôJZ°Rlq‹Ýþ÷¿ü'ÞØì†*ÈÆÖ¶J„Ô¹çÅ ) Åî7ÿÍêã…ØgÁ¬¨:…³ú/÷À7õã s - ±€-™Ãle6—pÖ-Lš˜tO„zˆÅ ‰¿.Ç$'$„¼@ Ó²¹ÖÓ'DAíï}@û·¡´±¹Î“*ÉÔt‡é‚eOíƒ,íK¦€þ­8/"°}ª¡êøVÇvÿ ý9”Ç¢¤äÓ÷YåÅ…¼H¼’P©½°°$3'åCBWšÀçÍÿPKýƒIô u’PKG9NM settings.xmlÝZ]sâ6}ï¯Èx¶3»Äl¾˜À$aÃ,Ih€m·o¾€YòHrÿ¾WRlâÈv§Ó<$[:çèêJ÷.¿¼ìढ‚7œÚaÕ9î Ÿòià ;•sçKó—K1™Pê¾ð¢¸®(Ї¨œÎU}ùºáD’×QTÕ9 @ÕµW!ðõ´úæèzL¶|òÂ(j83­ÃºëÎçóÃùñ¡S·vqqáÆo×CC ¡ˆŽg#Üœ³Ië >¡Ó¬(Ëћ󅯢̈́åÂbáGÕêgwùÿz´ (ËÊeÆV<„¨yÌ~²á4È cÆî¬z©j¿ö‹Wí«ÍÚp‘#§¹ö‡µ4/WÆYþ©P ñ‘ƒÕc£±á eý™ÂüÕ{œ¤y?ÏùN•1@KŠÐY¿Ô‹_R®få¸zréîâ¼ »~~r~”ýwêëYüqµvvœþèt–(ÿ¨vz–Ù8•€„Ê}x› æÉ›ÏA“‹,ŠaÞõ·d*-Ñœ¦ñ‹š%¾JêwÕÊ[ðc!î4µŒ |G ¾mâWð aʽ«œ„CaXŠ×¾Fï“)Ü9¥\•Gb~÷(‡TŠÌô0þ <Ý‘ø°t–¾À3TÎbú,R·„û T‹ÍÉâ-ïÍE[ëZ’éOKß{¸a4 œhè ¶ˆÍÖúeè’®¥ÚIµj»š]®2¶hµ=²À¸·Oœ~n£¹pšnæŸó†–>ÞšüWÔô„÷þÛRÜ·¡îEKãõ?.cïÅ•`bWb~gŒÈ㹫¿aÞ”3!ñèXž›0¼ÑÀ7à…wUì wÂ/Á]u-"4ò£ÞÓ^ôOSƒpš˜àjFøŲN(žfÀ¨ªòQÌ“÷à³åõ‹VIØ€øªµÜÖÿÚmiëÌi÷©}¶y%ˆTª¾vtf¹«ÿ€§kχÞÁüoŸð|È{UÛC›¤/ýÇ=Ï’h!Kä¸ÆZ;Î`RX àøñ/¬ãG9ëèª8k447Ü$=©UQŽäÀ —œþ)DðÀBl†à‘ðp³¶N{½*ד(ÐÙ3Ë‘Œƒß{:4­0d‹‘yM4)>j¶©Æ…+Œ{)݃±²ÕŸ\5#Ü0”¿·?|ŒPا±«4Ö\Dú‡JŒíÔ`Ž &RòJ¦«hzbC”RhNg®…¯Ø³sMÔ¬p-¾¥˜ˆi”3ÐÉY‰}ôº ÆàwDQådz3­#ä˜ú>ðW_ÊßZëaæ%¤ö« n눑áOA ,S(")Ùi¦­P3”m˜RnMÖ7Üß;?ïeüv %>E°ÂÏÕÔnGn‰öŠ?ä3Û àHÞRè#ýˆ{:Ú[ŽåkÙÎJ[ˆ'\Lémâ=•Q»nrìíRç&Á_N‡2FïP½/uÊOpߺ†£”&xÌaÐïËjNÇ CÊ \; i)úÍ‹oá׌­AüG ¾àl;À±Ú; *’0â; «UnhYƒÅa®"1Hb1lÙÚèc:2•$œ ¢ (ëÒ=ì·ˆ0ª·Íž§_Âìb6Ÿ‚clí‘…ˆR’Œ&ó fï‚Ef™¹(N{öDd…1m§³ïm¦È3|_~HÿÀ¯˜Peœá‡²œÂ?†Þß·°…>Æ-ŒŒÁ‚í‰aï«ÎãêÚÝù"„›öU™æßPKƒðääl#PKG9NMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKG9NMŸ.Ä++mimetypePKG9NMDÞpÑGGQThumbnails/thumbnail.pngPKG9NMû6j|À+Îmeta.xmlPKG9NMÄConfigurations2/progressbar/PKG9NMþConfigurations2/toolbar/PKG9NM4Configurations2/menubar/PKG9NMjConfigurations2/floater/PKG9NM Configurations2/statusbar/PKG9NM'ØConfigurations2/accelerator/current.xmlPKG9NM/Configurations2/toolpanel/PKG9NMgConfigurations2/popupmenu/PKG9NMŸConfigurations2/images/Bitmaps/PKG9NM›À# /7 Üstyles.xmlPKG9NMýƒIô u’ &content.xmlPKG9NMƒðääl# M4settings.xmlPKG9NM¿(ÖÞ9META-INF/manifest.xmlPK6ç:apache-buildstream-27ae392/doc/source/image-sources/arch-scheduler-queues.odg000066400000000000000000000363471514607367700273400ustar00rootroot00000000000000PKvPMMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPKvPMMnùrýýThumbnails/thumbnail.png‰PNG  IHDR‚ž£æ3[PLTEfiv%|%)~).xP5suEMRJRWKTYOX]S]cYdj_jpalsdpwfryhu|kxEu¥Gv«M~¡M{«P§Q~­Q~±myp}„**3…38‡8;‰;AA@‘@K“KM˜MP–PU™U[œ[Y¡Y`Ÿ`tƒtw‰wyŽy|”|~˜~e¢ed¨dj¥jn¨ns«sy®ywµw}±}~¹~U†ŸZ‹ž^žO¡Rƒ Uƒ«XŠ [‰¬U‚²Y…µ\†¸]ˆ¶_£b”žf˜i›r€‡u‚Šx†Žy‡|Š’`¯bµb‹»iŽ½`‘ d‘®g˜ d‘±g»h“·i’¾t™¶v˜½n œq£œv¨›z¬›}°›m”Ás•Àu›Å}œÄyŸÈ|¡É€Ž—€˜ƒ“œ†– Šš¤Žž¨„£„†¨†‰¬‰ƒ´ƒ„¾„‹³‹Œ¹Œ‚´š†¸šŠ½™‘½‘ ©’£­–§±—©³™«¶œ®¸Ÿ±¼ ³½‚ Æ€¥Ë…©Î‰¬ÐŽ°Ó’¬Î“µÕŸ¶Ô˜¹Ø£µÁ¦¹Ä¨»Æ©½É¡·Ô ¿ÜŽÄŽŽÁ™•Á•’Å™•É•–É™šÂš˜Ë˜¡Æ¡¦É¦ªÌª°Ï°²Ñ²ºÕº¾Ø¾­ÁͤÃÞ¯ÄЬÀÚ±ÅÑ´ÉÕ¶ÌظÎÚ»ÑݨÆà­Êâ±Îå»ËàµÑç¶Ñè½ÓàºÕê½ØëÃÛÃÈÞÈÁÔãÂØåÄÛêÈÞëÆßðÓÝëÍàÍÒäÒØçØÜéÜÈàîÇàðÎæôÖàìâíâæðæëòëìñöñöñóöùõøõ÷ùûþþþÿÿÿDZ­]IDATxÚí]SG–^'¾»½Úœ—dï\ö‚VBXòfãS°|¸êp Ø1 WP¥ÝòH°±)Œ1‡ø!äø×. Ë!NKx$$ ’c; 6»áÉZ(¡läù»nfz¤yÝÓ3Ó"fïêDK€Tô£»?}ïõûºgšïñU^¾·À>ûì°À>ûìË·DyY]ü×Ñn²ü¡ªèÞÜ&ËÌŸª€o»Uãß~v©Êج66»«‹‹«k·o|uç7ÕȀͷnéªy+0rkmßö¨zžÎTSðëÇψò´íëÿí~朇wã¯À7¿ÿí¥Ë¿½tiæWsiæ’øú÷ÿ­û7²'Í&óÉ\­ýv‹¥1Ã>|‡s%—ËE¬4’qXLõ§v¾‹*ؘc³g…ïç$óð'7ò¨üö›Ád<ð Kž± »3Wà ¶å½ aŠË¯¼§ûf)Q9b*²œ²—«­˜Ø vZJØÚc{ À†9[~팱t.kV\9΂™· ¼±&XÚ°9Á땽à|†–'c¼·Lj3·2´awÀøq°¸wdfáÛ“ÆÃ)Ôc~ŸqÔánÏ€³›Ø;Bf"Æä$¦²F#ƒF¯¼†A–ˆ,Yï.ˆX_3ø|¼%ÛA£4†[äìmúz`Çß/›-F([qŒöƒ» ðÚ¿>;ÍÄ4æ8`0fë‹Èƒ¸a%¸øú¬AÌ «é€·ŠY¥„ëÀÊ”Ý$ͱy¥Næµ ° E'‰Ï; R½_×"u–-à %+(š·”J Ø«œ>yKC=,åeƒ%¡"€ðé4™ëµŠÕ¦&€8õ™­X h‘£`ÇfÕlìò‡I‘µN­QÅ©€£é^T«„Í95Vêµ ¢ÇI²ÅÅuWÛä´•Bç»Ú÷ÌmŠb ×kw*êðjP¬ÓNô4>ž¼Ø”í®ž…LD=­gÑ€ƒÜ(Ƴ×õ ƽ8¢zŪ€RÍÅö‡Çü ­ãx@›¶Þyt,ì? ¢uÄ &&ͧ•ßùÅ2¤Á'=)B‚NE¯L„ý£þQ@‚3RšeÂ:%ŒcböªÈ@xôÄØÛWÞmãS˜Eü°l€wÆN¼ývX“عÆq`àÿÙÏÞ™8qÇ< ‘`âÊ¡‰ÿ ºÙ€Ï«õ<€Ð©ïœ8®¿ÛØè¿Oh• ‡„‹j°! 0 2Zس$cBÐà®’õ"“Ð+wÃ8%5…¹€ªà8Vhq°ƒf4NÅh.€gÒÆ à|>ß(¾i©µq»Ê=K ˜ž>á §@ž eX-ÄGÉdI @Ü ðá½òù($Ð'zå“Ú@°p®gpf–™ÁÃ!‘mÇÕ:AÀwøòŒP AÏ9©),©E )bÀ‘¢ÁΨhÜPèmD-Ì”žƒ¿XRÅçŠÀÒƒdÝ5/)PE']@ÆZröGj‹žG4å+ôϪ-}$æ¢Àáçê¥êvU ¥¶€ë±º8àhËâgE~ ªN¬( h~NN?‰µ5¯¸€{Nm1ß'æø}hÈì–hSl!] `áÚŸªM~É@T7ø¹ ÈYÖKœÙëWÁðLP€Vé¯ÞV“7¨ €» ê °ºúq0u{1#C »€~ ˆETàEÅ€’ÐC.0O‰’ È$ÓŒ’ @”h©z8n—APrÀgF唘^§Ê0 |…A0kãU ç¦]Î_ž#`+´S&œb,”:*¹€gVƒ3€¸px[clPÓRfÀ³€a*Ü>ûˆ,m>"”È&0ut¡\óSôcð‡3 ÑÀµÎ“MÌ¿ÇáœÚ€¸>Ruª[ŒšŽ À×ý)i1ãâ˜Rá~—XΗŠë|o(Š3 e-BՀř {ã€(×'Ö‚&ŽÈË«¨SSèËFâ‘cØ4XÞq@©ðÒ‡çÏÃ>¹Î£ A rrÑCe²ÙXÅPy± ÊaN~PÅ…Ç@ÕUý (Ç@1´{5¨0 ¯€rç˜Õ Z1 ogÀˆ19ì]Á\€Ë°óŠ\º_.ž~ôD¾PšJ k–k‹ÏP”Pƒ ãHq£zŠYˆˆʶT¹SØ/O¿dR°cÃàÄZå"!ç1žfËeNzÌ á °å!\ ª6'?…¯v|(â.À”MÊAbÈœÄpu‹ÁzõË΀òD#ÏGÅ®”ŠdqÑÝeðˆÈÊŸ§ä½ j‹‹x"T´bxæ å°²m*pt‹ž c´ùÊå°C €Ä€ÞO 3Á‚sšžsëÐmœ Æ#¼Q*ÜÍ’ÉÒÚZ@ù<%>”H­­k¯( RáçwÖµˆù1Z¤:‹I,ã T8\Çä0Ǽ0tµf¨³vøúU Êyì×Ó‡. uv¢‰¡Ä$À’õWk‡»‚-0™ 0`øZçp×…šëÃ48r|œüûΡŽ Ck;Á`2L=LRsPLÓÉÕô-ãå°y[hc-˜N¦•õ-º³L¦‚M;²$“·’éÅ`‰ôž%vЋ 4cÀÌ”.¨jÐÝÐøÂ8,±/‰‘Ĩ \àÃO)0¸€jM ÃÈ/xC9Ì@«Úð‰ s‚´á\ ál`.Ðÿ;ŠƧÁÉ"Àaõ4ø”ÃÊÅdçïrrŽ¢Ì¶pœN XúǹyP[|=àÂSáDO…|D¶q¹O„& ½Ýó„Á|‹w€é£³¥þ”’€fH…}½}£T„×(O¥1i¥>T$;éçtT®¿(ý§Ç€¨¯èTß!‡ Ä^¡î”Kˆ«`sTO æp`C€q61D`,†¼¼†Z–¿^ÎÆ+À_¨T *Ÿ'«nâ Ôà2»ŽÑd±2 æb¼žŽ©P:gÀý `a@¥ "œ¯ä3n¡ô OHŸ\ÙËÜâ—àw‚€Æä°Ò9ä!U ·‡c€^žŒnñÏ—“‹êÁhÈ]êOé1Í1íŸ$#ô|k1|hYXøDz.”¾_>Ëé1€k™]ødþø5KÈa‚çÈN- øt0Õ6¯ôGživ-‡Ÿàû‘E ËâÓÞPÏàËâõ¼‘~ÞŽ­ yC9ÜVÁÞ ‘ by@$FѺ™ …7”ÃD*lá åp3Á•Í¯$¾“’›a@J…“kOR XxŠؼý´Ráû ô4÷Ž©×Dq9 {%`ú ††k~~á­á‘Š ã.€p=õµ]]oHæ¿ë×ÉákovÔýx¨3hÌ€6Y_èì|ó‡×?•»Çôp#ùýŽ®Ú’®dwø7©àCṞ¢Èa{ž¢×RÁàgéäzÀ€ãÊ‚ÈmA §ƒë©ÏÕbHɶ!d2½\EÚnÆb°ÀX[Oד\Ðc(‡ hjg@ÑÊ3ªÁûê;)4Õ ”%$íP bÊ]/ÚrxJÃÝáM†lã1 îç åp7Ó‚H eüs…Кr‘­|¤w©AÊ2ê4@YÑ”Ã_Jrø˜j3UG3]#ímKðÒ2üå#Kz,ýhpn,¿ Æ{±TXY‘Sá3˜…ðhêà>:[ÞB˜Eý;Œï 4æpB‡A‡$“Vw”aL=ýnyÆ->ÝÄÎPN‰·H pÇíq Oô¢ßµ@,Bj)©¢d ~s{¦ñ!e…³´3äÿ.0AúY‰vArË•Ë&Ç&†Äý×Ò>,GQƒY¯©9ªtfx-9ÌÑÕ ¸U¦ŽZWÞ$V`á©b AÞV6f€‘^ñòÊa€Ž× RÐUƒìfsÔhwXIYw‡­Ë*ƒ 0óºÄTëð?9lÄ€ëLó™#gš•rÖ1@IdxüáÂå0¸5 ¹€¯]ü»ŠAssû4tfœK®zD®ßÜ,]ø[ž3€©^,Â;Æ+DÚˆ)gkëb? ],ݪšr½òÈrø1ÖÆÖöÖãƒr8î%p]&êooÍöÂL0ì'èïQ_#Ôe×ê ²J.ÉfÕRƒps41Ág‚sPƒYÐLI„Z  4fy5Èí~cDa¼ß—mgÈ»ÂÓ´€Îî0¼_R€2 …A&ÈÆ€›Ayì*Â1Ä€àÚOÞT¯€ ðì5Ï·T©0ð2À-¹S¨V@^3 °šL‹î ®è9t¡¦¦]û,+`Í. ï'ߨé¨íèøšp8`Eèó‘šÚšÎšÎŽ É€;€ákµÕt—(¹@ÖÁSðqòïj;~(t«b5¸øpuýaꋯ’ (šy*ÒiA ¯'Ÿ­ª‡Ã.ðçÀCÑbó³u’q?€`*õÅz*ùì¡Â€cª©cÀzr5½™J~õpKÙ°ûŽ7Ll8ø cÀr„g‚Ñ´c@+CÐËtb@Wˆh0 ±¶²¬ bw3m›‹¬È1£¥ÎÕâL  ì?r+À€@Yã}Þ–ű;ä5åðÅP€v#JNžè^$µÑ¨ÀE¹“éLr¸ï½Ö6©´¶µ¶~ <º[]ÊæhF49ü×–VT]þÞÚö¾G ‹Ýn‡Ráv¡îô@fïŸW¡È¤ €© ’å=°/€ÑR–Ãÿ4@ZégÚª®†b(Œu®¬8NÑ\Iw!šâj8Ù‰0çUD—¦¦|ý>ñK¼/Q,¸s?ÊAÖRÅ)å9Å}'1$OƒØ!0LjGÜ`wø>9 *bHD•“~€+¦6p SûXÉ+Pƒ" ü<â¸Ãîp$A@gwرB âÍQ“îÅÒïŠÓ ~B ƒÆ)#0àžìEvîÙTqiïÖ¢‚8ÆWÀ± qŠÂ²þ‚ˆ€V~œ¯€bHòiœÄ(½Ò€·GµoÕ®¦sÛñ¦â :;)vŽ8Ô†Ïêq·^HSäÙ4§5:$uJH™ äYQ“ ôÊRqžÖ g³iWÒPS@ñõš'œ4œ,Š'g‘÷u,„á䛃»ŽDòðž¢v¯ê |UÇè¼È¾JVx¼‹~ˆß²%ı?òª’— Ärò(ÈG2j‹`ñ`¼y‰]Šió‚h#›û„ŽÑ)˜Õ|ib½w_ #tÄ·ÇèÅN¼ä5N×ð¬°yìÑ£æT‡óxW ,þEï—º˜_îkt"Ù—¢ÕèÀÀ~0‰àÏFmŸwÎn`À[v €ãÖ7“Ô|œð_gÜÈ¢H¥å0´pàQµXŸß;0‰Q0Çø ‡“0<+N`4vÒŠñ†qüä FãwÀ7 Å.üdÉx4x«8ù:ÓYž£KõŠ“&¶`ãå5óˆéŒÕ¢)WÞ b°Øyý€Þ |Fÿc$bóG"“6/ã¸|Ö„X±8‹lÅãùã÷2¶±Œ -†M1–ê'¨îßJl¹¢3“OÙìv»ÅVÁ)Ð~»d‘`6ÈÙ‚…É©O±Ï/Éåo-¿øÓnØEÉçó…=¶(Uþ0ð”<"|æ?þ:üß(”¥Ííîj ›À·ÕÀÚÚgÒ¸ÿ¼½½¸Yu´IwMvtÕÖvÕvm_¯½>"𲺠]&ººº¾¶¸L-¦ªÏÔ1àeu¹@uÁÊfZO5Mƒ_—þS\Où1ðu5 ý×À—Ø¿ Ü•øÿ^öØ`€}öØ`€ê-ÿ¿ÀËcoIEND®B`‚PKvPMMmeta.xmlSËŽÓ0Ýó‘™mb;¯ÖV’‘X°EbW¥¶§;²Iù{§))­P—¾÷œ{Î}¸x>µMð&Œ•Z•GB1Í¥:–àÛîc¸ÏÕ»B¿¾J&(׬o…ra+\ŒTeéœ*AoÕµ•–ªº–:Fu'ÔB¡k4õBsäÔHõ«?œë(„Ã0DCis„˜}vrvÁu½i<Š3(1)Xˆ# ìäðQSvm©3ÂŽÙÚù¡‡gìQ(1’µ©^äÁˆÏ¾s˜E8J#üô"UÚßæû< V€}gôOÁĨEOzÙðŸeþVœ.ÕNGidëÃTa<œ^¹äÀª€W“‡÷~{õPKÄjUÿ¾+PKvPMMConfigurations2/menubar/PKvPMMConfigurations2/floater/PKvPMMConfigurations2/progressbar/PKvPMMConfigurations2/toolbar/PKvPMM'Configurations2/accelerator/current.xmlPKPKvPMMConfigurations2/statusbar/PKvPMMConfigurations2/toolpanel/PKvPMMConfigurations2/popupmenu/PKvPMMConfigurations2/images/Bitmaps/PKvPMM styles.xmlÝ[Ûnã8}߯04˜}St±“N¼ ö‚˜Ì.¦gæu@K”ÍnJ(*Nòû¸ÿ·_²U¤(Q¶äȹö¤ˆcV‘¬:u!«¤|üá6ç³*+&ŠK/: ½-‘²b}éýúË'ÿÜûáêOE–±„.S‘Ô9-”_©;N«L.ª¥!^zµ,–‚T¬Z$§ÕR%KQÒÂNZºÜK½•Ñ‹M®™ÝÙŠÞª©“‘·7—¬¦ï¬™ÝÙ©$Û©“‘0u§gbêäÛŠû™ð‘—D±)n9+¾^z¥Êel·Û“íüDÈu]\\šÚ œ´|e-¹æJ“€rŠ›UAt–7§ŠL•y]‘Š:_Q9¢ÈžUKI+`uÑ/§-äÎéù×Íz²wݬG`N6DNö3ÍÜw•y:ÝUæ©;7'j3bßóàˆúÇõ_É|ê^ÈÛƒ*‘¬œ¬¦ávç !ZQq‚ v-n†‹À|w¸·Ù·’)*öä {BxÒ".ò!Ѐ/ €Ã§7èò­Þ9㓵Þ'!› =òDðG5< $-…T- Ùô¤ »ÄmÊبœ§ ¤ZÖµLÓAVg@ú€àõoÝ~çõNƒÃŽp±ã:µ>4E3¹¹÷à„( § _pî ëö(ËD]¤Æ@z[RÉD¸ž¶ì­àZ‹‹G,ٜΠ½CQn3E«Òà2Bøyå³D”Kgv/kUÕ\ ™ï—Ÿ¤ùx"BÎoöq.±weOýLÀ‰Ÿ‘„ú)MxuõÑdëvxf¾£—Þ R¿V~ö™`R-+ÎÝ¥÷gRŠê/;|fЛõ–F~M Ð_Šœ=Ž’©Rã ‘LûF0]4X2›$2¾†pÿ _ÈoõaÌž "Uw•¢ù“d’‚¥z»Ù'ÂùŠ$_ÇeÛç}?IJG@sHO•  …fÜ܇¯>bÊ[~¥r¦7BþUJ±5b®ü›¸½ôÂY8‹ÃÙ<4ãô׌q?ÂÁMÞãÆ# ~¾ËáÒ ü‡¿7«krʪ’“;‡m6&AÍ£ÙEwBœž-P ü€ñ 2ÜwFHiFjÞ´ïµ$å†%žåm¾û%$(*ƒ‚7©”_)œ \Àñ»ùâì”,ñ3ÆyKù_d Di&–[XÊ¥¹ ÂÇïP%‘DoÖÛJ“0¿ù¤V¢* Ö(,¥Â°^nˆµ{Y‰ªuÈëµÁ/X^âqcèp‹¦þJR7í U–‚W¸Øû¹Hay.}µê9+RŠç1VMzDWWámÁ‚Ó @e…Þ6®VËŽzíi[WÔ‡2#[_oÞ ©dM{BåC@ZÓ+vôxQ*=ÆI±®É†h¡8ᔓÿúyo].#¤ØMh®myÌ ÄÍ&–v¿±”f7KøûOû{âuŒÓÛvm¹÷m©¶»sKúçO^gŸ^,Xøa`®ë€L½ãÃEÇ„‰XGp–znmYŠ¥A˜äÞƒ‘erÈ •I;ñ$Æ©ûô„âµÂºh\ùÀt¤MƸn5tMA×&jþFHv/ð¶ÁÊÖ`ú/u¥Xv§}°$)v)|ˆ%ŠOQ‡°Ja0Dã4SZ]‚dëC1ØT*,M1œAòE•ÛG£O¼$ZÝÏCü^ ›œUýäºãC.‡†fUsNÕÌqîpžùjH>Ö¤—ÞÿþûŸÖáœEŸÓsrVøœ¬€ÒÚú ÆÒMÖ·?ƒáðY îMIMý¼³8ý^GÙ!MÀ!~úÔðWêdjAxÍŸ ¡Èøù»ChñŒ¿K„NŸ ¡ødñ.:{6„æïŸχÏ;ÍÓçφÐâæé‹gDè}æé(|6ˆNÿX‰Ú!7[0Vxl,ÀÞPí¬QaSE4%D3h+ˆþ¨©RúcmÒ ëöÀ†šùQ~¯G5rL7 냘êÆrBæ„w«( e±yÕFñPv^þk¾âÛføfwâ[‚ÅyM}¨ë íYéßס7ÀsÀõFŽ‰asEƒÕû5ÀÍã“Ó(v m/ú (IÛ+%d„dÝhà[é’0$G•q½N˽LÙü©Œ ´À¸ƒ{^ˆ̵ķ®{­„Lñh|° FmÖrR)í—wÝû á3…î»ûv¦Ñº}}sÞÉ;…Ç´@ªqÖU ~€£I˜]5W°«‡¥Q1Þy7Y¿èÖ%ŒÎI;¶ ¡Ë"Ú¸Î˳;XÃTvõPKcHþ”6PKvPMM content.xmlí]moã¸þÞ_aèÚâî É¢Þån²è¡] Àn{èn‹k¿‰rt+Kª$ÇÉý‰~ìÿë/éPo¦Þl&N6NnHl‘3õ 9‡œhß¼½ÝÄ‹šQš\HDQ¥Mü4ˆ’õ…ô÷OïdGz{ù«7iF>]©¿ÝФ”ý4)ásÚI±ªk/¤mž¬R¯ˆŠUâmh±*ýUšÑ¤ÕZñÒ«ê^uIQÞÅÂê•0¯]ÒÛRT™Éöt½+ñ;W¼v{;Qe&  òêa*ª|[Är˜ê›Ì+£A/nã(ù|!]—e¶Z.w»²Ó•4_/‰ëºËª¶ë°ßÉeÛ<®¤IcÊnV,‰B–­ì†–žhÿ˜,ߥd»¹¢¹04^鬚å´x\60Åâuzãëf-<ºnÖ30û×^.<Î*áþPÑñ¡¢¼îÆ+¯gìë,?@eõëÃûý¸Ê7¢÷b²=¨ü<Ê„³–æõÓ4íºÊêÉ^uWSUcY_sÒ»ƒâ»<*iΉûÅ}/ö;ÄÓÍh G– !Ó6ä»IÄ€(f´e]Ý ÁlÓ?|xÿÑ¿¦o/–£¤(½dL±‰ba+€ìÌ õ’Hx(0ÙÑÔÉÙ`˜EÜ\æ4Kó²3P(Np­ÃèºÜÄó.ŒÕ¶¢ë<&E¡;úÜ8ù&¢»¯¤;˜î``V®þ˜J%ÄsÁA¢.™LçN`¨î‰+_wܦÛ$¨íPHo3šG¬Ê‹+µU¯ÞZqú€&>æZè‘TDãÖsu4ÙLšÊ›2LØ4[qÚ}ÎË7·bͱIšá°ÅÃò‹B/§Æç¿-YÌ(H­¹·ÔѤËv]Sû±bÙ„°¾‘Cϧr@ý¸¸|SóSW¼¨¯Y¿/¤÷]ï⣗€—iEajÞ]H¿õ²´øý@®.”½¦™¼¼¦ ®.O7^Ò“È¢Ò2¸ñò¨}Kñ®A“¡Pߘà—èÜéOÞ?¶‡1ãdºTÜ%ݜԧ<‚êv‹w^_yþçù¾e¿Hßå”΀ÆUÚƒåÜThʽm™2¾ðåªnŽT¿{ý 2Òݬég³–3oÍ=îXuT×¹—]G~[œy9 Hª ¹VbLxy µí6*rn„æeD‹ ¹¢ÌÓÏTÞE[_©Šj#ñU~§°ŒýJU-K—ú ˱ü3ÍᎰÄÛ+ë¦É”y ½éú0Šã®m×õ}×mjÒÌ󣞒¨êoúeÍÃ5…Ì«Á“{òušG?§ÌeÊ^­Á³ý´-Ê(¼ Þ°'÷÷b›ˆ”6R̘ò:Owò5Ö×àPC/. 6LW›(éJEwØc4ÅÍÓéŠjÛMqæ,x”Ë4cMÌAùUZ–le6QÓ°dš=¬Éë»·Ultr#æÐðѾøðyŠ1â‡ÔÍç5½ª8†5¶=QË:OÛëOcû µÚøR’&tÆe°œ1[Ù±uÃ꟨ÑFfq ¹.Æ/ÇÕ‰Ts^Tc»/Šk,?g6~^ÖZÅÆñsfãÇ~QãǹÿøI¯~¢~¹‹Êët[2[Á@º>Þm6´„XõGMýñ9ØêÀh3L»7Úzë ‘UÃYYñûQ¼ Vó*c€iêݹ¹µ—è\ëâí®i¾)x†Z ”OÙöÛ=`q=ðš˜çi;; øÎ>á}GKðûttÍü@Ôjª®/^mÑÏÐw'+{[G¬TöŠÈKfêØÉDLoëZqˆF«ñSM3ûp-vVõOºJã@ØÔJmí´jÏOL(wõ•º0¼ŸFŒÊVº·³«·I@ó8Jh;Üê)ñb±‘õ1laN>˜Ë÷ÂZxªßß URM‹ý3;"ñ1½¡qsã«mÓrQW²rè¤T_ÖU2K<¹þ÷ßÿtFäáLYé°°&ö® ¦[M[õbõðÜhËGX[ÃZ±…`[°Ø ºª$›áRæ[:+†Y­D—‡žô8h'àPdìx늆iN[^Bú£!DíU"d<"BΫDÈ|4„4Åx•Y†þ*ñ±ŸWê§GCÈx¥~Ú}D„^§Ÿ&ê£Ad¾,GÍU7kíålÂRSq•wÝE“™tù¦Š¯Y~Ri×ksvM¤ö¨û®ÛG¯Rœš-ê"ë*±InÓÜBo³¨ªð·ôC.®½ŒŽªžº­ëÞ^=Û¾ÝÄ«(€À#бþUìݱ¼{øH·e½-ß[8šÝnÕïÏ8§-ƒXÅVÜNä®Ra{×Í°Ë–ÍÐäše*òš¦l›þ®’g)·ß¥ÐˆºP±Ôæw K”çiÎátÇ.|u»K̼߯òÈié_gi¢ Îœ®ñ±À]_ ÞúÈ©W\HoC}ñ64àÇ„«•¸Ë(Ki„¸TΩ_zɺ;1Ú¤AF4eâꚢ™Ž©¶©;íþzIV¥ê€†m€áøy:‹÷Õ§»øgU^ƒó¾“©šO&ô¾ûîT „ú¯Å_ÚñCÿ½­ó4¹av i¾ÙÆ›,Òò*üZ]|[DÉ×oCøòu-‰£~óÍáF´a#ÐÛÅ·:±ô¥méÖae}¨ÌS~÷6Ô‡õŒ¡^™fjæP­:¡‘+ZCÅúë?úOb‘ÃÔ±+º¿íUzÛúÓ&~áû¡·Î¢šd\ÿ‰H—ªÿüµ ˜\Fd-&û-÷€œÇhD–b›r r r r rŒÇ hµ+‚ÓƒHç ‚kSGŽAŽAŽAŽAŽã˜u\ä˜!ÇD,ûªRB’A’A’A’A’ ™¡=7’™ǘÇâ½Ûéß¿:xìM&p3tÿø±·(jæÔ4éòûm?/›³ÈõÌr¦r“ õe7ÐCчØá—‡9ÈßÈß/”¿Ÿ9"¦¢Ž’ð@ã ô£èG_’}¶8hʃâ‰ÎË ÓðHYYYYF”e†yç¶Ýv,3LÀ3ddddQ–xP$™1ÉŒ2ð/‘eeeeDYf”:@ÎŒgNÍhÅúW§åÌ€‹â&=ðŽÑÃS±³5 ^/}Àîg‹€7>À¿p@Skˆuy˜>€$Ž$þBIü™£!(ú(}v0B?Š~ô%ùÑg †¦<(ìL°Ì0}veeeeDYfàA‘dÆ$3ÌÀs$$$$Q’fœÛ±Î9°Ì(}ÀBšAšAšAšAš¤™Qú€vfðÝ6Šƒg~ûéGDÀxûÀ<|ù€‡ÙÈáÈá/”ß;šHÀ„, …Т}InôùB¡‰Ü|õÀˆcðÍÈ1È1È1È1ä|ñÀQŽÁ÷ Ç Ç Ç Ç_84áAqÏmL2ÃÜÜsC’A’A’A’%™ákÎmÏíXf˜=€Š,ƒ,ƒ,ƒ,#Ê2Ãôsû{Ðs`™Q4ƒ4ƒ4ƒ4ƒ4#H3£ ýÌxæÔ ‚v¨ö¯N|ïÀì¸(n ßo‹‡ÿ¯i’+¤ùpÎ,p3ÿ/Ç8þ6¿içJ%0‹>ÓHUa$-Tø 2hV‡a™Î~IDSì=¦Ú…ä*ŽmvZ*»nÁËËzÞð¯@¨‹×ñ–Ív`R°MÇwÁ^œ/ä…Õún04>èпÑl]õò4×\IØ bêjõM6 Ý)t¢™²eëdQ}3 C“Æë Ks¡-¢Y.¿YöMól¦"³Tk%›·Øàˆ•ˆë°‘™ÈœôûÙÉ8n'×áídØ` bNõM6tÛ]¨²Œd;`.W­¿¦ªN™K¯Ì¥ÛÚYš‹¸{{ÓäíŬqÄ`PÁØ`ÚœÁÌûÌ:n0bi=‹Á¤‚ùTY¬šc›E›Y`*—0£Á7˜mÆüÌ2㟟Ϋ§BNÝòåå¥kß®‡†BmG¸=g›Ö|B§Ç¢,GoÏBlD› Ë…Yᕳ³swùÿz´ (;–ËŒ-y"Qó˜ýd+Âip,Œ»·ê¥ªÃÚ/7ÚW›µå"§±ö‡µ4®VÆYþ)Q ñ‘“Õc£±î eí™Â|ã=NܼŸç|§Ê ) Eè¬_êEˆ/)×N£Tý|qvåî½ ¼‹~~YÎ þ;õõ,½ZýTÉ t:‹U_9/¬‹_ HX¢Ü‡ðw¹`¿Yvú›\£æ]G¦Ò=Ái¿Hi‰¯’ú]µ²ÇüX„; -#Èß‘‚ïšx>!L¥Dïª'áP–üµ¯Ñûd wDN)WÅ‘˜ß=Ê!‘"3=ŒÿOw$>,œ¥/𠳘>‹Ô-á>Õds²xÍ{3‘Yk]K2àIcÉÛ“Þn ('ú‚-¬Ùzø@7 ]ܵTþx–òÆîª}®"¶hµ=²À¸·‹O|:o¡¹pîÑ?õtZúxCjò_QÓÞø¯Kq_‡ºM×ÿ¸ˆ ¼mÁľÄìÎþ1ÆsWÃ0¼(gBâÑIynÀðF߀ç ÜUÖî„_€)ºêZDhä6£ÞÓ^ôOƒpKО>…G±¬ò§0êƒêƒ|óø=8Oyý¢Ub6À^µÙ3ãÿiÚ])8íN¹—&l "$ª/W>§4ü?àÉÚ³¡w0ÿ;$<òAÕé¡MÒgEÿqų$ZÈ9®±Ö¶LK?þ…uü(f]e³FCsÃMÒ“XeH zÉéŸB<&Ä+I½n‚1ø‘W9™ÜLë9¦¾|ãKÙ[k=Ì\£˜Ô~µAÀÓ:bdøPGƒ”)‘”ì5ÓV¨G”-˜RnM©n¸p~ÖËøõ@J|Š`¹Ÿ«iº¹%ÚËÿÏÒfß@ò¦BéGÜÓÑÁr,[ÎtVZB<áb 'h煮Úu›ã`—:3 øb:”½Cõ¡Ô);Á­½u G!MpËaÐï‹jN[†!eP¬†´ýæÆÅ‹7÷k&­AüG ¾àl7Àå±Ú; *’0â{ «Un˜²³a®"1Hl1œ²õÐÇtd*I8DAPÔ¥k=ì·ˆ0ªwÍž¥_eaAv1›OÁ1¶öÈBD IFƒ‰y ³wÁ"³ÌL”1§ýøDd…1í¦³om¦È3|_~HÿÀÛL¨"ÎðÀÃYLáo¡÷-ÒBãFÆ`AÈÄ°·U綺v÷¾á&}U¦ñ7PKHÓ1šl#PKvPMMMETA-INF/manifest.xml­“Anà E÷9…ÅÞÐfU!;YTê ÒPDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPKvPMMŸ.Ä++mimetypePKvPMMnùrýýQThumbnails/thumbnail.pngPKvPMMÄjUÿ¾+„meta.xmlPKvPMMxConfigurations2/menubar/PKvPMM®Configurations2/floater/PKvPMMäConfigurations2/progressbar/PKvPMMConfigurations2/toolbar/PKvPMM'TConfigurations2/accelerator/current.xmlPKvPMM«Configurations2/statusbar/PKvPMMãConfigurations2/toolpanel/PKvPMMConfigurations2/popupmenu/PKvPMMSConfigurations2/images/Bitmaps/PKvPMMcHþ”6 styles.xmlPKvPMMT*$Vé Æ#content.xmlPKvPMMHÓ1šl# 2settings.xmlPKvPMM¿(ÖÞC7META-INF/manifest.xmlPK6›8apache-buildstream-27ae392/doc/source/image-sources/arch-scheduler-run.odg000066400000000000000000000335341514607367700266300ustar00rootroot00000000000000PK»ZMMŸ.Ä++mimetypeapplication/vnd.oasis.opendocument.graphicsPK»ZMM ›#VXXThumbnails/thumbnail.png‰PNG  IHDR±M“0òPLTE$)+)-0/474:>6=A;BF>FJ@GLAIMFNSGPTHPUKTZOX]PY_S]bV`fWbhYdj]hn^ipfffkkkbmtdpwfryiu}kxttt{{{4e¤7h¦8g¥:i¦=l¨An©Eq«Jv®Mx¯My°T}²ewkw…mz‚q~…_€ªUµ\ƒ¶Z…¸^ˆ¹r€‡t‚Šx†Ž}‹”`†·`†¸c‹»jŽ¼j¾r‡¢{«p“¾n•Áu–ÀsšÄz™ÂyŸÈ|¡É‹‹‹€˜ƒ“›•••ššš‡— ˆ— Šš£Žž¨ ª‘¢¬•¦±—©³š¬¶œ®¹°º£££««« ³¾´´´³¶¸»»»€žÅƒ Æ„¤É…©Î‰¥Ê©Ë‹­ÑŽ±Ó“­Î®Ä–¯Ðž³Í”²Óœ´Òœ»Öš»Ù¢µÁ¥¹Ä¨»Ç©¾É ·Ô£¹Õ ¿Üª¾Ø­ÁͤÃÞ¯ÄЭÁÚ°ÅѲÅÜ´ÉÕ¶ÉÛ¸ËÝ»ÑݨÆà¬Éâ±Íä¼Ìà´Ðæ·Óè¾Ôá¼×ë½ØëÃÃÃËËËÄÏÝÒÒÒÛÛÛÀÏâÄÓäÂØæÄÜëÉÖæËØçÌÙèÓÞëÉàîÆàðÎæôÖàìÚãîÞæðäääëëëàçñäêòéîôìñöïóøóóóóõùöøúþþþÿÿÿå¼»{!IDATxÚí]WGÏ»^ï%©}&1j0à”«=!µ¦Ð;T,Òblr²áH í…gòb¢Æ¨æh‰»…>"lÑ,Î?z³ Ê•ý 6ûÝY$»ìd¿Ÿý~f¾3óÙsà—s â uœ(:ò'Yw¢rºü‹ªÓuçj®C̶uĨ¸Æ§Žƒ¥öI˜ôL¼€©óDwžPã$<À<“tÂÕ$®¦òj"¯Ž¨aš.˜ÔÀK§x¬”*ÙzŽ²Å0[’Pg‹e‹Mµ€¸&ò§ž%\r8I—†’Ê7ÿ|”@Sy™- Ú ¼¸œÄÔ$àAU×ä œ¨/Kc”°›"òòŽÀò̧&c€ c1x£ŽS1hFñ8ˆÅc° £€†.ã¡ôNãL\.¨Ip ¯Æ}|ŠÔàXœƒÉ)A:âÜ¡‰«1u –çTü™Ã«ÎÓ{Üyˤ<à(ô®Ÿôª)ÌKLy|ôuœ¤(‡‹tMºbøì!¥â¸kp¸Ž’=pÌ% hT­ó‚IsLzqø•ìqy Á€wÒEL‘jЃ\ãòP— î\35%â­‰ €ËÇ€²C;YvéH’¢ý:úý…> Á@ïÉëÖH¯øò<àX§åù]Gª)é xQ@&¡d°¯· bÒG0OÊåšT»\˜ìàÞ6¡ÖÝÝÝÓCü!û¼(Ï•Ò@Z@fU<ì{h€dìîVSòàä@÷×ùTo¡»%¸Ôö÷ëÿõ¦3…ôÚrõ¶üß¿×X[ß+že äË~›¾Šþ7†Óƒy‚EÆ,{ÒSÀ#;A‹ÅXŠì@ °~‘ֿβËcýþÙ£@qÓo[1^­~øªZ-ŠþËY£À¶Å`[EÝðñÇ­œIÿì¯Æ³EˆÅ´ŽôÄ¥] óâ°o3¦køÆê-q €7’@‘¢ð\d®¶p‹$¯\Ù §v#™p2É4ZÀá™p„ìƒ9ô×ZÀ̓ѾïïŸëQ9ÖÀºé P ð•õT¯…ÀY, |©óYgÇýOÜŸ>ýÄ]CA3õÛŸûÆ`“'ÎV¤z¢»»ÙÝl&ÿ”¤pÀ@zïvJ[ûR 8ävA•šÖzt%µ&“ù–ÉlÒiK›­$þ¥µœpâ„<´7mç±Z@6{ye3|}e7n¨Bý'(ÒÛaâ-½-ífƒÖ@#±´½ÓÚþ€õ[‡»Äü€éÙ+—ºFÜ{ë„Å‹­I°—H„‚A“"ŒÔDÄe¡@ÁXàŠp½ untvuuÝûâ"“@Èq/š0hmKÕ¶÷Æ/‰5펈‹Ð'8Îâý1ü ž ,wéÏ àRGüF­5T± .é!%¦Àº‘¥ÙÎDé<ÁýȸÎø|ÿˆ1‚ôIM}„;¨mž®ðºýF±#J8Ö‡ÜÿÜ ›ê>8‘ƒeC@@_YÛ 8¼ñ+’°8Æ_‰åŸ.€«Ð1Iø?ë @€ŒìÎ#á@ øÖ fóM 3Ü,–›5R>n1›ÓÂü™%¾rÀÈÞoÉ<0ò. % ·TšNéôcKúDJS‚{·9‰-`Û øY™ì´u¶Ò–ÖE`S €ß.-LÛ€'ä±°¯ß“Ò(êD³€ÿI°¥ôß ˆw§ÒXØ1IIu”›n)ÀAJ Ì¿@q·ãhLH@QB X~kw Úæ¬Eé(p£ "Æ¥Ð|5ûºÂ(P0)`—Fÿlöµít„ îD»Õbqm¼X<[ÿåFÕé6 l\ €T ⑲:”¢_äÑpÿêQ³rõá©NJ^z Pb"´þÚ‚¨úýÇ ë…yIjˆU¼»µAž‹ªÿOþª®…ÓF$?‰ìiE&@p®¦så„Q`ýh€7¬ý]Ôç?W׽Č€°Z`]@Ͳ›ŒÕ›ÑxÃXsÀd èZ÷7t°ÝfjºsŠzŸŽÔJÀRw ý‘‡6óâ/,òzë˜kUé4–iìbÛd*ÐXÖ Y¾n*Ã:†(¦(1ÓÊ°@Êâ§ÿ[[†)?8Ô!ãö[ÎXlH„6ƒ40òÚ(Þó/!°ÎŸÛƒoîÙÒP—1„É%LF†\Ðlcð¾1,_ž#^oÅoô¿¬rfSÙd6ûýüËl2ÖÆÑîy!€@Ô±AÆ09,æõùOÞã"µ%y®2îváÊ·ç7>r_éøáó+•ç“ãKù9´{~@°Qq‚æý†‘¤Ë›Ç0ÊëÍ»jLìwKÅTUO>ëTÍ\ºÜñê’³ÂQóO Ì/ð &nœ I'ÚËM¬+ͦé@±Nhü‘d4N$ÃÑŽèqÁŠ-.("Â#EýúÈaþÕ=ùºUƒúœ!þ0”3õðt¦÷©*±;ól9@â P#ûëtcòݱþ©Oú^:SÉL8™‚Ú‡“O^¥RL6#8ðÒ sY)çÿdjïw6ÊÔ™z Hõöª®vt©>w×Ð?8À(·¬ÆÁ*(MM¸a¨D¡'>ŠŽt|üý•ÎkCçžv|:ä¾ÐWÉõ—1ÞètoL;¯mL»£C—ïl9n÷_¦ÜÎ;³wFþ=º2ÄLŒ½ èpžwþ¥CåvŸ ™JP`& ô÷ßíU, yѹò奾ïî^ûâòÊ(ÌyèÂq9øŽ7f¾p÷¾R½=yúÞUÕ3Õèù®oŸe¯f®Ž¨VTª¬ ™õe@r7±N&w£Ê-í8ïY¶Ê—nl즒ÉÜwÃÙd2M&+úço ŒömN;§“wÝÎðgλ#wf¿sÎFïÝÉŽÝINDUȸ٠"4D&a*ˆ#Ú8 :LíVùª9©j°û2$˜/pX繯wnþ°ñtæ^tvöîÊÌlÅÞa«£E¯ÅÛØâ#±•è<þÓõÞ¾gßGÎ{׎\Á‰ºÖ@«ªAI£Ä"_Ó×Þ}9ò´kzdæþw½—¾½ïaÔ_GˆfDøE‰•`Fýc¸ô0‚]-ZœçTÈ7ŒÙÍ­!vX2ðü)¬l1‚ôGææ ~Ô÷ˆÝÚ<êåŒýF4¾ !´@Ñj~ôøÑãÅGP«¶ÇôÇ£ÒepA¬>áSõG§°¡è?×7È<{<ð/øýs þ9¸ÕÊ<4?“(¹˜Ÿ?‡.±û°eŒ¡p¬”ŽÄ<&Â,qdþEmïÚ)£c\¦ÎÒý¾ÑãíM$r{¼ðzuíçfiõ¶^HððÞøW¼c¯9PdœÊH« %Nnkx™¯ ¹q-z‚'Oóó…öû%„êÿ qùÁÓç¡«ÅÆT†‰“‰a Çs!£> lÞ5'"lý‚2 k¯³GŽ*ÿ| f-±CZ‰ÄR0h7ß2hu›})²#`ýa‘Td™£Îb½×zŸž›ùó¥ðè…//~ZÀ z59©,Z]I¬6[p ª½#Ý0$÷mKj¾hçÝ‹³]#£×6GU5Bºþþw%‘BUq–Óƒl´<8ÕÆk- œÝ̬d_E7ÂáÝh¸f!%#Ø“vEIñÖ 9ôsò$]ST, БÎ^»x#ùª²bQ TFÿd|~€HÔ%5¥_YZÔ÷ ä캇HD4P’“ˆ¸°r¿Þ:`§bœàÖ™ZZ»¢œ]O/ðy‚ÃÔقú±lÇõFk WÄÐ^6ÝFYN0k INé^±QH,//‡&ìãÿø‘¡À4YWW_¬–d±¼½X|AËêjùsqø68ƒhlo0Ö4óvû„}b|bÊüfÿçgöÒ!ûÉK6d¾†üpjÍ»ÆdxÙb€Dk2àkg>x Hýª-Zb„B…4(¬ã•›@o[Hµ´ñKWåyí.…‘í €WJu瘳¥ê/­œs5/n%j P îˆÇù¸˜è×) x'éÓ0‘¢”¼4g&§“ƒ"4tÊ;â0‡NW)i€ÏSm¶SXU¶¦lYœ³–4†ÚI>4ù?´d˜F¾S—;IEND®B`‚PK»ZMMmeta.xmlSM›0¼÷W w¯` X©‡ž¶j¥f¥Þ"b{S·`#Û,é¿/6!%»Q•£ß›y3ïÃåã©kƒW®P²8B à’*&ä±Ï»Ïá<ÖJõò"('LÑ¡ãÒ†·M0Q¥!sªƒ–D5F"›Žb)Q=— …¬ÑÄ Í‘S+äï ü´¶'ŽãI¤ôâ¢( Ï.PF/¸~ЭG1 yË‚8ÂpÁ:‡÷šrص¥^s3eë‡r_5g]K)u1íhó¼õ¡Îïm:ÑÞ«è°!U]?iÚ«™6Rt÷–qØw®š±öÖR&Ï œ¦ÞØ&||ü®ÎàÿÝ—nÏg°:¼ÔË•¹uÔ¥_ ÕÜ{ 'A^ÇoCŒBœìpAâœlòY†’<Ý–ð£d”¼¥ÆˆÄ˜¤Y„óÇùÆQجʙ°ÓÙ }­úÛ.M¾l¾Ÿ%Þ¥¯Yôm¹©³7èsxƹäYéúI4ÿꇛGi„ž„NûÛlŸ¥Á °ïµúÅ©…uèáÓ Zâ³Ì¿Š³Âå«w“Æ ø¸:¸ ÓÝ ÒV Ž¬Kx5xxë³×PKtZ¾*PK»ZMMConfigurations2/floater/PK»ZMMConfigurations2/menubar/PK»ZMMConfigurations2/toolbar/PK»ZMMConfigurations2/progressbar/PK»ZMMConfigurations2/statusbar/PK»ZMM'Configurations2/accelerator/current.xmlPKPK»ZMMConfigurations2/toolpanel/PK»ZMMConfigurations2/popupmenu/PK»ZMMConfigurations2/images/Bitmaps/PK»ZMM styles.xmlÝ[_Û¸ï§0t¸¾Ñ’lïfífsh{PàöZ4w}=Ðe3¡D¢Öë|‰{ì÷ë'é ©?”-yåý›ÛÈFœræÇ™!g¤}ÿÃ]*&·L\f×^8 ¼ Ë"ólsíýúËGråýðáOïe’ðˆ­b•)Ë4)ô^°bÂY±²Äk¯TÙJÒ‚«Œ¦¬Xéh%s–ÕB+—{e–²#f²±â†Ù•ÖìNFÞŽ,]_Ù0»Ò±¢»±ÂÈ ˜ºâ‰+|W’HÉ4§šhq'xöåÚÛj¯|·ÛMwó©T?\.—¾¡6 G _^*a¸âÈg‚áb…NC¿æM™¦cõC^W¥¬L×L††jz´«¹b°€¹è—ã&re:þu»í]·›˜£-U£ýÌ0w]ew•yìʦToö÷Ê¿¢ùçæ§Ö¯T:v-äí@)ž6Ór»òRÊFU°ÁnÔÁ·Ï÷î$ûNqÍ”Ãd¨ˆÄeÚð…>pv‹.ßØr1ÚjàpšñÑÐ#ï‘«*Р_±\*Ý’ŒOº°Ê¬I[Šá”Ôšu£â¸—Ô™û> xÉ-g»ï¼ÎipÚ–Ž`Rë}"†Éͽ'ÂÀGž&|Á5ÚƒBmš£,‘eÛ}°²»œ)Ž$*ŒØª3ƒ»[B>`Êêüsfè œ‰:S4&õN#%I Â3™¯éNÖ*Š¹îÛ¾_þí#à‰9¿Zǹ̼õ©ŸH8ñ1³HÞÛlÝ Oì3*yíýÄ!õã'Ÿh)µf…ÀÙ_{¦¹,þrÀg½Igjä'–ÕøJ¦4ëpä\Go©âÆ7üñªÁ”É(Ýñ%”û‘}¦ÿ)OcæðŒP©Øš¥ÒII›å&©k}Öí˜÷Etü¨Í!=V(ªq{þðSØê Só«ä_•’;«!æÊ¿É»k/˜“Y0™v‚þ&„1ABÜ΂¯¸ðÀ„Ÿö)\ºÀ€ßfÁoÕì†ó"tOØ&C„á<œ,ƒY«ÄÅåµÀ0¾E†V“B+ùŒ§ÅÖUç#Ïjò#Œ³¸Wd™tèÕµ_±H×R!$÷‘–mð.ß}ßάi†%‡l<$f -EUÔ;Z9ÃFÑ|Ë#¯æ­žIÙ“)Í¡šA*#)$\`¿›/./è¢Z6áB4”w³eA IäjS™Û«j& >·JåTQ³Xg)CÂäKh©e‘S´†ÇLZV*ò-­2/³H—&™¹ÁiyšãYhépÅgd­…2 ÀÝÖ5ïOPuTÆ0½PD¯;žÎ³˜áeK:3 *bJ¿„Š‚5`ÁÑ  Ê¼ÀP6«aG»Ž¬- F ŠåŽ˜Å+$µ*YG©¡d HzÁ¿}¶Èµ4Û”tC,3¿ZÁ–ÿúéh^7%šfÛ–ç®yì ÄÕ"5í붦T«Õ„¿ÿ|¼&Þ»»gÕ†«w݆ºå‡+7¤üìµûÓ‰…zcÜÀ°0c ÅTÅÞùáâ$˜G {níxŒaD©wodÙºBÙÔNg(zLÞyjí0€+ŸGjŸ0ÆucÁp¤ º6…P#[©øW‰W1V¾­ÿ\š'{ãƒ9±…B fP•pvÊ8„µÔ+”>š`‰6ßlŠÝ€-MÅãÀ$p8L“».]â¾—XÛ~à_ðÓ9wÉ rÙXÂ2“u)ÓKÄq“ãÍ£%,˜¯½ÿý÷÷ÆáœIŸ32)ψ k 4{} øCé§Nµo‚„“q-…7&5uóÎâÂ8þ)KÏÀaöÌ©AÖ ŠxVƒðöš?B¡õó7‡Ðâ ºz“]<B³éâM"tùdÍß$>7š§¯ž ¡ÅÍÓË'Dèmæé0x2ˆ.þX‰Ú!W›?Txl,ÀÚPílÐ`[ET%D5XWÝQ[¥tÇš¥6í-³òa|oF rÜ4,뽘š®rgR¥T´³he±}w•ÉŒBÚYJ–Ul*;‡dT×[%ËÍ–T]'w‰c&½Ïx:¦’?t—çkp¸VxUõ@ìXmY׺@<²¸„mS¶ëãbRO·«öºž¯Š?¦¡~&PNg¦OÔÛ½iº,½ýã#*î‡÷‘ë†z¹5½Åq åcu‰&H+j ¤ ÝЩŽô(¦Úà†yÐä†cØè†eÀì†>lø‹ñ/–æ[jõ(’œ%=N)oû|²¡JmRpš_÷4½äú3‹ôŽk8^l[¼·÷U·_©j>y!GǶǦáÅ@‡,0z:`Íû„½³w§{g6l[gs›ÉÏEÓ¦šç„³Êe·¼àÆ¥Ÿ¥ÛõËáôÁžáó˜~ŸoØþ¥‰”ѪÿÓ¨ŽïlP}ü™Iü矠U;ˆñVÛÐ}«c§™à“LNPz¢øWè#Ë&ˆã€9å¡åZÆû×Ôôôµ®º\ÂAr¶UØ8O8|ñrÖ ]©í‹ºn›ÿl3í5ùÛ5òŒº¡{í¯‹©‘pp-^51ŒqäÅâ,GF“Â×°ið¥U`²>6ÇG‘½ˆŒÝŽÙùÛ1{ö³¸¹-íÛ¸Î$I-—ßÖŒ r»Êa{`4BÏþ>¬G`7}~Öa²e4~í+Ç(è+4³!8gáxŒÎ Œ£WIU¯R·MR÷5ÖRÄçC÷Liå›…nÑÓaâ‚ñ¢üæMŸÚf©ºfz_ÏîBv^é½›€—€ Ò°ÂÒ\_Ï~ï¦4ÿã_Õ8“:]3~yA\]Fc7ép¤aáA0xp³[ß|ºjÐe0I7ÆøÊ}K~qÿ^îÆLà™ R¾Î Ÿ¤SFCŸßNúÞ¢{×û8®[Ÿ÷Ytü>#d4tªó±¥P·»eAY¼ðdÞO»FøgG_?EíÍj=ëlXNÝ…0ܾè"ë‰.27]†^Óœº+Hø…\ ±Hê»™?kú­EäÜÉŠäSî—ÑLA7_ÎÄ&™µšaB~ÆêÏßÂñ+¸k†ÇÐgI™ª¨òâÉ£…Ü÷ “ÿ«ûÝì#É@MÈ 7ša0a‘"å0=£¨í;àÿj M]/,õwu óy€‹+/iþJ™C•Ý(\€ßû¥Ì‹0Xo3>0\¼ [B˜%5›jy‘ÑGyIÂÅÜmàF9Pz‡IÛj+ªi3…ëözŽ‚T³nN]ŸÕ–rAS6@dlµßÓ¢`‰Û)"AÁØÚ¦dÕí[R5“K×Ep˜U –¶ËìðÓ˜Ý z 3ñbÆÿ×(TÅQ¾Q ÅAè‹מÁå$4!ƒ^Cåÿ„Élx÷8½“ÖŸ †‹þ‰áÒ±~¨%‡˜ÞÿB¼â1,–´,˜ZGºƒ4Ý4\ýOq={¿ŽcR@ôgáìO¬ÚîD7¬Ž;é =ÅÎ^Øz6?q»=l´Ö%r/r9f?r©J›ä¼0C·?qCW°a6bçE9è£3þÏÚAÿ¥WâÁ¬¹|2`jª ©1«œVµfÛ±Ø aÖÀä¶ßt€ú½RáHõk—÷´Êör[QÙ'¼o/w< ¤½¾äHÔ*Në²ó]<üt·Ó¢³WÂZe7Ýd„ƶâ#²ª¨Ó!ê%›§NÍèà¶ýU3ÞÇÚßÓÈï «"4ƒ#·ãæ ÓÞËÏ^¸mŒÁW™ ÂÌ&Ú6ÖÉõ,„”…E¥½H‹ÆUõ$y«©M—Ó§©kœ#½·ôñþç^fzÔ-G-ë\{>zºµ&ÜSÝ^ÔÇXoÞð v˜µÚª©rsvÚs¥ëö 4?[?ÜÍ ’ñS°rs&:pˈ•/œÁ+sÐCΗnJúñÓ±íCßÎSnöPuGW¡%—oÕlÝ«È]³—´à-‹êvs8OUÙ<äÞœî´ù‰1Þ¸bs­êÍåše1fë£Ö|•DNi˜›[ë­¬ Ý¥uÅAõ³ÿQq£_e‹Ò2è¢é{ÄÍÍÝán¦(ÑAÓöˆ[£âšbàZšÛSÚtC’%{§Ç—„²§ûk.Á^Nù†B?ª¤J™jýYÏJêK˜ÊŒÞ’« ³½ ÕÞÁ°î2T{~qóëÙm I·?ü´gÂ,£™pœ¦=#’›]b±N ;ü_&¾œ¯p“E{²&¦~„$ƒcÕ±&Ív4¬;fc¨-$)©í-(e ?ÂÏm`Kwü·#ýÄÛ+pîà;jú±þ͘îÚï6@ ÐJ?4&HþUVo4k2hO¨Ð,.#—yŠvÚжÀoUé뇯žèoµÍˆ.>öâc/>öúØ ƒA9WmÔ¹êÇzPŒñnj:›dß÷ž®‡Z—v¢"Zãm¡àKúôhvÃ^e÷ˈdJV&_~UWWŒ¿©´ÒœÍ çdΉŒN$2Ή̎ê%]"Ñ%]"Ñg‰6Îuœ+:Op®çRÿpÃâØÈ4Šš>Šš¹šÁ½6U±°±s¯ÍÒBÆ؃ žÝ¼+é;öGq¤;JÓ££7Mˆ 4ÂÉÅidORˆ;^™=Ôo­ß´[DCÈl¯W BŠãh-æ–¶y=œ¿—ÃׄhÂUófÿŸ™¨——¿aÇB£È\oQªxkÛ¶*¦­z!i,ù5t òdâæ³ãŠüÐiâ%¹Í­ïÄuè¶Uiìüé7çôŽQ]Zî.‰@èR]J¢KIt)‰>«’Hë8WíX纻$ÂuJvÖ’è¯%)G®Sº…Tn$wx9UÒpM¬#š¤¬-$:…fOûùA…Á*wšÅÂ@oÝ¿PEhBãVÉÑTHý$éÕƒãÈH×-!Ç–uM×ø·bhº4g=ñ\P[Š!œÙ@ HÛ´D¤»x³»:ˆµ>Š5ÂZ¨Ì,¸¤ÁÍ2ô@gš'”bö(lSJ1ÝÖö•bPKØX¬ÅÀ*Ãv&c?‘§YÀÎZjþÄƨUm]lk¤N5‡“²ô§HÅ-ż¤â—Tü’Š_RñÏ*o&¿{uæTœëùSñw¤(³‘±—œ5›ŠcîÌšm‘5›­§29±qgÖl2Tɶ ž5CŒG˜gÍüÛhÖ¬!íĬù)ótmg~Ã(õ3äFÎp†ƒ±úrjöduGmoߪJ⺣6Ý´Æ,̵0kÈÂì ÃH PP²™R2䆆¤ÊXw,ÙV­ê Öñ…͆|ª…k¼maÍéÊëîlTàëh k{kukkksƒ5fæk0¥TYØ–l Ë\¶TA ßF¡6üâë¸nT±xùrÖ:îzDD™×‰ùBx+†5ñ÷]æ·aæ#ÿ{¨›ÿPK<*Öí _jPK»ZMM settings.xmlÝZ]sâ6}ï¯Èx¶3»Ä„$Ý… ì@fIBl»}öÔÈ’G’Cø÷½°‰ãN§yH&¶tÎÑÕ•î\|}ØÑHEo:'ÇUç¸'|ÊgMg<êV¾8_[?]ˆé”zÐð…ÀuEÖ8Dát®«×M'’¼!ˆ¢ªÁIª¡½†o¦5¶G7,ÙêÉ3£ü±é̵®»X,Ž§ÇBÎÜ“z½îÚ·›¡¡…PD[Áé·çlÓz‚Oé,-Êjôö|!Ä‹h3aµ0+¼V­ž¹«ÿ7£U@YZ.3¶â‰ DÍöÊV„Ó -Œ»·ê•ªÃÚë/Ú×›µå"5§µñ‡´.ÖÆYý©P ñ‘£õc£±é eã‰ÂâÅ{œ¸y¯ç|§Ê -ŒDèl^êeˆ/)×N«rZ?½p÷qޅ݇©Ž¯ÕsƒÿN}=C?ù\=?Ë t6UR«Ÿ¤Å¯$¬PîÃ3ø»\°ˆß+;ÝM.Ó(†EÏß‘©´DGpZÆ-R+}û«¤~O­í±?‚áNKËòÀw¥à»&~Ÿ¦2¢÷Ô“p$ KñÚ7è2ƒ["g”«òHÌï>åHQ€™î'§»–Î2x†ÊYÌ€Eê†pŸj³Y¾å½¹È¬µ®$™ ñ¤±äíÉî׌” Á–Öl}| Û†.öZ:¯V³®fŸ«Œ-ZïGŸ,1ìíâ¿œuP„\:-7õÏ—f6-¼!5ù¯¨é ïü·¥¸oC݉¶ÆëRÆÞ‰KÁľÄüÎþ1%ÆsWÃ0¼(çBâÑÉxn†ÀðF߀ ÜSÖn…_‚)zêJDhäKF½Ç<ëkŸ&á<4–àrNø ĪL(žfȨjòA,â÷ c*h¬³öªÍ¸­ÿû¬û¼ä¬»–=Ù¼D*HV_ûœqSÿ?¤=zÓ¿CÂó!TÚä|VôwX:K¢…,‘ã +m›À$°Àñã_XÇrÖÑS6i44×Üä<‰EQŽÜÀ —›þ)DpÏc"lŠØóp»´Nz½®Öã(ÐéÛ2«‘´±ï=ý™v²åX¼"š4;TãBGÆÆý„æÁ‡XØêO®š nHʇŸO;>F(ìÓšØUK."ýc%&ÙÔ`Š æQòR¦§h:b#”RhJe×ÂWìesEÔ¼p-~F-0%Ó(g¨ã“’ìÑë:˜€ßEU“ɽ´®êûÀ_|)g­‰k“Ù¯7xVGŒ êx˜1…"’’½^Ú5EÕØå¦Ï”ášûç罌ߤħVø¹šeÛ‘¢½âùDŽF'&ÖpŽÙ} 26 º–<à°Àaœ@L÷«\â%ÈU‚3‰éz•è«ÇÞ 9^RI[¡´ ¥õQèãʲçß¼|¦”qBàÙp=WXgžh´þ=¿BD?gô;ÝDå> ܈ot÷ PK¿(ÖÞPK»ZMMŸ.Ä++mimetypePK»ZMM ›#VXXQThumbnails/thumbnail.pngPK»ZMMtZ¾*ßmeta.xmlPK»ZMMÓConfigurations2/floater/PK»ZMM Configurations2/menubar/PK»ZMM?Configurations2/toolbar/PK»ZMMuConfigurations2/progressbar/PK»ZMM¯Configurations2/statusbar/PK»ZMM'çConfigurations2/accelerator/current.xmlPK»ZMM>Configurations2/toolpanel/PK»ZMMvConfigurations2/popupmenu/PK»ZMM®Configurations2/images/Bitmaps/PK»ZMM‘?ŒŠ0 07 ëstyles.xmlPK»ZMM<*Öí _j Scontent.xmlPK»ZMMƒÉUaj# y,settings.xmlPK»ZMM¿(ÖÞ¸1META-INF/manifest.xmlPK63apache-buildstream-27ae392/doc/source/images/000077500000000000000000000000001514607367700211345ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/images/arch-datamodel-context.svg000066400000000000000000001521221514607367700262070ustar00rootroot00000000000000 Command Line ArgumentsArguments specified on the command line have the final say for any user preference.User ConfigurationThe user configuration file overrides any of the previous settings.Project RecommendationsWhile we try to keep project data and user preferences as separate as possible, in some cases it is convenient for the project to suggest a value, such as which artifact server to use.Default ConfigurationBuildStream ships an internal YAML file with the same structure as the user configuration file, this file specifies all of the default configurations. Default Configuration Project Recommend User Configuration Command Line Arguments apache-buildstream-27ae392/doc/source/images/arch-datamodel-element-composition.svg000066400000000000000000002367601514607367700305300ustar00rootroot00000000000000 BuildStream Core DataPublic Domain Data Core Defaults Element Declaration Project Configuration Element Defaults Project Overrides Element Configuration Data Element Declaration Element Defaults Project Overrides Element Configuration Data is composited in the same way as Core Data and Public Data, except that it is not understood by the project configuration so there are no defaults. Composited the first time that a Element of a given “kind†is instantiated.This is cached on the given Element’s class data for later reuse. Composited every time against the cached class data below every time an Element is instantiated. apache-buildstream-27ae392/doc/source/images/arch-datamodel-element.svg000066400000000000000000002010771514607367700261600ustar00rootroot00000000000000 BuildStream Core DataCore data is understood by the BuildStream core. This includes the list of source declarations which are components of the Element.Element Configuration DataElement Configuration Data is defined by Element plugin classes. This data is internal to a given plugin.Public Domain DataPublic Domain Data is free form and not validated, this provides a way to declare data that is intended to be read back and processed by reverse dependencies. Elementkind: autotoolsdescription: Optional description about foodepends:- gtk+.bst- clutter.bstsources:- <source declarations here>config: configure-commands: (<): - cp %{datadir}/automake-*/config.{sub,guess} .public: bst: integration-commands: - update-flying-pony-cache -f ${datadir}/ponies apache-buildstream-27ae392/doc/source/images/arch-datamodel-source-composition.svg000066400000000000000000001557761514607367700304060ustar00rootroot00000000000000 BuildStream Core DataSource Configuration Data Source Declaration Project Overrides Composited the first time that a Source of a given “kind†is instantiated.This is cached on the given Source’s class data for later reuse.Note: In the case of Sources, defaults are provided in code. Composited every time against the cached class data below every time an Source is instantiated. Source Defaults apache-buildstream-27ae392/doc/source/images/arch-datamodel-source.svg000066400000000000000000001562421514607367700260320ustar00rootroot00000000000000 BuildStream Core DataCore data understood by BuildStream.Source Configuration DataSource Configuration Data is defined by Source plugin classes. Sourcekind: gitdirectory: sub/diruri: upstream:foo.gittrack: masterref: bbf775301a08b9a578ef7f647bc35fe13e816241 apache-buildstream-27ae392/doc/source/images/arch-dependency-model-build.svg000066400000000000000000001552431514607367700271130ustar00rootroot00000000000000 Build scope for the Application element Application ServiceRequired at runtime CompilerRequired at build time Base RuntimeRequired at all times Service DefinitionXML files needed to build the Service BootstrapSelf Hosting Compiler apache-buildstream-27ae392/doc/source/images/arch-dependency-model-runtime.svg000066400000000000000000001533141514607367700274740ustar00rootroot00000000000000 Runtime scope for the Application element Application ServiceRequired at runtime CompilerRequired at build time Base RuntimeRequired at all times Service DefinitionXML files needed to build the Service BootstrapSelf Hosting Compiler apache-buildstream-27ae392/doc/source/images/arch-dependency-model.svg000066400000000000000000001744321514607367700260170ustar00rootroot00000000000000 Application ServiceRequired at runtime CompilerRequired at build time Base RuntimeRequired at all times LegendBuild DependencyRuntime DependencyBuild & Runtime Dependency Service DefinitionXML files needed to build the Service BootstrapSelf Hosting Compiler apache-buildstream-27ae392/doc/source/images/arch-overview.svg000066400000000000000000012051771514607367700244530ustar00rootroot00000000000000 Frontend Main Entry Point (cli.py)Implements command line interface Main Application State (App)Initializes the “Streamâ€, handles logging and user interactions Loggerwidget.py Status Barstatus.py Completionscomplete.py Color profilesprofile.py Core, Frontend facing API ContextUser configuration, CLI arguments ProjectLoaded project.conf configuration StreamLoading of the project, implementation of all main user facing commands in the CLI Core Internals PlatformAbstract class providing platform specific routines. The Frontend implements the main command line interface, status bar, master log formatting, and any interactivity with the user. This is cleanly separated from the core BuildStream application. Scheduler(Scheduling of element processing jobs)The Scheduler is in charge of processing elements in job queues. Job(Abstract tasks) Queue(Abstract job queues) Loader(Load the elements in a project)The loader loads elements and sources from the project directory, transforming them into data structures ready for Element and Source instantiation LoadElement, Dependency(Loader data model) The Frontend creates the toplevel Project, Context and Stream, and receives Elements and Sources by way of loading the Stream. These represent the main active internal components which sit below the frontend facing business logic in terms of module stacking order. OptionsLoads and resolves project options, and processes conditional statements VariablesResolves a collection of variables, and performs substitutions on strings containing variables IncludesProcesses include statements and composites the resulting dictionaries YAML Parsing Utilities YAMLLow level utilities for parsing YAML, storing the provenance of loaded values for error reporting, composition of dictionaries and validation of loaded values utils.pyVarious miscellaneous utilities, includes the directory staging mechanics signals.pyUtilities and context managers for dealing with UNIX signal handling utils.pyVarious miscellaneous utilities, includes the directory staging mechanics WorkspacesHelper object for (de)serialization of the workspace configuration Low Level Subsystems Element, Source, and PluginData Model CAS Server CasServerContent Addressable Storage Server The CAS Server only needs to interact with Core Internals and lower levels of the stack. CasCacheContent Addressable Storage implementation, used by the artifact cache and CAS server FUSE / SafeHardlinksThe Fuse layer implements a copy on write hardlink experience for the Sandbox implementations SourceFactory / ElementFactoryLow level factory for instantiating Source and Element plugins, one plugin namespace per project ArtifactCacheArtifact Storage Module SandboxAbstract Class for running commands in the isolated build environment apache-buildstream-27ae392/doc/source/images/arch-program-flow.svg000066400000000000000000007217601514607367700252210ustar00rootroot00000000000000 Parse Command Line Load Context Enter Application Context Load Platform ArtifactCache Preflight Initialize Logging Exit with error Load Toplevel Project Initialize Stream Initialize Application Run Stream API Load Elements Resolve Keys Resolve Cached State Construct Scheduler Queues EventSwitch Log Line Logging Event Interrupt or Error Event App isInteractive Run SchedulerRun the event loop, and process the elements until they have passed through all of the queues, or until there is a terminate condition. Interactive Pause Scheduler Prompt User Resume Scheduler Set Terminate Condition Non Interactive Continue Set Terminate Condition Terminate Log Session Summary Clean Exit Exception Event apache-buildstream-27ae392/doc/source/images/arch-remote-execution.svg000066400000000000000000002141561514607367700260750ustar00rootroot00000000000000 Remote Execution service Local artifact cache push REAPI Remote artifact cache 1 pull Remote artifact cache 2 push pull Send (initial environments, commands) Retrieve (finished environments, logs) Cache manager(cascache.py) Sandbox.run(commands, flags, environment) Remote execution cache REAPI-compatible build service BuildStream Remote execution service implementation apache-buildstream-27ae392/doc/source/images/arch-scheduler-job.svg000066400000000000000000003473351514607367700253350ustar00rootroot00000000000000 Setup signal handlers Block signals fork() Unblock signals Main BuildStream Process Job Process Initialize subprocess logging redirection Unblock selected signals (ready) Run the job payload Log messages Start New Job Receive IPC Messages Report job result and any other state Collect results on job instance Collect exit status Exit with symbolic exit status Interruptible Job Harness Inform the job owner that a job completed, hand over the result Job ControlSuspend (SIGTSTP)Resume (SIGCONT)Terminate (SIGTERM)Kill (SIGKILL) Job Exited This process is a fork() of the main process without execve().This means it has access to the main process data model at the time of the fork(), any modifications made after this point are copy-on-write, and any side effects of the job need to be manually propagated back to the main process through the IPC. apache-buildstream-27ae392/doc/source/images/arch-scheduler-queue-ports.svg000066400000000000000000001742331514607367700270470ustar00rootroot00000000000000 Processed Failed Skipped Ready Skip Queue implementations report a “QueueStatus†for all of the elements which are in the input queue at all times.Skip elements go directly to the output queue without processing, and the scheduler only ever processes the Ready elements. Wait Elements In Element Processing After elements are either processed or skipped, they move to the Queue’s output queue where the scheduler can pick them up and and move them along to the next Queue.Elements which have passed through a Queue are also kept in status lists for bookkeeping purposes. Input Queue Output Queue apache-buildstream-27ae392/doc/source/images/arch-scheduler-queues.svg000066400000000000000000001623541514607367700260660ustar00rootroot00000000000000 Element Element Element Element Pull Element Element Element Element Fetch Element Element Element Element Build Element Element Element Element Track Element Element Element Element Push Elements In Elements Out apache-buildstream-27ae392/doc/source/images/arch-scheduler-run.svg000066400000000000000000002025761514607367700253640ustar00rootroot00000000000000 Scheduler.run() Wait Run Event Loop Job Done ReadyElementsRemain QueueJobs Yes Return No apache-buildstream-27ae392/doc/source/index.rst000066400000000000000000000023551514607367700215350ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. BuildStream documentation ========================= These docs cover everything you need to build and integrate software stacks using BuildStream. They begin with a basic introduction to BuildStream, background information on basic concepts, and a guide to the BuildStream command line interface. Later sections provide detailed information on BuildStream internals. .. toctree:: :maxdepth: 1 main_about main_install main_using main_core main_porting.rst CONTRIBUTING main_architecture main_glossary For any other information, including `how to install BuildStream `_, refer to `the BuildStream website `_. apache-buildstream-27ae392/doc/source/junctions/000077500000000000000000000000001514607367700217035ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/junctions/junction-elements.rst000066400000000000000000000077441514607367700261140ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _junction_elements: Junction elements ================= BuildStream's :mod:`junction ` elements are the mechanism which allow projects to interact and depend on eachother. Junction elements represent the BuildStream project you are depending, and behave much like other elements in the sense that they can be :ref:`fetched ` and :ref:`tracked ` like other elements, except that regular elements cannot *depend* on junctions directly, nor can junctions be :ref:`built `. Instead, junctions act like a window into another project you depend on, and allow elements of your project to depend on elements exposed by the project referenced by the junction. Projects which are junctioned by your project are referred to as *subprojects*. A simple example ---------------- .. note:: This example is distributed with BuildStream in the `doc/examples/junctions `_ subdirectory. Below is a simple example of bst file for the junction element, which we have called ``hello-junction.bst`` in this project: .. literalinclude:: ../../examples/junctions/elements/hello-junction.bst :language: yaml This element imports the autotools example subproject distributed with BuildStream in the `doc/examples/junctions/autotools `_ subdirectory. .. note:: For the sake of this example we are using a local source in a subdirectory of the example project. Since junctions allow interoperability of projects, it would be more common to use a junction to a remote project under separate revision control, possibly using a ``kind: git`` source. The below bst file describes the element ``callHello.bst``, which depends on the ``hello.bst`` element from the autotools example: .. literalinclude:: ../../examples/junctions/elements/callHello.bst :language: yaml Note how this element refers to the previously declared ``hello-junction.bst`` junction in its :ref:`dependency dictionary `. This dependency expresses that we are depending on the ``hello.bst`` element in the project which ``hello-junction.bst`` refers to. The ``callHello.bst`` element simply imports a ``callHello.sh`` shell script which calls the hello command provided by ``hello.bst``: .. literalinclude:: ../../examples/junctions/files/callHello.sh :language: shell Building and running -------------------- Building the ``callHello.bst`` element which requires an external project is just a matter of invoking :ref:`bst build ` in the regular way: .. raw:: html :file: ../sessions/junctions-build.html You can see that the hello.bst element and its dependencies from the autotools project have been built as a part of the pipeline for callHello.bst. We can now invoke :ref:`bst shell ` and run our ``callHello.sh`` script, which in turn also calls the ``hello`` program installed by the subproject's ``hello.bst`` element. .. raw:: html :file: ../sessions/junctions-shell.html Further reading --------------- For an example of junction elements being used in a real project, take a look at the `freedesktop-sdk junction `_ in the `gnome-build-meta `_ project. apache-buildstream-27ae392/doc/source/junctions/junction-includes.rst000066400000000000000000000234151514607367700260770ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _advanced_junction_includes: Subproject includes =================== We've already discussed how we can add optionality to projects and explored how we can perform conditional statements and include fragments of BuildStream YAML in the earlier :ref:`chapter about optionality and directives `. In this chapter we're going to explore how we can use :ref:`include directives ` to include YAML fragments from a subproject referred to by a :mod:`junction ` element, and how :ref:`project options ` can be specified in the configuration of your :mod:`junction `. .. note:: This example is distributed with BuildStream in the `doc/examples/junction-includes `_ subdirectory. Overview -------- It is a goal of BuildStream to provide developers and integrators with the tools they need to maintain software stacks which depend on eachother with least friction as possible, such that one can integrate upgrades of projects one depends on via :mod:`junction ` elements regularly and with the least hassle as possible. :ref:`Project options ` and :ref:`include directives ` combined form the basis on which projects can maximize on code sharing effectively, and the basis on which BuildStream projects can form reliable APIs. Project options ~~~~~~~~~~~~~~~ The :ref:`options ` which a project exposes is a fairly limited API surface, it allows one to configure a limited set of options advertized by the project maintainers, and the options will affect what kind of artifacts will be produced by the project. This kind of optionality however does not allow consumers to entirely redefine how artifacts are produced and how elements are configured. On the one hand, this limitation can be frustrating, as one constantly finds themselves requiring a feature that their subproject does not support *right now*. On the other hand, the limitation of features which a given project chooses to support is what guards downstream project consumers against consuming artifacts which are not supported by the upstream. Project options are designed to enforce a *separation of concerns*, where we expect that downstreams will either fork a project in order to support a new feature, or convince the upstream to start supporting a new feature. Furthermore, limited API surfaces for interdependent projects offers a possibility of API stability of projects, such that you can upgrade your dependencies with limited friction. Includes ~~~~~~~~ The :ref:`includes ` which a project might advertize as *"public"*, form the output of the API exchange between a project and its subproject(s). Cross-project include files allow a project to *inherit configuration* from a subproject. Include files can be used to define anything from the :ref:`variables ` one needs to have in context in order to build into or link into alternative system prefixes, what special compiler flags to use when building for a specific machine architecture, to customized :ref:`shell configurations ` to use when testing out applications in :ref:`bst shell `. This chapter will provide an example of the *mechanics* of cross project includes when combined with project optionality. Project structure ----------------- Project options ~~~~~~~~~~~~~~~ This example is comprised of two separate projects, both of which offer some project options. This is intended to emphasize how your toplevel project options can be used to select and configure options to use in the subprojects you depend on. For convenience, the subproject is stored in the subdirectory of the toplevel project, while in the real world the subproject is probably hosted elsewhere. First let's take a look at the options declared in the respective ``project.conf`` files. Toplevel ``project.conf`` ''''''''''''''''''''''''' .. literalinclude:: ../../examples/junction-includes/project.conf :language: yaml Subproject ``project.conf`` ''''''''''''''''''''''''''' .. literalinclude:: ../../examples/junction-includes/subproject/project.conf :language: yaml As we can see, these two projects both offer some arbitrarily named options. Conditional configuration of subproject ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The toplevel project here does some conditional configuration of the subproject. Toplevel ``elements/subproject-junction.bst`` ''''''''''''''''''''''''''''''''''''''''''''' .. literalinclude:: ../../examples/junction-includes/elements/subproject-junction.bst :language: yaml Here we can see that projects can use :ref:`conditional statements ` to make decisions about subproject configuration based on their own configuration. In this example, if the toplevel project is ``funky``, then it will configure its subproject with ``color`` set to ``blue``, otherwise it will use the ``red`` variant of the subproject ``color``. Including configuration from a subproject ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Here there are a couple of aspects to observe, namely how the toplevel project includes files across a junction boundary, and how that include file might be implemented. Toplevel ``elements/hello.bst`` ''''''''''''''''''''''''''''''' .. literalinclude:: ../../examples/junction-includes/elements/hello.bst :language: yaml Here we can see the same element which we discussed in the :ref:`autotools example `, except that we're including a file from the subproject. As explained in the :ref:`reference manual `, this is done by prefixing the include path with the local :mod:`junction ` element name and then a colon. Note that in this case, the API contract is simply that ``hello.bst`` is including ``paths.bst``, and has the expectation that ``paths.bst`` will in some way influence the ``variables``, nothing more. It can be that an include file is expected to create new variables, and it can be that the subproject might declare things differently depending on the subproject's own configuration, as we will observe next. Subproject ``include/paths.bst`` '''''''''''''''''''''''''''''''' .. literalinclude:: ../../examples/junction-includes/subproject/include/paths.bst :language: yaml Here, we can see the include file *itself* is making a :ref:`conditional statement `, in turn deciding what values to use depending on how the project was configured. This decision will provide valuable context for any file including ``paths.bst``, whether it be an element, a ``project.conf`` which applies the variable as a default for the entire project, whether it is being included by files in the local project, or whether it is being included by a downstream project which junctions this project, as is the case in this example. Using the project ----------------- At this stage, you have probably already reasoned out what would happen if we tried to build and run the project. Nevertheless, we will still present the outputs here for observation. Building the project normally ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Here we build the project without any special arguments. .. raw:: html :file: ../sessions/junction-includes-build-normal.html Building the project in funky mode ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Now let's see what happens when we build the project in funky mode .. raw:: html :file: ../sessions/junction-includes-build-funky.html As we can see, this time we've built the project into the ``/opt`` system prefix instead of the standard ``/usr`` prefix. Let's just take a step back now and summarize the process which went into this decision: * The toplevel ``project.conf`` exposes the boolean ``funky`` option * The toplevel junction ``subproject-junction.bst`` chooses to set the subproject ``color`` to ``blue`` when the toplevel project is ``funky`` * The subproject ``include/paths.bst`` include file decides to set the ``prefix`` to ``/opt`` in the case that the subproject is ``blue`` * The ``hello.bst`` includes the ``include/paths.bst`` file, in order to inherit its path configuration from the subproject Running the project in both modes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. raw:: html :file: ../sessions/junction-includes-shell-normal.html .. raw:: html :file: ../sessions/junction-includes-shell-funky.html As expected, the ``funky`` variant of the toplevel project installs the hello world program in the ``/opt`` prefix, and as such we need to call it from there. Summary ------- In this chapter we've discussed how :ref:`conditional statements ` and :ref:`include files ` play an essential role in the API surface of a project, and help to provide some configurability while preserving encapsulation of the API which a project exposes. We've also gone over the mechanics of how these concepts interact and presented an example which shows how project options can be used in a recursive context, and how includes can help not only to share code, but to provide context to dependent projects about how their subprojects are configured. apache-buildstream-27ae392/doc/source/junctions/junction-workspaces.rst000066400000000000000000000045431514607367700264530ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _junction_workspaces: Workspaces and subprojects ========================== When developping a project with :mod:`junctions ` and subprojects, you will probably end up needing to work on the subprojects as well. Opening :ref:`workspaces ` works mostly in the same way as it does with subprojects as it does for elements directly in your own project. .. note:: This section runs commands on the same example project presented in the :ref:`previous section `, which is distributed with BuildStream in the `doc/examples/junctions `_ subdirectory. Workspacing a junction ---------------------- Sometimes you need to work on the elements declared in a subproject directly. As the downstream consumer of a junctioned project, it makes sense that you might need to work on that project as well in order to satisfy the needs of your downstream project. You can easily work on your subproject by :ref:`opening a workspace ` on the junction element directly. .. raw:: html :file: ../sessions/junctions-workspace-open-subproject.html After opening a workspace on the junction element, the open workspace is used to define the subproject, allowing you to make changes to how the subproject is built, add new dependencies and configure the subproject in any way. Cross-junction workspaces ------------------------- You can open workspaces for elements in the project refered to by the junction using the syntax ``bst open ${junction-name}:{element-name}``. In this example, .. raw:: html :file: ../sessions/junctions-workspace-open.html This has opened a workspace for the hello.bst element from the autotools project. This workspace can now be used as normal. apache-buildstream-27ae392/doc/source/main_about.rst000066400000000000000000000011111514607367700225310ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. include:: ../../README.rst apache-buildstream-27ae392/doc/source/main_architecture.rst000066400000000000000000000015401514607367700241070ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Architecture ============ This section provides details on the overall BuildStream architecture. .. toctree:: :maxdepth: 2 arch_overview arch_program_flow arch_data_model arch_dependency_model arch_scheduler arch_cachekeys arch_caches arch_sandboxing arch_remote_execution apache-buildstream-27ae392/doc/source/main_core.rst000066400000000000000000000017601514607367700223610ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Reference ========= This section details the core API reference along with other more elaborate details about BuildStream internals. .. toctree:: :maxdepth: 2 core_format core_plugins core_framework core_additional .. This is a hidden toctree so that the autogenerated modules index is not orphaned, sort of a cheat because we would rather present a manual toctree for this part. .. toctree:: :hidden: buildstream apache-buildstream-27ae392/doc/source/main_glossary.rst000066400000000000000000000127061514607367700232760ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Glossary ======== .. glossary:: :sorted: ``.bst`` file The configuration for an :term:`Element `, represented in YAML format. Artifact The output collected after building an :term:`Element`. Artifacts can be built from :term:`Sources `, or pulled from a :term:`Remote Cache `, if available. Artifact name The :ref:`name of an artifact `, which can be used in various commands to operate directly on artifacts, without requiring the use of a :term:`Project`. Cache BuildStream leverages various caching techniques in order to avoid duplicating work. Depending on context, "Cache" might refer to BuildStream's :term:`local cache ` or a :term:`Remote Cache `. Core plugin A :term:`Plugin ` that is contained in the BuildStream package. These are built-in and don't need to be defined in the project configuration. See :ref:`plugin documentation ` for more details on core plugins. Dependency :term:`Elements ` in a BuildStream project can depend on other elements from the same project. The element dependent upon is called a dependency. See :ref:`Dependencies document ` for more details. Dependency configuration Additional custom YAML configuration which is used to define an :term:`Element's ` relationship with it's :term:`Dependency `. This is supported on limited :term:`Element ` implementations, and each :term:`Element ` defines what configuration it supports. See the :ref:`dependency documentation ` for details on dependency configuration. Element An atom of a :term:`BuildStream project `. Projects consist of zero or more elements. During the build process, BuildStream transforms :term:`Sources ` and :term:`Dependencies ` of an element into its output. The output is called an :term:`Artifact `. Configuration for elements is stored in form of :term:`.bst files <.bst file>`. See :ref:`Declaring Elements document ` for more details on element configurtion. External Plugin A :term:`Plugin ` that is defined in some package other than BuildStream. External plugins must be declared in :ref:`the project configuration `. For a list of known external plugin repositories, see :ref:`plugins_external`. Junction A special kind of :term:`Element `, that allows you to depend on elements from another project. See :mod:`Junction reference ` for details on how to configure junction elements. See :ref:`Junction guide ` for details on how to use junction elements. Local Cache To avoid duplicating work, BuildStream will cache sources, artifacts, logs, buildtrees etc. in a local cache directory. If these sources or artifacts are needed another time, BuildStream will use them from the cache. See :ref:`Local cache expiry ` section of the user guide for details on how to configure the local cache. Plugin BuildStream Plugins define types of :term:`Elements ` and :term:`Sources `. Hence, they come in two distinct varities - Element Plugins and Source Plugins. BuildStream supports some plugins :term:`out of the box `. It also has support for :term:`third party plugins `. Project A collection of :term:`Elements `. Elements in a project share some central configuration. See :ref:`projectconf` to learn how to configure BuildStream projects. Remote Cache A server setup for sharing BuildStream :term:`Sources ` and/or :term:`Artifacts `. See :ref:`cache server documentation ` for details on artifact caches. Source Sources describe the input to the build of an :term:`Element`. In general, an element can have zero or more sources. But, certain element plugins may restrict the number of allowed sources. Sources are defined in the :ref:`Sources ` section of :term:`Element ` configuration. Subproject Subprojects are :term:`projects ` which are referred to by a :term:`Junction`. Workspace Workspaces allow building one or more elements using a local, and potentially modified, copy of their sources. See :ref:`Workspaces guide ` for more details on how to use workspaces. apache-buildstream-27ae392/doc/source/main_install.rst000066400000000000000000000231021514607367700230710ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Installing from Source ====================== This page explains how to build and install this version of BuildStream from source. For general purpose installation instructions consult the `website `_. For full install instructions, read on: * :ref:`Install dependencies` * :ref:`Install BuildBox` * :ref:`Install BuildStream` * :ref:`Install completions` .. _install-dependencies: Installing Dependencies ----------------------- Runtime requirements ~~~~~~~~~~~~~~~~~~~~ BuildStream requires the following Python environment to run: - python3 >= 3.10 - PyPI packages as specified in `requirements.in `_. Some :mod:`Source ` plugins require specific tools installed on the host. Here is a commonly used subset based on the :ref:`core source plugins ` and `buildstream-plugins `_. - git (for ``git`` sources) - lzip (for ``.tar.lz`` support in ``tar`` sources) - patch (for ``patch`` sources) Some BuildBox tools used by BuildStream require additional host tools: - bubblewrap (for ``buildbox-run-bubblewrap``) - fusermount3 (for ``buildbox-fuse``) Install-time requirements ~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream contains Cython code which implies the following extra dependencies at install-time only: - C and C++ toolchain - Python development headers These instructions use ``pip3`` to install necessary PyPI packages. Packagers and integrators may use a different tool and can ignore the `pip` dependency below. Distribution-specific guides ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This table gives you a list of packages for specific distros: .. list-table:: * - **Distribution** - **Runtime requires** - **Install requires** * - Arch Linux - bubblewrap fuse3 git lzip patch python - gcc python-pip * - Debian - bubblewrap fuse3 git lzip patch python3 - g++ python3-dev python3-pip * - Fedora - bubblewrap fuse3 git lzip patch python3 - gcc-c++ python3-devel python3-pip * - Ubuntu - bubblewrap fuse3 git lzip patch python3 - g++ python3-dev python3-pip .. _install-buildbox: Installing BuildBox ------------------- BuildStream depends on the following tools from `BuildBox `_: * ``buildbox-casd`` (to manage local and remote content-addressed storage) * ``buildbox-fuse`` (to check out content from the local CAS) * ``buildbox-run-bubblewrap`` (to run element commands in a controlled sandbox) These components can be installed from binaries, or built from source. Install binaries ~~~~~~~~~~~~~~~~ Browse the `release history of static binaries here `_. Linux x86-64 users can download the `latest statically linked binaries here `_, The contents of the tarball should be extracted into a directory in ``PATH``, e.g., ``~/.local/bin``. Build from source ~~~~~~~~~~~~~~~~~ Buildbox components can be installed from their git repository. We recommend installing from the latest release tag. See the <"Installation" section here `_. Buildbox contains many components, BuildStream needs only ``buildbox-casd``, ``buildbox-fuse`` and ``buildbox-run-bubblewrap``, which can be selected by passing options ``-DTOOLS=OFF -DCASD=ON -DFUSE=ON -DRUN_BUBBLEWRAP=ON`` to CMake. Finally, configure buildbox-run-bubblewrap as the default buildbox-run implementation:: ln -sv buildbox-run-bubblewrap /usr/local/bin/buildbox-run .. _install-buildstream: Installing BuildStream from a git checkout ------------------------------------------ First, clone the repository. Please check the existing tags in the git repository and determine which version you want to install:: git clone https://github.com/apache/buildstream.git cd buildstream git checkout We recommend ``pip`` as a frontend to the underlying ``setuptools`` build system. The following command will build and install BuildStream into your user's homedir in ``~/.local``, and will attempt to fetch and install any required PyPI dependencies from the internet at the same time:: pip3 install --user . We do not recommend using Pip's `editable mode `_ (the ``-e`` flag). See `this issue `_ for discussion. If you want to stop Pip from fetching missing dependencies, use the ``--no-index`` and ``--no-deps`` options. Finally, check that the ``PATH`` variable contains the ``~/.local/bin`` directory. If it doesn't, you could add this to the end of your Bash configuration ``~/.bashrc`` and restart Bash:: export PATH="${PATH}:${HOME}/.local/bin" Note for packagers ~~~~~~~~~~~~~~~~~~ Distro packaging standards may recommend a specific installation method for Python packages. BuildStream can be installed with any build frontend that supports the `PEP517 standard `_. You are also welcome to use the underlying `setuptools `_ build backend directly. .. _install-virtual-environment: Installing in virtual environments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can consider installing BuildStream in a `Virtual Environment `_ if you want to install multiple versions of BuildStream, or to isolate BuildStream and its dependencies from other Python packages. Here is how to install BuildStream stable and development snapshot releases in virtual environments of their own:: # Install BuildStream stable in an environment called "venv-bst-stable" # (At time of writing, this will be BuildStream 1) python3 -m venv venv-bst-stable venv-bst-stable/bin/pip install BuildStream # Install BuildStream latest development snapshot in an environment # called "venv-bst-latest" # (At time of writing, this will be Buildstream 2) python3 -m venv venv-bst-latest venv-bst-latest/bin/pip install --pre BuildStream To start using BuildStream from the desired environment, you will need to activate it first. Activating it will automatically add ``bst`` to your ``PATH`` and set up other necessary environment variables:: # Use BuildStream stable from venv-bst-stable source venv-bst-stable/bin/activate bst --version # Use BuildStream latest from venv-bst-latest source venv-bst-latest/bin/activate bst --version # Once you are done, remember to deactivate the virtual environment deactivate If you do not want to manage your virtual environments manually, you can consider using `pipx `_. .. _install-completions: Installing completions ---------------------- BuildStream integrates with Bash and Zsh to provide helpful tab-completion. These completion scripts require manual installation. Bash completions ~~~~~~~~~~~~~~~~ Bash completions are provided by the ``bst`` completion script, available online (`src/buildstream/data/bst `_) and in your local Git clone at ``src/buildstream/data/bst``. To install for the current user, paste the contents of the completion script into the file ``~/.bash_completion``. To install completions system-wide, copy the completion script to the system-wide bash-completion installation path, which you can discover as follows:: pkg-config --variable=completionsdir bash-completion See the `bash-completion FAQ `_ for more information. Zsh completions ~~~~~~~~~~~~~~~~ Zsh completions are provided by the ``_bst`` completion script, available online (`src/buildstream/data/zsh/_bst `_) and in your local Git clone at ``src/buildstream/data/zsh/_bst``. Copy the above file to your Zsh completions location. Here are some instructions for vanilla Zsh, as well as the *Prezto* and *Oh My Zsh* frameworks: **Zsh**:: cp src/buildstream/data/zsh/_bst ~/.zfunc/_bst You must then add the following lines in your ``~/.zshrc``, if they do not already exist:: fpath+=~/.zfunc autoload -Uz compinit && compinit **Prezto**:: cp src/buildstream/data/zsh/_bst ~/.zprezto/modules/completion/external/src/_bst You may have to reset your zcompdump cache, if you have one, and then restart your shell:: rm ~/.zcompdump ${XDG_CACHE_HOME:-$HOME/.cache}/prezto/zcompdump **Oh My Zsh**:: mkdir $ZSH_CUSTOM/plugins/bst cp src/buildstream/data/zsh/_bst $ZSH_CUSTOM/plugins/bst/_bst You must then add ``bst`` to your plugins array in ``~/.zshrc``:: plugins( bst ... ) apache-buildstream-27ae392/doc/source/main_porting.rst000066400000000000000000000014431514607367700231110ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _main_porting: Porting guide ============= This document focuses on porting your existing BuildStream 1 projects to using BuildStream 2. .. toctree:: :maxdepth: 1 porting_user_configuration porting_command_line porting_project apache-buildstream-27ae392/doc/source/main_using.rst000066400000000000000000000016651514607367700225620ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _using: Using ===== This section includes user facing documentation including tutorials, guides and information on user preferences and configuration. .. toctree:: :maxdepth: 2 using_tutorial using_developing using_handling_files using_junctions using_config using_commands using_configuring_cache_server using_configuring_remote_execution using_examples apache-buildstream-27ae392/doc/source/plugin.rsttemplate000066400000000000000000000000331514607367700234470ustar00rootroot00000000000000.. automodule:: @@MODULE@@ apache-buildstream-27ae392/doc/source/porting_command_line.rst000066400000000000000000000137401514607367700246150ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _porting_command_line: Porting command line usage ========================== This document outlines breaking changes made to the :ref:`command line interface ` in BuildStream 2. :ref:`bst init ` ------------------------------- * The global ``--directory`` option is no longer observed by ``bst init``, instead the command accepts an optional target directory argument. * The ``--format-version`` option has been removed in favor of the new ``--min-version`` option. :ref:`bst build ` --------------------------------- * Tracking is no longer supported at build time and must be performed separately, this removes the ``--track``, ``--track-all``, ``--track-except``, ``--track-cross-junctions`` and ``--track-save`` options from the command. To track your elements in BuildStream 2, use the :ref:`bst source track ` command instead. * The ``--all`` option which was used to indicate that all dependencies should be built regardless of whether they are needed for producing the target elements has been removed in favor of adding the ``--deps`` option. To acheive the same functionality, use ``bst build --deps all ...``. :ref:`bst show ` ------------------------------- * The ``plan`` value is no longer supported as a value for the ``--deps`` option. * Values for the ``%{state}`` format have changed * :mod:`junction ` elements will display ``junction``, as these cannot be built * In the case a cached failed build artifact is found, then ``failed`` will be displayed * Due to changes in the scheduler, we may observe changes as to when ``waiting``, ``buildable``, ``fetch needed`` are displayed for a given element. :ref:`bst fetch ` ---------------------------------------- * This command has been removed as a top-level command and now exists as :ref:`bst source fetch ` * Tracking is no longer supported at fetch time and must be performed separately, this removes the ``--track`` and ``--track-cross-junctions`` options from the command. To track your elements in BuildStream 2, use the :ref:`bst source track ` command instead. * The ``plan`` value is no longer supported as a value for the ``--deps`` option. The default value for the ``--deps`` option is now ``none``. :ref:`bst track ` ---------------------------------------- * This command has been removed as a top-level command and now exists as :ref:`bst source track ` :ref:`bst pull ` ---------------------------------------- * This command has been removed as a top-level command and now exists as :ref:`bst artifact pull ` * The ``--remote`` option has been removed in favor the ``--artifact-remote`` option, which can be specified multiple times. * The values which can be specified by ``--artifact-remote`` options have a new format which is :ref:`documented here `. :ref:`bst push ` ---------------------------------------- * This command has been removed as a top-level command and now exists as :ref:`bst artifact push ` * The ``--remote`` option has been removed in favor the ``--artifact-remote`` option, which can be specified multiple times. * The values which can be specified by ``--artifact-remote`` options have a new format which is :ref:`documented here `. :ref:`bst checkout ` ------------------------------------------------ * This command has been removed as a top-level command and now exists as :ref:`bst artifact checkout ` * The trailing ``LOCATION`` argument has been removed in favor of a ``--directory`` option. **BuildStream 1:** .. code:: shell bst checkout element.bst ~/checkout **BuildStream 2:** .. code:: shell bst artifact checkout --directory ~/checkout element.bst :ref:`bst shell ` --------------------------------- * The ``--sysroot`` option has been completely removed. This is no longer needed for failed builds as the build tree will be cached in a failed build artifact. * Sources and artifacts required to produce the shell environment will now be downloaded automatically by default. :ref:`bst workspace open ` --------------------------------------------------- * The ``--track`` option is now removed. * The trailing ``LOCATION`` argument has been removed in favor of a ``--directory`` option. **BuildStream 1:** .. code:: shell bst workspace open element.bst ~/workspace **BuildStream 2:** .. code:: shell bst workspace open --directory ~/workspace element.bst :ref:`bst workspace reset ` ----------------------------------------------------- * The ``--track`` option is now removed. :ref:`bst source-bundle ` --------------------------------------------------- This command has been completely removed, but similar behavior can be achieved using the :ref:`bst source checkout ` command. **BuildStream 1:** .. code:: shell bst source-bundle --directory ~/bundle element.bst **BuildStream 2:** .. code:: shell bst source checkout \ --tar ~/sources.tgz \ --compression gz \ --include-build-scripts \ element.bst apache-buildstream-27ae392/doc/source/porting_project.rst000066400000000000000000000327261514607367700236430ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _porting_project: Porting the project format ========================== This document outlines breaking changes made to the project format in BuildStream 2. The project.conf ---------------- This section outlines breaking changes made to the :ref:`project.conf format `. Project name ~~~~~~~~~~~~ Various features related to :mod:`junction ` elements have been added which allow addressing projects by their :ref:`project names `. For this reason, it is important to ensure that your project names are appropriately unique. Project versioning ~~~~~~~~~~~~~~~~~~ Instead of maintaining a separate version number for the format and for BuildStream releases, projects now declare the minimum version of BuildStream 2 they depend on. The ``format-version`` attribute should be removed from your project.conf (if present) and the :ref:`min-version ` attribute must be added. Some attributes can only be specified in project.conf ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Whenever specifying any of the following toplevel project attributes, they must be specified inside the project.conf itself and cannot be :ref:`included ` from a separate file: * :ref:`name ` * :ref:`element-path ` * :ref:`min-version ` * :ref:`plugins ` Artifact cache configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The format for declaring :ref:`artifact caches ` which are associated to your project have been completely redesigned. **BuildStream 1:** .. code:: yaml # # We used to specify a single URL # artifacts: url: https://foo.com/artifacts **BuildStream 2:** .. code:: yaml # # Now we declare a list, and credentials have been split # out into a separate "auth" dictionary # artifacts: - url: https://foo.com:11001 auth: server-cert: server.crt Loading plugins ~~~~~~~~~~~~~~~ The format for :ref:`loading plugins ` has been completely redesigned. .. tip:: A new method for :ref:`loading plugins through junctions ` has been added. In the interest of ensuring strong determinism and reliability it is strongly recommended to use this new method. Local plugins ''''''''''''' Here is an example of how loading :ref:`local plugins ` has changed. **BuildStream 1:** .. code:: yaml plugins: - origin: local path: plugins/sources # # We used to specify version numbers, these no longer exist. # sources: mysource: 0 **BuildStream 2:** .. code:: yaml plugins: - origin: local path: plugins/sources # # Now we merely specify a list of plugins to load from # a given project local directory # sources: - mysource Pip plugins ''''''''''' Here is an example of how loading :ref:`pip plugins ` has changed. **BuildStream 1:** .. code:: yaml plugins: - origin: pip package-name: vegetables # # We used to specify version numbers, these no longer exist. # elements: potato: 0 **BuildStream 2:** .. code:: yaml plugins: - origin: pip # # We can now specify version constraints # package-name: vegetables>=1.2 # # Now we merely specify a list of plugins to load from # a given pip package that is expected to be installed # elements: - potato Core elements ------------- This section outlines breaking changes made to :ref:`core element plugins ` which may require you to make changes to your project. The :mod:`stack ` element ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Stack elements dependencies are now hard required to be both build and runtime dependencies. The :mod:`script ` element ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``layout`` attribute has now been removed in favor of dependency level configuration. Here is an example script which collects a manifest of all files in the hypothetical ``system.bst`` element, using a hypothetical base runtime element ``base-utilities.bst``. **BuildStream 1:** .. code:: yaml kind: script build-depends: - base-utilities.bst - system.bst config: # # The old format was redundant and required explicit layout # of the dependencies already declared above. # layout: - element: base-utilities.bst destination: / - element: system.bst destination: "%{build-root}" commands: - find %{build-root} > %{install-root}/manifest.log **BuildStream 2:** .. code:: yaml kind: script # # The default location is "/" so there is no need to configure # the "base-utilities.bst" dependency # build-depends: - base-utilities.bst - system.bst config: location: "%{build-root}" config: commands: - find %{build-root} > %{install-root}/manifest.log .. tip:: The ``location`` dependency level configuration is also supported by all :mod:`BuildElement ` plugins. The :mod:`junction ` element ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The YAML format for declaring junctions has not changed, however the way that multiple junctions interact in a loaded pipeline has changed significantly. Specifically, the :ref:`element name ` used to declare a junction no longer has any special significance, whereas in BuildStream 1 the junction's name is used to coalesce matching junctions in subprojects. BuildStream 2 offers more flexibility in this regard, and allows you to *inherit* a junction from a subproject, by using a :mod:`link ` element directly in place of a junction, and/or explicitly override the configuration of a subproject's junction using the new ``overrides`` configuration attribute which the junction element now provides. Consult the :mod:`junction ` element documentation for a more detailed explanation. Migrated plugins ---------------- A majority of the plugins which used to be considered core plugins have been removed from BuildStream in favor of a more modular and distributed approach. The remaining core plugins are :ref:`documented here `. Any core plugins which you have been using in BuildStream 1 which have been migrated to separate repositories will need to be accessed externally. +---------------+-----------------------------------------------------------------------------------------+ | Plugin | New location | +===============+=========================================================================================+ | **Element plugins** | +---------------+-----------------------------------------------------------------------------------------+ | make | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | autotools | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | cmake | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | distutils | `buildstream-plugins `__ (as setuptools) | +---------------+-----------------------------------------------------------------------------------------+ | pip | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | meson | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | qmake | `bst-plugins-experimental `_ | +---------------+-----------------------------------------------------------------------------------------+ | makemaker | `bst-plugins-experimental `_ | +---------------+-----------------------------------------------------------------------------------------+ | modulebuild | `bst-plugins-experimental `_ | +---------------+-----------------------------------------------------------------------------------------+ | **Source plugins** | +---------------+-----------------------------------------------------------------------------------------+ | bzr | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | git | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | patch | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | pip | `buildstream-plugins `__ | +---------------+-----------------------------------------------------------------------------------------+ | deb | `bst-plugins-experimental `_ | +---------------+-----------------------------------------------------------------------------------------+ | ostree | `bst-plugins-experimental `_ | +---------------+-----------------------------------------------------------------------------------------+ | zip | `bst-plugins-experimental `_ | +---------------+-----------------------------------------------------------------------------------------+ .. attention:: **YAML composition with externally loaded plugins** Note that when :ref:`YAML composition ` occurs with plugins loaded from external projects, the *plugin defaults* will be composited with *your project.conf* and not with the project.conf originating in the external project containing the plugin. Example of externally loaded plugin ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ It is recommended to transition directly :ref:`loading these plugins through junctions `, which can be done as follows. Create an alias for PyPI in your project.conf ''''''''''''''''''''''''''''''''''''''''''''' .. code:: yaml aliases: pypi: https://files.pythonhosted.org/packages/ Create buildstream-plugins-junction.bst ''''''''''''''''''''''''''''''''''''''' Create a junction which accesses the release tarball of the plugin repository. .. code:: yaml kind: junction sources: - kind: tar url: pypi:e2/d8/ed9e849a1386297f854f9fa0200f3fa108498c0fdb5c86468c1601c7e571/buildstream-plugins-1.91.0.tar.gz ref: 44c6ea15d15476b68d0767c1d410d416f71544e57be572201058f8b3d3b05f83 Declare the plugin you want to use in your project.conf ''''''''''''''''''''''''''''''''''''''''''''''''''''''' This will make the ``make`` and ``meson`` element plugins from the `buildstream-plugins `__ project available for use in your project. .. code:: yaml plugins: - origin: junction junction: buildstream-plugins-junction.bst elements: - make - meson Miscellaneous ------------- Here we list some miscellaneous breaking changes to the format in general. Element naming ~~~~~~~~~~~~~~ The names of elements have :ref:`become more restrictive `, for example they must have the ``.bst`` extension. Overlap whitelist ~~~~~~~~~~~~~~~~~ The :ref:`overlap whitelist `, which is the public data found on elements which indicate which files an element can overwrite, must now be expressed with absolute paths. Strip commands ~~~~~~~~~~~~~~ The default ``strip-commands`` which :mod:`BuildElement ` implementations use to split out debug symbols from binaries have been removed. This can be solved by declaring a value for the ``%{strip-binaries}`` variable which will be used for this purpose. apache-buildstream-27ae392/doc/source/porting_user_configuration.rst000066400000000000000000000033271514607367700260750ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _porting_user_configuration: Porting the buildstream.conf ============================ This document outlines breaking changes made to the :ref:`user configuration ` in BuildStream 2. Filename and parallel installation ---------------------------------- The default filename to load user configuration remains unchanged, however, if you plan to install and use both versions of BuildStream on the same host, it is recommended to keep your BuildStream 2 configuration in a file named ``buildstream2.conf``. Working directories ------------------- The ``builddir`` and ``artifactdir`` have been removed in favor of the new ``cachedir``. BuildStream 1: ~~~~~~~~~~~~~~ .. code:: yaml builddir: ${XDG_CACHE_HOME}/buildstream/build artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts BuildStream 2: ~~~~~~~~~~~~~~ .. code:: yaml cachedir: ${XDG_CACHE_HOME}/buildstream/cache Remote cache configuration -------------------------- The configuration for remote artifact caches has been completely redesigned, please refer to the :ref:`artifact cache configuration documentation ` for details on how to configure remotes in BuildStream 2. apache-buildstream-27ae392/doc/source/sample_plugin/000077500000000000000000000000001514607367700225265ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/sample_plugin/MANIFEST.in000066400000000000000000000000261514607367700242620ustar00rootroot00000000000000global-include *.yaml apache-buildstream-27ae392/doc/source/sample_plugin/setup.py000066400000000000000000000005541514607367700242440ustar00rootroot00000000000000from setuptools import setup, find_packages setup(name='BuildStream Autotools', version="0.1", description="A better autotools element for BuildStream", packages=find_packages(), include_package_data=True, entry_points={ 'buildstream.plugins.elements': [ 'autotools = elements.autotools' ] }) apache-buildstream-27ae392/doc/source/sessions-stored/000077500000000000000000000000001514607367700230335ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/sessions-stored/autotools-build.html000066400000000000000000001323141514607367700270530ustar00rootroot00000000000000 
user@host:~/autotools$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:54:03
  Project:       autotools (/home/user/autotools)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
   buildable 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
     waiting 7e7bdf36e3d6947ebf279f651fc492c0d20f919e114fb3af8196ac6d2863a8b7 base.bst 
     waiting 26721a3b05918bec80836da7b713f8f0227410c6a7fecc82a158e0bebe7baddf hello.bst 
===============================================================================
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   autotools/base-alpine/179c6ae9-build.4731.log
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Staging sources
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Staging sources
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS autotools/base-alpine/179c6ae9-build.4731.log
[--:--:--][7e7bdf36][   build:base.bst                      ] START   autotools/base/7e7bdf36-build.4738.log
[--:--:--][7e7bdf36][   build:base.bst                      ] START   Caching artifact
[00:00:00][7e7bdf36][   build:base.bst                      ] SUCCESS Caching artifact
[00:00:00][7e7bdf36][   build:base.bst                      ] SUCCESS autotools/base/7e7bdf36-build.4738.log
[--:--:--][26721a3b][   build:hello.bst                     ] START   autotools/hello/26721a3b-build.4742.log
[--:--:--][26721a3b][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][26721a3b][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][26721a3b][   build:hello.bst                     ] START   Staging sources
[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][26721a3b][   build:hello.bst                     ] START   Running configure-commands
[--:--:--][26721a3b][   build:hello.bst                     ] STATUS  Running command

    export NOCONFIGURE=1;
    
    if [ -x ./configure ]; then true;
    elif [ -x ./autogen ]; then ./autogen;
    elif [ -x ./autogen.sh ]; then ./autogen.sh;
    elif [ -x ./bootstrap ]; then ./bootstrap;
    elif [ -x ./bootstrap.sh ]; then ./bootstrap.sh;
    else autoreconf -ivf .;
    fi

[--:--:--][26721a3b][   build:hello.bst                     ] STATUS  Running command

    ./configure --prefix=/usr \
    --exec-prefix=/usr \
    --bindir=/usr/bin \
    --sbindir=/usr/sbin \
    --sysconfdir=/etc \
    --datadir=/usr/share \
    --includedir=/usr/include \
    --libdir=/usr/lib \
    --libexecdir=/usr/libexec \
    --localstatedir=/var \
    --sharedstatedir=/usr/com \
    --mandir=/usr/share/man \
    --infodir=/usr/share/info   

[00:00:03][26721a3b][   build:hello.bst                     ] SUCCESS Running configure-commands
[--:--:--][26721a3b][   build:hello.bst                     ] START   Running build-commands
[--:--:--][26721a3b][   build:hello.bst                     ] STATUS  Running command

    make

[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][26721a3b][   build:hello.bst                     ] START   Running install-commands
[--:--:--][26721a3b][   build:hello.bst                     ] STATUS  Running command

    make -j1 DESTDIR="/buildstream-install" install

[--:--:--][26721a3b][   build:hello.bst                     ] STATUS  Running command

    if false || false; then
      find "/buildstream-install" -name "*.la" -print0 | while read -d '' -r file; do
        if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        else
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        fi
      done
    fi

[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][26721a3b][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][26721a3b][   build:hello.bst                     ] START   Caching artifact
[00:00:00][26721a3b][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:04][26721a3b][   build:hello.bst                     ] SUCCESS autotools/hello/26721a3b-build.4742.log
[00:00:05][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     3
  Fetch Queue: processed 0, skipped 3, failed 0 
  Build Queue: processed 3, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/autotools-shell.html000066400000000000000000000242731514607367700270670ustar00rootroot00000000000000 
user@host:~/autotools$ bst shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hello World!
This is amhello 1.0.
apache-buildstream-27ae392/doc/source/sessions-stored/autotools-show-variables.html000066400000000000000000000177251514607367700307120ustar00rootroot00000000000000 
user@host:~/autotools$ bst show --deps none --format "%{vars}" hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
prefix: /usr
exec_prefix: /usr
bindir: /usr/bin
sbindir: /usr/sbin
libexecdir: /usr/libexec
datadir: /usr/share
sysconfdir: /etc
sharedstatedir: /usr/com
localstatedir: /var
lib: lib
libdir: /usr/lib
debugdir: /usr/lib/debug
includedir: /usr/include
docdir: /usr/share/doc
infodir: /usr/share/info
mandir: /usr/share/man
build-root: /buildstream/autotools/hello.bst
conf-root: .
install-root: /buildstream-install
strip-binaries: ''
fix-pyc-timestamps: "find \"/buildstream-install\" -name '*.pyc' -exec \\\n  dd if=/dev/zero\
  \ of={} bs=1 count=4 seek=4 conv=notrunc ';'"
project-name: autotools
max-jobs: '8'
autogen: "export NOCONFIGURE=1;\n\nif [ -x ./configure ]; then true;\nelif [ -x ./autogen\
  \ ]; then ./autogen;\nelif [ -x ./autogen.sh ]; then ./autogen.sh;\nelif [ -x ./bootstrap\
  \ ]; then ./bootstrap;\nelif [ -x ./bootstrap.sh ]; then ./bootstrap.sh;\nelse autoreconf\
  \ -ivf .;\nfi"
conf-global: ''
conf-local: ''
conf-extra: ''
conf-cmd: ./configure
conf-args: "--prefix=/usr \\\n--exec-prefix=/usr \\\n--bindir=/usr/bin \\\n--sbindir=/usr/sbin\
  \ \\\n--sysconfdir=/etc \\\n--datadir=/usr/share \\\n--includedir=/usr/include \\\
  \n--libdir=/usr/lib \\\n--libexecdir=/usr/libexec \\\n--localstatedir=/var \\\n\
  --sharedstatedir=/usr/com \\\n--mandir=/usr/share/man \\\n--infodir=/usr/share/info\
  \   "
configure: "./configure --prefix=/usr \\\n--exec-prefix=/usr \\\n--bindir=/usr/bin\
  \ \\\n--sbindir=/usr/sbin \\\n--sysconfdir=/etc \\\n--datadir=/usr/share \\\n--includedir=/usr/include\
  \ \\\n--libdir=/usr/lib \\\n--libexecdir=/usr/libexec \\\n--localstatedir=/var \\\
  \n--sharedstatedir=/usr/com \\\n--mandir=/usr/share/man \\\n--infodir=/usr/share/info\
  \   "
make: make
make-install: make -j1 DESTDIR="/buildstream-install" install
remove-libtool-modules: 'false'
remove-libtool-libraries: 'false'
delete-libtool-archives: "if false || false; then\n  find \"/buildstream-install\"\
  \ -name \"*.la\" -print0 | while read -d '' -r file; do\n    if grep '^shouldnotlink=yes$'\
  \ \"${file}\" &>/dev/null; then\n      if false; then\n        echo \"Removing ${file}.\"\
  \n        rm \"${file}\"\n      else\n        echo \"Not removing ${file}.\"\n \
  \     fi\n    else\n      if false; then\n        echo \"Removing ${file}.\"\n \
  \       rm \"${file}\"\n      else\n        echo \"Not removing ${file}.\"\n   \
  \   fi\n    fi\n  done\nfi"
command-subdir: doc/amhello
element-name: hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/composition-build.html000066400000000000000000002211531514607367700273650ustar00rootroot00000000000000 
user@host:~/composition$ bst build runtime-only.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:51:33
  Project:       composition (/home/user/composition)
  Targets:       runtime-only.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
   buildable afe1fb0c366992c006f58c49017f0b48031f6f397a1e2f0e73756d619de61ffc base/alpine.bst 
     waiting ba5d4acbd9563000d7f2cb65c446a280c9de8025995f4d6affb085174f7b43cc base.bst 
     waiting 9be5097f82d3d7186227d0ef406c8860dca858378e6c83e7c72d84c588ba2f36 libhello.bst 
     waiting 55edded75fed8a2ed639e4b6d38a24358ebf3d1de527c74839c27d5055fb77e9 hello.bst 
     waiting f61cf06345976e20b194912d4580c2f6f04cecb1fe0054bddc4f0404e0e47b0e runtime-only.bst 
===============================================================================
[--:--:--][afe1fb0c][   build:base/alpine.bst               ] START   composition/base-alpine/afe1fb0c-build.30826.log
[--:--:--][afe1fb0c][   build:base/alpine.bst               ] START   Staging sources
[00:00:00][afe1fb0c][   build:base/alpine.bst               ] SUCCESS Staging sources
[--:--:--][afe1fb0c][   build:base/alpine.bst               ] START   Caching artifact
[00:00:00][afe1fb0c][   build:base/alpine.bst               ] SUCCESS Caching artifact
[00:00:00][afe1fb0c][   build:base/alpine.bst               ] SUCCESS composition/base-alpine/afe1fb0c-build.30826.log
[--:--:--][ba5d4acb][   build:base.bst                      ] START   composition/base/ba5d4acb-build.30833.log
[--:--:--][ba5d4acb][   build:base.bst                      ] START   Caching artifact
[00:00:00][ba5d4acb][   build:base.bst                      ] SUCCESS Caching artifact
[00:00:00][ba5d4acb][   build:base.bst                      ] SUCCESS composition/base/ba5d4acb-build.30833.log
[--:--:--][9be5097f][   build:libhello.bst                  ] START   composition/libhello/9be5097f-build.30837.log
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Staging dependencies
[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Staging dependencies
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Integrating sandbox
[--:--:--][9be5097f][   build:libhello.bst                  ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Integrating sandbox
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Staging sources
[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Staging sources
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Running build-commands
[--:--:--][9be5097f][   build:libhello.bst                  ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Running build-commands
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Running install-commands
[--:--:--][9be5097f][   build:libhello.bst                  ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Running install-commands
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Running strip-commands
[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Running strip-commands
[--:--:--][9be5097f][   build:libhello.bst                  ] START   Caching artifact
[00:00:00][9be5097f][   build:libhello.bst                  ] SUCCESS Caching artifact
[00:00:01][9be5097f][   build:libhello.bst                  ] SUCCESS composition/libhello/9be5097f-build.30837.log
[--:--:--][55edded7][   build:hello.bst                     ] START   composition/hello/55edded7-build.30874.log
[--:--:--][55edded7][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][55edded7][   build:hello.bst                     ] START   Integrating sandbox
[--:--:--][55edded7][   build:hello.bst                     ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][55edded7][   build:hello.bst                     ] START   Staging sources
[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][55edded7][   build:hello.bst                     ] START   Running build-commands
[--:--:--][55edded7][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][55edded7][   build:hello.bst                     ] START   Running install-commands
[--:--:--][55edded7][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][55edded7][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][55edded7][   build:hello.bst                     ] START   Caching artifact
[00:00:00][55edded7][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:01][55edded7][   build:hello.bst                     ] SUCCESS composition/hello/55edded7-build.30874.log
[--:--:--][f61cf063][   build:runtime-only.bst              ] START   composition/runtime-only/f61cf063-build.30907.log
[--:--:--][f61cf063][   build:runtime-only.bst              ] START   Staging dependencies
[00:00:00][f61cf063][   build:runtime-only.bst              ] SUCCESS Staging dependencies
[--:--:--][f61cf063][   build:runtime-only.bst              ] START   Computing split
[00:00:00][f61cf063][   build:runtime-only.bst              ] SUCCESS Computing split
[--:--:--][f61cf063][   build:runtime-only.bst              ] START   Integrating sandbox
[--:--:--][f61cf063][   build:runtime-only.bst              ] STATUS  Running command

    ldconfig "/usr/lib"

[--:--:--][f61cf063][   build:runtime-only.bst              ] INFO    Integration modified 367, added 0 and removed 0 files
[00:00:00][f61cf063][   build:runtime-only.bst              ] SUCCESS Integrating sandbox
[--:--:--][f61cf063][   build:runtime-only.bst              ] START   Creating composition

    Including files from domains: runtime
    Excluding orphaned files

[--:--:--][f61cf063][   build:runtime-only.bst              ] INFO    Composing 287 files
[00:00:00][f61cf063][   build:runtime-only.bst              ] SUCCESS Creating composition
[--:--:--][f61cf063][   build:runtime-only.bst              ] START   Caching artifact
[00:00:00][f61cf063][   build:runtime-only.bst              ] SUCCESS Caching artifact
[00:00:01][f61cf063][   build:runtime-only.bst              ] SUCCESS composition/runtime-only/f61cf063-build.30907.log
[00:00:04][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       5
  Session:     5
  Fetch Queue: processed 0, skipped 5, failed 0 
  Build Queue: processed 5, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/composition-list-contents.html000066400000000000000000000252431514607367700310760ustar00rootroot00000000000000 
user@host:~/composition$ bst artifact list-contents runtime-only.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
  runtime-only.bst:
	lib
	lib/ld-musl-x86_64.so.1
	usr
	usr/bin
	usr/bin/2to3
	usr/bin/2to3-3.6
	usr/bin/[
	usr/bin/[[
	usr/bin/aclocal
	usr/bin/aclocal-1.15
	usr/bin/addr2line
	usr/bin/ar
	usr/bin/as
	usr/bin/autoconf
	usr/bin/autoheader
	usr/bin/autom4te
	usr/bin/automake
	usr/bin/automake-1.15
	usr/bin/autoreconf
	usr/bin/autoscan
	usr/bin/autoupdate
	usr/bin/awk
	usr/bin/basename
	usr/bin/beep
	usr/bin/blkdiscard
	usr/bin/bunzip2
	usr/bin/bzcat
	usr/bin/bzip2
	usr/bin/c++filt
	usr/bin/c_rehash
	usr/bin/cal
	usr/bin/cc
	usr/bin/ccmake
	usr/bin/chvt
	usr/bin/cksum
	usr/bin/clear
	usr/bin/cmake
	usr/bin/cmp
	usr/bin/comm
	usr/bin/cpack
	usr/bin/cpio
	usr/bin/crontab
	usr/bin/cryptpw
	usr/bin/ctest
	usr/bin/cut
	usr/bin/dc
	usr/bin/deallocvt
	usr/bin/diff
	usr/bin/dirname
	usr/bin/dos2unix
	usr/bin/du
	usr/bin/dumpleases
	usr/bin/dwp
	usr/bin/easy_install-3.6
	usr/bin/eject
	usr/bin/elfedit
	usr/bin/env
	usr/bin/expand
	usr/bin/expr
	usr/bin/factor
	usr/bin/fallocate
	usr/bin/find
	usr/bin/flock
	usr/bin/fold
	usr/bin/free
	usr/bin/fuser
	usr/bin/gdbm_dump
	usr/bin/gdbm_load
	usr/bin/gdbmtool
	usr/bin/getconf
	usr/bin/getent
	usr/bin/gprof
	usr/bin/groups
	usr/bin/hd
	usr/bin/head
	usr/bin/hello
	usr/bin/hexdump
	usr/bin/hostid
	usr/bin/iconv
	usr/bin/id
	usr/bin/ifnames
	usr/bin/install
	usr/bin/ipcrm
	usr/bin/ipcs
	usr/bin/killall
	usr/bin/ld
	usr/bin/ld.bfd
	usr/bin/ldd
	usr/bin/less
	usr/bin/logger
	usr/bin/lsof
	usr/bin/lsusb
	usr/bin/lzcat
	usr/bin/lzma
	usr/bin/lzopcat
	usr/bin/m4
	usr/bin/make
	usr/bin/md5sum
	usr/bin/mesg
	usr/bin/microcom
	usr/bin/mkfifo
	usr/bin/mkpasswd
	usr/bin/nc
	usr/bin/nl
	usr/bin/nm
	usr/bin/nmeter
	usr/bin/nohup
	usr/bin/nproc
	usr/bin/nsenter
	usr/bin/nslookup
	usr/bin/objcopy
	usr/bin/objdump
	usr/bin/od
	usr/bin/openvt
	usr/bin/passwd
	usr/bin/paste
	usr/bin/patch
	usr/bin/perl
	usr/bin/perl5.26.1
	usr/bin/perldoc
	usr/bin/pgrep
	usr/bin/pip3
	usr/bin/pip3.6
	usr/bin/pkill
	usr/bin/pmap
	usr/bin/pod2html
	usr/bin/pod2man
	usr/bin/pod2text
	usr/bin/pod2usage
	usr/bin/podchecker
	usr/bin/podselect
	usr/bin/printf
	usr/bin/pscan
	usr/bin/pstree
	usr/bin/pwdx
	usr/bin/pydoc3
	usr/bin/pydoc3.6
	usr/bin/python3
	usr/bin/python3.6
	usr/bin/python3.6m
	usr/bin/pyvenv
	usr/bin/pyvenv-3.6
	usr/bin/ranlib
	usr/bin/readelf
	usr/bin/readlink
	usr/bin/realpath
	usr/bin/renice
	usr/bin/reset
	usr/bin/resize
	usr/bin/scanelf
	usr/bin/seq
	usr/bin/setkeycodes
	usr/bin/setsid
	usr/bin/sha1sum
	usr/bin/sha256sum
	usr/bin/sha3sum
	usr/bin/sha512sum
	usr/bin/showkey
	usr/bin/shred
	usr/bin/shuf
	usr/bin/size
	usr/bin/smemcap
	usr/bin/sort
	usr/bin/split
	usr/bin/ssl_client
	usr/bin/strings
	usr/bin/strip
	usr/bin/sum
	usr/bin/tac
	usr/bin/tail
	usr/bin/tcc
	usr/bin/tee
	usr/bin/test
	usr/bin/time
	usr/bin/timeout
	usr/bin/top
	usr/bin/tr
	usr/bin/traceroute
	usr/bin/traceroute6
	usr/bin/truncate
	usr/bin/tty
	usr/bin/ttysize
	usr/bin/udhcpc6
	usr/bin/unexpand
	usr/bin/uniq
	usr/bin/unix2dos
	usr/bin/unlink
	usr/bin/unlzma
	usr/bin/unlzop
	usr/bin/unshare
	usr/bin/unxz
	usr/bin/unzip
	usr/bin/uptime
	usr/bin/uudecode
	usr/bin/uuencode
	usr/bin/vi
	usr/bin/vlock
	usr/bin/volname
	usr/bin/wc
	usr/bin/wget
	usr/bin/which
	usr/bin/whoami
	usr/bin/whois
	usr/bin/xargs
	usr/bin/xmlwf
	usr/bin/xxd
	usr/bin/xzcat
	usr/bin/yes
	usr/lib
	usr/lib/libarchive.so.13
	usr/lib/libarchive.so.13.3.2
	usr/lib/libbfd-2.28.so
	usr/lib/libbz2.so.1
	usr/lib/libbz2.so.1.0.6
	usr/lib/libc.so
	usr/lib/libcrypto.so.42
	usr/lib/libcrypto.so.42.0.0
	usr/lib/libcurl.so.4
	usr/lib/libcurl.so.4.5.0
	usr/lib/libexpat.so.1
	usr/lib/libexpat.so.1.6.7
	usr/lib/libffi.so.6
	usr/lib/libffi.so.6.0.4
	usr/lib/libformw.so.6
	usr/lib/libformw.so.6.0
	usr/lib/libgcc_s.so.1
	usr/lib/libgdbm.so.4
	usr/lib/libgdbm.so.4.0.0
	usr/lib/libgdbm_compat.so.4
	usr/lib/libgdbm_compat.so.4.0.0
	usr/lib/libhello.so
	usr/lib/liblz4.so.1
	usr/lib/liblz4.so.1.8.0
	usr/lib/liblzma.so.5
	usr/lib/liblzma.so.5.2.3
	usr/lib/libmenuw.so.6
	usr/lib/libmenuw.so.6.0
	usr/lib/libncursesw.so.6
	usr/lib/libncursesw.so.6.0
	usr/lib/libopcodes-2.28.so
	usr/lib/libpanelw.so.6
	usr/lib/libpanelw.so.6.0
	usr/lib/libpython3.6m.so.1.0
	usr/lib/libpython3.so
	usr/lib/libreadline.so.7
	usr/lib/libreadline.so.7.0
	usr/lib/librhash.so.0
	usr/lib/libsqlite3.so.0
	usr/lib/libsqlite3.so.0.8.6
	usr/lib/libssh2.so.1
	usr/lib/libssh2.so.1.0.1
	usr/lib/libssl.so.44
	usr/lib/libssl.so.44.0.1
	usr/lib/libstdc++.so.6
	usr/lib/libstdc++.so.6.0.22
	usr/lib/libuv.so.1
	usr/lib/libuv.so.1.0.0
	usr/sbin
	usr/sbin/add-shell
	usr/sbin/addgroup
	usr/sbin/adduser
	usr/sbin/arping
	usr/sbin/brctl
	usr/sbin/chpasswd
	usr/sbin/chroot
	usr/sbin/crond
	usr/sbin/delgroup
	usr/sbin/deluser
	usr/sbin/ether-wake
	usr/sbin/fbset
	usr/sbin/fdformat
	usr/sbin/killall5
	usr/sbin/loadfont
	usr/sbin/lspci
	usr/sbin/nanddump
	usr/sbin/nandwrite
	usr/sbin/nbd-client
	usr/sbin/ntpd
	usr/sbin/partprobe
	usr/sbin/powertop
	usr/sbin/rdate
	usr/sbin/rdev
	usr/sbin/readahead
	usr/sbin/readprofile
	usr/sbin/remove-shell
	usr/sbin/rfkill
	usr/sbin/sendmail
	usr/sbin/setfont
	usr/sbin/setlogcons
	usr/sbin/update-ca-certificates
apache-buildstream-27ae392/doc/source/sessions-stored/composition-shell.html000066400000000000000000000243031514607367700273730ustar00rootroot00000000000000 
user@host:~/composition$ bst shell runtime-only.bst -- hello audience

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:runtime-only.bst              ] START   Staging dependencies
[00:00:00][        ][    main:runtime-only.bst              ] SUCCESS Staging dependencies
[--:--:--][        ][    main:runtime-only.bst              ] START   Integrating sandbox
[00:00:00][        ][    main:runtime-only.bst              ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:runtime-only.bst              ] STATUS  Running command

    hello audience

Hello audience
apache-buildstream-27ae392/doc/source/sessions-stored/developing-build-after-changes-workspace.html000066400000000000000000000324041514607367700336560ustar00rootroot00000000000000 
user@host:~/workspace_hello$ bst build

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:51:52
  Project:       developing (/codethink/GNOME/buildstream/doc/examples/developing)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
      cached 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
      cached 125099c65de3a18455f11c5a4a7379df93d5638460906c7f183cfae1b21a651b base.bst 
      cached ec74614e1f763b990614c97d00ce3494c11b7a0729fc8a914f8e8e4580599949 hello.bst Workspace: /home/user/workspace_hello
===============================================================================
[00:00:00][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     0
  Fetch Queue: processed 0, skipped 0, failed 0 
  Build Queue: processed 0, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/developing-build-after-changes.html000066400000000000000000001227201514607367700316630ustar00rootroot00000000000000 
user@host:~/developing$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:51:51
  Project:       developing (/home/user/developing)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
   buildable 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
     waiting 125099c65de3a18455f11c5a4a7379df93d5638460906c7f183cfae1b21a651b base.bst 
     waiting ec74614e1f763b990614c97d00ce3494c11b7a0729fc8a914f8e8e4580599949 hello.bst Workspace: /home/user/developing/workspace_hello
===============================================================================
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   developing/base-alpine/179c6ae9-build.31085.log
[--:--:--][ec74614e][   fetch:hello.bst                     ] START   developing/hello/ec74614e-fetch.31086.log
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Staging sources
[00:00:00][ec74614e][   fetch:hello.bst                     ] SUCCESS developing/hello/ec74614e-fetch.31086.log
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Staging sources
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS developing/base-alpine/179c6ae9-build.31085.log
[--:--:--][125099c6][   build:base.bst                      ] START   developing/base/125099c6-build.31096.log
[--:--:--][125099c6][   build:base.bst                      ] START   Caching artifact
[00:00:00][125099c6][   build:base.bst                      ] SUCCESS Caching artifact
[00:00:00][125099c6][   build:base.bst                      ] SUCCESS developing/base/125099c6-build.31096.log
[--:--:--][ec74614e][   build:hello.bst                     ] START   developing/hello/ec74614e-build.31100.log
[--:--:--][ec74614e][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][ec74614e][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][ec74614e][   build:hello.bst                     ] START   Staging sources
[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][ec74614e][   build:hello.bst                     ] START   Running build-commands
[--:--:--][ec74614e][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][ec74614e][   build:hello.bst                     ] START   Running install-commands
[--:--:--][ec74614e][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][ec74614e][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][ec74614e][   build:hello.bst                     ] START   Caching artifact
[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:00][ec74614e][   build:hello.bst                     ] SUCCESS developing/hello/ec74614e-build.31100.log
[00:00:01][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     3
  Fetch Queue: processed 1, skipped 2, failed 0 
  Build Queue: processed 3, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/developing-close-workspace.html000066400000000000000000000147001514607367700311560ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace close hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] INFO    Closed workspace for hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/developing-discard-workspace.html000066400000000000000000000204611514607367700314630ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace close --remove-dir hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Removing workspace directory /home/user/developing/workspace_hello
[00:00:00][        ][    main:core activity                 ] SUCCESS Removing workspace directory /home/user/developing/workspace_hello
[--:--:--][        ][    main:core activity                 ] INFO    Closed workspace for hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/developing-reopen-workspace.html000066400000000000000000000166151514607367700313500ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace open --no-checkout --directory workspace_hello hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] INFO    Creating workspace for element hello.bst
[--:--:--][        ][    main:core activity                 ] INFO    Created a workspace for element: hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/developing-reset-workspace.html000066400000000000000000000515321514607367700311770ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace reset hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Removing workspace directory /home/user/developing/workspace_hello
[00:00:00][        ][    main:core activity                 ] SUCCESS Removing workspace directory /home/user/developing/workspace_hello
[--:--:--][        ][    main:core activity                 ] INFO    Closed workspace for hello.bst
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources
[--:--:--][        ][    main:core activity                 ] INFO    Creating workspace for element hello.bst
[--:--:--][        ][    main:hello.bst                     ] START   Staging sources to /home/user/developing/workspace_hello
[--:--:--][        ][    main:hello.bst                     ] START   Staging local files into CAS
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging local files into CAS
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging sources to /home/user/developing/workspace_hello
[--:--:--][        ][    main:core activity                 ] INFO    Created a workspace for element: hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/developing-shell-after-changes.html000066400000000000000000000242751514607367700317010ustar00rootroot00000000000000 
user@host:~/developing$ bst shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hello World
We can use workspaces!
apache-buildstream-27ae392/doc/source/sessions-stored/developing-soft-reset.html000066400000000000000000000147661514607367700301640ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace reset --soft hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] INFO    Reset workspace state for hello.bst at: /home/user/developing/workspace_hello
apache-buildstream-27ae392/doc/source/sessions-stored/developing-workspace-list.html000066400000000000000000000005531514607367700310250ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace list

workspaces:
- element: hello.bst
  directory: /home/user/developing/workspace_hello
apache-buildstream-27ae392/doc/source/sessions-stored/developing-workspace-open.html000066400000000000000000000315631514607367700310200ustar00rootroot00000000000000 
user@host:~/developing$ bst workspace open --directory workspace_hello hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources
[--:--:--][        ][    main:core activity                 ] INFO    Creating workspace for element hello.bst
[--:--:--][        ][    main:hello.bst                     ] START   Staging sources to /home/user/developing/workspace_hello
[--:--:--][        ][    main:hello.bst                     ] START   Staging local files into CAS
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging local files into CAS
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging sources to /home/user/developing/workspace_hello
[--:--:--][        ][    main:core activity                 ] INFO    Created a workspace for element: hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/directives-build-excited.html000066400000000000000000000743221514607367700306120ustar00rootroot00000000000000 
user@host:~/directives$ bst --option flavor excited build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:32
  Project:       directives (/home/user/directives)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Project Options
  flavor: excited

Pipeline
      cached 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
      cached c556a766f75b4d44070ae7eb39b344142736f5505b535299a6d4f5f1376de8eb base.bst 
   buildable 3d93bafadd3d0b00d16a558c46af4c7b3ca8d81ddcd0d0a184faee2c876ca12c hello.bst 
===============================================================================
[--:--:--][3d93bafa][   build:hello.bst                     ] START   directives/hello/3d93bafa-build.32121.log
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Staging sources
[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Running build-commands
[--:--:--][3d93bafa][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr" GREETING="Howdy there, and what a world it is !"

[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Running install-commands
[--:--:--][3d93bafa][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][3d93bafa][   build:hello.bst                     ] START   Caching artifact
[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:00][3d93bafa][   build:hello.bst                     ] SUCCESS directives/hello/3d93bafa-build.32121.log
[00:00:00][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     1
  Fetch Queue: processed 0, skipped 1, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/directives-build-normal.html000066400000000000000000001171731514607367700304570ustar00rootroot00000000000000 
user@host:~/directives$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:29
  Project:       directives (/home/user/directives)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Project Options
  flavor: normal

Pipeline
   buildable 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
     waiting c556a766f75b4d44070ae7eb39b344142736f5505b535299a6d4f5f1376de8eb base.bst 
     waiting cad7581043070b047af4834845fc584abd88ef51040aa6584186eeed10fea068 hello.bst 
===============================================================================
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   directives/base-alpine/179c6ae9-build.32015.log
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Staging sources
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Staging sources
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS directives/base-alpine/179c6ae9-build.32015.log
[--:--:--][c556a766][   build:base.bst                      ] START   directives/base/c556a766-build.32022.log
[--:--:--][c556a766][   build:base.bst                      ] START   Caching artifact
[00:00:00][c556a766][   build:base.bst                      ] SUCCESS Caching artifact
[00:00:00][c556a766][   build:base.bst                      ] SUCCESS directives/base/c556a766-build.32022.log
[--:--:--][cad75810][   build:hello.bst                     ] START   directives/hello/cad75810-build.32026.log
[--:--:--][cad75810][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][cad75810][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][cad75810][   build:hello.bst                     ] START   Staging sources
[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][cad75810][   build:hello.bst                     ] START   Running build-commands
[--:--:--][cad75810][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr" GREETING="Hello world !"

[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][cad75810][   build:hello.bst                     ] START   Running install-commands
[--:--:--][cad75810][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][cad75810][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][cad75810][   build:hello.bst                     ] START   Caching artifact
[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:00][cad75810][   build:hello.bst                     ] SUCCESS directives/hello/cad75810-build.32026.log
[00:00:01][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     3
  Fetch Queue: processed 0, skipped 3, failed 0 
  Build Queue: processed 3, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/directives-build-somber.html000066400000000000000000000742651514607367700304620ustar00rootroot00000000000000 
user@host:~/directives$ bst --option flavor somber build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:31
  Project:       directives (/home/user/directives)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Project Options
  flavor: somber

Pipeline
      cached 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
      cached c556a766f75b4d44070ae7eb39b344142736f5505b535299a6d4f5f1376de8eb base.bst 
   buildable 028e738455de72766973477172442f4c6d89eaff445894e69a29ed7d850cfb15 hello.bst 
===============================================================================
[--:--:--][028e7384][   build:hello.bst                     ] START   directives/hello/028e7384-build.32072.log
[--:--:--][028e7384][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][028e7384][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][028e7384][   build:hello.bst                     ] START   Staging sources
[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][028e7384][   build:hello.bst                     ] START   Running build-commands
[--:--:--][028e7384][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr" GREETING="Hey world."

[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][028e7384][   build:hello.bst                     ] START   Running install-commands
[--:--:--][028e7384][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][028e7384][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][028e7384][   build:hello.bst                     ] START   Caching artifact
[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:00][028e7384][   build:hello.bst                     ] SUCCESS directives/hello/028e7384-build.32072.log
[00:00:00][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     1
  Fetch Queue: processed 0, skipped 1, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/directives-shell-excited.html000066400000000000000000000243301514607367700306140ustar00rootroot00000000000000 
user@host:~/directives$ bst --option flavor excited shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Howdy there, and what a world it is !
apache-buildstream-27ae392/doc/source/sessions-stored/directives-shell-normal.html000066400000000000000000000242501514607367700304600ustar00rootroot00000000000000 
user@host:~/directives$ bst shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hello world !
apache-buildstream-27ae392/doc/source/sessions-stored/directives-shell-somber.html000066400000000000000000000242741514607367700304650ustar00rootroot00000000000000 
user@host:~/directives$ bst --option flavor somber shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hey world.
apache-buildstream-27ae392/doc/source/sessions-stored/filtering-list-contents-libhello-filtered.html000066400000000000000000000134141514607367700340770ustar00rootroot00000000000000 
user@host:~/filtering$ bst artifact list-contents libhello-filtered.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
  libhello-filtered.bst:
	usr
	usr/include
	usr/include/libhello.h
	usr/lib
	usr/lib/libhello.so
apache-buildstream-27ae392/doc/source/sessions-stored/filtering-list-contents-libhello.html000066400000000000000000000135001514607367700322770ustar00rootroot00000000000000 
user@host:~/filtering$ bst artifact list-contents libhello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
  libhello.bst:
	usr
	usr/include
	usr/include/libhello.h
	usr/lib
	usr/lib/libhello.so
	usr/share
	usr/share/libhello
	usr/share/libhello/default-person.txt
apache-buildstream-27ae392/doc/source/sessions-stored/filtering-shell-with-filter.html000066400000000000000000000262361514607367700312560ustar00rootroot00000000000000 
user@host:~/filtering$ bst --option use_filter True shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hello stranger
apache-buildstream-27ae392/doc/source/sessions-stored/filtering-shell-without-filter.html000066400000000000000000000262351514607367700320050ustar00rootroot00000000000000 
user@host:~/filtering$ bst --option use_filter False shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hello Sophia
apache-buildstream-27ae392/doc/source/sessions-stored/first-project-build.html000066400000000000000000000506571514607367700276260ustar00rootroot00000000000000 
user@host:~/first-project$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:51:55
  Project:       first-project (/home/user/first-project)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
   buildable 0cb3c97c0c64d82e88c84497408b02ecafd57398059fa8d2769435c2e3720c10 hello.bst 
===============================================================================
[--:--:--][0cb3c97c][   fetch:hello.bst                     ] START   first-project/hello/0cb3c97c-fetch.31328.log
[00:00:00][0cb3c97c][   fetch:hello.bst                     ] SUCCESS first-project/hello/0cb3c97c-fetch.31328.log
[--:--:--][0cb3c97c][   build:hello.bst                     ] START   first-project/hello/0cb3c97c-build.31332.log
[--:--:--][0cb3c97c][   build:hello.bst                     ] START   Staging sources
[00:00:00][0cb3c97c][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][0cb3c97c][   build:hello.bst                     ] START   Caching artifact
[00:00:00][0cb3c97c][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:00][0cb3c97c][   build:hello.bst                     ] SUCCESS first-project/hello/0cb3c97c-build.31332.log
[00:00:00][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       1
  Session:     1
  Fetch Queue: processed 1, skipped 0, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/first-project-checkout.html000066400000000000000000000261001514607367700303160ustar00rootroot00000000000000 
user@host:~/first-project$ bst artifact checkout --directory here hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] START   Checking out files in 'here'
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Checking out files in 'here'
apache-buildstream-27ae392/doc/source/sessions-stored/first-project-init.html000066400000000000000000000005541514607367700274610ustar00rootroot00000000000000 
user@host:~/first-project$ bst init --project-name first-project

Created project.conf at: /home/user/first-project/project.conf
apache-buildstream-27ae392/doc/source/sessions-stored/first-project-ls.html000066400000000000000000000004351514607367700271320ustar00rootroot00000000000000 
user@host:~/first-project$ ls ./here

hello.world
apache-buildstream-27ae392/doc/source/sessions-stored/first-project-show.html000066400000000000000000000134021514607367700274720ustar00rootroot00000000000000 
user@host:~/first-project$ bst show hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
      cached 0cb3c97c0c64d82e88c84497408b02ecafd57398059fa8d2769435c2e3720c10 hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/first-project-touch.html000066400000000000000000000004301514607367700276310ustar00rootroot00000000000000 
user@host:~/first-project$ touch hello.world
apache-buildstream-27ae392/doc/source/sessions-stored/integration-commands-build.html000066400000000000000000001646421514607367700311550ustar00rootroot00000000000000 
user@host:~/integration-commands$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:54:20
  Project:       integration-commands (/home/user/integration-commands)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
   buildable b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
     waiting c51be27603466b582ea90970d9b83bc058224017d83c36f71e33bc5b3b84b638 base.bst 
     waiting dade3a10309e0169eaa5ff94de2699500c488e503c59c3471530ffc1f088f936 libhello.bst 
     waiting 281f8501f66dc3f648eda7602a71492f6ebb18418483bfbcfa713500b67d4e95 hello.bst 
===============================================================================
[--:--:--][b795a72d][   build:base/alpine.bst               ] START   integration-commands/base-alpine/b795a72d-build.5834.log
[--:--:--][b795a72d][   build:base/alpine.bst               ] START   Staging sources
[00:00:00][b795a72d][   build:base/alpine.bst               ] SUCCESS Staging sources
[--:--:--][b795a72d][   build:base/alpine.bst               ] START   Caching artifact
[00:00:00][b795a72d][   build:base/alpine.bst               ] SUCCESS Caching artifact
[00:00:00][b795a72d][   build:base/alpine.bst               ] SUCCESS integration-commands/base-alpine/b795a72d-build.5834.log
[--:--:--][c51be276][   build:base.bst                      ] START   integration-commands/base/c51be276-build.5841.log
[--:--:--][c51be276][   build:base.bst                      ] START   Caching artifact
[00:00:00][c51be276][   build:base.bst                      ] SUCCESS Caching artifact
[00:00:00][c51be276][   build:base.bst                      ] SUCCESS integration-commands/base/c51be276-build.5841.log
[--:--:--][dade3a10][   build:libhello.bst                  ] START   integration-commands/libhello/dade3a10-build.5845.log
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Staging dependencies
[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Staging dependencies
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Integrating sandbox
[--:--:--][dade3a10][   build:libhello.bst                  ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Integrating sandbox
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Staging sources
[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Staging sources
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Running build-commands
[--:--:--][dade3a10][   build:libhello.bst                  ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Running build-commands
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Running install-commands
[--:--:--][dade3a10][   build:libhello.bst                  ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Running install-commands
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Running strip-commands
[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Running strip-commands
[--:--:--][dade3a10][   build:libhello.bst                  ] START   Caching artifact
[00:00:00][dade3a10][   build:libhello.bst                  ] SUCCESS Caching artifact
[00:00:01][dade3a10][   build:libhello.bst                  ] SUCCESS integration-commands/libhello/dade3a10-build.5845.log
[--:--:--][281f8501][   build:hello.bst                     ] START   integration-commands/hello/281f8501-build.5881.log
[--:--:--][281f8501][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][281f8501][   build:hello.bst                     ] START   Integrating sandbox
[--:--:--][281f8501][   build:hello.bst                     ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][281f8501][   build:hello.bst                     ] START   Staging sources
[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][281f8501][   build:hello.bst                     ] START   Running build-commands
[--:--:--][281f8501][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][281f8501][   build:hello.bst                     ] START   Running install-commands
[--:--:--][281f8501][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][281f8501][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][281f8501][   build:hello.bst                     ] START   Caching artifact
[00:00:00][281f8501][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:01][281f8501][   build:hello.bst                     ] SUCCESS integration-commands/hello/281f8501-build.5881.log
[00:00:02][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       4
  Session:     4
  Fetch Queue: processed 0, skipped 4, failed 0 
  Build Queue: processed 4, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/integration-commands-shell.html000066400000000000000000000262261514607367700311600ustar00rootroot00000000000000 
user@host:~/integration-commands$ bst shell hello.bst -- hello pony

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello pony

Hello pony
apache-buildstream-27ae392/doc/source/sessions-stored/junction-includes-build-funky.html000066400000000000000000001100151514607367700316030ustar00rootroot00000000000000 
user@host:~/junction-includes$ bst --option funky True build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:53:45
  Project:       subproject (/home/user/junction-includes)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Project Options
  funky: 1

Pipeline
      cached 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 subproject-junction.bst:base/alpine.bst 
      cached a41060b8e6ebd32de32417cd4576f3f21e168787c82302df104a8c463a7d1cf0 subproject-junction.bst:base.bst 
   buildable 7250d6eee607e51c1e63ee48a4c1dd28c2a775cc5c92678e85fadc9dca298147 hello.bst 
===============================================================================
[--:--:--][7250d6ee][   build:hello.bst                     ] START   subproject/hello/7250d6ee-build.3495.log
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Staging sources
[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Running configure-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] STATUS  Running command

    export NOCONFIGURE=1;
    
    if [ -x ./configure ]; then true;
    elif [ -x ./autogen ]; then ./autogen;
    elif [ -x ./autogen.sh ]; then ./autogen.sh;
    elif [ -x ./bootstrap ]; then ./bootstrap;
    elif [ -x ./bootstrap.sh ]; then ./bootstrap.sh;
    else autoreconf -ivf .;
    fi

[--:--:--][7250d6ee][   build:hello.bst                     ] STATUS  Running command

    ./configure --prefix=/opt \
    --exec-prefix=/opt \
    --bindir=/opt/bin \
    --sbindir=/opt/sbin \
    --sysconfdir=/etc \
    --datadir=/opt/share \
    --includedir=/opt/include \
    --libdir=/opt/lib \
    --libexecdir=/opt/libexec \
    --localstatedir=/var \
    --sharedstatedir=/opt/com \
    --mandir=/opt/share/man \
    --infodir=/opt/share/info   

[00:00:03][7250d6ee][   build:hello.bst                     ] SUCCESS Running configure-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Running build-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] STATUS  Running command

    make

[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Running install-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] STATUS  Running command

    make -j1 DESTDIR="/buildstream-install" install

[--:--:--][7250d6ee][   build:hello.bst                     ] STATUS  Running command

    if false || false; then
      find "/buildstream-install" -name "*.la" -print0 | while read -d '' -r file; do
        if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        else
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        fi
      done
    fi

[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][7250d6ee][   build:hello.bst                     ] START   Caching artifact
[00:00:00][7250d6ee][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:04][7250d6ee][   build:hello.bst                     ] SUCCESS subproject/hello/7250d6ee-build.3495.log
[00:00:04][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     1
  Fetch Queue: processed 0, skipped 1, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/junction-includes-build-normal.html000066400000000000000000001330151514607367700317440ustar00rootroot00000000000000 
user@host:~/junction-includes$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:53:40
  Project:       subproject (/home/user/junction-includes)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Project Options
  funky: 0

Pipeline
   buildable 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 subproject-junction.bst:base/alpine.bst 
     waiting a41060b8e6ebd32de32417cd4576f3f21e168787c82302df104a8c463a7d1cf0 subproject-junction.bst:base.bst 
     waiting 71d3352ef5aad1fef8d62c07c858ff7984cfe871bf9bdde6df16408e62f2f6a6 hello.bst 
===============================================================================
[--:--:--][179c6ae9][   build:subproject-junction.bst:base/alpine.bst] START   subproject/base-alpine/179c6ae9-build.1902.log
[--:--:--][179c6ae9][   build:subproject-junction.bst:base/alpine.bst] START   Staging sources
[00:00:00][179c6ae9][   build:subproject-junction.bst:base/alpine.bst] SUCCESS Staging sources
[--:--:--][179c6ae9][   build:subproject-junction.bst:base/alpine.bst] START   Caching artifact
[00:00:00][179c6ae9][   build:subproject-junction.bst:base/alpine.bst] SUCCESS Caching artifact
[00:00:00][179c6ae9][   build:subproject-junction.bst:base/alpine.bst] SUCCESS subproject/base-alpine/179c6ae9-build.1902.log
[--:--:--][a41060b8][   build:subproject-junction.bst:base.bst] START   subproject/base/a41060b8-build.1920.log
[--:--:--][a41060b8][   build:subproject-junction.bst:base.bst] START   Caching artifact
[00:00:00][a41060b8][   build:subproject-junction.bst:base.bst] SUCCESS Caching artifact
[00:00:00][a41060b8][   build:subproject-junction.bst:base.bst] SUCCESS subproject/base/a41060b8-build.1920.log
[--:--:--][71d3352e][   build:hello.bst                     ] START   subproject/hello/71d3352e-build.1940.log
[--:--:--][71d3352e][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][71d3352e][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][71d3352e][   build:hello.bst                     ] START   Staging sources
[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][71d3352e][   build:hello.bst                     ] START   Running configure-commands
[--:--:--][71d3352e][   build:hello.bst                     ] STATUS  Running command

    export NOCONFIGURE=1;
    
    if [ -x ./configure ]; then true;
    elif [ -x ./autogen ]; then ./autogen;
    elif [ -x ./autogen.sh ]; then ./autogen.sh;
    elif [ -x ./bootstrap ]; then ./bootstrap;
    elif [ -x ./bootstrap.sh ]; then ./bootstrap.sh;
    else autoreconf -ivf .;
    fi

[--:--:--][71d3352e][   build:hello.bst                     ] STATUS  Running command

    ./configure --prefix=/usr \
    --exec-prefix=/usr \
    --bindir=/usr/bin \
    --sbindir=/usr/sbin \
    --sysconfdir=/etc \
    --datadir=/usr/share \
    --includedir=/usr/include \
    --libdir=/usr/lib \
    --libexecdir=/usr/libexec \
    --localstatedir=/var \
    --sharedstatedir=/usr/com \
    --mandir=/usr/share/man \
    --infodir=/usr/share/info   

[00:00:03][71d3352e][   build:hello.bst                     ] SUCCESS Running configure-commands
[--:--:--][71d3352e][   build:hello.bst                     ] START   Running build-commands
[--:--:--][71d3352e][   build:hello.bst                     ] STATUS  Running command

    make

[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][71d3352e][   build:hello.bst                     ] START   Running install-commands
[--:--:--][71d3352e][   build:hello.bst                     ] STATUS  Running command

    make -j1 DESTDIR="/buildstream-install" install

[--:--:--][71d3352e][   build:hello.bst                     ] STATUS  Running command

    if false || false; then
      find "/buildstream-install" -name "*.la" -print0 | while read -d '' -r file; do
        if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        else
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        fi
      done
    fi

[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][71d3352e][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][71d3352e][   build:hello.bst                     ] START   Caching artifact
[00:00:00][71d3352e][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:04][71d3352e][   build:hello.bst                     ] SUCCESS subproject/hello/71d3352e-build.1940.log
[00:00:05][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     3
  Fetch Queue: processed 0, skipped 3, failed 0 
  Build Queue: processed 3, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/junction-includes-shell-funky.html000066400000000000000000000243511514607367700316220ustar00rootroot00000000000000 
user@host:~/junction-includes$ bst --option funky True shell hello.bst -- /opt/bin/hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    /opt/bin/hello

Hello World!
This is amhello 1.0.
apache-buildstream-27ae392/doc/source/sessions-stored/junction-includes-shell-normal.html000066400000000000000000000243251514607367700317570ustar00rootroot00000000000000 
user@host:~/junction-includes$ bst shell hello.bst -- /usr/bin/hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    /usr/bin/hello

Hello World!
This is amhello 1.0.
apache-buildstream-27ae392/doc/source/sessions-stored/junctions-build.html000066400000000000000000001765611514607367700270520ustar00rootroot00000000000000 
user@host:~/junctions$ bst build callHello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:55
  Project:       junctions (/home/user/junctions)
  Targets:       

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline

===============================================================================
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:55
  Project:       junctions (/home/user/junctions)
  Targets:       callHello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
fetch needed 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 hello-junction.bst:base/alpine.bst 
     waiting 7e7bdf36e3d6947ebf279f651fc492c0d20f919e114fb3af8196ac6d2863a8b7 hello-junction.bst:base.bst 
fetch needed 26721a3b05918bec80836da7b713f8f0227410c6a7fecc82a158e0bebe7baddf hello-junction.bst:hello.bst 
     waiting 18d029559c67773c834fe614312382190e080c08c9c6ce5b786552424ef9e69e callHello.bst 
===============================================================================
[--:--:--][179c6ae9][   fetch:hello-junction.bst:base/alpine.bst] START   autotools/base-alpine/179c6ae9-fetch.32733.log
[--:--:--][26721a3b][   fetch:hello-junction.bst:hello.bst  ] START   autotools/hello/26721a3b-fetch.32734.log
[--:--:--][18d02955][   fetch:callHello.bst                 ] START   junctions/callHello/18d02955-fetch.32735.log
[--:--:--][179c6ae9][   fetch:hello-junction.bst:base/alpine.bst] START   Fetching https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/integration-tests-base.v1.x86_64.tar.xz
[--:--:--][26721a3b][   fetch:hello-junction.bst:hello.bst  ] START   Fetching https://ftpmirror.gnu.org/gnu/automake/automake-1.16.tar.gz
[00:00:00][18d02955][   fetch:callHello.bst                 ] SUCCESS junctions/callHello/18d02955-fetch.32735.log
[00:00:02][179c6ae9][   fetch:hello-junction.bst:base/alpine.bst] SUCCESS Fetching https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/integration-tests-base.v1.x86_64.tar.xz
[00:00:03][26721a3b][   fetch:hello-junction.bst:hello.bst  ] SUCCESS Fetching https://ftpmirror.gnu.org/gnu/automake/automake-1.16.tar.gz
[00:00:03][26721a3b][   fetch:hello-junction.bst:hello.bst  ] SUCCESS autotools/hello/26721a3b-fetch.32734.log
[00:00:10][179c6ae9][   fetch:hello-junction.bst:base/alpine.bst] SUCCESS autotools/base-alpine/179c6ae9-fetch.32733.log
[--:--:--][179c6ae9][   build:hello-junction.bst:base/alpine.bst] START   autotools/base-alpine/179c6ae9-build.32750.log
[--:--:--][179c6ae9][   build:hello-junction.bst:base/alpine.bst] START   Staging sources
[00:00:00][179c6ae9][   build:hello-junction.bst:base/alpine.bst] SUCCESS Staging sources
[--:--:--][179c6ae9][   build:hello-junction.bst:base/alpine.bst] START   Caching artifact
[00:00:00][179c6ae9][   build:hello-junction.bst:base/alpine.bst] SUCCESS Caching artifact
[00:00:00][179c6ae9][   build:hello-junction.bst:base/alpine.bst] SUCCESS autotools/base-alpine/179c6ae9-build.32750.log
[--:--:--][7e7bdf36][   build:hello-junction.bst:base.bst   ] START   autotools/base/7e7bdf36-build.32755.log
[--:--:--][7e7bdf36][   build:hello-junction.bst:base.bst   ] START   Caching artifact
[00:00:00][7e7bdf36][   build:hello-junction.bst:base.bst   ] SUCCESS Caching artifact
[00:00:00][7e7bdf36][   build:hello-junction.bst:base.bst   ] SUCCESS autotools/base/7e7bdf36-build.32755.log
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   autotools/hello/26721a3b-build.32759.log
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Staging dependencies
[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Staging dependencies
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Integrating sandbox
[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Integrating sandbox
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Staging sources
[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Staging sources
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Running configure-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] STATUS  Running command

    export NOCONFIGURE=1;
    
    if [ -x ./configure ]; then true;
    elif [ -x ./autogen ]; then ./autogen;
    elif [ -x ./autogen.sh ]; then ./autogen.sh;
    elif [ -x ./bootstrap ]; then ./bootstrap;
    elif [ -x ./bootstrap.sh ]; then ./bootstrap.sh;
    else autoreconf -ivf .;
    fi

[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] STATUS  Running command

    ./configure --prefix=/usr \
    --exec-prefix=/usr \
    --bindir=/usr/bin \
    --sbindir=/usr/sbin \
    --sysconfdir=/etc \
    --datadir=/usr/share \
    --includedir=/usr/include \
    --libdir=/usr/lib \
    --libexecdir=/usr/libexec \
    --localstatedir=/var \
    --sharedstatedir=/usr/com \
    --mandir=/usr/share/man \
    --infodir=/usr/share/info   

[00:00:03][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Running configure-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Running build-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] STATUS  Running command

    make

[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Running build-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Running install-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] STATUS  Running command

    make -j1 DESTDIR="/buildstream-install" install

[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] STATUS  Running command

    if false || false; then
      find "/buildstream-install" -name "*.la" -print0 | while read -d '' -r file; do
        if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        else
          if false; then
            echo "Removing ${file}."
            rm "${file}"
          else
            echo "Not removing ${file}."
          fi
        fi
      done
    fi

[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Running install-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Running strip-commands
[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Running strip-commands
[--:--:--][26721a3b][   build:hello-junction.bst:hello.bst  ] START   Caching artifact
[00:00:00][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS Caching artifact
[00:00:04][26721a3b][   build:hello-junction.bst:hello.bst  ] SUCCESS autotools/hello/26721a3b-build.32759.log
[--:--:--][18d02955][   build:callHello.bst                 ] START   junctions/callHello/18d02955-build.1391.log
[--:--:--][18d02955][   build:callHello.bst                 ] START   Staging sources
[00:00:00][18d02955][   build:callHello.bst                 ] SUCCESS Staging sources
[--:--:--][18d02955][   build:callHello.bst                 ] START   Caching artifact
[00:00:00][18d02955][   build:callHello.bst                 ] SUCCESS Caching artifact
[00:00:00][18d02955][   build:callHello.bst                 ] SUCCESS junctions/callHello/18d02955-build.1391.log
[00:00:15][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       4
  Session:     4
  Fetch Queue: processed 3, skipped 1, failed 0 
  Build Queue: processed 4, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/junctions-shell.html000066400000000000000000000243541514607367700270520ustar00rootroot00000000000000 
user@host:~/junctions$ bst shell callHello.bst -- /bin/sh callHello.sh

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:callHello.bst                 ] START   Staging dependencies
[00:00:00][        ][    main:callHello.bst                 ] SUCCESS Staging dependencies
[--:--:--][        ][    main:callHello.bst                 ] START   Integrating sandbox
[00:00:00][        ][    main:callHello.bst                 ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:callHello.bst                 ] STATUS  Running command

    /bin/sh callHello.sh

Calling hello:
Hello World!
This is amhello 1.0.
apache-buildstream-27ae392/doc/source/sessions-stored/junctions-workspace-open-subproject.html000066400000000000000000000316321514607367700330530ustar00rootroot00000000000000 
user@host:~/junctions$ bst workspace open --directory workspace_subproject hello-junction.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources
[--:--:--][        ][    main:core activity                 ] INFO    Creating workspace for element hello-junction.bst
[--:--:--][        ][    main:hello-junction.bst            ] START   Staging sources to /home/user/junctions/workspace_subproject
[--:--:--][        ][    main:hello-junction.bst            ] START   Staging local files into CAS
[00:00:00][        ][    main:hello-junction.bst            ] SUCCESS Staging local files into CAS
[00:00:00][        ][    main:hello-junction.bst            ] SUCCESS Staging sources to /home/user/junctions/workspace_subproject
[--:--:--][        ][    main:core activity                 ] INFO    Created a workspace for element: hello-junction.bst
apache-buildstream-27ae392/doc/source/sessions-stored/junctions-workspace-open.html000066400000000000000000000260701514607367700306750ustar00rootroot00000000000000 
user@host:~/junctions$ bst workspace open --directory workspace_hello hello-junction.bst:hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources
[--:--:--][        ][    main:core activity                 ] INFO    Creating workspace for element hello.bst
[--:--:--][        ][    main:hello-junction.bst:hello.bst  ] START   Staging sources to /home/user/junctions/workspace_hello
[00:00:00][        ][    main:hello-junction.bst:hello.bst  ] SUCCESS Staging sources to /home/user/junctions/workspace_hello
[--:--:--][        ][    main:core activity                 ] INFO    Created a workspace for element: hello-junction.bst:hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/overlaps-build.html000066400000000000000000000715141514607367700266610ustar00rootroot00000000000000 
user@host:~/overlaps$ bst build runtime-only.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:53:26
  Project:       overlaps (/home/user/overlaps)
  Targets:       runtime-only.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
      cached afe1fb0c366992c006f58c49017f0b48031f6f397a1e2f0e73756d619de61ffc base/alpine.bst 
      cached 14a4e96021cd12177192902ea8a2ac5354aa358e2bf9e4d80c1e25639028ca0a base.bst 
      cached e2b5f713391cfd8a76593cdd689e3b1004ba649f50635ba615f0c865fe4c953a libhello.bst 
      cached 26037d070846c5adb7ecba45fb5b01b4d97e0535fdd03fba81a3cc86ff30e0e4 hello.bst 
   buildable af1fc1febe3cbed1ac5a71e48cd048dd6dc919f0a8e4658038480cd526dc001b runtime-only.bst 
===============================================================================
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] START   overlaps/runtime-only/af1fc1fe-build.1682.log
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] START   Staging dependencies
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] WARNING [overlaps]: Non-whitelisted overlaps detected

    Staged files overwrite existing files in staging area:
    /usr/share/doc/hello.txt: hello.bst is not permitted to overlap other elements, order hello.bst above libhello.bst 

[00:00:00][af1fc1fe][   build:runtime-only.bst              ] SUCCESS Staging dependencies
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] START   Computing split
[00:00:00][af1fc1fe][   build:runtime-only.bst              ] SUCCESS Computing split
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] START   Integrating sandbox
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] STATUS  Running command

    ldconfig "/usr/lib"

[--:--:--][af1fc1fe][   build:runtime-only.bst              ] INFO    Integration modified 367, added 0 and removed 0 files
[00:00:00][af1fc1fe][   build:runtime-only.bst              ] SUCCESS Integrating sandbox
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] START   Creating composition

    Including files from domains: runtime
    Excluding orphaned files

[--:--:--][af1fc1fe][   build:runtime-only.bst              ] INFO    Composing 287 files
[00:00:00][af1fc1fe][   build:runtime-only.bst              ] SUCCESS Creating composition
[--:--:--][af1fc1fe][   build:runtime-only.bst              ] START   Caching artifact
[00:00:00][af1fc1fe][   build:runtime-only.bst              ] SUCCESS Caching artifact
[00:00:01][af1fc1fe][   build:runtime-only.bst              ] SUCCESS overlaps/runtime-only/af1fc1fe-build.1682.log
[00:00:01][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       5
  Session:     1
  Fetch Queue: processed 0, skipped 1, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/running-commands-build.html000066400000000000000000001167261514607367700303120ustar00rootroot00000000000000 
user@host:~/running-commands$ bst build hello.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:51:20
  Project:       running-commands (/home/user/running-commands)
  Targets:       hello.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       Yes
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
   buildable 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
     waiting e2d966e322d011d1eeafd8e77fc7257e72cce29a813b67c66b087358e785873e base.bst 
     waiting 0d8e771194a9b434f43d1cd44355b2e992136fd0941ca5366405ad8f9d4e3e05 hello.bst 
===============================================================================
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   running-commands/base-alpine/179c6ae9-build.30666.log
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Staging sources
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Staging sources
[--:--:--][179c6ae9][   build:base/alpine.bst               ] START   Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS Caching artifact
[00:00:00][179c6ae9][   build:base/alpine.bst               ] SUCCESS running-commands/base-alpine/179c6ae9-build.30666.log
[--:--:--][e2d966e3][   build:base.bst                      ] START   running-commands/base/e2d966e3-build.30673.log
[--:--:--][e2d966e3][   build:base.bst                      ] START   Caching artifact
[00:00:00][e2d966e3][   build:base.bst                      ] SUCCESS Caching artifact
[00:00:00][e2d966e3][   build:base.bst                      ] SUCCESS running-commands/base/e2d966e3-build.30673.log
[--:--:--][0d8e7711][   build:hello.bst                     ] START   running-commands/hello/0d8e7711-build.30677.log
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Staging dependencies
[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Integrating sandbox
[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Staging sources
[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Staging sources
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Running build-commands
[--:--:--][0d8e7711][   build:hello.bst                     ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Running build-commands
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Running install-commands
[--:--:--][0d8e7711][   build:hello.bst                     ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Running install-commands
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Running strip-commands
[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Running strip-commands
[--:--:--][0d8e7711][   build:hello.bst                     ] START   Caching artifact
[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS Caching artifact
[00:00:00][0d8e7711][   build:hello.bst                     ] SUCCESS running-commands/hello/0d8e7711-build.30677.log
[00:00:01][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       3
  Session:     3
  Fetch Queue: processed 0, skipped 3, failed 0 
  Build Queue: processed 3, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/running-commands-shell.html000066400000000000000000000242541514607367700303140ustar00rootroot00000000000000 
user@host:~/running-commands$ bst shell hello.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello.bst                     ] START   Staging dependencies
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello.bst                     ] START   Integrating sandbox
[00:00:00][        ][    main:hello.bst                     ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello.bst                     ] STATUS  Running command

    hello

Hello World
apache-buildstream-27ae392/doc/source/sessions-stored/running-commands-show-after.html000066400000000000000000000142721514607367700312630ustar00rootroot00000000000000 
user@host:~/running-commands$ bst show hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
      cached 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
      cached e2d966e322d011d1eeafd8e77fc7257e72cce29a813b67c66b087358e785873e base.bst 
      cached 0d8e771194a9b434f43d1cd44355b2e992136fd0941ca5366405ad8f9d4e3e05 hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/running-commands-show-before.html000066400000000000000000000142721514607367700314240ustar00rootroot00000000000000 
user@host:~/running-commands$ bst show hello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
   buildable 179c6ae937e8e2ece3192ab8dc2a55053134a591c9ccd37507b56d11885fae23 base/alpine.bst 
     waiting e2d966e322d011d1eeafd8e77fc7257e72cce29a813b67c66b087358e785873e base.bst 
     waiting 0d8e771194a9b434f43d1cd44355b2e992136fd0941ca5366405ad8f9d4e3e05 hello.bst
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-build-dynamic-no-strict.html000066400000000000000000001037371514607367700326250ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst --no-strict build hello-dynamic.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:12
  Project:       strict-mode (/home/user/strict-mode)
  Targets:       hello-dynamic.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       No
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
      cached b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
      cached e4a4926893ce660306ca9c812fd8018fa41571c27f37fff4a9e6ab592d0333fe base.bst 
   buildable 3ea19fa473f6d9f4a7e67cb3bcafd0306486abdae7e126cd49c13cfb281df882 libhello.bst Workspace: /home/user/strict-mode/workspace_libhello
      cached 200162f545a171c13e9fa2eb0bf562742d6b8cecaeadbf6a0914748dc98c4dde hello-dynamic.bst 
===============================================================================
[--:--:--][3ea19fa4][   fetch:libhello.bst                  ] START   strict-mode/libhello/3ea19fa4-fetch.31676.log
[00:00:00][3ea19fa4][   fetch:libhello.bst                  ] SUCCESS strict-mode/libhello/3ea19fa4-fetch.31676.log
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   strict-mode/libhello/3ea19fa4-build.31680.log
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Staging dependencies
[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Staging dependencies
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Integrating sandbox
[--:--:--][3ea19fa4][   build:libhello.bst                  ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Integrating sandbox
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Staging sources
[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Staging sources
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Running build-commands
[--:--:--][3ea19fa4][   build:libhello.bst                  ] STATUS  Running command

    make PREFIX="/usr"

[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Running build-commands
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Running install-commands
[--:--:--][3ea19fa4][   build:libhello.bst                  ] STATUS  Running command

    make -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Running install-commands
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Running strip-commands
[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Running strip-commands
[--:--:--][3ea19fa4][   build:libhello.bst                  ] START   Caching artifact
[00:00:00][3ea19fa4][   build:libhello.bst                  ] SUCCESS Caching artifact
[00:00:01][3ea19fa4][   build:libhello.bst                  ] SUCCESS strict-mode/libhello/3ea19fa4-build.31680.log
[00:00:01][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       4
  Session:     1
  Fetch Queue: processed 1, skipped 0, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-build-static-no-strict.html000066400000000000000000000777001514607367700324700ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst --no-strict build hello-static.bst

[--:--:--][        ][    main:core activity                 ] START   Build
[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources

BuildStream Version 1.93.1+172.g03ccfcd3b
  Session Start: Friday, 17-04-2020 at 20:52:15
  Project:       strict-mode (/home/user/strict-mode)
  Targets:       hello-static.bst

User Configuration
  Configuration File:      /home/user/.config/buildstream.conf
  Cache Directory:         /home/user/.cache/buildstream
  Log Files:               /home/user/.cache/buildstream/logs
  Source Mirrors:          /home/user/.cache/buildstream/sources
  Build Area:              /home/user/.cache/buildstream/build
  Strict Build Plan:       No
  Maximum Fetch Tasks:     10
  Maximum Build Tasks:     4
  Maximum Push Tasks:      4
  Maximum Network Retries: 2

Pipeline
      cached b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
      cached e4a4926893ce660306ca9c812fd8018fa41571c27f37fff4a9e6ab592d0333fe base.bst 
      cached 3ea19fa473f6d9f4a7e67cb3bcafd0306486abdae7e126cd49c13cfb281df882 libhello.bst Workspace: /home/user/strict-mode/workspace_libhello
   buildable 1fe34abbad09b930e86d213b3c21507bf250baecd95effb24fd2223fc6017968 hello-static.bst 
===============================================================================
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   strict-mode/hello-static/1fe34abb-build.31821.log
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Staging dependencies
[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Staging dependencies
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Integrating sandbox
[--:--:--][1fe34abb][   build:hello-static.bst              ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Integrating sandbox
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Staging sources
[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Staging sources
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Running build-commands
[--:--:--][1fe34abb][   build:hello-static.bst              ] STATUS  Running command

    make -f Makefile.static PREFIX="/usr"

[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Running build-commands
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Running install-commands
[--:--:--][1fe34abb][   build:hello-static.bst              ] STATUS  Running command

    make -f Makefile.static -j1 PREFIX="/usr" DESTDIR="/buildstream-install" install

[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Running install-commands
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Running strip-commands
[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Running strip-commands
[--:--:--][1fe34abb][   build:hello-static.bst              ] START   Caching artifact
[00:00:00][1fe34abb][   build:hello-static.bst              ] SUCCESS Caching artifact
[00:00:01][1fe34abb][   build:hello-static.bst              ] SUCCESS strict-mode/hello-static/1fe34abb-build.31821.log
[00:00:01][        ][    main:core activity                 ] SUCCESS Build

Pipeline Summary
  Total:       4
  Session:     1
  Fetch Queue: processed 0, skipped 1, failed 0 
  Build Queue: processed 1, skipped 0, failed 0
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-run-dynamic-no-strict.html000066400000000000000000000262421514607367700323250ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst --no-strict shell hello-dynamic.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello-dynamic.bst             ] START   Staging dependencies
[00:00:00][        ][    main:hello-dynamic.bst             ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello-dynamic.bst             ] START   Integrating sandbox
[--:--:--][        ][    main:hello-dynamic.bst             ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][        ][    main:hello-dynamic.bst             ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello-dynamic.bst             ] STATUS  Running command

    hello

Good morning stranger
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-run-static-no-strict.html000066400000000000000000000262411514607367700321670ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst --no-strict shell hello-static.bst -- hello

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:hello-static.bst              ] START   Staging dependencies
[00:00:00][        ][    main:hello-static.bst              ] SUCCESS Staging dependencies
[--:--:--][        ][    main:hello-static.bst              ] START   Integrating sandbox
[--:--:--][        ][    main:hello-static.bst              ] STATUS  Running command

    ldconfig "/usr/lib"

[00:00:00][        ][    main:hello-static.bst              ] SUCCESS Integrating sandbox
[--:--:--][        ][    main:hello-static.bst              ] STATUS  Running command

    hello

Good morning stranger
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-show-dynamic-no-strict.html000066400000000000000000000150421514607367700324750ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst --no-strict show hello-dynamic.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
      cached b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
      cached e4a4926893ce660306ca9c812fd8018fa41571c27f37fff4a9e6ab592d0333fe base.bst 
   buildable 3ea19fa473f6d9f4a7e67cb3bcafd0306486abdae7e126cd49c13cfb281df882 libhello.bst Workspace: /home/user/strict-mode/workspace_libhello
      cached 200162f545a171c13e9fa2eb0bf562742d6b8cecaeadbf6a0914748dc98c4dde hello-dynamic.bst
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-show-dynamic-strict.html000066400000000000000000000147241514607367700320710ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst show hello-dynamic.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
      cached b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
      cached e4a4926893ce660306ca9c812fd8018fa41571c27f37fff4a9e6ab592d0333fe base.bst 
   buildable bbf7a5de337c55b6f4d32cbe3d47ed7f39ff9649bb5ff52edd44bd7c0e36e38e libhello.bst Workspace: /home/user/strict-mode/workspace_libhello
     waiting fd41ac4b5ebf2fccae28e727602f08d7ee2997fd36b96b597da183f86e2caf32 hello-dynamic.bst
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-show-initial.html000066400000000000000000000152201514607367700305600ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst show hello-static.bst hello-dynamic.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
      cached b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
      cached e4a4926893ce660306ca9c812fd8018fa41571c27f37fff4a9e6ab592d0333fe base.bst 
      cached 82aeedb247804d906e44722cadc8e36e6a12f84518f78e6ed44af56c11aec5fc libhello.bst 
      cached 7a164ef5f2a2e36ba63d454c1ab9e18b837d327c46f091305814e46ab6c71f2f hello-static.bst 
      cached 200162f545a171c13e9fa2eb0bf562742d6b8cecaeadbf6a0914748dc98c4dde hello-dynamic.bst
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-show-static-no-strict.html000066400000000000000000000150401514607367700323360ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst --no-strict show hello-static.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
      cached b795a72d92c05ed0810723f5c3ad40c53bba6e8b568a9d5da22a450a11e03659 base/alpine.bst 
      cached e4a4926893ce660306ca9c812fd8018fa41571c27f37fff4a9e6ab592d0333fe base.bst 
   buildable 3ea19fa473f6d9f4a7e67cb3bcafd0306486abdae7e126cd49c13cfb281df882 libhello.bst Workspace: /home/user/strict-mode/workspace_libhello
     waiting 1fe34abbad09b930e86d213b3c21507bf250baecd95effb24fd2223fc6017968 hello-static.bst
apache-buildstream-27ae392/doc/source/sessions-stored/strict-mode-workspace-open.html000066400000000000000000000316101514607367700311070ustar00rootroot00000000000000 
user@host:~/strict-mode$ bst workspace open --directory workspace_libhello libhello.bst

[--:--:--][        ][    main:core activity                 ] START   Loading elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Loading elements
[--:--:--][        ][    main:core activity                 ] START   Resolving elements
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving elements
[--:--:--][        ][    main:core activity                 ] START   Resolving cached state
[00:00:00][        ][    main:core activity                 ] SUCCESS Resolving cached state
[--:--:--][        ][    main:core activity                 ] START   Checking sources
[00:00:00][        ][    main:core activity                 ] SUCCESS Checking sources
[--:--:--][        ][    main:core activity                 ] INFO    Creating workspace for element libhello.bst
[--:--:--][        ][    main:libhello.bst                  ] START   Staging sources to /home/user/strict-mode/workspace_libhello
[--:--:--][        ][    main:libhello.bst                  ] START   Staging local files into CAS
[00:00:00][        ][    main:libhello.bst                  ] SUCCESS Staging local files into CAS
[00:00:00][        ][    main:libhello.bst                  ] SUCCESS Staging sources to /home/user/strict-mode/workspace_libhello
[--:--:--][        ][    main:core activity                 ] INFO    Created a workspace for element: libhello.bst
apache-buildstream-27ae392/doc/source/tutorial/000077500000000000000000000000001514607367700215325ustar00rootroot00000000000000apache-buildstream-27ae392/doc/source/tutorial/autotools.rst000066400000000000000000000133321514607367700243170ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _tutorial_autotools: Using the autotools element =========================== In :ref:`the last chapter ` we observed how the :mod:`manual ` element works, allowing one to specify and run commands manually in the process of constructing an *artifact*. In this chapter, we'll go over a mostly automated build of a similar hello world example. We will observe how our configurations of the :mod:`autotools ` element translate to configurations on the :mod:`manual ` element, and observe how :ref:`variable substitution ` works. .. note:: This example is distributed with BuildStream in the `doc/examples/autotools `_ subdirectory. Overview -------- Instead of using the :mod:`local ` source as we have been using in the previous examples, we're going to use a :mod:`tar ` source this time to obtain the ``automake`` release tarball directly from the upstream hosting. In this example we're going to build the example program included in the upstream ``automake`` tarball itself, and we're going to use the automated :mod:`autotools ` build element to do so. Project structure ----------------- ``project.conf`` ~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/autotools/project.conf :language: yaml Like the :ref:`last project.conf `, we've added another :ref:`source alias ` for ``gnu``, the location from which we're going to download the ``automake`` tarball. ``elements/base/alpine.bst`` and ``elements/base.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The alpine base and base stack element are defined in the same way as in the last chapter: :ref:`tutorial_running_commands`. ``elements/hello.bst`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/autotools/elements/hello.bst :language: yaml In this case, we haven't touched the element's ``config`` section at all, instead we just slightly override the bahavior of the :mod:`autotools ` build element by overriding the :ref:`command-subdir variable ` Looking at variables '''''''''''''''''''' Let's take a moment and observe how :ref:`element composition ` works with variables. As :ref:`the documentation ` mentions: * The initial settings of the ``project.conf`` variables are setup using BuildStream's :ref:`builtin defaults `. * After this, your local ``project.conf`` may override some variables on a project wide basis. Those will in turn be overridden by any defaults provided by element classes, such as the variables set in the documentation of the :mod:`autotools ` build element. The variables you set in your final ```` *element declarations*, will have the final say on the value of a particular variable. * Finally, the variables, which may be composed of other variables, are resolved after all composition has taken place. The variable we needed to override was ``command-subdir``, which is an automatic variable provided by the :mod:`BuildElement ` abstract class. This variable simply instructs the :mod:`BuildElement ` in which subdirectory of the ``%{build-root}`` to run its commands in. One can always display the resolved set of variables for a given element's configuration using :ref:`bst show `: .. raw:: html :file: ../sessions/autotools-show-variables.html As an exercise, we suggest that you modify the ``hello.bst`` element to set the prefix like so: .. code:: yaml variables: prefix: "/opt" And rerun the above :ref:`bst show ` command to observe how this changes the output. Observe where the variables are declared in the :ref:`builtin defaults ` and :mod:`autotools ` element documentation, and how overriding these effects the resolved set of variables. Using the project ----------------- Build the hello.bst element ~~~~~~~~~~~~~~~~~~~~~~~~~~~ To build the project, run :ref:`bst build ` in the following way: .. raw:: html :file: ../sessions/autotools-build.html Run the hello world program ~~~~~~~~~~~~~~~~~~~~~~~~~~~ We probably know by now what's going to happen, but let's run the program we've compiled anyway using :ref:`bst shell `: .. raw:: html :file: ../sessions/autotools-shell.html Summary ------- Now we've used an external :ref:`build element `, from the ``buildstream-plugins`` package and we've taken a look into :ref:`how variables work `. When browsing the :ref:`build elements ` in their respective documentation, we are now equipped with a good idea of what an element is going to do, based on their default YAML configuration and any configurations we have in our project. We can also now observe what variables are in effect for the build of a given element, using :ref:`bst show `. apache-buildstream-27ae392/doc/source/tutorial/directives.rst000066400000000000000000000147471514607367700244420ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _tutorial_directives: Optionality and directives ========================== In this chapter we're going to go over some of the more flexible constructs which BuildStream offers for :ref:`optionality `, and show how we can use :ref:`directives ` in the BuildStream YAML format. .. note:: This example is distributed with BuildStream in the `doc/examples/directives `_ subdirectory. Overview -------- This chapter's example will build another ``hello.c`` program which much resembles the program in the :ref:`running commands ` example, but here we're going to make the greeting string *configurable* using the C preprocessor. We'll be compiling the following C file: ``files/src/hello.c`` ~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/directives/files/src/hello.c :language: c And we're going to build it using ``make``, using the following Makefile: ``files/src/Makefile`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/directives/files/src/Makefile :language: Makefile Notice the addition of ``-DGREETING_MESSAGE="\"${GREETING}\""`` in the above Makefile, this will allow us to configure the greeting message from the ``hello.bst`` element declaration. We will need to add support to our project for *optionality*, and we'll have to make *conditional statements* to resolve what kind of greeting we want from the hello world program. Project structure ----------------- Since this project has much the same structure as the :ref:`running commands ` chapter did, we won't go over all of these elements in detail. Instead let's focus on the addition of the new :ref:`project options ` in ``project.conf``, the added file in the ``include/`` project subdirectory, and how these come together in the the ``hello.bst`` element. ``project.conf`` ~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/directives/project.conf :language: yaml Here, our ``project.conf`` declares a project option called ``flavor``, and this will inform what kind of greeting message we want to use when building the project. ``elements/hello.bst`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/directives/elements/hello.bst :language: yaml Notice the ``(@)`` symbol we've added in the ``variables:`` section, this symbol is used to invoke the :ref:`include directive `, which can be useful for code sharing between elements or simply to improve readability. In this case, we are compositing the content of ``include/greeting.bst`` into the :ref:`variables ` section of the element declaration, directives can however be used virtually anywhere in the BuildStream YAML format. ``include/greeting.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/directives/include/greeting.bst :language: yaml Here we can see the dictionary which will be composited into the ``variables:`` section of the ``hello.bst`` element described above. Note the usage of the ``(?)`` symbol at the toplevel of the YAML dictionary, this is how we perform :ref:`conditional statements ` in the BuildStream YAML format. This include file uses the ``flavor`` project option we declared in ``project.conf`` to decide what value will end up being assigned to the ``%{greeting}`` variable, which will ultimately be used in the ``hello.bst`` element. Using the project ----------------- Now that we have a project which uses options and conditional statements, lets build the project with a few different options and observe the outputs. Building hello.bst element with options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since the :ref:`flavor option ` we've declared above has a default, we can build it the first time using :ref:`bst build ` without any special command line options: .. raw:: html :file: ../sessions/directives-build-normal.html If we want to build the ``somber`` flavor, we just need to specify the additional ``--option`` command line option to :ref:`bst ` in order to inform BuildStream of the options we want. .. raw:: html :file: ../sessions/directives-build-somber.html Note that the ``--option`` option can be specified many times on the ``bst`` command line, so as to support projects which have multiple options. Finally lets get the ``excited`` flavor built as well: .. raw:: html :file: ../sessions/directives-build-excited.html If you observe the cache keys above, you will notice that while we have only three elements in the pipeline, counting ``base/alpine.bst``, ``base.bst`` and ``hello.bst``, we have actually built *five artifacts*, because the ``hello.bst`` is built differently each time, it has a different cache key and is stored separately in the artifact cache. Run the hello world program with options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Since the ``--option`` command line option to :ref:`bst ` is a main option, it can be used in any command. Let's run the ``hello`` program using :ref:`bst shell ` three times in a row, each time using a different option so we can observe the results. .. raw:: html :file: ../sessions/directives-shell-normal.html .. raw:: html :file: ../sessions/directives-shell-somber.html .. raw:: html :file: ../sessions/directives-shell-excited.html Summary ------- In this chapter we've demonstrated how to declare :ref:`project options `, how to use :ref:`conditional directives `, and also how to use :ref:`include directives `. To get more familliar with these concepts, you may want to explore the remaining :ref:`directives ` in the BuildStream YAML format, and also take a look at the various :ref:`types of project options ` that are also supported. apache-buildstream-27ae392/doc/source/tutorial/first-project.rst000066400000000000000000000102401514607367700250540ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _tutorial_first_project: Your first project ================== To get a feel for the basics, we'll start with the most basic BuildStream project we could think of. .. note:: This example is distributed with BuildStream in the `doc/examples/first-project `_ subdirectory. Creating the project -------------------- First, lets create the project itself using the convenience :ref:`bst init ` command to create a little project structure: .. raw:: html :file: ../sessions/first-project-init.html This will give you a :ref:`project.conf ` which will look like this: ``project.conf`` ~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/first-project/project.conf :language: yaml The :ref:`project.conf ` is a central point of configuration for your BuildStream project. Add some content ---------------- BuildStream processes directory trees as input and output, so let's just create a ``hello.world`` file for the project to have. .. raw:: html :file: ../sessions/first-project-touch.html Declare the element ------------------- Here we're going to declare a simple :mod:`import ` element which will import the ``hello.world`` file we've created in the previous step. Create ``elements/hello.bst`` with the following content: ``elements/hello.bst`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/first-project/elements/hello.bst :language: yaml The source ~~~~~~~~~~ The :mod:`local ` source used by the ``hello.bst`` element, can be used to access files or directories which are stored in the same repository as your BuildStream project. The ``hello.bst`` element uses the :mod:`local ` source to stage our local ``hello.world`` file. The element ~~~~~~~~~~~ The :mod:`import ` element can be used to simply add content directly to the output artifacts. In this case, it simply takes the ``hello.world`` file provided by its source and stages it directly to the artifact output root. .. tip:: In this example so far we've used two plugins, the :mod:`local ` source and the :mod:`import ` element. You can always browse the documentation for all plugins in the :ref:`plugins section ` of the manual. Build the element ----------------- In order to carry out the activities of the :mod:`import ` element we've declared, we're going to have to ask BuildStream to *build*. This process will collect all of the sources required for the specified ``hello.bst`` and get the backing :mod:`import ` element to generate an *artifact* for us. .. raw:: html :file: ../sessions/first-project-build.html Now the artifact is ready. Using :ref:`bst show `, we can observe that the artifact's state, which was reported as ``buildable`` in the :ref:`bst build ` command above, has now changed to ``cached``: .. raw:: html :file: ../sessions/first-project-show.html Observe the output ------------------ Now that we've finished building, we can checkout the output of the artifact we've created using :ref:`bst artifact checkout ` .. raw:: html :file: ../sessions/first-project-checkout.html And observe that the file we expect is there: .. raw:: html :file: ../sessions/first-project-ls.html Summary ------- In this section we've created our first BuildStream project from scratch, but it doesnt do much. We've observed the general structure of a BuildStream project, and we've run our first build. apache-buildstream-27ae392/doc/source/tutorial/integration-commands.rst000066400000000000000000000122341514607367700264100ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _tutorial_integration_commands: Integration commands ==================== Sometimes a software requires more configuration or processing than what is performed at installation time, otherwise it will not run properly. This is especially true in cases where a daemon or library interoperates with third party extensions and needs to maintain a system wide cache whenever its extensions are installed or removed; system wide font caches are an example of this. In these cases we use :ref:`integration commands ` to ensure that a runtime is ready to run after all of its components have been *staged*. .. note:: This example is distributed with BuildStream in the `doc/examples/integration-commands `_ subdirectory. Overview -------- In this chapter, we'll be exploring :ref:`integration commands `, which will be our first look at :ref:`public data `. Project structure ----------------- ``project.conf`` and ``elements/base.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The project.conf and base stack :mod:`stack ` element are configured in the same way as in the previous chapter: :ref:`tutorial_running_commands`. ``elements/base/alpine.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/integration-commands/elements/base/alpine.bst :language: yaml This is the same ``base/alpine.bst`` we've seen in previous chapters, except that we've added an :ref:`integration command `. This informs BuildStream that whenever the output of this element is expected to *run*, this command should be run first. In this case we are simply running ``ldconfig`` as a precautionary measure, to ensure that the runtime linker is ready to find any shared libraries we may have added to ``%{libdir}``. Looking at public data '''''''''''''''''''''' The :ref:`integration commands ` used here is the first time we've used any :ref:`builtin public data `. Public data is a free form portion of an element's configuration and is not necessarily understood by the element on which it is declared, public data is intended to be read by its reverse dependency elements. This allows annotations on some elements to inform elements later in the dependency chain about details of its artifact, or to suggest how it should be processed. ``elements/libhello.bst`` and ``elements/hello.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ These are basically manual elements very similar to the ones we've seen in the previous chapter: :ref:`tutorial_running_commands`. These produce a library and a hello program which uses the library, we will consider these irrelevant to the topic and leave examination of `their sources `_ as an exercise for the reader. Using the project ----------------- Build the hello.bst element ~~~~~~~~~~~~~~~~~~~~~~~~~~~ To build the project, run :ref:`bst build ` in the following way: .. raw:: html :file: ../sessions/integration-commands-build.html Observe in the build process above, the integration command declared on the ``base/alpine.bst`` element is run after staging the dependency artifacts into the build sandbox and before running any of the build commands, for both of the ``libhello.bst`` and ``hello.bst`` elements. BuildStream assumes that commands which are to be run in the build sandbox need to be run in an *integrated* sandbox. .. tip:: Integration commands can be taxing on your overall build process, because they need to run at the beginning of every build which :ref:`runtime depends ` on the element declaring them. For this reason, it is better to leave out more onerous tasks if they are not needed at software build time, and handle those specific tasks differently later in the pipeline, before deployment. Run the hello world program ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Unlike the previous chapters, this hello world program takes an argument, we can invoke the program using :ref:`bst shell `: .. raw:: html :file: ../sessions/integration-commands-shell.html Here we see again, the integration commands are also used when preparing the shell to launch a command. Summary ------- In this chapter we've observed how :ref:`integration commands ` work, and we now know about :ref:`public data `, which plugins can read from their dependencies in order to influence their build process. apache-buildstream-27ae392/doc/source/tutorial/running-commands.rst000066400000000000000000000204461514607367700255510ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _tutorial_running_commands: Running commands ================ In :ref:`the first chapter ` we only imported a file to create an artifact, this time lets run some commands inside the :ref:`isolated build sandbox `. .. note:: This example is distributed with BuildStream in the `doc/examples/running-commands `_ subdirectory. Overview -------- In this chapter, we'll be running commands inside the sandboxed execution environment and producing build output. We'll be compiling the following simple C file: ``files/src/hello.c`` ~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/running-commands/files/src/hello.c :language: c And we're going to build it using ``make``, using the following Makefile: ``files/src/Makefile`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/running-commands/files/src/Makefile :language: Makefile We'll be using the most fundamental :ref:`build element `, the :mod:`manual ` build element. The :mod:`manual ` element is the backbone on which all the other build elements are built, so understanding how it works at this level is helpful. Project structure ----------------- In this project we have a ``project.conf``, a directory with some source code, and 3 element declarations. Let's first take a peek at what we need to build using :ref:`bst show `: .. raw:: html :file: ../sessions/running-commands-show-before.html This time we have loaded a pipeline with 3 elements, let's go over what they do in detail. .. _tutorial_running_commands_project_conf: ``project.conf`` ~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/running-commands/project.conf :language: yaml Our ``project.conf`` is very much like the last one, except that we have defined a :ref:`source alias ` for ``alpine``. .. tip:: Using :ref:`source aliases ` for groups of sources which are generally hosted together is encouraged. This allows one to globally change the access scheme or URL for a group of repositories which belong together. ``elements/base/alpine.bst`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/running-commands/elements/base/alpine.bst :language: yaml This :mod:`import ` element uses a :mod:`tar ` source to download our Alpine Linux tarball to create our base runtime. This tarball is a sysroot which provides the C runtime libraries and some programs - this is what will be providing the programs we're going to run in this example. ``elements/base.bst`` ~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/running-commands/elements/base.bst :language: yaml This is just a symbolic :mod:`stack ` element which declares that anything which depends on it, will implicitly depend on ``base/alpine.bst``. It is typical to use stack elements in places where the implementing logical software stack could change, but you rather not have your higher level components carry knowledge about those changing components. Any element which :ref:`runtime depends ` on the ``base.bst`` will now be able to execute programs provided by the imported ``base/alpine.bst`` runtime. ``elements/hello.bst`` ~~~~~~~~~~~~~~~~~~~~~~ .. literalinclude:: ../../examples/running-commands/elements/hello.bst :language: yaml Finally we have the element which executes commands. Looking at the :mod:`manual ` element's documentation, we can see that the element configuration exposes four command lists: * ``configure-commands`` Commands which are run in preparation of a build. This is where you would normally call any configure stage build tools to configure the build how you like and generate some files needed for the build. * ``build-commands`` Commands to run the build, usually a build system will invoke the compiler for you here. * ``install-commands`` Commands to install the build results. Commands to install the build results into the target system, these should install files somewhere under ``%{install-root}``. * ``strip-commands`` Commands to doctor the build results after the install. Typically this involves stripping binaries of debugging symbols or stripping timestamps from build results to ensure reproducibility. .. tip:: All other :ref:`build elements ` implement exactly the same command lists too, except that they provide default commands specific to invoke the build systems they support. The :mod:`manual ` element however is the most basic and does not provide any default commands, so we have instructed it to use ``make`` to build and install our program. Using the project ----------------- Build the hello.bst element ~~~~~~~~~~~~~~~~~~~~~~~~~~~ To build the project, run :ref:`bst build ` in the following way: .. raw:: html :file: ../sessions/running-commands-build.html Now we've built our hello world program, using ``make`` and the C compiler provided by the Alpine Linux image. In the :ref:`first chapter ` we observed that the inputs and output of an element are *directory trees*. In this example, the directory tree generated by ``base/alpine.bst`` is consumed by ``hello.bst`` due to the :ref:`implicit runtime dependency ` introduced by ``base.bst``. .. tip:: All of the :ref:`dependencies ` which are required to run for the sake of a build, are staged at the root of the build sandbox. These comprise the runtime environment in which the depending element will run commands. The result is that the ``make`` program and C compiler provided by ``base/alpine.bst`` were already in ``$PATH`` and ready to run when the commands were needed by ``hello.bst``. Now observe that all of the elements in the loaded pipeline are ``cached``, the element is *built*: .. raw:: html :file: ../sessions/running-commands-show-after.html Run the hello world program ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Now that we've built everything, we can indulge ourselves in running the hello world program using :ref:`bst shell `: .. raw:: html :file: ../sessions/running-commands-shell.html Here, :ref:`bst shell ` created a runtime environment for running the ``hello.bst`` element. This was done by staging all of the dependencies of ``hello.bst`` including the ``hello.bst`` output itself into a directory. Once a directory with all of the dependencies was staged and ready, we ran the ``hello`` command from within the build sandbox environment. .. tip:: When specifying a command for :ref:`bst shell ` to run, we always specify ``--`` first. This is a commonly understood shell syntax to indicate that the remaining arguments are to be treated literally. Specifying ``--`` is optional and disambiguates BuildStream's arguments and options from those of the program being run by :ref:`bst shell `. Summary ------- In this chapter we've explored how to use the :mod:`manual ` element, which forms the basis of all build elements. We've also observed how the directory tree from the output *artifact* of one element is later *staged* at the root of the sandbox, as input for use by any build elements which :ref:`depend ` on that element. .. tip:: The way that elements consume their dependency input can vary across the different *kinds* of elements. This chapter describes how it works for :mod:`build elements ` implementations, which are the most commonly used element type. apache-buildstream-27ae392/doc/source/using_commands.rst000066400000000000000000000154331514607367700234350ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _commands: Commands ======== This page contains documentation for each BuildStream command, along with their possible options and arguments. Each command can be invoked on the command line, where, in most cases, this will be from the project's main directory. Commonly used parameters ------------------------ .. _invoking_specify_remotes: Remotes ~~~~~~~ Remote :ref:`cache servers ` can be specified on the command line for commands which may result in communicating with such servers. Any command which has arguments to specify a ``REMOTE``, such as ``--artifact-remote`` or ``--source-remote``, will override whatever was set in the user configuration, and will have an accompanying switch which allows the command to decide whether to ignore any remote :ref:`artifact ` or :ref:`source ` caches suggested by project configuration. Remotes can be specified on the command line either as a simple URI, or as a comma separated list of key value pairs. **Specifying a remote using a URI** .. code:: shell bst artifact push --remote https://artifacts.com/artifacts:8088 element.bst **Specifying a remote using key value pairs** .. code:: shell bst build --artifact-remote \ url=https://artifacts.com/artifacts:8088,type=index,server-cert=~/artifacts.cert \ element.bst Attributes '''''''''' Here is the list attributes which can be spefied when providing a ``REMOTE`` on the command line: * ``url`` The URL of the remote, possibly including a port number. * ``instance-name`` The instance name of this remote, used for sharding by some implementations. * ``type`` Whether this remote is to be used for indexing, storage or both, as explained in the corresponding :ref:`user configuration documentation ` * ``push`` Normally one need not specify this, as it is often inferred by the command being used. In some cases, like :ref:`bst build `, it can be useful to specify multiple remotes, and only allow pushing to some of the remotes. If unspecified, this is assumed to be ``True`` and BuildStream will attempt to push to the remote, but fallback to only pulling if insufficient credentials were provided. * ``server-cert``, ``client-cert``, ``client-key``: These keys specify the attributes of the :ref:`authentication configuration `. When specifying these on the command line, they are interpreted as paths relative to the current working directory. Top-level commands ------------------ .. The bst options e.g. bst --version, or bst --verbose etc. .. _invoking_bst: .. click:: buildstream._frontend:cli :prog: bst .. Further description of the command goes here ---- .. _invoking_artifact: .. click:: buildstream._frontend.cli:artifact :prog: bst artifact ---- .. the `bst init` command .. _invoking_init: .. click:: buildstream._frontend.cli:init :prog: bst init ---- .. the `bst build` command .. _invoking_build: .. click:: buildstream._frontend.cli:build :prog: bst build ---- .. _invoking_show: .. click:: buildstream._frontend.cli:show :prog: bst show ---- .. _invoking_shell: .. click:: buildstream._frontend.cli:shell :prog: bst shell ---- .. _invoking_source: .. click:: buildstream._frontend.cli:source :prog: bst source ---- .. _invoking_workspace: .. click:: buildstream._frontend.cli:workspace :prog: bst workspace .. _artifact_subcommands: Artifact subcommands -------------------- .. _artifact_names: Artifact names ~~~~~~~~~~~~~~ Various artifact subcommands accept either :ref:`element names `, which will operate on artifacts by deriving the artifact from local project state, or :term:`artifact names ` interchangeably as targets. Artifact names allow the user to operate directly on cached artifacts, without requiring local project data. An artifact name is composed of the following identifiers: * The :ref:`project name ` * The :ref:`element name `, without any trailing ``.bst`` extension * The cache key of the element at the time it was built. To compose an artifact name, simply join these using a forward slash (``/``) character, like so: ``//``. An artifact name might look like: ``project/target/788da21e7c1b5818b7e7b60f7eb75841057ff7e45d362cc223336c606fe47f27`` .. _invoking_artifact_checkout: .. click:: buildstream._frontend.cli:artifact_checkout :prog: bst artifact checkout ---- .. _invoking_artifact_log: .. click:: buildstream._frontend.cli:artifact_log :prog: bst artifact log ---- .. _invoking_artifact_pull: .. click:: buildstream._frontend.cli:artifact_pull :prog: bst artifact pull ---- .. _invoking_artifact_push: .. click:: buildstream._frontend.cli:artifact_push :prog: bst artifact push ---- .. _invoking_artifact_delete: .. click:: buildstream._frontend.cli:artifact_delete :prog: bst artifact delete ---- .. _invoking_artifact_show: .. click:: buildstream._frontend.cli:artifact_show :prog: bst artifact show ---- .. _invoking_artifact_list_contents: .. click:: buildstream._frontend.cli:artifact_list_contents :prog: bst artifact list-contents .. _source_subcommands: Source subcommands ------------------ .. _invoking_source_fetch: .. click:: buildstream._frontend.cli:source_fetch :prog: bst source fetch ---- .. _invoking_source_track: .. click:: buildstream._frontend.cli:source_track :prog: bst source track ---- .. _invoking_source_push: .. click:: buildstream._frontend.cli:source_push :prog: bst source push ---- .. _invoking_source_checkout: .. click:: buildstream._frontend.cli:source_checkout :prog: bst source checkout .. _workspace_subcommands: Workspace subcommands --------------------- .. _invoking_workspace_open: .. click:: buildstream._frontend.cli:workspace_open :prog: bst workspace open ---- .. _invoking_workspace_close: .. click:: buildstream._frontend.cli:workspace_close :prog: bst workspace close ---- .. _invoking_workspace_reset: .. click:: buildstream._frontend.cli:workspace_reset :prog: bst workspace reset ---- .. _invoking_workspace_list: .. click:: buildstream._frontend.cli:workspace_list :prog: bst workspace list apache-buildstream-27ae392/doc/source/using_config.rst000066400000000000000000001234341514607367700231020ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _user_config: User configuration ================== User configuration and preferences can be specified in a user provided configuration file, and in most cases these preferences can be overridden using :ref:`command line options `. Values that are not specified in a user configuration file assume their :ref:`default values `. Configuration file ------------------ Unless a configuration file is explicitly specified on the :ref:`command line ` when invoking ``bst``, an attempt is made to load user specific configuration from ``$XDG_CONFIG_HOME/buildstream.conf``. On most Linux based systems, the location will be ``~/.config/buildstream.conf`` .. note:: If you have have multiple major versions of BuildStream installed, you can have separate configuration files in your ``${XDG_CONFIG_HOME}``. You can do this by naming them according to the major versions of BuildStream you have installed. BuildStream 1 will load its configuration from ``$XDG_CONFIG_HOME/buildstream1.conf`` and BuildStream 2 will load its configuration from ``$XDG_CONFIG_HOME/buildstream2.conf``, while any version will fallback to ``$XDG_CONFIG_HOME/buildstream.conf``. Working directories ------------------- The working directories control where BuildStream will store data. While these will have sane defaults, you may want to change these directories depending on your partitioning scheme and where you may have extra space. Environment variables and ``~/`` home directory expressions are supported when specifying working directories. Working directories are configured at the toplevel of your configuration file, like so: .. code:: yaml # # Configure working directories # sourcedir: ~/buildstream/sources Attributes ~~~~~~~~~~ * ``sourcedir`` This is the location where BuildStream stores the source code it downloads for builds. * ``logdir`` This is the location where BuildStream stores log files of build command output and basically logs pertaining to any activity BuildStream orchestrates. * ``cachedir`` This is the location where BuildStream stores the local *CAS* (*Content Addressable Storage*). The *CAS* is used to cache anything and everything which BuildStream may reuse at a later time. .. attention:: While it may be beneficial at times to delete the entire cache directory manually in order to free up disk space, one should keep in mind that the ``sourcedir`` and ``logdir`` are configured as subdirectories of this directory when default configuration is used. Take care not to accidentally remove all your cached downloaded sources when deleting your cache. * ``workspacedir`` A default location for :ref:`opening workspaces `. .. note:: By default this is configured to ``.``, which is to say workspaces are created as a subdirectory of the current working directory by default. Because of this, the ``workspacedir`` directory is the only directory which is allowed to be declared as a relative path. .. _config_local_cache: Cache control ------------- Beyond deciding what directory you intend to place the cache, there are some controls on what is cached locally and how. These are controlled by the attributes of a ``cache`` dictionary at the toplevel of your configuration file, like so: .. code:: # # Control the cache # cache: # Use as much space as possible quota: infinity # Keep 5% of disk space available reserved-disk-space: 5% # Retain 80% of the cache on cleanup low-watermark: 80% # Avoid pulling large amounts of data we don't need locally pull-buildtrees: False # # Avoid caching build trees if we don't need them cache-buildtrees: auto # # Support CAS server as remote cache # Useful to minimize network traffic with remote execution # or to work with limited local disk space storage-service: url: https://cache-server.com/cas:11001 auth: server-cert: server.crt client-cert: client.crt client-key: client.key access-token: access.token access-token-reload-interval: 60 connection-config: keepalive-time: 60 retry-limit: 4 retry-delay: 1000 request-timeout: 300 Attributes ~~~~~~~~~~ * ``quota`` This controls how much data you allow BuildStream to cache locally. An attempt will be made to error out instead of exceeding the maximum quota which the user has allowed here. Given that it is impossible for BuildStream to know how much data a given build will create, this quota is implemented on a best effort basis. The ``quota`` can be specified in multiple ways: * The special ``infinity`` value This default value states that BuildStream can use as much space as is available on the filesystem where the cache resides. * A number in bytes. * A human readable number, suffixed in K, M, G or T E.g. ``250K`` being 250 kilobytes, ``100M`` being 100 megabytes, etc. * A percentage value, e.g. ``80%`` Percentage values are taken to represent a percentage of the partition size on the filesystem where the cache has been configured. * ``reserved-disk-space`` This controls how much disk space should remain available. If the amount of available disk space falls below the specified value, unused cache objects will be pruned even if the configured quota has not been exceeded. ``reserved-disk-space`` can be specified in the same way as ``quota``, with the exception of the special ``infinity`` value. The default is ``5%``. *Since: 2.4* * ``low-watermark`` This controls how much of the cache should be retained on cleanup. ``low-watermark`` is specified as a percentage of the effective cache quota as configured by ``quota`` and/or ``reserved-disk-space``. The default is ``80%``, which means that when cleanup is triggered, 20% of the cache will be pruned by removing CAS objects that haven't been used recently. *Since: 2.4* * ``pull-buildtrees`` Whether to pull *build trees* when downloading remote artifacts. The *build tree* of an artifact is the directory where a build took place, this is useful for :ref:`running a build shell ` in order to observe how an element was built or to debug how a build failed if the build failed remotely. Since build trees are rather expensive, the default is to not pull build trees for every artifact. If you need a build tree that exists remotely, it will be possible to download it as an option at the time you run a command which requires it. * ``cache-buildtrees`` Whether to cache build trees when creating artifacts, if build trees are cached locally and the client is configured to push to remote servers, then build trees will be pushed along with any uploaded artifacts. This configuration has three possible values: * ``never``: Never cache build trees * ``auto``: Only cache the build trees where necessary (e.g. for failed builds) * ``always``: Always cache the build tree. * ``storage-service`` An optional :ref:`service configuration ` to use a *Content Addressable Storage* service as a remote cache. Write access is required. This service is compatible with the *storage* service offered by :ref:`cache servers `. Without this option, all content is stored in the local cache. This includes CAS objects from fetched sources, build outputs and pulled artifacts. With this option, content is primarily stored in the remote cache and the local cache is populated only as needed. E.g. ``bst artifact checkout`` will download CAS objects on demand from the remote cache. This feature is incompatible with offline operation. This is primarily useful in combination with :ref:`remote execution ` to minimize downloads of build outputs, which may not be needed locally. The elimination of unnecessary downloads reduces total build time, especially if the bandwidth between the local system and the remote execution cluster is limited. .. tip:: Skip the ``storage-service`` option in the :ref:`remote execution ` configuration to use the same CAS service for caching and remote execution. It is also possible to configure this with local builds without remote execution. This enables operation with a small local cache even with large projects. However, for local builds this can result in a longer total build time due to additional network transfers. This is only recommended with a high bandwidth connection to a storage-service, ideally in a local network. Scheduler controls ------------------ Controls related to how the scheduler works are exposed as attributes of the toplevel ``scheduler`` dictionary, like so: .. code:: yaml # # Control the scheduler # scheduler: # Allow building up to four seperate elements at a time builders: 4 # Continue building as many elements as possible if anything fails on-error: continue Attributes ~~~~~~~~~~ * ``fetchers`` The number of concurrent tasks which download sources or artifacts. * ``pushers`` The number of concurrent tasks which upload sources or artifacts. * ``builders`` The number of concurrent tasks which build elements. .. note:: This does not control the number of processes in the scope of the build of a single element, but rather the number of elements which may be built in parallel. * ``network-retries`` The number of times to retry a task which failed due to network connectivity issues. * ``on-error`` What to do when a task fails and BuildStream is running in non-interactive mode. This can be set to the following values: * ``continue``: Continue with other tasks, a summary of errors will be printed at the end * ``quit``: Quit after all ongoing tasks have completed * ``terminate``: Abort any ongoing tasks and exit immediately .. note:: If BuildStream is running in interactive mode, then the ongoing build will be suspended and the user will be prompted and asked what to do when a task fails. Interactive mode is automatically enabled if BuildStream is connected to a terminal rather than being run automatically, or, it can be specified on the :ref:`command line `. Build controls -------------- Some aspects about how elements get built can be controlled by attributes of the ``build`` dictionary at the toplevel, like so: .. code:: yaml # # Build controls # build: # # Allow up to 4 parallel processes to execute within the scope of one build # max-jobs: 4 Attributes ~~~~~~~~~~ * ``max-jobs`` This is a best effort attempt to instruct build systems on how many parallel processes to use when building an element. It is supported by most popular build systems such as ``make``, ``cmake``, ``ninja``, etc, via environment variables such as ``MAXJOBS`` and similar command line options. When using the special value ``0``, BuildStream will allocate the number of threads available on the host and limit this with a hard coded value of ``8``, which was found to be an optimial number when building even on hosts with many cores. * ``retry-failed`` Try to build elements for which a failed build artifact is found when running :ref:`bst build `. This is useful in the case that a build has failed due to insufficient resources such as memory or disk space. * ``dependencies`` This instructs what dependencies of the target elements should be built, valid values for this attribute are: * ``none``: Only build elements required to generate the expected target artifacts * ``run``: Build required elements and their their runtime dependencies * ``all``: Build elements even if they are build dependencies of artifacts which are already cached Fetch controls -------------- Some aspects about how sources get fetched can be controlled by attributes of the ``fetch`` dictionary at the toplevel, like so: .. code:: yaml # # Fetch controls # fetch: # # Don't allow fetching from project defined alias or mirror URIs # source: user Attributes ~~~~~~~~~~ * ``source`` This controls what URIs are allowed to be accessed when fetching sources, valid values for this attribute are: * ``all``: Fetch from mirrors defined in :ref:`user configuration ` and :ref:`project configuration `, and also project defined :ref:`default alias URIs `. * ``aliases``: Only allow fetching from project defined :ref:`default alias URIs `. * ``mirrors``: Only allow fetching from mirrors defined in :ref:`user configuration ` and :ref:`project configuration ` * ``user``: Only allow fetching from mirrors defined in :ref:`user configuration ` Track controls -------------- Some aspects about how sources get tracked can be controlled by attributes of the ``track`` dictionary at the toplevel, like so: .. code:: yaml # # Track controls # track: # # Only track sources for new refs from project defined default alias URIs # source: aliases Attributes ~~~~~~~~~~ * ``source`` This controls what URIs are allowed to be accessed when tracking sources for new refs, valid values for this attribute are: * ``all``: Track from mirrors defined in :ref:`user configuration ` and :ref:`project configuration `, and also project defined :ref:`default alias URIs `. * ``aliases``: Only allow tracking from project defined :ref:`default alias URIs `. * ``mirrors``: Only allow tracking from mirrors defined in :ref:`user configuration ` and :ref:`project configuration ` * ``user``: Only allow tracking from mirrors defined in :ref:`user configuration ` Logging controls ---------------- Various aspects of how BuildStream presents output and UI can be controlled with attributes of the toplevel ``logging`` dictionary, like so: .. code:: yaml # # Control logging output # logging: # # Whether to be verbose # verbose: True Attributes ~~~~~~~~~~ * ``verbose`` Whether to use verbose logging. * ``debug`` Whether to print messages related to debugging BuildStream itself. * ``key-length`` When displaying abbreviated cache keys, this controls the number of characters of the cache key which should be printed. * ``throttle-ui-updates`` Whether the throttle updates to the status bar in interactive mode. If set to ``True``, then the status bar will be updated once per second. * ``error-lines`` The maximum number of lines to print in the main logging output related to an error processing an element, these will be the last lines found in the relevant element's stdout and stderr. * ``message-lines`` The maximum number of lines to print in a detailed message sent to the main logging output. * ``element-format`` The default format to use when printing out elements in :ref:`bst show ` output, and when printing the pipeline summary at the beginning of sessions. The format is specified as a string containing variables which will be expanded in the resulting string, variables must be specified using a leading percent sign and enclosed in curly braces, a colon can be specified in the variable to perform python style string alignments, e.g.: .. code:: yaml logging: # # Set the element format # element-format: | %{state: >12} %{full-key} %{name} %{workspace-dirs} Variable names which can be used in the element format consist of: * ``name`` The :ref:`element path `, which is the name of the element including any leading junctions. * ``key`` The abbreviated cache key, the length of which is controlled by the ``key-length`` logging configuration. * ``full-key`` The full cache key. * ``state`` The element state, this will be formatted as one of the following: * ``no reference``: If the element still needs to be :ref:`tracked `. * ``junction``: If the element is a junction and as such does not have any relevant state. * ``failed``: If the element has been built and the build has failed. * ``cached``: If the element has been successfully built and is present in the local cache. * ``fetch needed``: If the element cannot be built yet because the sources need to be :ref:`fetched `. * ``buildable``: If the element has all of its sources and build dependency artifacts cached locally. * ``waiting``: If the element has all of its sources cached but its build dependencies are not yet locally cached. * ``config`` The :ref:`element configuration `, formatted in YAML. * ``vars`` The resolved :ref:`element variables `, formatted as a simple YAML dictionary. * ``env`` The resolved :ref:`environment variables `, formatted as a simple YAML dictionary. * ``public`` The resolved :ref:`public data `, formatted in YAML. * ``workspaced`` If the element has an open workspace, this will expand to the string *"(workspaced)"*, otherwise it will expand to an empty string. * ``workspace-dirs`` If the element has an open workspace, this will expand to the workspace directory, prefixed with the text *"Workspace: "*, otherwise it will expand to an empty string. * ``deps`` A list of the :ref:`element paths ` of all dependency elements. * ``build-deps`` A list of the :ref:`element paths ` of all build dependency elements. * ``runtime-deps`` A list of the :ref:`element paths ` of all runtime dependency elements. * ``message-format`` The format to use for messages being logged in the aggregated main logging output. Similarly to the ``element-format``, The format is specified as a string containing variables which will be expanded in the resulting string, and variables must be specified using a leading percent sign and enclosed in curly braces, e.g.: .. code:: yaml logging: # # Set the message format # message-format: | [%{elapsed}][%{key}][%{element}] %{action} %{message} Variable names which can be used in the element format consist of: * ``elapsed`` If this message announces the completion of (successful or otherwise) of an activity, then this will expand to a time code showing how much time elapsed for the given activity, in the format: ``HH:MM:SS``, otherwise an empty time code will be displayed in the format: ``--:--:--``. * ``elapsed-us`` Similar to the ``elapsed`` variable, however the timecode will include microsecond precision. * ``wallclock`` This will display a timecode for each message displaying the local wallclock time, in the format ``HH:MM:SS``. * ``wallclock-us`` Similar to the ``wallclock`` variable, however the timecode will include microsecond precision. * ``key`` The abbreviated cache key of the element the message is related to, the length of which is controlled by the ``key-length`` logging configuration. If the message in question is not related to any element, then this will expand to whitespace of equal length. * ``element`` This will be formatted to an indicator consisting of the type of activity which is being performed on the element (e.g. *"build"* or *"fetch"* etc), and the :ref:`element path ` of the element this message is related to. If the message in question is not related to any element, then a string will be formatted to indicate that this message is related to a core activity instead. * ``action`` A classifier of the type of message this is, the possible values this will expand to are: * ``DEBUG`` This is a message related to debugging BuildStream itself * ``STATUS`` A message showing some detail of what is currently happening, this message will not be displayed unless verbose output is enabled. * ``INFO`` An informative message, this may be printed for instance when discovering a new ref for source code when running :ref:`bst source track `. * ``WARN`` A warning message. * ``ERROR`` An error message. * ``BUG`` A bug happened in BuildStream, this will usually be accompanied by a python stack trace. * ``START`` An activity related to an element started. Any ``START`` message will always be accompanied by a later ``SUCCESS``, ``FAILURE`` or ``SKIPPED`` message. * ``SUCCESS`` An activity related to an element completed successfully. * ``FAILURE`` An activity related to an element failed. * ``SKIPPED`` After strating this activity, it was discovered that no work was needed and the activity was skipped. * ``message`` The brief message, or the path to the corresponding log file, will be printed here. When this is a scheduler related message about the commencement or completion of an element related activity, then the path to the corresponding log for that activity will be printed here. If it is a message issued for any other reason, then the message text will be formatted here. .. note:: Messages issued by the core or by plugins are allowed to provide detailed accounts, these are the indented multiline messages which sometimes get displayed in the main aggregated logging output, and will be printed regardless of the logging ``message-format`` value. Remote services --------------- BuildStream can be configured to cooperate with remote caches and execution services. .. _config_remote_auth: Authentication ~~~~~~~~~~~~~~ BuildStream supports end to end encryption when communicating with remote services. All remote service configuration blocks come with an optional ``auth`` configuration block which allows one to specify the certificates and keys required for encrypted traffic. The ``auth`` configuration block looks like this: .. code:: yaml auth: server-cert: server.crt client-cert: client.crt client-key: client.key access-token: access.token access-token-reload-interval: 60 Attributes '''''''''' * ``server-cert`` The server certificate is used to verify the identity of the server instead of using the CA store from the operating system for verification. * ``client-cert`` The client certificate is used by the remote server to authenticate the client. * ``client-key`` The private client key corresponding to the specified client certificate. * ``access-token`` The path to a token for optional HTTP bearer authentication. * ``access-token-reload-interval`` The reload interval in minutes for the specified access token. If not specified, automatic reloading is disabled. Remote cache services may allow *downloading* artifacts and sources without authentication, in which case only ``server-cert`` is required for secure access (or no attributes at all if the CA store from the OS can be used). However, remote cache services should normally not allow *uploading* artifacts or sources without authentication. Authentication can be configured by setting ``access-token`` or both ``client-key`` and ``client-cert``. .. _config_connection_config: Connection configuration ~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream uses gRPC to communicate with remote services. All remote service configuration blocks come with an optional ``connection-config`` block which allows setting gRPC connection options. The ``connection-config`` block looks like this: .. code:: yaml connection-config: keepalive-time: 60 retry-limit: 4 retry-delay: 1000 request-timeout: 300 Attributes '''''''''' * ``keepalive-time`` The interval in seconds between gRPC PING frames. Disabled by default. `gRPC keepalive guide `_ * ``retry-limit`` The maximum number of retries, not including the original request. * ``retry-delay`` The initial backoff in milliseconds for retries. `gRPC retry guide `_ * ``request-timeout`` The timeout for gRPC requests in seconds. No timeout by default. `gRPC deadline guide `_ .. _config_cache_servers: Cache servers ~~~~~~~~~~~~~ BuildStream supports two types of cache servers, :ref:`source cache servers ` and :ref:`artifact cache servers `. These services allow you to store sources and build artifacts for later reuse, and share them among your peers. .. important:: **Storing and indexing** Cache servers are split into two separate services, the *index* and the *storage*. Sometimes these services are provided by the same server, and sometimes it is desirable to use different cache servers for indexing and storing data. In simple setups, it is possible to use the same cache server for indexing and storing of both sources and artifacts. However, when using :ref:`remote execution ` it is recommended to use the remote execution build cluster's ``storage-service`` as the *storage* service of your cache servers, which may require setting up your *index* service separately. When configuring cache servers, BuildStream will require both storage and indexing capabilities, otherwise no attempt will be made to fetch or push data to and from cache servers. Cache server configuration is declared in the following way: .. code:: yaml override-project-caches: false servers: - url: https://cache-server.com/cache:11001 instance-name: main type: all push: true auth: server-cert: server.crt client-cert: client.crt client-key: client.key connection-config: keepalive-time: 60 retry-limit: 4 retry-delay: 1000 request-timeout: 300 Attributes '''''''''' * ``override-project-caches`` Whether this user configuration overrides the project recommendations for :ref:`artifact caches ` or :ref:`source caches `. If this is false (which is the default), then project recommended cache servers will be observed after user specified caches. * ``servers`` This is the list of cache servers in the configuration block, every entry in the block represents a server which will be accessed in the specified order. * ``url`` Indicates the ``http`` or ``https`` url and optionally the port number of where the cache server is located. * ``instance-name`` Instance names separate different shards on the same endpoint (``url``). The instance name is optional, and not all cache server implementations support instance names. The instance name should be given to you by the service provider of each service. * ``type`` The type of service you intend to use this cache server for. If unspecified, the default value for this field is ``all``. * ``storage`` Use this cache service for storage. * ``index`` Use this cache service for index content expected to be present in one or more *storage* services. * ``all`` Use this cache service for both indexing and storing data. * ``push`` Set this to ``true`` if you intend to upload data to this cache server. Normally this requires additional credentials in the ``auth`` field. * ``auth`` The :ref:`authentication attributes ` to connect to this server. .. _config_cache_server_list: Cache server lists '''''''''''''''''' Cache servers are always specified as *lists* in the configuration, this allows *index* and *storage* services to be declared separately, and also allows for some redundancy. **Example:** .. code:: yaml - url: https://cache-server-1.com/index type: index - url: https://cache-server-1.com/storage type: storage - url: https://cache-server-2.com type: all When downloading data from a cache server, BuildStream will iterate over each *index* service one by one until it finds the reference to the data it is looking for, and then it will iterate over each *storage* service one by one, downloading the referenced data until all data is downloaded. When uploading data to a cache server, BuildStream will first upload the data to each *storage* service which was configured with the ``push`` attribute, and upon successful upload, it will proceed to upload the references to the uploaded data to each *index* service in the list. .. _config_artifact_caches: Artifact cache servers ~~~~~~~~~~~~~~~~~~~~~~ Using artifact :ref:`cache servers ` is an essential means of *build avoidance*, as it will allow you to avoid building an element which has already been built and uploaded to a common artifact server. Artifact cache servers can be declared in different ways, with differing priorities. Command line '''''''''''' Various commands which involve connecting to artifact servers allow :ref:`specifying remotes `, remotes specified on the command line replace all user configuration. Global caches ''''''''''''' To declare the global artifact server list, use the ``artifacts`` key at the toplevel of the user configuration. .. code:: yaml # # Configure a global artifact server for pushing and pulling artifacts # artifacts: override-project-caches: false servers: - url: https://artifacts.com/artifacts:11001 push: true auth: server-cert: server.crt client-cert: client.crt client-key: client.key Project overrides ''''''''''''''''' To declare artifact servers lists for individual projects, declare them in the :ref:`project specific section ` of the user configuration. Artifact server lists declared in this section will only be used for elements belonging to the specified project, and will be used instead of artifact cache servers declared in the global caches. .. code:: yaml # # Configure an artifact server for pushing and pulling artifacts from project "foo" # projects: foo: artifacts: override-project-caches: false servers: - url: https://artifacts.com/artifacts:11001 push: true auth: server-cert: server.crt client-cert: client.crt client-key: client.key Project recommendations ''''''''''''''''''''''' Projects can :ref:`recommend artifact cache servers ` in their individual project configuration files. These will only be used for elements belonging to their respective projects, and are the lowest priority configuration. .. _config_source_caches: Source cache servers ~~~~~~~~~~~~~~~~~~~~ Using source :ref:`cache servers ` enables BuildStream to cache source code referred to by your project and share those sources with peers who have access to the same source cache server. This can optimize your build times in the case that it is determined that an element needs to be rebuilt because of changes in the dependency graph, as BuildStream will first attempt to download the source code from the cache server before attempting to obtain it from an external source, which may suffer higher latencies. Source cache servers can be declared in different ways, with differing priorities. Command line '''''''''''' Various commands which involve connecting to source cache servers allow :ref:`specifying remotes `, remotes specified on the command line replace all user configuration. Global caches ''''''''''''' To declare the global source cache server list, use the ``source-caches`` key at the toplevel of the user configuration. .. code:: yaml # # Configure a global source cache server for pushing and pulling sources # source-caches: override-project-caches: false servers: - url: https://sources.com/sources:11001 push: true auth: server-cert: server.crt client-cert: client.crt client-key: client.key Project overrides ''''''''''''''''' To declare source cache servers lists for individual projects, declare them in the :ref:`project specific section ` of the user configuration. Source cache server lists declared in this section will only be used for elements belonging to the specified project, and will be used instead of source cache servers declared in the global caches. .. code:: yaml # # Configure a source cache server for pushing and pulling sources from project "foo" # projects: foo: source-caches: override-project-caches: false servers: - url: https://sources.com/sources:11001 push: true auth: server-cert: server.crt client-cert: client.crt client-key: client.key Project recommendations ''''''''''''''''''''''' Projects can :ref:`recommend source cache servers ` in their individual project configuration files. These will only be used for elements belonging to their respective projects, and are the lowest priority configuration. .. _user_config_remote_execution: Remote execution ~~~~~~~~~~~~~~~~ BuildStream supports building remotely using the `Google Remote Execution API (REAPI) `_. There is also limited support to build locally while still using a remote action cache. You can configure the remote execution services globally in your user configuration using the ``remote-execution`` key, like so: .. code:: yaml remote-execution: execution-service: url: http://execution.fallback.example.com:50051 instance-name: main storage-service: url: https://storage.fallback.example.com:11002 instance-name: main auth: server-cert: /keys/server.crt client-cert: /keys/client.crt client-key: /keys/client.key action-cache-service: url: http://cache.flalback.example.com:50052 instance-name: main Attributes '''''''''' * ``execution-service`` A :ref:`service configuration ` specifying how to connect with the main *execution service*, this service is the main controlling entity in a remote execution build cluster. Skip the configuration of this service to enable local builds with a remote action cache. * ``storage-service`` A :ref:`service configuration ` specifying how to connect with the *Content Addressable Storage* service, this is where build input and output is stored on the remote execution build cluster. This service is compatible with the *storage* service offered by :ref:`cache servers `. This is optional if a ``storage-service`` is configured in the :ref:`cache configuration `, in which case actual file contents of build outputs will only be downloaded as needed, e.g. on ``bst artifact checkout``. * ``action-cache-service`` A :ref:`service configuration ` specifying how to connect with the *action cache*, this service stores information about activities which clients request be performed by workers on the remote execution build cluster, and results of completed operations. This service is optional in a remote execution build cluster, if your remote execution service provides an action cache, then you should configure it here. For local builds with a remote action cache, set ``push`` to ``true`` to upload action results to the server. By default, action results are downloaded from the server but not uploaded. If remote execution is not enabled, the remote action cache is currently only used for REAPI clients in the sandbox, not for BuildStream element build jobs. For more information about REAPI clients in the sandbox, see ``remote-apis-socket`` in the :ref:`sandbox configuration `. .. _user_config_remote_execution_service: Remote execution service configuration '''''''''''''''''''''''''''''''''''''' Each of the distinct services are described by the same configuration block, which looks like this: .. code:: yaml url: https://storage.fallback.example.com:11002 instance-name: main auth: server-cert: /keys/server.crt client-cert: /keys/client.crt client-key: /keys/client.key connection-config: keepalive-time: 60 retry-limit: 4 retry-delay: 1000 request-timeout: 300 **Attributes:** * ``url`` Indicates the ``http`` or ``https`` url and optionally the port number of where the service is located. * ``instance-name`` The instance name is optional. Instance names separate different shards on the same endpoint (``url``). The instance name should be given to you by the service provider of each service. Not all service providers support instance names. * ``auth`` The :ref:`authentication attributes ` to connect to this server. .. _user_config_project_overrides: Project specific values ----------------------- The ``projects`` key can be used to specify project specific configurations, the supported configurations on a project wide basis are listed here. .. _user_config_strict_mode: Strict build plan ~~~~~~~~~~~~~~~~~ The strict build plan option decides whether you want elements to rebuild when their dependencies have changed. This is enabled by default, but recommended to turn off in developer scenarios where you might want to build a large system and test it quickly after modifying some low level component. **Example** .. code:: yaml projects: project-name: strict: False .. note:: It is always possible to override this at invocation time using the ``--strict`` and ``--no-strict`` command line options. .. _config_mirrors: Mirrors ~~~~~~~ Project defined :ref:`mirrors `, can be overridden with user configuration. This is helpful when you need to mirror all of the source code used by subprojects and ensure that your project can be built in perpetuity. **Example** .. code:: yaml projects: project-name: mirrors: - name: middle-earth aliases: foo: - http://www.middle-earth.com/foo/1 - http://www.middle-earth.com/foo/2 bar: - http://www.middle-earth.com/bar/1 - http://www.middle-earth.com/bar/2 - name: oz aliases: foo: - http://www.oz.com/foo bar: - http://www.oz.com/bar .. _config_default_mirror: Default mirror ~~~~~~~~~~~~~~ When using :ref:`mirrors `, one can specify which mirror should be used first. **Example** .. code:: yaml projects: project-name: default-mirror: oz .. note:: It is possible to override this at invocation time using the ``--default-mirror`` command-line option. Project options ~~~~~~~~~~~~~~~ One can specify values to use for :ref:`project options ` for the projects you use here, this avoids needing to specify the options on the command line every time. **Example** .. code:: yaml projects: # # Configure the debug flag offered by `project-name` # project-name: options: debug-build: True Source cache servers ~~~~~~~~~~~~~~~~~~~~ As already described in the section concerning configuration of :ref:`source cache servers `, these can be specified on a per project basis. Artifact cache servers ~~~~~~~~~~~~~~~~~~~~~~ As already described in the section concerning configuration of :ref:`artifact cache servers `, these can be specified on a per project basis. Remote execution configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Following the same format as the toplevel :ref:`remote execution configuration `, the global configuration can be overridden on a per project basis in this project override section. **Example** .. code:: yaml projects: project-name: # # If `project-name` is built as the toplevel project in this BuildStream session, # then use this remote execution configuration instead of any globally defined # remote execution configuration. # remote-execution: execution-service: url: http://execution.example.com:50051 instance-name: main .. note:: Only one remote execution service will be considered for any invocation of BuildStream. If you are building a project which has a junction into another subproject for which you have specified a project specific remote execution service for in your user configuration, then it will be ignored in the context of building that toplevel project. .. _config_defaults: Default configuration --------------------- The default BuildStream configuration is specified here for reference: .. literalinclude:: ../../src/buildstream/data/userconfig.yaml :language: yaml apache-buildstream-27ae392/doc/source/using_configuring_cache_server.rst000066400000000000000000000063571514607367700266640ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _cache_servers: Configuring Cache Servers ========================= BuildStream caches the results of builds in a local artifact cache, and will avoid building an element if there is a suitable build already present in the local artifact cache. Similarly it will cache sources and avoid pulling them if present in the local cache. See :ref:`caches ` for more details. In addition to the local caches, you can configure one or more remote caches and BuildStream will then try to pull a suitable object from one of the remotes, falling back to performing a local build or fetching a source if needed. On the client side, cache servers are declared and configured in :ref:`user configuration `, and since it is typical for projects to maintain their own cache servers, it is also possible for projects to provide recommended :ref:`artifact cache servers ` and :ref:`source cache servers ` through project configuration, so that downstream users can download from services provided by upstream projects by default. Setting up a remote cache ------------------------- BuildStream relies on the `ContentAddressableStorage protocol `_ in order to exchange data with remote services, in concert with the `remote asset protocol `_ in order to assign symbolic labels (such as :ref:`artifact names `) to identify stored content. As such, BuildStream is able to function with any implementations of these two services. Known implementations --------------------- Here are some details about known open source implementations of the required protocols Buildbarn ~~~~~~~~~ The `Buildbarn `_ project provides a remote execution service implementation for use in build tooling such as BuildStream, `Bazel `_ and `recc `_, the `bb-storage `_ and `bb-remote-asset `_ services are tested to work as cache service for BuildStream's artifact and source caches. A simple configuration to spin up the service using `docker compose `_ follows: .. literalinclude:: ../../.github/compose/ci.buildbarn.yml :language: yaml Visit the `bb-storage `_ and `bb-remote-asset `_ project pages to find more documentation about setting up services with authentication enabled. apache-buildstream-27ae392/doc/source/using_configuring_remote_execution.rst000066400000000000000000000065231514607367700276040ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _remote_execution_servers: Remote Execution Servers ======================== BuildStream supports building remotely using the `Google Remote Execution API (REAPI). `_, which has various known implementations. Some of these implementations include: * `BuildGrid `_ * `BuildBarn `_ * `Buildfarm `_ These various implementations implement the `Google Remote Execution API (REAPI) `_ to various degrees as these projects have different priorities. On the client side, the remote execution service to use can be specified in the :ref:`user configuration `. BuildStream specific requirements --------------------------------- In order for BuildStream to work correctly with a remote execution cluster, there are a couple of requirements that implementation needs to meet. Implementation of platform properties ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The remote execution service must properly implement `platform properties `_. This is crucial because BuildStream needs to be guaranteed the correct operating system and ISA which it requests from the service. Staging the input root as the filesystem root ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BuildStream requires that the *input root* given to the remote execution service be treated as the absolute *filesystem root*. This is because BuildStream provides guarantees that all build dependencies, including the base runtime and compilers, are defined by elements and run within a sandboxed and isolated build environment, but the `REAPI `_ was originally developped without this determinism and control in mind. Instead, typically it is up to the user to configure a cluster to use a docker image to build payloads with, rather than allowing the REAPI client to control the entire sandbox. Unfortunately the ability to dictate that the *input root* be treated as the *filesystem root* in a container on remote workers in the cluster is not yet standardized in the REAPI protocol. .. note:: The *input root* is referred to as the ``input_root_digest`` member of the ``Action`` message as defined in the `protocol `_ Example working configuration ----------------------------- A simple configuration to spin up the `BuildGrid `_ service using `docker compose `_ follows: .. literalinclude:: ../../.github/compose/ci.buildgrid.yml :language: yaml apache-buildstream-27ae392/doc/source/using_developing.rst000066400000000000000000000015351514607367700237660ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Developing ========== This section covers the features BuildStream provides for users to develop their integrated products, as well as developping their modules in the context of an integrated product. .. toctree:: :numbered: :maxdepth: 1 developing/workspaces.rst developing/strict-mode.rst apache-buildstream-27ae392/doc/source/using_examples.rst000066400000000000000000000015151514607367700234460ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _examples: Examples ======== This page contains documentation for real examples of BuildStream projects, described step by step. All run under CI, so you can trust they are maintained and work as expected. .. toctree:: :maxdepth: 1 examples/tar-mirror examples/git-mirror apache-buildstream-27ae392/doc/source/using_handling_files.rst000066400000000000000000000014531514607367700245770ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Handling files ============== This section covers the various aspects related to finer grained handling of files. .. toctree:: :numbered: :maxdepth: 1 handling-files/composition.rst handling-files/filtering.rst handling-files/overlaps.rst apache-buildstream-27ae392/doc/source/using_junctions.rst000066400000000000000000000014711514607367700236450ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Combining projects ================== This section covers the features which allow BuildStream projects to depend on eachother. .. toctree:: :numbered: :maxdepth: 1 junctions/junction-elements junctions/junction-workspaces junctions/junction-includes apache-buildstream-27ae392/doc/source/using_tutorial.rst000066400000000000000000000016341514607367700234750ustar00rootroot00000000000000.. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. .. _tutorial: Getting started =============== This is a step by step walkthrough meant help the user quickly get familiar with the fundamentals of creating and using BuildStream projects. .. toctree:: :numbered: :maxdepth: 1 tutorial/first-project tutorial/running-commands tutorial/autotools tutorial/integration-commands tutorial/directives apache-buildstream-27ae392/man/000077500000000000000000000000001514607367700163755ustar00rootroot00000000000000apache-buildstream-27ae392/man/bst-artifact-checkout.1000066400000000000000000000023031514607367700226430ustar00rootroot00000000000000.TH "BST ARTIFACT CHECKOUT" "1" "2025-05-09" "2.5" "bst artifact checkout Manual" .SH NAME bst\-artifact\-checkout \- Checkout contents of an artifact .SH SYNOPSIS .B bst artifact checkout [OPTIONS] [TARGET] .SH DESCRIPTION Checkout contents of an artifact .PP When this command is executed from a workspace directory, the default is to checkout the artifact of the workspace element. .SH OPTIONS .TP \fB\-f,\fP \-\-force Allow files to be overwritten .TP \fB\-d,\fP \-\-deps [run|build|none|all] The dependencies to checkout [default: run] .TP \fB\-\-integrate\fP / \-\-no\-integrate Whether to run integration commands .TP \fB\-\-hardlinks\fP Checkout hardlinks instead of copying if possible .TP \fB\-\-tar\fP LOCATION Create a tarball from the artifact contents instead of a file tree. If LOCATION is '-', the tarball will be dumped to the standard output. .TP \fB\-\-compression\fP [gz|xz|bz2] The compression option of the tarball created. .TP \fB\-\-directory\fP DIRECTORY The directory to checkout the artifact to .TP \fB\-\-artifact\-remote\fP REMOTE A remote for downloading artifacts .TP \fB\-\-ignore\-project\-artifact\-remotes\fP Ignore remote artifact cache servers recommended by projects apache-buildstream-27ae392/man/bst-artifact-delete.1000066400000000000000000000005561514607367700223100ustar00rootroot00000000000000.TH "BST ARTIFACT DELETE" "1" "2025-05-09" "2.5" "bst artifact delete Manual" .SH NAME bst\-artifact\-delete \- Remove artifacts from the local cache .SH SYNOPSIS .B bst artifact delete [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION Remove artifacts from the local cache .SH OPTIONS .TP \fB\-d,\fP \-\-deps [none|run|build|all] The dependencies to delete [default: none] apache-buildstream-27ae392/man/bst-artifact-list-contents.1000066400000000000000000000010521514607367700236440ustar00rootroot00000000000000.TH "BST ARTIFACT LIST-CONTENTS" "1" "2025-05-09" "2.5" "bst artifact list-contents Manual" .SH NAME bst\-artifact\-list-contents \- List the contents of an artifact .SH SYNOPSIS .B bst artifact list-contents [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION List the contents of an artifact. .PP Note that 'artifacts' can be element names, which must end in '.bst', or artifact references, which must be in the format `//`. .SH OPTIONS .TP \fB\-l,\fP \-\-long Provide more information about the contents of the artifact. apache-buildstream-27ae392/man/bst-artifact-log.1000066400000000000000000000005461514607367700216260ustar00rootroot00000000000000.TH "BST ARTIFACT LOG" "1" "2025-05-09" "2.5" "bst artifact log Manual" .SH NAME bst\-artifact\-log \- Show logs of artifacts .SH SYNOPSIS .B bst artifact log [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION Show build logs of artifacts .SH OPTIONS .TP \fB\-\-out\fP PATH Output logs to individual files in the specified path. If absent, logs are written to stdout. apache-buildstream-27ae392/man/bst-artifact-pull.1000066400000000000000000000025231514607367700220160ustar00rootroot00000000000000.TH "BST ARTIFACT PULL" "1" "2025-05-09" "2.5" "bst artifact pull Manual" .SH NAME bst\-artifact\-pull \- Pull a built artifact .SH SYNOPSIS .B bst artifact pull [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION Pull a built artifact from the configured remote artifact cache. .PP Specifying no elements will result in pulling the default targets of the project. If no default targets are configured, all project elements will be pulled. .PP When this command is executed from a workspace directory, the default is to pull the workspace element. .PP By default the artifact will be pulled one of the configured caches if possible, following the usual priority order. If the `--artifact-remote` flag is given, only the specified cache will be queried. .PP Specify `--deps` to control which artifacts to pull: .PP  none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies .SH OPTIONS .TP \fB\-d,\fP \-\-deps [build|none|run|all] The dependency artifacts to pull [default: none] .TP \fB\-\-artifact\-remote\fP REMOTE A remote for downloading artifacts .TP \fB\-\-ignore\-project\-artifact\-remotes\fP Ignore remote artifact cache servers recommended by projects apache-buildstream-27ae392/man/bst-artifact-push.1000066400000000000000000000025041514607367700220200ustar00rootroot00000000000000.TH "BST ARTIFACT PUSH" "1" "2025-05-09" "2.5" "bst artifact push Manual" .SH NAME bst\-artifact\-push \- Push a built artifact .SH SYNOPSIS .B bst artifact push [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION Push built artifacts to a remote artifact cache, possibly pulling them first. .PP Specifying no elements will result in pushing the default targets of the project. If no default targets are configured, all project elements will be pushed. .PP When this command is executed from a workspace directory, the default is to push the workspace element. .PP If bst has been configured to include build trees on artifact pulls, an attempt will be made to pull any required build trees to avoid the skipping of partial artifacts being pushed. .PP Specify `--deps` to control which artifacts to push: .PP  none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies .SH OPTIONS .TP \fB\-d,\fP \-\-deps [build|none|run|all] The dependencies to push [default: none] .TP \fB\-\-artifact\-remote\fP REMOTE A remote for uploading artifacts .TP \fB\-\-ignore\-project\-artifact\-remotes\fP Ignore remote artifact cache servers recommended by projects apache-buildstream-27ae392/man/bst-artifact-server.1000066400000000000000000000015071514607367700223510ustar00rootroot00000000000000.TH "BST-ARTIFACT-SERVER" "1" "2020-10-14" "" "bst-artifact-server Manual" .SH NAME bst-artifact-server \- CAS Artifact Server .SH SYNOPSIS .B bst-artifact-server [OPTIONS] REPO .SH OPTIONS .TP \fB\-p,\fP \-\-port INTEGER Port number [required] .TP \fB\-\-server\-key\fP TEXT Private server key for TLS (PEM-encoded) .TP \fB\-\-server\-cert\fP TEXT Public server certificate for TLS (PEM-encoded) .TP \fB\-\-client\-certs\fP TEXT Public client certificates for TLS (PEM-encoded) .TP \fB\-\-enable\-push\fP Allow clients to upload blobs and update artifact cache .TP \fB\-\-quota\fP INTEGER Maximum disk usage in bytes [default: 10000000000.0] .TP \fB\-\-index\-only\fP Only provide the BuildStream artifact and source services ("index"), not the CAS ("storage") .TP \fB\-\-log\-level\fP [warning|info|trace] The log level to launch with apache-buildstream-27ae392/man/bst-artifact-show.1000066400000000000000000000005531514607367700220230ustar00rootroot00000000000000.TH "BST ARTIFACT SHOW" "1" "2025-05-09" "2.5" "bst artifact show Manual" .SH NAME bst\-artifact\-show \- Show the cached state of artifacts .SH SYNOPSIS .B bst artifact show [OPTIONS] [ARTIFACTS]... .SH DESCRIPTION show the cached state of artifacts .SH OPTIONS .TP \fB\-d,\fP \-\-deps [build|run|all|none] The dependencies we also want to show [default: none] apache-buildstream-27ae392/man/bst-artifact.1000066400000000000000000000045141514607367700210460ustar00rootroot00000000000000.TH "BST ARTIFACT" "1" "2025-05-09" "2.5" "bst artifact Manual" .SH NAME bst\-artifact \- Manipulate cached artifacts. .SH SYNOPSIS .B bst artifact [OPTIONS] COMMAND [ARGS]... .SH DESCRIPTION Manipulate cached artifacts .PP Some subcommands take artifact references as arguments. Artifacts can be specified in two ways: .PP  - artifact refs: triples of the form // - element names .PP When elements are given, the artifact is looked up by observing the element and it's current cache key. .PP The commands also support shell-style wildcard expansion: `?` matches a single character, `*` matches zero or more characters but does not match the `/` path separator, and `**` matches zero or more characters including `/` path separators. .PP If the wildcard expression ends with `.bst`, then it will be used to search element names found in the project, otherwise, it will be used to search artifacts that are present in the local artifact cache. .PP Some example arguments are: .PP  - `myproject/hello/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f` - `myproject/he*/827637*` - `core/*.bst` (all elements in the core directory) - `**.bst` (all elements) - `myproject/**` (all artifacts from myproject) - `myproject/myelement/*` (all cached artifacts for a specific element) .SH COMMANDS .PP \fBshow\fP Show the cached state of artifacts See \fBbst artifact-show(1)\fP for full documentation on the \fBshow\fP command. .PP \fBcheckout\fP Checkout contents of an artifact See \fBbst artifact-checkout(1)\fP for full documentation on the \fBcheckout\fP command. .PP \fBpull\fP Pull a built artifact See \fBbst artifact-pull(1)\fP for full documentation on the \fBpull\fP command. .PP \fBpush\fP Push a built artifact See \fBbst artifact-push(1)\fP for full documentation on the \fBpush\fP command. .PP \fBlog\fP Show logs of artifacts See \fBbst artifact-log(1)\fP for full documentation on the \fBlog\fP command. .PP \fBlist-contents\fP List the contents of an artifact See \fBbst artifact-list-contents(1)\fP for full documentation on the \fBlist-contents\fP command. .PP \fBdelete\fP Remove artifacts from the local cache See \fBbst artifact-delete(1)\fP for full documentation on the \fBdelete\fP command. apache-buildstream-27ae392/man/bst-build.1000066400000000000000000000026141514607367700203470ustar00rootroot00000000000000.TH "BST BUILD" "1" "2025-05-09" "2.5" "bst build Manual" .SH NAME bst\-build \- Build elements in a pipeline .SH SYNOPSIS .B bst build [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Build elements in a pipeline .PP Specifying no elements will result in building the default targets of the project. If no default targets are configured, all project elements will be built. .PP When this command is executed from a workspace directory, the default is to build the workspace element. .PP Specify `--deps` to control which dependencies must be built: .PP  none: No dependencies, just the element itself build: Build time dependencies, excluding the element itself all: All dependencies .PP Dependencies that are consequently required to build the requested elements will be built on demand. .SH OPTIONS .TP \fB\-d,\fP \-\-deps [none|build|all] The dependencies to build .TP \fB\-\-artifact\-remote\fP REMOTE A remote for uploading and downloading artifacts .TP \fB\-\-source\-remote\fP REMOTE A remote for uploading and downloading cached sources .TP \fB\-\-ignore\-project\-artifact\-remotes\fP Ignore remote artifact cache servers recommended by projects .TP \fB\-\-ignore\-project\-source\-remotes\fP Ignore remote source cache servers recommended by projects .TP \fB\-r,\fP \-\-retry\-failed Try to build elements for which a failed build artifact is found apache-buildstream-27ae392/man/bst-help.1000066400000000000000000000003171514607367700201760ustar00rootroot00000000000000.TH "BST HELP" "1" "2025-05-09" "2.5" "bst help Manual" .SH NAME bst\-help \- Print usage information .SH SYNOPSIS .B bst help [OPTIONS] COMMAND .SH DESCRIPTION Print usage information about a given command apache-buildstream-27ae392/man/bst-init.1000066400000000000000000000012701514607367700202100ustar00rootroot00000000000000.TH "BST INIT" "1" "2025-05-09" "2.5" "bst init Manual" .SH NAME bst\-init \- Initialize a new BuildStream project .SH SYNOPSIS .B bst init [OPTIONS] [TARGET_DIRECTORY] .SH DESCRIPTION Initialize a new BuildStream project .PP Creates a new BuildStream project.conf in the project directory. .PP Unless `--project-name` is specified, this will be an interactive session. .SH OPTIONS .TP \fB\-\-project\-name\fP TEXT The project name to use .TP \fB\-\-min\-version\fP TEXT The required format version [default: 2.4] .TP \fB\-\-element\-path\fP PATH The subdirectory to store elements in [default: elements] .TP \fB\-f,\fP \-\-force Allow overwriting an existing project.conf apache-buildstream-27ae392/man/bst-shell.1000066400000000000000000000032361514607367700203600ustar00rootroot00000000000000.TH "BST SHELL" "1" "2025-05-09" "2.5" "bst shell Manual" .SH NAME bst\-shell \- Shell into an element's sandbox environment .SH SYNOPSIS .B bst shell [OPTIONS] [TARGET] [COMMAND]... .SH DESCRIPTION Run a command in the target element's sandbox environment .PP When this command is executed from a workspace directory, the default is to shell into the workspace element. .PP This will stage a temporary sysroot for running the target element, assuming it has already been built and all required artifacts are in the local cache. .PP Use '--' to separate a command from the options to bst, otherwise bst may respond to them instead. e.g. .PP  bst shell example.bst -- df -h .PP Use the --build option to create a temporary sysroot for building the element instead. .PP If no COMMAND is specified, the default is to attempt to run an interactive shell. .SH OPTIONS .TP \fB\-b,\fP \-\-build Stage dependencies and sources to build .TP \fB\-\-mount\fP HOSTPATH PATH Mount a file or directory into the sandbox .TP \fB\-\-isolate\fP Create an isolated build sandbox .TP \fB\-t,\fP \-\-use\-buildtree Stage a buildtree. Will fail if a buildtree is not available. pull-buildtrees configuration is needed if the buildtree is not available locally. .TP \fB\-\-artifact\-remote\fP REMOTE A remote for uploading and downloading artifacts .TP \fB\-\-source\-remote\fP REMOTE A remote for uploading and downloading cached sources .TP \fB\-\-ignore\-project\-artifact\-remotes\fP Ignore remote artifact cache servers recommended by projects .TP \fB\-\-ignore\-project\-source\-remotes\fP Ignore remote source cache servers recommended by projects apache-buildstream-27ae392/man/bst-show.1000066400000000000000000000053651514607367700202360ustar00rootroot00000000000000.TH "BST SHOW" "1" "2025-05-09" "2.5" "bst show Manual" .SH NAME bst\-show \- Show elements in the pipeline .SH SYNOPSIS .B bst show [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Show elements in the pipeline .PP Specifying no elements will result in showing the default targets of the project. If no default targets are configured, all project elements will be shown. .PP When this command is executed from a workspace directory, the default is to show the workspace element. .PP By default this will show all of the dependencies of the specified target element. .PP Specify ``--deps`` to control which elements to show: .PP  none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies .PP **FORMAT** .PP The ``--format`` option controls what should be printed for each element, the following symbols can be used in the format string: .PP  %{name} The element name %{description} The element description, on a single line (Since: 2.3) %{key} The abbreviated cache key (if all sources are consistent) %{full-key} The full cache key (if all sources are consistent) %{state} cached, buildable, waiting, inconsistent or junction %{config} The element configuration %{vars} Variable configuration %{env} Environment settings %{public} Public domain data %{workspaced} If the element is workspaced %{workspace-dirs} A list of workspace directories %{deps} A list of all dependencies %{build-deps} A list of build dependencies %{runtime-deps} A list of runtime dependencies %{source-info} Source provenance information %{artifact-cas-digest} The CAS digest of the built artifact .PP The value of the %{symbol} without the leading '%' character is understood as a pythonic formatting string, so python formatting features apply, example: .PP  bst show target.bst --format \ 'Name: %{name: ^20} Key: %{key: ^8} State: %{state}' .PP If you want to use a newline in a format string in bash, use the '$' modifier: .PP  bst show target.bst --format \ $'---------- %{name} ----------\n%{vars}' .SH OPTIONS .TP \fB\-\-except\fP PATH Except certain dependencies .TP \fB\-d,\fP \-\-deps [none|run|build|all] The dependencies to show [default: all] .TP \fB\-\-order\fP [stage|alpha] Staging or alphabetic ordering of dependencies [default: stage] .TP \fB\-f,\fP \-\-format FORMAT Format string for each element apache-buildstream-27ae392/man/bst-source-checkout.1000066400000000000000000000021311514607367700223450ustar00rootroot00000000000000.TH "BST SOURCE CHECKOUT" "1" "2025-05-09" "2.5" "bst source checkout Manual" .SH NAME bst\-source\-checkout \- Checkout sources of an element .SH SYNOPSIS .B bst source checkout [OPTIONS] [ELEMENT] .SH DESCRIPTION Checkout sources of an element to the specified location .PP When this command is executed from a workspace directory, the default is to checkout the sources of the workspace element. .SH OPTIONS .TP \fB\-f,\fP \-\-force Allow files to be overwritten .TP \fB\-\-except\fP PATH Except certain dependencies .TP \fB\-d,\fP \-\-deps [build|none|run|all] The dependencies whose sources to checkout [default: none] .TP \fB\-\-tar\fP LOCATION Create a tarball containing the sources instead of a file tree. .TP \fB\-\-compression\fP [gz|xz|bz2] The compression option of the tarball created. .TP \fB\-\-include\-build\-scripts\fP .PP .TP \fB\-\-directory\fP DIRECTORY The directory to checkout the sources to .TP \fB\-\-source\-remote\fP REMOTE A remote for downloading cached sources .TP \fB\-\-ignore\-project\-source\-remotes\fP Ignore remote source cache servers recommended by projects apache-buildstream-27ae392/man/bst-source-fetch.1000066400000000000000000000023651514607367700216420ustar00rootroot00000000000000.TH "BST SOURCE FETCH" "1" "2025-05-09" "2.5" "bst source fetch Manual" .SH NAME bst\-source\-fetch \- Fetch sources in a pipeline .SH SYNOPSIS .B bst source fetch [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Fetch sources required to build the pipeline .PP Specifying no elements will result in fetching the default targets of the project. If no default targets are configured, all project elements will be fetched. .PP When this command is executed from a workspace directory, the default is to fetch the workspace element. .PP By default this will only try to fetch sources for the specified elements. .PP Specify `--deps` to control which sources to fetch: .PP  none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies .SH OPTIONS .TP \fB\-\-except\fP PATH Except certain dependencies from fetching .TP \fB\-d,\fP \-\-deps [none|build|run|all] The dependencies to fetch [default: none] .TP \fB\-\-source\-remote\fP REMOTE A remote for downloading sources .TP \fB\-\-ignore\-project\-source\-remotes\fP Ignore remote source cache servers recommended by projects apache-buildstream-27ae392/man/bst-source-push.1000066400000000000000000000024201514607367700215200ustar00rootroot00000000000000.TH "BST SOURCE PUSH" "1" "2025-05-09" "2.5" "bst source push Manual" .SH NAME bst\-source\-push \- Push sources in a pipeline .SH SYNOPSIS .B bst source push [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Push sources required to build the pipeline .PP Specifying no elements will result in pushing the sources of the default targets of the project. If no default targets are configured, sources of all project elements will be pushed. .PP When this command is executed from a workspace directory, the default is to push the sources of the workspace element. .PP By default this will only try to push sources for the specified elements. .PP Specify `--deps` to control which sources to fetch: .PP  none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies .SH OPTIONS .TP \fB\-\-except\fP PATH Except certain dependencies from pushing .TP \fB\-d,\fP \-\-deps [none|build|run|all] The dependencies to push [default: none] .TP \fB\-\-source\-remote\fP REMOTE A remote for uploading sources .TP \fB\-\-ignore\-project\-source\-remotes\fP Ignore remote source cache servers recommended by projects apache-buildstream-27ae392/man/bst-source-track.1000066400000000000000000000024001514607367700216430ustar00rootroot00000000000000.TH "BST SOURCE TRACK" "1" "2025-05-09" "2.5" "bst source track Manual" .SH NAME bst\-source\-track \- Track new source references .SH SYNOPSIS .B bst source track [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Consults the specified tracking branches for new versions available to build and updates the project with any newly available references. .PP Specifying no elements will result in tracking the default targets of the project. If no default targets are configured, all project elements will be tracked. .PP When this command is executed from a workspace directory, the default is to track the workspace element. .PP If no default is declared, all elements in the project will be tracked .PP By default this will track just the specified element, but you can also update a whole tree of dependencies in one go. .PP Specify `--deps` to control which sources to track: .PP  none: No dependencies, just the specified elements all: All dependencies of all specified elements .SH OPTIONS .TP \fB\-\-except\fP PATH Except certain dependencies from tracking .TP \fB\-d,\fP \-\-deps [build|run|all|none] The dependencies to track [default: none] .TP \fB\-J,\fP \-\-cross\-junctions Allow crossing junction boundaries apache-buildstream-27ae392/man/bst-source.1000066400000000000000000000013701514607367700205460ustar00rootroot00000000000000.TH "BST SOURCE" "1" "2025-05-09" "2.5" "bst source Manual" .SH NAME bst\-source \- Manipulate sources for an element .SH SYNOPSIS .B bst source [OPTIONS] COMMAND [ARGS]... .SH DESCRIPTION Manipulate sources for an element .SH COMMANDS .PP \fBfetch\fP Fetch sources in a pipeline See \fBbst source-fetch(1)\fP for full documentation on the \fBfetch\fP command. .PP \fBpush\fP Push sources in a pipeline See \fBbst source-push(1)\fP for full documentation on the \fBpush\fP command. .PP \fBtrack\fP Track new source references See \fBbst source-track(1)\fP for full documentation on the \fBtrack\fP command. .PP \fBcheckout\fP Checkout sources of an element See \fBbst source-checkout(1)\fP for full documentation on the \fBcheckout\fP command. apache-buildstream-27ae392/man/bst-workspace-close.1000066400000000000000000000005511514607367700223470ustar00rootroot00000000000000.TH "BST WORKSPACE CLOSE" "1" "2025-05-09" "2.5" "bst workspace close Manual" .SH NAME bst\-workspace\-close \- Close workspaces .SH SYNOPSIS .B bst workspace close [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Close a workspace .SH OPTIONS .TP \fB\-\-remove\-dir\fP Remove the path that contains the closed workspace .TP \fB\-a,\fP \-\-all Close all open workspaces apache-buildstream-27ae392/man/bst-workspace-list.1000066400000000000000000000003241514607367700222130ustar00rootroot00000000000000.TH "BST WORKSPACE LIST" "1" "2025-05-09" "2.5" "bst workspace list Manual" .SH NAME bst\-workspace\-list \- List open workspaces .SH SYNOPSIS .B bst workspace list [OPTIONS] .SH DESCRIPTION List open workspaces apache-buildstream-27ae392/man/bst-workspace-open.1000066400000000000000000000014751514607367700222110ustar00rootroot00000000000000.TH "BST WORKSPACE OPEN" "1" "2025-05-09" "2.5" "bst workspace open Manual" .SH NAME bst\-workspace\-open \- Open a new workspace .SH SYNOPSIS .B bst workspace open [OPTIONS] ELEMENTS... .SH DESCRIPTION Open a workspace for manual source modification .SH OPTIONS .TP \fB\-\-no\-checkout\fP Do not checkout the source, only link to the given directory .TP \fB\-f,\fP \-\-force The workspace will be created even if the directory in which it will be created is not empty or if a workspace for that element already exists .TP \fB\-\-directory\fP DIRECTORY Only for use when a single Element is given: Set the directory to use to create the workspace .TP \fB\-\-source\-remote\fP REMOTE A remote for downloading cached sources .TP \fB\-\-ignore\-project\-source\-remotes\fP Ignore remote source cache servers recommended by projects apache-buildstream-27ae392/man/bst-workspace-reset.1000066400000000000000000000007101514607367700223610ustar00rootroot00000000000000.TH "BST WORKSPACE RESET" "1" "2025-05-09" "2.5" "bst workspace reset Manual" .SH NAME bst\-workspace\-reset \- Reset a workspace to its original state .SH SYNOPSIS .B bst workspace reset [OPTIONS] [ELEMENTS]... .SH DESCRIPTION Reset a workspace to its original state .SH OPTIONS .TP \fB\-\-soft\fP Mark workspace to re-execute configuration steps (if any) on next build. Does not alter workspace contents. .TP \fB\-a,\fP \-\-all Reset all open workspaces apache-buildstream-27ae392/man/bst-workspace.1000066400000000000000000000013611514607367700212440ustar00rootroot00000000000000.TH "BST WORKSPACE" "1" "2025-05-09" "2.5" "bst workspace Manual" .SH NAME bst\-workspace \- Manipulate developer workspaces .SH SYNOPSIS .B bst workspace [OPTIONS] COMMAND [ARGS]... .SH DESCRIPTION Manipulate developer workspaces .SH COMMANDS .PP \fBopen\fP Open a new workspace See \fBbst workspace-open(1)\fP for full documentation on the \fBopen\fP command. .PP \fBclose\fP Close workspaces See \fBbst workspace-close(1)\fP for full documentation on the \fBclose\fP command. .PP \fBreset\fP Reset a workspace to its original state See \fBbst workspace-reset(1)\fP for full documentation on the \fBreset\fP command. .PP \fBlist\fP List open workspaces See \fBbst workspace-list(1)\fP for full documentation on the \fBlist\fP command. apache-buildstream-27ae392/man/bst.1000066400000000000000000000057301514607367700172540ustar00rootroot00000000000000.TH "BST" "1" "2025-05-09" "2.5" "bst Manual" .SH NAME bst \- Build and manipulate BuildStream projects .SH SYNOPSIS .B bst [OPTIONS] COMMAND [ARGS]... .SH DESCRIPTION Build and manipulate BuildStream projects .PP Most of the main options override options in the user preferences configuration file. .SH OPTIONS .TP \fB\-\-version\fP .PP .TP \fB\-c,\fP \-\-config FILE Configuration file to use .TP \fB\-C,\fP \-\-directory DIRECTORY Project directory (default: current directory) .TP \fB\-\-on\-error\fP [continue|quit|terminate] What to do when an error is encountered .TP \fB\-\-fetchers\fP INTEGER Maximum simultaneous download tasks .TP \fB\-\-builders\fP INTEGER Maximum simultaneous build tasks .TP \fB\-\-pushers\fP INTEGER Maximum simultaneous upload tasks .TP \fB\-\-max\-jobs\fP INTEGER Number of parallel jobs allowed for a given build task .TP \fB\-\-network\-retries\fP INTEGER Maximum retries for network tasks .TP \fB\-\-no\-interactive\fP Force non interactive mode, otherwise this is automatically decided .TP \fB\-\-verbose\fP / \-\-no\-verbose Be extra verbose .TP \fB\-\-debug\fP / \-\-no\-debug Print debugging output .TP \fB\-\-error\-lines\fP INTEGER Maximum number of lines to show from a task log .TP \fB\-\-message\-lines\fP INTEGER Maximum number of lines to show in a detailed message .TP \fB\-\-log\-file\fP FILENAME A file to store the main log (allows storing the main log while in interactive mode) .TP \fB\-\-colors\fP / \-\-no\-colors Force enable/disable ANSI color codes in output .TP \fB\-\-strict\fP / \-\-no\-strict Elements must be rebuilt when their dependencies have changed .TP \fB\-o,\fP \-\-option OPTION VALUE Specify a project option .TP \fB\-\-default\-mirror\fP TEXT The mirror to fetch from first, before attempting other mirrors .TP \fB\-\-pull\-buildtrees\fP Include an element's build tree when pulling remote element artifacts .TP \fB\-\-cache\-buildtrees\fP [always|auto|never] Cache artifact build tree content on creation .SH COMMANDS .PP \fBhelp\fP Print usage information See \fBbst-help(1)\fP for full documentation on the \fBhelp\fP command. .PP \fBinit\fP Initialize a new BuildStream project See \fBbst-init(1)\fP for full documentation on the \fBinit\fP command. .PP \fBbuild\fP Build elements in a pipeline See \fBbst-build(1)\fP for full documentation on the \fBbuild\fP command. .PP \fBshow\fP Show elements in the pipeline See \fBbst-show(1)\fP for full documentation on the \fBshow\fP command. .PP \fBshell\fP Shell into an element's sandbox environment See \fBbst-shell(1)\fP for full documentation on the \fBshell\fP command. .PP \fBsource\fP Manipulate sources for an element See \fBbst-source(1)\fP for full documentation on the \fBsource\fP command. .PP \fBworkspace\fP Manipulate developer workspaces See \fBbst-workspace(1)\fP for full documentation on the \fBworkspace\fP command. .PP \fBartifact\fP Manipulate cached artifacts. See \fBbst-artifact(1)\fP for full documentation on the \fBartifact\fP command. apache-buildstream-27ae392/pyproject.toml000066400000000000000000000036421514607367700205430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # [build-system] requires = [ # We need at least version 41.2.0 to support python 3.9 "setuptools>=41.2.0", # In order to build wheels, and as required by PEP 517 "wheel", # Require a new enough cython for the python versions we support "Cython>=0.29.25", "packaging", ] build-backend = "setuptools.build_meta" [tool.black] line-length = 119 exclude = ''' ( /( \.eggs | \.git | \.mypy_cache | \.tox | _build | build | dist )/ | src/buildstream/_protos ) ''' [tool.cibuildwheel] build-frontend = "build" environment = { BST_BUNDLE_BUILDBOX = "1" } # The BuildBox binaries produced in buildbox-integration are linked against GLIBC 2.28 # from Rocky Linux 8.10. See: https://gitlab.com/BuildGrid/buildbox/buildbox-integration. # # The PyPA manylinux_2_28 platform tag identifies that our wheel will run on any x86_64 # OS with GLIBC >= 2.28. Following this setting, `cibuildwheel` builds the packages in # the corresponding manylinux_2_28 container image. See: https://github.com/pypa/manylinux manylinux-x86_64-image = "manylinux_2_28" skip = [ # PyPy may work, but nobody is testing it so avoid distributing prebuilt binaries. "pp*", # Skipping this niche archicture ~halves overall build time. "*_i686", # The prebuilt BuildBox binaries link against GLibc so will work on manylinux but not musllinux "*-musllinux_*", ] apache-buildstream-27ae392/requirements/000077500000000000000000000000001514607367700203455ustar00rootroot00000000000000apache-buildstream-27ae392/requirements/Makefile000066400000000000000000000007401514607367700220060ustar00rootroot00000000000000# Makefile for updating BuildStream's requirements files. # REQUIREMENTS_IN := $(wildcard *.in) REQUIREMENTS_TXT := $(REQUIREMENTS_IN:.in=.txt) PYTHON := python3 VENV := $(PYTHON) -m venv VENV_PIP = $(VENVDIR)/bin/pip .PHONY: all FORCE all: $(REQUIREMENTS_TXT) %.txt: %.in FORCE $(eval VENVDIR := $(shell mktemp -d $(CURDIR)/.bst-venv.XXXXXX)) $(VENV) $(VENVDIR) $(VENV_PIP) install -r $< $(VENV_PIP) freeze -r $< | grep -v pkg-resources > $@ rm -rf $(VENVDIR) FORCE: apache-buildstream-27ae392/requirements/cov-requirements.in000066400000000000000000000000711514607367700242030ustar00rootroot00000000000000coverage >= 6 pytest-cov >= 2.5.0 pytest >= 6.0.1 Cython apache-buildstream-27ae392/requirements/cov-requirements.txt000066400000000000000000000003641514607367700244210ustar00rootroot00000000000000coverage==7.11.0 pytest-cov==7.0.0 pytest==8.4.2 Cython==3.1.6 ## The following requirements were added by pip freeze: exceptiongroup==1.3.0 iniconfig==2.3.0 packaging==25.0 pluggy==1.6.0 Pygments==2.19.2 tomli==2.3.0 typing_extensions==4.15.0 apache-buildstream-27ae392/requirements/dev-requirements.in000066400000000000000000000002251514607367700241730ustar00rootroot00000000000000pexpect pylint # Pytest 6.0.0 doesn't play well with pylint pytest >= 6.0.1 pytest-datafiles >= 3.0 pytest-env pytest-xdist pytest-timeout pyftpdlib apache-buildstream-27ae392/requirements/dev-requirements.txt000066400000000000000000000007551514607367700244140ustar00rootroot00000000000000pexpect==4.9.0 pylint==4.0.2 # Pytest 6.0.0 doesn't play well with pylint pytest==8.4.2 pytest-datafiles==3.0.0 pytest-env==1.2.0 pytest-xdist==3.8.0 pytest-timeout==2.4.0 pyftpdlib==2.1.0 ## The following requirements were added by pip freeze: astroid==4.0.1 dill==0.4.0 exceptiongroup==1.3.0 execnet==2.1.1 iniconfig==2.3.0 isort==7.0.0 mccabe==0.7.0 packaging==25.0 platformdirs==4.5.0 pluggy==1.6.0 ptyprocess==0.7.0 Pygments==2.19.2 tomli==2.3.0 tomlkit==0.13.3 typing_extensions==4.15.0 apache-buildstream-27ae392/requirements/requirements.in000066400000000000000000000002341514607367700234170ustar00rootroot00000000000000Click >= 7.0, != 8.2.2 grpcio Jinja2 >= 2.10 packaging pluginbase protobuf <9,>=5.29 psutil ruamel.yaml >= 0.16.7 ruamel.yaml.clib >= 0.1.2 pyroaring ujson apache-buildstream-27ae392/requirements/requirements.txt000066400000000000000000000004341514607367700236320ustar00rootroot00000000000000click==8.3.0 grpcio==1.76.0 Jinja2==3.1.6 packaging==25.0 pluginbase==1.0.1 protobuf==6.33.0 psutil==7.1.3 ruamel.yaml==0.18.16 ruamel.yaml.clib==0.2.14 pyroaring==1.0.3 ujson==5.11.0 ## The following requirements were added by pip freeze: MarkupSafe==3.0.3 typing_extensions==4.15.0 apache-buildstream-27ae392/setup.cfg000066400000000000000000000034771514607367700174560ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # [options] python_requires = >=3.10 [versioneer] VCS = git style = pep440 versionfile_source = src/buildstream/_version.py versionfile_build = buildstream/_version.py tag_prefix = tag_regex = *.*.* parentdir_prefix = BuildStream- [tool:pytest] addopts = --verbose --basetemp ./tmp --durations=20 --timeout=1800 testpaths = tests norecursedirs = src tests/integration/project tests/plugins/loading tests/plugins/sample-plugins integration-cache tmp __pycache__ .eggs python_files = tests/*/*.py env = D:BST_TEST_SUITE=True D:GRPC_ENABLE_FORK_SUPPORT=0 markers = datafiles: data files for tests integration: run test only if --integration option is specified remoteexecution: run test only if --remote-execution option is specified remotecache: run tests only if --remote-cache option is specified xfail_strict=True [mypy] files = src warn_unused_configs = True warn_no_return = True # Ignore missing stubs for third-party packages. # In future, these should be re-enabled if/when stubs for them become available. [mypy-copyreg,grpc,pluginbase,psutil,pyroaring,ruamel,multiprocessing.forkserver,pkg_resources.extern] ignore_missing_imports=True # Ignore issues with generated files and vendored code [mypy-buildstream._protos.*,buildstream._version] ignore_errors = True apache-buildstream-27ae392/setup.py000077500000000000000000000333011514607367700173370ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Benjamin Schubert import os from pathlib import Path import re import sys ################################### # Ensure we have a version number # ################################### # Add local directory to the path, in order to be able to import versioneer sys.path.append(os.path.dirname(__file__)) import versioneer # pylint: disable=wrong-import-position version = versioneer.get_version() if version.startswith("0+untagged"): print( "Your git repository has no tags - BuildStream can't determine its version. Please run `git fetch --tags`.", file=sys.stderr, ) sys.exit(1) ################################################################## # Python requirements ################################################################## REQUIRED_PYTHON_MAJOR = 3 REQUIRED_PYTHON_MINOR = 10 if sys.version_info[0] != REQUIRED_PYTHON_MAJOR or sys.version_info[1] < REQUIRED_PYTHON_MINOR: print("BuildStream requires Python >= " + str(REQUIRED_PYTHON_MAJOR) + "." + str(REQUIRED_PYTHON_MINOR)) sys.exit(1) try: from setuptools import setup, find_packages, Command, Extension except ImportError: print( "BuildStream requires setuptools in order to build. Install it using" " your package manager (usually python3-setuptools) or via pip (pip3" " install setuptools)." ) sys.exit(1) ############################################################ # List the BuildBox binaries to ship in the wheel packages # ############################################################ # # BuildBox isn't widely available in OS distributions. To enable a "one click" # install for BuildStream, we bundle prebuilt BuildBox binaries in our binary # wheel packages. # # The binaries are provided by the buildbox-integration Gitlab project: # https://gitlab.com/BuildGrid/buildbox/buildbox-integration # # If you want to build a wheel with the BuildBox binaries included, set the # env var "BST_BUNDLE_BUILDBOX=1" when running setup.py. try: BUNDLE_BUILDBOX = int(os.environ.get("BST_BUNDLE_BUILDBOX", "0")) except ValueError: print("BST_BUNDLE_BUILDBOX must be an integer. Please set it to '1' to enable, '0' to disable", file=sys.stderr) raise SystemExit(1) def list_buildbox_binaries(): expected_binaries = [ "buildbox-casd", "buildbox-fuse", "buildbox-run", ] if BUNDLE_BUILDBOX: bst_package_dir = Path(__file__).parent.joinpath("src/buildstream") buildbox_dir = bst_package_dir.joinpath("subprojects", "buildbox") buildbox_binaries = [buildbox_dir.joinpath(name) for name in expected_binaries] missing_binaries = [path for path in buildbox_binaries if not path.is_file()] if missing_binaries: paths_text = "\n".join([" * {}".format(path) for path in missing_binaries]) print( "Expected BuildBox binaries were not found. " "Set BST_BUNDLE_BUILDBOX=0 or provide:\n\n" "{}\n".format(paths_text), file=sys.stderr, ) raise SystemExit(1) for path in buildbox_binaries: if path.is_symlink(): print( "Bundled BuildBox binaries must not be symlinks. Please fix {}".format(path), file=sys.stderr, ) raise SystemExit(1) return [str(path.relative_to(bst_package_dir)) for path in buildbox_binaries] else: return [] ########################################### # List the pre-built man pages to install # ########################################### # # Man pages are automatically generated however it was too difficult # to integrate with setuptools as a step of the build (FIXME !). # # To update the man pages in tree before a release, run: # # tox -e man # # Then commit the result. # def list_man_pages(): bst_dir = os.path.dirname(os.path.abspath(__file__)) man_dir = os.path.join(bst_dir, "man") try: man_pages = os.listdir(man_dir) return [os.path.join("man", page) for page in man_pages] except FileNotFoundError: # Do not error out when 'man' directory does not exist return [] ###################################################### # List the data files needed by buildstream._testing # ###################################################### # # List the datafiles which need to be installed for the # buildstream._testing package # def list_testing_datafiles(): bst_dir = Path(os.path.dirname(os.path.abspath(__file__))) data_dir = bst_dir.joinpath("src", "buildstream", "_testing", "_sourcetests", "project") return [str(f) for f in data_dir.rglob("*")] ##################################################### # gRPC command for code generation # ##################################################### class BuildGRPC(Command): """Command to generate project *_pb2.py modules from proto files.""" description = "build gRPC protobuf modules" user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): try: import grpc_tools.command except ImportError: print( "BuildStream requires grpc_tools in order to build gRPC modules.\n" "Install it via pip (pip3 install grpcio-tools)." ) sys.exit(1) protos_root = "src/buildstream/_protos" grpc_tools.command.build_package_protos(protos_root) # Postprocess imports in generated code for root, _, files in os.walk(protos_root): for filename in files: if filename.endswith(".py"): path = os.path.join(root, filename) with open(path, "r", encoding="utf-8") as f: code = f.read() # All protos are in buildstream._protos code = re.sub(r"^from ", r"from buildstream._protos.", code, flags=re.MULTILINE) # Except for the core google.protobuf protos code = re.sub( r"^from buildstream._protos.google.protobuf", r"from google.protobuf", code, flags=re.MULTILINE ) with open(path, "w", encoding="utf-8") as f: f.write(code) def get_cmdclass(): cmdclass = { "build_grpc": BuildGRPC, } cmdclass.update(versioneer.get_cmdclass()) return cmdclass ##################################################### # Gather requirements # ##################################################### with open("requirements/requirements.in", encoding="utf-8") as install_reqs: install_requires = install_reqs.read().splitlines() ##################################################### # Prepare package description from README # ##################################################### with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "README.rst"), encoding="utf-8") as readme: long_description = readme.read() ##################################################### # Setup Cython and extensions # ##################################################### try: ENABLE_CYTHON_TRACE = int(os.environ.get("BST_CYTHON_TRACE", "0")) except ValueError: print("BST_CYTHON_TRACE must be an integer. Please set it to '1' to enable, '0' to disable", file=sys.stderr) raise SystemExit(1) extension_macros = [("CYTHON_TRACE", ENABLE_CYTHON_TRACE)] def cythonize(extensions, **kwargs): # We want to make sure that generated Cython code is never # included in the source distribution. # # This is because Cython will generate some code which accesses # internal API from CPython, as such we cannot guarantee that # the C code we generated when creating the distribution will # be valid for the target CPython interpretor. # if "sdist" in sys.argv: return extensions try: from Cython.Build import cythonize as _cythonize except ImportError: print( "Cython is required when building BuildStream from sources. " "Please install it using your package manager (usually 'python3-cython') " "or pip (pip install cython).", file=sys.stderr, ) raise SystemExit(1) return _cythonize(extensions, **kwargs) def register_cython_module(module_name, dependencies=None): def files_from_module(modname): basename = "src/{}".format(modname.replace(".", "/")) return "{}.pyx".format(basename), "{}.pxd".format(basename) if dependencies is None: dependencies = [] implementation_file, definition_file = files_from_module(module_name) assert os.path.exists(implementation_file) depends = [] if os.path.exists(definition_file): depends.append(definition_file) for module in dependencies: imp_file, def_file = files_from_module(module) assert os.path.exists(imp_file), "Dependency file not found: {}".format(imp_file) assert os.path.exists(def_file), "Dependency declaration file not found: {}".format(def_file) depends.append(imp_file) depends.append(def_file) BUILD_EXTENSIONS.append( Extension( name=module_name, sources=[implementation_file], depends=depends, define_macros=extension_macros, ) ) BUILD_EXTENSIONS = [] register_cython_module("buildstream.node") register_cython_module("buildstream._loader.loadelement", dependencies=["buildstream.node"]) register_cython_module("buildstream._yaml", dependencies=["buildstream.node"]) register_cython_module("buildstream._types") register_cython_module("buildstream._utils") register_cython_module("buildstream._variables", dependencies=["buildstream.node"]) ##################################################### # Main setup() Invocation # ##################################################### setup( name="BuildStream", version=version, cmdclass=get_cmdclass(), author="The Apache Software Foundation", author_email="dev@buildstream.apache.org", classifiers=[ "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: POSIX", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Build Tools", ], description="A framework for modelling build pipelines in YAML", license="Apache License Version 2.0", long_description=long_description, long_description_content_type="text/x-rst; charset=UTF-8", url="https://buildstream.build", project_urls={ "Source": "https://github.com/apache/buildstream", "Documentation": "https://docs.buildstream.build", "Tracker": "https://github.com/apache/buildstream/issues", "Mailing List": "https://lists.apache.org/list.html?dev@buildstream.apache.org", }, python_requires="~={}.{}".format(REQUIRED_PYTHON_MAJOR, REQUIRED_PYTHON_MINOR), package_dir={"": "src"}, packages=find_packages(where="src", exclude=("subprojects", "tests", "tests.*")), package_data={ "buildstream": [ "py.typed", "plugins/*/*.py", "plugins/*/*.yaml", "data/*.yaml", "data/*.sh.in", *list_buildbox_binaries(), *list_testing_datafiles(), ] }, data_files=[ # This is a weak attempt to integrate with the user nicely, # installing things outside of the python package itself with pip is # not recommended, but there seems to be no standard structure for # addressing this; so just installing this here. # # These do not get installed in developer mode (`pip install --user -e .`) # # The completions are ignored by bash unless it happens to be installed # in the right directory; this is more like a weak statement that we # attempt to install bash completion scriptlet. # ("share/man/man1", list_man_pages()), ("share/bash-completion/completions", [os.path.join("src", "buildstream", "data", "bst")]), ], install_requires=install_requires, entry_points={"console_scripts": ["bst = buildstream._frontend:cli"]}, ext_modules=cythonize( BUILD_EXTENSIONS, compiler_directives={ # Version of python to use # https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#arguments "language_level": "3", # Enable line tracing when requested only, this is needed in order to generate coverage. "linetrace": bool(ENABLE_CYTHON_TRACE), "profile": os.environ.get("BST_CYTHON_PROFILE", False), }, ), zip_safe=False, ) apache-buildstream-27ae392/src/000077500000000000000000000000001514607367700164115ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/000077500000000000000000000000001514607367700207245ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/__init__.py000066400000000000000000000032411514607367700230350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Plugin author facing APIs import os if "_BST_COMPLETION" not in os.environ: # Special sauce to get the version from versioneer from ._version import get_versions __version__ = get_versions()["version"] del get_versions from .utils import UtilError, ProgramNotFoundError from .sandbox import Sandbox, SandboxCommandError from .storage import Directory, DirectoryError, FileType, FileStat from .types import CoreWarnings, OverlapAction, FastEnum, SourceRef from .node import MappingNode, Node, ProvenanceInformation, ScalarNode, SequenceNode from .plugin import Plugin from .source import ( Source, SourceError, SourceFetcher, SourceInfo, SourceInfoMedium, SourceVersionType, ) from .sourcemirror import SourceMirror, SourceMirrorError from .downloadablefilesource import DownloadableFileSource from .element import Element, ElementError, DependencyConfiguration from .buildelement import BuildElement from .scriptelement import ScriptElement apache-buildstream-27ae392/src/buildstream/__main__.py000066400000000000000000000022141514607367700230150ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ################################################################## # Private Entry Point # ################################################################## # # This allows running the cli when BuildStream is uninstalled, # as long as BuildStream repo is in PYTHONPATH, one can run it # with: # # python3 -m buildstream [program args] # # This is used when we need to run BuildStream before installing, # like when we build documentation. # if __name__ == "__main__": # pylint: disable=no-value-for-parameter from ._frontend.cli import cli cli() apache-buildstream-27ae392/src/buildstream/_artifact.py000066400000000000000000000612641514607367700232430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tom Pollard # Tristan Van Berkom """ Artifact ========= Implementation of the Artifact class which aims to 'abstract' direct artifact composite interaction away from Element class """ import os import tempfile from typing import Dict, Tuple from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto from . import _yaml from . import utils from .node import Node from .types import _Scope from .storage._casbaseddirectory import CasBasedDirectory from .sandbox._config import SandboxConfig from ._variables import Variables # An Artifact class to abstract artifact operations # from the Element class # # Args: # element (Element): The Element object # context (Context): The BuildStream context # strong_key (str): The elements strong cache key, dependent on context # strict_key (str): The elements strict cache key # weak_key (str): The elements weak cache key # class Artifact: version = 2 def __init__(self, element, context, *, strong_key=None, strict_key=None, weak_key=None): self._element = element self._context = context self._cache_key = strong_key self._strict_key = strict_key self._weak_cache_key = weak_key self._artifactdir = context.artifactdir self._cas = context.get_cascache() self._tmpdir = context.tmpdir self._proto = None self._metadata_keys = None # Strong, strict and weak key tuple extracted from the artifact self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact self._metadata_workspaced_dependencies = None # List of which dependencies are workspaced from the artifact self._cached = None # Boolean of whether the artifact is cached # strong_key(): # # A property which evaluates to the strong key, regardless of whether # it was the strong key that the Artifact object was initialized with # or whether it was the strong key loaded from artifact metadata. # @property def strong_key(self) -> str: if self.cached(): key, _, _ = self.get_metadata_keys() else: key = self._cache_key return key # strict_key(): # # A property which evaluates to the strict key, regardless of whether # it was the strict key that the Artifact object was initialized with # or whether it was the strict key loaded from artifact metadata. # @property def strict_key(self) -> str: if self.cached(): _, key, _ = self.get_metadata_keys() else: key = self._strict_key return key # weak_key(): # # A property which evaluates to the weak key, regardless of whether # it was the weak key that the Artifact object was initialized with # or whether it was the weak key loaded from artifact metadata. # @property def weak_key(self) -> str: if self.cached(): _, _, key = self.get_metadata_keys() else: key = self._weak_cache_key return key # get_files(): # # Get a virtual directory for the artifact files content # # Returns: # (Directory): The virtual directory object # def get_files(self): files_digest = self._get_field_digest("files") return CasBasedDirectory(self._cas, digest=files_digest) # get_buildroot(): # # Get a virtual directory for the artifact buildroot content # # Returns: # (Directory): The virtual directory object # def get_buildroot(self): buildroot_digest = self._get_field_digest("buildroot") return CasBasedDirectory(self._cas, digest=buildroot_digest) # get_buildtree(): # # Get a virtual directory for the artifact buildtree content # # Returns: # (Directory): The virtual directory object # def get_buildtree(self): buildtree_digest = self._get_field_digest("buildtree") return CasBasedDirectory(self._cas, digest=buildtree_digest) # get_sources(): # # Get a virtual directory for the artifact sources # # Returns: # (Directory): The virtual directory object # def get_sources(self): sources_digest = self._get_field_digest("sources") return CasBasedDirectory(self._cas, digest=sources_digest) # get_logs(): # # Get the paths of the artifact's logs # # Returns: # (list): A list of object paths # def get_logs(self): artifact = self._get_proto() logfile_paths = [] for logfile in artifact.logs: logfile_paths.append(self._cas.objpath(logfile.digest)) return logfile_paths # get_extract_key(): # # Get the key used to extract the artifact # # Returns: # (str): The key # def get_extract_key(self): return self._cache_key or self._weak_cache_key # cache(): # # Create the artifact and commit to cache # # Args: # buildrootvdir (Directory): The root directory of the build sandbox # sandbox_build_dir (Directory): Virtual Directory object for the sandbox build-root # collectvdir (Directory): Virtual Directoy object from within the sandbox for collection # sourcesvdir (Directory): Virtual Directoy object for the staged sources # buildresult (tuple): bool, short desc and detailed desc of result # publicdata (dict): dict of public data to commit to artifact metadata # variables (Variables): The element's Variables # environment (dict): dict of the element's environment variables # sandboxconfig (SandboxConfig): The element's SandboxConfig # buildsandbox (Sandbox): The element's configured build sandbox # def cache( self, *, buildrootvdir, sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata, variables, environment, sandboxconfig, buildsandbox, ): context = self._context element = self._element size = 0 filesvdir = None buildtreevdir = None artifact = ArtifactProto() artifact.version = self.version # Store result artifact.build_success = buildresult[0] artifact.build_error = buildresult[1] artifact.build_error_details = "" if not buildresult[2] else buildresult[2] # Store keys artifact.strong_key = self._cache_key artifact.strict_key = self._strict_key artifact.weak_key = self._weak_cache_key artifact.was_workspaced = bool(element._get_workspace()) properties = ["mtime"] if artifact.was_workspaced else [] # Store files if collectvdir is not None: filesvdir = CasBasedDirectory(cas_cache=self._cas) filesvdir._import_files_internal(collectvdir, properties=properties, collect_result=False) artifact.files.CopyFrom(filesvdir._get_digest()) size += filesvdir._get_size() with tempfile.TemporaryDirectory() as tmpdir: files_to_capture = [] # Store public data tmpname = os.path.join(tmpdir, "public_data") _yaml.roundtrip_dump(publicdata, tmpname) files_to_capture.append((tmpname, artifact.public_data)) # Store low diversity metadata, this metadata must have a high # probability of deduplication, such as environment variables # and SandboxConfig. # sandbox_dict = sandboxconfig.to_dict() low_diversity_dict = {"environment": environment, "sandbox-config": sandbox_dict} low_diversity_node = Node.from_dict(low_diversity_dict) tmpname = os.path.join(tmpdir, "low_diversity_meta") _yaml.roundtrip_dump(low_diversity_node, tmpname) files_to_capture.append((tmpname, artifact.low_diversity_meta)) # Store high diversity metadata, this metadata is expected to diverge # for every element and as such cannot be deduplicated. # # The Variables object supports being converted directly to a dictionary variables_dict = dict(variables) high_diversity_dict = {"variables": variables_dict} high_diversity_node = Node.from_dict(high_diversity_dict) tmpname = os.path.join(tmpdir, "high_diversity_meta") _yaml.roundtrip_dump(high_diversity_node, tmpname) files_to_capture.append((tmpname, artifact.high_diversity_meta)) # Store log file log_filename = context.messenger.get_log_filename() if log_filename: log = artifact.logs.add() log.name = os.path.basename(log_filename) files_to_capture.append((log_filename, log.digest)) # Capture queued files and store returned digests digests = self._cas.add_objects(paths=[entry[0] for entry in files_to_capture]) # add_objects() should guarantee this. # `zip(..., strict=True)` could be used in Python 3.10+ assert len(files_to_capture) == len(digests) for entry, digest in zip(files_to_capture, digests): entry[1].CopyFrom(digest) # store build dependencies for e in element._dependencies(_Scope.BUILD): new_build = artifact.build_deps.add() new_build.project_name = e.project_name new_build.element_name = e.name new_build.cache_key = e._get_cache_key() new_build.was_workspaced = bool(e._get_workspace()) # Store build tree if sandbox_build_dir is not None: buildtreevdir = CasBasedDirectory(cas_cache=self._cas) buildtreevdir._import_files_internal(sandbox_build_dir, properties=properties, collect_result=False) artifact.buildtree.CopyFrom(buildtreevdir._get_digest()) # Store sources if sourcesvdir is not None: artifact.sources.CopyFrom(sourcesvdir._get_digest()) # Store build root if buildrootvdir is not None: rootvdir = CasBasedDirectory(cas_cache=self._cas) rootvdir._import_files_internal(buildrootvdir, properties=properties, collect_result=False) artifact.buildroot.CopyFrom(rootvdir._get_digest()) if buildsandbox is not None: sandbox_env = buildsandbox._get_configured_environment() if sandbox_env: for key, value in sorted(sandbox_env.items()): artifact.buildsandbox.environment.add(name=key, value=value) for directory in buildsandbox._get_marked_directories(): artifact.buildsandbox.marked_directories.append(directory) artifact.buildsandbox.working_directory = buildsandbox._get_work_directory() for subsandbox in buildsandbox._get_subsandboxes(): vdir = subsandbox.get_virtual_directory() digest = artifact.buildsandbox.subsandbox_digests.add() digest.CopyFrom(vdir._get_digest()) os.makedirs(os.path.dirname(os.path.join(self._artifactdir, element.get_artifact_name())), exist_ok=True) keys = utils._deduplicate([self._cache_key, self._weak_cache_key]) for key in keys: path = os.path.join(self._artifactdir, element.get_artifact_name(key=key)) with utils.save_file_atomic(path, mode="wb") as f: f.write(artifact.SerializeToString()) # cached_buildroot() # # Check if artifact is cached with expected buildroot. A # buildroot will not be present if the rest of the partial artifact # is not cached. # # Returns: # (bool): True if artifact cached with buildroot, False if # missing expected buildroot. Note this only confirms # if a buildroot is present, not its contents. # def cached_buildroot(self): buildroot_digest = self._get_field_digest("buildroot") if buildroot_digest: return self._cas.contains_directory(buildroot_digest) else: return False # buildroot_exists() # # Check if artifact was created with a buildroot. This does not check # whether the buildroot is present in the local cache. # # Returns: # (bool): True if artifact was created with buildroot # def buildroot_exists(self): artifact = self._get_proto() return bool(str(artifact.buildroot)) # cached_buildtree() # # Check if artifact is cached with expected buildtree. A # buildtree will not be present if the rest of the partial artifact # is not cached. # # Returns: # (bool): True if artifact cached with buildtree, False if # missing expected buildtree. Note this only confirms # if a buildtree is present, not its contents. # def cached_buildtree(self): buildtree_digest = self._get_field_digest("buildtree") if buildtree_digest: return self._cas.contains_directory(buildtree_digest) else: return False # buildtree_exists() # # Check if artifact was created with a buildtree. This does not check # whether the buildtree is present in the local cache. # # Returns: # (bool): True if artifact was created with buildtree # def buildtree_exists(self): artifact = self._get_proto() return bool(str(artifact.buildtree)) # cached_sources() # # Check if artifact is cached with sources. # # Returns: # (bool): True if artifact is cached with sources, False if sources # are not available. # def cached_sources(self): sources_digest = self._get_field_digest("sources") if sources_digest: return self._cas.contains_directory(sources_digest) else: return False # load_public_data(): # # Loads the public data from the cached artifact # # Returns: # (dict): The artifacts cached public data # def load_public_data(self): # Load the public data from the artifact artifact = self._get_proto() with self._cas.open(artifact.public_data) as meta_file: meta_str = meta_file.read() data = _yaml.load_data(meta_str, file_name="public.yaml") return data # load_sandbox_config(): # # Loads the sandbox configuration from the cached artifact # # Returns: # The stored SandboxConfig object # def load_sandbox_config(self) -> SandboxConfig: # Load the sandbox data from the artifact artifact = self._get_proto() meta_file = self._cas.objpath(artifact.low_diversity_meta) data = _yaml.load(meta_file, shortname="low-diversity-meta.yaml") # Extract the sandbox data config = data.get_mapping("sandbox-config") # Return a SandboxConfig return SandboxConfig.new_from_node(config) # load_environment(): # # Loads the environment variables from the cached artifact # # Returns: # The environment variables # def load_environment(self) -> Dict[str, str]: # Load the sandbox data from the artifact artifact = self._get_proto() meta_file = self._cas.objpath(artifact.low_diversity_meta) data = _yaml.load(meta_file, shortname="low-diversity-meta.yaml") # Extract the environment config = data.get_mapping("environment") # Return the environment return config.strip_node_info() # load_variables(): # # Loads the element variables from the cached artifact # # Returns: # The element variables # def load_variables(self) -> Variables: # Load the sandbox data from the artifact artifact = self._get_proto() meta_file = self._cas.objpath(artifact.high_diversity_meta) data = _yaml.load(meta_file, shortname="high-diversity-meta.yaml") # Extract the variables node and return the new Variables instance variables_node = data.get_mapping("variables") return Variables(variables_node) # load_build_result(): # # Load the build result from the cached artifact # # Returns: # (bool): Whether the artifact of this element present in the artifact cache is of a success # (str): Short description of the result # (str): Detailed description of the result # def load_build_result(self): artifact = self._get_proto() build_result = (artifact.build_success, artifact.build_error, artifact.build_error_details) return build_result # get_metadata_keys(): # # Retrieve the strong and weak keys from the given artifact. # # Returns: # The strong key # The strict key # The weak key # def get_metadata_keys(self) -> Tuple[str, str, str]: if self._metadata_keys is not None: return self._metadata_keys # Extract proto artifact = self._get_proto() strong_key = artifact.strong_key strict_key = artifact.strict_key weak_key = artifact.weak_key self._metadata_keys = (strong_key, strict_key, weak_key) return self._metadata_keys # get_metadata_workspaced(): # # Retrieve the hash of dependency from the given artifact. # # Returns: # (bool): Whether the given artifact was workspaced # def get_metadata_workspaced(self): if self._metadata_workspaced is not None: return self._metadata_workspaced # Extract proto artifact = self._get_proto() self._metadata_workspaced = artifact.was_workspaced return self._metadata_workspaced # get_metadata_workspaced_dependencies(): # # Retrieve the hash of workspaced dependencies keys from the given artifact. # # Returns: # (list): List of which dependencies are workspaced # def get_metadata_workspaced_dependencies(self): if self._metadata_workspaced_dependencies is not None: return self._metadata_workspaced_dependencies # Extract proto artifact = self._get_proto() self._metadata_workspaced_dependencies = [ dep.element_name for dep in artifact.build_deps if dep.was_workspaced ] return self._metadata_workspaced_dependencies # get_dependency_artifact_names() # # Retrieve the artifact names of all of the dependencies in _Scope.BUILD # # Returns: # (list [str]): A list of refs of all build dependencies in staging order. # def get_dependency_artifact_names(self): # XXX: The pylint disable is necessary due to upstream issue: # https://github.com/PyCQA/pylint/issues/850 from .element import _get_normal_name # pylint: disable=cyclic-import artifact = self._get_proto() try: dependency_refs = [ os.path.join(dep.project_name, _get_normal_name(dep.element_name), dep.cache_key) for dep in artifact.build_deps ] except AttributeError: # If the artifact has no dependencies, the build_deps attribute # will be missing from the proto. dependency_refs = [] return dependency_refs # query_cache(): # # Check whether the artifact corresponding to the stored cache key is # available. This also checks whether all required parts of the artifact # are available, which may depend on command and configuration. The cache # key used for querying is dependent on the current context. # # Returns: # (bool): Whether artifact is in local cache # def query_cache(self): artifact = self._load_proto() if not artifact: self._cached = False return False # Check whether 'files' subdirectory is available, with or without file contents if str(artifact.files) and not self._cas.contains_directory(artifact.files): self._cached = False return False # Check whether public data and logs are available logfile_digests = [logfile.digest for logfile in artifact.logs] digests = [artifact.low_diversity_meta, artifact.high_diversity_meta, artifact.public_data] + logfile_digests if not self._cas.contains_files(digests): self._cached = False return False self._proto = artifact self._cached = True return True # cached() # # Return whether the artifact is available in the local cache. This must # be called after `query_cache()` or `set_cached()`. # # Returns: # (bool): Whether artifact is in local cache # def cached(self, *, buildtree=False): assert self._cached is not None ret = self._cached if buildtree: ret = ret and (self.cached_buildtree() or not self.buildtree_exists()) return ret # cached_logs() # # Check if the artifact is cached with log files. # # Returns: # (bool): True if artifact is cached with logs, False if # element not cached or missing logs. # def cached_logs(self): # Log files are currently considered an essential part of an artifact. # If the artifact is cached, its log files are available as well. return self._element._cached() # set_cached() # # Mark the artifact as cached without querying the filesystem. # This is used as optimization when we know the artifact is available. # def set_cached(self): self._proto = self._load_proto() assert self._proto self._cached = True # pull() # # Pull artifact from remote artifact repository into local artifact cache. # # Args: # pull_buildtrees (bool): Whether to pull buildtrees or not # # Returns: True if the artifact has been downloaded, False otherwise # def pull(self, *, pull_buildtrees): artifacts = self._context.artifactcache pull_key = self.get_extract_key() if not artifacts.pull(self._element, pull_key, pull_buildtrees=pull_buildtrees): return False self.set_cached() # Add reference for the other key (weak key when pulling with strong key, # strong key when pulling with weak key) for key in self.get_metadata_keys(): artifacts.link_key(self._element, pull_key, key) return True def configure_sandbox(self, sandbox): artifact = self._get_proto() if artifact.HasField("buildsandbox") and artifact.buildsandbox.environment: env = {} for env_var in artifact.buildsandbox.environment: env[env_var.name] = env_var.value else: env = self.load_environment() sandbox.set_environment(env) if artifact.HasField("buildsandbox"): for directory in artifact.buildsandbox.marked_directories: sandbox.mark_directory(directory) if artifact.buildsandbox.working_directory: sandbox.set_work_directory(artifact.buildsandbox.working_directory) # load_proto() # # Returns: # (Artifact): Artifact proto # def _load_proto(self): key = self.get_extract_key() proto_path = os.path.join(self._artifactdir, self._element.get_artifact_name(key=key)) artifact = ArtifactProto() try: with open(proto_path, mode="r+b") as f: artifact.ParseFromString(f.read()) except FileNotFoundError: return None os.utime(proto_path) return artifact # _get_proto() # # Returns: # (Artifact): Artifact proto # def _get_proto(self): return self._proto # _get_field_digest() # # Returns: # (Digest): Digest of field specified # def _get_field_digest(self, field): artifact_proto = self._get_proto() digest = getattr(artifact_proto, field) if not str(digest): return None return digest apache-buildstream-27ae392/src/buildstream/_artifactcache.py000066400000000000000000000413321514607367700242210ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat import os from ._assetcache import AssetCache from ._cas.casremote import BlobNotFound from ._exceptions import ArtifactError, AssetCacheError, CASError, CASRemoteError from ._protos.buildstream.v2 import artifact_pb2 from . import utils REMOTE_ASSET_ARTIFACT_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:artifact:{}" # An ArtifactCache manages artifacts. # # Args: # context (Context): The BuildStream context # class ArtifactCache(AssetCache): def __init__(self, context): super().__init__(context) # create artifact directory self._basedir = context.artifactdir os.makedirs(self._basedir, exist_ok=True) # preflight(): # # Preflight check. # def preflight(self): self.cas.preflight() # contains(): # # Check whether the artifact for the specified Element is already available # in the local artifact cache. # # Args: # element (Element): The Element to check # key (str): The cache key to use # # Returns: True if the artifact is in the cache, False otherwise # def contains(self, element, key): ref = element.get_artifact_name(key) return os.path.exists(os.path.join(self._basedir, ref)) # list_artifacts(): # # List artifacts in this cache in LRU order. # # Args: # glob (str): An option glob expression to be used to list artifacts satisfying the glob # # Returns: # ([str]) - A list of artifact names as generated in LRU order # def list_artifacts(self, *, glob=None): return [ref for _, ref in sorted(list(self.list_refs_mtimes(self._basedir, glob_expr=glob)))] # remove(): # # Removes the artifact for the specified ref from the local # artifact cache. # # Args: # ref (artifact_name): The name of the artifact to remove (as # generated by `Element.get_artifact_name`) # def remove(self, ref): try: self.remove_ref(ref) except AssetCacheError as e: raise ArtifactError("{}".format(e)) from e # push(): # # Push committed artifact to remote repository. # # Args: # element (Element): The Element whose artifact is to be pushed # artifact (Artifact): The artifact being pushed # # Returns: # (bool): True if any remote was updated, False if no pushes were required # # Raises: # (ArtifactError): if there was an error # def push(self, element, artifact): project = element._get_project() display_key = element._get_display_key() index_remotes, storage_remotes = self.get_remotes(project.name, True) artifact_proto = artifact._get_proto() artifact_digest = self.cas.add_object(buffer=artifact_proto.SerializeToString()) pushed = False # First push our files to all storage remotes, so that they # can perform file checks on their end for remote in storage_remotes: remote.init() element.status("Pushing data from artifact {} -> {}".format(display_key.brief, remote)) if self._push_artifact_blobs(artifact, artifact_digest, remote): element.info("Pushed data from artifact {} -> {}".format(display_key.brief, remote)) else: element.info( "Remote ({}) already has all data of artifact {} cached".format(remote, display_key.brief) ) for remote in index_remotes: remote.init() element.status("Pushing artifact {} -> {}".format(display_key.brief, remote)) if self._push_artifact_proto(element, artifact, artifact_digest, remote): element.info("Pushed artifact {} -> {}".format(display_key.brief, remote)) pushed = True else: element.info("Remote ({}) already has artifact {} cached".format(remote, display_key.brief)) return pushed # pull(): # # Pull artifact from one of the configured remote repositories. # # Args: # element (Element): The Element whose artifact is to be fetched # key (str): The cache key to use # pull_buildtrees (bool): Whether to pull buildtrees or not # # Returns: # (bool): True if pull was successful, False if artifact was not available # def pull(self, element, key, *, pull_buildtrees=False): artifact_digest = None display_key = key[: self.context.log_key_length] project = element._get_project() artifact_name = element.get_artifact_name(key=key) uri = REMOTE_ASSET_ARTIFACT_URN_TEMPLATE.format(artifact_name) index_remotes, storage_remotes = self.get_remotes(project.name, False) errors = [] # Start by pulling our artifact proto, so that we know which # blobs to pull for remote in index_remotes: remote.init() try: element.status("Pulling artifact {} <- {}".format(display_key, remote)) response = remote.fetch_blob([uri]) if response: artifact_digest = response.blob_digest break element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) except AssetCacheError as e: element.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors and not artifact_digest: raise ArtifactError( "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors), temporary=True, ) # If we don't have an artifact, we can't exactly pull our # artifact if not artifact_digest: return False errors = [] # If we do, we can pull it! for remote in storage_remotes: remote.init() try: element.status("Pulling data for artifact {} <- {}".format(display_key, remote)) if self._pull_artifact_storage(element, key, artifact_digest, remote, pull_buildtrees=pull_buildtrees): element.info("Pulled artifact {} <- {}".format(display_key, remote)) return True element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) except BlobNotFound as e: # Not all blobs are available on this remote element.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) continue except CASError as e: element.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors: raise ArtifactError( "Failed to pull artifact {}".format(display_key), detail="\n".join(str(e) for e in errors), temporary=True, ) return False # link_key(): # # Add a key for an existing artifact. # # Args: # element (Element): The Element whose artifact is to be linked # oldkey (str): An existing cache key for the artifact # newkey (str): A new cache key for the artifact # def link_key(self, element, oldkey, newkey): oldref = element.get_artifact_name(oldkey) newref = element.get_artifact_name(newkey) if oldref == newref: # The two refs are identical, nothing to do return utils.safe_link(os.path.join(self._basedir, oldref), os.path.join(self._basedir, newref)) # check_remotes_for_element() # # Check if the element is available in any of the remotes # # Args: # element (Element): The element to check # # Returns: # (bool): True if the element is available remotely # def check_remotes_for_element(self, element): project = element._get_project() index_remotes, _ = self.get_remotes(project.name, False) # If there are no remotes if not index_remotes: return False ref = element.get_artifact_name() for remote in index_remotes: remote.init() if self._query_remote(ref, remote): return True return False ################################################ # Local Private Methods # ################################################ # _push_artifact_blobs() # # Push the blobs that make up an artifact to the remote server. # # Args: # artifact (Artifact): The artifact whose blobs to push. # remote (CASRemote): The remote to push the blobs to. # # Returns: # (bool) - True if we uploaded anything, False otherwise. # # Raises: # ArtifactError: If we fail to push blobs (*unless* they're # already there or we run out of space on the server). # def _push_artifact_blobs(self, artifact, artifact_digest, remote): artifact_proto = artifact._get_proto() try: if artifact_proto.HasField("files"): self.cas._send_directory(remote, artifact_proto.files) if artifact_proto.HasField("buildtree"): try: self.cas._send_directory(remote, artifact_proto.buildtree) except FileNotFoundError: pass if artifact_proto.HasField("sources"): try: self.cas._send_directory(remote, artifact_proto.sources) except FileNotFoundError: pass if artifact_proto.HasField("buildroot"): try: self.cas._send_directory(remote, artifact_proto.buildroot) except FileNotFoundError: pass if artifact_proto.HasField("buildsandbox"): for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests: self.cas._send_directory(remote, subsandbox_digest) digests = [artifact_digest, artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta] if artifact_proto.HasField("public_data"): digests.append(artifact_proto.public_data) for log_file in artifact_proto.logs: digests.append(log_file.digest) self.cas.send_blobs(remote, digests) except CASRemoteError as cas_error: if cas_error.reason != "cache-too-full": raise ArtifactError("Failed to push artifact blobs: {}".format(cas_error), temporary=True) return False return True # _push_artifact_proto() # # Pushes the artifact proto to remote. # # Args: # element (Element): The element # artifact (Artifact): The related artifact being pushed # remote (AssetRemote): Remote to push to # # Returns: # (bool): Whether we pushed the artifact. # # Raises: # ArtifactError: If the push fails for any reason except the # artifact already existing. # def _push_artifact_proto(self, element, artifact, artifact_digest, remote): artifact_proto = artifact._get_proto() keys = list(utils._deduplicate([artifact_proto.strong_key, artifact_proto.weak_key])) artifact_names = [element.get_artifact_name(key=key) for key in keys] uris = [REMOTE_ASSET_ARTIFACT_URN_TEMPLATE.format(artifact_name) for artifact_name in artifact_names] try: response = remote.fetch_blob(uris) # Skip push if artifact is already on the server if response and response.blob_digest == artifact_digest: return False except AssetCacheError as e: raise ArtifactError("{}".format(e), temporary=True) from e referenced_directories = [] if artifact_proto.HasField("files"): referenced_directories.append(artifact_proto.files) if artifact_proto.HasField("buildtree"): referenced_directories.append(artifact_proto.buildtree) if artifact_proto.HasField("sources"): referenced_directories.append(artifact_proto.sources) if artifact_proto.HasField("buildroot"): referenced_directories.append(artifact_proto.buildroot) if artifact_proto.HasField("buildsandbox"): for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests: referenced_directories.append(subsandbox_digest) referenced_blobs = [artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta] + [ log_file.digest for log_file in artifact_proto.logs ] if artifact_proto.HasField("public_data"): referenced_blobs.append(artifact_proto.public_data) try: remote.push_blob( uris, artifact_digest, references_blobs=referenced_blobs, references_directories=referenced_directories, ) except AssetCacheError as e: raise ArtifactError("{}".format(e), temporary=True) from e return True # _pull_artifact_storage(): # # Pull artifact blobs from the given remote. # # Args: # element (Element): element to pull # key (str): The specific key for the artifact to pull # remote (CASRemote): remote to pull from # pull_buildtree (bool): whether to pull buildtrees or not # # Returns: # (bool): True if we pulled any blobs. # # Raises: # ArtifactError: If the pull failed for any reason except the # blobs not existing on the server. # def _pull_artifact_storage(self, element, key, artifact_digest, remote, pull_buildtrees=False): artifact_name = element.get_artifact_name(key=key) try: # Fetch and parse artifact proto self.cas.fetch_blobs(remote, [artifact_digest]) artifact = artifact_pb2.Artifact() with self.cas.open(artifact_digest, "rb") as f: artifact.ParseFromString(f.read()) # Write the artifact proto to cache artifact_path = os.path.join(self._basedir, artifact_name) os.makedirs(os.path.dirname(artifact_path), exist_ok=True) with utils.save_file_atomic(artifact_path, mode="wb") as f: f.write(artifact.SerializeToString()) if artifact.HasField("files"): self.cas.fetch_directory(remote, artifact.files) if pull_buildtrees: if artifact.HasField("buildtree"): self.cas.fetch_directory(remote, artifact.buildtree) if artifact.HasField("sources"): self.cas.fetch_directory(remote, artifact.sources) if artifact.HasField("buildroot"): self.cas.fetch_directory(remote, artifact.buildroot) if artifact.HasField("buildsandbox"): for subsandbox_digest in artifact.buildsandbox.subsandbox_digests: self.cas.fetch_directory(remote, subsandbox_digest) digests = [artifact.low_diversity_meta, artifact.high_diversity_meta] if artifact.HasField("public_data"): digests.append(artifact.public_data) for log_digest in artifact.logs: digests.append(log_digest.digest) self.cas.fetch_blobs(remote, digests) except BlobNotFound: return False except CASRemoteError as e: raise ArtifactError("{}".format(e), temporary=True) from e return True # _query_remote() # # Args: # ref (str): The artifact ref # remote (AssetRemote): The remote we want to check # # Returns: # (bool): True if the ref exists in the remote, False otherwise. # def _query_remote(self, ref, remote): uri = REMOTE_ASSET_ARTIFACT_URN_TEMPLATE.format(ref) try: response = remote.fetch_blob([uri]) return bool(response) except AssetCacheError as e: raise ArtifactError("{}".format(e), temporary=True) from e apache-buildstream-27ae392/src/buildstream/_artifactelement.py000066400000000000000000000136311514607367700246100ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # James Ennis # Tristan Van Berkom from typing import TYPE_CHECKING, Optional, Dict from contextlib import suppress from . import Element from . import _cachekey from ._artifactproject import ArtifactProject from ._exceptions import ArtifactElementError from ._loader import LoadElement from .node import Node if TYPE_CHECKING: from ._context import Context from ._state import Task # ArtifactElement() # # Object to be used for directly processing an artifact # # Args: # context (Context): The Context object # ref (str): The artifact ref # class ArtifactElement(Element): # A hash of ArtifactElement by ref __instantiated_artifacts: Dict[str, "ArtifactElement"] = {} def __init__(self, context, ref): project_name, element_name, key = verify_artifact_ref(ref) project = ArtifactProject(project_name, context) load_element = LoadElement(Node.from_dict({}), element_name, project.loader) # NOTE element has no .bst suffix super().__init__(context, project, load_element, None, artifact_key=key) ######################################################## # Public API # ######################################################## # new_from_artifact_name(): # # Recursively instantiate a new ArtifactElement instance, and its # dependencies from an artifact name # # Args: # artifact_name: The artifact name # context: The Context object # task: A task object to report progress to # # Returns: # (ArtifactElement): A newly created Element instance # @classmethod def new_from_artifact_name(cls, artifact_name: str, context: "Context", task: Optional["Task"] = None): # Initial lookup for already loaded artifact. with suppress(KeyError): return cls.__instantiated_artifacts[artifact_name] # Instantiate the element, this can result in having a different # artifact name, if we loaded the artifact by it's weak key then # we will have the artifact loaded via it's strong key. element = ArtifactElement(context, artifact_name) artifact_name = element.get_artifact_name() # Perform a second lookup, avoid loading the same artifact # twice, even if we've loaded it both with weak and strong keys. with suppress(KeyError): return cls.__instantiated_artifacts[artifact_name] # Now cache the loaded artifact cls.__instantiated_artifacts[artifact_name] = element # Walk the dependencies and load recursively artifact = element._get_artifact() for dep_artifact_name in artifact.get_dependency_artifact_names(): dependency = ArtifactElement.new_from_artifact_name(dep_artifact_name, context, task) element._add_build_dependency(dependency) return element # clear_artifact_name_cache() # # Clear the internal artifact refs cache # # When loading ArtifactElements from artifact refs, we cache already # instantiated ArtifactElements in order to not have to load the same # ArtifactElements twice. This clears the cache. # # It should be called whenever we are done loading all artifacts in order # to save memory. # @classmethod def clear_artifact_name_cache(cls): cls.__instantiated_artifacts = {} ######################################################## # Override internal Element methods # ######################################################## def _load_artifact(self, *, pull, strict=None): # pylint: disable=useless-super-delegation # Always operate in strict mode as artifact key has been specified explicitly. return super()._load_artifact(pull=pull, strict=True) # Once we've finished loading an artifact, we assume the # state of the loaded artifact. This is also used if the # artifact is loaded after pulling. # def _load_artifact_done(self): self._mimic_artifact() super()._load_artifact_done() ######################################################## # Implement Element abstract methods # ######################################################## def configure(self, node): pass def preflight(self): pass def configure_sandbox(self, sandbox): install_root = self.get_variable("install-root") # Tell the sandbox to mount the build root and install root sandbox.mark_directory(install_root) # verify_artifact_ref() # # Verify that a ref string matches the format of an artifact # # Args: # ref (str): The artifact ref # # Returns: # project (str): The project's name # element (str): The element's name # key (str): The cache key # # Raises: # ArtifactElementError if the ref string does not match # the expected format # def verify_artifact_ref(ref): try: project, element, key = ref.split("/", 2) # This will raise a Value error if unable to split # Explicitly raise a ValueError if the key length is not as expected if not _cachekey.is_key(key): raise ValueError except ValueError: raise ArtifactElementError("Artifact: {} is not of the expected format".format(ref)) return project, element, key apache-buildstream-27ae392/src/buildstream/_artifactproject.py000066400000000000000000000051241514607367700246230ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # from contextlib import suppress from typing import TYPE_CHECKING from ._project import Project from ._context import Context from ._loader import Loader if TYPE_CHECKING: from typing import Dict # ArtifactProject() # # A project instance to be used as the project for an ArtifactElement. # # This is basically a simplified Project implementation which ensures that # we do not accidentally infer any data from a possibly present local project # when processing an ArtifactElement. # # Args: # project_name: The name of this project # class ArtifactProject(Project): __loaded_artifact_projects = {} # type: Dict[str, ArtifactProject] def __init__(self, project_name: str, context: Context): # # Chain up to the Project constructor, and allow it to initialize # without loading anything # super().__init__(None, context, search_for_project=False) # Fill in some necessities # self.name = project_name self.element_path = "" # This needs to be set to avoid Loader crashes self.loader = Loader(self) # get_artifact_project(): # # Gets a reference to an ArtifactProject for the given # project name, possibly instantiating one if needed. # # Args: # project_name: The project name # context: The Context # # Returns: # An ArtifactProject with the given project_name # @classmethod def get_artifact_project(cls, project_name: str, context: Context) -> "ArtifactProject": with suppress(KeyError): return cls.__loaded_artifact_projects[project_name] project = cls(project_name, context) cls.__loaded_artifact_projects[project_name] = project return project # clear_project_cache(): # # Clears the cache of loaded projects, this can be called directly # after completing a full load. # @classmethod def clear_project_cache(cls): cls.__loaded_artifact_projects = {} apache-buildstream-27ae392/src/buildstream/_assetcache.py000066400000000000000000000427331514607367700235510ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # import os import re from typing import List, Dict, Tuple, Iterable, Optional import grpc from . import utils from ._cas import CASRemote, CASCache, CASDProcessManager from ._exceptions import AssetCacheError, RemoteError from ._remotespec import RemoteSpec, RemoteType from ._remote import BaseRemote from ._protos.build.bazel.remote.asset.v1 import remote_asset_pb2 from ._protos.build.buildgrid import local_cas_pb2 from ._protos.google.rpc import code_pb2 class AssetRemote(BaseRemote): def __init__(self, spec, casd): super().__init__(spec) self.casd = casd self.instance_name = None self.fetch_service = None self.push_service = None def _configure_protocols(self): local_cas = self.casd.get_local_cas() request = local_cas_pb2.GetInstanceNameForRemotesRequest() self.spec.to_localcas_remote(request.remote_asset) response = local_cas.GetInstanceNameForRemotes(request) self.instance_name = response.instance_name self.fetch_service = self.casd.get_asset_fetch() self.push_service = self.casd.get_asset_push() # _check(): # # Check if this remote provides everything required for the # particular kind of remote. This is expected to be called as part # of check() # # Raises: # RemoteError: If the upstream has a problem # def _check(self): request = remote_asset_pb2.FetchBlobRequest() if self.instance_name: request.instance_name = self.instance_name try: self.fetch_service.FetchBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT: # Expected error as the request doesn't specify any URIs. pass elif e.code() == grpc.StatusCode.UNIMPLEMENTED: raise RemoteError( "Configured remote does not implement the Remote Asset " "Fetch service. Please check remote configuration." ) else: raise RemoteError("Remote initialisation failed with status {}: {}".format(e.code().name, e.details())) if self.spec.push: request = remote_asset_pb2.PushBlobRequest() if self.instance_name: request.instance_name = self.instance_name try: self.push_service.PushBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT: # Expected error as the request doesn't specify any URIs. pass elif e.code() == grpc.StatusCode.UNIMPLEMENTED: raise RemoteError( "Configured remote does not implement the Remote Asset " "Push service. Please check remote configuration." ) else: raise RemoteError( "Remote initialisation failed with status {}: {}".format(e.code().name, e.details()) ) # fetch_blob(): # # Resolve URIs to a CAS blob digest. # # Args: # uris (list of str): The URIs to resolve. Multiple URIs should represent # the same content available at different locations. # qualifiers (list of Qualifier): Optional qualifiers sub-specifying the # content to fetch. # # Returns # (FetchBlobResponse): The asset server response or None if the resource # is not available. # # Raises: # AssetCacheError: If the upstream has a problem # def fetch_blob(self, uris, *, qualifiers=None): request = remote_asset_pb2.FetchBlobRequest() if self.instance_name: request.instance_name = self.instance_name request.uris.extend(uris) if qualifiers: request.qualifiers.extend(qualifiers) try: response = self.fetch_service.FetchBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: return None raise AssetCacheError("FetchBlob failed with status {}: {}".format(e.code().name, e.details())) from e if response.status.code == code_pb2.NOT_FOUND: return None if response.status.code != code_pb2.OK: raise AssetCacheError("FetchBlob failed with response status {}".format(response.status.code)) return response # fetch_directory(): # # Resolve URIs to a CAS Directory digest. # # Args: # uris (list of str): The URIs to resolve. Multiple URIs should represent # the same content available at different locations. # qualifiers (list of Qualifier): Optional qualifiers sub-specifying the # content to fetch. # # Returns # (FetchDirectoryResponse): The asset server response or None if the resource # is not available. # # Raises: # AssetCacheError: If the upstream has a problem # def fetch_directory(self, uris, *, qualifiers=None): request = remote_asset_pb2.FetchDirectoryRequest() if self.instance_name: request.instance_name = self.instance_name request.uris.extend(uris) if qualifiers: request.qualifiers.extend(qualifiers) try: response = self.fetch_service.FetchDirectory(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: return None raise AssetCacheError("FetchDirectory failed with status {}: {}".format(e.code().name, e.details())) from e if response.status.code == code_pb2.NOT_FOUND: return None if response.status.code != code_pb2.OK: raise AssetCacheError("FetchDirectory failed with response status {}".format(response.status.code)) return response # push_blob(): # # Associate a CAS blob digest to URIs. # # Args: # uris (list of str): The URIs to associate with the blob digest. # blob_digest (Digest): The CAS blob to associate. # qualifiers (list of Qualifier): Optional qualifiers sub-specifying the # content that is being pushed. # references_blobs (list of Digest): Referenced blobs that need to not expire # before expiration of this association. # references_directories (list of Digest): Referenced directories that need to not expire # before expiration of this association. # # Raises: # AssetCacheError: If the upstream has a problem # def push_blob(self, uris, blob_digest, *, qualifiers=None, references_blobs=None, references_directories=None): request = remote_asset_pb2.PushBlobRequest() if self.instance_name: request.instance_name = self.instance_name request.uris.extend(uris) request.blob_digest.CopyFrom(blob_digest) if qualifiers: request.qualifiers.extend(qualifiers) if references_blobs: request.references_blobs.extend(references_blobs) if references_directories: request.references_directories.extend(references_directories) try: self.push_service.PushBlob(request) except grpc.RpcError as e: raise AssetCacheError("PushBlob failed with status {}: {}".format(e.code().name, e.details())) from e # push_directory(): # # Associate a CAS Directory digest to URIs. # # Args: # uris (list of str): The URIs to associate with the blob digest. # directory_digest (Digest): The CAS Direcdtory to associate. # qualifiers (list of Qualifier): Optional qualifiers sub-specifying the # content that is being pushed. # references_blobs (list of Digest): Referenced blobs that need to not expire # before expiration of this association. # references_directories (list of Digest): Referenced directories that need to not expire # before expiration of this association. # # Raises: # AssetCacheError: If the upstream has a problem # def push_directory( self, uris, directory_digest, *, qualifiers=None, references_blobs=None, references_directories=None ): request = remote_asset_pb2.PushDirectoryRequest() if self.instance_name: request.instance_name = self.instance_name request.uris.extend(uris) request.root_directory_digest.CopyFrom(directory_digest) if qualifiers: request.qualifiers.extend(qualifiers) if references_blobs: request.references_blobs.extend(references_blobs) if references_directories: request.references_directories.extend(references_directories) try: self.push_service.PushDirectory(request) except grpc.RpcError as e: raise AssetCacheError("PushDirectory failed with status {}: {}".format(e.code().name, e.details())) from e # RemotePair() # # A pair of remotes which corresponds to a RemoteSpec, we # need separate remote objects for the index and the storage so # we store them together for each RemoteSpec here. # # Either members of the RemotePair may be None, in case that # the user specified a diffrerent RemoteSpec for indexing and # for storage. # # Both members may also be None, in the case that we were unable # to establish a connection to this remote at initialization time. # class RemotePair: def __init__(self, casd: CASDProcessManager, spec: RemoteSpec): self.index: Optional[AssetRemote] = None self.storage: Optional[CASRemote] = None self.error: Optional[str] = None try: if spec.remote_type in [RemoteType.INDEX, RemoteType.ALL]: index = AssetRemote(spec, casd) index.check() self.index = index if spec.remote_type in [RemoteType.STORAGE, RemoteType.ALL]: storage = CASRemote(spec, casd) storage.check() self.storage = storage except RemoteError as e: self.error = str(e) # Base Asset Cache for Caches to derive from # class AssetCache: def __init__(self, context): self.context = context self.cas: CASCache = context.get_cascache() # Table of RemotePair objects self._remotes: Dict[RemoteSpec, RemotePair] = {} # Table of prioritized RemoteSpecs which are valid for each project self._project_specs: Dict[str, List[RemoteSpec]] = {} self._has_fetch_remotes: bool = False self._has_push_remotes: bool = False self._basedir = None # setup_remotes(): # # Sets up which remotes to use # # Args: # specs: The active remote specs # project_specs: List of specs for each project # def setup_remotes(self, specs: Iterable[RemoteSpec], project_specs: Dict[str, List[RemoteSpec]]): # Hold on to the project specs self._project_specs = project_specs casd = self.context.get_casd() for spec in specs: # This can be called multiple times, ensure that we only try # to instantiate each remote once. # if spec in self._remotes: continue remote = RemotePair(casd, spec) if remote.error: self.context.messenger.warn("Failed to initialize remote {}: {}".format(spec.url, remote.error)) self._remotes[spec] = remote # Determine overall existance of push or fetch remotes self._has_fetch_remotes = any(remote.storage for _, remote in self._remotes.items()) and any( remote.index for _, remote in self._remotes.items() ) self._has_push_remotes = any(spec.push and remote.storage for spec, remote in self._remotes.items()) and any( spec.push and remote.index for spec, remote in self._remotes.items() ) # get_remotes(): # # List the index remotes and storage remotes available for fetching # # Args: # project_name: The project name # push: Whether pushing is required for this remote # # Returns: # index_remotes: The index remotes # storage_remotes: The storage remotes # def get_remotes(self, project_name: str, push: bool) -> Tuple[List[AssetRemote], List[CASRemote]]: try: project_specs = self._project_specs[project_name] except KeyError: # Technically this shouldn't happen, but here is a defensive return none the less. return [], [] index_remotes = [] storage_remotes = [] for spec in project_specs: if push and not spec.push: continue remote = self._remotes[spec] if remote.index: index_remotes.append(remote.index) if remote.storage: storage_remotes.append(remote.storage) return index_remotes, storage_remotes # has_fetch_remotes(): # # Check whether any remote repositories are available for fetching. # # Args: # plugin (Plugin): The Plugin to check # # Returns: True if any remote repositories are configured, False otherwise # def has_fetch_remotes(self, *, plugin=None): if not self._has_fetch_remotes: # No project has fetch remotes return False elif plugin is None: # At least one (sub)project has fetch remotes return True else: project = plugin._get_project() index_remotes, storage_remotes = self.get_remotes(project.name, False) # Check whether the specified element's project has fetch remotes return index_remotes and storage_remotes # has_push_remotes(): # # Check whether any remote repositories are available for pushing. # # Args: # element (Element): The Element to check # # Returns: True if any remote repository is configured, False otherwise # def has_push_remotes(self, *, plugin=None): if not self._has_push_remotes: # No project has push remotes return False elif plugin is None: # At least one (sub)project has push remotes return True else: project = plugin._get_project() index_remotes, storage_remotes = self.get_remotes(project.name, True) # Check whether the specified element's project has fetch remotes return bool(index_remotes and storage_remotes) # list_refs_mtimes() # # List refs in a directory, given a base path. Also returns the # associated mtimes # # Args: # base_path (str): Base path to traverse over # glob_expr (str|None): Optional glob expression to match against files # # Returns: # (iter (mtime, filename)]): iterator of tuples of mtime and refs # def list_refs_mtimes(self, base_path, *, glob_expr=None): path = base_path if glob_expr is not None: globdir = os.path.dirname(glob_expr) if not any(c in "*?[" for c in globdir): # path prefix contains no globbing characters so # append the glob to optimise the os.walk() path = os.path.join(base_path, globdir) regexer = None if glob_expr: expression = utils._glob2re(glob_expr) regexer = re.compile(expression) for root, _, files in os.walk(path): for filename in files: ref_path = os.path.join(root, filename) relative_path = os.path.relpath(ref_path, base_path) # Relative to refs head if regexer is None or regexer.match(relative_path): # Obtain the mtime (the time a file was last modified) yield (os.path.getmtime(ref_path), relative_path) # remove_ref() # # Removes a ref. # # This also takes care of pruning away directories which can # be removed after having removed the given ref. # # Args: # ref (str): The ref to remove # # Raises: # (AssetCacheError): If the ref didnt exist, or a system error # occurred while removing it # def remove_ref(self, ref): try: utils._remove_path_with_parents(self._basedir, ref) except FileNotFoundError as e: raise AssetCacheError("Could not find ref '{}'".format(ref)) from e except OSError as e: raise AssetCacheError("System error while removing ref '{}': {}".format(ref, e)) from e apache-buildstream-27ae392/src/buildstream/_cachekey.py000066400000000000000000000033301514607367700232100ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import hashlib import ujson # Internal record of the size of a cache key _CACHEKEY_SIZE = len(hashlib.sha256().hexdigest()) # Hex digits _HEX_DIGITS = "0123456789abcdef" # is_key() # # Check if the passed in string *could be* a cache key. This basically checks # that the length matches a sha256 hex digest, and that the string does not # contain any non-hex characters and is fully lower case. # # Args: # key (str): The string to check # # Returns: # (bool): Whether or not `key` could be a cache key # def is_key(key): if len(key) != _CACHEKEY_SIZE: return False return not any(ch not in _HEX_DIGITS for ch in key) # generate_key() # # Generate an sha256 hex digest from the given value. The value # can be a simple value or recursive dictionary with lists etc, # anything simple enough to serialize. # # Args: # value: A value to get a key for # # Returns: # (str): An sha256 hex digest of the given value # def generate_key(value): ustring = ujson.dumps(value, sort_keys=True, escape_forward_slashes=False).encode("utf-8") return hashlib.sha256(ustring).hexdigest() apache-buildstream-27ae392/src/buildstream/_cas/000077500000000000000000000000001514607367700216315ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_cas/__init__.py000066400000000000000000000013721514607367700237450ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .cascache import CASCache, CASLogLevel from .casdprocessmanager import CASDProcessManager from .casremote import CASRemote apache-buildstream-27ae392/src/buildstream/_cas/cascache.py000066400000000000000000000641041514607367700237420ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jürg Billeter import itertools import os import stat import contextlib import time from typing import Optional, List import threading import grpc from .._protos.google.rpc import code_pb2 from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .._protos.build.buildgrid import local_cas_pb2 from .. import utils from ..types import FastEnum, SourceRef from .._exceptions import CASCacheError from .casremote import CASRemote, _CASBatchRead, _CASBatchUpdate, BlobNotFound _BUFFER_SIZE = 65536 # Refresh interval for disk usage of local cache in seconds _CACHE_USAGE_REFRESH = 5 class CASLogLevel(FastEnum): WARNING = "warning" INFO = "info" TRACE = "trace" # A CASCache manages a CAS repository as specified in the Remote Execution API. # # Args: # path (str): The root directory for the CAS repository # casd (CASDProcessManager): The buildbox-casd manager # remote_cache (bool): True if a CAS server is configured as a remote cache (storage-service) # class CASCache: def __init__(self, path, *, casd, remote_cache=False): self.casdir = os.path.join(path, "cas") self.tmpdir = os.path.join(path, "tmp") os.makedirs(self.tmpdir, exist_ok=True) self._cache_usage_monitor = None self._remote_cache = remote_cache self._casd = casd if casd: self._cache_usage_monitor = _CASCacheUsageMonitor(self._casd) self._cache_usage_monitor.start() else: assert not self._remote_cache self._default_remote = CASRemote(None, casd) self._default_remote.init() # preflight(): # # Preflight check. # def preflight(self): if not os.path.join(self.casdir, "objects"): raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir)) # release_resources(): # # Release resources used by CASCache. # def release_resources(self): if self._cache_usage_monitor: self._cache_usage_monitor.stop() self._cache_usage_monitor.join() def get_default_remote(self): return self._default_remote # contains_files(): # # Check whether file digests exist in the local CAS cache # # Args: # digest (Digest): The file digest to check # # Returns: True if the files are in the cache, False otherwise # def contains_files(self, digests): return len(self.missing_blobs(digests)) == 0 # contains_directory(): # # Check whether the specified directory, subdirectories and files are in the # cache, i.e non dangling. # # Args: # digest (Digest): The directory digest to check # # Returns: True if the directory is available in the local cache # def contains_directory(self, digest): local_cas = self._casd.get_local_cas() # Without a remote cache, `FetchTree` simply checks the local cache. request = local_cas_pb2.FetchTreeRequest() request.root_digest.CopyFrom(digest) # Always fetch Directory protos as they are needed to enumerate subdirectories and files. # Don't implicitly fetch file blobs from the remote cache as we don't need them. request.fetch_file_blobs = not self._remote_cache try: local_cas.FetchTree(request) if not self._remote_cache: return True except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: return False if e.code() == grpc.StatusCode.UNIMPLEMENTED: raise CASCacheError("Unsupported buildbox-casd version: FetchTree unimplemented") from e raise # Make sure everything is available in the remote cache (storage-service) request = local_cas_pb2.UploadTreeRequest() request.root_digest.CopyFrom(digest) try: local_cas.UploadTree(request) return True except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: return False if e.code() == grpc.StatusCode.UNIMPLEMENTED: # Fallback path if buildbox-casd is too old to support UploadTree missing_blobs = self.missing_blobs_for_directory(digest, remote=self._default_remote) return not missing_blobs raise # checkout(): # # Checkout the specified directory digest. # # Args: # dest (str): The destination path # tree (Digest): The directory digest to extract # can_link (bool): Whether we can create hard links in the destination # def checkout(self, dest, tree, *, can_link=False, _fetch=True): if _fetch: # We need the files in the local cache self.ensure_tree(tree) os.makedirs(dest, exist_ok=True) directory = remote_execution_pb2.Directory() with open(self.objpath(tree), "rb") as f: directory.ParseFromString(f.read()) for filenode in directory.files: # regular file, create hardlink fullpath = os.path.join(dest, filenode.name) node_properties = filenode.node_properties if node_properties.HasField("mtime"): mtime = utils._parse_protobuf_timestamp(node_properties.mtime) else: mtime = None if can_link and mtime is None: utils.safe_link(self.objpath(filenode.digest), fullpath) else: utils.safe_copy(self.objpath(filenode.digest), fullpath, copystat=False) if mtime is not None: utils._set_file_mtime(fullpath, mtime) if filenode.is_executable: st = os.stat(fullpath) mode = st.st_mode if mode & stat.S_IRUSR: mode |= stat.S_IXUSR if mode & stat.S_IRGRP: mode |= stat.S_IXGRP if mode & stat.S_IROTH: mode |= stat.S_IXOTH os.chmod(fullpath, mode) for dirnode in directory.directories: fullpath = os.path.join(dest, dirnode.name) self.checkout(fullpath, dirnode.digest, can_link=can_link, _fetch=False) for symlinknode in directory.symlinks: # symlink fullpath = os.path.join(dest, symlinknode.name) os.symlink(symlinknode.target, fullpath) node_properties = directory.node_properties if node_properties.HasField("mtime"): mtime = utils._parse_protobuf_timestamp(node_properties.mtime) utils._set_file_mtime(dest, mtime) # ensure_tree(): # # Make sure all blobs referenced by the given directory tree are available # in the local cache when using a remote cache. # # Args: # tree (Digest): The digest of the tree # def ensure_tree(self, tree): if self._remote_cache: local_cas = self._casd.get_local_cas() request = local_cas_pb2.FetchTreeRequest() request.root_digest.CopyFrom(tree) request.fetch_file_blobs = True local_cas.FetchTree(request) # fetch_directory(): # # Fetches remote directory and adds it to content addressable store. # # This recursively fetches directory objects and files. # # Args: # remote (Remote): The remote to use. # dir_digest (Digest): Digest object for the directory to fetch. # def fetch_directory(self, remote, dir_digest): local_cas = self._casd.get_local_cas() request = local_cas_pb2.FetchTreeRequest() request.instance_name = remote.local_cas_instance_name request.root_digest.CopyFrom(dir_digest) request.fetch_file_blobs = False try: local_cas.FetchTree(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise BlobNotFound( dir_digest.hash, "Failed to fetch directory tree {}: {}: {}".format(dir_digest.hash, e.code().name, e.details()), ) from e raise CASCacheError( "Failed to fetch directory tree {}: {}: {}".format(dir_digest.hash, e.code().name, e.details()) ) from e required_blobs = self.required_blobs_for_directory(dir_digest) self.fetch_blobs(remote, required_blobs) # pull_tree(): # # Pull a single Tree rather than a ref. # Does not update local refs. # # Args: # remote (CASRemote): The remote to pull from # digest (Digest): The digest of the tree # def pull_tree(self, remote, digest): try: remote.init() digest = self._fetch_tree(remote, digest) return digest except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: raise return None # objpath(): # # Return the path of an object based on its digest. # # Args: # digest (Digest): The digest of the object # # Returns: # (str): The path of the object # def objpath(self, digest): return os.path.join(self.casdir, "objects", digest.hash[:2], digest.hash[2:]) # open(): # # Open file read-only by CAS digest and return a corresponding file object. # # Args: # digest (Digest): The digest of the object # mode (str): An optional string that specifies the mode in which the file is opened. # def open(self, digest, mode="r"): if mode not in ["r", "rb"]: raise ValueError("Unsupported mode: `{}`".format(mode)) objpath = self.objpath(digest) if self._remote_cache and not os.path.exists(objpath): batch = _CASBatchRead(self._default_remote) batch.add(digest) batch.send() return open(objpath, mode=mode) # pylint: disable=consider-using-with,unspecified-encoding # add_object(): # # Hash and write object to CAS. # # Args: # path (str): Path to file to add # buffer (bytes): Byte buffer to add # instance_name (str): casd instance_name for remote CAS # # Returns: # (Digest): The digest of the added object # # Either `path` or `buffer` must be passed, but not both. # def add_object(self, *, path=None, buffer=None, instance_name=None): # Exactly one of the two parameters has to be specified assert (path is None) != (buffer is None) if path is None: digests = self.add_objects(buffers=[buffer], instance_name=instance_name) else: digests = self.add_objects(paths=[path], instance_name=instance_name) assert len(digests) == 1 return digests[0] # add_objects(): # # Hash and write objects to CAS. # # Args: # paths (List[str]): Paths to files to add # buffers (List[bytes]): Byte buffers to add # instance_name (str): casd instance_name for remote CAS # # Returns: # (List[Digest]): The digests of the added objects # # Either `paths` or `buffers` must be passed, but not both. # def add_objects(self, *, paths=None, buffers=None, instance_name=None): # Exactly one of the two parameters has to be specified assert (paths is None) != (buffers is None) digests = [] with contextlib.ExitStack() as stack: if paths is None: paths = [] for buffer in buffers: tmp = stack.enter_context(self._temporary_object()) tmp.write(buffer) tmp.flush() paths.append(tmp.name) request = local_cas_pb2.CaptureFilesRequest() if instance_name: request.instance_name = instance_name for path in paths: request.path.append(path) local_cas = self._casd.get_local_cas() response = local_cas.CaptureFiles(request) if len(response.responses) != len(paths): raise CASCacheError( "Expected {} responses from CaptureFiles, got {}".format(len(paths), len(response.responses)) ) for path, blob_response in zip(paths, response.responses): if blob_response.status.code == code_pb2.RESOURCE_EXHAUSTED: raise CASCacheError("Cache too full", reason="cache-too-full") if blob_response.status.code != code_pb2.OK: raise CASCacheError("Failed to capture blob {}: {}".format(path, blob_response.status.code)) digest = remote_execution_pb2.Digest() digest.CopyFrom(blob_response.digest) digests.append(digest) return digests # import_directory(): # # Import directory tree into CAS. # # Args: # path (str): Path to directory to import # properties Optional[List[str]]: List of properties to request # # Returns: # (Digest): The digest of the imported directory # def import_directory(self, path: str, properties: Optional[List[str]] = None) -> SourceRef: local_cas = self._casd.get_local_cas() request = local_cas_pb2.CaptureTreeRequest() request.path.append(path) if properties: for _property in properties: request.node_properties.append(_property) response = local_cas.CaptureTree(request) if len(response.responses) != 1: raise CASCacheError("Expected 1 response from CaptureTree, got {}".format(len(response.responses))) tree_response = response.responses[0] if tree_response.status.code == code_pb2.RESOURCE_EXHAUSTED: raise CASCacheError("Cache too full", reason="cache-too-full") if tree_response.status.code != code_pb2.OK: raise CASCacheError("Failed to capture tree {}: {}".format(path, tree_response.status)) treepath = self.objpath(tree_response.tree_digest) tree = remote_execution_pb2.Tree() with open(treepath, "rb") as f: tree.ParseFromString(f.read()) root_directory = tree.root.SerializeToString() return utils._message_digest(root_directory) # stage_directory(): # # A contextmanager to stage a CAS directory tree in the local filesystem. # # This makes the specified directory tree temporarily available for local # filesystem access. This may use FUSE, hardlinks, reflinks or file copies, # depending on the system. The implementation makes sure that BuildStream # and subprocesses cannot corrupt the cache by modifying staged files. # # Args: # directory_digest (Digest): The digest of a directory # # Yields: # (str): The local filesystem path # @contextlib.contextmanager def stage_directory(self, directory_digest): local_cas = self._casd.get_local_cas() request = local_cas_pb2.StageTreeRequest() request.root_digest.CopyFrom(directory_digest) # Specify the credentials used to access the staged tree request.access_credentials.uid = os.geteuid() done_event = threading.Event() def request_iterator(): yield request # Wait until staged tree is no longer needed done_event.wait() # A second (empty) request indicates that the staging location can be cleaned up yield local_cas_pb2.StageTreeRequest() response_stream = local_cas.StageTree(request_iterator()) try: # Read primary response and yield staging location response = next(response_stream) yield response.path # Staged tree is no longer needed done_event.set() # Wait for cleanup to complete next(response_stream) except StopIteration as e: raise CASCacheError("Unexpected end of response stream for StageTree") from e # missing_blobs_for_directory(): # # Determine which blobs of a directory tree are missing on the remote. # # Args: # digest (Digest): The directory digest # # Returns: List of missing Digest objects # def missing_blobs_for_directory(self, digest, *, remote=None): required_blobs = self.required_blobs_for_directory(digest) return self.missing_blobs(required_blobs, remote=remote) # missing_blobs(): # # Determine which blobs are missing locally or on the remote. # # Args: # blobs ([Digest]): List of directory digests to check # # Returns: List of missing Digest objects # def missing_blobs(self, blobs, *, remote=None): cas = self._casd.get_cas() if remote: instance_name = remote.local_cas_instance_name else: instance_name = "" missing_blobs = {} # Limit size of FindMissingBlobs request for required_blobs_group in _grouper(iter(blobs), 512): request = remote_execution_pb2.FindMissingBlobsRequest(instance_name=instance_name) for required_digest in required_blobs_group: d = request.blob_digests.add() d.CopyFrom(required_digest) try: response = cas.FindMissingBlobs(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT and e.details().startswith("Invalid instance name"): raise CASCacheError("Unsupported buildbox-casd version: FindMissingBlobs failed") from e raise for missing_digest in response.missing_blob_digests: d = remote_execution_pb2.Digest() d.CopyFrom(missing_digest) missing_blobs[d.hash] = d return missing_blobs.values() # required_blobs_for_directory(): # # Generator that returns the Digests of all blobs in the tree specified by # the Digest of the toplevel Directory object. # def required_blobs_for_directory(self, directory_digest, *, excluded_subdirs=None, _fetch_tree=True): if not excluded_subdirs: excluded_subdirs = [] if self._remote_cache and _fetch_tree: # Ensure we have the directory protos in the local cache local_cas = self._casd.get_local_cas() request = local_cas_pb2.FetchTreeRequest() request.root_digest.CopyFrom(directory_digest) request.fetch_file_blobs = False local_cas.FetchTree(request) # parse directory, and recursively add blobs yield directory_digest directory = remote_execution_pb2.Directory() with open(self.objpath(directory_digest), "rb") as f: directory.ParseFromString(f.read()) for filenode in directory.files: yield filenode.digest for dirnode in directory.directories: if dirnode.name not in excluded_subdirs: yield from self.required_blobs_for_directory(dirnode.digest, _fetch_tree=False) ################################################ # Local Private Methods # ################################################ # _temporary_object(): # # Returns: # (file): A file object to a named temporary file. # # Create a named temporary file with 0o0644 access rights. @contextlib.contextmanager def _temporary_object(self): with utils._tempnamedfile(dir=self.tmpdir) as f: os.chmod(f.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) yield f def _fetch_tree(self, remote, digest): self.fetch_blobs(remote, [digest]) tree = remote_execution_pb2.Tree() with self.open(digest, "rb") as f: tree.ParseFromString(f.read()) dirbuffers = [tree.root.SerializeToString()] for directory in tree.children: dirbuffers.append(directory.SerializeToString()) dirdigests = self.add_objects(buffers=dirbuffers) # The digest of the root directory return dirdigests[0] # fetch_blobs(): # # Fetch blobs from remote CAS. Optionally returns missing blobs that could # not be fetched. # # Args: # remote (CASRemote): The remote repository to fetch from # digests (list): The Digests of blobs to fetch # allow_partial (bool): True to return missing blobs, False to raise a # BlobNotFound error if a blob is missing # # Returns: The Digests of the blobs that were not available on the remote CAS # def fetch_blobs(self, remote, digests, *, allow_partial=False): if self._remote_cache: # Determine blobs missing in the remote cache and only fetch those digests = self.missing_blobs(digests) missing_blobs = [] if allow_partial else None remote.init() batch = _CASBatchRead(remote) for digest in digests: if digest.hash: batch.add(digest) batch.send(missing_blobs=missing_blobs) if self._remote_cache: # Upload fetched blobs to the remote cache as we can't transfer # blobs directly from another remote to the remote cache batch = _CASBatchUpdate(self._default_remote) for digest in digests: if missing_blobs is None or digest not in missing_blobs: # pylint: disable=unsupported-membership-test batch.add(digest) batch.send() return missing_blobs # send_blobs(): # # Upload blobs to remote CAS. # # Args: # remote (CASRemote): The remote repository to upload to # digests (list): The Digests of Blobs to upload # def send_blobs(self, remote, digests): if self._remote_cache: # First fetch missing blobs from the remote cache as we can't # transfer blobs directly from the remote cache to another remote. remote_missing_blobs = self.missing_blobs(digests, remote=remote) batch = _CASBatchRead(self._default_remote) for digest in remote_missing_blobs: batch.add(digest) batch.send() batch = _CASBatchUpdate(remote) for digest in digests: batch.add(digest) batch.send() def _send_directory(self, remote, digest): required_blobs = list(self.required_blobs_for_directory(digest)) # Upload any blobs missing on the server. # buildbox-casd will call FindMissingBlobs before the actual upload # and skip blobs that already exist on the server. self.send_blobs(remote, required_blobs) # get_cache_usage(): # # Fetches the current usage of the CAS local cache. # # Returns: # (CASCacheUsage): The current status # def get_cache_usage(self): return self._cache_usage_monitor.get_cache_usage() # _CASCacheUsage # # A simple object to report the current CAS cache usage details. # # Args: # used_size (int): Total size used by the local cache, in bytes. # quota_size (int): Disk quota for the local cache, in bytes. # class _CASCacheUsage: def __init__(self, used_size, quota_size): self.used_size = used_size self.quota_size = quota_size if self.quota_size is None: self.used_percent = 0 else: self.used_percent = int(self.used_size * 100 / self.quota_size) # Formattable into a human readable string # def __str__(self): if self.used_size is None: return "unknown" elif self.quota_size is None: return utils._pretty_size(self.used_size, dec_places=1) else: return "{} / {} ({}%)".format( utils._pretty_size(self.used_size, dec_places=1), utils._pretty_size(self.quota_size, dec_places=1), self.used_percent, ) # _CASCacheUsageMonitor # # This manages the subprocess that tracks cache usage information via # buildbox-casd. # class _CASCacheUsageMonitor(threading.Thread): def __init__(self, casd): super().__init__() self._casd = casd self._disk_usage = None self._disk_quota = None self._should_stop = False def get_cache_usage(self): return _CASCacheUsage(self._disk_usage, self._disk_quota) def stop(self): self._should_stop = True def run(self): local_cas = self._casd.get_local_cas() while not self._should_stop: try: # Ask buildbox-casd for current value request = local_cas_pb2.GetLocalDiskUsageRequest() response = local_cas.GetLocalDiskUsage(request) # Update values in shared memory self._disk_usage = response.size_bytes disk_quota = response.quota_bytes if disk_quota == 0: # Quota == 0 means there is no quota self._disk_quota = None else: self._disk_quota = disk_quota except grpc.RpcError: # Terminate loop when buildbox-casd becomes unavailable break # Sleep until next refresh for _ in range(_CACHE_USAGE_REFRESH * 10): if self._should_stop: break time.sleep(0.1) def _grouper(iterable, n): while True: try: current = next(iterable) except StopIteration: return yield itertools.chain([current], itertools.islice(iterable, n - 1)) apache-buildstream-27ae392/src/buildstream/_cas/casdprocessmanager.py000066400000000000000000000451531514607367700260570ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import contextlib import threading import os import re import random import shutil import stat import subprocess import sys import tempfile import time from subprocess import CalledProcessError import psutil import grpc from .._protos.build.bazel.remote.asset.v1 import remote_asset_pb2_grpc from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc from .._protos.build.buildgrid import local_cas_pb2_grpc from .._protos.google.bytestream import bytestream_pb2_grpc from .._protos.google.longrunning import operations_pb2_grpc from .. import _site from .. import utils from .._exceptions import CASCacheError _CASD_MAX_LOGFILES = 10 _CASD_TIMEOUT = 300 # in seconds # # Minimum required version of buildbox-casd # _REQUIRED_CASD_MAJOR = 1 _REQUIRED_CASD_MINOR = 2 _REQUIRED_CASD_MICRO = 6 # CASDProcessManager # # This manages the subprocess that runs buildbox-casd. # # Args: # path (str): The root directory for the CAS repository # log_dir (str): The directory for the logs # log_level (LogLevel): Log level to give to buildbox-casd for logging # cache_quota (int): User configured cache quota # reserved (int): User configured reserved disk space # remote_cache_spec (RemoteSpec): Optional remote cache server # protect_session_blobs (bool): Disable expiry for blobs used in the current session # messenger (Messenger): The messenger to report warnings through the UI # class CASDProcessManager: def __init__( self, path, log_dir, log_level, cache_quota, remote_cache_spec, protect_session_blobs, messenger, *, reserved=None, low_watermark=None, local_jobs=None ): os.makedirs(path, exist_ok=True) self._log_dir = log_dir self._socket_path = self._make_socket_path(path) self._connection_string = "unix:" + self._socket_path # Early version check self._check_casd_version(messenger) casd_args = [self.__buildbox_casd()] casd_args.append("--bind=" + self._connection_string) casd_args.append("--log-level=" + log_level.value) if cache_quota is not None: casd_args.append("--quota-high={}".format(int(cache_quota))) if low_watermark is not None: casd_args.append("--quota-low={}%".format(int(low_watermark * 100))) if reserved is not None: casd_args.append("--reserved={}".format(int(reserved))) if protect_session_blobs: casd_args.append("--protect-session-blobs") if local_jobs is not None: try: buildbox_run = utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox") casd_args.append("--buildbox-run={}".format(buildbox_run)) casd_args.append("--jobs={}".format(local_jobs)) except utils.ProgramNotFoundError: # Not fatal as buildbox-run is not needed for remote execution # and buildbox-casd local execution will never be used if # buildbox-run is not available. pass if remote_cache_spec: casd_args.append("--cas-remote={}".format(remote_cache_spec.url)) if remote_cache_spec.instance_name: casd_args.append("--cas-instance={}".format(remote_cache_spec.instance_name)) if remote_cache_spec.server_cert_file: casd_args.append("--cas-server-cert={}".format(remote_cache_spec.server_cert_file)) if remote_cache_spec.client_key_file: casd_args.append("--cas-client-key={}".format(remote_cache_spec.client_key_file)) casd_args.append("--cas-client-cert={}".format(remote_cache_spec.client_cert_file)) if remote_cache_spec.access_token_file: casd_args.append("--cas-access-token={}".format(remote_cache_spec.access_token_file)) if remote_cache_spec.access_token_reload_interval is not None: casd_args.append( "--cas-token-reload-interval={}".format(remote_cache_spec.access_token_reload_interval) ) if remote_cache_spec.keepalive_time is not None: casd_args.append("--cas-keepalive-time={}".format(remote_cache_spec.keepalive_time)) if remote_cache_spec.retry_limit is not None: casd_args.append("--cas-retry-limit={}".format(remote_cache_spec.retry_limit)) if remote_cache_spec.retry_delay is not None: casd_args.append("--cas-retry-delay={}".format(remote_cache_spec.retry_delay)) if remote_cache_spec.request_timeout is not None: casd_args.append("--cas-request-timeout={}".format(remote_cache_spec.request_timeout)) casd_args.append(path) self._start_time = time.time() self._logfile = self._rotate_and_get_next_logfile() # Create a new process group for buildbox-casd such that SIGINT won't reach it. if sys.version_info >= (3, 11): process_group_kwargs = {"process_group": 0} else: process_group_kwargs = {"preexec_fn": os.setpgrp} with open(self._logfile, "w", encoding="utf-8") as logfile_fp: # The frontend will take care of terminating buildbox-casd. self.process = subprocess.Popen( # pylint: disable=consider-using-with casd_args, cwd=path, stdout=logfile_fp, stderr=subprocess.STDOUT, env=self.__buildbox_casd_env(), **process_group_kwargs ) self._casd_channel = None self._bytestream = None self._casd_cas = None self._local_cas = None self._asset_fetch = None self._asset_push = None self._exec_service = None self._operations_service = None self._ac_service = None self._shutdown_requested = False self._lock = threading.Lock() def __buildbox_casd(self): return utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox") def __buildbox_casd_env(self): env = os.environ.copy() # buildbox-casd needs to have buildbox-fuse in its PATH at runtime, # otherwise it will fallback to the HardLinkStager backend. bundled_buildbox_dir = os.path.join(_site.subprojects, "buildbox") if os.path.exists(bundled_buildbox_dir): path = env.get("PATH", "").split(os.pathsep) path = [bundled_buildbox_dir] + path env["PATH"] = os.pathsep.join(path) return env # _check_casd_version() # # Check for minimal acceptable version of buildbox-casd. # # If the version is unacceptable, then an error is raised. # # If buildbox-casd was built without version information available (or has reported # version information with a string which we are unprepared to parse), then # a warning is produced to inform the user. # def _check_casd_version(self, messenger): # # We specify a trailing "path" argument because some versions of buildbox-casd # require specifying the storage path even for invoking the --version option. # casd_args = [self.__buildbox_casd()] casd_args.append("--version") casd_args.append("/") try: version_output = subprocess.check_output(casd_args) except CalledProcessError as e: raise CASCacheError("Error checking buildbox-casd version") from e version_output = version_output.decode("utf-8") version_match = re.match(r".*buildbox-casd (\d+).(\d+).(\d+).*", version_output) if version_match: version_major = int(version_match.group(1)) version_minor = int(version_match.group(2)) version_micro = int(version_match.group(3)) acceptable_version = True if version_major < _REQUIRED_CASD_MAJOR: acceptable_version = False elif version_major == _REQUIRED_CASD_MAJOR: if version_minor < _REQUIRED_CASD_MINOR: acceptable_version = False elif version_minor == _REQUIRED_CASD_MINOR: if version_micro < _REQUIRED_CASD_MICRO: acceptable_version = False if not acceptable_version: raise CASCacheError( "BuildStream requires buildbox-casd >= {}.{}.{}".format( _REQUIRED_CASD_MAJOR, _REQUIRED_CASD_MINOR, _REQUIRED_CASD_MICRO ), detail="Currently installed: {}".format(version_output), ) elif messenger: messenger.warn( "Unable to determine buildbox-casd version", detail="buildbox-casd reported: {}".format(version_output) ) # _make_socket_path() # # Create a path to the CASD socket, ensuring that we don't exceed # the socket path limit. # # Note that we *may* exceed the path limit if the python-chosen # tmpdir path is very long, though this should be /tmp. # # Args: # path (str): The root directory for the CAS repository. # # Returns: # (str) - The path to the CASD socket. # def _make_socket_path(self, path): self._socket_tempdir = tempfile.mkdtemp(prefix="buildstream") # mkdtemp will create this directory in the "most secure" # way. This translates to "u+rwx,go-rwx". # # This is a good thing, generally, since it prevents us # from leaking sensitive information to other users, but # it's a problem for the workflow for userchroot, since # the setuid casd binary will not share a uid with the # user creating the tempdir. # # Instead, we chmod the directory 755, and only place a # symlink to the CAS directory in here, which will allow the # CASD process RWX access to a directory without leaking build # information. os.chmod( self._socket_tempdir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, ) os.symlink(path, os.path.join(self._socket_tempdir, "cas")) # FIXME: There is a potential race condition here; if multiple # instances happen to create the same socket path, at least # one will try to talk to the same server as us. # # There's no real way to avoid this from our side; we'd need # buildbox-casd to tell us that it could not create a fresh # socket. # # We could probably make this even safer by including some # thread/process-specific information, but we're not really # supporting this use case anyway; it's mostly here fore # testing, and to help more gracefully handle the situation. # # Note: this uses the same random string generation principle # as cpython, so this is probably a safe file name. random_name = "".join([random.choice("abcdefghijklmnopqrstuvwxyz0123456789_") for _ in range(8)]) socket_name = "casserver-{}.sock".format(random_name) return os.path.join(self._socket_tempdir, "cas", socket_name) # _rotate_and_get_next_logfile() # # Get the logfile to use for casd # # This will ensure that we don't create too many casd log files by # rotating the logs and only keeping _CASD_MAX_LOGFILES logs around. # # Returns: # (str): the path to the log file to use # def _rotate_and_get_next_logfile(self): try: existing_logs = sorted(os.listdir(self._log_dir)) except FileNotFoundError: os.makedirs(self._log_dir) else: while len(existing_logs) >= _CASD_MAX_LOGFILES: logfile_to_delete = existing_logs.pop(0) os.remove(os.path.join(self._log_dir, logfile_to_delete)) return os.path.join(self._log_dir, str(self._start_time) + ".log") # release_resources() # # Terminate the process and release related resources. # def release_resources(self, messenger=None): self._shutdown_requested = True with self._lock: if self._casd_channel: self._ac_service = None self._operations_service = None self._exec_service = None self._asset_push = None self._asset_fetch = None self._local_cas = None self._casd_cas = None self._bytestream = None self._casd_channel.close() self._casd_channel = None self._terminate(messenger) self.process = None shutil.rmtree(self._socket_tempdir) # _terminate() # # Terminate the buildbox casd process. # def _terminate(self, messenger=None): return_code = self.process.poll() if return_code is not None: # buildbox-casd is already dead if messenger: messenger.bug( "Buildbox-casd died during the run. Exit code: {}, Logs: {}".format(return_code, self._logfile) ) return self.process.terminate() try: # Don't print anything if buildbox-casd terminates quickly return_code = self.process.wait(timeout=0.5) except subprocess.TimeoutExpired: if messenger: cm = messenger.timed_activity("Terminating buildbox-casd") else: cm = contextlib.suppress() with cm: try: return_code = self.process.wait(timeout=15) except subprocess.TimeoutExpired: self.process.kill() self.process.wait(timeout=15) if messenger: messenger.warn("Buildbox-casd didn't exit in time and has been killed") return if return_code != 0 and messenger: messenger.bug( "Buildbox-casd didn't exit cleanly. Exit code: {}, Logs: {}".format(return_code, self._logfile) ) def _establish_connection(self): with self._lock: if self._casd_channel is not None: return while not os.path.exists(self._socket_path): # casd is not ready yet, try again after a 10ms delay, # but don't wait for more than specified timeout period if time.time() > self._start_time + _CASD_TIMEOUT: raise CASCacheError("Timed out waiting for buildbox-casd to become ready") if self._shutdown_requested: # Shutdown has been requested, we can exit return # check that process is still alive try: proc = psutil.Process(self.process.pid) if proc.status() == psutil.STATUS_ZOMBIE: proc.wait() if not proc.is_running(): if self._shutdown_requested: return raise CASCacheError("buildbox-casd process died before connection could be established") except psutil.NoSuchProcess: if self._shutdown_requested: return raise CASCacheError("buildbox-casd process died before connection could be established") time.sleep(0.01) self._casd_channel = grpc.insecure_channel(self._connection_string) self._bytestream = bytestream_pb2_grpc.ByteStreamStub(self._casd_channel) self._casd_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self._casd_channel) self._local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(self._casd_channel) self._asset_fetch = remote_asset_pb2_grpc.FetchStub(self._casd_channel) self._asset_push = remote_asset_pb2_grpc.PushStub(self._casd_channel) self._exec_service = remote_execution_pb2_grpc.ExecutionStub(self._casd_channel) self._operations_service = operations_pb2_grpc.OperationsStub(self._casd_channel) self._ac_service = remote_execution_pb2_grpc.ActionCacheStub(self._casd_channel) # get_cas(): # # Return ContentAddressableStorage stub for buildbox-casd channel. # def get_cas(self): if self._casd_channel is None: self._establish_connection() return self._casd_cas # get_local_cas(): # # Return LocalCAS stub for buildbox-casd channel. # def get_local_cas(self): if self._local_cas is None: self._establish_connection() return self._local_cas def get_bytestream(self): if self._bytestream is None: self._establish_connection() return self._bytestream # get_asset_fetch(): # # Return Remote Asset Fetch stub for buildbox-casd channel. # def get_asset_fetch(self): if self._casd_channel is None: self._establish_connection() return self._asset_fetch # get_asset_push(): # # Return Remote Asset Push stub for buildbox-casd channel. # def get_asset_push(self): if self._casd_channel is None: self._establish_connection() return self._asset_push # get_exec_service(): # # Return Remote Execution stub for buildbox-casd channel. # def get_exec_service(self): if self._casd_channel is None: self._establish_connection() return self._exec_service # get_operations_service(): # # Return Operations stub for buildbox-casd channel. # def get_operations_service(self): if self._casd_channel is None: self._establish_connection() return self._operations_service # get_ac_service(): # # Return Action Cache stub for buildbox-casd channel. # def get_ac_service(self): if self._casd_channel is None: self._establish_connection() return self._ac_service apache-buildstream-27ae392/src/buildstream/_cas/casremote.py000066400000000000000000000127561514607367700242000ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from .._protos.google.rpc import code_pb2 from .._protos.build.buildgrid import local_cas_pb2 from .._remote import BaseRemote from .._exceptions import CASRemoteError # The default limit for gRPC messages is 4 MiB. # Limit payload to 1 MiB to leave sufficient headroom for metadata. _MAX_PAYLOAD_BYTES = 1024 * 1024 # How many digests to put in a single gRPC message. # A 256-bit hash requires 64 bytes of space (hexadecimal encoding). # 80 bytes provide sufficient space for hash, size, and protobuf overhead. _MAX_DIGESTS = _MAX_PAYLOAD_BYTES / 80 class BlobNotFound(CASRemoteError): def __init__(self, blob, msg): self.blob = blob super().__init__(msg) # Represents a single remote CAS cache. # class CASRemote(BaseRemote): def __init__(self, spec, casd, **kwargs): super().__init__(spec, **kwargs) self.casd = casd self.local_cas_instance_name = None # check_remote # _configure_protocols(): # # Configure remote-specific protocols. This method should *never* # be called outside of init(). # def _configure_protocols(self): if not self.spec: # Remote cache (handled by default instance in casd) self.local_cas_instance_name = "" return local_cas = self.casd.get_local_cas() request = local_cas_pb2.GetInstanceNameForRemotesRequest() self.spec.to_localcas_remote(request.content_addressable_storage) response = local_cas.GetInstanceNameForRemotes(request) self.local_cas_instance_name = response.instance_name # Represents a batch of blobs queued for fetching. # class _CASBatchRead: def __init__(self, remote): self._remote = remote self._requests = [] self._request = None self._sent = False def add(self, digest): assert not self._sent if not self._request or len(self._request.blob_digests) >= _MAX_DIGESTS: self._request = local_cas_pb2.FetchMissingBlobsRequest() self._request.instance_name = self._remote.local_cas_instance_name self._requests.append(self._request) request_digest = self._request.blob_digests.add() request_digest.CopyFrom(digest) def send(self, *, missing_blobs=None): assert not self._sent self._sent = True if not self._requests: return local_cas = self._remote.casd.get_local_cas() for request in self._requests: batch_response = local_cas.FetchMissingBlobs(request) for response in batch_response.responses: if response.status.code == code_pb2.NOT_FOUND: if missing_blobs is None: raise BlobNotFound( response.digest.hash, "Failed to download blob {}: {}".format(response.digest.hash, response.status.code), ) missing_blobs.append(response.digest) if response.status.code != code_pb2.OK: raise CASRemoteError( "Failed to download blob {}: {}".format(response.digest.hash, response.status.code) ) if response.digest.size_bytes != len(response.data): raise CASRemoteError( "Failed to download blob {}: expected {} bytes, received {} bytes".format( response.digest.hash, response.digest.size_bytes, len(response.data) ) ) # Represents a batch of blobs queued for upload. # class _CASBatchUpdate: def __init__(self, remote): self._remote = remote self._requests = [] self._request = None self._sent = False def add(self, digest): assert not self._sent if not self._request or len(self._request.blob_digests) >= _MAX_DIGESTS: self._request = local_cas_pb2.UploadMissingBlobsRequest() self._request.instance_name = self._remote.local_cas_instance_name self._requests.append(self._request) request_digest = self._request.blob_digests.add() request_digest.CopyFrom(digest) def send(self): assert not self._sent self._sent = True if not self._requests: return local_cas = self._remote.casd.get_local_cas() for request in self._requests: batch_response = local_cas.UploadMissingBlobs(request) for response in batch_response.responses: if response.status.code != code_pb2.OK: if response.status.code == code_pb2.RESOURCE_EXHAUSTED: reason = "cache-too-full" else: reason = None raise CASRemoteError( "Failed to upload blob {}: {}".format(response.digest.hash, response.status.code), reason=reason, ) apache-buildstream-27ae392/src/buildstream/_cas/casserver.py000066400000000000000000000212031514607367700241760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jürg Billeter from concurrent import futures from enum import Enum import contextlib import logging import os import sys import grpc import click from .._protos.build.bazel.remote.asset.v1 import remote_asset_pb2_grpc from .. import _signals from .._protos.build.bazel.remote.execution.v2 import ( remote_execution_pb2, remote_execution_pb2_grpc, ) from .._protos.google.bytestream import bytestream_pb2_grpc from .casdprocessmanager import CASDProcessManager # The default limit for gRPC messages is 4 MiB. # Limit payload to 1 MiB to leave sufficient headroom for metadata. _MAX_PAYLOAD_BYTES = 1024 * 1024 # LogLevel(): # # Manage log level choices using click. # class LogLevel(click.Choice): # Levels(): # # Represents the actual buildbox-casd log level. # class Levels(Enum): WARNING = "warning" INFO = "info" TRACE = "trace" def __init__(self): super().__init__([m.lower() for m in LogLevel.Levels._member_names_]) # pylint: disable=no-member def convert(self, value, param, ctx) -> "LogLevel.Levels": if isinstance(value, LogLevel.Levels): value = value.value return LogLevel.Levels(super().convert(value, param, ctx)) @classmethod def get_logging_equivalent(cls, level) -> int: equivalents = { cls.Levels.WARNING: logging.WARNING, cls.Levels.INFO: logging.INFO, cls.Levels.TRACE: logging.DEBUG, } return equivalents[level] # create_server(): # # Create gRPC CAS artifact server as specified in the Remote Execution API. # # Args: # repo (str): Path to CAS repository # enable_push (bool): Whether to allow blob uploads and artifact updates # index_only (bool): Whether to store CAS blobs or only artifacts # @contextlib.contextmanager def create_server(repo, *, enable_push, quota, index_only, log_level=LogLevel.Levels.WARNING): logger = logging.getLogger("buildstream._cas.casserver") logger.setLevel(LogLevel.get_logging_equivalent(log_level)) handler = logging.StreamHandler(sys.stderr) handler.setFormatter(logging.Formatter(fmt="%(levelname)s: %(funcName)s: %(message)s")) logger.addHandler(handler) casd = CASDProcessManager( os.path.abspath(repo), os.path.join(os.path.abspath(repo), "logs"), log_level, quota, None, False, None ) try: # Use max_workers default from Python 3.5+ max_workers = (os.cpu_count() or 1) * 5 server = grpc.server(futures.ThreadPoolExecutor(max_workers)) if not index_only: bytestream_pb2_grpc.add_ByteStreamServicer_to_server( _ByteStreamServicer(casd, enable_push=enable_push), server ) remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server( _ContentAddressableStorageServicer(casd, enable_push=enable_push), server ) remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(_CapabilitiesServicer(), server) # Remote Asset API remote_asset_pb2_grpc.add_FetchServicer_to_server(_FetchServicer(casd), server) if enable_push: remote_asset_pb2_grpc.add_PushServicer_to_server(_PushServicer(casd), server) # Ensure we have the signal handler set for SIGTERM # This allows threads from GRPC to call our methods that do register # handlers at exit. with _signals.terminator(lambda: None): yield server finally: casd.release_resources() class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer): def __init__(self, casd, *, enable_push): super().__init__() self.bytestream = casd.get_bytestream() self.enable_push = enable_push self.logger = logging.getLogger("buildstream._cas.casserver") def Read(self, request, context): self.logger.debug("Reading %s", request.resource_name) try: yield from self.bytestream.Read(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) def Write(self, request_iterator, context): # Note that we can't easily give more information because the # data is stuck in an iterator that will be consumed if read. self.logger.debug("Writing data") try: ret = self.bytestream.Write(request_iterator) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddressableStorageServicer): def __init__(self, casd, *, enable_push): super().__init__() self.cas = casd.get_cas() self.enable_push = enable_push self.logger = logging.getLogger("buildstream._cas.casserver") def FindMissingBlobs(self, request, context): self.logger.info("Finding '%s'", request.blob_digests) try: ret = self.cas.FindMissingBlobs(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret def BatchReadBlobs(self, request, context): self.logger.info("Reading '%s'", request.digests) try: ret = self.cas.BatchReadBlobs(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret def BatchUpdateBlobs(self, request, context): self.logger.info("Updating: '%s'", [request.digest for request in request.requests]) try: ret = self.cas.BatchUpdateBlobs(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer): def __init__(self): self.logger = logging.getLogger("buildstream._cas.casserver") def GetCapabilities(self, request, context): self.logger.info("Retrieving capabilities") response = remote_execution_pb2.ServerCapabilities() cache_capabilities = response.cache_capabilities cache_capabilities.digest_functions.append(remote_execution_pb2.DigestFunction.SHA256) cache_capabilities.action_cache_update_capabilities.update_enabled = False cache_capabilities.max_batch_total_size_bytes = _MAX_PAYLOAD_BYTES cache_capabilities.symlink_absolute_path_strategy = remote_execution_pb2.SymlinkAbsolutePathStrategy.ALLOWED response.deprecated_api_version.major = 2 response.low_api_version.major = 2 response.high_api_version.major = 2 return response class _FetchServicer(remote_asset_pb2_grpc.FetchServicer): def __init__(self, casd): super().__init__() self.fetch = casd.get_asset_fetch() self.logger = logging.getLogger("buildstream._cas.casserver") def FetchBlob(self, request, context): self.logger.debug("FetchBlob '%s'", request.uris) try: ret = self.fetch.FetchBlob(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret def FetchDirectory(self, request, context): self.logger.debug("FetchDirectory '%s'", request.uris) try: ret = self.fetch.FetchDirectory(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret class _PushServicer(remote_asset_pb2_grpc.PushServicer): def __init__(self, casd): super().__init__() self.push = casd.get_asset_push() self.logger = logging.getLogger("buildstream._cas.casserver") def PushBlob(self, request, context): self.logger.debug("PushBlob '%s'", request.uris) try: ret = self.push.PushBlob(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret def PushDirectory(self, request, context): self.logger.debug("PushDirectory '%s'", request.uris) try: ret = self.push.PushDirectory(request) except grpc.RpcError as err: context.abort(err.code(), err.details()) return ret apache-buildstream-27ae392/src/buildstream/_context.py000066400000000000000000000771371514607367700231400ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from typing import TYPE_CHECKING, List, Dict, Set, Optional, Iterable import os import shutil from . import utils from . import _site from . import _yaml from ._exceptions import LoadError from .exceptions import LoadErrorReason from ._messenger import Messenger from ._profile import Topics, PROFILER from ._platform import Platform from ._artifactcache import ArtifactCache from ._elementsourcescache import ElementSourcesCache from ._remotespec import RemoteSpec, RemoteExecutionSpec from ._sourcecache import SourceCache from ._cas import CASCache, CASDProcessManager, CASLogLevel from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction, _SourceUriPolicy from ._workspaces import Workspaces, WorkspaceProjectCache from .node import Node, MappingNode if TYPE_CHECKING: # pylint: disable=cyclic-import from ._project import Project # pylint: enable=cyclic-import # _CacheConfig # # A convenience object for parsing artifact/source cache configurations # class _CacheConfig: def __init__(self, override_projects: bool, remote_specs: List[RemoteSpec]): self.override_projects: bool = override_projects self.remote_specs: List[RemoteSpec] = remote_specs @classmethod def new_from_node(cls, node: MappingNode) -> "_CacheConfig": node.validate_keys(["override-project-caches", "servers"]) servers = node.get_sequence("servers", default=[], allowed_types=[MappingNode]) override_projects: bool = node.get_bool("override-project-caches", default=False) remote_specs: List[RemoteSpec] = [RemoteSpec.new_from_node(node) for node in servers] return cls(override_projects, remote_specs) # Context() # # The Context object holds all of the user preferences # and context for a given invocation of BuildStream. # # This is a collection of data from configuration files and command # line arguments and consists of information such as where to store # logs and artifacts, where to perform builds and cache downloaded sources, # verbosity levels and basically anything pertaining to the context # in which BuildStream was invoked. # class Context: def __init__(self, *, use_casd: bool = True) -> None: # Whether we are running as part of a test suite. This is only relevant # for developing BuildStream itself. self.is_running_in_test_suite: bool = "BST_TEST_SUITE" in os.environ # Filename indicating which configuration file was used, or None for the defaults self.config_origin: Optional[str] = None # The directory under which other directories are based self.cachedir: Optional[str] = None # The directory where various sources are stored self.sourcedir: Optional[str] = None # The directory where build sandboxes will be created self.builddir: Optional[str] = None # The directory for CAS self.casdir: Optional[str] = None # Whether to use casd - meant for interfaces such as # completion where casd is not required self.use_casd: bool = use_casd # Whether we are going to build, this is required for some conditional # functionality to take place only in the case that we are building. self.build: bool = False # The directory for artifact protos self.artifactdir: Optional[str] = None # The directory for temporary files self.tmpdir: Optional[str] = None # Default root location for workspaces self.workspacedir: Optional[str] = None # The global remote execution configuration self.remote_execution_specs: Optional[RemoteExecutionSpec] = None # The configured artifact cache remote specs for each project self.project_artifact_cache_specs: Dict[str, List[RemoteSpec]] = {} # The configured source cache remote specs for each project self.project_source_cache_specs: Dict[str, List[RemoteSpec]] = {} # The directory to store build logs self.logdir: Optional[str] = None # The abbreviated cache key length to display in the UI self.log_key_length: Optional[int] = None # Whether debug mode is enabled self.log_debug: Optional[int] = None # Whether verbose mode is enabled self.log_verbose: Optional[int] = None # Maximum number of lines to print from build logs self.log_error_lines: Optional[int] = None # Maximum number of lines to print in the master log for a detailed message self.log_message_lines: Optional[int] = None # Format string for printing the pipeline at startup time self.log_element_format: Optional[str] = None # Format string for printing message lines in the master log self.log_message_format: Optional[str] = None # Wether to rate limit the updating of the bst output where applicable self.log_throttle_updates: Optional[int] = None # Maximum number of fetch or refresh tasks self.sched_fetchers: Optional[int] = None # Maximum number of build tasks self.sched_builders: Optional[int] = None # Maximum number of push tasks self.sched_pushers: Optional[int] = None # Maximum number of retries for network tasks self.sched_network_retries: Optional[int] = None # What to do when a build fails in non interactive mode self.sched_error_action: Optional[str] = None # Maximum jobs per build self.build_max_jobs: Optional[int] = None # Retry any existing failed builds self.build_retry_failed: Optional[bool] = None # Control which dependencies to build self.build_dependencies: Optional[_PipelineSelection] = None # Control which URIs can be accessed when fetching sources self.fetch_source: Optional[str] = None # Control which URIs can be accessed when tracking sources self.track_source: Optional[str] = None # Size of the artifact cache in bytes self.config_cache_quota: Optional[int] = None # User specified cache quota, used for display messages self.config_cache_quota_string: Optional[str] = None # Reserved disk space for local cache in bytes self.config_cache_reserved: Optional[int] = None # Low watermark for local cache (ratio relative to effective quota) self.config_cache_low_watermark: Optional[float] = None # Remote cache server self.remote_cache_spec: Optional[RemoteSpec] = None # Whether or not to attempt to pull build trees globally self.pull_buildtrees: Optional[bool] = None # Whether or not to cache build trees on artifact creation self.cache_buildtrees: Optional[str] = None # Don't shoot the messenger self.messenger: Messenger = Messenger() # Make sure the XDG vars are set in the environment before loading anything self._init_xdg() # # Private variables # # Whether elements must be rebuilt when their dependencies have changed self._strict_build_plan: Optional[bool] = None # Lists of globally configured cache configurations self._global_artifact_cache_config: _CacheConfig = _CacheConfig(False, []) self._global_source_cache_config: _CacheConfig = _CacheConfig(False, []) # Set of all actively configured remote specs self._active_artifact_cache_specs: Set[RemoteSpec] = set() self._active_source_cache_specs: Set[RemoteSpec] = set() self._platform: Optional[Platform] = None self._artifactcache: Optional[ArtifactCache] = None self._elementsourcescache: Optional[ElementSourcesCache] = None self._sourcecache: Optional[SourceCache] = None self._projects: List["Project"] = [] self._project_overrides: MappingNode = Node.from_dict({}) self._workspaces: Optional[Workspaces] = None self._workspace_project_cache: WorkspaceProjectCache = WorkspaceProjectCache() self._casd: Optional[CASDProcessManager] = None self._cascache: Optional[CASCache] = None # __enter__() # # Called when entering the with-statement context. # def __enter__(self) -> "Context": return self # __exit__() # # Called when exiting the with-statement context. # def __exit__(self, exc_type, exc_value, traceback) -> None: if self._cascache: self._cascache.release_resources() if self._casd: self._casd.release_resources(self.messenger) self._casd = None # load() # # Loads the configuration files # # Args: # config: The user specified configuration file, if any # # Raises: # LoadError # # This will first load the BuildStream default configuration and then # override that configuration with the configuration file indicated # by *config*, if any was specified. # @PROFILER.profile(Topics.LOAD_CONTEXT, "load") def load(self, config: Optional[str] = None) -> None: # If a specific config file is not specified, default to trying # a $XDG_CONFIG_HOME/buildstream.conf file # if not config: # # Support parallel installations of BuildStream by first # trying a (major point) version specific configuration file # and then falling back to buildstream.conf. # for config_filename in ("buildstream2.conf", "buildstream.conf"): default_config = os.path.join(os.environ["XDG_CONFIG_HOME"], config_filename) if os.path.exists(default_config): config = default_config break # Load default config # defaults = _yaml.load(_site.default_user_config, shortname="userconfig.yaml") if config: self.config_origin = os.path.abspath(config) # Here we use the fullpath as the shortname as well, as it is useful to have # a fullpath displayed in errors for the user configuration user_config = _yaml.load(config, shortname=config) user_config._composite(defaults) # Give obsoletion warnings if "builddir" in defaults: raise LoadError("builddir is obsolete, use cachedir", LoadErrorReason.INVALID_DATA) if "artifactdir" in defaults: raise LoadError("artifactdir is obsolete", LoadErrorReason.INVALID_DATA) defaults.validate_keys( [ "cachedir", "sourcedir", "builddir", "logdir", "scheduler", "build", "fetch", "track", "artifacts", "source-caches", "logging", "projects", "cache", "workspacedir", "remote-execution", ] ) for directory in ["cachedir", "sourcedir", "logdir", "workspacedir"]: # Allow the ~ tilde expansion and any environment variables in # path specification in the config files. # path = defaults.get_str(directory) path = os.path.expanduser(path) path = os.path.expandvars(path) path = os.path.normpath(path) setattr(self, directory, path) # Relative paths don't make sense in user configuration. The exception is # workspacedir where `.` is useful as it will be combined with the name # specified on the command line. if not os.path.isabs(path) and not (directory == "workspacedir" and path == "."): raise LoadError("{} must be an absolute path".format(directory), LoadErrorReason.INVALID_DATA) # add directories not set by users assert self.cachedir self.tmpdir = os.path.join(self.cachedir, "tmp") self.casdir = os.path.join(self.cachedir, "cas") self.builddir = os.path.join(self.cachedir, "build") self.artifactdir = os.path.join(self.cachedir, "artifacts", "refs") # Move old artifact cas to cas if it exists and create symlink old_casdir = os.path.join(self.cachedir, "artifacts", "cas") if os.path.exists(old_casdir) and not os.path.islink(old_casdir) and not os.path.exists(self.casdir): os.rename(old_casdir, self.casdir) os.symlink(self.casdir, old_casdir) # Cleanup old extract directories old_extractdir = os.path.join(self.cachedir, "extract") if os.path.isdir(old_extractdir): shutil.rmtree(old_extractdir, ignore_errors=True) # Load quota configuration # We need to find the first existing directory in the path of our # casdir - the casdir may not have been created yet. cache = defaults.get_mapping("cache") cache.validate_keys( ["quota", "reserved-disk-space", "low-watermark", "storage-service", "pull-buildtrees", "cache-buildtrees"] ) cas_volume = self.casdir while not os.path.exists(cas_volume): cas_volume = os.path.dirname(cas_volume) self.config_cache_quota_string = cache.get_str("quota") try: self.config_cache_quota = utils._parse_size(self.config_cache_quota_string, cas_volume) except utils.UtilError as e: raise LoadError( "{}\nPlease specify the value in bytes or as a % of full disk space.\n" "\nValid values are, for example: 800M 10G 1T 50%\n".format(str(e)), LoadErrorReason.INVALID_DATA, ) from e cache_reserved_string = cache.get_str("reserved-disk-space") try: self.config_cache_reserved = utils._parse_size(cache_reserved_string, cas_volume) if self.config_cache_reserved is None: provenance = cache.get_scalar("reserved-disk-space").get_provenance() raise LoadError( "{}: Please specify the value in bytes or as a % of full disk space.\n" "\nValid values are, for example: 2G 5%\n".format(provenance), LoadErrorReason.INVALID_DATA, ) except utils.UtilError as e: raise LoadError( "{}\nPlease specify the value in bytes or as a % of full disk space.\n" "\nValid values are, for example: 2G 5%\n".format(str(e)), LoadErrorReason.INVALID_DATA, ) from e low_watermark_string = cache.get_str("low-watermark") try: self.config_cache_low_watermark = utils._parse_percentage(low_watermark_string) except utils.UtilError as e: raise LoadError( "{}\nPlease specify the value as a % of the cache quota.".format(str(e)), LoadErrorReason.INVALID_DATA, ) from e remote_cache = cache.get_mapping("storage-service", default=None) if remote_cache: self.remote_cache_spec = RemoteSpec.new_from_node(remote_cache) # Load global artifact cache configuration cache_config = defaults.get_mapping("artifacts", default={}) self._global_artifact_cache_config = _CacheConfig.new_from_node(cache_config) # Load global source cache configuration cache_config = defaults.get_mapping("source-caches", default={}) self._global_source_cache_config = _CacheConfig.new_from_node(cache_config) # Load the global remote execution config remote_execution = defaults.get_mapping("remote-execution", default=None) if remote_execution: self.remote_execution_specs = self._load_remote_execution(remote_execution) # Load pull build trees configuration self.pull_buildtrees = cache.get_bool("pull-buildtrees") # Load cache build trees configuration self.cache_buildtrees = cache.get_enum("cache-buildtrees", _CacheBuildTrees) # Load logging config logging = defaults.get_mapping("logging") logging.validate_keys( [ "key-length", "verbose", "error-lines", "message-lines", "debug", "element-format", "message-format", "throttle-ui-updates", ] ) self.log_key_length = logging.get_int("key-length") self.log_debug = logging.get_bool("debug") self.log_verbose = logging.get_bool("verbose") self.log_error_lines = logging.get_int("error-lines") self.log_message_lines = logging.get_int("message-lines") self.log_element_format = logging.get_str("element-format") self.log_message_format = logging.get_str("message-format") self.log_throttle_updates = logging.get_bool("throttle-ui-updates") # Load scheduler config scheduler = defaults.get_mapping("scheduler") scheduler.validate_keys(["on-error", "fetchers", "builders", "pushers", "network-retries"]) self.sched_error_action = scheduler.get_enum("on-error", _SchedulerErrorAction) self.sched_fetchers = scheduler.get_int("fetchers") self.sched_builders = scheduler.get_int("builders") self.sched_pushers = scheduler.get_int("pushers") self.sched_network_retries = scheduler.get_int("network-retries") # Load build config build = defaults.get_mapping("build") build.validate_keys(["max-jobs", "retry-failed", "dependencies"]) self.build_max_jobs = build.get_int("max-jobs") self.build_retry_failed = build.get_bool("retry-failed") dependencies = build.get_str("dependencies") if dependencies not in ["none", "run", "all"]: provenance = build.get_scalar("dependencies").get_provenance() raise LoadError( "{}: Invalid value for 'dependencies'. Choose 'none', 'run', or 'all'.".format(provenance), LoadErrorReason.INVALID_DATA, ) self.build_dependencies = _PipelineSelection(dependencies) # Load fetch config fetch = defaults.get_mapping("fetch") fetch.validate_keys(["source"]) self.fetch_source = fetch.get_enum("source", _SourceUriPolicy) # Load track config track = defaults.get_mapping("track") track.validate_keys(["source"]) self.track_source = track.get_enum("source", _SourceUriPolicy) # Load per-projects overrides self._project_overrides = defaults.get_mapping("projects", default={}) # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. for overrides_project in self._project_overrides.keys(): overrides = self._project_overrides.get_mapping(overrides_project) overrides.validate_keys( ["artifacts", "source-caches", "options", "strict", "default-mirror", "remote-execution", "mirrors"] ) @property def platform(self) -> Platform: if not self._platform: self._platform = Platform.create_instance() return self._platform @property def artifactcache(self) -> ArtifactCache: if not self._artifactcache: self._artifactcache = ArtifactCache(self) return self._artifactcache @property def elementsourcescache(self) -> ElementSourcesCache: if not self._elementsourcescache: self._elementsourcescache = ElementSourcesCache(self) return self._elementsourcescache @property def sourcecache(self) -> SourceCache: if not self._sourcecache: self._sourcecache = SourceCache(self) return self._sourcecache @property def effective_build_max_jobs(self) -> int: # Based on some testing (mainly on AWS), maximum effective # max-jobs value seems to be around 8-10 if we have enough cores # users should set values based on workload and build infrastructure return self.build_max_jobs or self.platform.get_cpu_count(8) # add_project(): # # Add a project to the context. # # Args: # project: The project to add # def add_project(self, project: "Project") -> None: if not self._projects: self._workspaces = Workspaces(project, self._workspace_project_cache) self._projects.append(project) # get_projects(): # # Return the list of projects in the context. # # Returns: # The list of projects # def get_projects(self) -> Iterable["Project"]: return self._projects # get_toplevel_project(): # # Return the toplevel project, the one which BuildStream was # invoked with as opposed to a junctioned subproject. # # Returns: # (Project): The toplevel Project object, or None # def get_toplevel_project(self) -> "Project": # # It is an error to call this before a toplevel # project is added # return self._projects[0] # initialize_remotes() # # This will resolve what remotes each loaded project will interact # with an initialize the underlying asset cache modules. # # Note that this can be called more than once, in the case that # Stream() has loaded additional projects during the load cycle # and some state needs to be recalculated. # # Args: # connect_artifact_cache: Whether to try to contact remote artifact caches # connect_source_cache: Whether to try to contact remote source caches # artifact_remotes: Artifact cache remotes specified on the commmand line # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore artifact remotes specified by projects # def initialize_remotes( self, connect_artifact_cache: bool, connect_source_cache: bool, artifact_remotes: Iterable[RemoteSpec] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, ) -> None: # Ensure all projects are fully loaded. for project in self._projects: project.ensure_fully_loaded() # # If the global remote execution specs have been overridden by the # toplevel project, then adjust them now that we're all loaded. # project = self.get_toplevel_project() if project: override_node = self.get_overrides(project.name) remote_execution = override_node.get_mapping("remote-execution", default=None) if remote_execution: self.remote_execution_specs = self._load_remote_execution(remote_execution) # # Maintain our list of remote specs for artifact and source caches # for project in self._projects: artifact_specs: List[RemoteSpec] = [] source_specs: List[RemoteSpec] = [] if connect_artifact_cache: artifact_specs = self._resolve_specs_for_project( project, artifact_remotes, ignore_project_artifact_remotes, self._global_artifact_cache_config, "artifacts", "artifact_cache_specs", ) if connect_source_cache: source_specs = self._resolve_specs_for_project( project, source_remotes, ignore_project_source_remotes, self._global_source_cache_config, "source-caches", "source_cache_specs", ) # Advertize the per project remote specs publicly for the frontend self.project_artifact_cache_specs[project.name] = artifact_specs self.project_source_cache_specs[project.name] = source_specs # # Now that we know which remote specs are going to be used, maintain # our total set of overall active remote specs, this helps the asset cache # modules to maintain a remote connection for the required remotes. # for spec in artifact_specs: self._active_artifact_cache_specs.add(spec) for spec in source_specs: self._active_source_cache_specs.add(spec) # Now initialize the underlying asset caches # with self.messenger.timed_activity("Initializing remote caches", silent_nested=True): self.artifactcache.setup_remotes(self._active_artifact_cache_specs, self.project_artifact_cache_specs) self.elementsourcescache.setup_remotes(self._active_source_cache_specs, self.project_source_cache_specs) self.sourcecache.setup_remotes(self._active_source_cache_specs, self.project_source_cache_specs) # get_workspaces(): # # Return a Workspaces object containing a list of workspaces. # # Returns: # The Workspaces object # def get_workspaces(self) -> Workspaces: # # It is an error to call this early on before the Workspaces # has been instantiated # assert self._workspaces return self._workspaces # get_workspace_project_cache(): # # Return the WorkspaceProjectCache object used for this BuildStream invocation # # Returns: # The WorkspaceProjectCache object # def get_workspace_project_cache(self) -> WorkspaceProjectCache: return self._workspace_project_cache # get_overrides(): # # Fetch the override dictionary for the active project. This returns # a node loaded from YAML. # # Args: # project_name: The project name # # Returns: # The overrides dictionary for the specified project # def get_overrides(self, project_name: str) -> MappingNode: return self._project_overrides.get_mapping(project_name, default={}) # get_strict(): # # Fetch whether we are strict or not # # Returns: # Whether or not to use strict build plan # def get_strict(self) -> bool: if self._strict_build_plan is None: # Either we're not overridden or we've never worked it out before # so work out if we should be strict, and then cache the result toplevel = self.get_toplevel_project() overrides = self.get_overrides(toplevel.name) self._strict_build_plan = overrides.get_bool("strict", default=True) # If it was set by the CLI, it overrides any config # Ditto if we've already computed this, then we return the computed # value which we cache here too. return self._strict_build_plan def get_casd(self) -> CASDProcessManager: if self._casd is None: if self.log_debug: log_level = CASLogLevel.TRACE elif self.log_verbose: log_level = CASLogLevel.INFO else: log_level = CASLogLevel.WARNING assert self.logdir is not None, "log_directory is required for casd" log_dir = os.path.join(self.logdir, "_casd") assert self.sched_builders is not None, "builders configuration is required" self._casd = CASDProcessManager( self.cachedir, log_dir, log_level, self.config_cache_quota, self.remote_cache_spec, protect_session_blobs=True, messenger=self.messenger, reserved=self.config_cache_reserved, low_watermark=self.config_cache_low_watermark, local_jobs=self.sched_builders * self.effective_build_max_jobs, ) return self._casd def get_cascache(self) -> CASCache: if self._cascache is None: self._cascache = CASCache(self.cachedir, casd=self.get_casd(), remote_cache=bool(self.remote_cache_spec)) return self._cascache ###################################################### # Private methods # ###################################################### # _resolve_specs_for_project() # # Helper function to resolve which remote specs apply for a given project # # Args: # project: The project # cli_remotes: The remotes specified in the CLI # cli_override: Whether the CLI decided to override project suggestions # global_config: The global user configuration for this remote type # override_key: The key to lookup project overrides for this remote type # project_attribute: The Project attribute for project suggestions # # Returns: # The resolved remotes for this project. # def _resolve_specs_for_project( self, project: "Project", cli_remotes: Iterable[RemoteSpec], cli_override: bool, global_config: _CacheConfig, override_key: str, project_attribute: str, ) -> List[RemoteSpec]: # Early return if the CLI is taking full control if cli_override and cli_remotes: return list(cli_remotes) # Obtain the overrides override_node = self.get_overrides(project.name) override_config_node = override_node.get_mapping(override_key, default={}) override_config = _CacheConfig.new_from_node(override_config_node) # # Decide on what remotes to use from user config, if any # # Priority CLI -> Project overrides -> Global config # remotes: List[RemoteSpec] if cli_remotes: remotes = list(cli_remotes) elif override_config.remote_specs: remotes = override_config.remote_specs else: remotes = global_config.remote_specs # If any of the configs have disabled project remotes, return now # if cli_override or override_config.override_projects or global_config.override_projects: return remotes # If there are any project recommendations, append them at the end project_remotes = getattr(project, project_attribute) remotes = list(utils._deduplicate(remotes + project_remotes)) return remotes # Force the resolved XDG variables into the environment, # this is so that they can be used directly to specify # preferred locations of things from user configuration # files. def _init_xdg(self) -> None: if not os.environ.get("XDG_CACHE_HOME"): os.environ["XDG_CACHE_HOME"] = os.path.expanduser("~/.cache") if not os.environ.get("XDG_CONFIG_HOME"): os.environ["XDG_CONFIG_HOME"] = os.path.expanduser("~/.config") if not os.environ.get("XDG_DATA_HOME"): os.environ["XDG_DATA_HOME"] = os.path.expanduser("~/.local/share") def _load_remote_execution(self, node: MappingNode) -> Optional[RemoteExecutionSpec]: return RemoteExecutionSpec.new_from_node(node, remote_cache=bool(self.remote_cache_spec)) apache-buildstream-27ae392/src/buildstream/_elementproxy.py000066400000000000000000000153671514607367700242040ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from typing import TYPE_CHECKING, cast, Optional, Iterator, Dict, List, Sequence from .types import _Scope, OverlapAction from .utils import FileListResult from ._pluginproxy import PluginProxy if TYPE_CHECKING: from typing import Any from .node import MappingNode, ScalarNode, SequenceNode from .sandbox import Sandbox from .source import Source from .element import Element # pylint: disable=cyclic-import # ElementProxy() # # A PluginProxy for Element instances. # # Refer to the Element class for the documentation for these APIs. # class ElementProxy(PluginProxy): def __lt__(self, other): return self.name < other.name ############################################################## # Exposed proxied APIs # ############################################################## @property def project_name(self): return cast("Element", self._plugin).project_name @property def normal_name(self): return cast("Element", self._plugin).normal_name def sources(self) -> Iterator["Source"]: return cast("Element", self._plugin).sources() def dependencies( self, selection: Optional[Sequence["Element"]] = None, *, recurse: bool = True ) -> Iterator["Element"]: # # When dependencies() is called on a dependency of the main plugin Element, # we simply reroute the call to the original owning element, while specifying # this element as the selection. # # This ensures we only allow returning dependencies in the _Scope.RUN scope # of this element. # if selection is None: selection = [cast("Element", self._plugin)] # Return the iterable from the called generator, this is more performant than yielding from it return cast("Element", self._owner).dependencies(selection, recurse=recurse) def search(self, name: str) -> Optional["Element"]: # # Similarly to dependencies() above, we only search in the _Scope.RUN # of dependencies of the active element plugin. # return cast("Element", self._plugin)._search(_Scope.RUN, name) def node_subst_vars(self, node: "ScalarNode") -> str: return cast("Element", self._plugin).node_subst_vars(node) def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str]: return cast("Element", self._plugin).node_subst_sequence_vars(node) def compute_manifest( self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True ) -> str: return cast("Element", self._plugin).compute_manifest(include=include, exclude=exclude, orphans=orphans) def get_artifact_name(self, key: Optional[str] = None) -> str: return cast("Element", self._plugin).get_artifact_name(key=key) def stage_artifact( self, sandbox: "Sandbox", *, path: Optional[str] = None, action: str = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True ) -> FileListResult: owner = cast("Element", self._owner) element = cast("Element", self._plugin) overlap_collector = owner._overlap_collectors.get(sandbox) assert overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()" with overlap_collector.session(action, path): result = element._stage_artifact( sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans, owner=owner ) return result def stage_dependency_artifacts( self, sandbox: "Sandbox", selection: Optional[Sequence["Element"]] = None, *, path: Optional[str] = None, action: str = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True ) -> None: # # Same approach used here as in Element.dependencies() # if selection is None: selection = [cast("Element", self._plugin)] cast("Element", self._owner).stage_dependency_artifacts( sandbox, selection, path=path, action=action, include=include, exclude=exclude, orphans=orphans ) def integrate(self, sandbox: "Sandbox") -> None: cast("Element", self._plugin).integrate(sandbox) def get_public_data(self, domain: str) -> "MappingNode[Any]": return cast("Element", self._plugin).get_public_data(domain) def get_environment(self) -> Dict[str, str]: return cast("Element", self._plugin).get_environment() def get_variable(self, varname: str) -> Optional[str]: return cast("Element", self._plugin).get_variable(varname) ############################################################## # Element Internal APIs # ############################################################## # # Some functions the Element expects to call directly on the # proxy. # def _dependencies(self, scope, *, recurse=True, visited=None): # # We use a return statement even though this is a generator, simply # to avoid the generator overhead of yielding each element. # return cast("Element", self._plugin)._dependencies(scope, recurse=recurse, visited=visited) def _file_is_whitelisted(self, path): return cast("Element", self._plugin)._file_is_whitelisted(path) def _stage_artifact( self, sandbox: "Sandbox", *, path: Optional[str] = None, action: str = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True, owner: Optional["Element"] = None ) -> FileListResult: owner = cast("Element", self._owner) element = cast("Element", self._plugin) return element._stage_artifact( sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans, owner=owner ) apache-buildstream-27ae392/src/buildstream/_elementsources.py000066400000000000000000000365071514607367700245050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from contextlib import contextmanager from typing import TYPE_CHECKING, Iterator from . import _cachekey from ._exceptions import SkipJob from ._context import Context from ._protos.buildstream.v2 import source_pb2 from .plugin import Plugin from .storage._casbaseddirectory import CasBasedDirectory if TYPE_CHECKING: from typing import List # pylint: disable=cyclic-import from .source import Source from ._project import Project # pylint: enable=cyclic-import # An ElementSources object represents the combined sources of an element. class ElementSources: def __init__(self, context: Context, project: "Project", plugin: Plugin): self._context = context self._project = project self._plugin = plugin self._sources = [] # type: List[Source] self._sourcecache = context.sourcecache # Source cache self._elementsourcescache = context.elementsourcescache # Cache of staged element sources self._is_resolved = False # Whether the source is fully resolved or not self._cached = None # If the sources are known to be successfully cached in CAS self._cache_key = None # Our cached cache key self._proto = None # The cached Source proto # get_project(): # # Return the project associated with this object # def get_project(self): return self._project # add_source(): # # Append source to this list of element sources. # # Args: # source (Source): The source to add # def add_source(self, source): self._sources.append(source) # sources(): # # A generator function to enumerate the element sources # # Yields: # Source: The individual sources # def sources(self) -> Iterator["Source"]: yield from self._sources # track(): # # Calls track() on the Element sources # # Raises: # SourceError: If one of the element sources has an error # # Returns: # (list): A list of Source object ids and their new references # def track(self, workspace): refs = [] for source in self._sources: old_ref = source.get_ref() if source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK: with self._stage_previous_sources(source) as staging_directory: new_ref = source._track(previous_sources_dir=staging_directory) else: new_ref = source._track() refs.append((source._unique_id, new_ref, old_ref != new_ref)) # Complimentary warning that the new ref will be unused. if old_ref != new_ref and workspace: detail = ( "This source has an open workspace.\n" + "To start using the new reference, please close the existing workspace." ) source.warn("Updated reference will be ignored as source has open workspace", detail=detail) # Sources which do not implement track() will return None, produce # a SKIP message in the UI if all sources produce None # if all(ref is None for _, ref, _ in refs): raise SkipJob("Element sources are not trackable") return refs # stage_and_cache(): # # Stage the element sources to a directory in CAS # def stage_and_cache(self): vdir = self._stage() source_proto = source_pb2.Source() source_proto.files.CopyFrom(vdir._get_digest()) self._elementsourcescache.store_proto(self, source_proto) self._proto = source_proto self._cached = True # get_files(): # # Get a virtual directory for the staged source files # # Returns: # (Directory): The virtual directory object # def get_files(self): # Assert sources are cached assert self.cached() cas = self._context.get_cascache() return CasBasedDirectory(cas, digest=self._proto.files) # fetch_done() # # Indicates that fetching the sources for this element has been done. # # Args: # fetched_original (bool): Whether the original sources had been asked (and fetched) or not # def fetch_done(self, fetched_original): self._proto = self._elementsourcescache.load_proto(self) assert self._proto self._cached = True for source in self._sources: source._fetch_done(fetched_original) # push() # # Push the element's sources. # # Returns: # (bool): True if the remote was updated, False if it already existed # and no updated was required # def push(self): pushed = False for source in self.sources(): if source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH or source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: continue if self._sourcecache.contains(source) and self._sourcecache.push(source): pushed = True if self._elementsourcescache.push(self, self._plugin): pushed = True return pushed # init_workspace(): # # Initialises a new workspace from the element sources. # # Args: # directory (str): Path of the workspace to init # def init_workspace(self, directory: str): for source in self.sources(): if source._directory: srcdir = os.path.join(directory, source._directory) else: srcdir = directory os.makedirs(srcdir, exist_ok=True) source._init_workspace(srcdir) # fetch(): # # Fetch the combined or individual element sources. # # Raises: # SourceError: If one of the element sources has an error # def fetch(self): if self._cached is None: self.query_cache() if self.cached(): return # Try to fetch staged sources from remote source cache if self._elementsourcescache.has_fetch_remotes() and self._elementsourcescache.pull(self, self._plugin): self.fetch_done(False) return # Otherwise, fetch individual sources self.fetch_sources() # fetch_sources(): # # Fetch the individual element sources. # # Args: # fetch_original (bool): Always fetch original source # stop (Source): Only fetch sources listed before this source # # Raises: # SourceError: If one of the element sources has an error # def fetch_sources(self, *, fetch_original=False, stop=None): for source in self._sources: if source == stop: break if ( fetch_original or source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH or source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE ): # Source depends on previous sources, it cannot be stored in # CAS-based source cache on its own. Fetch original source # if it's not in the plugin-specific cache yet. if not source._is_cached(): self._fetch_original_source(source) else: self._fetch_source(source) # get_unique_key(): # # Return something which uniquely identifies the combined sources of the # element. # # Returns: # (str, list, dict): A string, list or dictionary as unique identifier # def get_unique_key(self): assert self.is_resolved() result = [] for source in self._sources: key_dict = {"key": source._get_unique_key(), "name": source.get_kind()} if source._directory: key_dict["directory"] = source._directory result.append(key_dict) return result # get_cache_key(): # # Return cache key for the combined element sources # def get_cache_key(self): return self._cache_key # get_brief_display_key() # # Returns an abbreviated cache key for display purposes # # Returns: # (str): An abbreviated hex digest cache key for this Element # def get_brief_display_key(self): context = self._context key = self._cache_key length = min(len(key), context.log_key_length) return key[:length] # query_cache(): # # Check if the element sources are cached in CAS, generating the source # cache keys if needed. # # Returns: # (bool): True if the element sources are cached # def query_cache(self): cas = self._context.get_cascache() elementsourcescache = self._elementsourcescache source_proto = elementsourcescache.load_proto(self) if not source_proto: self._cached = False return False if not cas.contains_directory(source_proto.files): self._cached = False return False self._proto = source_proto self._cached = True return True # can_query_cache(): # # Returns whether the cache status is available. # # Returns: # (bool): True if cache status is available # def can_query_cache(self): return self._cached is not None # cached() # # Return whether the element sources are cached in CAS. This must be # called only when all sources are resolved. # # Returns: # (bool): True if the element sources are cached # def cached(self): assert self._cached is not None return self._cached # is_resolved(): # # Get whether all sources of the element are resolved # # Returns: # (bool): True if all element sources are resolved # def is_resolved(self): return self._is_resolved # cached_original(): # # Get whether all the sources of the element have their own cached # copy of their sources. # # Returns: # (bool): True if all element sources have the original sources cached # def cached_original(self): return all(source._is_cached() for source in self._sources) # update_resolved_state(): # # Updates source's resolved state # # An element's source state must be resolved before it may compute # cache keys, because the source's ref, whether defined in yaml or # from the workspace, is a component of the element's cache keys. # def update_resolved_state(self): if self._is_resolved: # Already resolved return for source in self._sources: if not source.is_resolved(): return # Source is resolved, generate its cache key source._generate_key() self._is_resolved = True # Also generate the cache key for the combined element sources unique_key = self.get_unique_key() self._cache_key = _cachekey.generate_key(unique_key) # preflight(): # # A internal wrapper for calling the abstract preflight() method on # the element and its sources. # def preflight(self): # Ensure that the first source does not need access to previous sources if self._sources and self._sources[0]._requires_previous_sources(): from .element import ElementError # pylint: disable=cyclic-import raise ElementError( "{}: {} cannot be the first source of an element " "as it requires access to previous sources".format(self, self._sources[0]) ) # Preflight the sources for source in self.sources(): source._preflight() # _fetch_source(): # # Fetch a single source into the local CAS-based source cache # # Args: # source (Source): The source to fetch # def _fetch_source(self, source): # Cannot store a source in the CAS-based source cache on its own # if the source depends on previous sources. assert not source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH and not source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE if self._sourcecache.contains(source): # Already cached return cached_original = source._is_cached() if not cached_original: if self._sourcecache.has_fetch_remotes() and self._sourcecache.pull(source): # Successfully fetched individual source from remote source cache return # Unable to fetch source from remote source cache, fall back to # fetching the original source. source._fetch() # Stage original source into the local CAS-based source cache self._sourcecache.commit(source) # _fetch_source(): # # Fetch a single original source # # Args: # source (Source): The source to fetch # def _fetch_original_source(self, source): if source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH: with self._stage_previous_sources(source) as staging_directory: source._fetch(previous_sources_dir=staging_directory) else: source._fetch() # _stage(): # # Stage the element sources # # Args: # stop (Source): Only stage sources listed before this source # def _stage(self, *, stop=None): cas = self._context.get_cascache() vdir = CasBasedDirectory(cas) for source in self._sources: if source == stop: break if source._directory: vsubdir = vdir.open_directory(source._directory.lstrip(os.path.sep), create=True) else: vsubdir = vdir if source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH or source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: if source.BST_STAGE_VIRTUAL_DIRECTORY: source._stage(vsubdir) else: # Stage previous sources with cas.stage_directory(vsubdir._get_digest()) as tmpdir: # Stage current source source._stage(tmpdir) # Capture modified tree vsubdir._clear() vsubdir.import_files(tmpdir, collect_result=False) else: source_dir = self._sourcecache.export(source) vsubdir.import_files(source_dir, collect_result=False) return vdir # Context manager that stages sources in a cas based or temporary file # based directory @contextmanager def _stage_previous_sources(self, source): self.fetch_sources(stop=source) vdir = self._stage(stop=source) if source._directory: vdir = vdir.open_directory(source._directory, create=True) if source.BST_STAGE_VIRTUAL_DIRECTORY: yield vdir else: cas = self._context.get_cascache() with cas.stage_directory(vdir._get_digest()) as tempdir: yield tempdir apache-buildstream-27ae392/src/buildstream/_elementsourcescache.py000066400000000000000000000253551514607367700254700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from ._cas.casremote import BlobNotFound from ._assetcache import AssetCache from ._exceptions import AssetCacheError, CASError, CASRemoteError, SourceCacheError from . import utils from ._protos.buildstream.v2 import source_pb2 REMOTE_ASSET_SOURCE_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:source:{}" # Class that keeps config of remotes and deals with caching of sources. # # Args: # context (Context): The Buildstream context # class ElementSourcesCache(AssetCache): def __init__(self, context): super().__init__(context) self._basedir = os.path.join(context.cachedir, "elementsources") os.makedirs(self._basedir, exist_ok=True) # load_proto(): # # Load source proto from local cache. # # Args: # sources (ElementSources): The sources whose proto we want to load # def load_proto(self, sources): ref = sources.get_cache_key() path = self._source_path(ref) if not os.path.exists(path): return None source_proto = source_pb2.Source() with open(path, "r+b") as f: source_proto.ParseFromString(f.read()) return source_proto def store_proto(self, sources, proto): ref = sources.get_cache_key() path = self._source_path(ref) with utils.save_file_atomic(path, "w+b") as f: f.write(proto.SerializeToString()) # pull(): # # Attempts to pull sources from configured remote source caches. # # Args: # sources (ElementSources): The sources we want to fetch # # Returns: # (bool): True if pull successful, False if not # def pull(self, sources, plugin): project = sources.get_project() ref = sources.get_cache_key() display_key = sources.get_brief_display_key() uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(ref) index_remotes, storage_remotes = self.get_remotes(project.name, False) source_digest = None errors = [] # Start by pulling our source proto, so that we know which # blobs to pull for remote in index_remotes: remote.init() try: plugin.status("Pulling source {} <- {}".format(display_key, remote)) response = remote.fetch_blob([uri]) if response: source_digest = response.blob_digest break plugin.info("Remote ({}) does not have source {} cached".format(remote, display_key)) except AssetCacheError as e: plugin.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors and not source_digest: raise SourceCacheError( "Failed to pull source {}".format(display_key), detail="\n".join(str(e) for e in errors), temporary=True, ) # If we don't have a source proto, we can't pull source files if not source_digest: return False errors = [] for remote in storage_remotes: remote.init() try: plugin.status("Pulling data for source {} <- {}".format(display_key, remote)) if self._pull_source_storage(ref, source_digest, remote): plugin.info("Pulled source {} <- {}".format(display_key, remote)) return True plugin.info("Remote ({}) does not have source {} cached".format(remote, display_key)) except BlobNotFound as e: # Not all blobs are available on this remote plugin.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) continue except CASError as e: plugin.warn("Could not pull from remote {}: {}".format(remote, e)) errors.append(e) if errors: raise SourceCacheError( "Failed to pull source {}".format(display_key), detail="\n".join(str(e) for e in errors) ) return False # push(): # # Push sources to remote repository. # # Args: # sources (ElementSources): The sources to be pushed # # Returns: # (bool): True if any remote was updated, False if no pushes were required # # Raises: # (SourceCacheError): if there was an error # def push(self, sources, plugin): project = sources.get_project() ref = sources.get_cache_key() display_key = sources.get_brief_display_key() uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(ref) index_remotes, storage_remotes = self.get_remotes(project.name, True) source_proto = self.load_proto(sources) source_digest = self.cas.add_object(buffer=source_proto.SerializeToString()) pushed = False # First push our files to all storage remotes, so that they # can perform file checks on their end for remote in storage_remotes: remote.init() plugin.status("Pushing data from source {} -> {}".format(display_key, remote)) if self._push_source_blobs(source_proto, source_digest, remote): plugin.info("Pushed data from source {} -> {}".format(display_key, remote)) else: plugin.info("Remote ({}) already has all data of source {} cached".format(remote, display_key())) for remote in index_remotes: remote.init() plugin.status("Pushing source {} -> {}".format(display_key, remote)) if self._push_source_proto(uri, source_proto, source_digest, remote): plugin.info("Pushed source {} -> {}".format(display_key, remote)) pushed = True else: plugin.info("Remote ({}) already has source {} cached".format(remote, display_key)) return pushed def _get_source(self, ref): path = self._source_path(ref) source_proto = source_pb2.Source() try: with open(path, "r+b") as f: source_proto.ParseFromString(f.read()) return source_proto except FileNotFoundError as e: raise SourceCacheError("Attempted to access unavailable source: {}".format(e)) from e def _source_path(self, ref): return os.path.join(self._basedir, ref) # _push_source_blobs() # # Push the blobs that make up an source to the remote server. # # Args: # source_proto: The source proto whose blobs to push. # source_digest: The digest of the source proto. # remote (CASRemote): The remote to push the blobs to. # # Returns: # (bool) - True if we uploaded anything, False otherwise. # # Raises: # SourceCacheError: If we fail to push blobs (*unless* they're # already there or we run out of space on the server). # def _push_source_blobs(self, source_proto, source_digest, remote): try: # Push source files self.cas._send_directory(remote, source_proto.files) # Push source proto self.cas.send_blobs(remote, [source_digest]) except CASRemoteError as cas_error: if cas_error.reason != "cache-too-full": raise SourceCacheError("Failed to push source blobs: {}".format(cas_error)) return False return True # _push_source_proto() # # Pushes the source proto to remote. # # Args: # source_proto: The source proto. # source_digest: The digest of the source proto. # remote (AssetRemote): Remote to push to # # Returns: # (bool): Whether we pushed the source. # # Raises: # SourceCacheError: If the push fails for any reason except the # source already existing. # def _push_source_proto(self, uri, source_proto, source_digest, remote): try: response = remote.fetch_blob([uri]) # Skip push if source is already on the server if response and response.blob_digest == source_digest: return False except AssetCacheError as e: raise SourceCacheError("Error checking source cache: {}".format(e), temporary=True) from e referenced_directories = [source_proto.files] try: remote.push_blob( [uri], source_digest, references_directories=referenced_directories, ) except AssetCacheError as e: raise SourceCacheError("Failed to push source: {}".format(e), temporary=True) return True # _pull_source_storage(): # # Pull source blobs from the given remote. # # Args: # key (str): The specific key for the source to pull # remote (CASRemote): remote to pull from # # Returns: # (bool): True if we pulled any blobs. # # Raises: # SourceCacheError: If the pull failed for any reason except the # blobs not existing on the server. # def _pull_source_storage(self, key, source_digest, remote): try: # Fetch and parse source proto self.cas.fetch_blobs(remote, [source_digest]) source = source_pb2.Source() with self.cas.open(source_digest, "rb") as f: source.ParseFromString(f.read()) # Write the source proto to cache source_path = os.path.join(self._basedir, key) with utils.save_file_atomic(source_path, mode="wb") as f: f.write(source.SerializeToString()) self.cas.fetch_directory(remote, source.files) except BlobNotFound: return False except CASRemoteError as e: raise SourceCacheError("Failed to pull source: {}".format(e), temporary=True) return True def _push_source(self, source_ref, remote): uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(source_ref) try: remote.init() source_proto = self._get_source(source_ref) remote.push_directory([uri], source_proto.files) return True except AssetCacheError as e: raise SourceCacheError("Failed to push source: {}".format(e), temporary=True) return False apache-buildstream-27ae392/src/buildstream/_exceptions.py000066400000000000000000000207071514607367700236240ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Tiago Gomes import os from .exceptions import ErrorDomain # Disable pylint warnings for whole file here: # pylint: disable=global-statement # The last raised exception, this is used in test cases only _last_exception = None _last_task_error_domain = None _last_task_error_reason = None # get_last_exception() # # Fetches the last exception from the main process # # Used by regression tests # def get_last_exception(): global _last_exception le = _last_exception _last_exception = None return le # get_last_task_error() # # Fetches the last exception from a task # # Used by regression tests # def get_last_task_error(): if "BST_TEST_SUITE" not in os.environ: raise BstError("Getting the last task error is only supported when running tests") global _last_task_error_domain global _last_task_error_reason d = _last_task_error_domain r = _last_task_error_reason _last_task_error_domain = _last_task_error_reason = None return (d, r) # set_last_task_error() # # Sets the last exception of a task # # This is set by some internals to inform regression # tests about how things failed in a machine readable way # def set_last_task_error(domain, reason): if "BST_TEST_SUITE" in os.environ: global _last_task_error_domain global _last_task_error_reason _last_task_error_domain = domain _last_task_error_reason = reason # BstError is an internal base exception class for BuildStream # exceptions. # # The sole purpose of using the base class is to add additional # context to exceptions raised by plugins in child tasks, this # context can then be communicated back to the main process. # class BstError(Exception): def __init__(self, message, *, detail=None, domain=None, reason=None, temporary=False): global _last_exception super().__init__(message) # Additional error detail, these are used to construct detail # portions of the logging messages when encountered. # self.detail = detail # A sandbox can be created to debug this error self.sandbox = False # When this exception occurred during the handling of a job, indicate # whether or not there is any point retrying the job. # self.temporary = temporary # Error domain and reason # self.domain = domain self.reason = reason # Hold on to the last raised exception for testing purposes if "BST_TEST_SUITE" in os.environ: _last_exception = self # PluginError # # Raised on plugin related errors. # # This exception is raised either by the plugin loading process, # or by the base :class:`.Plugin` element itself. # class PluginError(BstError): def __init__(self, message, *, reason=None, detail=None, temporary=False): super().__init__(message, domain=ErrorDomain.PLUGIN, detail=detail, reason=reason, temporary=temporary) # LoadError # # Raised while loading some YAML. # # Args: # message (str): human readable error explanation # reason (LoadErrorReason): machine readable error reason # # This exception is raised when loading or parsing YAML, or when # interpreting project YAML # class LoadError(BstError): def __init__(self, message, reason, *, detail=None): super().__init__(message, detail=detail, domain=ErrorDomain.LOAD, reason=reason) # ImplError # # Raised when a :class:`.Source` or :class:`.Element` plugin fails to # implement a mandatory method # class ImplError(BstError): def __init__(self, message, reason=None): super().__init__(message, domain=ErrorDomain.IMPL, reason=reason) # PlatformError # # Raised if the current platform is not supported. class PlatformError(BstError): def __init__(self, message, reason=None, detail=None): super().__init__(message, domain=ErrorDomain.PLATFORM, reason=reason, detail=detail) # SandboxError # # Raised when errors are encountered by the sandbox implementation # class SandboxError(BstError): def __init__(self, message, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) # SandboxUnavailableError # # Raised when a specific sandbox backend is unsupported on the current host. # class SandboxUnavailableError(BstError): def __init__(self, message, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) # AssetCacheError # # Raised when errors are encountered in either type of cache # class AssetCacheError(BstError): def __init__(self, message, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason) # SourceCacheError # # Raised when errors are encountered in the source caches # class SourceCacheError(BstError): def __init__(self, message, detail=None, reason=None, temporary=False): super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason, temporary=temporary) # ArtifactError # # Raised when errors are encountered in the artifact caches # class ArtifactError(BstError): def __init__(self, message, *, detail=None, reason=None, temporary=False): super().__init__(message, detail=detail, domain=ErrorDomain.ARTIFACT, reason=reason, temporary=temporary) # RemoteError # # Raised when errors are encountered in Remotes # class RemoteError(BstError): def __init__(self, message, *, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.REMOTE, reason=reason) # CASError # # Raised when errors are encountered in the CAS # class CASError(BstError): def __init__(self, message, *, detail=None, reason=None, temporary=False): super().__init__(message, detail=detail, domain=ErrorDomain.CAS, reason=reason, temporary=temporary) # CASRemoteError # # Raised when errors are encountered in the remote CAS class CASRemoteError(CASError): pass # CASCacheError # # Raised when errors are encountered in the local CASCacheError # class CASCacheError(CASError): pass # PipelineError # # Raised from pipeline operations # class PipelineError(BstError): def __init__(self, message, *, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.PIPELINE, reason=reason) # StreamError # # Raised when a stream operation fails # class StreamError(BstError): def __init__(self, message=None, *, detail=None, reason=None, terminated=False): # The empty string should never appear to a user, # this only allows us to treat this internal error as # a BstError from the frontend. if message is None: message = "" super().__init__(message, detail=detail, domain=ErrorDomain.STREAM, reason=reason) self.terminated = terminated # AppError # # Raised from the frontend App directly # class AppError(BstError): def __init__(self, message, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.APP, reason=reason) # CachedFailure # # Raised from a child process within a job to indicate that the failure was cached # class CachedFailure(BstError): pass # SkipJob # # Raised from a child process within a job when the job should be # considered skipped by the parent process. # class SkipJob(Exception): pass # ArtifactElementError # # Raised when errors are encountered by artifact elements # class ArtifactElementError(BstError): def __init__(self, message, *, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason) # ProfileError # # Raised when a user provided profile choice isn't valid # class ProfileError(BstError): def __init__(self, message, detail=None, reason=None): super().__init__(message, detail=detail, domain=ErrorDomain.PROFILE, reason=reason) apache-buildstream-27ae392/src/buildstream/_frontend/000077500000000000000000000000001514607367700227025ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_frontend/__init__.py000066400000000000000000000014411514607367700250130ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os from .cli import cli if "_BST_COMPLETION" not in os.environ: from .profile import Profile from .status import Status from .widget import LogLine apache-buildstream-27ae392/src/buildstream/_frontend/app.py000066400000000000000000001176151514607367700240470ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from contextlib import contextmanager import os import sys import threading import traceback import datetime from textwrap import TextWrapper import click from click import UsageError # Import various buildstream internals from .._context import Context from .._project import Project from .._exceptions import BstError, StreamError, LoadError, AppError from ..exceptions import LoadErrorReason from .._message import Message, MessageType, unconditional_messages from .._stream import Stream from ..types import _SchedulerErrorAction, _Scope from .. import node from .. import utils from ..utils import UtilError # Import frontend assets from .profile import Profile from .status import Status from .widget import LogLine # Intendation for all logging INDENT = 4 # App() # # Main Application State # # Args: # main_options (dict): The main CLI options of the `bst` # command, before any subcommand # class App: def __init__(self, main_options): # # Public members # self.context = None # The Context object self.stream = None # The Stream object self.project = None # The toplevel Project object self.logger = None # The LogLine object self.interactive = None # Whether we are running in interactive mode self.colors = None # Whether to use colors in logging # # Private members # self._session_start = datetime.datetime.now() self._session_name = None self._main_options = main_options # Main CLI options, before any command self._status = None # The Status object self._fail_messages = {} # Failure messages by unique plugin id self._interactive_failures = None # Whether to handle failures interactively self._started = False # Whether a session has started self._set_project_dir = False # Whether -C option was used self._state = None # Frontend reads this and registers callbacks # UI Colors Profiles self._content_profile = Profile(fg="yellow") self._format_profile = Profile(fg="cyan", dim=True) self._success_profile = Profile(fg="green") self._error_profile = Profile(fg="red", dim=True) self._detail_profile = Profile(dim=True) # Cached messages self._cached_message_lock = threading.Lock() self._cached_message_text = "" self._cache_messages = None # # Early initialization # is_a_tty = sys.stdout.isatty() and sys.stderr.isatty() # Enable interactive mode if we're attached to a tty if main_options["no_interactive"]: self.interactive = False else: self.interactive = is_a_tty # Handle errors interactively if we're in interactive mode # and --on-error was not specified on the command line if main_options.get("on_error") is not None: self._interactive_failures = False else: self._interactive_failures = self.interactive # Use color output if we're attached to a tty, unless # otherwise specified on the command line if main_options["colors"] is None: self.colors = is_a_tty elif main_options["colors"]: self.colors = True else: self.colors = False if main_options["directory"]: self._set_project_dir = True else: main_options["directory"] = os.getcwd() # create() # # Should be used instead of the regular constructor. # # This will select a platform specific App implementation # # Args: # The same args as the App() constructor # @classmethod def create(cls, *args, **kwargs): if sys.platform.startswith("linux"): # Use an App with linux specific features from .linuxapp import LinuxApp # pylint: disable=cyclic-import return LinuxApp(*args, **kwargs) else: # The base App() class is default return App(*args, **kwargs) # initialized() # # Context manager to initialize the application and optionally run a session # within the context manager. # # This context manager will take care of catching errors from within the # context and report them consistently, so the CLI need not take care of # reporting the errors and exiting with a consistent error status. # # Args: # session_name (str): The name of the session, or None for no session # # Note that the except_ argument may have a subtly different meaning depending # on the activity performed on the Pipeline. In normal circumstances the except_ # argument excludes elements from the `elements` list. In a build session, the # except_ elements are excluded from the tracking plan. # # If a session_name is provided, we treat the block as a session, and print # the session header and summary, and time the main session from startup time. # @contextmanager def initialized(self, *, session_name=None): directory = self._main_options["directory"] config = self._main_options["config"] self._session_name = session_name # Instantiate Context with Context() as context: self.context = context # # Load the configuration # try: self.context.load(config) except BstError as e: self._error_exit(e, "Error loading user configuration") # Override things in the context from our command line options, # the command line when used, trumps the config files. # override_map = { "strict": "_strict_build_plan", "debug": "log_debug", "verbose": "log_verbose", "error_lines": "log_error_lines", "message_lines": "log_message_lines", "on_error": "sched_error_action", "fetchers": "sched_fetchers", "builders": "sched_builders", "pushers": "sched_pushers", "max_jobs": "build_max_jobs", "network_retries": "sched_network_retries", "pull_buildtrees": "pull_buildtrees", "cache_buildtrees": "cache_buildtrees", } for cli_option, context_attr in override_map.items(): option_value = self._main_options.get(cli_option) if option_value is not None: setattr(self.context, context_attr, option_value) try: self.context.platform except BstError as e: self._error_exit(e, "Error instantiating platform") # Create the stream right away, we'll need to pass it around. self.stream = Stream( self.context, self._session_start, session_start_callback=self.session_start_cb, interrupt_callback=self._interrupt_handler, ticker_callback=self._render_cached_messages, ) self._state = self.stream.get_state() # Register callbacks with the State self._state.register_task_failed_callback(self._job_failed) # Create the logger right before setting the message handler self.logger = LogLine( self.context, self._state, self._content_profile, self._format_profile, self._success_profile, self._error_profile, self._detail_profile, indent=INDENT, ) # Propagate pipeline feedback to the user self.context.messenger.set_message_handler(self._message_handler) # Check if throttling frontend updates to tick rate self._cache_messages = self.context.log_throttle_updates # Allow the Messenger to write status messages self.context.messenger.set_render_status_cb(self._render_status) # Preflight the artifact cache after initializing logging, # this can cause messages to be emitted. try: self.context.artifactcache.preflight() except BstError as e: self._error_exit(e, "Error instantiating artifact cache") # Now that we have a logger and message handler, # we can override the global exception hook. sys.excepthook = self._global_exception_handler # Initialize the parts of Stream that have side-effects self.stream.init() # Create our status printer, only available in interactive self._status = Status( self.context, self._state, self._content_profile, self._format_profile, self._success_profile, self._error_profile, self.stream, ) # Mark the beginning of the session if session_name: self._message(MessageType.START, session_name) # # Load the Project # try: self.project = Project( directory, self.context, cli_options=self._main_options["option"], default_mirror=self._main_options.get("default_mirror"), fetch_subprojects=self.stream.fetch_subprojects, ) except LoadError as e: # If there was no project.conf at all then there was just no project found. # # Don't error out in this case, as Stream() supports some operations which # do not require a project. If Stream() requires a project and it is missing, # then it will raise an error. # if e.reason != LoadErrorReason.MISSING_PROJECT_CONF: self._error_exit(e, "Error loading project") except BstError as e: self._error_exit(e, "Error loading project") # Set the project on the Stream, this can be None if there is no project. # self.stream.set_project(self.project) # Run the body of the session here, once everything is loaded try: yield except BstError as e: # Check that any cached messages are printed self._render_cached_messages() # Print a nice summary if this is a session if session_name: elapsed = self._state.elapsed_time() if isinstance(e, StreamError) and e.terminated: # pylint: disable=no-member self._message(MessageType.WARN, session_name + " Terminated", elapsed=elapsed) else: self._message(MessageType.FAIL, session_name, elapsed=elapsed) # Notify session failure self._notify("{} failed".format(session_name), e) if self._started: self._print_summary() # Exit with the error self._error_exit(e) except RecursionError: # Check that any cached messages are printed self._render_cached_messages() click.echo( "RecursionError: Dependency depth is too large. Maximum recursion depth exceeded.", err=True ) sys.exit(-1) else: # Check that any cached messages are printed self._render_cached_messages() # No exceptions occurred, print session time and summary if session_name: self._message(MessageType.SUCCESS, session_name, elapsed=self._state.elapsed_time()) if self._started: self._print_summary() # Notify session success self._notify("{} succeeded".format(session_name), "") # init_project() # # Initialize a new BuildStream project, either with the explicitly passed options, # or by starting an interactive session if project_name is not specified and the # application is running in interactive mode. # # Args: # project_name (str): The project name, must be a valid symbol name # min_version (str): The minimum required version of BuildStream # element_path (str): The subdirectory to store elements in # force (bool): Allow overwriting an existing project.conf # target_directory (str): The target directory the project should be initialized in # def init_project( self, project_name, min_version, element_path, force=False, target_directory=None, ): if target_directory: directory = os.path.abspath(target_directory) else: directory = self._main_options["directory"] directory = os.path.abspath(directory) project_path = os.path.join(directory, "project.conf") try: if self._set_project_dir: raise AppError( "Attempted to use -C or --directory with init.", reason="init-with-set-directory", detail="Please use 'bst init {}' instead.".format(directory), ) # Abort if the project.conf already exists, unless `--force` was specified in `bst init` if not force and os.path.exists(project_path): raise AppError("A project.conf already exists at: {}".format(project_path), reason="project-exists") if project_name: # If project name was specified, user interaction is not desired, just # perform some validation and write the project.conf node._assert_symbol_name(project_name, "project name") self._assert_min_version(min_version) self._assert_element_path(element_path) elif not self.interactive: raise AppError( "Cannot initialize a new project without specifying the project name", reason="unspecified-project-name", ) else: # Collect the parameters using an interactive session project_name, min_version, element_path = self._init_project_interactive( project_name, min_version, element_path ) # Create the directory if it doesnt exist try: os.makedirs(directory, exist_ok=True) except IOError as e: raise AppError("Error creating project directory {}: {}".format(directory, e)) from e # Create the elements sub-directory if it doesnt exist elements_path = os.path.join(directory, element_path) try: os.makedirs(elements_path, exist_ok=True) except IOError as e: raise AppError("Error creating elements sub-directory {}: {}".format(elements_path, e)) from e # Dont use ruamel.yaml here, because it doesnt let # us programatically insert comments or whitespace at # the toplevel. try: with open(project_path, "w", encoding="utf-8") as f: f.write( "# Unique project name\n" + "name: {}\n\n".format(project_name) + "# Required BuildStream version\n" + "min-version: {}\n\n".format(min_version) + "# Subdirectory where elements are stored\n" + "element-path: {}\n".format(element_path) ) except IOError as e: raise AppError("Error writing {}: {}".format(project_path, e)) from e except BstError as e: self._error_exit(e) click.echo("", err=True) click.echo("Created project.conf at: {}".format(project_path), err=True) sys.exit(0) # shell_prompt(): # # Creates a prompt for a shell environment, using ANSI color codes # if they are available in the execution context. # # Args: # element (Element): The element # # Returns: # (str): The formatted prompt to display in the shell # def shell_prompt(self, element): element_name = element._get_full_name() display_key = element._get_display_key() if self.colors: dim_key = not display_key.strict prompt = ( self._format_profile.fmt("[") + self._content_profile.fmt(display_key.brief, dim=dim_key) + self._format_profile.fmt("@") + self._content_profile.fmt(element_name) + self._format_profile.fmt(":") + self._content_profile.fmt("$PWD") + self._format_profile.fmt("]$") + " " ) else: prompt = "[{}@{}:${{PWD}}]$ ".format(display_key.brief, element_name) return prompt # cleanup() # # Cleans up application state # # This is called by Click at exit time # def cleanup(self): if self.stream: self.stream.cleanup() ############################################################ # Abstract Class Methods # ############################################################ # notify() # # Notify the user of something which occurred, this # is intended to grab attention from the user. # # This is guaranteed to only be called in interactive mode # # Args: # title (str): The notification title # text (str): The notification text # def notify(self, title, text): pass ############################################################ # Local Functions # ############################################################ # Local function for calling the notify() virtual method # def _notify(self, title, text): if self.interactive: self.notify(str(title), str(text)) # Local message propagator # def _message(self, message_type, message, **kwargs): self.context.messenger.message(Message(message_type, message, **kwargs)) # Flush any potentially cached messages immediately self._render_cached_messages() # Exception handler # def _global_exception_handler(self, etype, value, tb, exc=True): # Print the regular BUG message formatted = None if exc: # Format the exception & traceback by default formatted = "".join(traceback.format_exception(etype, value, tb)) self._message(MessageType.BUG, str(value), detail=formatted) # If the scheduler has started, try to terminate all jobs gracefully, # otherwise exit immediately. if self.stream.running: self.stream.terminate() else: sys.exit(-1) # # Cache messages # # Args: # message (Message): The message to cache # # Returns: # (str): The rendered text of only this message # def _cache_message(self, message): text = self.logger.render(message) with self._cached_message_lock: self._cached_message_text += text return text # # Render cached messages in case throttling messages during regular sessions # def _render_cached_messages(self): # First clear the status area if self._status: self._status.clear() # Render pending messages with self._cached_message_lock: if self._cached_message_text: click.echo(self._cached_message_text, nl=False, err=True) self._cached_message_text = "" # Render the status area again self._render_status() # # Render status, this is used in some timed messages while not running the scheduler, # and also used to render the status bar in regular sessions. # def _render_status(self): # If we're suspended or terminating, then dont render the status area if self._status and self.stream and not (self.stream.suspended or self.stream.terminated): self._status.render() # # Handle ^C SIGINT interruptions in the scheduling main loop # def _interrupt_handler(self): # Only handle ^C interactively in interactive mode if not self.interactive: self._status.clear() self.stream.terminate() return # Here we can give the user some choices, like whether they would # like to continue, abort immediately, or only complete processing of # the currently ongoing tasks. We can also print something more # intelligent, like how many tasks remain to complete overall. with self._interrupted(): click.echo( "\nUser interrupted with ^C\n" + "\n" "Choose one of the following options:\n" + " (c)ontinue - Continue queueing jobs as much as possible\n" + " (q)uit - Exit after all ongoing jobs complete\n" + " (t)erminate - Terminate any ongoing jobs and exit\n" + "\n" + "Pressing ^C again will terminate jobs and exit\n", err=True, ) try: choice = click.prompt( "Choice:", value_proc=_prefix_choice_value_proc(["continue", "quit", "terminate"]), default="continue", err=True, ) except (click.Abort, SystemError): # In some cases, the readline buffer underlying the prompt gets corrupted on the second CTRL+C # This throws a SystemError, which doesn't seem to be problematic for the rest of the program # Ensure a newline after automatically printed '^C' click.echo("", err=True) choice = "terminate" if choice == "terminate": click.echo("\nTerminating all jobs at user request\n", err=True) self.stream.terminate() else: if choice == "quit": click.echo("\nCompleting ongoing tasks before quitting\n", err=True) self.stream.quit() elif choice == "continue": click.echo("\nContinuing\n", err=True) # Callback that a job has failed # # XXX: This accesses the core directly, which is discouraged. # Removing use of the core would require delegating to Shell # the creation of an interactive shell, and the retrying of jobs. # # Args: # task_id (str): The unique identifier of the task # element (tuple): If an element job failed a tuple of Element instance unique_id & display key # def _job_failed(self, task_id, element=None): task = self._state.tasks[task_id] # Flush any pending messages when handling a failure self._render_cached_messages() # Dont attempt to handle a failure if the user has already opted to # terminate if not self.stream.terminated: if element: # Get the last failure message for additional context failure = self._fail_messages.get(task.full_name) # XXX This is dangerous, sometimes we get the job completed *before* # the failure message reaches us ?? if not failure: self._status.clear() click.echo( "\n\n\nBUG: Message handling out of sync, " + "unable to retrieve failure message for element {}\n\n\n\n\n".format(task.full_name), err=True, ) else: self._handle_failure(element, task, failure) else: # Not an element_job, we don't handle the failure click.echo("\nTerminating all jobs\n", err=True) self.stream.terminate() def _handle_failure(self, element, task, failure): full_name = task.full_name # Handle non interactive mode setting of what to do when a job fails. if not self._interactive_failures: if self.context.sched_error_action == _SchedulerErrorAction.TERMINATE: self.stream.terminate() elif self.context.sched_error_action == _SchedulerErrorAction.QUIT: self.stream.quit() elif self.context.sched_error_action == _SchedulerErrorAction.CONTINUE: pass return # Interactive mode for element failures with self._interrupted(): summary = ( "\n{} failure on element: {}\n".format(failure.action_name, full_name) + "\n" + "Choose one of the following options:\n" + " (c)ontinue - Continue queueing jobs as much as possible\n" + " (q)uit - Exit after all ongoing jobs complete\n" + " (t)erminate - Terminate any ongoing jobs and exit\n" + " (r)etry - Retry this job\n" ) if failure.logfile: summary += " (l)og - View the full log file\n" if failure.sandbox: summary += " (s)hell - Drop into a shell in the failed build sandbox\n" summary += "\nPressing ^C will terminate jobs and exit\n" choices = ["continue", "quit", "terminate", "retry"] if failure.logfile: choices += ["log"] if failure.sandbox: choices += ["shell"] choice = "" while choice not in ["continue", "quit", "terminate", "retry"]: click.echo(summary, err=True) self._notify("BuildStream failure", "{} on element {}".format(failure.action_name, full_name)) try: choice = click.prompt( "Choice:", default="continue", err=True, value_proc=_prefix_choice_value_proc(choices) ) except (click.Abort, SystemError): # In some cases, the readline buffer underlying the prompt gets corrupted on the second CTRL+C # This throws a SystemError, which doesn't seem to be problematic for the rest of the program # Ensure a newline after automatically printed '^C' click.echo("", err=True) choice = "terminate" # Handle choices which you can come back from # if choice == "shell": click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True) try: unique_id, _ = element self.stream.shell( None, _Scope.BUILD, self.shell_prompt, isolate=True, usebuildtree=True, unique_id=unique_id, ) except BstError as e: click.echo("Error while attempting to create interactive shell: {}".format(e), err=True) elif choice == "log": with open(failure.logfile, "r", encoding="utf-8") as logfile: content = logfile.read() click.echo_via_pager(content) if choice == "terminate": click.echo("\nTerminating all jobs\n", err=True) self.stream.terminate() else: if choice == "quit": click.echo("\nCompleting ongoing tasks before quitting\n", err=True) self.stream.quit() elif choice == "continue": click.echo("\nContinuing with other non failing elements\n", err=True) elif choice == "retry": click.echo("\nRetrying failed job\n", err=True) unique_id = element[0] self.stream.retry_job(task.action_name, unique_id) # # Print the session heading if we've loaded a pipeline and there # is going to be a session # def session_start_cb(self): self._started = True if self._session_name: self.logger.print_heading(self.project, self.stream, log_file=self._main_options["log_file"]) # # Print a summary of the queues # def _print_summary(self): # Ensure all status & messages have been processed self._render_cached_messages() click.echo("", err=True) try: self.logger.print_summary(self.stream, self._main_options["log_file"]) except BstError as e: self._error_exit(e) # _error_exit() # # Exit with an error # # This will print the passed error to stderr and exit the program # with -1 status # # Args: # error (BstError): A BstError exception to print # prefix (str): An optional string to prepend to the error message # def _error_exit(self, error, prefix=None): click.echo("", err=True) if self.context is None or self.context.log_debug is None: # Context might not be initialized, default to cmd debug = self._main_options["debug"] else: debug = self.context.log_debug if debug: main_error = "\n\n" + traceback.format_exc() else: main_error = str(error) if prefix is not None: main_error = "{}: {}".format(prefix, main_error) click.echo(main_error, err=True) if error.detail: indent = " " * INDENT detail = "\n" + indent + indent.join(error.detail.splitlines(True)) click.echo(detail, err=True) sys.exit(-1) # # Handle messages from the pipeline # def _message_handler(self, message, is_silenced): # Drop status messages from the UI if not verbose, we'll still see # info messages and status messages will still go to the log files. if not self.context.log_verbose and message.message_type == MessageType.STATUS: return # Hold on to the failure messages if message.message_type in [MessageType.FAIL, MessageType.BUG] and message.element_name is not None: self._fail_messages[message.element_name] = message # Send to frontend if appropriate if is_silenced and (message.message_type not in unconditional_messages): return # Cache the message text = self._cache_message(message) # If we're not rate limiting messaging, or the scheduler tick isn't active then render if not self._cache_messages or not self.stream.running: self._render_cached_messages() # Additionally log to a file if self._main_options["log_file"]: click.echo(text, file=self._main_options["log_file"], color=False, nl=False) @contextmanager def _interrupted(self): self._status.clear() try: with self.stream.suspend(): yield finally: self._render_cached_messages() # Some validation routines for project initialization # def _assert_min_version(self, min_version): bst_major, bst_minor = utils._get_bst_api_version() message = "The minimum version must be a known version of BuildStream {}".format(bst_major) # Validate the version format try: min_version_major, min_version_minor = utils._parse_version(min_version) except UtilError as e: raise AppError(str(e), reason="invalid-min-version") from e # Validate that this version can be loaded by the installed version of BuildStream if min_version_major != bst_major or min_version_minor > bst_minor: raise AppError(message, reason="invalid-min-version") def _assert_element_path(self, element_path): message = "The element path cannot be an absolute path or contain any '..' components\n" # Validate the path is not absolute if os.path.isabs(element_path): raise AppError(message, reason="invalid-element-path") # Validate that the path does not contain any '..' components path = element_path while path: split = os.path.split(path) path = split[0] basename = split[1] if basename == "..": raise AppError(message, reason="invalid-element-path") # _init_project_interactive() # # Collect the user input for an interactive session for App.init_project() # # Args: # project_name (str): The project name, must be a valid symbol name # min_version (str): The minimum BuildStream version # element_path (str): The subdirectory to store elements in # # Returns: # project_name (str): The user selected project name # min_version (int): The user selected minimum BuildStream version # element_path (str): The user selected element path # def _init_project_interactive(self, project_name, min_version, element_path): bst_major, bst_minor = utils._get_bst_api_version() def project_name_proc(user_input): try: node._assert_symbol_name(user_input, "project name") except LoadError as e: message = "{}\n\n{}\n".format(e, e.detail) raise UsageError(message) from e return user_input def min_version_proc(user_input): try: self._assert_min_version(user_input) except AppError as e: raise UsageError(str(e)) from e return user_input def element_path_proc(user_input): try: self._assert_element_path(user_input) except AppError as e: raise UsageError(str(e)) from e return user_input w = TextWrapper(initial_indent=" ", subsequent_indent=" ", width=79) # Collect project name click.echo("", err=True) click.echo(self._content_profile.fmt("Choose a unique name for your project"), err=True) click.echo(self._format_profile.fmt("-------------------------------------"), err=True) click.echo("", err=True) click.echo( self._detail_profile.fmt( w.fill( "The project name is a unique symbol for your project and will be used " "to distinguish your project from others in user preferences, namespacing " "of your project's artifacts in shared artifact caches, and in any case where " "BuildStream needs to distinguish between multiple projects." ) ), err=True, ) click.echo("", err=True) click.echo( self._detail_profile.fmt( w.fill( "The project name must contain only alphanumeric characters, " "may not start with a digit, and may contain dashes or underscores." ) ), err=True, ) click.echo("", err=True) project_name = click.prompt(self._content_profile.fmt("Project name"), value_proc=project_name_proc, err=True) click.echo("", err=True) # Collect minimum BuildStream version click.echo( self._content_profile.fmt("Select the minimum required BuildStream version for your project"), err=True ) click.echo( self._format_profile.fmt("----------------------------------------------------------------"), err=True ) click.echo("", err=True) click.echo( self._detail_profile.fmt( w.fill( "The minimum version is used to provide users who build your project " "with a helpful error message in the case that they do not have a recent " "enough version of BuildStream to support all the features which your " "project uses." ) ), err=True, ) click.echo("", err=True) click.echo( self._detail_profile.fmt( w.fill( "The lowest version allowed is {major}.0, the currently installed version of BuildStream is {major}.{minor}".format( major=bst_major, minor=bst_minor ) ) ), err=True, ) click.echo("", err=True) min_version = click.prompt( self._content_profile.fmt("Minimum version"), value_proc=min_version_proc, default=min_version, err=True, ) click.echo("", err=True) # Collect element path click.echo(self._content_profile.fmt("Select the element path"), err=True) click.echo(self._format_profile.fmt("-----------------------"), err=True) click.echo("", err=True) click.echo( self._detail_profile.fmt( w.fill( "The element path is a project subdirectory where element .bst files are stored " "within your project." ) ), err=True, ) click.echo("", err=True) click.echo( self._detail_profile.fmt( w.fill( "Elements will be displayed in logs as filenames relative to " "the element path, and similarly, dependencies must be expressed as filenames " "relative to the element path." ) ), err=True, ) click.echo("", err=True) element_path = click.prompt( self._content_profile.fmt("Element path"), value_proc=element_path_proc, default=element_path, err=True ) return (project_name, min_version, element_path) # # Return a value processor for partial choice matching. # The returned values processor will test the passed value with all the item # in the 'choices' list. If the value is a prefix of one of the 'choices' # element, the element is returned. If no element or several elements match # the same input, a 'click.UsageError' exception is raised with a description # of the error. # # Note that Click expect user input errors to be signaled by raising a # 'click.UsageError' exception. That way, Click display an error message and # ask for a new input. # def _prefix_choice_value_proc(choices): def value_proc(user_input): remaining_candidate = [choice for choice in choices if choice.startswith(user_input)] if not remaining_candidate: raise UsageError("Expected one of {}, got {}".format(choices, user_input)) if len(remaining_candidate) == 1: return remaining_candidate[0] else: raise UsageError("Ambiguous input. '{}' can refer to one of {}".format(user_input, remaining_candidate)) return value_proc apache-buildstream-27ae392/src/buildstream/_frontend/cli.py000066400000000000000000001652451514607367700240400ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import re import sys from functools import partial from typing import TYPE_CHECKING import shutil import click from .. import _yaml from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError if TYPE_CHECKING or click.Command.__bases__ == (object,): # Click >= 8.2 ClickCommandBaseClass = click.Command ClickGroupBaseClass = click.Group else: # Click < 8.2 ClickCommandBaseClass = click.BaseCommand ClickGroupBaseClass = click.MultiCommand ################################################################## # Helper classes and methods for Click # ################################################################## class FastEnumType(click.Choice): def __init__(self, enum, options=None): self._enum = enum if options is None: options = enum.values() else: options = [option.value for option in options] super().__init__(options) def convert(self, value, param, ctx): # This allows specifying default values as instances of the # enum if isinstance(value, self._enum): value = value.value return self._enum(super().convert(value, param, ctx)) class RemoteSpecType(click.ParamType): name = "remote" def __init__(self, purpose=RemoteSpecPurpose.ALL): self.purpose = purpose def convert(self, value, param, ctx): spec = None try: spec = RemoteSpec.new_from_string(value, self.purpose) except RemoteError as e: self.fail("Failed to interpret remote: {}".format(e)) return spec def __repr__(self): return "REMOTE" ################################################################## # Override of click's main entry point # ################################################################## # search_command() # # Helper function to get a command and context object # for a given command. # # Args: # commands (list): A list of command words following `bst` invocation # context (click.Context): An existing toplevel context, or None # # Returns: # context (click.Context): The context of the associated command, or None # def search_command(args, *, context=None): if context is None: context = cli.make_context("bst", args, resilient_parsing=True) # Loop into the deepest command command = cli command_ctx = context for cmd in args: command = command_ctx.command.get_command(command_ctx, cmd) if command is None: return None command_ctx = command.make_context(command.name, [command.name], parent=command_ctx, resilient_parsing=True) return command_ctx # Completion for completing command names as help arguments def complete_commands(cmd, args, incomplete): command_ctx = search_command(args[1:]) if command_ctx and command_ctx.command and isinstance(command_ctx.command, ClickGroupBaseClass): return [ subcommand + " " for subcommand in command_ctx.command.list_commands(command_ctx) if not command_ctx.command.get_command(command_ctx, subcommand).hidden ] return [] # Special completion for completing the bst elements in a project dir def complete_target(args, incomplete): """ :param args: full list of args typed before the incomplete arg :param incomplete: the incomplete text to autocomplete :return: all the possible user-specified completions for the param """ from .. import utils project_conf = "project.conf" # First resolve the directory, in case there is an # active --directory/-C option # base_directory = "." idx = -1 try: idx = args.index("-C") except ValueError: try: idx = args.index("--directory") except ValueError: pass if idx >= 0 and len(args) > idx + 1: base_directory = args[idx + 1] else: # Check if this directory or any of its parent directories # contain a project config file base_directory, _ = utils._search_upward_for_files(base_directory, [project_conf]) if base_directory is None: # No project_conf was found in base_directory or its parents, no need # to try loading any project conf and avoid os.path NoneType TypeError. return [] else: project_file = os.path.join(base_directory, project_conf) try: project = _yaml.load(project_file, shortname=project_conf) except LoadError: # If there is no project conf in context, just dont # even bother trying to complete anything. return [] # The project is not required to have an element-path element_directory = project.get_str("element-path", default="") # If a project was loaded, use its element-path to # adjust our completion's base directory if element_directory: base_directory = os.path.join(base_directory, element_directory) complete_list = [] for p in complete_path("File", incomplete, base_directory=base_directory): if p.endswith(".bst ") or p.endswith("/"): complete_list.append(p) return complete_list def complete_artifact(orig_args, args, incomplete): from .._context import Context with Context(use_casd=False) as ctx: config = None if orig_args: for i, arg in enumerate(orig_args): if arg in ("-c", "--config"): try: config = orig_args[i + 1] except IndexError: pass if args: for i, arg in enumerate(args): if arg in ("-c", "--config"): try: config = args[i + 1] except IndexError: pass ctx.load(config) # element targets are valid artifact names complete_list = complete_target(args, incomplete) complete_list.extend(ref for ref in ctx.artifactcache.list_artifacts() if ref.startswith(incomplete)) return complete_list def override_completions(orig_args, cmd, cmd_param, args, incomplete): """ :param orig_args: original, non-completion args :param cmd_param: command definition :param args: full list of args typed before the incomplete arg :param incomplete: the incomplete text to autocomplete :return: all the possible user-specified completions for the param """ if cmd.name == "help": return complete_commands(cmd, args, incomplete) # We can't easily extend click's data structures without # modifying click itself, so just do some weak special casing # right here and select which parameters we want to handle specially. if isinstance(cmd_param.type, click.Path): if cmd_param.name in ("elements", "element", "except_"): return complete_target(args, incomplete) if cmd_param.name in ("artifacts", "target"): return complete_artifact(orig_args, args, incomplete) raise CompleteUnhandled() def validate_output_streams(): if sys.platform == "win32": # Windows does not support 'fcntl', the module is unavailable there as # of Python 3.7, therefore early-out here. return import fcntl for stream in (sys.stdout, sys.stderr): fileno = stream.fileno() flags = fcntl.fcntl(fileno, fcntl.F_GETFL) if flags & os.O_NONBLOCK: click.echo("{} is currently set to O_NONBLOCK, try opening a new shell".format(stream.name), err=True) sys.exit(-1) def override_main(self, args=None, prog_name=None, complete_var=None, standalone_mode=True, **extra): # Hook for the Bash completion. This only activates if the Bash # completion is actually enabled, otherwise this is quite a fast # noop. if main_bashcomplete(self, prog_name, partial(override_completions, args)): # If we're running tests we cant just go calling exit() # from the main process. # # The below is a quicker exit path for the sake # of making completions respond faster. if "BST_TEST_SUITE" not in os.environ: sys.stdout.flush() sys.stderr.flush() os._exit(0) # Regular client return for test cases return # Check output file descriptor at earliest opportunity, to # provide a reasonable error message instead of a stack trace # in the case that it is non-blocking. validate_output_streams() original_main(self, args=args, prog_name=prog_name, complete_var=None, standalone_mode=standalone_mode, **extra) original_main = ClickCommandBaseClass.main # Disable type checking since mypy doesn't support assigning to a method. # See https://github.com/python/mypy/issues/2427. ClickCommandBaseClass.main = override_main # type: ignore ################################################################## # Main Options # ################################################################## def print_version(ctx, param, value): if not value or ctx.resilient_parsing: return from .. import __version__ click.echo(__version__) ctx.exit() @click.group(context_settings={"help_option_names": ["-h", "--help"]}) @click.option("--version", is_flag=True, callback=print_version, expose_value=False, is_eager=True) @click.option( "--config", "-c", type=click.Path(exists=True, dir_okay=False, readable=True), help="Configuration file to use" ) @click.option( "--directory", "-C", default=None, # Set to os.getcwd() later. type=click.Path(file_okay=False, readable=True), help="Project directory (default: current directory)", ) @click.option( "--on-error", default=None, type=FastEnumType(_SchedulerErrorAction), help="What to do when an error is encountered", ) @click.option("--fetchers", type=click.INT, default=None, help="Maximum simultaneous download tasks") @click.option("--builders", type=click.INT, default=None, help="Maximum simultaneous build tasks") @click.option("--pushers", type=click.INT, default=None, help="Maximum simultaneous upload tasks") @click.option( "--max-jobs", type=click.INT, default=None, help="Number of parallel jobs allowed for a given build task" ) @click.option("--network-retries", type=click.INT, default=None, help="Maximum retries for network tasks") @click.option( "--no-interactive", is_flag=True, help="Force non interactive mode, otherwise this is automatically decided" ) @click.option("--verbose/--no-verbose", default=None, help="Be extra verbose") @click.option("--debug/--no-debug", default=None, help="Print debugging output") @click.option("--error-lines", type=click.INT, default=None, help="Maximum number of lines to show from a task log") @click.option( "--message-lines", type=click.INT, default=None, help="Maximum number of lines to show in a detailed message" ) @click.option( "--log-file", type=click.File(mode="w", encoding="UTF-8"), help="A file to store the main log (allows storing the main log while in interactive mode)", ) @click.option("--colors/--no-colors", default=None, help="Force enable/disable ANSI color codes in output") @click.option( "--strict/--no-strict", default=None, is_flag=True, help="Elements must be rebuilt when their dependencies have changed", ) @click.option( "--option", "-o", type=click.Tuple([str, str]), multiple=True, metavar="OPTION VALUE", help="Specify a project option", ) @click.option("--default-mirror", default=None, help="The mirror to fetch from first, before attempting other mirrors") @click.option( "--pull-buildtrees", is_flag=True, default=None, help="Include an element's build tree when pulling remote element artifacts", ) @click.option( "--cache-buildtrees", default=None, type=FastEnumType(_CacheBuildTrees), help="Cache artifact build tree content on creation", ) @click.pass_context def cli(context, **kwargs): """Build and manipulate BuildStream projects Most of the main options override options in the user preferences configuration file. """ from .app import App # Create the App, giving it the main arguments context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) # Configure colors context.color = context.obj.colors ################################################################## # Help Command # ################################################################## @cli.command(name="help", short_help="Print usage information", context_settings={"help_option_names": []}) @click.argument("command", nargs=-1, metavar="COMMAND") @click.pass_context def help_command(ctx, command): """Print usage information about a given command""" command_ctx = search_command(command, context=ctx.parent) if not command_ctx: click.echo("Not a valid command: '{} {}'".format(ctx.parent.info_name, " ".join(command)), err=True) sys.exit(-1) click.echo(command_ctx.command.get_help(command_ctx), err=True) # Hint about available sub commands if isinstance(command_ctx.command, ClickGroupBaseClass): detail = " " if command: detail = " {} ".format(" ".join(command)) click.echo( "\nFor usage on a specific command: {} help{}COMMAND".format(ctx.parent.info_name, detail), err=True ) ################################################################## # Init Command # ################################################################## @cli.command(short_help="Initialize a new BuildStream project") @click.option("--project-name", type=click.STRING, help="The project name to use") @click.option( "--min-version", type=click.STRING, default="2.7", show_default=True, help="The required format version", ) @click.option( "--element-path", type=click.Path(), default="elements", show_default=True, help="The subdirectory to store elements in", ) @click.option("--force", "-f", is_flag=True, help="Allow overwriting an existing project.conf") @click.argument("target-directory", nargs=1, required=False, type=click.Path(file_okay=False, writable=True)) @click.pass_obj def init(app, project_name, min_version, element_path, force, target_directory): """Initialize a new BuildStream project Creates a new BuildStream project.conf in the project directory. Unless `--project-name` is specified, this will be an interactive session. """ app.init_project(project_name, min_version, element_path, force, target_directory) ################################################################## # Build Command # ################################################################## @cli.command(short_help="Build elements in a pipeline") @click.option( "--deps", "-d", default=None, type=FastEnumType( _PipelineSelection, [_PipelineSelection.NONE, _PipelineSelection.BUILD, _PipelineSelection.RUN, _PipelineSelection.ALL], ), help="The dependencies to build", ) @click.option( "--artifact-remote", "artifact_remotes", type=RemoteSpecType(), multiple=True, help="A remote for uploading and downloading artifacts", ) @click.option( "--source-remote", "source_remotes", type=RemoteSpecType(), multiple=True, help="A remote for uploading and downloading cached sources", ) @click.option( "--ignore-project-artifact-remotes", is_flag=True, help="Ignore remote artifact cache servers recommended by projects", ) @click.option( "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.option( "--retry-failed", "-r", is_flag=True, help="Try to build elements for which a failed build artifact is found", ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def build( app, elements, deps, artifact_remotes, source_remotes, ignore_project_artifact_remotes, ignore_project_source_remotes, retry_failed, ): """Build elements in a pipeline Specifying no elements will result in building the default targets of the project. If no default targets are configured, all project elements will be built. When this command is executed from a workspace directory, the default is to build the workspace element. Specify `--deps` to control which dependencies must be built: \b none: No dependencies, just the element itself build: Build time dependencies, excluding the element itself run: The element and its runtime dependencies all: All dependencies Dependencies that are consequently required to build the requested elements will be built on demand. """ with app.initialized(session_name="Build"): ignore_junction_targets = False if deps is None: deps = app.context.build_dependencies if not elements: elements = app.stream.get_default_targets() # Junction elements cannot be built, exclude them from default targets ignore_junction_targets = True app.stream.build( elements, selection=deps, ignore_junction_targets=ignore_junction_targets, artifact_remotes=artifact_remotes, source_remotes=source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, retry_failed=retry_failed, ) ################################################################## # Show Command # ################################################################## @cli.command(short_help="Show elements in the pipeline") @click.option( "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" ) @click.option( "--deps", "-d", default=_PipelineSelection.ALL, show_default=True, type=FastEnumType( _PipelineSelection, [ _PipelineSelection.NONE, _PipelineSelection.RUN, _PipelineSelection.BUILD, _PipelineSelection.ALL, ], ), help="The dependencies to show", ) @click.option( "--order", default="stage", show_default=True, type=click.Choice(["stage", "alpha"]), help="Staging or alphabetic ordering of dependencies", ) @click.option( "--format", "-f", "format_", metavar="FORMAT", default=None, type=click.STRING, help="Format string for each element", ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def show(app, elements, deps, except_, order, format_): """Show elements in the pipeline Specifying no elements will result in showing the default targets of the project. If no default targets are configured, all project elements will be shown. When this command is executed from a workspace directory, the default is to show the workspace element. By default this will show all of the dependencies of the specified target element. Specify ``--deps`` to control which elements to show: \b none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies **FORMAT** The ``--format`` option controls what should be printed for each element, the following symbols can be used in the format string: \b %{name} The element name %{kind} The element kind (Since: 2.6) %{description} The element description, on a single line (Since: 2.3) %{key} The abbreviated cache key (if all sources are consistent) %{full-key} The full cache key (if all sources are consistent) %{state} cached, buildable, waiting, inconsistent or junction %{config} The element configuration %{vars} Variable configuration %{env} Environment settings %{public} Public domain data %{workspaced} If the element is workspaced %{workspace-dirs} A list of workspace directories %{deps} A list of all dependencies %{build-deps} A list of build dependencies %{runtime-deps} A list of runtime dependencies %{source-info} Source provenance information %{artifact-cas-digest} The CAS digest of the built artifact The value of the %{symbol} without the leading '%' character is understood as a pythonic formatting string, so python formatting features apply, example: \b bst show target.bst --format \\ 'Name: %{name: ^20} Key: %{key: ^8} State: %{state}' If you want to use a newline in a format string in bash, use the '$' modifier: \b bst show target.bst --format \\ $'---------- %{name} ----------\\n%{vars}' """ with app.initialized(): if not format_: format_ = app.context.log_element_format # First determine whether we need to go about querying the local cache # and spending time setting up remotes. state_match = re.search(r"%(\{(state)[^%]*?\})", format_) key_match = re.search(r"%(\{(key)[^%]*?\})", format_) full_key_match = re.search(r"%(\{(full-key)[^%]*?\})", format_) artifact_cas_digest_match = re.search(r"%(\{(artifact-cas-digest)[^%]*?\})", format_) need_state = bool(state_match or key_match or full_key_match or artifact_cas_digest_match) if not elements: elements = app.stream.get_default_targets() dependencies = app.stream.load_selection( elements, selection=deps, except_targets=except_, need_state=need_state ) # Don't spend time interrogating the cache if we don't need to show element state if need_state: app.stream.query_cache(dependencies, need_state=True) if order == "alpha": dependencies = sorted(dependencies) report = app.logger.show_pipeline(dependencies, format_) click.echo(report) ################################################################## # Shell Command # ################################################################## @cli.command(short_help="Shell into an element's sandbox environment") @click.option("--build", "-b", "build_", is_flag=True, help="Stage dependencies and sources to build") @click.option( "--mount", type=click.Tuple([click.Path(exists=True), str]), multiple=True, metavar="HOSTPATH PATH", help="Mount a file or directory into the sandbox", ) @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") @click.option( "--use-buildtree", "-t", "cli_buildtree", is_flag=True, help=( "Stage a buildtree. Will fail if a buildtree is not available. " "pull-buildtrees configuration is needed if the buildtree is not available locally." ), ) @click.option( "--artifact-remote", "artifact_remotes", type=RemoteSpecType(), multiple=True, help="A remote for uploading and downloading artifacts", ) @click.option( "--source-remote", "source_remotes", type=RemoteSpecType(), multiple=True, help="A remote for uploading and downloading cached sources", ) @click.option( "--ignore-project-artifact-remotes", is_flag=True, help="Ignore remote artifact cache servers recommended by projects", ) @click.option( "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("target", required=False, type=click.Path(readable=False)) @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell( app, target, command, mount, isolate, build_, cli_buildtree, artifact_remotes, source_remotes, ignore_project_artifact_remotes, ignore_project_source_remotes, ): """Run a command in the target element's sandbox environment When this command is executed from a workspace directory, the default is to shell into the workspace element. This will stage a temporary sysroot for running the target element, assuming it has already been built and all required artifacts are in the local cache. Use '--' to separate a command from the options to bst, otherwise bst may respond to them instead. e.g. \b bst shell example.bst -- df -h Use the --build option to create a temporary sysroot for building the element instead. If no COMMAND is specified, the default is to attempt to run an interactive shell. """ # Buildtree can only be used with build shells if cli_buildtree: build_ = True scope = _Scope.BUILD if build_ else _Scope.RUN with app.initialized(): if not target: target = app.stream.get_default_target() if not target: raise AppError('Missing argument "TARGET".') mounts = [_HostMount(path, host_path) for host_path, path in mount] try: exitcode = app.stream.shell( target, scope, app.shell_prompt, mounts=mounts, isolate=isolate, command=command, usebuildtree=cli_buildtree, artifact_remotes=artifact_remotes, source_remotes=source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e # If there were no errors, we return the shell's exit code here. sys.exit(exitcode) ################################################################## # Source Command # ################################################################## @cli.group(short_help="Manipulate sources for an element") def source(): """Manipulate sources for an element""" ################################################################## # Source Fetch Command # ################################################################## @source.command(name="fetch", short_help="Fetch sources in a pipeline") @click.option( "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies from fetching", ) @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [ _PipelineSelection.NONE, _PipelineSelection.BUILD, _PipelineSelection.RUN, _PipelineSelection.ALL, ], ), help="The dependencies to fetch", ) @click.option( "--source-remote", "source_remotes", type=RemoteSpecType(RemoteSpecPurpose.PULL), multiple=True, help="A remote for downloading sources", ) @click.option( "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def source_fetch(app, elements, deps, except_, source_remotes, ignore_project_source_remotes): """Fetch sources required to build the pipeline Specifying no elements will result in fetching the default targets of the project. If no default targets are configured, all project elements will be fetched. When this command is executed from a workspace directory, the default is to fetch the workspace element. By default this will only try to fetch sources for the specified elements. Specify `--deps` to control which sources to fetch: \b none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies """ with app.initialized(session_name="Fetch"): if not elements: elements = app.stream.get_default_targets() app.stream.fetch( elements, selection=deps, except_targets=except_, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) ################################################################## # Source Push Command # ################################################################## @source.command(name="push", short_help="Push sources in a pipeline") @click.option( "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies from pushing", ) @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [ _PipelineSelection.NONE, _PipelineSelection.BUILD, _PipelineSelection.RUN, _PipelineSelection.ALL, ], ), help="The dependencies to push", ) @click.option( "--source-remote", "source_remotes", type=RemoteSpecType(RemoteSpecPurpose.PUSH), multiple=True, help="A remote for uploading sources", ) @click.option( "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def source_push(app, elements, deps, except_, source_remotes, ignore_project_source_remotes): """Push sources required to build the pipeline Specifying no elements will result in pushing the sources of the default targets of the project. If no default targets are configured, sources of all project elements will be pushed. When this command is executed from a workspace directory, the default is to push the sources of the workspace element. By default this will only try to push sources for the specified elements. Specify `--deps` to control which sources to fetch: \b none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies """ with app.initialized(session_name="Push"): if not elements: elements = app.stream.get_default_targets() app.stream.source_push( elements, selection=deps, except_targets=except_, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) ################################################################## # Source Track Command # ################################################################## @source.command(name="track", short_help="Track new source references") @click.option( "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies from tracking", ) @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.BUILD, _PipelineSelection.RUN, _PipelineSelection.ALL, _PipelineSelection.NONE], ), help="The dependencies to track", ) @click.option("--cross-junctions", "-J", is_flag=True, help="Allow crossing junction boundaries") @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def source_track(app, elements, deps, except_, cross_junctions): """Consults the specified tracking branches for new versions available to build and updates the project with any newly available references. Specifying no elements will result in tracking the default targets of the project. If no default targets are configured, all project elements will be tracked. When this command is executed from a workspace directory, the default is to track the workspace element. If no default is declared, all elements in the project will be tracked By default this will track just the specified element, but you can also update a whole tree of dependencies in one go. Specify `--deps` to control which sources to track: \b none: No dependencies, just the specified elements all: All dependencies of all specified elements """ with app.initialized(session_name="Track"): if not elements: elements = app.stream.get_default_targets() # Substitute 'none' for 'redirect' so that element redirections # will be done if deps == _PipelineSelection.NONE: deps = _PipelineSelection.REDIRECT app.stream.track(elements, selection=deps, except_targets=except_, cross_junctions=cross_junctions) ################################################################## # Source Checkout Command # ################################################################## @source.command(name="checkout", short_help="Checkout sources of an element") @click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") @click.option( "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" ) @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.BUILD, _PipelineSelection.NONE, _PipelineSelection.RUN, _PipelineSelection.ALL], ), help="The dependencies whose sources to checkout", ) @click.option( "--tar", default=None, metavar="LOCATION", type=click.Path(), help="Create a tarball containing the sources instead of a file tree.", ) @click.option( "--compression", default=None, type=click.Choice(["gz", "xz", "bz2"]), help="The compression option of the tarball created.", ) @click.option("--include-build-scripts", "build_scripts", is_flag=True) @click.option( "--directory", default="source-checkout", type=click.Path(file_okay=False), help="The directory to checkout the sources to", ) @click.option( "--source-remote", "source_remotes", type=RemoteSpecType(RemoteSpecPurpose.PULL), multiple=True, help="A remote for downloading cached sources", ) @click.option( "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("element", required=False, type=click.Path(readable=False)) @click.pass_obj def source_checkout( app, element, directory, force, deps, except_, tar, compression, build_scripts, source_remotes, ignore_project_source_remotes, ): """Checkout sources of an element to the specified location When this command is executed from a workspace directory, the default is to checkout the sources of the workspace element. """ if tar and directory != "source-checkout": click.echo("ERROR: options --directory and --tar conflict", err=True) sys.exit(-1) if compression and not tar: click.echo("ERROR: --compression specified without --tar", err=True) sys.exit(-1) # Set the location depending on whether --tar/--directory were specified # Note that if unset, --directory defaults to "source-checkout" location = tar if tar else directory with app.initialized(): if not element: element = app.stream.get_default_target() if not element: raise AppError('Missing argument "ELEMENT".') app.stream.source_checkout( element, location=location, force=force, deps=deps, except_targets=except_, tar=bool(tar), compression=compression, include_build_scripts=build_scripts, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) ################################################################## # Workspace Command # ################################################################## @cli.group(short_help="Manipulate developer workspaces") def workspace(): """Manipulate developer workspaces""" ################################################################## # Workspace Open Command # ################################################################## @workspace.command(name="open", short_help="Open a new workspace") @click.option("--no-checkout", is_flag=True, help="Do not checkout the source, only link to the given directory") @click.option( "--force", "-f", is_flag=True, help="The workspace will be created even if the directory in which it will be created is not empty " + "or if a workspace for that element already exists", ) @click.option( "--directory", type=click.Path(file_okay=False), default=None, help="Only for use when a single Element is given: Set the directory to use to create the workspace", ) @click.option( "--source-remote", "source_remotes", type=RemoteSpecType(RemoteSpecPurpose.PULL), multiple=True, help="A remote for downloading cached sources", ) @click.option( "--ignore-project-source-remotes", is_flag=True, help="Ignore remote source cache servers recommended by projects" ) @click.argument("elements", nargs=-1, type=click.Path(readable=False), required=True) @click.pass_obj def workspace_open(app, no_checkout, force, directory, source_remotes, ignore_project_source_remotes, elements): """Open a workspace for manual source modification""" with app.initialized(): app.stream.workspace_open( elements, no_checkout=no_checkout, force=force, custom_dir=directory, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) ################################################################## # Workspace Close Command # ################################################################## @workspace.command(name="close", short_help="Close workspaces") @click.option("--remove-dir", is_flag=True, help="Remove the path that contains the closed workspace") @click.option("--all", "-a", "all_", is_flag=True, help="Close all open workspaces") @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def workspace_close(app, remove_dir, all_, elements): """Close a workspace""" with app.initialized(): if not (all_ or elements): # NOTE: I may need to revisit this when implementing multiple projects # opening one workspace. element = app.stream.get_default_target() if element: elements = (element,) else: raise AppError("No elements specified") # Early exit if we specified `all` and there are no workspaces if all_ and not app.stream.workspace_exists(): click.echo("No open workspaces to close", err=True) sys.exit(0) if all_: elements = [element_name for element_name, _ in app.context.get_workspaces().list()] elements = app.stream.redirect_element_names(elements) # Check that the workspaces in question exist, and that it's safe to # remove them. nonexisting = [] for element_name in elements: if not app.stream.workspace_exists(element_name): nonexisting.append(element_name) if nonexisting: raise AppError("Workspace does not exist", detail="\n".join(nonexisting)) for element_name in elements: app.stream.workspace_close(element_name, remove_dir=remove_dir) ################################################################## # Workspace Reset Command # ################################################################## @workspace.command(name="reset", short_help="Reset a workspace to its original state") @click.option( "--soft", is_flag=True, help="Mark workspace to re-execute configuration steps (if any) on next build. Does not alter workspace contents.", ) @click.option("--all", "-a", "all_", is_flag=True, help="Reset all open workspaces") @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def workspace_reset(app, soft, all_, elements): """Reset a workspace to its original state""" # Check that the workspaces in question exist with app.initialized(): if not (all_ or elements): element = app.stream.get_default_target() if element: elements = (element,) else: raise AppError("No elements specified to reset") if all_ and not app.stream.workspace_exists(): raise AppError("No open workspaces to reset") if all_: elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list()) app.stream.workspace_reset(elements, soft=soft) ################################################################## # Workspace List Command # ################################################################## @workspace.command(name="list", short_help="List open workspaces") @click.pass_obj def workspace_list(app): """List open workspaces""" with app.initialized(): app.stream.workspace_list() ############################################################# # Artifact Commands # ############################################################# @cli.group(short_help="Manipulate cached artifacts.") def artifact(): """Manipulate cached artifacts Some subcommands take artifact references as arguments. Artifacts can be specified in two ways: \b - artifact refs: triples of the form // - element names When elements are given, the artifact is looked up by observing the element and it's current cache key. The commands also support shell-style wildcard expansion: `?` matches a single character, `*` matches zero or more characters but does not match the `/` path separator, and `**` matches zero or more characters including `/` path separators. If the wildcard expression ends with `.bst`, then it will be used to search element names found in the project, otherwise, it will be used to search artifacts that are present in the local artifact cache. Some example arguments are: \b - `myproject/hello/8276376b077eda104c812e6ec2f488c7c9eea211ce572c83d734c10bf241209f` - `myproject/he*/827637*` - `core/*.bst` (all elements in the core directory) - `**.bst` (all elements) - `myproject/**` (all artifacts from myproject) - `myproject/myelement/*` (all cached artifacts for a specific element) """ # Note that the backticks in the above docstring are important for the # generated docs. When sphinx is generating rst output from the help output # of this command, the asterisks will be interpreted as emphasis tokens if # they are not somehow escaped. ############################################################# # Artifact show Command # ############################################################# @artifact.command(name="show", short_help="Show the cached state of artifacts") @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.BUILD, _PipelineSelection.RUN, _PipelineSelection.ALL, _PipelineSelection.NONE], ), help="The dependencies we also want to show", ) @click.option( "--artifact-remote", "artifact_remotes", type=RemoteSpecType(RemoteSpecPurpose.PULL), multiple=True, help="A remote for downloading artifacts (Since: 2.7)", ) @click.option( "--ignore-project-artifact-remotes", is_flag=True, help="Ignore remote artifact cache servers recommended by projects (Since: 2.7)", ) @click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_show(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts): """Show the cached state of artifacts""" with app.initialized(): targets = app.stream.artifact_show( artifacts, selection=deps, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) click.echo(app.logger.show_state_of_artifacts(targets)) sys.exit(0) ##################################################################### # Artifact Checkout Command # ##################################################################### @artifact.command(name="checkout", short_help="Checkout contents of an artifact") @click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") @click.option( "--deps", "-d", default=_PipelineSelection.RUN, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.RUN, _PipelineSelection.BUILD, _PipelineSelection.NONE, _PipelineSelection.ALL], ), help="The dependencies to checkout", ) @click.option("--integrate/--no-integrate", default=None, is_flag=True, help="Whether to run integration commands") @click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible") @click.option( "--tar", default=None, metavar="LOCATION", type=click.Path(), help="Create a tarball from the artifact contents instead " "of a file tree. If LOCATION is '-', the tarball " "will be dumped to the standard output.", ) @click.option( "--compression", default=None, type=click.Choice(["gz", "xz", "bz2"]), help="The compression option of the tarball created.", ) @click.option( "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to" ) @click.option( "--artifact-remote", "artifact_remotes", type=RemoteSpecType(RemoteSpecPurpose.PULL), multiple=True, help="A remote for downloading artifacts", ) @click.option( "--ignore-project-artifact-remotes", is_flag=True, help="Ignore remote artifact cache servers recommended by projects", ) @click.argument("target", required=False, type=click.Path(readable=False)) @click.pass_obj def artifact_checkout( app, force, deps, integrate, hardlinks, tar, compression, directory, artifact_remotes, ignore_project_artifact_remotes, target, ): """Checkout contents of an artifact When this command is executed from a workspace directory, the default is to checkout the artifact of the workspace element. """ from .. import utils if hardlinks and tar: click.echo("ERROR: options --hardlinks and --tar conflict", err=True) sys.exit(-1) if tar and directory: click.echo("ERROR: options --directory and --tar conflict", err=True) sys.exit(-1) if not tar: if compression: click.echo("ERROR: --compression can only be provided if --tar is provided", err=True) sys.exit(-1) else: location = tar try: inferred_compression = utils._get_compression(tar) except UtilError as e: click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True) sys.exit(-1) if compression and inferred_compression != "" and inferred_compression != compression: click.echo( "WARNING: File extension and compression differ." "File extension has been overridden by --compression", err=True, ) if not compression: compression = inferred_compression with app.initialized(): if not target: target = app.stream.get_default_target() if not target: raise AppError('Missing argument "ELEMENT".') if not tar: if directory is None: location = os.path.abspath(os.path.join(os.getcwd(), target)) if location[-4:] == ".bst": location = location[:-4] else: location = directory app.stream.checkout( target, location=location, force=force, selection=deps, integrate=True if integrate is None else integrate, hardlinks=hardlinks, compression=compression, tar=bool(tar), artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) ################################################################ # Artifact Pull Command # ################################################################ @artifact.command(name="pull", short_help="Pull a built artifact") @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.BUILD, _PipelineSelection.NONE, _PipelineSelection.RUN, _PipelineSelection.ALL], ), help="The dependency artifacts to pull", ) @click.option( "--artifact-remote", "artifact_remotes", type=RemoteSpecType(RemoteSpecPurpose.PULL), multiple=True, help="A remote for downloading artifacts", ) @click.option( "--ignore-project-artifact-remotes", is_flag=True, help="Ignore remote artifact cache servers recommended by projects", ) @click.argument("artifacts", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def artifact_pull(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts): """Pull a built artifact from the configured remote artifact cache. Specifying no elements will result in pulling the default targets of the project. If no default targets are configured, all project elements will be pulled. When this command is executed from a workspace directory, the default is to pull the workspace element. By default the artifact will be pulled one of the configured caches if possible, following the usual priority order. If the `--artifact-remote` flag is given, only the specified cache will be queried. Specify `--deps` to control which artifacts to pull: \b none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies """ with app.initialized(session_name="Pull"): ignore_junction_targets = False if not artifacts: artifacts = app.stream.get_default_targets() # Junction elements cannot be pulled, exclude them from default targets ignore_junction_targets = True app.stream.pull( artifacts, selection=deps, ignore_junction_targets=ignore_junction_targets, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) ################################################################## # Artifact Push Command # ################################################################## @artifact.command(name="push", short_help="Push a built artifact") @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.BUILD, _PipelineSelection.NONE, _PipelineSelection.RUN, _PipelineSelection.ALL], ), help="The dependencies to push", ) @click.option( "--artifact-remote", "artifact_remotes", type=RemoteSpecType(RemoteSpecPurpose.PUSH), multiple=True, help="A remote for uploading artifacts", ) @click.option( "--ignore-project-artifact-remotes", is_flag=True, help="Ignore remote artifact cache servers recommended by projects", ) @click.argument("artifacts", nargs=-1, type=click.Path(readable=False)) @click.pass_obj def artifact_push(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts): """Push built artifacts to a remote artifact cache, possibly pulling them first. Specifying no elements will result in pushing the default targets of the project. If no default targets are configured, all project elements will be pushed. When this command is executed from a workspace directory, the default is to push the workspace element. If bst has been configured to include build trees on artifact pulls, an attempt will be made to pull any required build trees to avoid the skipping of partial artifacts being pushed. Specify `--deps` to control which artifacts to push: \b none: No dependencies, just the element itself run: Runtime dependencies, including the element itself build: Build time dependencies, excluding the element itself all: All dependencies """ with app.initialized(session_name="Push"): ignore_junction_targets = False if not artifacts: artifacts = app.stream.get_default_targets() # Junction elements cannot be pushed, exclude them from default targets ignore_junction_targets = True app.stream.push( artifacts, selection=deps, ignore_junction_targets=ignore_junction_targets, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) ################################################################ # Artifact Log Command # ################################################################ @artifact.command(name="log", short_help="Show logs of artifacts") @click.option( "--out", type=click.Path(file_okay=True, writable=True), help="Output logs to individual files in the specified path. If absent, logs are written to stdout.", ) @click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_log(app, artifacts, out): """Show build logs of artifacts""" with app.initialized(): artifact_logs = app.stream.artifact_log(artifacts) if not out: try: for log in list(artifact_logs.values()): with open(log[0], "r", encoding="utf-8") as f: data = f.read() click.echo_via_pager(data) except (OSError, FileNotFoundError): click.echo("Error: file cannot be opened", err=True) sys.exit(1) else: try: os.mkdir(out) except FileExistsError: click.echo("Error: {} already exists".format(out), err=True) sys.exit(1) for name, log_files in artifact_logs.items(): if len(log_files) > 1: os.mkdir(name) for log in log_files: dest = os.path.join(out, name, log) shutil.copy(log, dest) # make a dir and write in log files else: log_name = os.path.splitext(name)[0] + ".log" dest = os.path.join(out, log_name) shutil.copy(log_files[0], dest) # write a log file ################################################################ # Artifact List-Contents Command # ################################################################ @artifact.command(name="list-contents", short_help="List the contents of an artifact") @click.option( "--long", "-l", "long_", is_flag=True, help="Provide more information about the contents of the artifact." ) @click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_list_contents(app, artifacts, long_): """List the contents of an artifact. Note that 'artifacts' can be element names, which must end in '.bst', or artifact references, which must be in the format `//`. """ with app.initialized(): elements_to_files = app.stream.artifact_list_contents(artifacts) if not elements_to_files: click.echo("None of the specified artifacts are cached.", err=True) sys.exit(1) else: click.echo(app.logger.pretty_print_element_contents(elements_to_files, long_)) sys.exit(0) ################################################################### # Artifact Delete Command # ################################################################### @artifact.command(name="delete", short_help="Remove artifacts from the local cache") @click.option( "--deps", "-d", default=_PipelineSelection.NONE, show_default=True, type=FastEnumType( _PipelineSelection, [_PipelineSelection.NONE, _PipelineSelection.RUN, _PipelineSelection.BUILD, _PipelineSelection.ALL], ), help="The dependencies to delete", ) @click.argument("artifacts", type=click.Path(), nargs=-1) @click.pass_obj def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" with app.initialized(): app.stream.artifact_delete(artifacts, selection=deps) apache-buildstream-27ae392/src/buildstream/_frontend/complete.py000066400000000000000000000323451514607367700250730ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This module was forked from the python click library, Included # original copyright notice from the Click library and following disclaimer # as per their LICENSE requirements. # # Copyright (c) 2014 by Armin Ronacher. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # import collections.abc import copy import os from typing import TYPE_CHECKING import click from click.core import Option, Argument try: # Click >= 8.2 from click.shell_completion import split_arg_string except ImportError: # Click < 8.2 from click.parser import split_arg_string # type: ignore if TYPE_CHECKING or click.Command.__bases__ == (object,): # Click >= 8.2 ClickGroupBaseClass = click.Group else: # Click < 8.2 ClickGroupBaseClass = click.MultiCommand WORDBREAK = "=" COMPLETION_SCRIPT = """ %(complete_func)s() { local IFS=$'\n' COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\ COMP_CWORD=$COMP_CWORD \\ %(autocomplete_var)s=complete $1 ) ) return 0 } complete -F %(complete_func)s -o nospace %(script_names)s """ # An exception for our custom completion handler to # indicate that it does not want to handle completion # for this parameter # class CompleteUnhandled(Exception): pass def complete_path(path_type, incomplete, base_directory="."): """Helper method for implementing the completions() method for File and Path parameter types. """ # Try listing the files in the relative or absolute path # specified in `incomplete` minus the last path component, # otherwise list files starting from the current working directory. entries = [] base_path = "" # This is getting a bit messy listed_base_directory = False if os.path.sep in incomplete: split = incomplete.rsplit(os.path.sep, 1) base_path = split[0] # If there was nothing on the left of the last separator, # we are completing files in the filesystem root base_path = os.path.join(base_directory, base_path) else: incomplete_base_path = os.path.join(base_directory, incomplete) if os.path.isdir(incomplete_base_path): base_path = incomplete_base_path try: if base_path: if os.path.isdir(base_path): entries = [os.path.join(base_path, e) for e in os.listdir(base_path)] else: entries = os.listdir(base_directory) listed_base_directory = True except OSError: # If for any reason the os reports an error from os.listdir(), just # ignore this and avoid a stack trace pass base_directory_slash = base_directory if not base_directory_slash.endswith(os.sep): base_directory_slash += os.sep base_directory_len = len(base_directory_slash) def entry_is_dir(entry): if listed_base_directory: entry = os.path.join(base_directory, entry) return os.path.isdir(entry) def fix_path(path): # Append slashes to any entries which are directories, or # spaces for other files since they cannot be further completed if entry_is_dir(path) and not path.endswith(os.sep): path = path + os.sep else: path = path + " " # Remove the artificial leading path portion which # may have been prepended for search purposes. if path.startswith(base_directory_slash): path = path[base_directory_len:] return path return [ # Return an appropriate path for each entry fix_path(e) for e in sorted(entries) # Filter out non directory elements when searching for a directory, # the opposite is fine, however. if not (path_type == "Directory" and not entry_is_dir(e)) ] # Instead of delegating completions to the param type, # hard code all of buildstream's completions here. # # This whole module should be removed in favor of more # generic code in click once this issue is resolved: # https://github.com/pallets/click/issues/780 # def get_param_type_completion(param_type, incomplete): if isinstance(param_type, click.Choice): return [c + " " for c in param_type.choices] elif isinstance(param_type, click.File): return complete_path("File", incomplete) elif isinstance(param_type, click.Path): # Workaround click 8.x API break: # # https://github.com/pallets/click/issues/2037 # if param_type.file_okay and not param_type.dir_okay: path_type = "File" elif param_type.dir_okay and not param_type.file_okay: path_type = "Directory" else: path_type = "Path" return complete_path(path_type, incomplete) return [] def resolve_ctx(cli, prog_name, args): """ Parse into a hierarchy of contexts. Contexts are connected through the parent variable. :param cli: command definition :param prog_name: the program that is running :param args: full list of args typed before the incomplete arg :return: the final context/command parsed """ ctx = cli.make_context(prog_name, args, resilient_parsing=True) args_remaining = ctx.protected_args + ctx.args while ctx is not None and args_remaining: if isinstance(ctx.command, ClickGroupBaseClass): cmd = ctx.command.get_command(ctx, args_remaining[0]) if cmd is None: return None ctx = cmd.make_context(args_remaining[0], args_remaining[1:], parent=ctx, resilient_parsing=True) args_remaining = ctx.protected_args + ctx.args else: ctx = ctx.parent return ctx def start_of_option(param_str): """ :param param_str: param_str to check :return: whether or not this is the start of an option declaration (i.e. starts "-" or "--") """ return param_str and param_str[:1] == "-" def is_incomplete_option(all_args, cmd_param): """ :param all_args: the full original list of args supplied :param cmd_param: the current command paramter :return: whether or not the last option declaration (i.e. starts "-" or "--") is incomplete and corresponds to this cmd_param. In other words whether this cmd_param option can still accept values """ if cmd_param.is_flag: return False last_option = None for index, arg_str in enumerate(reversed([arg for arg in all_args if arg != WORDBREAK])): if index + 1 > cmd_param.nargs: break if start_of_option(arg_str): last_option = arg_str return bool(last_option and last_option in cmd_param.opts) def is_incomplete_argument(current_params, cmd_param): """ :param current_params: the current params and values for this argument as already entered :param cmd_param: the current command parameter :return: whether or not the last argument is incomplete and corresponds to this cmd_param. In other words whether or not the this cmd_param argument can still accept values """ current_param_values = current_params[cmd_param.name] if current_param_values is None: return True if cmd_param.nargs == -1: return True if ( isinstance(current_param_values, collections.abc.Iterable) and cmd_param.nargs > 1 and len(current_param_values) < cmd_param.nargs ): return True return False def get_user_autocompletions(args, incomplete, cmd, cmd_param, override): """ :param args: full list of args typed before the incomplete arg :param incomplete: the incomplete text of the arg to autocomplete :param cmd_param: command definition :param override: a callable (cmd_param, args, incomplete) that will be called to override default completion based on parameter type. Should raise 'CompleteUnhandled' if it could not find a completion. :return: all the possible user-specified completions for the param """ # Use the type specific default completions unless it was overridden try: return override(cmd=cmd, cmd_param=cmd_param, args=args, incomplete=incomplete) except CompleteUnhandled: return get_param_type_completion(cmd_param.type, incomplete) or [] def get_choices(cli, prog_name, args, incomplete, override): """ :param cli: command definition :param prog_name: the program that is running :param args: full list of args typed before the incomplete arg :param incomplete: the incomplete text of the arg to autocomplete :param override: a callable (cmd_param, args, incomplete) that will be called to override default completion based on parameter type. Should raise 'CompleteUnhandled' if it could not find a completion. :return: all the possible completions for the incomplete """ all_args = copy.deepcopy(args) ctx = resolve_ctx(cli, prog_name, args) if ctx is None: return # In newer versions of bash long opts with '='s are partitioned, but it's easier to parse # without the '=' if start_of_option(incomplete) and WORDBREAK in incomplete: partition_incomplete = incomplete.partition(WORDBREAK) all_args.append(partition_incomplete[0]) incomplete = partition_incomplete[2] elif incomplete == WORDBREAK: incomplete = "" choices = [] found_param = False if start_of_option(incomplete): # completions for options for param in ctx.command.params: if isinstance(param, Option): choices.extend( [ param_opt + " " for param_opt in param.opts + param.secondary_opts if param_opt not in all_args or param.multiple ] ) found_param = True if not found_param: # completion for option values by choices for cmd_param in ctx.command.params: if isinstance(cmd_param, Option) and is_incomplete_option(all_args, cmd_param): choices.extend(get_user_autocompletions(all_args, incomplete, ctx.command, cmd_param, override)) found_param = True break if not found_param: # completion for argument values by choices for cmd_param in ctx.command.params: if isinstance(cmd_param, Argument) and is_incomplete_argument(ctx.params, cmd_param): choices.extend(get_user_autocompletions(all_args, incomplete, ctx.command, cmd_param, override)) found_param = True break if not found_param and isinstance(ctx.command, ClickGroupBaseClass): # completion for any subcommands choices.extend( [cmd + " " for cmd in ctx.command.list_commands(ctx) if not ctx.command.get_command(ctx, cmd).hidden] ) if ( not start_of_option(incomplete) and ctx.parent is not None and isinstance(ctx.parent.command, ClickGroupBaseClass) and ctx.parent.command.chain ): # completion for chained commands visible_commands = [ cmd for cmd in ctx.parent.command.list_commands(ctx.parent) if not ctx.parent.command.get_command(ctx.parent, cmd).hidden ] remaining_commands = set(visible_commands) - set(ctx.parent.protected_args) choices.extend([cmd + " " for cmd in remaining_commands]) for item in choices: if item.startswith(incomplete): yield item def do_complete(cli, prog_name, override): cwords = split_arg_string(os.environ["COMP_WORDS"]) cword = int(os.environ["COMP_CWORD"]) args = cwords[1:cword] try: incomplete = cwords[cword] except IndexError: incomplete = "" for item in get_choices(cli, prog_name, args, incomplete, override): click.echo(item) # Main function called from main.py at startup here # def main_bashcomplete(cmd, prog_name, override): """Internal handler for the bash completion support.""" if "_BST_COMPLETION" in os.environ: do_complete(cmd, prog_name, override) return True return False apache-buildstream-27ae392/src/buildstream/_frontend/linuxapp.py000066400000000000000000000034631514607367700251220ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os import click from .app import App # This trick is currently only supported on some terminals, # avoid using it where it can cause garbage to be printed # to the terminal. # def _osc_777_supported(): term = os.environ.get("TERM") if term and (term.startswith("xterm") or term.startswith("vte")): # Since vte version 4600, upstream silently ignores # the OSC 777 without printing garbage to the terminal. # # For distros like Fedora who have patched vte, this # will trigger a desktop notification and bring attention # to the terminal. # vte_version = os.environ.get("VTE_VERSION") try: vte_version_int = int(vte_version) except (ValueError, TypeError): return False if vte_version_int >= 4600: return True return False # A linux specific App implementation # class LinuxApp(App): def notify(self, title, text): # Currently we only try this notification method # of sending an escape sequence to the terminal # if _osc_777_supported(): click.echo("\033]777;notify;{};{}\007".format(title, text), err=True) apache-buildstream-27ae392/src/buildstream/_frontend/profile.py000066400000000000000000000044751514607367700247260ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import re import copy import click # Profile() # # A class for formatting text with ansi color codes # # Kwargs: # The same keyword arguments which can be used with click.style() # class Profile: def __init__(self, **kwargs): self._kwargs = dict(kwargs) # fmt() # # Format some text with ansi color codes # # Args: # text (str): The text to format # # Kwargs: # Keyword arguments to apply on top of the base click.style() # arguments # def fmt(self, text, **kwargs): kwargs = dict(kwargs) fmtargs = copy.copy(self._kwargs) fmtargs.update(kwargs) return click.style(text, **fmtargs) # fmt_subst() # # Substitute a variable of the %{varname} form, formatting # only the substituted text with the given click.style() configurations # # Args: # text (str): The text to format, with possible variables # varname (str): The variable name to substitute # value (str): The value to substitute the variable with # # Kwargs: # Keyword arguments to apply on top of the base click.style() # arguments # def fmt_subst(self, text, varname, value, **kwargs): def subst_callback(match): # Extract and format the "{(varname)...}" portion of the match inner_token = match.group(1) formatted = inner_token.format(**{varname: value}) # Colorize after the pythonic format formatting, which may have padding return self.fmt(formatted, **kwargs) # Lazy regex, after our word, match anything that does not have '%' return re.sub(r"%(\{(" + varname + r")[^%]*?\})", subst_callback, text) apache-buildstream-27ae392/src/buildstream/_frontend/status.py000066400000000000000000000461231514607367700246050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os import sys import curses from collections import OrderedDict import shutil import click # Import a widget internal for formatting time codes from .widget import TimeCode # Status() # # A widget for formatting overall status. # # Note that the render() and clear() methods in this class are # simply noops in the case that the application is not connected # to a terminal, or if the terminal does not support ANSI escape codes. # # Args: # context (Context): The Context # state (State): The state data from the Core # content_profile (Profile): Formatting profile for content text # format_profile (Profile): Formatting profile for formatting text # success_profile (Profile): Formatting profile for success text # error_profile (Profile): Formatting profile for error text # stream (Stream): The Stream # class Status: # Table of the terminal capabilities we require and use _TERM_CAPABILITIES = {"move_up": "cuu1", "move_x": "hpa", "clear_eol": "el"} def __init__(self, context, state, content_profile, format_profile, success_profile, error_profile, stream): self._context = context self._state = state self._content_profile = content_profile self._format_profile = format_profile self._success_profile = success_profile self._error_profile = error_profile self._stream = stream self._jobs = OrderedDict() self._last_lines = 0 # Number of status lines we last printed to console self._spacing = 1 self._header = _StatusHeader( context, state, content_profile, format_profile, success_profile, error_profile, stream ) self._term_width, _ = shutil.get_terminal_size() self._alloc_lines = 0 self._alloc_columns = None self._need_alloc = True self._term_caps = self._init_terminal() state.register_task_added_callback(self._add_job) state.register_task_removed_callback(self._remove_job) state.register_task_changed_callback(self._job_changed) # clear() # # Clear the status area, it is necessary to call # this before printing anything to the console if # a status area is in use. # # To print some logging to the output and then restore # the status, use the following: # # status.clear() # ... print something to console ... # status.render() # def clear(self): if not self._term_caps: return for _ in range(self._last_lines): self._move_up() self._clear_line() self._last_lines = 0 # render() # # Render the status area. # # If you are not printing a line in addition to rendering # the status area, for instance in a timeout, then it is # not necessary to call clear(). def render(self): if not self._term_caps: return elapsed = self._state.elapsed_time() self.clear() self._check_term_width() self._allocate() # Nothing to render, early return if self._alloc_lines == 0: return # Before rendering the actual lines, we need to add some line # feeds for the amount of lines we intend to print first, and # move cursor position back to the first line for _ in range(self._alloc_lines + self._header.lines): click.echo("", err=True) for _ in range(self._alloc_lines + self._header.lines): self._move_up() # Render the one line header text = self._header.render(self._term_width, elapsed) click.echo(text, err=True) # Now we have the number of columns, and an allocation for # alignment of each column n_columns = len(self._alloc_columns) for line in self._job_lines(n_columns): text = "" for job in line: column = line.index(job) text += job.render(self._alloc_columns[column] - job.size, elapsed) # Add spacing between columns if column < (n_columns - 1): text += " " * self._spacing # Print the line click.echo(text, err=True) # Track what we printed last, for the next clear self._last_lines = self._alloc_lines + self._header.lines ################################################### # Private Methods # ################################################### # _job_changed() # # Reacts to a specified job being changed # # Args: # task_id (str): The unique identifier of the task # def _job_changed(self, task_id): task = self._state.tasks[task_id] job = self._jobs[task_id] if job.update(task): self._need_alloc = True # _init_terminal() # # Initialize the terminal and return the resolved terminal # capabilities dictionary. # # Returns: # (dict|None): The resolved terminal capabilities dictionary, # or None if the terminal does not support all # of the required capabilities. # def _init_terminal(self): # We need both output streams to be connected to a terminal if not (sys.stdout.isatty() and sys.stderr.isatty()): return None # Initialized terminal, curses might decide it doesnt # support this terminal try: curses.setupterm(os.environ.get("TERM", "dumb")) except curses.error: return None term_caps = {} # Resolve the string capabilities we need for the capability # names we need. # for capname, capval in self._TERM_CAPABILITIES.items(): code = curses.tigetstr(capval) # If any of the required capabilities resolve empty strings or None, # then we don't have the capabilities we need for a status bar on # this terminal. if not code: return None # Decode sequences as latin1, as they are always 8-bit bytes, # so when b'\xff' is returned, this must be decoded to u'\xff'. # # This technique is employed by the python blessings library # as well, and should provide better compatibility with most # terminals. # term_caps[capname] = code.decode("latin1") return term_caps def _check_term_width(self): term_width, _ = shutil.get_terminal_size() if self._term_width != term_width: self._term_width = term_width self._need_alloc = True def _move_up(self): assert self._term_caps is not None # Explicitly move to beginning of line, fixes things up # when there was a ^C or ^Z printed to the terminal. move_x = curses.tparm(self._term_caps["move_x"].encode("latin1"), 0) move_x = move_x.decode("latin1") move_up = curses.tparm(self._term_caps["move_up"].encode("latin1")) move_up = move_up.decode("latin1") click.echo(move_x + move_up, nl=False, err=True) def _clear_line(self): assert self._term_caps is not None clear_eol = curses.tparm(self._term_caps["clear_eol"].encode("latin1")) clear_eol = clear_eol.decode("latin1") click.echo(clear_eol, nl=False, err=True) def _allocate(self): if not self._need_alloc: return # State when there is no jobs to display alloc_lines = 0 alloc_columns = [] line_length = 0 # Test for the widest width which fits columnized jobs for columns in reversed(range(len(self._jobs))): alloc_lines, alloc_columns = self._allocate_columns(columns + 1) # If the sum of column widths with spacing in between # fits into the terminal width, this is a good allocation. line_length = sum(alloc_columns) + (columns * self._spacing) if line_length < self._term_width: break self._alloc_lines = alloc_lines self._alloc_columns = alloc_columns self._need_alloc = False def _job_lines(self, columns): jobs_list = list(self._jobs.values()) for i in range(0, len(self._jobs), columns): yield jobs_list[i : i + columns] # Returns an array of integers representing the maximum # length in characters for each column, given the current # list of jobs to render. # def _allocate_columns(self, columns): column_widths = [0 for _ in range(columns)] lines = 0 for line in self._job_lines(columns): line_len = len(line) lines += 1 for col in range(columns): if col < line_len: job = line[col] column_widths[col] = max(column_widths[col], job.size) return lines, column_widths # _add_job() # # Adds a job to track in the status area # # Args: # task_id (str): The unique identifier of the task # def _add_job(self, task_id): task = self._state.tasks[task_id] elapsed = task.elapsed_offset job = _StatusJob( self._context, task.action_name, task.full_name, self._content_profile, self._format_profile, elapsed ) self._jobs[task_id] = job self._need_alloc = True # _remove_job() # # Removes a job currently being tracked in the status area # # Args: # task_id (str): The unique identifier of the task # def _remove_job(self, task_id): del self._jobs[task_id] self._need_alloc = True # _StatusHeader() # # A delegate object for rendering the header part of the Status() widget # # Args: # context (Context): The Context # content_profile (Profile): Formatting profile for content text # format_profile (Profile): Formatting profile for formatting text # success_profile (Profile): Formatting profile for success text # error_profile (Profile): Formatting profile for error text # stream (Stream): The Stream # class _StatusHeader: def __init__(self, context, state, content_profile, format_profile, success_profile, error_profile, stream): # # Public members # self.lines = 3 # # Private members # self._content_profile = content_profile self._format_profile = format_profile self._success_profile = success_profile self._error_profile = error_profile self._stream = stream self._state = state self._time_code = TimeCode(context, content_profile, format_profile) self._context = context def render(self, line_length, elapsed): project = self._context.get_toplevel_project() line_length = max(line_length, 80) # # Line 1: Session time, project name, session / total elements # # ========= 00:00:00 project-name (143/387) ========= # session = str(len(self._stream.session_elements)) total = str(len(self._stream.total_elements)) size = 0 text = "" size += len(total) + len(session) + 4 # Size for (N/N) with a leading space size += 8 # Size of time code size += len(project.name) + 1 text += self._time_code.render_time(elapsed) text += " " + self._content_profile.fmt(project.name) text += ( " " + self._format_profile.fmt("(") + self._content_profile.fmt(session) + self._format_profile.fmt("/") + self._content_profile.fmt(total) + self._format_profile.fmt(")") ) line1 = self._centered(text, size, line_length, "=") # # Line 2: Dynamic list of queue status reports # # (Sources Fetched:0 117 0)→ (Built:4 0 0) # size = 0 text = "" # Format and calculate size for each queue progress for index, task_group in enumerate(self._state.task_groups.values()): # Add spacing if index > 0: size += 2 text += self._format_profile.fmt("→ ") group_text, group_size = self._render_task_group(task_group) size += group_size text += group_text line2 = self._centered(text, size, line_length, " ") # # Line 3: Cache usage percentage report # # ~~~~~~ cache: 44.2G / 64G (69%) ~~~~~~ # cas = self._context.get_cascache() usage = cas.get_cache_usage() usage_string = str(usage) if usage.used_size is None: # Cache usage is unknown size = 0 text = "" else: size = 21 size += len(usage_string) if usage.used_percent >= 95: formatted_usage = self._error_profile.fmt(usage_string) elif usage.used_percent >= 80: formatted_usage = self._content_profile.fmt(usage_string) else: formatted_usage = self._success_profile.fmt(usage_string) text = ( self._format_profile.fmt("~~~~~~ ") + self._content_profile.fmt("cache") + self._format_profile.fmt(": ") + formatted_usage + self._format_profile.fmt(" ~~~~~~") ) line3 = self._centered(text, size, line_length, " ") return line1 + "\n" + line2 + "\n" + line3 ################################################### # Private Methods # ################################################### def _render_task_group(self, group): processed = str(group.processed_tasks) skipped = str(group.skipped_tasks) failed = str(len(group.failed_tasks)) size = 5 # Space for the formatting '[', ':', ' ', ' ' and ']' size += len(group.complete_name) size += len(processed) + len(skipped) + len(failed) text = ( self._format_profile.fmt("(") + self._content_profile.fmt(group.complete_name) + self._format_profile.fmt(":") + self._success_profile.fmt(processed) + " " + self._content_profile.fmt(skipped) + " " + self._error_profile.fmt(failed) + self._format_profile.fmt(")") ) return (text, size) def _centered(self, text, size, line_length, fill): remaining = line_length - size remaining -= 2 final_text = self._format_profile.fmt(fill * (remaining // 2)) + " " final_text += text final_text += " " + self._format_profile.fmt(fill * (remaining // 2)) return final_text # _StatusJob() # # A delegate object for rendering a job in the status area # # Args: # context (Context): The Context # action_name (str): The action performed # full_name (str): The name of the job # content_profile (Profile): Formatting profile for content text # format_profile (Profile): Formatting profile for formatting text # elapsed (datetime): The offset into the session when this job is created # class _StatusJob: def __init__(self, context, action_name, full_name, content_profile, format_profile, elapsed): # # Public members # self.action_name = action_name # The action name self.size = None # The number of characters required to render self.full_name = full_name # # Private members # self._offset = elapsed self._content_profile = content_profile self._format_profile = format_profile self._time_code = TimeCode(context, content_profile, format_profile) self._current_progress = None # Progress tally to render self._maximum_progress = None # Progress tally to render self.size = self.calculate_size() # calculate_size() # # Calculates the amount of space the job takes up when rendered # # Returns: # int: The size of the job when rendered # def calculate_size(self): # Calculate the size needed to display size = 10 # Size of time code with brackets size += len(self.action_name) size += len(self.full_name) size += 3 # '[' + ':' + ']' if self._current_progress is not None: size += len(str(self._current_progress)) size += 1 # ':' if self._maximum_progress is not None: size += len(str(self._maximum_progress)) size += 1 # '/' return size # update() # # Synchronises its internal data with the provided Task, # and returns whether its size has changed # # Args: # task (Task): The task associated with this job # # Returns: # bool: Whether the size of the job has changed # def update(self, task): changed = False size_changed = False if task.current_progress != self._current_progress: changed = True self._current_progress = task.current_progress if task.maximum_progress != self._maximum_progress: changed = True self._maximum_progress = task.maximum_progress if changed: old_size = self.size self.size = self.calculate_size() if self.size != old_size: size_changed = True return size_changed # render() # # Render the Job, return a rendered string # # Args: # padding (int): Amount of padding to print in order to align with columns # elapsed (datetime): The session elapsed time offset # def render(self, padding, elapsed): text = ( self._format_profile.fmt("[") + self._time_code.render_time(elapsed - self._offset) + self._format_profile.fmt("]") ) text += ( self._format_profile.fmt("[") + self._content_profile.fmt(self.action_name) + self._format_profile.fmt(":") + self._content_profile.fmt(self.full_name) ) if self._current_progress is not None: text += self._format_profile.fmt(":") + self._content_profile.fmt(str(self._current_progress)) if self._maximum_progress is not None: text += self._format_profile.fmt("/") + self._content_profile.fmt(str(self._maximum_progress)) # Add padding before terminating ']' terminator = (" " * padding) + "]" text += self._format_profile.fmt(terminator) return text apache-buildstream-27ae392/src/buildstream/_frontend/widget.py000066400000000000000000001224521514607367700245450ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import datetime import os from contextlib import ExitStack from mmap import mmap import re import textwrap import click from .profile import Profile from ..types import _Scope from .. import _yaml from .. import __version__ as bst_version from .. import FileType from .._exceptions import BstError, ImplError from .._message import MessageType from .._artifactelement import ArtifactElement # These messages are printed a bit differently ERROR_MESSAGES = [MessageType.FAIL, MessageType.ERROR, MessageType.BUG] class FormattingError(Exception): pass # Widget() # # Args: # content_profile (Profile): The profile to use for rendering content # format_profile (Profile): The profile to use for rendering formatting # # An abstract class for printing output columns in our text UI. # class Widget: def __init__(self, context, content_profile, format_profile): # The context self.context = context # The content profile self.content_profile = content_profile # The formatting profile self.format_profile = format_profile # render() # # Renders a string to be printed in the UI # # Args: # message (Message): A message to print # # Returns: # (str): The string this widget prints for the given message # def render(self, message): raise ImplError("{} does not implement render()".format(type(self).__name__)) # Used to add fixed text between columns class FixedText(Widget): def __init__(self, context, text, content_profile, format_profile): super().__init__(context, content_profile, format_profile) self.text = text def render(self, message): return self.format_profile.fmt(self.text) # Used to add the wallclock time this message was created at class WallclockTime(Widget): def __init__(self, context, content_profile, format_profile, output_format=False): self._output_format = output_format super().__init__(context, content_profile, format_profile) def render(self, message): fields = [ self.content_profile.fmt("{:02d}".format(x)) for x in [ message.creation_time.hour, message.creation_time.minute, message.creation_time.second, ] ] text = self.format_profile.fmt(":").join(fields) if self._output_format == "us": text += self.content_profile.fmt(".{:06d}".format(message.creation_time.microsecond)) return text # A widget for rendering the debugging column class Debug(Widget): def render(self, message): element_name = "n/a" if message.element_name is None else message.element_name text = self.format_profile.fmt("pid:") text += self.content_profile.fmt("{: <5}".format(message.pid)) text += self.format_profile.fmt("element name:") text += self.content_profile.fmt("{: <30}".format(element_name)) return text # A widget for rendering the time codes class TimeCode(Widget): def __init__(self, context, content_profile, format_profile, microseconds=False): self._microseconds = microseconds super().__init__(context, content_profile, format_profile) def render(self, message): return self.render_time(message.elapsed) def render_time(self, elapsed): if elapsed is None: fields = [self.content_profile.fmt("--") for i in range(3)] else: hours, remainder = divmod(int(elapsed.total_seconds()), 60 * 60) minutes, seconds = divmod(remainder, 60) fields = [self.content_profile.fmt("{0:02d}".format(field)) for field in [hours, minutes, seconds]] text = self.format_profile.fmt(":").join(fields) if self._microseconds: if elapsed is not None: text += self.content_profile.fmt(".{0:06d}".format(elapsed.microseconds)) else: text += self.content_profile.fmt(".------") return text # A widget for rendering the MessageType class TypeName(Widget): _action_colors = { MessageType.DEBUG: "cyan", MessageType.STATUS: "cyan", MessageType.INFO: "magenta", MessageType.WARN: "yellow", MessageType.START: "blue", MessageType.SUCCESS: "green", MessageType.FAIL: "red", MessageType.SKIPPED: "yellow", MessageType.ERROR: "red", MessageType.BUG: "red", } def render(self, message): return self.content_profile.fmt( "{: <7}".format(message.message_type.upper()), bold=True, dim=True, fg=self._action_colors[message.message_type], ) # A widget for displaying the Element name class ElementName(Widget): def render(self, message): action_name = message.action_name element_name = message.task_element_name or message.element_name if element_name is not None: name = "{: <30}".format(element_name) else: name = "core activity" name = "{: <30}".format(name) if not action_name: action_name = "Main" return ( self.content_profile.fmt("{: >8}".format(action_name.lower())) + self.format_profile.fmt(":") + self.content_profile.fmt(name) ) # A widget for displaying the primary message text class MessageText(Widget): def render(self, message): return message.message # A widget for formatting the element cache key class CacheKey(Widget): def __init__(self, context, content_profile, format_profile, err_profile): super().__init__(context, content_profile, format_profile) self._err_profile = err_profile self._key_length = context.log_key_length def render(self, message): if not self._key_length: return "" if message.element_name is None: return " " * self._key_length dim = False key = " " * self._key_length element_key = message.task_element_key or message.element_key if element_key: key = element_key.brief dim = not element_key.strict if message.message_type in ERROR_MESSAGES: text = self._err_profile.fmt(key) else: text = self.content_profile.fmt(key, dim=dim) return text # A widget for formatting the log file class LogFile(Widget): def __init__(self, context, content_profile, format_profile, err_profile): super().__init__(context, content_profile, format_profile) self._err_profile = err_profile self._logdir = context.logdir def render(self, message): return self.render_abbrev(message) def render_abbrev(self, message, abbrev=True): if message.logfile and message.scheduler: logfile = message.logfile if abbrev and self._logdir != "" and logfile.startswith(self._logdir): logfile = logfile[len(self._logdir) :] logfile = logfile.lstrip(os.sep) if message.message_type in ERROR_MESSAGES: text = self._err_profile.fmt(logfile) else: text = self.content_profile.fmt(logfile, dim=True) else: text = "" return text # START and SUCCESS messages are expected to have no useful # information in the message text, so we display the logfile name for # these messages, and the message text for other types. # class MessageOrLogFile(Widget): def __init__(self, context, content_profile, format_profile, err_profile): super().__init__(context, content_profile, format_profile) self._message_widget = MessageText(context, content_profile, format_profile) self._logfile_widget = LogFile(context, content_profile, format_profile, err_profile) def render(self, message): # Show the log file only in the main start/success messages if message.logfile and message.scheduler and message.message_type in [MessageType.START, MessageType.SUCCESS]: text = self._logfile_widget.render(message) else: text = self._message_widget.render(message) return text # LogLine # # A widget for formatting a log line # # Args: # context (Context): The Context # state (State): The state data from the Core # content_profile (Profile): Formatting profile for content text # format_profile (Profile): Formatting profile for formatting text # success_profile (Profile): Formatting profile for success text # error_profile (Profile): Formatting profile for error text # detail_profile (Profile): Formatting profile for detail text # indent (int): Number of spaces to use for general indentation # class LogLine(Widget): def __init__( self, context, state, content_profile, format_profile, success_profile, err_profile, detail_profile, indent=4 ): super().__init__(context, content_profile, format_profile) self._columns = [] self._failure_messages = {} self._success_profile = success_profile self._err_profile = err_profile self._detail_profile = detail_profile self._indent = " " * indent self._log_lines = context.log_error_lines self._message_lines = context.log_message_lines self._resolved_keys = None self._state = state self._logfile_widget = LogFile(context, content_profile, format_profile, err_profile) if context.log_debug: self._columns.extend([Debug(context, content_profile, format_profile)]) self.logfile_variable_names = { "elapsed": TimeCode(context, content_profile, format_profile, microseconds=False), "elapsed-us": TimeCode(context, content_profile, format_profile, microseconds=True), "wallclock": WallclockTime(context, content_profile, format_profile), "wallclock-us": WallclockTime(context, content_profile, format_profile, output_format="us"), "key": CacheKey(context, content_profile, format_profile, err_profile), "element": ElementName(context, content_profile, format_profile), "action": TypeName(context, content_profile, format_profile), "message": MessageOrLogFile(context, content_profile, format_profile, err_profile), } logfile_tokens = self._parse_logfile_format(context.log_message_format, content_profile, format_profile) self._columns.extend(logfile_tokens) # show_pipeline() # # Display a list of elements in the specified format. # # The formatting string is the one currently documented in `bst show`, this # is used in pipeline session headings and also to implement `bst show`. # # Args: # dependencies (list of Element): A list of Element objects # format_: A formatting string, as specified by `bst show` # # Returns: # (str): The formatted list of elements # def show_pipeline(self, dependencies, format_): report = "" p = Profile() for element in dependencies: line = format_ key = element._get_display_key() dim_keys = not key.strict # Guarantee that description is reported on a single line. description = " ".join(element._description.splitlines()) line = p.fmt_subst(line, "name", element._get_full_name(), fg="blue", bold=True) line = p.fmt_subst(line, "kind", element.get_kind(), fg="cyan") line = p.fmt_subst(line, "key", key.brief, fg="yellow", dim=dim_keys) line = p.fmt_subst(line, "full-key", key.full, fg="yellow", dim=dim_keys) line = p.fmt_subst(line, "description", description, fg="yellow", dim=dim_keys) try: if not element._has_all_sources_resolved(): line = p.fmt_subst(line, "state", "no reference", fg="red") else: if element.get_kind() == "junction": line = p.fmt_subst(line, "state", "junction", fg="magenta") elif not element._can_query_cache(): line = p.fmt_subst(line, "state", "waiting", fg="blue") elif element._cached_failure(): line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_success(): line = p.fmt_subst(line, "state", "cached", fg="magenta") elif not element._can_query_source_cache(): line = p.fmt_subst(line, "state", "waiting", fg="blue") elif element._fetch_needed(): line = p.fmt_subst(line, "state", "fetch needed", fg="red") elif element._buildable(): line = p.fmt_subst(line, "state", "buildable", fg="green") else: line = p.fmt_subst(line, "state", "waiting", fg="blue") except BstError as e: # Provide context to plugin error e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),) raise e # Element configuration if "%{config" in format_: line = p.fmt_subst( line, "config", _yaml.roundtrip_dump_string(element._Element__config), ) # Variables if "%{vars" in format_: variables = dict(element._Element__variables) line = p.fmt_subst(line, "vars", _yaml.roundtrip_dump_string(variables)) # Environment if "%{env" in format_: environment = element._Element__environment line = p.fmt_subst(line, "env", _yaml.roundtrip_dump_string(environment)) # Public if "%{public" in format_: public = element._Element__public line = p.fmt_subst(line, "public", _yaml.roundtrip_dump_string(public)) # Workspaced if "%{workspaced" in format_: line = p.fmt_subst(line, "workspaced", "(workspaced)" if element._get_workspace() else "", fg="yellow") # Workspace-dirs if "%{workspace-dirs" in format_: workspace = element._get_workspace() if workspace is not None: path = workspace.get_absolute_path() if path.startswith("~/"): path = os.path.join(os.getenv("HOME", "/root"), path[2:]) line = p.fmt_subst(line, "workspace-dirs", "Workspace: {}".format(path)) else: line = p.fmt_subst(line, "workspace-dirs", "") # Dependencies if "%{deps" in format_: deps = [e._get_full_name() for e in element._dependencies(_Scope.ALL, recurse=False)] line = p.fmt_subst(line, "deps", _yaml.roundtrip_dump_string(deps).rstrip("\n")) # Build Dependencies if "%{build-deps" in format_: build_deps = [e._get_full_name() for e in element._dependencies(_Scope.BUILD, recurse=False)] line = p.fmt_subst(line, "build-deps", _yaml.roundtrip_dump_string(build_deps).rstrip("\n")) # Runtime Dependencies if "%{runtime-deps" in format_: runtime_deps = [e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False)] line = p.fmt_subst(line, "runtime-deps", _yaml.roundtrip_dump_string(runtime_deps).rstrip("\n")) # Source Information if "%{source-info" in format_: # Get all the SourceInfo objects # all_source_infos = [] for source in element.sources(): source_infos = source.collect_source_info() if source_infos is not None: serialized_sources = [] for s in source_infos: serialized = s.serialize() serialized_sources.append(serialized) all_source_infos += serialized_sources # Dump the SourceInfo provenance objects in yaml format line = p.fmt_subst(line, "source-info", _yaml.roundtrip_dump_string(all_source_infos)) # Artifact CAS Digest if "%{artifact-cas-digest" in format_: artifact = element._get_artifact() if artifact.cached(): artifact_files = artifact.get_files() # We call the private CasBasedDirectory._get_digest() for # the moment, we should make it public on Directory. artifact_digest = artifact_files._get_digest() formated_artifact_digest = "{}/{}".format(artifact_digest.hash, artifact_digest.size_bytes) line = p.fmt_subst(line, "artifact-cas-digest", formated_artifact_digest) else: line = p.fmt_subst(line, "artifact-cas-digest", "") element.warn("Cannot obtain CAS digest because artifact is not cached") report += line + "\n" return report.rstrip("\n") # print_heading() # # A message to be printed at program startup, indicating # some things about user configuration and BuildStream version # and so on. # # Args: # toplevel_project (Project): The toplevel project we were invoked from, or None # stream (Stream): The stream # log_file (file): An optional file handle for additional logging # def print_heading(self, toplevel_project, stream, *, log_file): context = self.context starttime = datetime.datetime.now() text = "" def format_spec(spec): if spec.instance_name: return "{} (instance: {})".format(spec.url, spec.instance_name) return spec.url self._resolved_keys = {element: element._get_cache_key() for element in stream.session_elements} # Main invocation context text += "\n" text += self.content_profile.fmt("BuildStream Version {}\n".format(bst_version), bold=True) values = {} values["Session Start"] = starttime.strftime("%A, %d-%m-%Y at %H:%M:%S") if toplevel_project: values["Project"] = "{} ({})".format(toplevel_project.name, toplevel_project.directory) values["Targets"] = ", ".join([t.name for t in stream.targets]) text += self._format_values(values) # User configurations text += "\n" text += self.content_profile.fmt("User Configuration\n", bold=True) values = {} values["Configuration File"] = "Default Configuration" if not context.config_origin else context.config_origin values["Cache Directory"] = context.cachedir values["Log Files"] = context.logdir values["Source Mirrors"] = context.sourcedir values["Build Area"] = context.builddir values["Strict Build Plan"] = "Yes" if context.get_strict() else "No" values["Maximum Fetch Tasks"] = context.sched_fetchers values["Maximum Build Tasks"] = context.sched_builders values["Maximum Push Tasks"] = context.sched_pushers values["Maximum Network Retries"] = context.sched_network_retries if context.remote_cache_spec: values["Cache Storage Service"] = format_spec(context.remote_cache_spec) text += self._format_values(values) if context.remote_execution_specs: specs = context.remote_execution_specs text += "\n" text += self.content_profile.fmt("Remote Execution Configuration\n", bold=True) values = {} if specs.exec_spec: values["Execution Service"] = format_spec(specs.exec_spec) re_storage_spec = specs.storage_spec or context.remote_cache_spec values["Storage Service"] = format_spec(re_storage_spec) if specs.action_spec: values["Action Cache Service"] = format_spec(specs.action_spec) text += self._format_values(values) # Print information about each loaded project # text += "\n" if toplevel_project: loaded_projects = toplevel_project.loaded_projects() else: loaded_projects = [] for project_info in loaded_projects: project = project_info.project # Project title line text += ( self.content_profile.fmt("Project", bold=True) + self.format_profile.fmt(": ", bold=True) + self.content_profile.fmt(project.name, bold=True) ) text += "\n" # Details on how the project was loaded # values = {} if project.junction: values["Junction path"] = project_info.project.junction._get_full_name() if project_info.provenance: values["Loaded by"] = str(project_info.provenance) text += self._format_values(values) # Print out duplicate declarations if project_info.duplicates: text += self.format_profile.fmt("{}Declared duplicate by:\n".format(self._indent)) for duplicate in project_info.duplicates: text += self.content_profile.fmt("{}{}\n".format(self._indent * 2, duplicate)) # Print out internal declarations if project_info.internal: text += self.format_profile.fmt("{}Declared internal by:\n".format(self._indent)) for internal in project_info.internal: text += self.content_profile.fmt("{}{}\n".format(self._indent * 2, internal)) text += "\n" # Project Options values = {} project.options.printable_variables(values) if values: text += self.format_profile.fmt("{}Project Options\n".format(self._indent)) text += self._format_values(values, indent=2) text += "\n" # Plugins text += self._format_plugins( {p: d for p, _, _, d in project.element_factory.list_plugins()}, {p: d for p, _, _, d in project.source_factory.list_plugins()}, ) # Artifact cache servers specs = context.project_artifact_cache_specs.get(project.name) if specs: text += self.format_profile.fmt("{}Artifact cache servers\n".format(self._indent)) text += self._format_list(specs, indent=2) text += "\n" # Source cache servers specs = context.project_source_cache_specs.get(project.name) if specs: text += self.format_profile.fmt("{}Source cache servers\n".format(self._indent)) text += self._format_list(specs, indent=2) text += "\n" # Pipeline state text += self.content_profile.fmt("Pipeline\n", bold=True) text += self.show_pipeline(stream.total_elements, context.log_element_format) text += "\n" # Separator line before following output text += self.format_profile.fmt("=" * 79 + "\n") click.echo(text, nl=False, err=True) if log_file: click.echo(text, file=log_file, color=False, nl=False) # print_summary() # # Print a summary of activities at the end of a session # # Args: # stream (Stream): The Stream # log_file (file): An optional file handle for additional logging # def print_summary(self, stream, log_file): # Early silent return if there are no queues, can happen # only in the case that the stream early returned due to # an inconsistent pipeline state. if not self._state.task_groups: return text = "" assert self._resolved_keys is not None elements = sorted(e for (e, k) in self._resolved_keys.items() if k != e._get_cache_key()) if elements: text += self.content_profile.fmt("Resolved key Summary\n", bold=True) text += self.show_pipeline(elements, self.context.log_element_format) text += "\n\n" if self._failure_messages: values = {} for element_name, message in sorted(self._failure_messages.items()): for group in self._state.task_groups.values(): # Exclude the failure messages if the job didn't ultimately fail # (e.g. succeeded on retry) if element_name in group.failed_tasks: values[element_name] = self._render(message) if values: text += self.content_profile.fmt("Failure Summary\n", bold=True) text += self._format_values(values, style_value=False) text += self.content_profile.fmt("Pipeline Summary\n", bold=True) values = {} values["Total"] = self.content_profile.fmt(str(len(stream.total_elements))) values["Session"] = self.content_profile.fmt(str(len(stream.session_elements))) processed_maxlen = 1 skipped_maxlen = 1 failed_maxlen = 1 for group in self._state.task_groups.values(): processed_maxlen = max(len(str(group.processed_tasks)), processed_maxlen) skipped_maxlen = max(len(str(group.skipped_tasks)), skipped_maxlen) failed_maxlen = max(len(str(len(group.failed_tasks))), failed_maxlen) for group in self._state.task_groups.values(): processed = str(group.processed_tasks) skipped = str(group.skipped_tasks) failed = str(len(group.failed_tasks)) processed_align = " " * (processed_maxlen - len(processed)) skipped_align = " " * (skipped_maxlen - len(skipped)) failed_align = " " * (failed_maxlen - len(failed)) status_text = ( self.content_profile.fmt("processed ") + self._success_profile.fmt(processed) + self.format_profile.fmt(", ") + processed_align ) status_text += ( self.content_profile.fmt("skipped ") + self.content_profile.fmt(skipped) + self.format_profile.fmt(", ") + skipped_align ) status_text += self.content_profile.fmt("failed ") + self._err_profile.fmt(failed) + " " + failed_align values["{} Queue".format(group.name)] = status_text text += self._format_values(values, style_value=False) click.echo(text, nl=False, err=True) if log_file: click.echo(text, file=log_file, color=False, nl=False) ################################################### # Widget Abstract Methods # ################################################### def render(self, message): # Track logfiles for later use element_name = message.element_name if message.message_type in ERROR_MESSAGES and element_name is not None: self._failure_messages[element_name] = message return self._render(message) ################################################### # Private Methods # ################################################### def _parse_logfile_format(self, format_string, content_profile, format_profile): logfile_tokens = [] while format_string: if format_string.startswith("%%"): logfile_tokens.append(FixedText(self.context, "%", content_profile, format_profile)) format_string = format_string[2:] continue m = re.search(r"^%\{([^\}]+)\}", format_string) if m is not None: variable = m.group(1) format_string = format_string[m.end(0) :] if variable not in self.logfile_variable_names: raise FormattingError("'{0}' is not a valid log variable name.".format(variable)) logfile_tokens.append(self.logfile_variable_names[variable]) else: m = re.search("^[^%]+", format_string) if m is not None: text = FixedText(self.context, m.group(0), content_profile, format_profile) format_string = format_string[m.end(0) :] logfile_tokens.append(text) else: # No idea what to do now raise FormattingError( "'{0}' could not be parsed into a valid logging format.".format(format_string) ) return logfile_tokens def _render(self, message): # Render the column widgets first text = "" for widget in self._columns: text += widget.render(message) text += "\n" extra_nl = False # Now add some custom things if message.detail: # Identify frontend messages, we never abbreviate these frontend_message = not message.element_name # Split and truncate message detail down to message_lines lines lines = message.detail.splitlines(True) n_lines = len(lines) abbrev = False if message.message_type not in ERROR_MESSAGES and not frontend_message and n_lines > self._message_lines: lines = lines[0 : self._message_lines] if self._message_lines > 0: abbrev = True else: lines[n_lines - 1] = lines[n_lines - 1].rstrip("\n") detail = self._indent + self._indent.join(lines) text += "\n" if message.message_type in ERROR_MESSAGES: text += self._err_profile.fmt(detail, bold=True) else: text += self._detail_profile.fmt(detail) if abbrev: text += self._indent + self.content_profile.fmt( "Message contains {} additional lines".format(n_lines - self._message_lines), dim=True ) text += "\n" extra_nl = True if message.scheduler and message.message_type == MessageType.FAIL: text += "\n" if self.context is not None and not self.context.log_verbose: text += self._indent + self._err_profile.fmt("Log file: ") text += self._indent + self._logfile_widget.render(message) + "\n" elif self._log_lines > 0: text += ( self._indent + self._err_profile.fmt("Printing the last {} lines from log file:".format(self._log_lines)) + "\n" ) text += self._indent + self._logfile_widget.render_abbrev(message, abbrev=False) + "\n" text += self._indent + self._err_profile.fmt("=" * 70) + "\n" log_content = self._read_last_lines(message.logfile) log_content = textwrap.indent(log_content, self._indent) text += self._detail_profile.fmt(log_content) text += "\n" text += self._indent + self._err_profile.fmt("=" * 70) + "\n" extra_nl = True if extra_nl: text += "\n" return text def _read_last_lines(self, logfile): with ExitStack() as stack: # mmap handles low-level memory details, allowing for # faster searches f = stack.enter_context(open(logfile, "r+", encoding="utf-8")) log = stack.enter_context(mmap(f.fileno(), os.path.getsize(f.name))) count = 0 end = log.size() - 1 while count < self._log_lines and end >= 0: location = log.rfind(b"\n", 0, end) count += 1 # If location is -1 (none found), this will print the # first character because of the later +1 end = location # end+1 is correct whether or not a newline was found at # that location. If end is -1 (seek before beginning of file) # then we get the first characther. If end is a newline position, # we discard it and only want to print the beginning of the next # line. lines = log[(end + 1) :].splitlines() return "\n".join([line.decode("utf-8") for line in lines]).rstrip() # _format_plugins() # # Formats the plugins loaded by a project # # Args: # element_plugins (dict): Dict of element plugin kind and display string tuples # source_plugins (dict): Dict of source plugin kind and display string tuples # # Returns: # (str): The formatted text # def _format_plugins(self, element_plugins, source_plugins): text = "" if element_plugins: text += self.format_profile.fmt("{}Element Plugins\n".format(self._indent)) text += self._format_values(element_plugins, style_key=True, indent=2) text += "\n" if source_plugins: text += self.format_profile.fmt("{}Source Plugins\n".format(self._indent)) text += self._format_values(source_plugins, style_key=True, indent=2) text += "\n" return text # _format_values() # # Formats an indented dictionary of titles / values, ensuring # the values are aligned. # # Args: # values (dict): A dictionary # style_key (bool): Whether to use the content profile for the keys # style_value (bool): Whether to use the content profile for the values # indent (number): Number of initial indentation levels # # Returns: # (str): The formatted values # def _format_values(self, values, *, style_key=False, style_value=True, indent=1): text = "" max_key_len = 0 for key, value in values.items(): max_key_len = max(len(key), max_key_len) for key, value in values.items(): key = str(key) text += self._indent * indent if style_key: text += self.content_profile.fmt(key) else: text += self.format_profile.fmt(key) text += self.format_profile.fmt(":") # Special case for values containing newlines if isinstance(value, str) and "\n" in value: text += "\n" text += textwrap.indent(value, self._indent * indent) continue # Alignment spacing text += " {}".format(" " * (max_key_len - len(key))) # Print the value if style_value: text += self.content_profile.fmt(str(value)) else: text += str(value) text += "\n" return text # _format_list() # # Formats an indented list of values, ensuring the values have bullets. # # Args: # values (list): A list of values to format as strings (or strings) # indent (number): Number of initial indentation levels (must be >= 1) # # Returns: # (str): The formatted values # def _format_list(self, values, *, indent=1): text = "" # We need at least 2 leading spaces inside the indentation in order # to print bullets. assert indent >= 1 # Indent string in case of multiline values indent_string = self._indent * indent # Prepare the indented bullet right away indent_bullet = self.format_profile.fmt(indent_string[:-2] + "* ") for value in values: value = str(value) # In case of newlines, replace each newline in the value with a trailing # indent and strip the result. value = value.replace("\n", "\n" + indent_string) value = value.strip() # Append the bullet text += indent_bullet # Append the value text += self.content_profile.fmt(value) text += "\n" return text # pretty_print_element_contents() # # Formats a dictionary of elements and their contents in a human readable way # # Args: # values (Dict[str, Directory]): A dictionary # style_value (bool): Whether to use the content profile for the values # list_long (bool): whether to display verbose information about files # # Returns: # (str): The formatted element contents # def pretty_print_element_contents(self, values, long_=False, style_value=True): text = "" for element_name, directory in values.items(): text += self.format_profile.fmt(" {}:".format(element_name)) rendered_files = [] for filename in directory.list_relative_paths(): filestat = directory.stat(filename) rendered_files.append(self._get_filestats(directory, filename, filestat, list_long=long_)) value_list = "\n\t" + "\n\t".join(rendered_files) if not rendered_files: message = "\n\tThis element has no associated artifacts" if style_value: text += self.content_profile.fmt(message) else: text += message elif style_value: text += self.content_profile.fmt(value_list) else: text += value_list text += "\n" return text # show_state_of_artifacts() # # Show the cached status of artifacts # # Example output: # # "cached foo.bst" <- If cached locally # "failed foo.bst" <- If cached locally as a failure # "available foo.bst" <- If available to download from a remote # "not cached foo.bst" <- If not cached/available remotely. # # Note that artifact names may also be displayed here. # # Args: # targets (list [Element]): Elements (or ArtifactElements) we wish to show the # cached status of # def show_state_of_artifacts(self, targets): report = "" p = Profile() for element in targets: # # Here we selectively show the element name or artifact name # depending on whether we were asked about an artifact or an element. # if isinstance(element, ArtifactElement): element_name = element.get_artifact_name() else: element_name = element._get_full_name() line = "%{state: >12} %{name}" line = p.fmt_subst(line, "name", element_name, fg="yellow") if element._cached_success(): line = p.fmt_subst(line, "state", "cached", fg="magenta") elif element._cached(): line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_remotely(): line = p.fmt_subst(line, "state", "available", fg="green") else: line = p.fmt_subst(line, "state", "not cached", fg="bright_red") report += line + "\n" return report # _get_filestats() # # Gets the necessary information from a dictionary # # Args: # directory (Directory): The base directory # filename (str): A filename inside the directory # filestat (FileStat): A FileStat for the filename # list_long (Bool): whether to display verbose information about artifacts # # Returns: # (str): The information about the element # def _get_filestats(self, directory, filename, filestat, list_long=False): if list_long: size = str(filestat.size) # Support files up to 99G, meaning maximum characters is 11 max_v_len = 11 if filestat.file_type == FileType.DIRECTORY: return ( "drwxr-xr-x dir {}".format(size) + "{} ".format(" " * (max_v_len - len(size))) + "{}".format(filename) ) elif filestat.file_type == FileType.SYMLINK: target = directory.readlink(filename) return ( "lrwxrwxrwx link {}".format(size) + "{} ".format(" " * (max_v_len - len(size))) + "{} -> {}".format(filename, target) ) elif filestat.executable: return ( "-rwxr-xr-x exe {}".format(size) + "{} ".format(" " * (max_v_len - len(size))) + "{}".format(filename) ) else: return ( "-rw-r--r-- reg {}".format(size) + "{} ".format(" " * (max_v_len - len(size))) + "{}".format(filename) ) return filename apache-buildstream-27ae392/src/buildstream/_includes.py000066400000000000000000000207711514607367700232520ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from . import _yaml from .node import MappingNode, ScalarNode, SequenceNode from ._variables import Variables from ._exceptions import LoadError from .exceptions import LoadErrorReason # Includes() # # This takes care of processing include directives "(@)". # # Args: # loader (Loader): The Loader object # copy_tree (bool): Whether to make a copy, of tree in # provenance. Should be true if intended to be # serialized. class Includes: def __init__(self, loader, *, copy_tree=False): self._loader = loader self._loaded = {} self._copy_tree = copy_tree # process() # # Process recursively include directives in a YAML node. # # Args: # node (dict): A YAML node # only_local (bool): Whether to ignore junction files # process_project_options (bool): Whether to process options from current project def process(self, node, *, only_local=False, process_project_options=True): self._process(node, only_local=only_local, process_project_options=process_project_options) # _process() # # Process recursively include directives in a YAML node. This # method is a recursively called on loaded nodes from files. # # Args: # node (dict): A YAML node # included (set): Fail for recursion if trying to load any files in this set # current_loader (Loader): Use alternative loader (for junction files) # only_local (bool): Whether to ignore junction files # process_project_options (bool): Whether to process options from current project def _process(self, node, *, included=None, current_loader=None, only_local=False, process_project_options=True): if current_loader is None: current_loader = self._loader if process_project_options: current_loader.project.options.process_node(node) self._process_node( node, included=included, only_local=only_local, current_loader=current_loader, process_project_options=process_project_options, ) # _process_node() # # Process recursively include directives in a YAML node. This # method is recursively called on all nodes. # # Args: # node (dict): A YAML node # included (set): Fail for recursion if trying to load any files in this set # current_loader (Loader): Use alternative loader (for junction files) # only_local (bool): Whether to ignore junction files # process_project_options (bool): Whether to process options from current project def _process_node( self, node, *, included=None, current_loader=None, only_local=False, process_project_options=True ): if included is None: included = set() includes_node = node.get_node("(@)", allowed_types=[ScalarNode, SequenceNode], allow_none=True) if includes_node: if type(includes_node) is ScalarNode: # pylint: disable=unidiomatic-typecheck includes = [includes_node] else: includes = includes_node del node["(@)"] for include in reversed(includes): if only_local and ":" in include.as_str(): continue include_node, file_path, sub_loader = self._include_file(include, current_loader) if file_path in included: include_provenance = includes_node.get_provenance() raise LoadError( "{}: trying to recursively include {}".format(include_provenance, file_path), LoadErrorReason.RECURSIVE_INCLUDE, ) # Because the included node will be modified, we need # to copy it so that we do not modify the toplevel # node of the provenance. include_node = include_node.clone() try: included.add(file_path) self._process( include_node, included=included, current_loader=sub_loader, only_local=only_local, process_project_options=process_project_options or current_loader != sub_loader, ) finally: included.remove(file_path) include_node._composite_under(node) for value in node.values(): self._process_value( value, included=included, current_loader=current_loader, only_local=only_local, process_project_options=process_project_options, ) # _include_file() # # Load include YAML file from with a loader. # # Args: # include (ScalarNode): file path relative to loader's project directory. # Can be prefixed with junctio name. # loader (Loader): Loader for the current project. def _include_file(self, include, loader): include_str = include.as_str() shortname = include_str if ":" in include_str: junction, include_str = include_str.rsplit(":", 1) current_loader = loader.get_loader(junction, include) current_loader.project.ensure_fully_loaded() else: current_loader = loader project = current_loader.project directory = project.directory file_path = os.path.join(directory, include_str) key = (current_loader, file_path) if key not in self._loaded: try: self._loaded[key] = _yaml.load( file_path, shortname=shortname, project=project, copy_tree=self._copy_tree ) except LoadError as e: raise LoadError("{}: {}".format(include.get_provenance(), e), e.reason, detail=e.detail) from e # If the include is from a subproject, we need to expand variables # in the context of the subproject's variables, the subproject is # guaranteed at this stage to be fully loaded. # if current_loader != loader: assert ( current_loader.project.base_variables is not None ), "{}: Attempted to include file from a subproject that isn't fully loaded".format( include.get_provenance() ) variables_node = current_loader.project.base_variables.clone() variables = Variables(variables_node) variables.expand(self._loaded[key]) return self._loaded[key], file_path, current_loader # _process_value() # # Select processing for value that could be a list or a dictionary. # # Args: # value: Value to process. Can be a list or a dictionary. # included (set): Fail for recursion if trying to load any files in this set # current_loader (Loader): Use alternative loader (for junction files) # only_local (bool): Whether to ignore junction files # process_project_options (bool): Whether to process options from current project def _process_value( self, value, *, included=None, current_loader=None, only_local=False, process_project_options=True ): value_type = type(value) if value_type is MappingNode: self._process_node( value, included=included, current_loader=current_loader, only_local=only_local, process_project_options=process_project_options, ) elif value_type is SequenceNode: for v in value: self._process_value( v, included=included, current_loader=current_loader, only_local=only_local, process_project_options=process_project_options, ) apache-buildstream-27ae392/src/buildstream/_loader/000077500000000000000000000000001514607367700223315ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_loader/__init__.py000066400000000000000000000014701514607367700244440ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .types import Symbol from .metasource import MetaSource from .loadelement import LoadElement, Dependency, DependencyType from .loadcontext import LoadContext from .loader import Loader apache-buildstream-27ae392/src/buildstream/_loader/listsort.c000066400000000000000000001155531514607367700243720ustar00rootroot00000000000000/* * Based on listobject.c from Python version 3.12.9. */ #include "Python.h" #include /* Reverse a slice of a list in place, from lo up to (exclusive) hi. */ static void reverse_slice(PyObject **lo, PyObject **hi) { assert(lo && hi); --hi; while (lo < hi) { PyObject *t = *lo; *lo = *hi; *hi = t; ++lo; --hi; } } /* Lots of code for an adaptive, stable, natural mergesort. There are many * pieces to this algorithm; read listsort.txt for overviews and details. */ /* A sortslice contains a pointer to an array of keys and a pointer to * an array of corresponding values. In other words, keys[i] * corresponds with values[i]. If values == NULL, then the keys are * also the values. * * Several convenience routines are provided here, so that keys and * values are always moved in sync. */ typedef struct { PyObject **keys; PyObject **values; } sortslice; Py_LOCAL_INLINE(void) sortslice_copy(sortslice *s1, Py_ssize_t i, sortslice *s2, Py_ssize_t j) { s1->keys[i] = s2->keys[j]; if (s1->values != NULL) s1->values[i] = s2->values[j]; } Py_LOCAL_INLINE(void) sortslice_copy_incr(sortslice *dst, sortslice *src) { *dst->keys++ = *src->keys++; if (dst->values != NULL) *dst->values++ = *src->values++; } Py_LOCAL_INLINE(void) sortslice_copy_decr(sortslice *dst, sortslice *src) { *dst->keys-- = *src->keys--; if (dst->values != NULL) *dst->values-- = *src->values--; } Py_LOCAL_INLINE(void) sortslice_memcpy(sortslice *s1, Py_ssize_t i, sortslice *s2, Py_ssize_t j, Py_ssize_t n) { memcpy(&s1->keys[i], &s2->keys[j], sizeof(PyObject *) * n); if (s1->values != NULL) memcpy(&s1->values[i], &s2->values[j], sizeof(PyObject *) * n); } Py_LOCAL_INLINE(void) sortslice_memmove(sortslice *s1, Py_ssize_t i, sortslice *s2, Py_ssize_t j, Py_ssize_t n) { memmove(&s1->keys[i], &s2->keys[j], sizeof(PyObject *) * n); if (s1->values != NULL) memmove(&s1->values[i], &s2->values[j], sizeof(PyObject *) * n); } Py_LOCAL_INLINE(void) sortslice_advance(sortslice *slice, Py_ssize_t n) { slice->keys += n; if (slice->values != NULL) slice->values += n; } /* Comparison function: ms->key_compare, which is set at run-time in * listsort_impl to optimize for various special cases. * Returns -1 on error, 1 if x < y, 0 if x >= y. */ #define ISLT(X, Y) (*(ms->key_compare))(X, Y, ms) /* Compare X to Y via "<". Goto "fail" if the comparison raises an error. Else "k" is set to true iff X. X and Y are PyObject*s. */ #define IFLT(X, Y) if ((k = ISLT(X, Y)) < 0) goto fail; \ if (k) /* The maximum number of entries in a MergeState's pending-runs stack. * For a list with n elements, this needs at most floor(log2(n)) + 1 entries * even if we didn't force runs to a minimal length. So the number of bits * in a Py_ssize_t is plenty large enough for all cases. */ #define MAX_MERGE_PENDING (SIZEOF_SIZE_T * 8) /* When we get into galloping mode, we stay there until both runs win less * often than MIN_GALLOP consecutive times. See listsort.txt for more info. */ #define MIN_GALLOP 7 /* Avoid malloc for small temp arrays. */ #define MERGESTATE_TEMP_SIZE 256 /* One MergeState exists on the stack per invocation of mergesort. It's just * a convenient way to pass state around among the helper functions. */ struct s_slice { sortslice base; Py_ssize_t len; /* length of run */ int power; /* node "level" for powersort merge strategy */ }; typedef struct s_MergeState MergeState; struct s_MergeState { /* This controls when we get *into* galloping mode. It's initialized * to MIN_GALLOP. merge_lo and merge_hi tend to nudge it higher for * random data, and lower for highly structured data. */ Py_ssize_t min_gallop; Py_ssize_t listlen; /* len(input_list) - read only */ PyObject **basekeys; /* base address of keys array - read only */ /* 'a' is temp storage to help with merges. It contains room for * alloced entries. */ sortslice a; /* may point to temparray below */ Py_ssize_t alloced; /* A stack of n pending runs yet to be merged. Run #i starts at * address base[i] and extends for len[i] elements. It's always * true (so long as the indices are in bounds) that * * pending[i].base + pending[i].len == pending[i+1].base * * so we could cut the storage for this, but it's a minor amount, * and keeping all the info explicit simplifies the code. */ int n; struct s_slice pending[MAX_MERGE_PENDING]; /* 'a' points to this when possible, rather than muck with malloc. */ PyObject *temparray[MERGESTATE_TEMP_SIZE]; /* This is the function we will use to compare two keys, * even when none of our special cases apply and we have to use * safe_object_compare. */ int (*key_compare)(PyObject *, PyObject *, MergeState *); /* This function is used by unsafe_object_compare to optimize comparisons * when we know our list is type-homogeneous but we can't assume anything else. * In the pre-sort check it is set equal to Py_TYPE(key)->tp_richcompare */ PyObject *(*key_richcompare)(PyObject *, PyObject *, int); }; /* binarysort is the best method for sorting small arrays: it does few compares, but can do data movement quadratic in the number of elements. [lo, hi) is a contiguous slice of a list, and is sorted via binary insertion. This sort is stable. On entry, must have lo <= start <= hi, and that [lo, start) is already sorted (pass start == lo if you don't know!). If islt() complains return -1, else 0. Even in case of error, the output slice will be some permutation of the input (nothing is lost or duplicated). */ static int binarysort(MergeState *ms, sortslice lo, PyObject **hi, PyObject **start) { Py_ssize_t k; PyObject **l, **p, **r; PyObject *pivot; assert(lo.keys <= start && start <= hi); /* assert [lo, start) is sorted */ if (lo.keys == start) ++start; for (; start < hi; ++start) { /* set l to where *start belongs */ l = lo.keys; r = start; pivot = *r; /* Invariants: * pivot >= all in [lo, l). * pivot < all in [r, start). * The second is vacuously true at the start. */ assert(l < r); do { p = l + ((r - l) >> 1); IFLT(pivot, *p) r = p; else l = p+1; } while (l < r); assert(l == r); /* The invariants still hold, so pivot >= all in [lo, l) and pivot < all in [l, start), so pivot belongs at l. Note that if there are elements equal to pivot, l points to the first slot after them -- that's why this sort is stable. Slide over to make room. Caution: using memmove is much slower under MSVC 5; we're not usually moving many slots. */ for (p = start; p > l; --p) *p = *(p-1); *l = pivot; if (lo.values != NULL) { Py_ssize_t offset = lo.values - lo.keys; p = start + offset; pivot = *p; l += offset; for (p = start + offset; p > l; --p) *p = *(p-1); *l = pivot; } } return 0; fail: return -1; } /* Return the length of the run beginning at lo, in the slice [lo, hi). lo < hi is required on entry. "A run" is the longest ascending sequence, with lo[0] <= lo[1] <= lo[2] <= ... or the longest descending sequence, with lo[0] > lo[1] > lo[2] > ... Boolean *descending is set to 0 in the former case, or to 1 in the latter. For its intended use in a stable mergesort, the strictness of the defn of "descending" is needed so that the caller can safely reverse a descending sequence without violating stability (strict > ensures there are no equal elements to get out of order). Returns -1 in case of error. */ static Py_ssize_t count_run(MergeState *ms, PyObject **lo, PyObject **hi, int *descending) { Py_ssize_t k; Py_ssize_t n; assert(lo < hi); *descending = 0; ++lo; if (lo == hi) return 1; n = 2; IFLT(*lo, *(lo-1)) { *descending = 1; for (lo = lo+1; lo < hi; ++lo, ++n) { IFLT(*lo, *(lo-1)) ; else break; } } else { for (lo = lo+1; lo < hi; ++lo, ++n) { IFLT(*lo, *(lo-1)) break; } } return n; fail: return -1; } /* Locate the proper position of key in a sorted vector; if the vector contains an element equal to key, return the position immediately to the left of the leftmost equal element. [gallop_right() does the same except returns the position to the right of the rightmost equal element (if any).] "a" is a sorted vector with n elements, starting at a[0]. n must be > 0. "hint" is an index at which to begin the search, 0 <= hint < n. The closer hint is to the final result, the faster this runs. The return value is the int k in 0..n such that a[k-1] < key <= a[k] pretending that *(a-1) is minus infinity and a[n] is plus infinity. IOW, key belongs at index k; or, IOW, the first k elements of a should precede key, and the last n-k should follow key. Returns -1 on error. See listsort.txt for info on the method. */ static Py_ssize_t gallop_left(MergeState *ms, PyObject *key, PyObject **a, Py_ssize_t n, Py_ssize_t hint) { Py_ssize_t ofs; Py_ssize_t lastofs; Py_ssize_t k; assert(key && a && n > 0 && hint >= 0 && hint < n); a += hint; lastofs = 0; ofs = 1; IFLT(*a, key) { /* a[hint] < key -- gallop right, until * a[hint + lastofs] < key <= a[hint + ofs] */ const Py_ssize_t maxofs = n - hint; /* &a[n-1] is highest */ while (ofs < maxofs) { IFLT(a[ofs], key) { lastofs = ofs; assert(ofs <= (PY_SSIZE_T_MAX - 1) / 2); ofs = (ofs << 1) + 1; } else /* key <= a[hint + ofs] */ break; } if (ofs > maxofs) ofs = maxofs; /* Translate back to offsets relative to &a[0]. */ lastofs += hint; ofs += hint; } else { /* key <= a[hint] -- gallop left, until * a[hint - ofs] < key <= a[hint - lastofs] */ const Py_ssize_t maxofs = hint + 1; /* &a[0] is lowest */ while (ofs < maxofs) { IFLT(*(a-ofs), key) break; /* key <= a[hint - ofs] */ lastofs = ofs; assert(ofs <= (PY_SSIZE_T_MAX - 1) / 2); ofs = (ofs << 1) + 1; } if (ofs > maxofs) ofs = maxofs; /* Translate back to positive offsets relative to &a[0]. */ k = lastofs; lastofs = hint - ofs; ofs = hint - k; } a -= hint; assert(-1 <= lastofs && lastofs < ofs && ofs <= n); /* Now a[lastofs] < key <= a[ofs], so key belongs somewhere to the * right of lastofs but no farther right than ofs. Do a binary * search, with invariant a[lastofs-1] < key <= a[ofs]. */ ++lastofs; while (lastofs < ofs) { Py_ssize_t m = lastofs + ((ofs - lastofs) >> 1); IFLT(a[m], key) lastofs = m+1; /* a[m] < key */ else ofs = m; /* key <= a[m] */ } assert(lastofs == ofs); /* so a[ofs-1] < key <= a[ofs] */ return ofs; fail: return -1; } /* Exactly like gallop_left(), except that if key already exists in a[0:n], finds the position immediately to the right of the rightmost equal value. The return value is the int k in 0..n such that a[k-1] <= key < a[k] or -1 if error. The code duplication is massive, but this is enough different given that we're sticking to "<" comparisons that it's much harder to follow if written as one routine with yet another "left or right?" flag. */ static Py_ssize_t gallop_right(MergeState *ms, PyObject *key, PyObject **a, Py_ssize_t n, Py_ssize_t hint) { Py_ssize_t ofs; Py_ssize_t lastofs; Py_ssize_t k; assert(key && a && n > 0 && hint >= 0 && hint < n); a += hint; lastofs = 0; ofs = 1; IFLT(key, *a) { /* key < a[hint] -- gallop left, until * a[hint - ofs] <= key < a[hint - lastofs] */ const Py_ssize_t maxofs = hint + 1; /* &a[0] is lowest */ while (ofs < maxofs) { IFLT(key, *(a-ofs)) { lastofs = ofs; assert(ofs <= (PY_SSIZE_T_MAX - 1) / 2); ofs = (ofs << 1) + 1; } else /* a[hint - ofs] <= key */ break; } if (ofs > maxofs) ofs = maxofs; /* Translate back to positive offsets relative to &a[0]. */ k = lastofs; lastofs = hint - ofs; ofs = hint - k; } else { /* a[hint] <= key -- gallop right, until * a[hint + lastofs] <= key < a[hint + ofs] */ const Py_ssize_t maxofs = n - hint; /* &a[n-1] is highest */ while (ofs < maxofs) { IFLT(key, a[ofs]) break; /* a[hint + ofs] <= key */ lastofs = ofs; assert(ofs <= (PY_SSIZE_T_MAX - 1) / 2); ofs = (ofs << 1) + 1; } if (ofs > maxofs) ofs = maxofs; /* Translate back to offsets relative to &a[0]. */ lastofs += hint; ofs += hint; } a -= hint; assert(-1 <= lastofs && lastofs < ofs && ofs <= n); /* Now a[lastofs] <= key < a[ofs], so key belongs somewhere to the * right of lastofs but no farther right than ofs. Do a binary * search, with invariant a[lastofs-1] <= key < a[ofs]. */ ++lastofs; while (lastofs < ofs) { Py_ssize_t m = lastofs + ((ofs - lastofs) >> 1); IFLT(key, a[m]) ofs = m; /* key < a[m] */ else lastofs = m+1; /* a[m] <= key */ } assert(lastofs == ofs); /* so a[ofs-1] <= key < a[ofs] */ return ofs; fail: return -1; } /* Conceptually a MergeState's constructor. */ static void merge_init(MergeState *ms, Py_ssize_t list_size, int has_keyfunc, sortslice *lo) { assert(ms != NULL); if (has_keyfunc) { /* The temporary space for merging will need at most half the list * size rounded up. Use the minimum possible space so we can use the * rest of temparray for other things. In particular, if there is * enough extra space, listsort() will use it to store the keys. */ ms->alloced = (list_size + 1) / 2; /* ms->alloced describes how many keys will be stored at ms->temparray, but we also need to store the values. Hence, ms->alloced is capped at half of MERGESTATE_TEMP_SIZE. */ if (MERGESTATE_TEMP_SIZE / 2 < ms->alloced) ms->alloced = MERGESTATE_TEMP_SIZE / 2; ms->a.values = &ms->temparray[ms->alloced]; } else { ms->alloced = MERGESTATE_TEMP_SIZE; ms->a.values = NULL; } ms->a.keys = ms->temparray; ms->n = 0; ms->min_gallop = MIN_GALLOP; ms->listlen = list_size; ms->basekeys = lo->keys; } /* Free all the temp memory owned by the MergeState. This must be called * when you're done with a MergeState, and may be called before then if * you want to free the temp memory early. */ static void merge_freemem(MergeState *ms) { assert(ms != NULL); if (ms->a.keys != ms->temparray) { PyMem_Free(ms->a.keys); ms->a.keys = NULL; } } /* Ensure enough temp memory for 'need' array slots is available. * Returns 0 on success and -1 if the memory can't be gotten. */ static int merge_getmem(MergeState *ms, Py_ssize_t need) { int multiplier; assert(ms != NULL); if (need <= ms->alloced) return 0; multiplier = ms->a.values != NULL ? 2 : 1; /* Don't realloc! That can cost cycles to copy the old data, but * we don't care what's in the block. */ merge_freemem(ms); if ((size_t)need > PY_SSIZE_T_MAX / sizeof(PyObject *) / multiplier) { PyErr_NoMemory(); return -1; } ms->a.keys = (PyObject **)PyMem_Malloc(multiplier * need * sizeof(PyObject *)); if (ms->a.keys != NULL) { ms->alloced = need; if (ms->a.values != NULL) ms->a.values = &ms->a.keys[need]; return 0; } PyErr_NoMemory(); return -1; } #define MERGE_GETMEM(MS, NEED) ((NEED) <= (MS)->alloced ? 0 : \ merge_getmem(MS, NEED)) /* Merge the na elements starting at ssa with the nb elements starting at * ssb.keys = ssa.keys + na in a stable way, in-place. na and nb must be > 0. * Must also have that ssa.keys[na-1] belongs at the end of the merge, and * should have na <= nb. See listsort.txt for more info. Return 0 if * successful, -1 if error. */ static Py_ssize_t merge_lo(MergeState *ms, sortslice ssa, Py_ssize_t na, sortslice ssb, Py_ssize_t nb) { Py_ssize_t k; sortslice dest; int result = -1; /* guilty until proved innocent */ Py_ssize_t min_gallop; assert(ms && ssa.keys && ssb.keys && na > 0 && nb > 0); assert(ssa.keys + na == ssb.keys); if (MERGE_GETMEM(ms, na) < 0) return -1; sortslice_memcpy(&ms->a, 0, &ssa, 0, na); dest = ssa; ssa = ms->a; sortslice_copy_incr(&dest, &ssb); --nb; if (nb == 0) goto Succeed; if (na == 1) goto CopyB; min_gallop = ms->min_gallop; for (;;) { Py_ssize_t acount = 0; /* # of times A won in a row */ Py_ssize_t bcount = 0; /* # of times B won in a row */ /* Do the straightforward thing until (if ever) one run * appears to win consistently. */ for (;;) { assert(na > 1 && nb > 0); k = ISLT(ssb.keys[0], ssa.keys[0]); if (k) { if (k < 0) goto Fail; sortslice_copy_incr(&dest, &ssb); ++bcount; acount = 0; --nb; if (nb == 0) goto Succeed; if (bcount >= min_gallop) break; } else { sortslice_copy_incr(&dest, &ssa); ++acount; bcount = 0; --na; if (na == 1) goto CopyB; if (acount >= min_gallop) break; } } /* One run is winning so consistently that galloping may * be a huge win. So try that, and continue galloping until * (if ever) neither run appears to be winning consistently * anymore. */ ++min_gallop; do { assert(na > 1 && nb > 0); min_gallop -= min_gallop > 1; ms->min_gallop = min_gallop; k = gallop_right(ms, ssb.keys[0], ssa.keys, na, 0); acount = k; if (k) { if (k < 0) goto Fail; sortslice_memcpy(&dest, 0, &ssa, 0, k); sortslice_advance(&dest, k); sortslice_advance(&ssa, k); na -= k; if (na == 1) goto CopyB; /* na==0 is impossible now if the comparison * function is consistent, but we can't assume * that it is. */ if (na == 0) goto Succeed; } sortslice_copy_incr(&dest, &ssb); --nb; if (nb == 0) goto Succeed; k = gallop_left(ms, ssa.keys[0], ssb.keys, nb, 0); bcount = k; if (k) { if (k < 0) goto Fail; sortslice_memmove(&dest, 0, &ssb, 0, k); sortslice_advance(&dest, k); sortslice_advance(&ssb, k); nb -= k; if (nb == 0) goto Succeed; } sortslice_copy_incr(&dest, &ssa); --na; if (na == 1) goto CopyB; } while (acount >= MIN_GALLOP || bcount >= MIN_GALLOP); ++min_gallop; /* penalize it for leaving galloping mode */ ms->min_gallop = min_gallop; } Succeed: result = 0; Fail: if (na) sortslice_memcpy(&dest, 0, &ssa, 0, na); return result; CopyB: assert(na == 1 && nb > 0); /* The last element of ssa belongs at the end of the merge. */ sortslice_memmove(&dest, 0, &ssb, 0, nb); sortslice_copy(&dest, nb, &ssa, 0); return 0; } /* Merge the na elements starting at pa with the nb elements starting at * ssb.keys = ssa.keys + na in a stable way, in-place. na and nb must be > 0. * Must also have that ssa.keys[na-1] belongs at the end of the merge, and * should have na >= nb. See listsort.txt for more info. Return 0 if * successful, -1 if error. */ static Py_ssize_t merge_hi(MergeState *ms, sortslice ssa, Py_ssize_t na, sortslice ssb, Py_ssize_t nb) { Py_ssize_t k; sortslice dest, basea, baseb; int result = -1; /* guilty until proved innocent */ Py_ssize_t min_gallop; assert(ms && ssa.keys && ssb.keys && na > 0 && nb > 0); assert(ssa.keys + na == ssb.keys); if (MERGE_GETMEM(ms, nb) < 0) return -1; dest = ssb; sortslice_advance(&dest, nb-1); sortslice_memcpy(&ms->a, 0, &ssb, 0, nb); basea = ssa; baseb = ms->a; ssb.keys = ms->a.keys + nb - 1; if (ssb.values != NULL) ssb.values = ms->a.values + nb - 1; sortslice_advance(&ssa, na - 1); sortslice_copy_decr(&dest, &ssa); --na; if (na == 0) goto Succeed; if (nb == 1) goto CopyA; min_gallop = ms->min_gallop; for (;;) { Py_ssize_t acount = 0; /* # of times A won in a row */ Py_ssize_t bcount = 0; /* # of times B won in a row */ /* Do the straightforward thing until (if ever) one run * appears to win consistently. */ for (;;) { assert(na > 0 && nb > 1); k = ISLT(ssb.keys[0], ssa.keys[0]); if (k) { if (k < 0) goto Fail; sortslice_copy_decr(&dest, &ssa); ++acount; bcount = 0; --na; if (na == 0) goto Succeed; if (acount >= min_gallop) break; } else { sortslice_copy_decr(&dest, &ssb); ++bcount; acount = 0; --nb; if (nb == 1) goto CopyA; if (bcount >= min_gallop) break; } } /* One run is winning so consistently that galloping may * be a huge win. So try that, and continue galloping until * (if ever) neither run appears to be winning consistently * anymore. */ ++min_gallop; do { assert(na > 0 && nb > 1); min_gallop -= min_gallop > 1; ms->min_gallop = min_gallop; k = gallop_right(ms, ssb.keys[0], basea.keys, na, na-1); if (k < 0) goto Fail; k = na - k; acount = k; if (k) { sortslice_advance(&dest, -k); sortslice_advance(&ssa, -k); sortslice_memmove(&dest, 1, &ssa, 1, k); na -= k; if (na == 0) goto Succeed; } sortslice_copy_decr(&dest, &ssb); --nb; if (nb == 1) goto CopyA; k = gallop_left(ms, ssa.keys[0], baseb.keys, nb, nb-1); if (k < 0) goto Fail; k = nb - k; bcount = k; if (k) { sortslice_advance(&dest, -k); sortslice_advance(&ssb, -k); sortslice_memcpy(&dest, 1, &ssb, 1, k); nb -= k; if (nb == 1) goto CopyA; /* nb==0 is impossible now if the comparison * function is consistent, but we can't assume * that it is. */ if (nb == 0) goto Succeed; } sortslice_copy_decr(&dest, &ssa); --na; if (na == 0) goto Succeed; } while (acount >= MIN_GALLOP || bcount >= MIN_GALLOP); ++min_gallop; /* penalize it for leaving galloping mode */ ms->min_gallop = min_gallop; } Succeed: result = 0; Fail: if (nb) sortslice_memcpy(&dest, -(nb-1), &baseb, 0, nb); return result; CopyA: assert(nb == 1 && na > 0); /* The first element of ssb belongs at the front of the merge. */ sortslice_memmove(&dest, 1-na, &ssa, 1-na, na); sortslice_advance(&dest, -na); sortslice_advance(&ssa, -na); sortslice_copy(&dest, 0, &ssb, 0); return 0; } /* Merge the two runs at stack indices i and i+1. * Returns 0 on success, -1 on error. */ static Py_ssize_t merge_at(MergeState *ms, Py_ssize_t i) { sortslice ssa, ssb; Py_ssize_t na, nb; Py_ssize_t k; assert(ms != NULL); assert(ms->n >= 2); assert(i >= 0); assert(i == ms->n - 2 || i == ms->n - 3); ssa = ms->pending[i].base; na = ms->pending[i].len; ssb = ms->pending[i+1].base; nb = ms->pending[i+1].len; assert(na > 0 && nb > 0); assert(ssa.keys + na == ssb.keys); /* Record the length of the combined runs; if i is the 3rd-last * run now, also slide over the last run (which isn't involved * in this merge). The current run i+1 goes away in any case. */ ms->pending[i].len = na + nb; if (i == ms->n - 3) ms->pending[i+1] = ms->pending[i+2]; --ms->n; /* Where does b start in a? Elements in a before that can be * ignored (already in place). */ k = gallop_right(ms, *ssb.keys, ssa.keys, na, 0); if (k < 0) return -1; sortslice_advance(&ssa, k); na -= k; if (na == 0) return 0; /* Where does a end in b? Elements in b after that can be * ignored (already in place). */ nb = gallop_left(ms, ssa.keys[na-1], ssb.keys, nb, nb-1); if (nb <= 0) return nb; /* Merge what remains of the runs, using a temp array with * min(na, nb) elements. */ if (na <= nb) return merge_lo(ms, ssa, na, ssb, nb); else return merge_hi(ms, ssa, na, ssb, nb); } /* Two adjacent runs begin at index s1. The first run has length n1, and * the second run (starting at index s1+n1) has length n2. The list has total * length n. * Compute the "power" of the first run. See listsort.txt for details. */ static int powerloop(Py_ssize_t s1, Py_ssize_t n1, Py_ssize_t n2, Py_ssize_t n) { int result = 0; assert(s1 >= 0); assert(n1 > 0 && n2 > 0); assert(s1 + n1 + n2 <= n); /* midpoints a and b: * a = s1 + n1/2 * b = s1 + n1 + n2/2 = a + (n1 + n2)/2 * * Those may not be integers, though, because of the "/2". So we work with * 2*a and 2*b instead, which are necessarily integers. It makes no * difference to the outcome, since the bits in the expansion of (2*i)/n * are merely shifted one position from those of i/n. */ Py_ssize_t a = 2 * s1 + n1; /* 2*a */ Py_ssize_t b = a + n1 + n2; /* 2*b */ /* Emulate a/n and b/n one bit a time, until bits differ. */ for (;;) { ++result; if (a >= n) { /* both quotient bits are 1 */ assert(b >= a); a -= n; b -= n; } else if (b >= n) { /* a/n bit is 0, b/n bit is 1 */ break; } /* else both quotient bits are 0 */ assert(a < b && b < n); a <<= 1; b <<= 1; } return result; } /* The next run has been identified, of length n2. * If there's already a run on the stack, apply the "powersort" merge strategy: * compute the topmost run's "power" (depth in a conceptual binary merge tree) * and merge adjacent runs on the stack with greater power. See listsort.txt * for more info. * * It's the caller's responsibility to push the new run on the stack when this * returns. * * Returns 0 on success, -1 on error. */ static int found_new_run(MergeState *ms, Py_ssize_t n2) { assert(ms); if (ms->n) { assert(ms->n > 0); struct s_slice *p = ms->pending; Py_ssize_t s1 = p[ms->n - 1].base.keys - ms->basekeys; /* start index */ Py_ssize_t n1 = p[ms->n - 1].len; int power = powerloop(s1, n1, n2, ms->listlen); while (ms->n > 1 && p[ms->n - 2].power > power) { if (merge_at(ms, ms->n - 2) < 0) return -1; } assert(ms->n < 2 || p[ms->n - 2].power < power); p[ms->n - 1].power = power; } return 0; } /* Regardless of invariants, merge all runs on the stack until only one * remains. This is used at the end of the mergesort. * * Returns 0 on success, -1 on error. */ static int merge_force_collapse(MergeState *ms) { struct s_slice *p = ms->pending; assert(ms); while (ms->n > 1) { Py_ssize_t n = ms->n - 2; if (n > 0 && p[n-1].len < p[n+1].len) --n; if (merge_at(ms, n) < 0) return -1; } return 0; } /* Compute a good value for the minimum run length; natural runs shorter * than this are boosted artificially via binary insertion. * * If n < 64, return n (it's too small to bother with fancy stuff). * Else if n is an exact power of 2, return 32. * Else return an int k, 32 <= k <= 64, such that n/k is close to, but * strictly less than, an exact power of 2. * * See listsort.txt for more info. */ static Py_ssize_t merge_compute_minrun(Py_ssize_t n) { Py_ssize_t r = 0; /* becomes 1 if any 1 bits are shifted off */ assert(n >= 0); while (n >= 64) { r |= n & 1; n >>= 1; } return n + r; } static void reverse_sortslice(sortslice *s, Py_ssize_t n) { reverse_slice(s->keys, &s->keys[n]); if (s->values != NULL) reverse_slice(s->values, &s->values[n]); } /* Here we define custom comparison functions to optimize for the cases one commonly * encounters in practice: homogeneous lists, often of one of the basic types. */ /* This struct holds the comparison function and helper functions * selected in the pre-sort check. */ /* These are the special case compare functions. * ms->key_compare will always point to one of these: */ /* Heterogeneous compare: default, always safe to fall back on. */ static int safe_object_compare(PyObject *v, PyObject *w, MergeState *ms) { /* No assumptions necessary! */ return PyObject_RichCompareBool(v, w, Py_LT); } /* Homogeneous compare: safe for any two comparable objects of the same type. * (ms->key_richcompare is set to ob_type->tp_richcompare in the * pre-sort check.) */ static int unsafe_object_compare(PyObject *v, PyObject *w, MergeState *ms) { PyObject *res_obj; int res; /* No assumptions, because we check first: */ if (Py_TYPE(v)->tp_richcompare != ms->key_richcompare) return PyObject_RichCompareBool(v, w, Py_LT); assert(ms->key_richcompare != NULL); res_obj = (*(ms->key_richcompare))(v, w, Py_LT); if (res_obj == Py_NotImplemented) { Py_DECREF(res_obj); return PyObject_RichCompareBool(v, w, Py_LT); } if (res_obj == NULL) return -1; if (PyBool_Check(res_obj)) { res = (res_obj == Py_True); } else { res = PyObject_IsTrue(res_obj); } Py_DECREF(res_obj); /* Note that we can't assert * res == PyObject_RichCompareBool(v, w, Py_LT); * because of evil compare functions like this: * lambda a, b: int(random.random() * 3) - 1) * (which is actually in test_sort.py) */ return res; } /* An adaptive, stable, natural mergesort. See listsort.txt. * Returns Py_None on success, NULL on error. Even in case of error, the * list will be some permutation of its input state (nothing is lost or * duplicated). */ /*[clinic input] list.sort * key as keyfunc: object = None Sort the list in ascending order and return None. The sort is in-place (i.e. the list itself is modified) and stable (i.e. the order of two equal elements is maintained). If a key function is given, apply it once to each list item and sort them, ascending or descending, according to their function values. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc) /*[clinic end generated code: output=57b9f9c5e23fbe42 input=a74c4cd3ec6b5c08]*/ { MergeState ms; Py_ssize_t nremaining; Py_ssize_t minrun; sortslice lo; Py_ssize_t saved_ob_size, saved_allocated; PyObject **saved_ob_item; PyObject **final_ob_item; PyObject *result = NULL; /* guilty until proved innocent */ Py_ssize_t i; PyObject **keys; assert(self != NULL); assert(PyList_Check(self)); if (keyfunc == Py_None) keyfunc = NULL; /* The list is temporarily made empty, so that mutations performed * by comparison functions can't affect the slice of memory we're * sorting (allowing mutations during sorting is a core-dump * factory, since ob_item may change). */ saved_ob_size = Py_SIZE(self); saved_ob_item = self->ob_item; saved_allocated = self->allocated; Py_SET_SIZE(self, 0); self->ob_item = NULL; self->allocated = -1; /* any operation will reset it to >= 0 */ if (keyfunc == NULL) { keys = NULL; lo.keys = saved_ob_item; lo.values = NULL; } else { if (saved_ob_size < MERGESTATE_TEMP_SIZE/2) /* Leverage stack space we allocated but won't otherwise use */ keys = &ms.temparray[saved_ob_size+1]; else { keys = PyMem_Malloc(sizeof(PyObject *) * saved_ob_size); if (keys == NULL) { PyErr_NoMemory(); goto keyfunc_fail; } } for (i = 0; i < saved_ob_size ; i++) { keys[i] = PyObject_CallOneArg(keyfunc, saved_ob_item[i]); if (keys[i] == NULL) { for (i=i-1 ; i>=0 ; i--) Py_DECREF(keys[i]); if (saved_ob_size >= MERGESTATE_TEMP_SIZE/2) PyMem_Free(keys); goto keyfunc_fail; } } lo.keys = keys; lo.values = saved_ob_item; } /* The pre-sort check: here's where we decide which compare function to use. * How much optimization is safe? We test for homogeneity with respect to * several properties that are expensive to check at compare-time, and * set ms appropriately. */ if (saved_ob_size > 1) { /* Assume the first element is representative of the whole list. */ PyTypeObject* key_type = Py_TYPE(lo.keys[0]); int keys_are_all_same_type = 1; /* Prove that assumption by checking every key. */ for (i=0; i < saved_ob_size; i++) { PyObject *key = lo.keys[i]; if (!Py_IS_TYPE(key, key_type)) { keys_are_all_same_type = 0; break; } } /* Choose the best compare, given what we now know about the keys. */ if (keys_are_all_same_type && (ms.key_richcompare = key_type->tp_richcompare) != NULL) { ms.key_compare = unsafe_object_compare; } else { ms.key_compare = safe_object_compare; } } /* End of pre-sort check: ms is now set properly! */ merge_init(&ms, saved_ob_size, keys != NULL, &lo); nremaining = saved_ob_size; if (nremaining < 2) goto succeed; /* March over the array once, left to right, finding natural runs, * and extending short natural runs to minrun elements. */ minrun = merge_compute_minrun(nremaining); do { int descending; Py_ssize_t n; /* Identify next run. */ n = count_run(&ms, lo.keys, lo.keys + nremaining, &descending); if (n < 0) goto fail; if (descending) reverse_sortslice(&lo, n); /* If short, extend to min(minrun, nremaining). */ if (n < minrun) { const Py_ssize_t force = nremaining <= minrun ? nremaining : minrun; if (binarysort(&ms, lo, lo.keys + force, lo.keys + n) < 0) goto fail; n = force; } /* Maybe merge pending runs. */ assert(ms.n == 0 || ms.pending[ms.n -1].base.keys + ms.pending[ms.n-1].len == lo.keys); if (found_new_run(&ms, n) < 0) goto fail; /* Push new run on stack. */ assert(ms.n < MAX_MERGE_PENDING); ms.pending[ms.n].base = lo; ms.pending[ms.n].len = n; ++ms.n; /* Advance to find next run. */ sortslice_advance(&lo, n); nremaining -= n; } while (nremaining); if (merge_force_collapse(&ms) < 0) goto fail; assert(ms.n == 1); assert(keys == NULL ? ms.pending[0].base.keys == saved_ob_item : ms.pending[0].base.keys == &keys[0]); assert(ms.pending[0].len == saved_ob_size); lo = ms.pending[0].base; succeed: result = Py_None; fail: if (keys != NULL) { for (i = 0; i < saved_ob_size; i++) Py_DECREF(keys[i]); if (saved_ob_size >= MERGESTATE_TEMP_SIZE/2) PyMem_Free(keys); } if (self->allocated != -1 && result != NULL) { /* The user mucked with the list during the sort, * and we don't already have another error to report. */ PyErr_SetString(PyExc_ValueError, "list modified during sort"); result = NULL; } merge_freemem(&ms); keyfunc_fail: final_ob_item = self->ob_item; i = Py_SIZE(self); Py_SET_SIZE(self, saved_ob_size); self->ob_item = saved_ob_item; self->allocated = saved_allocated; if (final_ob_item != NULL) { /* we cannot use _list_clear() for this because it does not guarantee that the list is really empty when it returns */ while (--i >= 0) { Py_XDECREF(final_ob_item[i]); } PyMem_Free(final_ob_item); } Py_XINCREF(result); return result; } #undef IFLT #undef ISLT static int _list_sort(PyObject *v, PyObject *keyfunc) { if (v == NULL || !PyList_Check(v)) { PyErr_BadInternalCall(); return -1; } v = list_sort_impl((PyListObject *)v, keyfunc); if (v == NULL) return -1; Py_DECREF(v); return 0; } apache-buildstream-27ae392/src/buildstream/_loader/loadcontext.py000066400000000000000000000205051514607367700252310ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .._exceptions import LoadError from ..exceptions import LoadErrorReason from ..types import _ProjectInformation # ProjectLoaders() # # An object representing all of the loaders for a given project. # class ProjectLoaders: def __init__(self, project_name): # The project name self._name = project_name # A list of all loaded loaders for this project self._collect = [] # register_loader() # # Register a Loader for this project # # Args: # loader (Loader): The loader to register # def register_loader(self, loader): assert loader.project.name == self._name self._collect.append(loader) # assert_loaders(): # # Asserts the validity of loaders for this project # # Raises: # (LoadError): In case there is a CONFLICTING_JUNCTION error # def assert_loaders(self): duplicates = {} internal = {} primary = [] for loader in self._collect: duplicating, internalizing = self._search_project_relationships(loader) if duplicating: duplicates[loader] = duplicating if internalizing: internal[loader] = internalizing if not (duplicating or internalizing): primary.append(loader) if len(primary) > 1: self._raise_conflict(duplicates, internal) elif primary and duplicates: self._raise_conflict(duplicates, internal) # loaded_projects() # # A generator which yeilds all of the instances # of this loaded project. # # Yields: # (_ProjectInformation): A descriptive project information object # def loaded_projects(self): for loader in self._collect: duplicating, internalizing = self._search_project_relationships(loader) yield _ProjectInformation( loader.project, loader.provenance_node, [str(l) for l in duplicating], [str(l) for l in internalizing] ) # _search_project_relationships() # # Searches this loader's ancestry for projects which mark this # loader as internal or duplicate # # Args: # loader (Loader): The loader to search for duplicate markers of # # Returns: # (list): A list of Loader objects who's project has marked # this junction as a duplicate # (list): A list of Loader objects who's project has marked # this junction as internal # def _search_project_relationships(self, loader): duplicates = [] internal = [] for parent in loader.ancestors(): if parent.project.junction_is_duplicated(self._name, loader): duplicates.append(parent) if parent.project.junction_is_internal(loader): internal.append(parent) return duplicates, internal # _raise_conflict() # # Raises the LoadError indicating there was a conflict, this # will list all of the instances in which the project has # been loaded as the LoadError detail string # # Args: # duplicates (dict): A table of duplicating Loaders, indexed # by duplicated Loader # internals (dict): A table of Loaders which mark a loader as internal, # indexed by internal Loader # # Raises: # (LoadError): In case there is a CONFLICTING_JUNCTION error # def _raise_conflict(self, duplicates, internals): explanation = ( "Internal projects do not cause any conflicts. Conflicts can also be avoided\n" + "by marking every instance of the project as a duplicate." ) lines = [self._loader_description(loader, duplicates, internals) for loader in self._collect] detail = "{}\n{}".format("\n".join(lines), explanation) raise LoadError( "Project '{}' was loaded in multiple contexts".format(self._name), LoadErrorReason.CONFLICTING_JUNCTION, detail=detail, ) # _loader_description() # # Args: # loader (Loader): The loader to describe # duplicates (dict): A table of duplicating Loaders, indexed # by duplicated Loader # internals (dict): A table of Loaders which mark a loader as internal, # indexed by internal Loader # # Returns: # (str): A string representing how this loader was loaded # def _loader_description(self, loader, duplicates, internals): line = "{}\n".format(loader) # Mention projects which have marked this project as a duplicate duplicating = duplicates.get(loader) if duplicating: for dup in duplicating: line += " Duplicated by: {}\n".format(dup) # Mention projects which have marked this project as internal internalizing = internals.get(loader) if internalizing: for internal in internalizing: line += " Internal to: {}\n".format(internal) return line # LoaderContext() # # An object to keep track of overall context during the load process. # # Args: # context (Context): The invocation context # class LoadContext: def __init__(self, context): # Keep track of global context required throughout the recursive load self.context = context self.rewritable = False self.fetch_subprojects = None self.task = None # A table of all Loaders, indexed by project name self._loaders = {} # set_rewritable() # # Sets whether the projects are to be loaded in a rewritable fashion, # this is used for tracking and is slightly more expensive in load time. # # Args: # task (Task): The task to report progress on # def set_rewritable(self, rewritable): self.rewritable = rewritable # set_task() # # Sets the task for progress reporting. # # Args: # task (Task): The task to report progress on # def set_task(self, task): self.task = task # set_fetch_subprojects() # # Sets the task for progress reporting. # # Args: # task (callable): The callable for loading subprojects # def set_fetch_subprojects(self, fetch_subprojects): self.fetch_subprojects = fetch_subprojects # assert_loaders() # # Asserts that there are no conflicting projects loaded. # # Raises: # (LoadError): A CONFLICTING_JUNCTION LoadError in the case of a conflict # def assert_loaders(self): for _, loaders in self._loaders.items(): loaders.assert_loaders() # register_loader() # # Registers a new loader in the load context, possibly # raising an error in the case of a conflict. # # This must be called after a recursive load process has completed, # and after the pipeline is resolved (which is to say that all related # Plugin derived objects have been instantiated). # # Args: # loader (Loader): The Loader object to register into context # def register_loader(self, loader): project = loader.project try: project_loaders = self._loaders[project.name] except KeyError: project_loaders = ProjectLoaders(project.name) self._loaders[project.name] = project_loaders project_loaders.register_loader(loader) # loaded_projects() # # A generator which yeilds all of the loaded projects # # Yields: # (_ProjectInformation): A descriptive project information object # def loaded_projects(self): for _, project_loaders in self._loaders.items(): yield from project_loaders.loaded_projects() apache-buildstream-27ae392/src/buildstream/_loader/loadelement.pyi000066400000000000000000000015441514607367700253510ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import List from ..node import Node, ScalarNode def extract_depends_from_node(node: Node) -> List[Dependency]: ... class Dependency: ... class DependencyType: ... class LoadElement: first_pass: bool kind: str name: str description: str node: Node link_target: ScalarNode apache-buildstream-27ae392/src/buildstream/_loader/loadelement.pyx000066400000000000000000000540001514607367700253630ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from functools import cmp_to_key from pyroaring import BitMap, FrozenBitMap # pylint: disable=no-name-in-module from .._exceptions import LoadError from ..exceptions import LoadErrorReason from ..node cimport MappingNode, Node, ProvenanceInformation, ScalarNode, SequenceNode from .types import Symbol # Counter to get ids to LoadElements cdef int _counter = 0 cdef int _next_synthetic_counter(): global _counter _counter += 1 return _counter # DependencyType # # A bitfield to represent dependency types # cpdef enum DependencyType: # A build dependency BUILD = 0x001 # A runtime dependency RUNTIME = 0x002 # Both build and runtime dependencies ALL = 0x003 # Some forward declared lists, avoid creating these lists repeatedly # cdef list _filename_allowed_types=[ScalarNode, SequenceNode] cdef list _valid_dependency_keys = [Symbol.FILENAME, Symbol.TYPE, Symbol.JUNCTION, Symbol.STRICT, Symbol.CONFIG] cdef list _valid_typed_dependency_keys = [Symbol.FILENAME, Symbol.JUNCTION, Symbol.STRICT, Symbol.CONFIG] cdef list _valid_element_keys = [ 'kind', 'depends', 'sources', 'sandbox', 'variables', 'environment', 'environment-nocache', 'config', 'public', 'description', 'build-depends', 'runtime-depends', ] # Dependency(): # # Early stage data model for dependencies objects, the LoadElement has # Dependency objects which in turn refer to other LoadElements in the data # model. # # The constructor is incomplete, normally dependencies are loaded # via the Dependency.load() API below. The constructor arguments are # only used as a convenience to create the dummy Dependency objects # at the toplevel of the load sequence in the Loader. # # Args: # element (LoadElement): a LoadElement on which there is a dependency # dep_type (DependencyType): the type of dependency this dependency link is # cdef class Dependency: cdef readonly LoadElement element # The resolved LoadElement cdef readonly int dep_type # The dependency type (runtime or build or both) cdef readonly str name # The project local dependency name cdef readonly str junction # The junction path of the dependency name, if any cdef readonly bint strict # Whether this is a strict dependency cdef readonly list config_nodes # The custom config nodes for Element.configure_dependencies() cdef readonly Node node # The original node of the dependency def __cinit__(self, LoadElement element = None, int dep_type = DependencyType.ALL): self.element = element self.dep_type = dep_type self.name = None self.junction = None self.strict = False self.config_nodes = None self.node = None # path # # The path of the dependency represented as a single string, # instead of junction and name being separate. # @property def path(self): if self.junction is not None: return "{}:{}".format(self.junction, self.name) return self.name # set_element() # # Sets the resolved LoadElement # # When Dependencies are initially loaded, the `element` member # will be None until later on when the Loader loads the LoadElement # objects based on the Dependency `name` and `junction`, the Loader # will then call this to resolve the `element` member. # # Args: # element (LoadElement): The resolved LoadElement # cpdef set_element(self, element: LoadElement): self.element = element # load() # # Load dependency attributes from a Node, and validate it # # Args: # dep (Node): A node to load the dependency from # junction (str): The junction name, or None # name (str): The element name # default_dep_type (DependencyType): The default dependency type # cdef load(self, Node dep, str junction, str name, int default_dep_type): cdef str parsed_type cdef MappingNode config_node cdef ProvenanceInformation provenance self.junction = junction self.name = name self.node = dep self.element = None if type(dep) is ScalarNode: self.dep_type = default_dep_type or DependencyType.ALL elif type(dep) is MappingNode: if default_dep_type: ( dep).validate_keys(_valid_typed_dependency_keys) self.dep_type = default_dep_type else: ( dep).validate_keys(_valid_dependency_keys) # Resolve the DependencyType parsed_type = ( dep).get_str( Symbol.TYPE, None) if parsed_type is None or parsed_type == Symbol.ALL: self.dep_type = DependencyType.ALL elif parsed_type == Symbol.BUILD: self.dep_type = DependencyType.BUILD elif parsed_type == Symbol.RUNTIME: self.dep_type = DependencyType.RUNTIME else: provenance = dep.get_scalar(Symbol.TYPE).get_provenance() raise LoadError("{}: Dependency type '{}' is not 'build', 'runtime' or 'all'" .format(provenance, parsed_type), LoadErrorReason.INVALID_DATA) self.strict = ( dep).get_bool( Symbol.STRICT, False) config_node = ( dep).get_mapping( Symbol.CONFIG, None) if config_node: if self.dep_type == DependencyType.RUNTIME: raise LoadError("{}: Specifying 'config' for a runtime dependency is not allowed" .format(config_node.get_provenance()), LoadErrorReason.INVALID_DATA) self.config_nodes = [config_node] # Here we disallow explicitly setting 'strict' to False. # # This is in order to keep the door open to allowing the project.conf # set the default of dependency 'strict'-ness which might be useful # for projects which use mostly static linking and the like, in which # case we can later interpret explicitly non-strict dependencies # as an override of the project default. # if self.strict == False and Symbol.STRICT in dep: provenance = dep.get_scalar(Symbol.STRICT).get_provenance() raise LoadError("{}: Setting 'strict' to False is unsupported" .format(provenance), LoadErrorReason.INVALID_DATA) else: raise LoadError("{}: Dependency is not specified as a string or a dictionary".format(self.node.get_provenance()), LoadErrorReason.INVALID_DATA) # Only build dependencies are allowed to be strict # if self.strict and self.dep_type == DependencyType.RUNTIME: raise LoadError("{}: Runtime dependency {} specified as `strict`.".format(self.node.get_provenance(), self.name), LoadErrorReason.INVALID_DATA, detail="Only dependencies required at build time may be declared `strict`.") # merge() # # Merge the attributes of an existing dependency into this dependency # # Args: # other (Dependency): The dependency to merge into this one # cpdef merge(self, Dependency other): self.dep_type = self.dep_type | other.dep_type self.strict = self.strict or other.strict if self.config_nodes and other.config_nodes: self.config_nodes.extend(other.config_nodes) else: self.config_nodes = self.config_nodes or other.config_nodes # LoadElement(): # # A transient object breaking down what is loaded allowing us to # do complex operations in multiple passes. # # Args: # node (dict): A YAML loaded dictionary # name (str): The element name # loader (Loader): The Loader object for this element # cdef class LoadElement: cdef readonly MappingNode node cdef readonly str name cdef readonly str full_name cdef readonly str description cdef readonly str kind cdef int node_id cdef readonly bint first_pass cdef readonly object _loader cdef readonly ScalarNode link_target # TODO: if/when pyroaring exports symbols, we could type this statically cdef object _dep_cache cdef readonly list dependencies cdef readonly bint fully_loaded # This is True if dependencies were also loaded def __cinit__(self, MappingNode node, str filename, object loader): # # Public members # self.kind = None # The Element kind self.node = node # The YAML node self.name = filename # The element name self.full_name = None # The element full name (with associated junction) self.node_id = _next_synthetic_counter() self.link_target = None # The target of a link element (ScalarNode) self.fully_loaded = False # Whether we entered the loop to load dependencies or not # # Private members # self._loader = loader # The Loader object self._dep_cache = None # The dependency cache, to speed up depends() # # Initialization # if loader.project.junction: # dependency is in subproject, qualify name self.full_name = '{}:{}'.format(loader.project.junction._get_full_name(), self.name) else: # dependency is in top-level project self.full_name = self.name self.dependencies = [] # Ensure the root node is valid self.node.validate_keys(_valid_element_keys) self.kind = node.get_str(Symbol.KIND, default=None) self.description = node.get_str(Symbol.DESCRIPTION, default=None) self.first_pass = self.kind in ("junction", "link") # # If this is a link, resolve it right away and just # store the link target and provenance # if self.kind == 'link': # Avoid cyclic import here from ..element import Element element = Element._new_from_load_element(self) # Custom error for link dependencies, since we don't completely # parse their dependencies we cannot rely on the built-in ElementError. deps = extract_depends_from_node(self.node) if deps: raise LoadError( "{}: Dependencies are forbidden for 'link' elements".format(element), LoadErrorReason.LINK_FORBIDDEN_DEPENDENCIES ) self.link_target = element.target_node # We don't count progress for junction elements or link # as they do not represent real elements in the build graph. # # We check for a `None` kind, to avoid reporting progress for # the virtual toplevel element used to load the pipeline. # if self._loader.load_context.task and self.kind is not None and not self.first_pass: self._loader.load_context.task.add_current_progress() # project # # A property reporting the Project in which this element resides. # @property def project(self): return self._loader.project # junction # # A property reporting the junction element accessing this # element, if any. # @property def junction(self): return self._loader.project.junction # depends(): # # Checks if this element depends on another element, directly # or indirectly. # # Args: # other (LoadElement): Another LoadElement # # Returns: # (bool): True if this LoadElement depends on 'other' # def depends(self, LoadElement other not None): self._ensure_depends_cache() return other.node_id in self._dep_cache # mark_fully_loaded() # # Sets the fully loaded state on this load element # # This state bit is used by the Loader to distinguish # between an element which has only been shallow loaded # and an element which has entered the loop which loads # it's dependencies. # # Args: # element (LoadElement): The resolved LoadElement # def mark_fully_loaded(self): self.fully_loaded = True ########################################### # Private Methods # ########################################### cdef void _ensure_depends_cache(self): cdef Dependency dep if self._dep_cache: return self._dep_cache = BitMap() for dep in self.dependencies: elt = dep.element # Ensure the cache of the element we depend on elt._ensure_depends_cache() # We depend on this element self._dep_cache.add(elt.node_id) # And we depend on everything this element depends on self._dep_cache.update(elt._dep_cache) self._dep_cache = FrozenBitMap(self._dep_cache) # Sort algorithm copied from Python 3.12 cdef extern from "listsort.c": int _list_sort(object list, object keyfunc) except -1 # This comparison function does not impose a total ordering, which means # that the order of the sorted list depends on the order of inputs and # implementation details of the sort algorithm. Always use the sort # algorithm from Python 3.12 to ensure a deterministic result for a # given input order. def _dependency_cmp(Dependency dep_a, Dependency dep_b): cdef LoadElement element_a = dep_a.element cdef LoadElement element_b = dep_b.element # Sort on inter element dependency first if element_a.depends(element_b): return 1 elif element_b.depends(element_a): return -1 # If there are no inter element dependencies, place # runtime only dependencies last if dep_a.dep_type != dep_b.dep_type: if dep_a.dep_type == DependencyType.RUNTIME: return 1 elif dep_b.dep_type == DependencyType.RUNTIME: return -1 # All things being equal, string comparison. if element_a.name > element_b.name: return 1 elif element_a.name < element_b.name: return -1 # Sort local elements before junction elements # and use string comparison between junction elements if element_a.junction and element_b.junction: if element_a.junction > element_b.junction: return 1 elif element_a.junction < element_b.junction: return -1 elif element_a.junction: return -1 elif element_b.junction: return 1 # This wont ever happen return 0 # sort_dependencies(): # # Sort dependencies of each element by their dependencies, # so that direct dependencies which depend on other direct # dependencies (directly or indirectly) appear later in the # list. # # This avoids the need for performing multiple topological # sorts throughout the build process. # # Args: # element (LoadElement): The element to sort # visited (set): a list of elements that should not be treated because # because they already have been treated. # This is useful when wanting to sort dependencies of # multiple top level elements that might have a common # part. # def sort_dependencies(LoadElement element, set visited): cdef list working_elements = [element] cdef Dependency dep if element in visited: return visited.add(element) # Now dependency sort, we ensure that if any direct dependency # directly or indirectly depends on another direct dependency, # it is found later in the list. while working_elements: element = working_elements.pop() for dep in element.dependencies: if dep.element not in visited: visited.add(dep.element) working_elements.append(dep.element) _list_sort(element.dependencies, cmp_to_key(_dependency_cmp)) # _parse_dependency_filename(): # # Parse the filename of a dependency with the already provided parsed junction # name, if any. # # This will validate that the filename node does not contain `:` if # the junction is already specified, and otherwise it will appropriately # split the filename string and decompose it into a junction and filename. # # Args: # node (ScalarNode): The ScalarNode of the filename # junction (str): The already parsed junction, or None # # Returns: # (str): The junction component of the dependency filename # (str): The filename component of the dependency filename # cdef tuple _parse_dependency_filename(ScalarNode node, str junction): cdef str name = node.as_str() if junction is not None: if ':' in name: raise LoadError( "{}: Dependency {} contains `:` in its name. " "`:` characters are not allowed in filename when " "junction attribute is specified.".format(node.get_provenance(), name), LoadErrorReason.INVALID_DATA) elif ':' in name: junction, name = name.rsplit(':', maxsplit=1) return junction, name # _list_dependency_node_files(): # # List the filename, junction tuples associated with a dependency node, # this supports the `filename` attribute being expressed as a list, so # that multiple dependencies can be expressed with the common attributes. # # Args: # node (Node): A YAML loaded dictionary # # Returns: # (list): A list of filenames for `node` # cdef list _list_dependency_node_files(Node node): cdef list files = [] cdef str junction cdef tuple parsed_filename cdef Node filename_node cdef Node filename_iter cdef object filename_iter_object # The node can be a single filename declaration # if type(node) is ScalarNode: parsed_filename = _parse_dependency_filename(node, None) files.append(parsed_filename) # Otherwise it is a dictionary # elif type(node) is MappingNode: junction = ( node).get_str( Symbol.JUNCTION, None) filename_node = ( node).get_node( Symbol.FILENAME, allowed_types=_filename_allowed_types) if type(filename_node) is ScalarNode: parsed_filename = _parse_dependency_filename(filename_node, junction) files.append(parsed_filename) else: # The filename attribute is a list, iterate here for filename_iter_object in ( filename_node).value: filename_iter = filename_iter_object if type(filename_iter_object) is not ScalarNode: raise LoadError( "{}: Expected string while parsing the filename list".format(filename_iter.get_provenance()), LoadErrorReason.INVALID_DATA ) parsed_filename = _parse_dependency_filename(filename_iter, junction) files.append(parsed_filename) else: raise LoadError("{}: Dependency is not specified as a string or a dictionary".format(node.get_provenance()), LoadErrorReason.INVALID_DATA) return files # _extract_depends_from_node(): # # Helper for extract_depends_from_node to get dependencies of a particular type # # Adds to an array of Dependency objects from a given dict node 'node', # allows both strings and dicts for expressing the dependency. # # After extracting depends, the symbol is deleted from the node # # Args: # node (Node): A YAML loaded dictionary # key (str): the key on the Node corresponding to the dependency type # default_dep_type (DependencyType): type to give to the dependency # acc (dict): a dict in which to add the loaded dependencies # cdef void _extract_depends_from_node(Node node, str key, int default_dep_type, dict acc) except *: cdef SequenceNode depends = node.get_sequence(key, []) cdef Dependency existing_dep cdef object dep_node_object cdef Node dep_node cdef object deptup_object cdef tuple deptup cdef str junction cdef str filename for dep_node_object in depends.value: dep_node = dep_node_object for deptup_object in _list_dependency_node_files(dep_node): deptup = deptup_object junction = deptup[0] filename = deptup[1] dependency = Dependency() dependency.load(dep_node, junction, filename, default_dep_type) # Accumulate dependencies, merging any matching elements along the way existing_dep = acc.get(deptup, None) if existing_dep is not None: existing_dep.merge(dependency) else: acc[deptup] = dependency # Now delete the field, we dont want it anymore node.safe_del(key) # extract_depends_from_node(): # # Creates an array of Dependency objects from a given dict node 'node', # allows both strings and dicts for expressing the dependency and # throws a comprehensive LoadError in the case that the node is malformed. # # After extracting depends, the symbol is deleted from the node # # Args: # node (Node): A YAML loaded dictionary # # Returns: # (list): a list of Dependency objects # def extract_depends_from_node(Node node): cdef dict acc = {} _extract_depends_from_node(node, Symbol.BUILD_DEPENDS, DependencyType.BUILD, acc) _extract_depends_from_node(node, Symbol.RUNTIME_DEPENDS, DependencyType.RUNTIME, acc) _extract_depends_from_node(node, Symbol.DEPENDS, 0, acc) return [dep for dep in acc.values()] apache-buildstream-27ae392/src/buildstream/_loader/loader.py000066400000000000000000001231531514607367700241560ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os from contextlib import suppress from .._exceptions import LoadError from ..exceptions import LoadErrorReason from .. import _yaml from ..element import Element from ..node import Node from .._profile import Topics, PROFILER from .._includes import Includes from .._utils import valid_chars_name from ..types import _KeyStrength from .types import Symbol from . import loadelement from .loadelement import LoadElement, Dependency, DependencyType, extract_depends_from_node # Loader(): # # The Loader class does the heavy lifting of parsing target # bst files and ultimately transforming them into a list of LoadElements # ready for instantiation by the core. # # Args: # project (Project): The toplevel Project object # parent (Loader): A parent Loader object, in the case this is a junctioned Loader # provenance_node (Node): The provenance of the reference to this project's junction # class Loader: def __init__(self, project, *, parent=None, provenance_node=None): # Ensure we have an absolute path for the base directory basedir = project.element_path if not os.path.isabs(basedir): basedir = os.path.abspath(basedir) # # Public members # self.load_context = project.load_context # The LoadContext self.project = project # The associated Project self.provenance_node = provenance_node # The provenance of whence this loader was instantiated self.loaded = None # The number of loaded Elements # # Private members # self._options = project.options # Project options (OptionPool) self._basedir = basedir # Base project directory self._first_pass_options = project.first_pass_config.options # Project options (OptionPool) self._parent = parent # The parent loader self._alternative_parents = [] # Overridden parent loaders self._meta_elements = {} # Dict of resolved meta elements by name self._elements = {} # Dict of elements self._links = {} # Dict of link target target paths indexed by link element paths self._loaders = {} # Dict of junction loaders self._loader_search_provenances = {} # Dictionary of provenance nodes of ongoing child loader searches self._includes = Includes(self, copy_tree=True) assert project.name is not None self.load_context.register_loader(self) # The __str__ of a Loader is used to clearly identify the Loader, # the junction is was loaded as, and the provenance causing the # junction to be loaded. # def __str__(self): project_name = self.project.name if self.project.junction: junction_name = self.project.junction._get_full_name() if self.provenance_node: provenance = "({}): {}".format(junction_name, self.provenance_node.get_provenance()) else: provenance = "({})".format(junction_name) else: provenance = "(toplevel)" return "{} {}".format(project_name, provenance) # load(): # # Loads the project based on the parameters given to the constructor # # Args: # targets (list of str): Target, element-path relative bst filenames in the project # # Raises: LoadError # # Returns: # (list): The corresponding LoadElement instances matching the `targets` # def load(self, targets): for filename in targets: if os.path.isabs(filename): # XXX Should this just be an assertion ? # Expect that the caller gives us the right thing at least ? raise LoadError( "Target '{}' was not specified as a relative " "path to the base project directory: {}".format(filename, self._basedir), LoadErrorReason.INVALID_DATA, ) # First pass, recursively load files and populate our table of LoadElements # target_elements = [] for target in targets: with PROFILER.profile(Topics.LOAD_PROJECT, target): _junction, name, loader = self._parse_name(target, None) element = loader._load_file(name, None) target_elements.append(element) # # Now that we've resolved the dependencies, scan them for circular dependencies # # Set up a dummy element that depends on all top-level targets # to resolve potential circular dependencies between them dummy_target = LoadElement(Node.from_dict({}), "", self) # Pylint is not very happy with Cython and can't understand 'dependencies' is a list dummy_target.dependencies.extend( # pylint: disable=no-member Dependency(element, DependencyType.RUNTIME) for element in target_elements ) with PROFILER.profile(Topics.CIRCULAR_CHECK, "_".join(targets)): self._check_circular_deps(dummy_target) # # Sort direct dependencies of elements by their dependency ordering # # Keep a list of all visited elements, to not sort twice the same visited_elements = set() for element in target_elements: loader = element._loader with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name): loadelement.sort_dependencies(element, visited_elements) self._clean_caches() # Cache how many Elements have just been loaded if self.load_context.task: self.loaded = self.load_context.task.current_progress return target_elements # get_loader(): # # Obtains the appropriate loader for the specified junction # # If `load_subprojects` is enabled, then this function will # either return the desired loader or raise a LoadError. If # `load_subprojects` is disabled, then it can also return None # in the case that a loader could not be found. In either case, # a non-existant file in a loaded project will result in a LoadError. # # Args: # name (str): Name of junction, may have multiple `:` in the name # provenance_node (Node): The provenance from where this loader was requested # load_subprojects (bool): Whether to load subprojects on demand # # Returns: # (Loader): loader for sub-project # def get_loader(self, name, provenance_node, *, load_subprojects=True): junction_path = name.split(":") loader = self # # In this case we are attempting to load a subproject element via the # command line instead of referencing the subproject through a project # element or otherwise. # if provenance_node is None and load_subprojects: self.project.ensure_fully_loaded() circular_provenance_node = self._loader_search_provenances.get(name, None) if circular_provenance_node and load_subprojects: assert provenance_node detail = None if circular_provenance_node is not provenance_node: detail = "Already searching for '{}' at: {}".format(name, circular_provenance_node.get_provenance()) raise LoadError( "{}: Circular reference while searching for '{}'".format(provenance_node.get_provenance(), name), LoadErrorReason.CIRCULAR_REFERENCE, detail=detail, ) if load_subprojects and provenance_node: self._loader_search_provenances[name] = provenance_node for junction_name in junction_path: loader = loader._get_loader(junction_name, provenance_node, load_subprojects=load_subprojects) if not loader: # `loader` should never be None if `load_subprojects` is True assert not load_subprojects return None if load_subprojects and provenance_node: del self._loader_search_provenances[name] return loader # ancestors() # # This will traverse all active loaders in the ancestry for which this # project is reachable using a relative path. # # Yields: # (Loader): Each loader in the ancestry # def ancestors(self): traversed = {} def foreach_parent(parent): while parent: if parent in traversed: return traversed[parent] = True yield parent parent = parent._parent # Yield from the direct/active ancestry yield from foreach_parent(self._parent) # Yield from alternative parents which have been replaced by # overrides in the ancestry. for parent in self._alternative_parents: yield from foreach_parent(parent) ########################################### # Private Methods # ########################################### # _load_file_no_deps(): # # Load a bst file as a LoadElement # # This loads a bst file into a LoadElement but does no work to resolve # the element's dependencies. The dependencies must be resolved properly # before the LoadElement makes its way out of the loader. # # Args: # filename (str): The element-path relative bst file # provenance_node (Node): The location from where the file was referred to, or None # # Returns: # (LoadElement): A partially-loaded LoadElement # def _load_file_no_deps(self, filename, provenance_node=None): self._assert_element_name(filename, provenance_node) # Load the data and process any conditional statements therein fullpath = os.path.join(self._basedir, filename) try: node = _yaml.load( fullpath, shortname=filename, copy_tree=self.load_context.rewritable, project=self.project ) except LoadError as e: if e.reason == LoadErrorReason.MISSING_FILE: if self.project.junction: message = "Could not find element '{}' in project referred to by junction element '{}'".format( filename, self.project.junction.name ) else: message = "Could not find element '{}' in elements directory '{}'".format(filename, self._basedir) if provenance_node: message = "{}: {}".format(provenance_node.get_provenance(), message) # If we can't find the file, try to suggest plausible # alternatives by stripping the element-path from the given # filename, and verifying that it exists. detail = None elements_dir = os.path.relpath(self._basedir, self.project.directory) element_relpath = os.path.relpath(filename, elements_dir) if filename.startswith(elements_dir) and os.path.exists(os.path.join(self._basedir, element_relpath)): detail = "Did you mean '{}'?".format(element_relpath) raise LoadError(message, LoadErrorReason.MISSING_FILE, detail=detail) from e # Otherwise, we don't know the reason, so just raise raise kind = node.get_str(Symbol.KIND) if kind in ("junction", "link"): self._first_pass_options.process_node(node) else: self.project.ensure_fully_loaded() self._includes.process(node) element = LoadElement(node, filename, self) self._elements[filename] = element # # Update link caches in the ancestry # if element.link_target is not None: link_path = filename target_path = element.link_target.as_str() # pylint: disable=no-member # First resolve the link in this loader's cache # self._resolve_link(link_path, target_path) # Now resolve the link in parent project loaders # loader = self while loader._parent: junction = loader.project.junction link_path = junction.name + ":" + link_path target_path = junction.name + ":" + target_path # Resolve the link loader = loader._parent loader._resolve_link(link_path, target_path) return element # _resolve_link(): # # Resolves a link in the loader's link cache. # # This will first insert the new link -> target relationship # into the cache, and will also update any existing targets # which might have pointed to this link, to point to the new # target instead. # # Args: # link_path (str): The local project relative real path to a link # target_path (str): The new target for this link # def _resolve_link(self, link_path, target_path): self._links[link_path] = target_path for cached_link_path, cached_target_path in self._links.items(): if self._expand_link(cached_target_path) == link_path: self._links[cached_link_path] = target_path # _expand_link(): # # Expands any links in the provided path and returns a real path with # known link elements substituted for their targets. # # Args: # path (str): A project relative path # # Returns: # (str): The same path with any links expanded # def _expand_link(self, path): # FIXME: This simply returns the first link, maybe # this needs to be more iterative, or sorted by # number of path components, or smth for link, target in self._links.items(): if path.startswith(link): return target + path[len(link) :] return path # _load_one_file(): # # A helper function to load a single file within the _load_file() process, # this allows us to handle redirections more consistently. # # Args: # filename (str): The element-path relative bst file # provenance_node (Node): The location from where the file was referred to, or None # load_subprojects (bool): Whether to load subprojects # # Returns: # (LoadElement): A LoadElement, which might be shallow loaded or fully loaded, # or None, if loading of the subproject is disabled. # def _load_one_file(self, filename, provenance_node, *, load_subprojects=True): element = None # First check the cache, the cache might contain shallow loaded # elements. # try: element = self._elements[filename] # If the cached element has already entered the loop which loads # it's dependencies, it is fully loaded and any further checks in # this function are expected to have already been performed. # if element.fully_loaded: return element except KeyError: # Shallow load if it's not yet loaded. element = self._load_file_no_deps(filename, provenance_node) # Check if there was an override for this element # override = self._search_for_override_element(filename) if override: # # If there was an override for the element, then it was # implicitly fully loaded by _search_for_override_element(), # return override # If this element is a link then we need to resolve it, and return # the linked element instead of this one. # if element.link_target is not None: link_target = element.link_target.as_str() # pylint: disable=no-member _, filename, loader = self._parse_name(link_target, element.link_target, load_subprojects=load_subprojects) if not loader: # `loader` should never be None if `load_subprojects` is True assert not load_subprojects return None # # Redirect the loading of the file and it's dependencies to the appropriate loader, # which might or might not be the same loader. # return loader._load_file(filename, element.link_target, load_subprojects=load_subprojects) return element # _load_file(): # # Semi-Iteratively load bst files # # The "Semi-" qualification is because where junctions get involved there # is a measure of recursion, though this is limited only to the points at # which junctions are crossed. # # Args: # filename (str): The element-path relative bst file # provenance_node (Node): The location from where the file was referred to, or None # load_subprojects (bool): Whether to load subprojects # # Returns: # (LoadElement): A loaded LoadElementor None, if loading of the subproject is disabled. # def _load_file(self, filename, provenance_node, *, load_subprojects=True): top_element = self._load_one_file(filename, provenance_node, load_subprojects=load_subprojects) if not top_element: # `_load_one_file` should not return None if `load_subprojects` is True assert not load_subprojects return None # Already loaded dependencies for a fully loaded element, early return. # if top_element.fully_loaded: return top_element # # Mark the top element here as "fully loaded", so that we will avoid trying to # load it's dependencies more than once. # top_element.mark_fully_loaded() dependencies = extract_depends_from_node(top_element.node) # The loader queue is a stack of tuples # [0] is the LoadElement instance # [1] is a stack of Dependency objects to load # [2] is a Dict[LoadElement, Dependency] of loaded dependencies loader_queue = [(top_element, list(reversed(dependencies)), {})] # Load all dependency files for the new LoadElement while loader_queue: if loader_queue[-1][1]: current_element = loader_queue[-1] # Process the first dependency of the last loaded element dep = current_element[1].pop() if dep.junction: loader = self.get_loader(dep.junction, dep.node) dep_element = loader._load_file(dep.name, dep.node) else: dep_element = self._load_one_file(dep.name, dep.node, load_subprojects=load_subprojects) if not dep_element: # `_load_one_file` should not return None if `load_subprojects` is True assert not load_subprojects return None # If the loaded element is not fully loaded, queue up the dependencies to be loaded in this loop. # if not dep_element.fully_loaded: # Mark the dep_element as fully_loaded, as we're already queueing it's deps dep_element.mark_fully_loaded() dep_deps = extract_depends_from_node(dep_element.node) loader_queue.append((dep_element, list(reversed(dep_deps)), {})) # Pylint is not very happy about Cython and can't understand 'node' is a 'MappingNode' if dep_element.node.get_str(Symbol.KIND) == "junction": # pylint: disable=no-member raise LoadError( "{}: Cannot depend on junction".format(dep.node.get_provenance()), LoadErrorReason.INVALID_DATA, ) # We've now resolved the element for this dependency, lets set the resolved # LoadElement on the dependency and append the dependency to the owning # LoadElement dependency list. dep.set_element(dep_element) dep_dict = current_element[2] if dep.element in dep_dict: # Duplicate LoadElement in dependency list, this can happen if a dependency is # a link element that points to an element that is already a dependency. dep_dict[dep.element].merge(dep) else: current_element[0].dependencies.append(dep) # pylint: disable=no-member dep_dict[dep.element] = dep else: # And pop the element off the queue loader_queue.pop() # Nothing more in the queue, return the top level element we loaded. return top_element # _check_circular_deps(): # # Detect circular dependencies on LoadElements with # dependencies already resolved. # # Args: # element (str): The element to check # # Raises: # (LoadError): In case there was a circular dependency error # @staticmethod def _check_circular_deps(top_element): sequence = [top_element] sequence_indices = [0] check_elements = set(sequence) validated = set() while sequence: this_element = sequence[-1] index = sequence_indices[-1] if index < len(this_element.dependencies): element = this_element.dependencies[index].element sequence_indices[-1] = index + 1 if element in check_elements: # Create `chain`, the loop of element dependencies from this # element back to itself, by trimming everything before this # element from the sequence under consideration. chain = [element.full_name for element in sequence[sequence.index(element) :]] chain.append(element.full_name) raise LoadError( ("Circular dependency detected at element: {}\n" + "Dependency chain: {}").format( element.full_name, " -> ".join(chain) ), LoadErrorReason.CIRCULAR_DEPENDENCY, ) if element not in validated: # We've not already validated this element, so let's # descend into it to check it out sequence.append(element) sequence_indices.append(0) check_elements.add(element) # Otherwise we'll head back around the loop to validate the # next dependency in this entry else: # Done with entry, pop it off, indicate we're no longer # in its chain, and mark it valid sequence.pop() sequence_indices.pop() check_elements.remove(this_element) validated.add(this_element) # _search_for_local_override(): # # Search this project's active override list for an override, while # considering any link elements. # # Args: # override_path (str): The real relative path to search for # # Returns: # (ScalarNode): The overridding node from this project's junction, or None # def _search_for_local_override(self, override_path): junction = self.project.junction if junction is None: return None # Try the override without any link substitutions first with suppress(KeyError): return junction.overrides[override_path] # # If we did not get an exact match here, we might still have # an override where a link was used to specify the override. # for override_key, override_node in junction.overrides.items(): resolved_path = self._expand_link(override_key) if resolved_path == override_path: return override_node return None # _search_for_overrides(): # # Search for parent loaders which have an override for the specified element, # returning a list of loaders with the highest level overriding loader at the # end of the list, and the closest ancestor being at the beginning of the list. # # Args: # filename (str): The local element name # # Returns: # (list): A list of loaders which override this element # def _search_for_overrides(self, filename): loader = self override_path = filename # Collect any overrides to this junction in the ancestry # overriding_loaders = [] while loader._parent: junction = loader.project.junction override_node = loader._search_for_local_override(override_path) if override_node: overriding_loaders.append((loader._parent, override_node)) override_path = junction.name + ":" + override_path loader = loader._parent return overriding_loaders # _search_for_override_loader(): # # Search parent projects an override of the junction specified by @filename, # returning the loader object which should be used in place of the local # junction specified by @filename. # # This function is called once for each direct child while looking up # child loaders, after which point the child loader is cached in the `_loaders` # table. This function also has the side effect of recording alternative parents # of a child loader in the case that the child loader is overridden. # # Args: # filename (str): Junction name # # Returns: # (Loader): The loader to use, in case @filename was overridden, otherwise None. # def _search_for_override_loader(self, filename): overriding_loaders = self._search_for_overrides(filename) # If there are any overriding loaders, use the highest one in # the ancestry to lookup the loader for this project. # if overriding_loaders: overriding_loader, override_node = overriding_loaders[-1] loader = overriding_loader.get_loader(override_node.as_str(), override_node) # # Record alternative loaders which were overridden. # # When a junction is overridden by another higher priority junction, # the resulting loader is still reachable with the original element paths, # which will now traverse override redirections. # # In order to iterate over every project/loader in the ancestry which can # reach the actually selected loader, we need to keep track of the parent # loaders of all overridden junctions. # if loader is not self: loader._alternative_parents.append(self) del overriding_loaders[-1] loader._alternative_parents.extend(l for l, _ in overriding_loaders) return loader # No overrides were found in the ancestry # return None # _search_for_override_element(): # # Search parent projects an override of the element specified by @filename, # returning the loader object which should be used in place of the local # element specified by @filename. # # Args: # filename (str): Junction name # # Returns: # (Loader): The loader to use, in case @filename was overridden, otherwise None. # def _search_for_override_element(self, filename): element = None # If there are any overriding loaders, use the highest one in # the ancestry to lookup the element which should be used in place # of @filename. # overriding_loaders = self._search_for_overrides(filename) if overriding_loaders: overriding_loader, override_node = overriding_loaders[-1] _, filename, loader = overriding_loader._parse_name(override_node.as_str(), override_node) element = loader._load_file(filename, override_node) return element # _get_loader(): # # Return loader for specified junction # # Args: # filename (str): Junction name # load_subprojects (bool): Whether to load subprojects # provenance_node (Node): The location from where the file was referred to, or None # # Raises: LoadError # # Returns: A Loader or None if specified junction does not exist # def _get_loader(self, filename, provenance_node, *, load_subprojects=True): loader = None # return previously determined result if filename in self._loaders: return self._loaders[filename] # Local function to conditionally resolve the provenance prefix string def provenance_str(): if provenance_node is not None: return "{}: ".format(provenance_node.get_provenance()) return "" # # Search the ancestry for an overridden loader to use in place # of using the locally defined junction. # override_loader = self._search_for_override_loader(filename) if override_loader: self._loaders[filename] = override_loader return override_loader # # Load the junction file # self._load_file(filename, provenance_node, load_subprojects=load_subprojects) # At this point we've loaded the LoadElement load_element = self._elements[filename] # If the loaded element is a link, then just follow it # immediately and move on to the target. # if load_element.link_target: _, filename, loader = self._parse_name( load_element.link_target.as_str(), load_element.link_target, load_subprojects=load_subprojects ) if not loader: # `loader` should never be None if `load_subprojects` is True assert not load_subprojects return None return loader.get_loader(filename, load_element.link_target, load_subprojects=load_subprojects) # If we're only performing a lookup, we're done here. # if not load_subprojects: return None if load_element.kind != "junction": raise LoadError( "{}{}: Expected junction but element kind is {}".format(provenance_str(), filename, load_element.kind), LoadErrorReason.INVALID_DATA, ) # We check that junctions have no dependencies a little # early. This is cheating, since we don't technically know # that junctions aren't allowed to have dependencies. # # However, this makes progress reporting more intuitive # because we don't need to load dependencies of an element # that shouldn't have any, and therefore don't need to # duplicate the load count for elements that shouldn't be. # # We also fail slightly earlier (since we don't need to go # through the entire loading process), which is nice UX. It # would be nice if this could be done for *all* element types, # but since we haven't loaded those yet that's impossible. if load_element.dependencies: # Use the first dependency in the list as provenance p = load_element.dependencies[0].node.get_provenance() raise LoadError( "{}: Dependencies are forbidden for 'junction' elements".format(p), LoadErrorReason.INVALID_JUNCTION ) element = Element._new_from_load_element(load_element) # Handle the case where a subproject has no ref # if not element._has_all_sources_resolved(): detail = "Try tracking the junction element with `bst source track {}`".format(filename) raise LoadError( "{}Subproject has no ref for junction: {}".format(provenance_str(), filename), LoadErrorReason.SUBPROJECT_INCONSISTENT, detail=detail, ) # Handle the case where a subproject needs to be fetched # element._query_source_cache() if element._should_fetch(): self.load_context.fetch_subprojects([element]) sources = list(element.sources()) if len(sources) == 1 and sources[0]._get_local_path(): # Optimization for junctions with a single local source basedir = sources[0]._get_local_path() else: # Stage sources element._set_required() # Note: We use _KeyStrength.WEAK here because junctions # cannot have dependencies, therefore the keys are # equivalent. # # Since the element has not necessarily been given a # strong cache key at this point (in a non-strict build # that is set *after* we complete building/pulling, which # we haven't yet for this element), # element._get_cache_key() can fail if used with the # default _KeyStrength.STRONG. basedir = os.path.join( self.project.directory, ".bst", "staged-junctions", filename, element._get_cache_key(_KeyStrength.WEAK) ) if not os.path.exists(basedir): os.makedirs(basedir, exist_ok=True) element._stage_sources_at(basedir) # Load the project project_dir = os.path.join(basedir, element.path) try: from .._project import Project # pylint: disable=cyclic-import project = Project( project_dir, self.load_context.context, junction=element, parent_loader=self, search_for_project=False, provenance_node=provenance_node, ) except LoadError as e: if e.reason == LoadErrorReason.MISSING_PROJECT_CONF: message = ( provenance_str() + "Could not find the project.conf file in the project " "referred to by junction element '{}'.".format(element.name) ) if element.path: message += " Was expecting it at path '{}' in the junction's source.".format(element.path) raise LoadError(message=message, reason=LoadErrorReason.INVALID_JUNCTION) from e # Otherwise, we don't know the reason, so just raise raise loader = project.loader self._loaders[filename] = loader # Now we've loaded a junction and it's project, we need to try to shallow # load the overrides of this project and any projects in the ancestry which # have overrides referring to this freshly loaded project. # # This is to ensure that link elements have been resolved as much as possible # before we try to look for an override. # iter_loader = loader while iter_loader._parent: iter_loader._shallow_load_overrides() iter_loader = iter_loader._parent return loader # _shallow_load_overrides(): # # Loads any of the override elements on this loader's junction # def _shallow_load_overrides(self): if not self.project.junction: return junction = self.project.junction # Iterate over the keys, we want to ensure that links are resolved for # override paths specified in junctions, while the targets of these paths # are not consequential. # for override_path, override_target in junction.overrides.items(): # Ensure that we resolve indirect links, in case that shallow loading # an element results in loading a link, we need to discover if it's # target is also a link. # path = override_path provenance_node = override_target while path is not None: path, provenance_node = self._shallow_load_path(path, provenance_node) # _shallow_load_path() # # Perform a shallow load of an element by it's relative path, this is # used to load elements which might be specified by their path and might # not be used in the resulting load, like paths to elements overridden by # junctions. # # It is only important to shallow load these referenced elements in case # they are links which need to be known later on. # # Args: # path (str): The path to load # provenance_node (Node): The node to use for provenance # # Returns: # (str): The target of the loaded link element, if it was a link element # and it could be loaded presently, otherwise None. # (ScalarNode): The link target real node, if a link target was returned # def _shallow_load_path(self, path, provenance_node): if ":" in path: junction, element_name = path.rsplit(":", 1) target_loader = self.get_loader(junction, provenance_node, load_subprojects=False) # Subproject not loaded, discard this shallow load attempt # if target_loader is None: return None, None else: junction = None element_name = path target_loader = self # If the element is already loaded in the target loader, then there # is no need for a shallow load. try: element = target_loader._elements[element_name] except KeyError: # Shallow load the the element. element = target_loader._load_file_no_deps(element_name, provenance_node) if element.link_target: link_target = element.link_target.as_str() if junction: return "{}:{}".format(junction, link_target), element.link_target return link_target, element.link_target return None, None # _parse_name(): # # Get junction and base name of element along with loader for the sub-project # # Args: # name (str): Name of target # provenance_node (Node): The provenance node # load_subprojects (bool): Whether to load subprojects # # Returns: # (tuple): - (str): name of the junction element # - (str): name of the element # - (Loader): loader for sub-project # def _parse_name(self, name, provenance_node, *, load_subprojects=True): # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. junction_path = name.rsplit(":", 1) if len(junction_path) == 1: return None, junction_path[-1], self else: loader = self.get_loader(junction_path[-2], provenance_node, load_subprojects=load_subprojects) return junction_path[-2], junction_path[-1], loader # _warn(): # # Print a warning message, checks warning_token against project configuration # # Args: # brief (str): The brief message # warning_token (str): An optional configurable warning assosciated with this warning, # this will cause PluginError to be raised if this warning is configured as fatal. # # Raises: # (:class:`.LoadError`): When warning_token is considered fatal by the project configuration # def _warn(self, brief, *, warning_token=None): if warning_token: if self.project._warning_is_fatal(warning_token): raise LoadError(brief, warning_token) self.load_context.context.messenger.warn(brief) # _assert_element_name(): # # Raises an error if any of the specified elements have invalid names. # # Valid filenames should end with ".bst" extension. # # Args: # filename (str): The element name # provenance_node (Node): The provenance node, or None # # Raises: # (:class:`.LoadError`): If the element name is invalid # def _assert_element_name(self, filename, provenance_node): error_message = None error_reason = None if not filename.endswith(".bst"): error_message = "Element '{}' does not have expected file extension `.bst`".format(filename) error_reason = LoadErrorReason.BAD_ELEMENT_SUFFIX elif not valid_chars_name(filename): error_message = "Element '{}' has invalid characters.".format(filename) error_reason = LoadErrorReason.BAD_CHARACTERS_IN_NAME if error_message: if provenance_node is not None: error_message = "{}: {}".format(provenance_node.get_provenance(), error_message) raise LoadError(error_message, error_reason) # _clean_caches() # # Clean internal loader caches, recursively # # When loading the elements, the loaders use caches in order to not load the # same element twice. These are kept after loading and prevent garbage # collection. Cleaning them explicitely is required. # def _clean_caches(self): for loader in self._loaders.values(): # value may be None with nested junctions without overrides if loader is not None: loader._clean_caches() self._meta_elements = {} self._elements = {} apache-buildstream-27ae392/src/buildstream/_loader/metasource.py000066400000000000000000000033041514607367700250520ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom class MetaSource: # MetaSource() # # An abstract object holding data suitable for constructing a Source # # Args: # element_name: The name of the owning element # element_index: The index of the source in the owning element's source list # element_kind: The kind of the owning element # kind: The kind of the source # directory: A subdirectory where to stage the source # provenance: The user provided provenance information (e.g. homepage, issue tracking, etc). # config: The configuration data for the source # first_pass: This source will be used with first project pass configuration (used for junctions). # def __init__(self, element_name, element_index, element_kind, kind, directory, provenance, config, first_pass): self.element_name = element_name self.element_index = element_index self.element_kind = element_kind self.kind = kind self.directory = directory self.provenance = provenance self.config = config self.first_pass = first_pass apache-buildstream-27ae392/src/buildstream/_loader/types.py000066400000000000000000000024001514607367700240430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Symbol(): # # A simple object to denote the symbols we load with from YAML # class Symbol: FILENAME = "filename" KIND = "kind" DESCRIPTION = "description" DEPENDS = "depends" BUILD_DEPENDS = "build-depends" RUNTIME_DEPENDS = "runtime-depends" SOURCES = "sources" CONFIG = "config" VARIABLES = "variables" ENVIRONMENT = "environment" ENV_NOCACHE = "environment-nocache" PUBLIC = "public" TYPE = "type" BUILD = "build" RUNTIME = "runtime" ALL = "all" DIRECTORY = "directory" JUNCTION = "junction" SANDBOX = "sandbox" STRICT = "strict" PROVENANCE = "provenance" apache-buildstream-27ae392/src/buildstream/_message.py000066400000000000000000000066501514607367700230700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import datetime import os from typing import Optional from .types import _DisplayKey # Types of status messages. # class MessageType: DEBUG = "debug" # Debugging message STATUS = "status" # Status message, verbose details INFO = "info" # Informative messages WARN = "warning" # Warning messages ERROR = "error" # Error messages BUG = "bug" # An unhandled exception was raised in a plugin LOG = "log" # Messages for log files _only_, never in the frontend # Timed Messages: SUCCESS and FAIL have duration timestamps START = "start" # Status start message SUCCESS = "success" # Successful status complete message FAIL = "failure" # Failing status complete message SKIPPED = "skipped" # Messages which should be reported regardless of whether # they are currently silenced or not unconditional_messages = [MessageType.INFO, MessageType.WARN, MessageType.FAIL, MessageType.ERROR, MessageType.BUG] # Message object # class Message: def __init__( self, message_type: str, message: str, *, task_element_name: Optional[str] = None, task_element_key: Optional[_DisplayKey] = None, element_name: Optional[str] = None, element_key: Optional[_DisplayKey] = None, detail: Optional[str] = None, action_name: Optional[str] = None, elapsed: Optional[datetime.timedelta] = None, logfile: Optional[str] = None, sandbox: bool = False, scheduler: bool = False ): self.message_type: str = message_type # Message type self.message: str = message # The message string self.task_element_name: Optional[str] = task_element_name # The name of the issuing task element self.task_element_key: Optional[_DisplayKey] = task_element_key # The DisplayKey of the issuing task element self.element_name: Optional[str] = element_name # The name of the issuing element self.element_key: Optional[_DisplayKey] = element_key # The DisplayKey of the issuing element self.detail: Optional[str] = detail # An additional detail string self.action_name: Optional[str] = action_name # Name of the task queue (fetch, refresh, build, etc) self.elapsed: Optional[datetime.timedelta] = elapsed # The elapsed time, in timed messages self.logfile: Optional[str] = logfile # The log file path where commands took place self.sandbox: bool = sandbox # Whether the error that caused this message used a sandbox self.pid: int = os.getpid() # The process pid self.scheduler: bool = scheduler # Whether this is a scheduler level message self.creation_time: datetime.datetime = datetime.datetime.now() if message_type in (MessageType.SUCCESS, MessageType.FAIL): assert elapsed is not None apache-buildstream-27ae392/src/buildstream/_messenger.py000066400000000000000000000505371514607367700234370ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Angelos Evripiotis import os import datetime import threading from contextlib import contextmanager from typing import Optional, Callable, Iterator, TextIO from .types import _DisplayKey from . import _signals from ._exceptions import BstError from ._message import Message, MessageType, unconditional_messages from ._state import State, Task from ._version import get_versions _RENDER_INTERVAL: datetime.timedelta = datetime.timedelta(seconds=1) # Time in seconds for which we decide that we want to display subtask information _DISPLAY_LIMIT: datetime.timedelta = datetime.timedelta(seconds=3) # If we're in the test suite, we need to ensure that we don't set a limit if "BST_TEST_SUITE" in os.environ: _DISPLAY_LIMIT = datetime.timedelta(seconds=0) # TimeData class to contain times in an object that can be passed around # and updated from different places class _TimeData: __slots__ = ["start_time"] def __init__(self, start_time: datetime.datetime) -> None: self.start_time: datetime.datetime = start_time # _JobInfo # # Information about a job, used as a part of thread local storage # in order to fill in some Message parameters automatically. # class _JobInfo: def __init__(self, action_name: str, element_name: str, element_key: _DisplayKey) -> None: self.action_name = action_name self.element_name = element_name self.element_key = element_key # _MessengerLocal # # Thread local storage for the messenger # class _MessengerLocal(threading.local): def __init__(self) -> None: super().__init__() # The open file handle for this task self.log_handle: Optional[TextIO] = None # The filename for this task self.log_filename: Optional[str] = None # Level of silent messages depth in this task self.silence_scope_depth: int = 0 # Job self.job: Optional[_JobInfo] = None # Messenger() # # The messenger object. # # This is used to propagate messages either from the main context or # from task contexts in such a way that messages are propagated to # the frontend and also optionally recorded to a task log file when # the message is issued from a task context. # class Messenger: def __init__(self) -> None: self._state: Optional[State] = None # The State object # # State related to simple tasks, these drive the status bar # when ongoing activities occur outside of an active scheduler # self._active_simple_tasks: int = 0 # Number of active simple tasks self._next_render: Optional[datetime.datetime] = None # The time of the next render self._render_status_cb: Optional[Callable[[], None]] = None # The render callback # Thread local storage self._locals: _MessengerLocal = _MessengerLocal() # The callback to call when propagating messages # # FIXME: The message handler is currently not strongly typed, # as it uses a kwarg, we cannot declare it with Callable. # We can use `Protocol` to strongly type this with python >= 3.8 self._message_handler = None # Save the bst version to record in log files # self._bst_version = get_versions()["version"] # setup_new_action_context() # # Setup the thread local context for a new task, some message # components are filled in automatically based on the action context. # # Args: # action_name: The action name # element_name: The element name # element_key: The element's DisplayKey # def setup_new_action_context(self, action_name: str, element_name: str, element_key: _DisplayKey) -> None: self._locals.silence_scope_depth = 0 self._locals.job = _JobInfo(action_name, element_name, element_key) # set_message_handler() # # Sets the handler for any status messages propagated through # the messenger. # def set_message_handler(self, handler) -> None: self._message_handler = handler # set_state() # # Sets the State object within the Messenger # # Args: # state: The state to set # def set_state(self, state: State) -> None: self._state = state # set_render_status_cb() # # Sets the callback to use to render status # # Args: # callback: The Callback to be notified # def set_render_status_cb(self, callback: Callable[[], None]) -> None: self._render_status_cb = callback # message(): # # Proxies a message back to the caller, this is the central # point through which all messages pass. # # Args: # message: A Message object # def message(self, message: Message) -> None: # If we are recording messages, dump a copy into the open log file. self._record_message(message) # Always add the log filename automatically message.logfile = self._locals.log_filename is_silenced = self._silent_messages() job = self._locals.job if job is not None: # Automatically add message information from the job context message.action_name = job.action_name message.task_element_name = job.element_name message.task_element_key = job.element_key # Don't forward LOG messages from jobs if message.message_type == MessageType.LOG: return # Don't forward JOB messages if they are currently silent if is_silenced and (message.message_type not in unconditional_messages): return # Send it off to the log handler (can be the frontend, # or it can be the child task which will propagate # to the frontend) assert self._message_handler self._message_handler(message, is_silenced=is_silenced) # status(): # # A core facing convenience method for issuing STATUS messages # # Args: # brief: The brief status message # detail: An optional detailed message # kwargs: Additional Message constructor keyword arguments # def status(self, brief: str, *, detail: Optional[str] = None, **kwargs) -> None: message = Message(MessageType.STATUS, brief, detail=detail, **kwargs) self.message(message) # info(): # # A core facing convenience method for issuing INFO messages # # Args: # brief: The brief info message # detail: An optional detailed message # kwargs: Additional Message constructor keyword arguments # def info(self, brief: str, *, detail: Optional[str] = None, **kwargs) -> None: message = Message(MessageType.INFO, brief, detail=detail, **kwargs) self.message(message) # warn(): # # A core facing convenience method for issuing WARN messages # # Args: # brief: The brief warning message # detail: An optional detailed message # kwargs: Additional Message constructor keyword arguments # def warn(self, brief: str, *, detail: Optional[str] = None, **kwargs) -> None: message = Message(MessageType.WARN, brief, detail=detail, **kwargs) self.message(message) # error(): # # A core facing convenience method for issuing ERROR messages # # Args: # brief: The brief error message # detail: An optional detailed message # kwargs: Additional Message constructor keyword arguments # def error(self, brief: str, *, detail: Optional[str] = None, **kwargs) -> None: message = Message(MessageType.ERROR, brief, detail=detail, **kwargs) self.message(message) # bug(): # # A core facing convenience method for issuing BUG messages # # Args: # brief: The brief bug message # detail: An optional detailed message # kwargs: Additional Message constructor keyword arguments # def bug(self, brief: str, *, detail: Optional[str] = None, **kwargs) -> None: message = Message(MessageType.BUG, brief, detail=detail, **kwargs) self.message(message) # silence() # # A context manager to silence messages, this behaves in # the same way as the `silent_nested` argument of the # timed_activity() context manager: all but # _message.unconditional_messages will be silenced. # # Args: # actually_silence: Whether to actually do the silencing, if # False then this context manager does not # affect anything. # @contextmanager def silence(self, *, actually_silence: bool = True) -> Iterator[None]: if not actually_silence: yield None return self._locals.silence_scope_depth += 1 try: yield None finally: assert self._locals.silence_scope_depth > 0 self._locals.silence_scope_depth -= 1 # timed_activity() # # Context manager for performing timed activities and logging those # # Args: # activity_name: The name of the activity # detail: An optional detailed message, can be multiline output # silent_nested: If True, all nested messages are silenced except for unconditionaly ones # kwargs: Remaining Message() constructor keyword arguments. # @contextmanager def timed_activity( self, activity_name: str, *, detail: Optional[str] = None, silent_nested: bool = False, **kwargs ) -> Iterator[None]: with self.timed_suspendable() as timedata: try: # Push activity depth for status messages message = Message(MessageType.START, activity_name, detail=detail, **kwargs) self.message(message) with self.silence(actually_silence=silent_nested): yield None except BstError: # Note the failure in status messages and reraise, the scheduler # expects an error when there is an error. elapsed = datetime.datetime.now() - timedata.start_time message = Message(MessageType.FAIL, activity_name, elapsed=elapsed, **kwargs) self.message(message) raise elapsed = datetime.datetime.now() - timedata.start_time message = Message(MessageType.SUCCESS, activity_name, elapsed=elapsed, **kwargs) self.message(message) # simple_task() # # Context manager for creating a task to report progress to. # # Args: # activity_name: The name of the activity # task_name: Optionally, the task name for the frontend during this task # detail: An optional detailed message, can be multiline output # silent_nested: If True, all nested messages are silenced except for unconditionaly ones # kwargs: Remaining Message() constructor keyword arguments. # # Yields: # Task: A Task object that represents this activity, principally used to report progress # @contextmanager def simple_task( self, activity_name: str, *, task_name: Optional[str] = None, detail: Optional[str] = None, silent_nested: bool = False, **kwargs ) -> Iterator[Optional[Task]]: # Bypass use of State when none exists (e.g. tests) if not self._state: with self.timed_activity(activity_name, detail=detail, silent_nested=silent_nested, **kwargs): yield None return if not task_name: task_name = activity_name with self.timed_suspendable() as timedata: try: message = Message(MessageType.START, activity_name, detail=detail, **kwargs) self.message(message) task = self._state.add_task(task_name, activity_name, task_name) task.set_task_changed_callback(self._render_status) self._active_simple_tasks += 1 if not self._next_render: self._next_render = datetime.datetime.now() + _RENDER_INTERVAL with self.silence(actually_silence=silent_nested): yield task except BstError: elapsed = datetime.datetime.now() - timedata.start_time message = Message(MessageType.FAIL, activity_name, elapsed=elapsed, **kwargs) self.message(message) raise finally: self._state.remove_task(task_name) self._active_simple_tasks -= 1 if self._active_simple_tasks == 0: self._next_render = None elapsed = datetime.datetime.now() - timedata.start_time detail = None if task.current_progress is not None and elapsed > _DISPLAY_LIMIT: if task.maximum_progress is not None: detail = "{} of {} subtasks processed".format(task.current_progress, task.maximum_progress) else: detail = "{} subtasks processed".format(task.current_progress) message = Message(MessageType.SUCCESS, activity_name, elapsed=elapsed, detail=detail, **kwargs) self.message(message) # recorded_messages() # # Records all messages in a log file while the context manager # is active. # # In addition to automatically writing all messages to the # specified logging file, an open file handle for process stdout # and stderr will be available via the Messenger.get_log_handle() API, # and the full logfile path will be available via the # Messenger.get_log_filename() API. # # Args: # filename: A logging directory relative filename, # the pid and .log extension will be automatically # appended # # logdir: The path to the log file directory. # # Yields: # The fully qualified log filename # @contextmanager def recorded_messages(self, filename: str, logdir: str) -> Iterator[str]: # We dont allow recursing in this context manager, and # we also do not allow it in the main process. assert not hasattr(self._locals, "log_handle") or self._locals.log_handle is None assert not hasattr(self._locals, "log_filename") or self._locals.log_filename is None # Create the fully qualified logfile in the log directory, # appending the timestamp and .log extension at the end. timestamp = datetime.datetime.now() self._locals.log_filename = os.path.join( logdir, "{}.{}.log".format(filename, timestamp.strftime("%Y%m%d-%H%M%S")) ) # Ensure the directory exists first directory = os.path.dirname(self._locals.log_filename) os.makedirs(directory, exist_ok=True) with open(self._locals.log_filename, "a", encoding="utf-8") as logfile: # Write one last line to the log and flush it to disk def flush_log(): # If the process currently had something happening in the I/O stack # then trying to reenter the I/O stack will fire a runtime error. # # So just try to flush as well as we can at SIGTERM time try: logfile.write("\n\nForcefully terminated\n") logfile.flush() except RuntimeError: os.fsync(logfile.fileno()) # Unconditionally record date and buildstream version at the beginning of any log file # starttime = datetime.datetime.now() logfile.write( "BuildStream {} - {}\n".format(self._bst_version, starttime.strftime("%A, %d-%m-%Y at %H:%M:%S")) ) self._locals.log_handle = logfile with _signals.terminator(flush_log): yield self._locals.log_filename self._locals.log_handle = None self._locals.log_filename = None # get_log_handle() # # Fetches the active log handle, this will return the active # log file handle when the Messenger.recorded_messages() context # manager is active # # Returns: # The active logging file handle, or None # def get_log_handle(self) -> Optional[TextIO]: return self._locals.log_handle # get_log_filename() # # Fetches the active log filename, this will return the active # log filename when the Messenger.recorded_messages() context # manager is active # # Returns: # The active logging filename, or None # def get_log_filename(self) -> Optional[str]: return self._locals.log_filename # timed_suspendable() # # A contextmanager that allows an activity to be suspended and can # adjust for clock drift caused by suspending # # Yields: # An object that contains the time the activity started # @contextmanager def timed_suspendable(self) -> Iterator[_TimeData]: # Note: timedata needs to be in a namedtuple so that values can be # yielded that will change timedata = _TimeData(start_time=datetime.datetime.now()) stopped_time = None def stop_time(): nonlocal stopped_time stopped_time = datetime.datetime.now() def resume_time(): sleep_time = datetime.datetime.now() - stopped_time timedata.start_time += sleep_time with _signals.suspendable(stop_time, resume_time): yield timedata # _silent_messages(): # # Returns: # (bool): Whether messages are currently being silenced # def _silent_messages(self) -> bool: return self._locals.silence_scope_depth > 0 # _record_message() # # Records the message if recording is enabled # # Args: # message: The message to record # def _record_message(self, message: Message) -> None: if self._locals.log_handle is None: return INDENT = " " EMPTYTIME = "--:--:--" template = "[{timecode: <8}] {type: <7}" # If this message is associated with an element or source plugin, print the # full element name and key for the instance. element_key = "" if message.element_key: template += " [{element_key}]" element_key = message.element_key.brief element_name = "" if message.element_name: template += " {element_name}" element_name = message.element_name template += ": {message}" detail = "" if message.detail is not None: template += "\n\n{detail}" detail = message.detail.rstrip("\n") detail = INDENT + INDENT.join(detail.splitlines(True)) timecode = EMPTYTIME if message.message_type in (MessageType.SUCCESS, MessageType.FAIL): assert message.elapsed is not None hours, remainder = divmod(int(message.elapsed.total_seconds()), 60**2) minutes, seconds = divmod(remainder, 60) timecode = "{0:02d}:{1:02d}:{2:02d}".format(hours, minutes, seconds) text = template.format( timecode=timecode, element_key=element_key, element_name=element_name, type=message.message_type.upper(), message=message.message, detail=detail, ) # Write to the open log file self._locals.log_handle.write("{}\n".format(text)) self._locals.log_handle.flush() # _render_status() # # Calls the render status callback set in the messenger, but only if a # second has passed since it last rendered. # def _render_status(self) -> None: assert self._next_render # self._render_status_cb() now = datetime.datetime.now() if self._render_status_cb and now >= self._next_render: self._render_status_cb() self._next_render = now + _RENDER_INTERVAL apache-buildstream-27ae392/src/buildstream/_options/000077500000000000000000000000001514607367700225565ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_options/__init__.py000066400000000000000000000012351514607367700246700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .optionpool import OptionPool apache-buildstream-27ae392/src/buildstream/_options/option.py000066400000000000000000000057601514607367700244500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from typing import TYPE_CHECKING from ..node import _assert_symbol_name if TYPE_CHECKING: from typing import Optional # Shared symbols for validation purposes # OPTION_SYMBOLS = ["type", "description", "variable"] # Option() # # An abstract class representing a project option. # # Concrete classes must be created to handle option types, # the loaded project options is a collection of typed Option # instances. # class Option: # Subclasses use this to specify the type name used # for the yaml format and error messages OPTION_TYPE = None # type: Optional[str] def __init__(self, name, definition, pool): self.name = name self.variable = None self.value = None self.pool = pool self.load(definition) # load() # # Loads the option attributes from the descriptions # in the project.conf # # Args: # node (dict): The loaded YAML dictionary describing # the option def load(self, node): # We don't use the description, but we do require that options have a # description. node.get_str("description") self.variable = node.get_str("variable", default=None) # Assert valid symbol name for variable name if self.variable is not None: _assert_symbol_name(self.variable, "variable name", ref_node=node.get_node("variable")) # load_value() # # Loads the value of the option in string form. # # Args: # node (Mapping): The YAML loaded key/value dictionary # to load the value from # def load_value(self, node): pass # pragma: nocover # set_value() # # Sets the value of an option from a string passed # to buildstream on the command line # # Args: # value (str): The value in string form # def set_value(self, value): pass # pragma: nocover # get_value() # # Gets the value of an option in string form, this # is for the purpose of exporting option values to # variables which must be in string form. # # Returns: # (str): The value in string form # def get_value(self): pass # pragma: nocover # resolve() # # Called on each option once, after all configuration # and cli options have been passed. # def resolve(self): pass # pragma: nocover apache-buildstream-27ae392/src/buildstream/_options/optionarch.py000066400000000000000000000057461514607367700253120ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .._exceptions import LoadError, PlatformError from ..exceptions import LoadErrorReason from .._platform import Platform from .optionenum import OptionEnum # OptionArch # # An enumeration project option which does not allow # definition of a default value, but instead tries to set # the default value to the machine architecture introspected # using `uname` # # Note that when using OptionArch in a project, it will automatically # bail out of the host machine `uname` reports a machine architecture # not supported by the project, in the case that no option was # specifically specified # class OptionArch(OptionEnum): OPTION_TYPE: str = "arch" def load(self, node): super().load_special(node, allow_default_definition=False) def load_default_value(self, node): arch = Platform.get_host_arch() default_value = None for index, value in enumerate(self.values): try: canonical_value = Platform.canonicalize_arch(value) if default_value is None and canonical_value == arch: default_value = value # Do not terminate the loop early to ensure we validate # all values in the list. except PlatformError as e: provenance = node.get_sequence("values").scalar_at(index).get_provenance() prefix = "" if provenance: prefix = "{}: ".format(provenance) raise LoadError( "{}Invalid value for {} option '{}': {}".format(prefix, self.OPTION_TYPE, self.name, e), LoadErrorReason.INVALID_DATA, ) if default_value is None: # Host architecture is not supported by the project. # Do not raise an error here as the user may override it. # If the user does not override it, an error will be raised # by resolve()/validate(). default_value = arch return default_value def resolve(self): # Validate that the default machine arch reported by uname() is # explicitly supported by the project, only if it was not # overridden by user configuration or cli. # # If the value is specified on the cli or user configuration, # then it will already be valid. # self.validate(self.value) apache-buildstream-27ae392/src/buildstream/_options/optionbool.py000066400000000000000000000030131514607367700253110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .._exceptions import LoadError from ..exceptions import LoadErrorReason from .option import Option, OPTION_SYMBOLS # OptionBool # # A boolean project option # class OptionBool(Option): OPTION_TYPE: str = "bool" def load(self, node): super().load(node) node.validate_keys(OPTION_SYMBOLS + ["default"]) self.value = node.get_bool("default") def load_value(self, node): self.value = node.get_bool(self.name) def set_value(self, value): if value in ("True", "true"): self.value = True elif value in ("False", "false"): self.value = False else: raise LoadError( "Invalid value for boolean option {}: {}".format(self.name, value), LoadErrorReason.INVALID_DATA ) def get_value(self): if self.value: return "1" else: return "0" apache-buildstream-27ae392/src/buildstream/_options/optioneltmask.py000066400000000000000000000027211514607367700260230ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .. import utils from .optionflags import OptionFlags # OptionEltMask # # A flags option which automatically only allows element # names as values. # class OptionEltMask(OptionFlags): OPTION_TYPE: str = "element-mask" def load(self, node): # Ask the parent constructor to disallow value definitions, # we define those automatically only. super().load_special(node, allow_value_definitions=False) # Here we want all valid elements as possible values, # but we'll settle for just the relative filenames # of files ending with ".bst" in the project element directory def load_valid_values(self, node): values = [] for filename in utils.list_relative_paths(self.pool.element_path): if filename.endswith(".bst"): values.append(filename) return values apache-buildstream-27ae392/src/buildstream/_options/optionenum.py000066400000000000000000000053371514607367700253350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .._exceptions import LoadError from ..exceptions import LoadErrorReason from .option import Option, OPTION_SYMBOLS # OptionEnum # # An enumeration project option # class OptionEnum(Option): OPTION_TYPE: str = "enum" def __init__(self, name, definition, pool): self.values = None super().__init__(name, definition, pool) def load(self, node): self.load_special(node) def load_special(self, node, allow_default_definition=True): super().load(node) valid_symbols = OPTION_SYMBOLS + ["values"] if allow_default_definition: valid_symbols += ["default"] node.validate_keys(valid_symbols) self.values = node.get_str_list("values", default=[]) if not self.values: raise LoadError( "{}: No values specified for {} option '{}'".format( node.get_provenance(), self.OPTION_TYPE, self.name ), LoadErrorReason.INVALID_DATA, ) # Allow subclass to define the default value self.value = self.load_default_value(node) def load_value(self, node): value_node = node.get_scalar(self.name) self.value = value_node.as_str() self.validate(self.value, value_node) def set_value(self, value): self.validate(value) self.value = value def get_value(self): return self.value def validate(self, value, node=None): if value not in self.values: if node is not None: provenance = node.get_provenance() prefix = "{}: ".format(provenance) else: prefix = "" raise LoadError( "{}Invalid value for {} option '{}': {}\n".format(prefix, self.OPTION_TYPE, self.name, value) + "Valid values: {}".format(", ".join(self.values)), LoadErrorReason.INVALID_DATA, ) def load_default_value(self, node): value_node = node.get_scalar("default") value = value_node.as_str() self.validate(value, value_node) return value apache-buildstream-27ae392/src/buildstream/_options/optionflags.py000066400000000000000000000062151514607367700254610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .._exceptions import LoadError from ..exceptions import LoadErrorReason from .option import Option, OPTION_SYMBOLS # OptionFlags # # A flags project option # class OptionFlags(Option): OPTION_TYPE: str = "flags" def __init__(self, name, definition, pool): self.values = None super().__init__(name, definition, pool) def load(self, node): self.load_special(node) def load_special(self, node, allow_value_definitions=True): super().load(node) valid_symbols = OPTION_SYMBOLS + ["default"] if allow_value_definitions: valid_symbols += ["values"] node.validate_keys(valid_symbols) # Allow subclass to define the valid values self.values = self.load_valid_values(node) if not self.values: raise LoadError( "{}: No values specified for {} option '{}'".format( node.get_provenance(), self.OPTION_TYPE, self.name ), LoadErrorReason.INVALID_DATA, ) value_node = node.get_sequence("default", default=[]) self.value = value_node.as_str_list() self.validate(self.value, value_node) def load_value(self, node): value_node = node.get_sequence(self.name) self.value = sorted(value_node.as_str_list()) self.validate(self.value, value_node) def set_value(self, value): # Strip out all whitespace, allowing: "value1, value2 , value3" stripped = "".join(value.split()) # Get the comma separated values list_value = stripped.split(",") self.validate(list_value) self.value = sorted(list_value) def get_value(self): return ",".join(self.value) def validate(self, value, node=None): for flag in value: if flag not in self.values: if node is not None: provenance = node.get_provenance() prefix = "{}: ".format(provenance) else: prefix = "" raise LoadError( "{}Invalid value for flags option '{}': {}\n".format(prefix, self.name, value) + "Valid values: {}".format(", ".join(self.values)), LoadErrorReason.INVALID_DATA, ) def load_valid_values(self, node): # Allow the more descriptive error to raise when no values # exist rather than bailing out here (by specifying default_value) return node.get_str_list("values", default=[]) apache-buildstream-27ae392/src/buildstream/_options/optionos.py000066400000000000000000000021721514607367700250040ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman import platform from .optionenum import OptionEnum # OptionOS # class OptionOS(OptionEnum): OPTION_TYPE: str = "os" def load(self, node): super().load_special(node, allow_default_definition=False) def load_default_value(self, node): return platform.uname().system def resolve(self): # Validate that the default OS reported by uname() is explicitly # supported by the project, if not overridden by user config or cli. self.validate(self.value) apache-buildstream-27ae392/src/buildstream/_options/optionpool.py000066400000000000000000000261371514607367700253430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # import jinja2 from .._exceptions import LoadError from ..exceptions import LoadErrorReason from ..node import MappingNode, SequenceNode, _assert_symbol_name from ..types import FastEnum from .optionbool import OptionBool from .optionenum import OptionEnum from .optionflags import OptionFlags from .optioneltmask import OptionEltMask from .optionarch import OptionArch from .optionos import OptionOS _OPTION_TYPES = { OptionBool.OPTION_TYPE: OptionBool, OptionEnum.OPTION_TYPE: OptionEnum, OptionFlags.OPTION_TYPE: OptionFlags, OptionEltMask.OPTION_TYPE: OptionEltMask, OptionArch.OPTION_TYPE: OptionArch, OptionOS.OPTION_TYPE: OptionOS, } class OptionTypes(FastEnum): BOOL = OptionBool.OPTION_TYPE ENUM = OptionEnum.OPTION_TYPE FLAG = OptionFlags.OPTION_TYPE ELT_MASK = OptionEltMask.OPTION_TYPE ARCH = OptionArch.OPTION_TYPE OS = OptionOS.OPTION_TYPE class OptionPool: def __init__(self, element_path): # We hold on to the element path for the sake of OptionEltMask self.element_path = element_path # # Private members # self._options = {} # The Options self._variables = None # The Options resolved into typed variables self._environment = None self._init_environment() # load() # # Loads the options described in the project.conf # # Args: # node (dict): The loaded YAML options # def load(self, options): for option_name, option_definition in options.items(): # Assert that the option name is a valid symbol _assert_symbol_name(option_name, "option name", ref_node=option_definition, allow_dashes=False) opt_type_name = option_definition.get_enum("type", OptionTypes) opt_type = _OPTION_TYPES[opt_type_name.value] option = opt_type(option_name, option_definition, self) self._options[option_name] = option # load_yaml_values() # # Loads the option values specified in a key/value # dictionary loaded from YAML # # Args: # node (dict): The loaded YAML options # def load_yaml_values(self, node): for option_name, option_value in node.items(): try: option = self._options[option_name] except KeyError as e: p = option_value.get_provenance() raise LoadError( "{}: Unknown option '{}' specified".format(p, option_name), LoadErrorReason.INVALID_DATA ) from e option.load_value(node) # load_cli_values() # # Loads the option values specified in a list of tuples # collected from the command line # # Args: # cli_options (list): A list of (str, str) tuples # ignore_unknown (bool): Whether to silently ignore unknown options. # def load_cli_values(self, cli_options, *, ignore_unknown=False): for option_name, option_value in cli_options: try: option = self._options[option_name] except KeyError as e: if not ignore_unknown: raise LoadError( "Unknown option '{}' specified on the command line".format(option_name), LoadErrorReason.INVALID_DATA, ) from e else: option.set_value(option_value) # resolve() # # Resolves the loaded options, this is just a step which must be # performed after loading all options and their values, and before # ever trying to evaluate an expression # def resolve(self): self._variables = {} for option_name, option in self._options.items(): # Delegate one more method for options to # do some last minute validation once any # overrides have been performed. # option.resolve() self._variables[option_name] = option.value # export_variables() # # Exports the option values which are declared # to be exported, to the passed dictionary. # # Variable values are exported in string form # # Args: # variables (dict): A variables dictionary # def export_variables(self, variables): for _, option in self._options.items(): if option.variable: variables[option.variable] = option.get_value() # printable_variables() # # Exports all option names and string values # to the passed dictionary in alphabetical order. # # Args: # variables (dict): A variables dictionary # def printable_variables(self, variables): for key in sorted(self._options): variables[key] = self._options[key].get_value() # process_node() # # Args: # node (node): A YAML Loaded dictionary # restricted (List[str]): A list of restricted keys # # Restricted keys, if found in a conditional block, will raise # an error if specified such that they would be composited at the # root of "node", # def process_node(self, node, *, restricted=None): # A conditional will result in composition, which can # in turn add new conditionals to the root. # # Keep processing conditionals on the root node until # all directly nested conditionals are resolved. # while self._process_one_node(node, restricted=restricted): pass # Now recurse into nested dictionaries and lists # and process any indirectly nested conditionals. # for value in node.values(): value_type = type(value) if value_type is MappingNode: self.process_node(value) elif value_type is SequenceNode: self._process_list(value) ####################################################### # Private Methods # ####################################################### # _evaluate() # # Evaluates a jinja2 style expression with the loaded options in context. # # Args: # expression (str): The jinja2 style expression # # Returns: # (bool): Whether the expression resolved to a truthy value or a falsy one. # # Raises: # LoadError: If the expression failed to resolve for any reason # def _evaluate(self, expression): # # Variables must be resolved at this point. # try: template_string = "{{% if {} %}} True {{% else %}} False {{% endif %}}".format(expression) template = self._environment.from_string(template_string) context = template.new_context(self._variables, shared=True) result = template.root_render_func(context) evaluated = jinja2.utils.concat(result) val = evaluated.strip() if val == "True": return True elif val == "False": return False else: # pragma: nocover raise LoadError( "Failed to evaluate expression: {}".format(expression), LoadErrorReason.EXPRESSION_FAILED ) except jinja2.exceptions.TemplateError as e: raise LoadError( "Failed to evaluate expression ({}): {}".format(expression, e), LoadErrorReason.EXPRESSION_FAILED ) # Recursion assistent for lists, in case there # are lists of lists. # def _process_list(self, values): for value in values: value_type = type(value) if value_type is MappingNode: self.process_node(value) elif value_type is SequenceNode: self._process_list(value) # Process a single conditional, resulting in composition # at the root level on the passed node # # Return true if a conditional was processed. # def _process_one_node(self, node, *, restricted=None): conditions = node.get_sequence("(?)", default=None) assertion = node.get_str("(!)", default=None) # Process assersions first, we want to abort on the first encountered # assertion in a given dictionary, and not lose an assertion due to # it being overwritten by a later assertion which might also trigger. if assertion is not None: p = node.get_scalar("(!)").get_provenance() raise LoadError("{}: {}".format(p, assertion.strip()), LoadErrorReason.USER_ASSERTION) if conditions is not None: del node["(?)"] for condition in conditions: tuples = list(condition.items()) if len(tuples) > 1: provenance = condition.get_provenance() raise LoadError( "{}: Conditional statement has more than one key".format(provenance), LoadErrorReason.INVALID_DATA, ) expression, value = tuples[0] try: apply_fragment = self._evaluate(expression) except LoadError as e: # Prepend the provenance of the error provenance = condition.get_provenance() raise LoadError("{}: {}".format(provenance, e), e.reason) from e if type(value) is not MappingNode: # pylint: disable=unidiomatic-typecheck provenance = condition.get_provenance() raise LoadError( "{}: Only values of type 'dict' can be composed.".format(provenance), LoadErrorReason.ILLEGAL_COMPOSITE, ) # Observe restricted keys that are not allowed to be conditional for key in restricted or []: if key in value: provenance = value.get_node(key).get_provenance() raise LoadError( "{}: The '{}' key cannot be specified conditionally.".format(provenance, key), LoadErrorReason.ILLEGAL_COMPOSITE, ) # Apply the yaml fragment if its condition evaluates to true if apply_fragment: value._composite(node) return True return False def _init_environment(self): # jinja2 environment, with default globals cleared out of the way self._environment = jinja2.Environment(undefined=jinja2.StrictUndefined) self._environment.globals = [] apache-buildstream-27ae392/src/buildstream/_overlapcollector.py000066400000000000000000000311531514607367700250170ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os from contextlib import contextmanager from typing import TYPE_CHECKING, Optional, List, Tuple from .plugin import Plugin from .types import CoreWarnings, OverlapAction from .utils import FileListResult if TYPE_CHECKING: from typing import Dict # pylint: disable=cyclic-import from .element import Element # pylint: enable=cyclic-import # OverlapCollector() # # Collects results of Element.stage_artifact() and saves # them in order to raise a proper overlap error at the end # of staging. # # Args: # element (Element): The element for which we are staging artifacts # class OverlapCollector: def __init__(self, element: "Element"): # The Element we are staging for, on which we'll issue warnings self._element = element # type: Element # The list of sessions self._sessions = [] # type: List[OverlapCollectorSession] # The active session, if any self._session = None # type: Optional[OverlapCollectorSession] # session() # # Create a session for collecting overlaps, calls to OverlapCollector.collect_stage_result() # are expected to always occur within the context of a session (this context manager). # # Upon exiting this context, warnings and/or errors will be issued for any overlaps # which occurred either as a result of overlapping files within this session, or # as a result of files staged during this session, overlapping with files staged in # previous sessions in this OverlapCollector. # # Args: # action (OverlapAction): The action to take for this overall session's overlaps with other sessions # location (str): The Sandbox relative location this session was created for # @contextmanager def session(self, action: str, location: Optional[str]): assert self._session is None, "Stage session already started" if location is None: location = "/" self._session = OverlapCollectorSession(self._element, action, location) # Run code body where staging results can be collected. yield # Issue warnings for the current session, passing along previously completed sessions self._session.warnings(self._sessions) # Store the newly ended session and end the session self._sessions.append(self._session) self._session = None # collect_stage_result() # # Collect and accumulate results of Element.stage_artifact() # # Args: # element (Element): The name of the element staged # result (FileListResult): The result of Element.stage_artifact() # def collect_stage_result(self, element: "Element", result: FileListResult): assert self._session is not None, "Staging files outside of staging session" self._session.collect_stage_result(element, result) # OverlapCollectorSession() # # Collect the results of a single session # # Args: # element (Element): The element for which we are staging artifacts # action (OverlapAction): The action to take for this overall session's overlaps with other sessions # location (str): The Sandbox relative location this session was created for # class OverlapCollectorSession: def __init__(self, element: "Element", action: str, location: str): # The Element we are staging for, on which we'll issue warnings self._element = element # type: Element # The OverlapAction for this session self._action = action # type: str # The Sandbox relative directory this session was created for self._location = location # type: str # Dictionary of files which were ignored (See FileListResult()), keyed by element unique ID self._ignored = {} # type: Dict[int, List[str]] # Dictionary of files which were staged, keyed by element unique ID self._files_written = {} # type: Dict[int, List[str]] # Dictionary of element IDs which overlapped, keyed by the file they overlap on self._overlaps = {} # type: Dict[str, List[int]] # collect_stage_result() # # Collect and accumulate results of Element.stage_artifact() # # Args: # element (Element): The name of the element staged # result (FileListResult): The result of Element.stage_artifact() # def collect_stage_result(self, element: "Element", result: FileListResult): for overwritten_file in result.overwritten: overlap_list = None try: overlap_list = self._overlaps[overwritten_file] except KeyError: # Create a fresh list # self._overlaps[overwritten_file] = overlap_list = [] # Search files which were staged in this session, start the # list off with the bottom most element # for element_id, staged_files in self._files_written.items(): if overwritten_file in staged_files: overlap_list.append(element_id) break # Add the currently staged element to the overlap list, it might be # the only element in the list if it overlaps with a file staged # from a previous session. # overlap_list.append(element._unique_id) # Record written files and ignored files. # self._files_written[element._unique_id] = result.files_written if result.ignored: self._ignored[element._unique_id] = result.ignored # warnings() # # Issue any warnings as a batch as a result of staging artifacts, # based on the results collected with collect_stage_result(). # # Args: # sessions (list): List of previously completed sessions # def warnings(self, sessions: List["OverlapCollectorSession"]): # Collect a table of filenames which overlapped something from outside of this session. # external_overlaps = {} # type: Dict[str, int] # # First issue the warnings for this session # if self._overlaps: overlap_warning = False detail = "Staged files overwrite existing files in staging area: {}\n".format(self._location) for filename, element_ids in self._overlaps.items(): # If there is only one element in the overlap list, it means it has # overlapped a file from a previous session. # # Ignore it and handle the warning below # if len(element_ids) == 1: external_overlaps[filename] = element_ids[0] continue # Filter whitelisted elements out of the list of overlapping elements # # Ignore the bottom-most element as it does not overlap anything. # overlapping_element_ids = element_ids[1:] warning_elements = self._filter_whitelisted(filename, overlapping_element_ids) if warning_elements: overlap_warning = True detail += self._overlap_detail(filename, warning_elements, element_ids) if overlap_warning: self._element.warn( "Non-whitelisted overlaps detected", detail=detail, warning_token=CoreWarnings.OVERLAPS ) if self._ignored: detail = "Not staging files which would replace non-empty directories in staging area: {}\n".format( self._location ) for element_id, ignored_filenames in self._ignored.items(): element = Plugin._lookup(element_id) detail += "\nFrom {}:\n".format(element._get_full_name()) detail += " " + " ".join( ["{}\n".format(os.path.join(self._location, filename)) for filename in ignored_filenames] ) self._element.warn( "Not staging files which would have replaced non-empty directories", detail=detail, warning_token=CoreWarnings.UNSTAGED_FILES, ) if external_overlaps and self._action != OverlapAction.IGNORE: detail = "Detected file overlaps while staging elements into: {}\n".format(self._location) # Find the session responsible for the overlap # for filename, element_id in external_overlaps.items(): absolute_filename = os.path.join(self._location, filename) overlapped_id, location = self._search_stage_element(absolute_filename, sessions) element = Plugin._lookup(element_id) overlapped = Plugin._lookup(overlapped_id) detail += "{}: {} overlaps files previously staged by {} in: {}\n".format( absolute_filename, element._get_full_name(), overlapped._get_full_name(), location ) if self._action == OverlapAction.WARNING: self._element.warn("Overlaps detected", detail=detail, warning_token=CoreWarnings.OVERLAPS) else: from .element import ElementError raise ElementError("Overlaps detected", detail=detail, reason="overlaps") # _search_stage_element() # # Search the sessions list for the element responsible for staging the given file # # Args: # filename (str): The sandbox relative file which was overwritten # sessions (List[OverlapCollectorSession]) # # Returns: # element_id (int): The unique ID of the element responsible # location (str): The sandbox relative staging location where element_id was staged # def _search_stage_element(self, filename: str, sessions: List["OverlapCollectorSession"]) -> Tuple[int, str]: for session in reversed(sessions): for element_id, staged_files in session._files_written.items(): if any( staged_file for staged_file in staged_files if os.path.join(session._location, staged_file) == filename ): return element_id, session._location assert False, "Could not find element responsible for staging: {}".format(filename) # Silence the linter with an unreachable return statement return None, None # _filter_whitelisted() # # Args: # filename (str): The staging session relative filename # element_ids (List[int]): Ordered list of elements # # Returns: # (List[Element]): The list of element objects which are not whitelisted # def _filter_whitelisted(self, filename: str, element_ids: List[int]): overlap_elements = [] for element_id in element_ids: element = Plugin._lookup(element_id) if not element._file_is_whitelisted(filename): overlap_elements.append(element) return overlap_elements # _overlap_detail() # # Get a string to describe overlaps on a filename # # Args: # filename (str): The filename being overlapped # overlap_elements (List[Element]): A list of Elements overlapping # element_ids (List[int]): The ordered ID list of elements which staged this file # def _overlap_detail(self, filename, overlap_elements, element_ids): filename = os.path.join(self._location, filename) if overlap_elements: overlap_element_names = [element._get_full_name() for element in overlap_elements] overlap_order_elements = [Plugin._lookup(element_id) for element_id in element_ids] overlap_order_names = [element._get_full_name() for element in overlap_order_elements] return "{}: {} {} not permitted to overlap other elements, order {} \n".format( filename, " and ".join(overlap_element_names), "is" if len(overlap_element_names) == 1 else "are", " above ".join(reversed(overlap_order_names)), ) else: return "" apache-buildstream-27ae392/src/buildstream/_pipeline.py000066400000000000000000000304631514607367700232500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # Tristan Maat import itertools from collections import OrderedDict from operator import itemgetter from typing import List, Iterator from pyroaring import BitMap # pylint: disable=no-name-in-module from .element import Element from .types import _PipelineSelection, _Scope from ._context import Context from ._exceptions import PipelineError # dependencies() # # Generator function to iterate over the dependencies of multiple # targets in the specified scope, while guaranteeing that a given # element is never yielded more than once. # # Args: # targets: The target Elements to loop over # scope: An integer value from the _Scope enum, the scope to iterate over # recurse: Whether to recurse into dependencies # # Yields: # Elements in the scope of the specified target elements # def dependencies(targets: List[Element], scope: int, *, recurse: bool = True) -> Iterator[Element]: # Keep track of 'visited' in this scope, so that all targets # share the same context. visited = (BitMap(), BitMap()) for target in targets: yield from target._dependencies(scope, recurse=recurse, visited=visited) # get_selection() # # Gets a full list of elements based on a toplevel # list of element targets # # Various commands define a --deps option to specify what elements to # use in the result, this function reports a list that is appropriate for # the selected option. # # Args: # context: The invocation context # targets: The target Elements # mode: A value from PipelineSelection enumeration # silent: Whether to silence messages # depth_sort: Whether to sort the elements by depth (for an optimal build plan) # # Returns: # A list of Elements appropriate for the specified selection mode # def get_selection( context: Context, targets: List[Element], mode: str, *, silent: bool = True, depth_sort: bool = False ) -> List[Element]: def redirect_and_log() -> List[Element]: # Redirect and log if permitted elements: List[Element] = [] for t in targets: new_elm = t._get_source_element() if new_elm != t and not silent: context.messenger.info("Element '{}' redirected to '{}'".format(t.name, new_elm.name)) if new_elm not in elements: elements.append(new_elm) return elements def plan_all() -> List[Element]: return _Planner().plan(targets) def plan_build() -> List[Element]: build_targets = list(dependencies(targets, _Scope.BUILD, recurse=False)) return _Planner().plan(build_targets) selection_table = { _PipelineSelection.REDIRECT: redirect_and_log, } if depth_sort: # # Depth sorting is used with `bst build` and assumes a dynamic build planning # mode (Stream() will only mark the toplevel elements as "required", and all # elements will be built on demand). # # In this case, the `none` and `run` selection modes can potentially include # dependencies, and which ones will be dynamically resolved at build time, so # it is essentially equivalent to the `all` selection. # selection_table[_PipelineSelection.NONE] = plan_all selection_table[_PipelineSelection.ALL] = plan_all selection_table[_PipelineSelection.RUN] = plan_all selection_table[_PipelineSelection.BUILD] = plan_build else: selection_table[_PipelineSelection.NONE] = lambda: targets selection_table[_PipelineSelection.ALL] = lambda: list(dependencies(targets, _Scope.ALL)) selection_table[_PipelineSelection.RUN] = lambda: list(dependencies(targets, _Scope.RUN)) selection_table[_PipelineSelection.BUILD] = lambda: list(dependencies(targets, _Scope.BUILD)) return selection_table[mode]() # except_elements(): # # This function calculates the intersection of the `except_targets` # element dependencies and the `targets` dependencies, and removes # that intersection from the `elements` list, returning the result. # # Args: # targets: List of toplevel targetted elements # elements: The list to remove elements from # except_targets: List of toplevel except targets # # Returns: # The elements list with the intersected exceptions removed # # Important notes on the behavior # =============================== # # * Except elements can be completely outside of the scope # of targets. # # * When the dependencies of except elements intersect with # dependencies of targets, those dependencies are removed # from the result. # # * If a target is found within the intersection of excepted # elements, that target and it's dependencies are considered # exempt from the exception intersection. # # Example: # # (t1) (e1) # / \ / # (o) (o) ( ) # / \ / \ # (o) (x) ( ) # \ / \ # (o) (x) ( ) # \ / # (x) # / \ # (x) (t2) # / \ / \ # (x) (x) (o) # / \ # (o) (o) # # Here we have a mockup graph with 2 target elements (t1) and (t2), # and one except element (e1) which lies outside of the graph. # # - ( ) elements are ignored, they were never in the element list # - (o) elements will be included in the result # - (x) elements are removed from the graph # # Note how (t2) reintroduces portions of the graph which were otherwise # tainted by being depended on indirectly by the (e1) except element. # def except_elements(targets: List[Element], elements: List[Element], except_targets: List[Element]) -> List[Element]: if not except_targets: return elements targeted: List[Element] = list(dependencies(targets, _Scope.ALL)) visited: List[Element] = [] def find_intersection(element: Element) -> Iterator[Element]: if element in visited: return visited.append(element) # Intersection elements are those that are also in # 'targeted', as long as we don't recurse into them. if element in targeted: yield element else: for dep in element._dependencies(_Scope.ALL, recurse=False): yield from find_intersection(dep) # Build a list of 'intersection' elements, i.e. the set of # elements that lie on the border closest to excepted elements # between excepted and target elements. intersection = list(itertools.chain.from_iterable(find_intersection(element) for element in except_targets)) # Now use this set of elements to traverse the targeted # elements, except 'intersection' elements and their unique # dependencies. queue = [] visited = [] queue.extend(targets) while queue: element = queue.pop() if element in visited or element in intersection: continue visited.append(element) queue.extend(element._dependencies(_Scope.ALL, recurse=False)) # That looks like a lot, but overall we only traverse (part # of) the graph twice. This could be reduced to once if we # kept track of parent elements, but is probably not # significant. # Ensure that we return elements in the same order they were # in before. return [element for element in elements if element in visited] # assert_consistent() # # Asserts that the given list of elements are in a consistent state, that # is to say that all sources are consistent and can at least be fetched. # # Consequently it also means that cache keys can be resolved. # # Args: # context: The invocation context # elements: The elements to assert consistency on # # Raises: # PipelineError: If the elements are inconsistent. # def assert_consistent(context: Context, elements: List[Element]) -> None: inconsistent = [] inconsistent_workspaced = [] for element in elements: if not element._has_all_sources_resolved(): if element._get_workspace(): inconsistent_workspaced.append(element) else: inconsistent.append(element) if inconsistent: detail = "Exact versions are missing for the following elements:\n\n" for element in inconsistent: detail += " Element: {} is inconsistent\n".format(element._get_full_name()) for source in element.sources(): if not source.is_resolved(): detail += " {} is missing ref\n".format(source) detail += "\n" detail += "Try tracking these elements first with `bst source track`\n" raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline") if inconsistent_workspaced: detail = "Some workspaces exist but are not closed\n" + "Try closing them with `bst workspace close`\n\n" for element in inconsistent_workspaced: detail += " " + element._get_full_name() + "\n" raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced") # assert_sources_cached() # # Asserts that sources for the given list of elements are cached. # # Args: # context: The invocation context # elements: The elements to assert cached source state for # # Raises: # PipelineError: If the elements have uncached sources # def assert_sources_cached(context: Context, elements: List[Element]): uncached = [] with context.messenger.timed_activity("Checking sources"): for element in elements: if element._fetch_needed(): uncached.append(element) if uncached: detail = "Sources are not cached for the following elements:\n\n" for element in uncached: detail += " Following sources for element: {} are not cached:\n".format(element._get_full_name()) for source in element.sources(): if not source._is_cached(): detail += " {}\n".format(source) detail += "\n" detail += ( "Try fetching these elements first with `bst source fetch`,\n" + "or run this command with `--fetch` option\n" ) raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources") # _Planner() # # An internal object used for constructing build plan # from a given resolved toplevel element, using depth # sorting for more efficient processing. # class _Planner: def __init__(self): self.depth_map = OrderedDict() self.visiting_elements = set() # Here we want to traverse the same element more than once when # it is reachable from multiple places, with the interest of finding # the deepest occurance of every element def plan_element(self, element, depth): if element in self.visiting_elements: # circular dependency, already being processed return prev_depth = self.depth_map.get(element) if prev_depth is not None and prev_depth >= depth: # element and dependencies already processed at equal or greater depth return self.visiting_elements.add(element) for dep in element._dependencies(_Scope.RUN, recurse=False): self.plan_element(dep, depth) for dep in element._dependencies(_Scope.BUILD, recurse=False): self.plan_element(dep, depth + 1) self.depth_map[element] = depth self.visiting_elements.remove(element) def plan(self, roots): for root in roots: self.plan_element(root, 0) depth_sorted = sorted(self.depth_map.items(), key=itemgetter(1), reverse=True) # Set the depth of each element for index, item in enumerate(depth_sorted): item[0]._set_depth(index) return [item[0] for item in depth_sorted] apache-buildstream-27ae392/src/buildstream/_platform/000077500000000000000000000000001514607367700227075ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_platform/__init__.py000066400000000000000000000012161514607367700250200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat from .platform import Platform apache-buildstream-27ae392/src/buildstream/_platform/platform.py000066400000000000000000000133541514607367700251130ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat import os import platform import psutil from .._exceptions import PlatformError, SandboxUnavailableError from ..sandbox import SandboxDummy class Platform: # Platform() # # A class to manage platform-specific details. Currently holds the # sandbox factory as well as platform helpers. # def __init__(self): self._use_dummy_sandbox = False self._use_dummy_sandbox_reason = None self._setup_sandbox() def _setup_sandbox(self): from ..sandbox._sandboxbuildboxrun import SandboxBuildBoxRun # pylint: disable=cyclic-import try: SandboxBuildBoxRun._setup() except SandboxUnavailableError as e: self._use_dummy_sandbox = True self._use_dummy_sandbox_reason = str(e) @classmethod def create_instance(cls) -> "Platform": return Platform() def get_cpu_count(self, cap=None): # `psutil.Process.cpu_affinity()` is not available on all platforms. # So, fallback to getting the total cpu count in cases where it is not # available. dummy_process = psutil.Process() if hasattr(dummy_process, "cpu_affinity"): cpu_count = len(dummy_process.cpu_affinity()) else: cpu_count = os.cpu_count() if cap is None: return cpu_count else: return min(cpu_count, cap) @staticmethod def get_host_os(): system = platform.uname().system.lower() if system == "darwin" and platform.mac_ver()[0]: # mac_ver() returns a non-empty release string on macOS. return "macos" else: return system # canonicalize_arch(): # # This returns the canonical, OS-independent architecture name # or raises a PlatformError if the architecture is unknown. # @staticmethod def canonicalize_arch(arch): # Note that these are all expected to be lowercase, as we want a # case-insensitive lookup. Windows can report its arch in ALLCAPS. aliases = { "aarch32": "aarch32", "aarch64": "aarch64", "aarch64-be": "aarch64-be", "amd64": "x86-64", "arm": "aarch32", "armv8l": "aarch64", "armv8b": "aarch64-be", "i386": "x86-32", "i486": "x86-32", "i586": "x86-32", "i686": "x86-32", "la64v100": "la64v100", "loong64": "la64v100", "loongarch64": "la64v100", "power-isa-be": "power-isa-be", "power-isa-le": "power-isa-le", "powerpc": "power-isa-be", "powerpc64": "power-isa-be", # Used in GCC/LLVM "powerpc64le": "power-isa-le", # Used in GCC/LLVM "ppc64": "power-isa-be", "ppc64le": "power-isa-le", "riscv32": "rv32g", "riscv64": "rv64g", "rv32g": "rv32g", "rv64g": "rv64g", "sparc": "sparc-v9", "sparc64": "sparc-v9", "sparc-v9": "sparc-v9", "sun4v": "sparc-v9", "x86-32": "x86-32", "x86-64": "x86-64", } try: return aliases[arch.replace("_", "-").lower()] except KeyError: raise PlatformError("Unknown architecture: {}".format(arch)) # get_host_arch(): # # This returns the architecture of the host machine. The possible values # map from uname -m in order to be a OS independent list. # # Returns: # (string): String representing the architecture @staticmethod def get_host_arch(): uname = platform.uname() if uname.system.lower() == "aix": # IBM AIX systems reports their serial number as the machine # hardware identifier. So, we need to look at the reported processor # in this case. return Platform.canonicalize_arch(uname.processor) else: # Otherwise, use the hardware identifier from uname return Platform.canonicalize_arch(uname.machine) ################################################################## # Sandbox functions # ################################################################## # create_sandbox(): # # Create a build sandbox suitable for the environment # # Args: # args (dict): The arguments to pass to the sandbox constructor # kwargs (file): The keyword arguments to pass to the sandbox constructor # # Returns: # (Sandbox) A sandbox # def create_sandbox(self, *args, **kwargs): # pylint: disable=method-hidden from ..sandbox._sandboxbuildboxrun import SandboxBuildBoxRun # pylint: disable=cyclic-import if self._use_dummy_sandbox: kwargs["dummy_reason"] = self._use_dummy_sandbox_reason else: try: SandboxBuildBoxRun.check_sandbox_config(kwargs["config"]) return SandboxBuildBoxRun(*args, **kwargs) except SandboxUnavailableError as e: kwargs["dummy_reason"] = str(e) return SandboxDummy(*args, **kwargs) apache-buildstream-27ae392/src/buildstream/_pluginfactory/000077500000000000000000000000001514607367700237515ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_pluginfactory/__init__.py000066400000000000000000000032211514607367700260600ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from .pluginorigin import PluginOrigin, PluginOriginType, PluginType from .pluginoriginlocal import PluginOriginLocal from .pluginoriginpip import PluginOriginPip from .pluginoriginjunction import PluginOriginJunction from .sourcefactory import SourceFactory from .elementfactory import ElementFactory from .sourcemirrorfactory import SourceMirrorFactory # load_plugin_origin() # # Load a PluginOrigin from the YAML in project.conf # # Args: # project (Project): The project from whence this origin is loaded # origin_node (MappingNode): The node defining this origin # # Returns: # (PluginOrigin): The newly created PluginOrigin # def load_plugin_origin(project, origin_node): origin_type = origin_node.get_enum("origin", PluginOriginType) if origin_type == PluginOriginType.LOCAL: origin = PluginOriginLocal() elif origin_type == PluginOriginType.PIP: origin = PluginOriginPip() elif origin_type == PluginOriginType.JUNCTION: origin = PluginOriginJunction() else: assert False, "unreachable" origin.initialize(project, origin_node) return origin apache-buildstream-27ae392/src/buildstream/_pluginfactory/elementfactory.py000066400000000000000000000037501514607367700273510ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from typing import TYPE_CHECKING, Type, cast from .pluginfactory import PluginFactory from .pluginorigin import PluginType from .._loader import LoadElement from ..element import Element if TYPE_CHECKING: from .._context import Context from .._project import Project # A ElementFactory creates Element instances # in the context of a given factory # # Args: # plugin_base (PluginBase): The main PluginBase object to work with # class ElementFactory(PluginFactory): def __init__(self, plugin_base): super().__init__(plugin_base, PluginType.ELEMENT) # create(): # # Create an Element object. # # Args: # context (object): The Context object for processing # project (object): The project object # load_element (object): The LoadElement # # Returns: A newly created Element object of the appropriate kind # # Raises: # PluginError (if the kind lookup failed) # LoadError (if the element itself took issue with the config) # def create(self, context: "Context", project: "Project", load_element: LoadElement) -> Element: plugin_type, default_config = self.lookup(context.messenger, load_element.kind, load_element.node) element_type = cast(Type[Element], plugin_type) element = element_type(context, project, load_element, default_config) return element apache-buildstream-27ae392/src/buildstream/_pluginfactory/pluginfactory.py000066400000000000000000000357151514607367700272240ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os from typing import Tuple, Type, Iterator, Optional from pluginbase import PluginSource from .. import utils from .. import _site from ..plugin import Plugin from ..source import Source from ..sourcemirror import SourceMirror from ..element import Element from ..node import Node from ..utils import UtilError from .._exceptions import PluginError from .._messenger import Messenger from .pluginorigin import PluginOrigin, PluginType # A Context for loading plugin types # # Args: # plugin_base (PluginBase): The main PluginBase object to work with # plugin_type (PluginType): The type of plugin to load # # Since multiple pipelines can be processed recursively # within the same interpretor, it's important that we have # one context associated to the processing of a given pipeline, # this way sources and element types which are particular to # a given BuildStream project are isolated to their respective # Pipelines. # class PluginFactory: def __init__(self, plugin_base, plugin_type): # For pickling across processes, make sure this context has a unique # identifier, which we prepend to the identifier of each PluginSource. # This keeps plugins loaded during the first and second pass distinct # from eachother. self._identifier = str(id(self)) self._plugin_type = plugin_type # The kind of plugins this factory loads self._types = {} # Plugin type lookup table by kind self._origins = {} # PluginOrigin lookup table by kind self._allow_deprecated = {} # Lookup table to check if a plugin is allowed to be deprecated self._plugin_base = plugin_base # The PluginBase object # The PluginSource objects need to be kept in scope for the lifetime # of the loaded plugins, otherwise the PluginSources delete the plugin # modules when they go out of scope. # # FIXME: Instead of keeping this table, we can call: # # PluginBase.make_plugin_source(..., persist=True) # # The persist attribute avoids this behavior. This is not currently viable # because the BuildStream data model (projects and elements) does not properly # go out of scope when the CLI completes, causing errors to occur when # invoking BuildStream multiple times during tests. # self._sources = {} # A mapping of (location, kind) -> PluginSource objects self._init_site_source() # Initialize the PluginSource object for core plugins def _init_site_source(self): if self._plugin_type == PluginType.SOURCE: self._site_plugins_path = _site.source_plugins elif self._plugin_type == PluginType.ELEMENT: self._site_plugins_path = _site.element_plugins elif self._plugin_type == PluginType.SOURCE_MIRROR: self._site_plugins_path = _site.source_mirror_plugins self._site_source = self._plugin_base.make_plugin_source( searchpath=[self._site_plugins_path], identifier=self._identifier + "site", ) ###################################################### # Public Methods # ###################################################### # register_plugin_origin(): # # Registers the PluginOrigin to use for the given plugin kind # # Args: # kind (str): The kind identifier of the Plugin # origin (PluginOrigin): The PluginOrigin providing the plugin # allow_deprecated (bool): Whether this plugin kind is allowed to be used in a deprecated state # def register_plugin_origin(self, kind: str, origin: PluginOrigin, allow_deprecated: bool): if kind in self._origins: raise PluginError( "More than one {} plugin registered as kind '{}'".format(self._plugin_type, kind), reason="duplicate-plugin", ) self._origins[kind] = origin self._allow_deprecated[kind] = allow_deprecated # lookup(): # # Fetches a type loaded from a plugin in this plugin context # # Args: # messenger (Messenger): The messenger # kind (str): The kind of Plugin to create # provenance_node (Node): The node from where the plugin was referenced # # Returns: # (type): The type associated with the given kind # (str): A path to the YAML file holding the plugin's defaults, or None # # Raises: PluginError # def lookup(self, messenger: Messenger, kind: str, provenance_node: Node) -> Tuple[Type[Plugin], Optional[str]]: plugin_type, defaults = self._ensure_plugin(kind, provenance_node) # We can be called with None for the messenger here in the # case that we've been pickled through the scheduler (see jobpickler.py), # # In this case we know that we've already initialized and do not need # to warn about deprecated plugins a second time. if messenger is None: return plugin_type, defaults # After looking up the type, issue a warning if it's deprecated # # We do this here because we want to issue one warning for each time the # plugin is used. # if plugin_type.BST_PLUGIN_DEPRECATED and not self._allow_deprecated[kind]: messenger.warn( "{}: Using deprecated plugin '{}'".format(provenance_node.get_provenance(), kind), detail=plugin_type.BST_PLUGIN_DEPRECATION_MESSAGE, ) return plugin_type, defaults # list_plugins(): # # A generator which yields all of the plugins which have been loaded # # Yields: # (str): The plugin kind # (type): The loaded plugin type # (str): The default yaml file, if any # (str): The display string describing how the plugin was loaded # def list_plugins(self) -> Iterator[Tuple[str, Type[Plugin], str, str]]: for kind, (plugin_type, defaults, display) in self._types.items(): yield kind, plugin_type, defaults, display # get_plugin_paths(): # # Gets the directory on disk where the plugin itself is located, # and a full path to the plugin's accompanying YAML file for # it's defaults (if any). # # Args: # kind (str): The plugin kind # # Returns: # (str): The full path to the directory containing the plugin # (str): The full path to the accompanying .yaml file containing # the plugin's preferred defaults. # (str): The explanatory display string describing how this plugin was loaded # def get_plugin_paths(self, kind: str) -> Tuple[Optional[str], Optional[str], Optional[str]]: try: origin = self._origins[kind] except KeyError: return None, None, None return origin.get_plugin_paths(kind, self._plugin_type) ###################################################### # Private Methods # ###################################################### # _ensure_plugin(): # # Ensures that a plugin is loaded, delegating the work of getting # the plugin materials from the respective PluginOrigin # # Args: # kind (str): The plugin kind to load # provenance (str): The provenance of whence the plugin was referred to in the project # # Returns: # (type): The loaded type # (str): The full path the the yaml file containing defaults, or None # # Raises: # (PluginError): In case something went wrong loading the plugin # def _ensure_plugin(self, kind: str, provenance_node: Node) -> Tuple[Type[Plugin], Optional[str]]: if kind not in self._types: # Get the directory on disk where the plugin exists, and # the optional accompanying .yaml file for the plugin, should # one have been provided. # location, defaults, display = self.get_plugin_paths(kind) if location: # Make the PluginSource object # source = self._plugin_base.make_plugin_source( searchpath=[location], identifier=self._identifier + location + kind, ) # Keep a reference on the PluginSources (see comment in __init__) # self._sources[(location, kind)] = source else: # Try getting it from the core plugins if kind not in self._site_source.list_plugins(): raise PluginError( "{}: No {} plugin registered for kind '{}'".format( provenance_node.get_provenance(), self._plugin_type, kind ), reason="plugin-not-found", ) source = self._site_source defaults = os.path.join(self._site_plugins_path, "{}.yaml".format(kind)) if not os.path.exists(defaults): defaults = None display = "core plugin" self._types[kind] = (self._load_plugin(source, kind), defaults, display) type_, defaults, _ = self._types[kind] return type_, defaults # _load_plugin(): # # Loads the actual plugin type from the PluginSource # # Args: # source (PluginSource): The PluginSource # kind (str): The plugin kind to load # # Returns: # (type): The loaded type # # Raises: # (PluginError): In case something went wrong loading the plugin # def _load_plugin(self, source: PluginSource, kind: str) -> Type[Plugin]: try: plugin = source.load_plugin(kind) except ImportError as e: raise PluginError("Failed to load {} plugin '{}': {}".format(self._plugin_type, kind, e)) from e try: plugin_type = plugin.setup() except AttributeError as e: raise PluginError( "{} plugin '{}' did not provide a setup() function".format(self._plugin_type, kind), reason="missing-setup-function", ) from e except TypeError as e: raise PluginError( "setup symbol in {} plugin '{}' is not a function".format(self._plugin_type, kind), reason="setup-is-not-function", ) from e self._assert_plugin(kind, plugin_type) self._assert_min_version(kind, plugin_type) return plugin_type # _assert_plugin(): # # Performs assertions on the loaded plugin # # Args: # kind (str): The plugin kind to load # plugin_type (type): The loaded plugin type # # Raises: # (PluginError): In case something went wrong loading the plugin # def _assert_plugin(self, kind: str, plugin_type: Type[Plugin]): if kind in self._types: raise PluginError( "Tried to register {} plugin for existing kind '{}' " "(already registered {})".format(self._plugin_type, kind, self._types[kind].__name__) ) base_type: Type[Plugin] if self._plugin_type == PluginType.SOURCE: base_type = Source elif self._plugin_type == PluginType.ELEMENT: base_type = Element elif self._plugin_type == PluginType.SOURCE_MIRROR: base_type = SourceMirror try: if not issubclass(plugin_type, base_type): raise PluginError( "{} plugin '{}' returned type '{}', which is not a subclass of {}".format( self._plugin_type, kind, plugin_type.__name__, base_type.__name__ ), reason="setup-returns-bad-type", ) except TypeError as e: raise PluginError( "{} plugin '{}' returned something that is not a type (expected subclass of {})".format( self._plugin_type, kind, self._plugin_type ), reason="setup-returns-not-type", ) from e # _assert_min_version(): # # Performs the version checks on the loaded plugin type, # ensuring that the loaded plugin is intended to work # with this version of BuildStream. # # Args: # kind (str): The plugin kind to load # plugin_type (type): The loaded plugin type # # Raises: # (PluginError): In case something went wrong loading the plugin # def _assert_min_version(self, kind, plugin_type): if plugin_type.BST_MIN_VERSION is None: raise PluginError( "{} plugin '{}' did not specify BST_MIN_VERSION".format(self._plugin_type, kind), reason="missing-min-version", detail="Are you trying to use a BuildStream 1 plugin with a BuildStream 2 project ?", ) try: min_version_major, min_version_minor = utils._parse_version(plugin_type.BST_MIN_VERSION) except UtilError as e: raise PluginError( "{} plugin '{}' specified malformed BST_MIN_VERSION: {}".format( self._plugin_type, kind, plugin_type.BST_MIN_VERSION ), reason="malformed-min-version", detail="BST_MIN_VERSION must be specified as 'MAJOR.MINOR' with " + "numeric major and minor minimum required version numbers", ) from e bst_major, bst_minor = utils._get_bst_api_version() if min_version_major != bst_major: raise PluginError( "{} plugin '{}' requires BuildStream {}, but is being loaded with BuildStream {}".format( self._plugin_type, kind, min_version_major, bst_major ), reason="incompatible-major-version", detail="You will need to find the correct version of this plugin for your project.", ) if min_version_minor > bst_minor: raise PluginError( "{} plugin '{}' requires BuildStream {}, but is being loaded with BuildStream {}.{}".format( self._plugin_type, kind, plugin_type.BST_MIN_VERSION, bst_major, bst_minor ), reason="incompatible-minor-version", detail="Please upgrade to BuildStream {}".format(plugin_type.BST_MIN_VERSION), ) apache-buildstream-27ae392/src/buildstream/_pluginfactory/pluginorigin.py000066400000000000000000000136251514607367700270400ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from ..types import FastEnum from ..node import ScalarNode, MappingNode from .._exceptions import LoadError from ..exceptions import LoadErrorReason # PluginType() # # A type of plugin # class PluginType(FastEnum): # A Source plugin SOURCE = "source" # An Element plugin ELEMENT = "element" # A SourceMirror plugin SOURCE_MIRROR = "source-mirror" def __str__(self): return str(self.value) # PluginOriginType: # # An enumeration depicting the type of plugin origin # class PluginOriginType(FastEnum): # A local plugin LOCAL = "local" # A pip plugin PIP = "pip" # A plugin loaded via a junction JUNCTION = "junction" # PluginConfiguration: # # An object representing the configuration of a single # plugin in the origin. # class PluginConfiguration: def __init__(self, kind, allow_deprecated): self.kind = kind self.allow_deprecated = allow_deprecated # PluginOrigin # # Base class holding common properties of all origins. # class PluginOrigin: # Common fields valid for all plugin origins _COMMON_CONFIG_KEYS = ["origin", "sources", "elements", "source-mirrors", "allow-deprecated"] def __init__(self, origin_type): # Public self.origin_type = origin_type # The PluginOriginType self.elements = {} # A dictionary of PluginConfiguration self.sources = {} # A dictionary of PluginConfiguration objects self.source_mirrors = {} # A dictionary of PluginConfiguration objects self.provenance_node = None self.project = None # Private self._kinds = {} self._allow_deprecated = False # initialize() # # Initializes the origin, resulting in loading the origin # node. # # This is the bottom half of the initialization, it is done # separately because load_plugin_origin() needs to stay in # __init__.py in order to avoid cyclic dependencies between # PluginOrigin and it's subclasses. # # Args: # project (Project): The project this PluginOrigin was loaded for # origin_node (MappingNode): The node defining this origin # def initialize(self, project, origin_node): self.provenance_node = origin_node self.project = project self.load_config(origin_node) # Parse commonly defined aspects of PluginOrigins self._allow_deprecated = origin_node.get_bool("allow-deprecated", False) element_sequence = origin_node.get_sequence("elements", []) self._load_plugin_configurations(element_sequence, self.elements) source_sequence = origin_node.get_sequence("sources", []) self._load_plugin_configurations(source_sequence, self.sources) source_mirror_sequence = origin_node.get_sequence("source-mirrors", []) self._load_plugin_configurations(source_mirror_sequence, self.source_mirrors) ############################################## # Abstract methods # ############################################## # get_plugin_paths(): # # Abstract method for loading the details about a specific plugin, # the PluginFactory uses this to get the assets needed to actually # load the plugins. # # Args: # kind (str): The plugin # plugin_type (PluginType): The kind of plugin to load # # Returns: # (str): The full path to the directory containing the plugin # (str): The full path to the accompanying .yaml file containing # the plugin's preferred defaults. # (str): The explanatory display string describing how this plugin was loaded # def get_plugin_paths(self, kind, plugin_type): pass # load_config() # # Abstract method for loading data from the origin node, this # method should not load the source and element lists. # # Args: # origin_node (MappingNode): The node defining this origin # def load_config(self, origin_node): pass ############################################## # Private methods # ############################################## # _load_plugin_configurations() # # Helper function to load the list of source or element # PluginConfigurations # # Args: # sequence_node (SequenceNode): The list of configurations # dictionary (dict): The location to store the results # def _load_plugin_configurations(self, sequence_node, dictionary): for node in sequence_node: # Parse as a simple string if type(node) is ScalarNode: # pylint: disable=unidiomatic-typecheck kind = node.as_str() conf = PluginConfiguration(kind, self._allow_deprecated) # Parse as a dictionary elif type(node) is MappingNode: # pylint: disable=unidiomatic-typecheck node.validate_keys(["kind", "allow-deprecated"]) kind = node.get_str("kind") allow_deprecated = node.get_bool("allow-deprecated", self._allow_deprecated) conf = PluginConfiguration(kind, allow_deprecated) else: p = node.get_provenance() raise LoadError( "{}: Plugin is not specified as a string or a dictionary".format(p), LoadErrorReason.INVALID_DATA ) dictionary[kind] = conf apache-buildstream-27ae392/src/buildstream/_pluginfactory/pluginoriginjunction.py000066400000000000000000000066701514607367700306140ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from .._exceptions import PluginError from .pluginorigin import PluginType, PluginOrigin, PluginOriginType # PluginOriginJunction # # PluginOrigin for junction plugins # class PluginOriginJunction(PluginOrigin): def __init__(self): super().__init__(PluginOriginType.JUNCTION) # The junction element name through which to load plugins self._junction = None def get_plugin_paths(self, kind, plugin_type): # Get access to the project indicated by the junction, # possibly loading it as a side effect. # loader = self.project.loader.get_loader(self._junction, self.provenance_node) project = loader.project # Now get the appropriate PluginFactory object # if plugin_type == PluginType.SOURCE: factory = project.source_factory elif plugin_type == PluginType.ELEMENT: factory = project.element_factory elif plugin_type == PluginType.SOURCE_MIRROR: factory = project.source_mirror_factory else: assert False, "unreachable" # Now ask for the paths from the subproject PluginFactory try: location, defaults, display = factory.get_plugin_paths(kind) except PluginError as e: # Add some context to an error raised by loading a plugin from a subproject # raise PluginError( "{}: Error loading {} plugin '{}' from project '{}' referred to by junction '{}': {}".format( self.provenance_node.get_provenance(), plugin_type, kind, project.name, self._junction, e ), reason="junction-plugin-load-error", detail=e.detail, ) from e if not location: # Raise a helpful error if the referred plugin type is not found in a subproject # # Note that this can also bubble up through the above error when looking for # a plugin from a subproject which in turn requires the same plugin from it's # subproject. # raise PluginError( "{}: project '{}' referred to by junction '{}' does not declare a {} plugin named: '{}'".format( self.provenance_node.get_provenance(), project.name, self._junction, plugin_type, kind ), reason="junction-plugin-not-found", ) # Use the resolved project path for the display string rather than the user configured junction path project_path = "toplevel project" if project.junction: project_path = project.junction._get_full_name() return location, defaults, "junction: {} ({})".format(project_path, display) def load_config(self, origin_node): origin_node.validate_keys(["junction", *PluginOrigin._COMMON_CONFIG_KEYS]) self._junction = origin_node.get_str("junction") apache-buildstream-27ae392/src/buildstream/_pluginfactory/pluginoriginlocal.py000066400000000000000000000027051514607367700300500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from .pluginorigin import PluginOrigin, PluginOriginType # PluginOriginLocal # # PluginOrigin for local plugins # class PluginOriginLocal(PluginOrigin): def __init__(self): super().__init__(PluginOriginType.LOCAL) # Project relative path to where plugins from this origin are found self._path = None def get_plugin_paths(self, kind, plugin_type): path = os.path.join(self.project.directory, self._path) defaults = os.path.join(path, "{}.yaml".format(kind)) if not os.path.exists(defaults): defaults = None return path, defaults, "project directory: {}".format(self._path) def load_config(self, origin_node): origin_node.validate_keys(["path", *PluginOrigin._COMMON_CONFIG_KEYS]) path_node = origin_node.get_scalar("path") self._path = self.project.get_path_from_node(path_node, check_is_dir=True) apache-buildstream-27ae392/src/buildstream/_pluginfactory/pluginoriginpip.py000066400000000000000000000076171514607367700275550ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from pathlib import Path from .._exceptions import PluginError from .pluginorigin import PluginType, PluginOrigin, PluginOriginType # PluginOriginPip # # PluginOrigin for pip plugins # class PluginOriginPip(PluginOrigin): def __init__(self): super().__init__(PluginOriginType.PIP) # The pip package name to extract plugins from # self._package_name = None def get_plugin_paths(self, kind, plugin_type): from packaging.requirements import Requirement, InvalidRequirement from importlib.metadata import distribution, PackageNotFoundError from importlib.util import find_spec # Sources and elements are looked up in separate # entrypoint groups from the same package. # if plugin_type == PluginType.SOURCE: entrypoint_group = "buildstream.plugins.sources" elif plugin_type == PluginType.ELEMENT: entrypoint_group = "buildstream.plugins.elements" elif plugin_type == PluginType.SOURCE_MIRROR: entrypoint_group = "buildstream.plugins.sourcemirrors" else: assert False, "unreachable" try: package = Requirement(self._package_name) except InvalidRequirement as e: raise PluginError( "{}: Malformed package-name '{}' encountered: {}".format( self.provenance_node.get_provenance(), self._package_name, e ), reason="package-malformed-requirement", ) from e try: dist = distribution(package.name) except PackageNotFoundError as e: raise PluginError( "{}: Failed to load {} plugin '{}': {}".format( self.provenance_node.get_provenance(), plugin_type, kind, e ), reason="package-not-found", ) from e if not package.specifier.contains(dist.version, prereleases=True): raise PluginError( "{}: Version conflict encountered while loading {} plugin '{}'".format( self.provenance_node.get_provenance(), plugin_type, kind ), detail="{} {} is installed but {} is required".format(dist.name, dist.version, package), reason="package-version-conflict", ) try: entrypoint = dist.entry_points.select(group=entrypoint_group)[kind] except KeyError as e: raise PluginError( "{}: Pip package {} does not contain a {} plugin named '{}'".format( self.provenance_node.get_provenance(), self._package_name, plugin_type, kind ), reason="plugin-not-found", ) location = Path(find_spec(entrypoint.module).origin) defaults = location.with_suffix(".yaml") if not defaults.exists(): # The plugin didn't have an accompanying YAML file defaults = None directory = str(location.parent) return ( directory, str(defaults), "python package '{}' version {} at: {}".format(dist.name, dist.version, directory), ) def load_config(self, origin_node): origin_node.validate_keys(["package-name", *PluginOrigin._COMMON_CONFIG_KEYS]) self._package_name = origin_node.get_str("package-name") apache-buildstream-27ae392/src/buildstream/_pluginfactory/sourcefactory.py000066400000000000000000000040541514607367700272160ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from typing import TYPE_CHECKING, Type, cast from .pluginfactory import PluginFactory from .pluginorigin import PluginType from ..source import Source from .._loader import MetaSource from .._variables import Variables if TYPE_CHECKING: from .._context import Context from .._project import Project # A SourceFactory creates Source instances # in the context of a given factory # # Args: # plugin_base (PluginBase): The main PluginBase object to work with # class SourceFactory(PluginFactory): def __init__(self, plugin_base): super().__init__(plugin_base, PluginType.SOURCE) # create(): # # Create a Source object. # # Args: # context (object): The Context object for processing # project (object): The project object # meta (object): The loaded MetaSource # variables (Variables): The variables available to the source # # Returns: # A newly created Source object of the appropriate kind # # Raises: # PluginError (if the kind lookup failed) # LoadError (if the source itself took issue with the config) # def create(self, context: "Context", project: "Project", meta: MetaSource, variables: Variables) -> Source: plugin_type, _ = self.lookup(context.messenger, meta.kind, meta.config) source_type = cast(Type[Source], plugin_type) source = source_type(context, project, meta, variables) return source apache-buildstream-27ae392/src/buildstream/_pluginfactory/sourcemirrorfactory.py000066400000000000000000000043761514607367700304600ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from typing import TYPE_CHECKING, Type, cast from .pluginfactory import PluginFactory from .pluginorigin import PluginType from ..node import MappingNode from ..plugin import Plugin from ..sourcemirror import SourceMirror if TYPE_CHECKING: from .._context import Context from .._project import Project # A SourceMirrorFactory creates SourceMirror instances # in the context of a given factory # # Args: # plugin_base (PluginBase): The main PluginBase object to work with # class SourceMirrorFactory(PluginFactory): def __init__(self, plugin_base): super().__init__(plugin_base, PluginType.SOURCE_MIRROR) # create(): # # Create a SourceMirror object. # # Args: # context (object): The Context object for processing # project (object): The project object # node (MappingNode): The node where the mirror was defined # # Returns: # A newly created SourceMirror object of the appropriate kind # # Raises: # PluginError (if the kind lookup failed) # LoadError (if the source mirror itself took issue with the config) # def create(self, context: "Context", project: "Project", node: MappingNode) -> SourceMirror: plugin_type: Type[Plugin] # Shallow parsing to get the custom plugin type, delegate the remainder # of the parsing to SourceMirror # kind = node.get_str("kind", "default") plugin_type, _ = self.lookup(context.messenger, kind, node) source_mirror_type = cast(Type[SourceMirror], plugin_type) source_mirror = source_mirror_type(context, project, node) return source_mirror apache-buildstream-27ae392/src/buildstream/_pluginproxy.py000066400000000000000000000065041514607367700240420ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .plugin import Plugin # PluginProxyError() # # The PluginProxyError is raised by PluginProxy objects when an illegal # method call is called by a Plugin. # # This exception does not derive from BstError because it is not a user # facing error but a Plugin author facing error; the result of a # PluginProxyError being raised is that BuildStream treats it as an # unhandled exception, and issues a BUG message with a helpful stacktrace # which can be helpful for the Plugin author to fix their bugs. # class PluginProxyError(Exception): pass # PluginProxy() # # Base class for proxies to Plugin instances. # # Proxies are handed off to Plugin implementations whenever they observe the data # model, like when the Element observes it's dependencies, this allows the core to # do some police work and raise errors when plugins attempt to perform illegal method # calls. # # Refer to the Plugin class for the documentation for these APIs. # # In this file we simply raise a PluginProxyError() in the case that a Plugin tries to # call an illegal API, or we forward the method call along to the underlying Plugin # instance if the given method call is considered legal. # # Args: # owner (Plugin): The owning plugin, i.e. the plugin which this proxy was given to # plugin (Plugin): The proxied plugin, i.e. the plugin this proxy is attached to # class PluginProxy: def __init__(self, owner: Plugin, plugin: Plugin): # These members are considered internal, they are accessed by subclasses # which extend the PluginProxy, but hidden from the client Plugin implementations # which the proxy objects are handed off to. # self._owner = owner # The Plugin this proxy was given to / created for self._plugin = plugin # The Plugin this proxy was created as a proxy for # We use the fallback __getattr__ method to trigger plugin author facing # errors for any methods we have not explicitly redirected to the underlying # plugin object. # def __getattr__(self, name): if hasattr(self._plugin, name): raise PluginProxyError( "{}: Illegal attempt to access attribute '{}' on plugin: {}".format(self._owner, name, self._plugin) ) raise AttributeError("{}: Has no attribute '{}'".format(self._plugin, name)) ############################################################## # Exposed proxied APIs # ############################################################## @property def name(self): return self._plugin.name def get_kind(self) -> str: return self._plugin.get_kind() @property def _unique_id(self): return self._plugin._unique_id apache-buildstream-27ae392/src/buildstream/_profile.py000066400000000000000000000124311514607367700230760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # James Ennis # Benjamin Schubert import contextlib import cProfile import pstats import os import datetime import time from ._exceptions import ProfileError # Use the topic values here to decide what to profile # by setting them in the BST_PROFILE environment variable. # # Multiple topics can be set with the ':' separator. # # E.g.: # # BST_PROFILE=circ-dep-check:sort-deps bst # # The special 'all' value will enable all profiles. class Topics: CIRCULAR_CHECK = "circ-dep-check" SORT_DEPENDENCIES = "sort-deps" LOAD_CONTEXT = "load-context" LOAD_PROJECT = "load-project" LOAD_PIPELINE = "load-pipeline" LOAD_SELECTION = "load-selection" SCHEDULER = "scheduler" ALL = "all" class _Profile: def __init__(self, key, message): self.profiler = cProfile.Profile() self._additional_pstats_files = [] self.key = key self.message = message self.start_time = time.time() filename_template = os.path.join( os.getcwd(), "profile-{}-{}".format( datetime.datetime.fromtimestamp(self.start_time).strftime("%Y%m%dT%H%M%S"), self.key.replace("/", "-").replace(".", "-"), ), ) self.log_filename = "{}.log".format(filename_template) self.cprofile_filename = "{}.cprofile".format(filename_template) def __enter__(self): self.start() def __exit__(self, _exc_type, _exc_value, traceback): self.stop() self.save() def merge(self, profile): self._additional_pstats_files.append(profile.cprofile_filename) def start(self): self.profiler.enable() def stop(self): self.profiler.disable() def save(self): heading = "\n".join( [ "-" * 64, "Profile for key: {}".format(self.key), "Started at: {}".format(self.start_time), "\n\t{}".format(self.message) if self.message else "", "-" * 64, "", # for a final new line ] ) with open(self.log_filename, "a", encoding="utf-8") as fp: stats = pstats.Stats(self.profiler, *self._additional_pstats_files, stream=fp) # Create the log file fp.write(heading) stats.sort_stats("cumulative") stats.print_stats() # Dump the cprofile stats.dump_stats(self.cprofile_filename) class _Profiler: def __init__(self, settings): self.active_topics = set() self.enabled_topics = set() self._active_profilers = [] self._valid_topics = False if settings: self.enabled_topics = set(settings.split(":")) @contextlib.contextmanager def profile(self, topic, key, message=None): # Check if the user enabled topics are valid # NOTE: This is done in the first PROFILER.profile() call and # not __init__ to ensure we handle the exception. This also means # we cannot test for the exception due to the early instantiation and # how the environment is set in the test invocation. if not self._valid_topics: self._check_valid_topics() if not self._is_profile_enabled(topic): yield return if self._active_profilers: # we are in a nested profiler, stop the parent self._active_profilers[-1].stop() key = "{}-{}".format(topic, key) assert key not in self.active_topics self.active_topics.add(key) profiler = _Profile(key, message) self._active_profilers.append(profiler) with profiler: yield self.active_topics.remove(key) # Remove the last profiler from the list self._active_profilers.pop() if self._active_profilers: # We were in a previous profiler, add the previous results to it # and reenable it. parent_profiler = self._active_profilers[-1] parent_profiler.merge(profiler) parent_profiler.start() def _is_profile_enabled(self, topic): return topic in self.enabled_topics or Topics.ALL in self.enabled_topics def _check_valid_topics(self): non_valid_topics = [topic for topic in self.enabled_topics if topic not in vars(Topics).values()] if non_valid_topics: raise ProfileError("Provided BST_PROFILE topics do not exist: {}".format(", ".join(non_valid_topics))) self._valid_topics = True # Export a profiler to be used by BuildStream PROFILER = _Profiler(os.getenv("BST_PROFILE")) apache-buildstream-27ae392/src/buildstream/_project.py000066400000000000000000001405421514607367700231110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Tiago Gomes from typing import TYPE_CHECKING, Optional, Dict, Union, List, Sequence import os import urllib.parse from pathlib import Path from pluginbase import PluginBase from . import utils from . import _site from . import _yaml from ._variables import Variables from .utils import UtilError from ._profile import Topics, PROFILER from ._exceptions import LoadError from .exceptions import LoadErrorReason from ._options import OptionPool from .node import ScalarNode, MappingNode, ProvenanceInformation, _assert_symbol_name from ._pluginfactory import ElementFactory, SourceFactory, SourceMirrorFactory, load_plugin_origin from .types import CoreWarnings, _HostMount, _SourceUriPolicy from ._projectrefs import ProjectRefs, ProjectRefStorage from ._loader import Loader, LoadContext from .element import Element from ._includes import Includes from ._workspaces import WORKSPACE_PROJECT_FILE from ._remotespec import RemoteSpec from .sourcemirror import SourceMirror from .source import AliasSubstitution, SourceError if TYPE_CHECKING: from ._context import Context from .plugins.elements.junction import JunctionElement # Project Configuration file _PROJECT_CONF_FILE = "project.conf" # Represents project configuration that can have different values for junctions. class ProjectConfig: def __init__(self): self.options = None # OptionPool self.base_variables = None # The base set of variables self.element_overrides = {} # Element specific configurations self.source_overrides = {} # Source specific configurations self.mirrors = {} # Dictionary of SourceMirror objects self.default_mirror = None # The name of the preferred mirror. self._aliases = None # Aliases dictionary # Project() # # The Project Configuration # # Args: # directory: The project directory, or None for dummy ArtifactProjects # context: The invocation context # junction: The junction Element causing this project to be loaded # cli_options: The project options specified on the command line # default_mirror: The default mirror specified on the command line # parent_loader: The parent loader # provenance_node: The YAML provenance causing this project to be loaded # search_for_project: Whether to search for a project directory, e.g. from workspace metadata or parent directories # class Project: def __init__( self, directory: Optional[str], context: "Context", *, junction: Optional["JunctionElement"] = None, cli_options: Optional[Dict[str, str]] = None, default_mirror: Optional[str] = None, parent_loader: Optional[Loader] = None, provenance_node: Optional[ProvenanceInformation] = None, search_for_project: bool = True, fetch_subprojects=None ): # # Public members # self.name: str = "" # The project name self.directory: Optional[str] = directory # The project directory self.element_path: Optional[str] = None # The project relative element path self.load_context: LoadContext # The LoadContext self.loader: Optional[Loader] = None # The loader associated to this project self.junction: Optional["JunctionElement"] = junction # The junction Element object, if this is a subproject self.ref_storage: Optional[ProjectRefStorage] = None # Where to store source refs self.refs: Optional[ProjectRefs] = None self.junction_refs: Optional[ProjectRefs] = None self.disallow_subproject_uris: bool = False self.config: ProjectConfig = ProjectConfig() self.first_pass_config: ProjectConfig = ProjectConfig() self.base_environment: Union[MappingNode, Dict[str, str]] = {} # The base set of environment variables self.base_env_nocache: List[str] = [] # The base nocache mask (list) for the environment # Remote specs for communicating with remote services self.artifact_cache_specs: List[RemoteSpec] = [] # Artifact caches self.source_cache_specs: List[RemoteSpec] = [] # Source caches self.element_factory: Optional[ElementFactory] = None # ElementFactory for loading elements self.source_factory: Optional[SourceFactory] = None # SourceFactory for loading sources self.source_mirror_factory: Optional[SourceMirrorFactory] = None # SourceMirrorFactory self.sandbox: Optional[MappingNode] = None self.splits: Optional[MappingNode] = None self.source_provenance_attributes: Optional[ MappingNode ] = None # Source provenance attributes and their description # # Private members # self._context: "Context" = context # The invocation Context self._invoked_from_workspace_element: Optional[str] = None self._absolute_directory_path: Optional[Path] = None self._default_targets: Optional[List[str]] = None # Default target elements self._default_mirror: Optional[str] = default_mirror # The name of the preferred mirror. self._cli_options: Optional[Dict[str, str]] = cli_options self._fatal_warnings: List[str] = [] # A list of warnings which should trigger an error self._shell_command: List[str] = [] # The default interactive shell command self._shell_environment: Dict[str, str] = {} # Statically set environment vars self._shell_host_files: List[_HostMount] = [] # A list of HostMount objects self._mirror_override: bool = False # Whether mirrors have been declared in user configuration # This is a lookup table of lists indexed by project, # the child dictionaries are lists of ScalarNodes indicating # junction names self._junction_duplicates: Dict[str, List[str]] = {} # A list of project relative junctions to consider as 'internal', # stored as ScalarNodes. self._junction_internal: List[str] = [] self._partially_loaded: bool = False self._fully_loaded: bool = False self._project_includes: Optional[Includes] = None # # Initialization body # if parent_loader: self.load_context = parent_loader.load_context else: self.load_context = LoadContext(self._context) self.load_context.set_fetch_subprojects(fetch_subprojects) if search_for_project: self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory) if self.directory: self._absolute_directory_path = Path(self.directory).resolve() self.refs = ProjectRefs(self.directory, "project.refs") self.junction_refs = ProjectRefs(self.directory, "junction.refs") self._context.add_project(self) if self.directory: with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")): self._load(parent_loader=parent_loader, provenance_node=provenance_node) else: self._fully_loaded = True self._partially_loaded = True @property def options(self): return self.config.options @property def base_variables(self): return self.config.base_variables @property def element_overrides(self): return self.config.element_overrides @property def source_overrides(self): return self.config.source_overrides ######################################################## # Public Methods # ######################################################## # get_alias_url(): # # Fetch the value of a URL alias # # Args: # alias: The alias # first_pass: Whether to use first pass configuration (for junctions) # # Returns: # The alias substitution # def get_alias_url(self, alias: str, *, first_pass: bool = False) -> Optional[str]: if first_pass: config = self.first_pass_config else: config = self.config if config._aliases: return config._aliases.get_str(alias, default=None) return None # translate_url(): # # Translates the given url which may be specified with an alias # into a fully qualified url. # # Args: # url (str): A url, which may be using an alias # first_pass (bool): Whether to use first pass configuration (for junctions) # # Returns: # str: The fully qualified url, with aliases resolved # # This method is provided for :class:`.Source` objects to resolve # fully qualified urls based on the shorthand which is allowed # to be specified in the YAML def translate_url(self, url, *, source, first_pass=False): if url and utils._ALIAS_SEPARATOR in url: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) alias_url = self.get_alias_url(url_alias, first_pass=first_pass) if alias_url: if self.junction: parent_project = self.junction._get_project() parent_alias = self.junction.get_parent_alias(url_alias) if parent_alias: # Delegate translation to parent project return parent_project.translate_url( parent_alias + utils._ALIAS_SEPARATOR + url_body, source=source, first_pass=first_pass ) elif parent_project.disallow_subproject_uris: raise SourceError( "{}: Parent project did not provide a mapping for alias '{}' and disallowed usage of unmapped aliases".format( source, url_alias ), reason="missing-alias-mapping", ) url = alias_url + url_body return url # get_shell_config() # # Gets the project specified shell configuration # # Returns: # (list): The shell command # (dict): The shell environment # (list): The list of _HostMount objects # def get_shell_config(self): return (self._shell_command, self._shell_environment, self._shell_host_files) # get_path_from_node() # # Fetches the project path from a dictionary node and validates it # # Paths are asserted to never lead to a directory outside of the project # directory. In addition, paths can not point to symbolic links, fifos, # sockets and block/character devices. # # The `check_is_file` and `check_is_dir` parameters can be used to # perform additional validations on the path. Note that an exception # will always be raised if both parameters are set to ``True``. # # Args: # node (ScalarNode): A Node loaded from YAML containing the path to validate # check_is_file (bool): If ``True`` an error will also be raised # if path does not point to a regular file. # Defaults to ``False`` # check_is_dir (bool): If ``True`` an error will be also raised # if path does not point to a directory. # Defaults to ``False`` # Returns: # (str): The project path # # Raises: # (LoadError): In case that the project path is not valid or does not # exist # def get_path_from_node(self, node, *, check_is_file=False, check_is_dir=False): path_str = node.as_str() path = Path(path_str) full_path = self._absolute_directory_path / path if full_path.is_symlink(): provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' must not point to " "symbolic links ".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND, ) if path.parts and path.parts[0] == "..": provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' first component must " "not be '..'".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID, ) try: full_resolved_path = full_path.resolve(strict=True) except FileNotFoundError: provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' does not exist".format(provenance, path_str), LoadErrorReason.MISSING_FILE ) is_inside = self._absolute_directory_path in full_resolved_path.parents or ( full_resolved_path == self._absolute_directory_path ) if not is_inside: provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' must not lead outside of the " "project directory".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID, ) if path.is_absolute(): provenance = node.get_provenance() raise LoadError( "{}: Absolute path: '{}' invalid.\n" "Please specify a path relative to the project's root.".format(provenance, path), LoadErrorReason.PROJ_PATH_INVALID, ) if full_resolved_path.is_socket() or (full_resolved_path.is_fifo() or full_resolved_path.is_block_device()): provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' points to an unsupported " "file kind".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND, ) if check_is_file and not full_resolved_path.is_file(): provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' is not a regular file".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND, ) if check_is_dir and not full_resolved_path.is_dir(): provenance = node.get_provenance() raise LoadError( "{}: Specified path '{}' is not a directory".format(provenance, path_str), LoadErrorReason.PROJ_PATH_INVALID_KIND, ) return path_str # create_element() # # Instantiate and return an element # # Args: # load_element (LoadElement): The LoadElement # # Returns: # (Element): A newly created Element object of the appropriate kind # def create_element(self, load_element): return self.element_factory.create(self._context, self, load_element) # create_source() # # Instantiate and return a Source # # Args: # meta (MetaSource): The loaded MetaSource # variables (Variables): The list of variables available to the source # # Returns: # (Source): A newly created Source object of the appropriate kind # def create_source(self, meta, variables): return self.source_factory.create(self._context, self, meta, variables) # alias_exists() # # Returns the URI for a given alias, if it exists # # Args: # alias (str): The alias. # first_pass (bool): Whether to use first pass configuration (for junctions) # # Returns: # bool: Whether the alias is declared in the scope of this project # def alias_exists(self, alias, *, source, first_pass=False): if first_pass: config = self.first_pass_config else: config = self.config if self.junction: parent_project = self.junction._get_project() parent_alias = self.junction.get_parent_alias(alias) if parent_alias: if parent_project.alias_exists(parent_alias, source=source, first_pass=first_pass): return True else: raise SourceError( "{}: Mapped alias '{}' for subproject alias '{}' is invalid in the parent project".format( self.junction, parent_alias, alias ), reason="invalid-source-alias", ) elif parent_project.disallow_subproject_uris: raise SourceError( "{}: Parent project did not provide a mapping for alias '{}' and disallowed usage of unmapped aliases".format( source, alias ), reason="missing-alias-mapping", ) return config._aliases.get_str(alias, default=None) is not None # get_alias_uris() # # Args: # alias: The alias. # first_pass: Whether to use first pass configuration (for junctions) # tracking: Whether we want the aliases for tracking (otherwise assume fetching) # # Returns: # A list of alias substitutions found. Each substitution is either a string # (to be prepended to the rest of the URL instead of the alias) or a # SourceMirror object. # def get_alias_uris( self, alias: str, *, first_pass: bool = False, tracking: bool = False ) -> Sequence[Optional[AliasSubstitution]]: if first_pass: config = self.first_pass_config else: config = self.config if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test return [None] if self.junction: parent_project = self.junction._get_project() parent_alias = self.junction.get_parent_alias(alias) if parent_alias: # Delegate translation to parent project return parent_project.get_alias_uris(parent_alias, first_pass=first_pass, tracking=tracking) elif parent_project.disallow_subproject_uris: return [None] uri_list: List[Union[SourceMirror, str]] = [] policy = self._context.track_source if tracking else self._context.fetch_source if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.MIRRORS) or ( policy == _SourceUriPolicy.USER and self._mirror_override ): for mirror_name, mirror in config.mirrors.items(): list_to_add = mirror._get_alias_uris(alias) if mirror_name == config.default_mirror: uri_list = list_to_add + uri_list else: uri_list += list_to_add if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.ALIASES): uri_list.append(config._aliases.get_str(alias)) return [AliasSubstitution(alias, mirror) for mirror in uri_list] # load_elements() # # Loads elements from target names. # # Args: # targets (list): Target names # # Returns: # (list): A list of loaded Element # def load_elements(self, targets): with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task: self.load_context.set_task(task) load_elements = self.loader.load(targets) self.load_context.set_task(None) with self._context.messenger.simple_task("Resolving elements", silent_nested=True) as task: if task: task.set_maximum_progress(self.loader.loaded) elements = [Element._new_from_load_element(load_element, task) for load_element in load_elements] Element._clear_meta_elements_cache() # Assert loaders after resolving everything, this is because plugin # loading (across junction boundaries) can also be the cause of # conflicting projects. # self.load_context.assert_loaders() # Now warn about any redundant source references which may have # been discovered in the resolve() phase. redundant_refs = Element._get_redundant_source_refs() if redundant_refs: detail = "The following inline specified source references will be ignored:\n\n" lines = ["{}:{}".format(source._get_provenance(), ref) for source, ref in redundant_refs] detail += "\n".join(lines) self._context.messenger.warn("Ignoring redundant source references", detail=detail) return elements # ensure_fully_loaded() # # Ensure project has finished loading. At first initialization, a # project can only load junction elements. Other elements require # project to be fully loaded. # def ensure_fully_loaded(self): if self._fully_loaded: return assert self._partially_loaded # Ensure that the parent project is fully loaded first if self.junction: self.junction._get_project().ensure_fully_loaded() if self._fully_loaded: # The parent project triggered the loading of this project, # don't load this subproject twice. return # Here we mark the project as fully loaded right away, # before doing the work. # # This function will otherwise reenter itself infinitely: # # * Ensuring the invariant that a parent project is fully # loaded before completing the load of this project, will # trigger this function when completing the load of subprojects. # # * Completing the load of this project may include processing # some `(@)` include directives, which can directly trigger # the loading of subprojects. # self._fully_loaded = True self._load_second_pass() # get_default_target() # # Attempts to interpret which element the user intended to run a command on. # This is for commands that only accept a single target element and thus, # this only uses the workspace element (if invoked from workspace directory) # and does not use the project default targets. # def get_default_target(self): return self._invoked_from_workspace_element # get_default_targets() # # Attempts to interpret which elements the user intended to run a command on. # This is for commands that accept multiple target elements. # def get_default_targets(self): # If _invoked_from_workspace_element has a value, # a workspace element was found before a project config # Therefore the workspace does not contain a project if self._invoked_from_workspace_element: return (self._invoked_from_workspace_element,) # Default targets from project configuration if self._default_targets: return tuple(self._default_targets) # If default targets are not configured, default to all project elements default_targets = [] for root, dirs, files in os.walk(self.element_path): # Do not recurse down the ".bst" directory which is where we stage # junctions and other BuildStream internals. if ".bst" in dirs: dirs.remove(".bst") for file in files: if file.endswith(".bst"): rel_dir = os.path.relpath(root, self.element_path) rel_file = os.path.join(rel_dir, file).lstrip("./") default_targets.append(rel_file) return tuple(default_targets) # junction_is_duplicated() # # Check whether this loader is specified as a duplicate by # this project. # # Args: # project_name: (str): The project name # loader (Loader): The loader to check for # # Returns: # (bool): Whether the loader is specified as duplicate # def junction_is_duplicated(self, project_name, loader): junctions = self._junction_duplicates.get(project_name, {}) # Iterate over all paths specified by this project and see # if we find a match for the specified loader. # # Using the regular `Loader.get_loader()` codepath from this # project ensures that we will find the correct loader relative # to this project, regardless of any overrides or link elements # which might have been used in the project. # for dup_path in junctions: search = self.loader.get_loader(dup_path.as_str(), dup_path, load_subprojects=False) if loader is search: return True return False # junction_is_internal() # # Check whether this loader is specified as internal to # this project. # # Args: # loader (Loader): The loader to check for # # Returns: # (bool): Whether the loader is specified as internal # def junction_is_internal(self, loader): # Iterate over all paths specified by this project and see # if we find a match for the specified loader. # # Using the regular `Loader.get_loader()` codepath from this # project ensures that we will find the correct loader relative # to this project, regardless of any overrides or link elements # which might have been used in the project. # for internal_path in self._junction_internal: search = self.loader.get_loader(internal_path.as_str(), internal_path, load_subprojects=False) if loader is search: return True return False # loaded_projects() # # A generator which yields all the projects in context of a loaded # pipeline, including the self project. # # Projects will be yielded in the order in which they were loaded # for the current session's pipeline. # # This is used by the frontend to print information about all the # loaded projects. # # Yields: # (_ProjectInformation): A descriptive project information object # def loaded_projects(self): yield from self.load_context.loaded_projects() ######################################################## # Private Methods # ######################################################## # _validate_toplevel_node() # # Validates the toplevel project.conf keys # # Args: # node (MappingNode): The toplevel project.conf node # first_pass (bool): Whether this is the first or second pass # def _validate_toplevel_node(self, node, *, first_pass=False): node.validate_keys( [ "min-version", "element-path", "variables", "environment", "environment-nocache", "split-rules", "elements", "plugins", "aliases", "name", "defaults", "artifacts", "options", "fail-on-overlap", "shell", "fatal-warnings", "ref-storage", "sandbox", "mirrors", "sources", "source-caches", "junctions", "source-provenance-attributes", "(@)", "(?)", ] ) # Keys which are invalid if specified outside of project.conf if not first_pass: invalid_keys = {"name", "element-path", "min-version", "plugins"} for invalid_key in invalid_keys: invalid_node = node.get_node(invalid_key, allow_none=True) if invalid_node: provenance = invalid_node.get_provenance() if ( provenance._shortname != "project.conf" and provenance._filename != _site.default_project_config ): raise LoadError( "{}: Unexpected key: {}".format(provenance, invalid_key), LoadErrorReason.INVALID_DATA, detail="The '{}' configuration must be specified in project.conf".format(invalid_key), ) # _validate_version() # # Asserts that we have a BuildStream installation which is recent # enough for the project required version # # Args: # config_node (dict) - YaML node of the configuration file. # # Raises: LoadError if there was a problem with the project.conf # def _validate_version(self, config_node): bst_major, bst_minor = utils._get_bst_api_version() # Use a custom error message for the absence of the required "min-version" # as this may be an indication that we are trying to load a BuildStream 1 project. # min_version_node = config_node.get_scalar("min-version", None) if min_version_node.is_none(): p = config_node.get_provenance() raise LoadError( "{}: Dictionary did not contain expected key 'min-version'".format(p), LoadErrorReason.INVALID_DATA, # # TODO: Provide a link to documentation on how to install # BuildStream 1 in a venv # detail="If you are trying to use a BuildStream 1 project, " + "please install BuildStream 1 to use this project.", ) # Parse the project declared minimum required BuildStream version min_version = min_version_node.as_str() try: min_version_major, min_version_minor = utils._parse_version(min_version) except UtilError as e: p = min_version_node.get_provenance() raise LoadError( "{}: {}\n".format(p, e), LoadErrorReason.INVALID_DATA, detail="The min-version must be specified as MAJOR.MINOR with " + "numeric major and minor minimum required version numbers", ) from e # Future proofing, in case there is ever a BuildStream 3 if min_version_major != bst_major: p = min_version_node.get_provenance() raise LoadError( "{}: Version mismatch".format(p), LoadErrorReason.UNSUPPORTED_PROJECT, detail="Project requires BuildStream {}, ".format(min_version_major) + "but BuildStream {} is installed.\n".format(bst_major) + "Please use BuildStream {} with this project.".format(min_version_major), ) # Check minimal minor point requirement is satisfied if min_version_minor > bst_minor: p = min_version_node.get_provenance() raise LoadError( "{}: Version mismatch".format(p), LoadErrorReason.UNSUPPORTED_PROJECT, detail="Project requires at least BuildStream {}.{}, ".format(min_version_major, min_version_minor) + "but BuildStream {}.{} is installed.\n".format(bst_major, bst_minor) + "Please upgrade BuildStream.", ) # _load(): # # Loads the project configuration file in the project # directory process the first pass. # # Raises: LoadError if there was a problem with the project.conf # def _load(self, *, parent_loader=None, provenance_node=None): # Load builtin default projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE) self._default_config_node = _yaml.load(_site.default_project_config, shortname="projectconfig.yaml") # Load project local config and override the builtin try: self._project_conf = _yaml.load(projectfile, shortname=_PROJECT_CONF_FILE, project=self) except LoadError as e: # Raise a more specific error here if e.reason == LoadErrorReason.MISSING_FILE: raise LoadError(str(e), LoadErrorReason.MISSING_PROJECT_CONF) from e # Otherwise re-raise the original exception raise pre_config_node = self._default_config_node.clone() self._project_conf._composite(pre_config_node) # Assert project's minimum required version early, before validating toplevel keys self._validate_version(pre_config_node) self._validate_toplevel_node(pre_config_node, first_pass=True) # The project name, element path and option declarations # are constant and cannot be overridden by option conditional statements # FIXME: we should be keeping node information for further composition here self.name = self._project_conf.get_str("name") # Validate that project name is a valid symbol name _assert_symbol_name(self.name, "project name", ref_node=pre_config_node.get_node("name")) self.element_path = os.path.join( self.directory, self.get_path_from_node(pre_config_node.get_scalar("element-path"), check_is_dir=True) ) self.config.options = OptionPool(self.element_path) self.first_pass_config.options = OptionPool(self.element_path) defaults = pre_config_node.get_mapping("defaults") defaults.validate_keys(["targets"]) self._default_targets = defaults.get_str_list("targets") # Fatal warnings self._fatal_warnings = pre_config_node.get_str_list("fatal-warnings", default=[]) # Junction configuration junctions_node = pre_config_node.get_mapping("junctions", default={}) junctions_node.validate_keys(["duplicates", "internal", "disallow-subproject-uris"]) if self.junction and self.junction._get_project().disallow_subproject_uris: # If the parent project doesn't allow subproject URIs, this must # be enforced for nested subprojects as well. self.disallow_subproject_uris = True # The `disallow-subproject-uris` flag also implies fatal `unaliased-url` in subprojects # to ensure no subproject URIs escape the parent project's control. if CoreWarnings.UNALIASED_URL not in self._fatal_warnings: self._fatal_warnings.append(CoreWarnings.UNALIASED_URL) else: self.disallow_subproject_uris = junctions_node.get_bool("disallow-subproject-uris", default=False) # Parse duplicates junction_duplicates = junctions_node.get_mapping("duplicates", default={}) for project_name, junctions in junction_duplicates.items(): self._junction_duplicates[project_name] = junctions # Parse internal self._junction_internal = junctions_node.get_sequence("internal", default=[]) self.loader = Loader(self, parent=parent_loader, provenance_node=provenance_node) self._project_includes = Includes(self.loader, copy_tree=False) project_conf_first_pass = self._project_conf.clone() self._project_includes.process(project_conf_first_pass, only_local=True, process_project_options=False) config_no_include = self._default_config_node.clone() project_conf_first_pass._composite(config_no_include) # Plugin factories must be defined in project.conf, not included from elsewhere. self._load_plugin_factories(config_no_include) self._load_pass(config_no_include, self.first_pass_config, ignore_unknown=True) # Use separate file for storing source references ref_storage_node = pre_config_node.get_scalar("ref-storage") self.ref_storage = ref_storage_node.as_str() if self.ref_storage not in [ProjectRefStorage.INLINE, ProjectRefStorage.PROJECT_REFS]: p = ref_storage_node.get_provenance() raise LoadError( "{}: Invalid value '{}' specified for ref-storage".format(p, self.ref_storage), LoadErrorReason.INVALID_DATA, ) if self.ref_storage == ProjectRefStorage.PROJECT_REFS: self.junction_refs.load(self.first_pass_config.options) # _load_second_pass() # # Process the second pass of loading the project configuration. # def _load_second_pass(self): project_conf_second_pass = self._project_conf.clone() self._project_includes.process(project_conf_second_pass, process_project_options=False) config = self._default_config_node.clone() project_conf_second_pass._composite(config) self._load_pass(config, self.config) self._validate_toplevel_node(config, first_pass=False) # # Now all YAML composition is done, from here on we just load # the values from our loaded configuration dictionary. # # Load artifact remote specs caches = config.get_sequence("artifacts", default=[], allowed_types=[MappingNode]) for node in caches: spec = RemoteSpec.new_from_node(node, self.directory) self.artifact_cache_specs.append(spec) # Load source cache remote specs caches = config.get_sequence("source-caches", default=[], allowed_types=[MappingNode]) for node in caches: spec = RemoteSpec.new_from_node(node, self.directory) self.source_cache_specs.append(spec) # Load sandbox environment variables self.base_environment = config.get_mapping("environment") self.base_env_nocache = config.get_str_list("environment-nocache") # Load sandbox configuration self.sandbox = config.get_mapping("sandbox") # Load project split rules self.splits = config.get_mapping("split-rules") # Support backwards compatibility for fail-on-overlap fail_on_overlap = config.get_scalar("fail-on-overlap", None) # Deprecation check if not fail_on_overlap.is_none(): self._context.messenger.warn( "Use of fail-on-overlap within project.conf " + "is deprecated. Consider using fatal-warnings instead.", ) if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap.as_bool(): self._fatal_warnings.append(CoreWarnings.OVERLAPS) # Load project.refs if it exists, this may be ignored. if self.ref_storage == ProjectRefStorage.PROJECT_REFS: self.refs.load(self.options) # Parse shell options shell_options = config.get_mapping("shell") shell_options.validate_keys(["command", "environment", "host-files"]) self._shell_command = shell_options.get_str_list("command") # Perform environment expansion right away shell_environment = shell_options.get_mapping("environment", default={}) for key in shell_environment.keys(): value = shell_environment.get_str(key) self._shell_environment[key] = os.path.expandvars(value) # Host files is parsed as a list for convenience host_files = shell_options.get_sequence("host-files", default=[]) for host_file in host_files: if isinstance(host_file, ScalarNode): mount = _HostMount(host_file.as_str()) else: # Some validation host_file.validate_keys(["path", "host_path", "optional"]) # Parse the host mount path = host_file.get_str("path") host_path = host_file.get_str("host_path", default=None) optional = host_file.get_bool("optional", default=False) mount = _HostMount(path, host_path, optional) self._shell_host_files.append(mount) # We don't want to combine the source provenance attributes that a project defines with the defaults # If the project config defines source-provenance-attributes use that, otherwise fall back to the defaults # This is purely to maintain backwards compatibility with homepage and issue-tracker being available self.source_provenance_attributes = project_conf_second_pass.get_mapping( "source-provenance-attributes", None ) or config.get_mapping("source-provenance-attributes") # _load_pass(): # # Loads parts of the project configuration that are different # for first and second pass configurations. # # Args: # config (dict) - YaML node of the configuration file. # output (ProjectConfig) - ProjectConfig to load configuration onto. # ignore_unknown (bool) - Whether option loader shoud ignore unknown options. # def _load_pass(self, config, output, *, ignore_unknown=False): # Load project options options_node = config.get_mapping("options", default={}) output.options.load(options_node) if self.junction: # load before user configuration output.options.load_yaml_values(self.junction.options) # Collect option values specified in the user configuration overrides = self._context.get_overrides(self.name) override_options = overrides.get_mapping("options", default={}) output.options.load_yaml_values(override_options) if self._cli_options: output.options.load_cli_values(self._cli_options, ignore_unknown=ignore_unknown) # We're done modifying options, now we can use them for substitutions output.options.resolve() # # Now resolve any conditionals in the remaining configuration, # any conditionals specified for project option declarations, # or conditionally specifying the project name; will be ignored. # # Specify any options that would be ignored in the restrict list # so as to raise an appropriate error. # output.options.process_node( config, restricted=[ "min-version", "name", "element-path", "junctions", "defaults", "fatal-warnings", "ref-storage", "options", "plugins", ], ) # Element and Source type configurations will be composited later onto # element/source types, so we delete it from here and run our final # assertion after. output.element_overrides = config.get_mapping("elements", default={}) output.source_overrides = config.get_mapping("sources", default={}) config.safe_del("elements") config.safe_del("sources") config._assert_fully_composited() # Load base variables output.base_variables = config.get_mapping("variables") # Add the project name as a default variable output.base_variables["project-name"] = self.name # Extend variables with automatic variables and option exports # Initialize it as a string as all variables are processed as strings. output.base_variables["max-jobs"] = str(self._context.effective_build_max_jobs) # Export options into variables, if that was requested output.options.export_variables(output.base_variables) # Prepare a Variables instance for substitution of source alias and # source mirror values. # # This allows substitution of any project-level variables, plus the special # variables which allow resolving project relative directories on the host. # toplevel_project = self._context.get_toplevel_project() variables_node = output.base_variables.clone() variables_node["project-root"] = str(self._absolute_directory_path) variables_node["toplevel-root"] = str(toplevel_project._absolute_directory_path) variables_node["project-root-uri"] = "file://" + urllib.parse.quote(str(self._absolute_directory_path)) variables_node["toplevel-root-uri"] = "file://" + urllib.parse.quote( str(toplevel_project._absolute_directory_path) ) variables = Variables(variables_node) # Override default_mirror if not set by command-line output.default_mirror = self._default_mirror or overrides.get_str("default-mirror", default=None) # Source url aliases output._aliases = config.get_mapping("aliases", default={}) # First try mirrors specified in user configuration, user configuration # is allowed to completely disable mirrors by specifying an empty list, # so we check for a None value here too. # mirrors_node = overrides.get_sequence("mirrors", default=None) if mirrors_node is None: mirrors_node = config.get_sequence("mirrors", default=[]) else: self._mirror_override = True # Perform variable substitutions in source mirror definitions, # even if the mirrors are specified in user configuration. variables.expand(mirrors_node) # Collect SourceMirror objects for mirror_node in mirrors_node: mirror = self.source_mirror_factory.create(self._context, self, mirror_node) output.mirrors[mirror.name] = mirror if not output.default_mirror: output.default_mirror = mirror.name # Perform variable substitutions in source aliases variables.expand(output._aliases) # _find_project_dir() # # Returns path of the project directory, if a configuration file is found # in given directory or any of its parent directories. # # Args: # directory (str) - directory from where the command was invoked # # Raises: # LoadError if project.conf is not found # # Returns: # (str) - the directory that contains the project, and # (str) - the name of the element required to find the project, or None # def _find_project_dir(self, directory): workspace_element = None config_filenames = [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE] found_directory, filename = utils._search_upward_for_files(directory, config_filenames) if filename == _PROJECT_CONF_FILE: project_directory = found_directory elif filename == WORKSPACE_PROJECT_FILE: workspace_project_cache = self._context.get_workspace_project_cache() workspace_project = workspace_project_cache.get(found_directory) project_directory = workspace_project.get_default_project_path() workspace_element = workspace_project.get_default_element() else: raise LoadError( "None of {names} found in '{path}' or any of its parent directories".format( names=config_filenames, path=directory ), LoadErrorReason.MISSING_PROJECT_CONF, ) return project_directory, workspace_element # _load_plugin_factories() # # Loads the plugin factories # # Args: # config (MappingNode): The main project.conf node in the first pass # def _load_plugin_factories(self, config): # Create the factories pluginbase = PluginBase(package="buildstream.plugins") self.element_factory = ElementFactory(pluginbase) self.source_factory = SourceFactory(pluginbase) self.source_mirror_factory = SourceMirrorFactory(pluginbase) # Load the plugin origins and register them to their factories origins = config.get_sequence("plugins", default=[]) for origin_node in origins: origin = load_plugin_origin(self, origin_node) for kind, conf in origin.elements.items(): self.element_factory.register_plugin_origin(kind, origin, conf.allow_deprecated) for kind, conf in origin.sources.items(): self.source_factory.register_plugin_origin(kind, origin, conf.allow_deprecated) for kind, conf in origin.source_mirrors.items(): self.source_mirror_factory.register_plugin_origin(kind, origin, conf.allow_deprecated) # _warning_is_fatal(): # # Returns true if the warning in question should be considered fatal based on # the project configuration. # # Args: # warning_str (str): The warning configuration string to check against # # Returns: # (bool): True if the warning should be considered fatal and cause an error. # def _warning_is_fatal(self, warning_str): return warning_str in self._fatal_warnings apache-buildstream-27ae392/src/buildstream/_projectrefs.py000066400000000000000000000114711514607367700237670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os from . import _yaml from .node import _new_synthetic_file from ._exceptions import LoadError from .exceptions import LoadErrorReason # ProjectRefStorage() # # Indicates the type of ref storage class ProjectRefStorage: # Source references are stored inline # INLINE = "inline" # Source references are stored in a central project.refs file # PROJECT_REFS = "project.refs" # ProjectRefs() # # The project.refs file management # # Args: # directory (str): The project directory # base_name (str): The project.refs basename # class ProjectRefs: def __init__(self, directory, base_name): directory = os.path.abspath(directory) self._fullpath = os.path.join(directory, base_name) self._base_name = base_name self._toplevel_node = None self._toplevel_save = None # load() # # Load the project.refs file # # Args: # options (OptionPool): To resolve conditional statements # def load(self, options): try: self._toplevel_node = _yaml.load(self._fullpath, shortname=self._base_name, copy_tree=True) provenance = self._toplevel_node.get_provenance() self._toplevel_save = provenance._toplevel # Process any project options immediately options.process_node(self._toplevel_node) # Run any final assertions on the project.refs, just incase there # are list composition directives or anything left unprocessed. self._toplevel_node._assert_fully_composited() except LoadError as e: if e.reason != LoadErrorReason.MISSING_FILE: raise # Ignore failure if the file doesnt exist, it'll be created and # for now just assumed to be empty self._toplevel_node = _new_synthetic_file(self._fullpath) self._toplevel_save = self._toplevel_node self._toplevel_node.validate_keys(["projects"]) # Ensure we create our toplevel entry point on the fly here for node in [self._toplevel_node, self._toplevel_save]: if "projects" not in node: node["projects"] = {} # lookup_ref() # # Fetch the ref node for a given Source. If the ref node does not # exist and `write` is specified, it will be automatically created. # # Args: # project (str): The project to lookup # element (str): The element name to lookup # source_index (int): The index of the Source in the specified element # write (bool): Whether we want to read the node or write to it # # Returns: # (node): The YAML dictionary where the ref is stored # def lookup_ref(self, project, element, source_index, *, write=False): node = self._lookup(self._toplevel_node, project, element, source_index) if write: # If we couldnt find the orignal, create a new one. # if node is None: node = self._lookup(self._toplevel_save, project, element, source_index, ensure=True) return node # _lookup() # # Looks up a ref node in the project.refs file, creates one if ensure is True. # def _lookup(self, toplevel, project, element, source_index, *, ensure=False): projects = toplevel.get_mapping("projects") # Fetch the project try: project_node = projects.get_mapping(project) except LoadError: if not ensure: return None projects[project] = {} project_node = projects.get_mapping(project) # Fetch the element try: element_list = project_node.get_sequence(element) except LoadError: if not ensure: return None project_node[element] = [] element_list = project_node.get_sequence(element) # Fetch the source index try: node = element_list.mapping_at(source_index) except IndexError: if not ensure: return None while len(element_list) <= source_index: element_list.append({}) node = element_list.mapping_at(source_index) return node apache-buildstream-27ae392/src/buildstream/_protos/000077500000000000000000000000001514607367700224115ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/__init__.py000066400000000000000000000000001514607367700245100ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/000077500000000000000000000000001514607367700235105ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/__init__.py000066400000000000000000000000001514607367700256070ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/000077500000000000000000000000001514607367700246055ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/__init__.py000066400000000000000000000000001514607367700267040ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/000077500000000000000000000000001514607367700261005ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/__init__.py000066400000000000000000000000001514607367700301770ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/000077500000000000000000000000001514607367700272175ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/__init__.py000066400000000000000000000000001514607367700313160ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/v1/000077500000000000000000000000001514607367700275455ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/v1/__init__.py000066400000000000000000000000001514607367700316440ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/v1/remote_asset.proto000066400000000000000000000572621514607367700333400ustar00rootroot00000000000000// Copyright 2020 The Bazel Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package build.bazel.remote.asset.v1; import "build/bazel/remote/execution/v2/remote_execution.proto"; import "google/api/annotations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "google/rpc/status.proto"; option csharp_namespace = "Build.Bazel.Remote.Asset.v1"; option go_package = "github.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1;remoteasset"; option java_multiple_files = true; option java_outer_classname = "RemoteAssetProto"; option java_package = "build.bazel.remote.asset.v1"; option objc_class_prefix = "RA"; // The Remote Asset API provides a mapping from a URI and Qualifiers to // Digests. // // Multiple URIs may be used to refer to the same content. For example, the // same tarball may exist at multiple mirrors and thus be retrievable from // multiple URLs. When URLs are used, these should refer to actual content as // Fetch service implementations may choose to fetch the content directly // from the origin. For example, the HEAD of a git repository's active branch // can be referred to as: // // uri: https://github.com/bazelbuild/remote-apis.git // // URNs may be used to strongly identify content, for instance by using the // uuid namespace identifier: urn:uuid:f81d4fae-7dec-11d0-a765-00a0c91e6bf6. // This is most applicable to named content that is Push'd, where the URN // serves as an agreed-upon key, but carries no other inherent meaning. // // Service implementations may choose to support only URLs, only URNs for // Push'd content, only other URIs for which the server and client agree upon // semantics of, or any mixture of the above. // Qualifiers are used to disambiguate or sub-select content that shares a URI. // This may include specifying a particular commit or branch, in the case of // URIs referencing a repository; they could also be used to specify a // particular subdirectory of a repository or tarball. Qualifiers may also be // used to ensure content matches what the client expects, even when there is // no ambiguity to be had - for example, a qualifier specifying a checksum // value. // // In cases where the semantics of the request are not immediately clear from // the URL and/or qualifiers - e.g. dictated by URL scheme - it is recommended // to use an additional qualifier to remove the ambiguity. The `resource_type` // qualifier is recommended for this purpose. // // Qualifiers may be supplied in any order. message Qualifier { // The "name" of the qualifier, for example "resource_type". // No separation is made between 'standard' and 'nonstandard' // qualifiers, in accordance with https://tools.ietf.org/html/rfc6648, // however implementers *SHOULD* take care to avoid ambiguity. string name = 1; // The "value" of the qualifier. Semantics will be dictated by the name. string value = 2; } // The Fetch service resolves or fetches assets referenced by URI and // Qualifiers, returning a Digest for the content in // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. // // As with other services in the Remote Execution API, any call may return an // error with a [RetryInfo][google.rpc.RetryInfo] error detail providing // information about when the client should retry the request; clients SHOULD // respect the information provided. service Fetch { // Resolve or fetch referenced assets, making them available to the caller and // other consumers in the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. // // Servers *MAY* fetch content that they do not already have cached, for any // URLs they support. // // Servers *SHOULD* ensure that referenced files are present in the CAS at the // time of the response, and (if supported) that they will remain available // for a reasonable period of time. The lifetimes of the referenced blobs *SHOULD* // be increased if necessary and applicable. // In the event that a client receives a reference to content that is no // longer present, it *MAY* re-issue the request with // `oldest_content_accepted` set to a more recent timestamp than the original // attempt, to induce a re-fetch from origin. // // Servers *MAY* cache fetched content and reuse it for subsequent requests, // subject to `oldest_content_accepted`. // // Servers *MAY* support the complementary [Push][build.bazel.remote.asset.v1.Push] // API and allow content to be directly inserted for use in future fetch // responses. // // Servers *MUST* ensure Fetch'd content matches all the specified // qualifiers except in the case of previously Push'd resources, for which // the server *MAY* trust the pushing client to have set the qualifiers // correctly, without validation. // // Servers not implementing the complementary [Push][build.bazel.remote.asset.v1.Push] // API *MUST* reject requests containing qualifiers it does not support. // // Servers *MAY* transform assets as part of the fetch. For example a // tarball fetched by [FetchDirectory][build.bazel.remote.asset.v1.Fetch.FetchDirectory] // might be unpacked, or a Git repository // fetched by [FetchBlob][build.bazel.remote.asset.v1.Fetch.FetchBlob] // might be passed through `git-archive`. // // Errors handling the requested assets will be returned as gRPC Status errors // here; errors outside the server's control will be returned inline in the // `status` field of the response (see comment there for details). // The possible RPC errors include: // * `INVALID_ARGUMENT`: One or more arguments were invalid, such as a // qualifier that is not supported by the server. // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to // perform the requested operation. The client may retry after a delay. // * `UNAVAILABLE`: Due to a transient condition the operation could not be // completed. The client should retry. // * `INTERNAL`: An internal error occurred while performing the operation. // The client should retry. // * `DEADLINE_EXCEEDED`: The fetch could not be completed within the given // RPC deadline. The client should retry for at least as long as the value // provided in `timeout` field of the request. // // In the case of unsupported qualifiers, the server *SHOULD* additionally // send a [BadRequest][google.rpc.BadRequest] error detail where, for each // unsupported qualifier, there is a `FieldViolation` with a `field` of // `qualifiers.name` and a `description` of `"{qualifier}" not supported` // indicating the name of the unsupported qualifier. rpc FetchBlob(FetchBlobRequest) returns (FetchBlobResponse) { option (google.api.http) = { post: "/v1/{instance_name=**}/assets:fetchBlob" body: "*" }; } rpc FetchDirectory(FetchDirectoryRequest) returns (FetchDirectoryResponse) { option (google.api.http) = { post: "/v1/{instance_name=**}/assets:fetchDirectory" body: "*" }; } } // A request message for // [Fetch.FetchBlob][build.bazel.remote.asset.v1.Fetch.FetchBlob]. message FetchBlobRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The timeout for the underlying fetch, if content needs to be retrieved from // origin. // // If unset, the server *MAY* apply an implementation-defined timeout. // // If set, and the user-provided timeout exceeds the RPC deadline, the server // *SHOULD* keep the fetch going after the RPC completes, to be made // available for future Fetch calls. The server may also enforce (via clamping // and/or an INVALID_ARGUMENT error) implementation-defined minimum and // maximum timeout values. // // If this timeout is exceeded on an attempt to retrieve content from origin // the client will receive DEADLINE_EXCEEDED in [FetchBlobResponse.status]. google.protobuf.Duration timeout = 2; // The oldest content the client is willing to accept, as measured from the // time it was Push'd or when the underlying retrieval from origin was // started. // Upon retries of Fetch requests that cannot be completed within a single // RPC, clients *SHOULD* provide the same value for subsequent requests as the // original, to simplify combining the request with the previous attempt. // // If unset, the client *SHOULD* accept content of any age. google.protobuf.Timestamp oldest_content_accepted = 3; // The URI(s) of the content to fetch. These may be resources that the server // can directly fetch from origin, in which case multiple URIs *SHOULD* // represent the same content available at different locations (such as an // origin and secondary mirrors). These may also be URIs for content known to // the server through other mechanisms, e.g. pushed via the [Push][build.bazel.remote.asset.v1.Push] // service. // // Clients *MUST* supply at least one URI. Servers *MAY* match any one of the // supplied URIs. repeated string uris = 4; // Qualifiers sub-specifying the content to fetch - see comments on // [Qualifier][build.bazel.remote.asset.v1.Qualifier]. // The same qualifiers apply to all URIs. // // Specified qualifier names *MUST* be unique. repeated Qualifier qualifiers = 5; // The digest function the server must use to compute the digest. // // If unset, the server SHOULD default to SHA256. build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 6; } // A response message for // [Fetch.FetchBlob][build.bazel.remote.asset.v1.Fetch.FetchBlob]. message FetchBlobResponse { // If the status has a code other than `OK`, it indicates that the operation // was unable to be completed for reasons outside the servers' control. // The possible fetch errors include: // * `DEADLINE_EXCEEDED`: The operation could not be completed within the // specified timeout. // * `NOT_FOUND`: The requested asset was not found at the specified location. // * `PERMISSION_DENIED`: The request was rejected by a remote server, or // requested an asset from a disallowed origin. // * `ABORTED`: The operation could not be completed, typically due to a // failed consistency check. // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to // perform the requested operation. The client may retry after a delay. google.rpc.Status status = 1; // The uri from the request that resulted in a successful retrieval, or from // which the error indicated in `status` was obtained. string uri = 2; // Any qualifiers known to the server and of interest to clients. repeated Qualifier qualifiers = 3; // A minimum timestamp the content is expected to be available through. // Servers *MAY* omit this field, if not known with confidence. google.protobuf.Timestamp expires_at = 4; // The result of the fetch, if the status had code `OK`. // The digest of the file's contents, available for download through the CAS. build.bazel.remote.execution.v2.Digest blob_digest = 5; // This field SHOULD be set to the digest function that was used by the server // to compute [FetchBlobResponse.blob_digest]. // Clients could use this to determine whether the server honors // [FetchBlobRequest.digest_function] that was set in the request. // // If unset, clients SHOULD default to use SHA256 regardless of the requested // [FetchBlobRequest.digest_function]. build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 6; } // A request message for // [Fetch.FetchDirectory][build.bazel.remote.asset.v1.Fetch.FetchDirectory]. message FetchDirectoryRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The timeout for the underlying fetch, if content needs to be retrieved from // origin. This value is allowed to exceed the RPC deadline, in which case the // server *SHOULD* keep the fetch going after the RPC completes, to be made // available for future Fetch calls. // // If this timeout is exceeded on an attempt to retrieve content from origin // the client will receive DEADLINE_EXCEEDED in [FetchDirectoryResponse.status]. google.protobuf.Duration timeout = 2; // The oldest content the client is willing to accept, as measured from the // time it was Push'd or when the underlying retrieval from origin was // started. // Upon retries of Fetch requests that cannot be completed within a single // RPC, clients *SHOULD* provide the same value for subsequent requests as the // original, to simplify combining the request with the previous attempt. // // If unset, the client *SHOULD* accept content of any age. google.protobuf.Timestamp oldest_content_accepted = 3; // The URI(s) of the content to fetch. These may be resources that the server // can directly fetch from origin, in which case multiple URIs *SHOULD* // represent the same content available at different locations (such as an // origin and secondary mirrors). These may also be URIs for content known to // the server through other mechanisms, e.g. pushed via the [Push][build.bazel.remote.asset.v1.Push] // service. // // Clients *MUST* supply at least one URI. Servers *MAY* match any one of the // supplied URIs. repeated string uris = 4; // Qualifiers sub-specifying the content to fetch - see comments on // [Qualifier][build.bazel.remote.asset.v1.Qualifier]. // The same qualifiers apply to all URIs. // // Specified qualifier names *MUST* be unique. repeated Qualifier qualifiers = 5; // The digest function the server must use to compute the digest. // // If unset, the server SHOULD default to SHA256. build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 6; } // A response message for // [Fetch.FetchDirectory][build.bazel.remote.asset.v1.Fetch.FetchDirectory]. message FetchDirectoryResponse { // If the status has a code other than `OK`, it indicates that the operation // was unable to be completed for reasons outside the servers' control. // The possible fetch errors include: // * `DEADLINE_EXCEEDED`: The operation could not be completed within the // specified timeout. // * `NOT_FOUND`: The requested asset was not found at the specified location. // * `PERMISSION_DENIED`: The request was rejected by a remote server, or // requested an asset from a disallowed origin. // * `ABORTED`: The operation could not be completed, typically due to a // failed consistency check. // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to // perform the requested operation. The client may retry after a delay. google.rpc.Status status = 1; // The uri from the request that resulted in a successful retrieval, or from // which the error indicated in `status` was obtained. string uri = 2; // Any qualifiers known to the server and of interest to clients. repeated Qualifier qualifiers = 3; // A minimum timestamp the content is expected to be available through. // Servers *MAY* omit this field, if not known with confidence. google.protobuf.Timestamp expires_at = 4; // The result of the fetch, if the status had code `OK`. // the root digest of a directory tree, suitable for fetching via // [ContentAddressableStorage.GetTree]. build.bazel.remote.execution.v2.Digest root_directory_digest = 5; // This field SHOULD be set to the digest function that was used by the server // to compute [FetchBlobResponse.root_directory_digest]. // Clients could use this to determine whether the server honors // [FetchDirectoryRequest.digest_function] that was set in the request. // // If unset, clients SHOULD default to use SHA256 regardless of the requested // [FetchDirectoryRequest.digest_function]. build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 6; } // The Push service is complementary to the Fetch, and allows for // associating contents of URLs to be returned in future Fetch API calls. // // As with other services in the Remote Execution API, any call may return an // error with a [RetryInfo][google.rpc.RetryInfo] error detail providing // information about when the client should retry the request; clients SHOULD // respect the information provided. service Push { // These APIs associate the identifying information of a resource, as // indicated by URI and optionally Qualifiers, with content available in the // CAS. For example, associating a repository url and a commit id with a // Directory Digest. // // Servers *SHOULD* only allow trusted clients to associate content, and *MAY* // only allow certain URIs to be pushed. // // Clients *MUST* ensure associated content is available in CAS prior to // pushing. // // Clients *MUST* ensure the Qualifiers listed correctly match the contents, // and Servers *MAY* trust these values without validation. // Fetch servers *MAY* require exact match of all qualifiers when returning // content previously pushed, or allow fetching content with only a subset of // the qualifiers specified on Push. // // Clients can specify expiration information that the server *SHOULD* // respect. Subsequent requests can be used to alter the expiration time. // // A minimal compliant Fetch implementation may support only Push'd content // and return `NOT_FOUND` for any resource that was not pushed first. // Alternatively, a compliant implementation may choose to not support Push // and only return resources that can be Fetch'd from origin. // // Errors will be returned as gRPC Status errors. // The possible RPC errors include: // * `INVALID_ARGUMENT`: One or more arguments to the RPC were invalid. // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to // perform the requested operation. The client may retry after a delay. // * `UNAVAILABLE`: Due to a transient condition the operation could not be // completed. The client should retry. // * `INTERNAL`: An internal error occurred while performing the operation. // The client should retry. rpc PushBlob(PushBlobRequest) returns (PushBlobResponse) { option (google.api.http) = { post: "/v1/{instance_name=**}/assets:pushBlob" body: "*" }; } rpc PushDirectory(PushDirectoryRequest) returns (PushDirectoryResponse) { option (google.api.http) = { post: "/v1/{instance_name=**}/assets:pushDirectory" body: "*" }; } } // A request message for // [Push.PushBlob][build.bazel.remote.asset.v1.Push.PushBlob]. message PushBlobRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The URI(s) of the content to associate. If multiple URIs are specified, the // pushed content will be available to fetch by specifying any of them. repeated string uris = 2; // Qualifiers sub-specifying the content that is being pushed - see comments // on [Qualifier][build.bazel.remote.asset.v1.Qualifier]. // The same qualifiers apply to all URIs. repeated Qualifier qualifiers = 3; // A time after which this content should stop being returned via [FetchBlob][build.bazel.remote.asset.v1.Fetch.FetchBlob]. // Servers *MAY* expire content early, e.g. due to storage pressure. google.protobuf.Timestamp expire_at = 4; // The blob to associate. build.bazel.remote.execution.v2.Digest blob_digest = 5; // Referenced blobs or directories that need to not expire before expiration // of this association, in addition to `blob_digest` itself. // These fields are hints - clients *MAY* omit them, and servers *SHOULD* // respect them, at the risk of increased incidents of Fetch responses // indirectly referencing unavailable blobs. repeated build.bazel.remote.execution.v2.Digest references_blobs = 6; repeated build.bazel.remote.execution.v2.Digest references_directories = 7; // The digest function that was used to compute the blob digest. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the action digest hash and the digest functions announced // in the server's capabilities. build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 8; } // A response message for // [Push.PushBlob][build.bazel.remote.asset.v1.Push.PushBlob]. message PushBlobResponse { /* empty */ } // A request message for // [Push.PushDirectory][build.bazel.remote.asset.v1.Push.PushDirectory]. message PushDirectoryRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The URI(s) of the content to associate. If multiple URIs are specified, the // pushed content will be available to fetch by specifying any of them. repeated string uris = 2; // Qualifiers sub-specifying the content that is being pushed - see comments // on [Qualifier][build.bazel.remote.asset.v1.Qualifier]. // The same qualifiers apply to all URIs. repeated Qualifier qualifiers = 3; // A time after which this content should stop being returned via // [FetchDirectory][build.bazel.remote.asset.v1.Fetch.FetchDirectory]. // Servers *MAY* expire content early, e.g. due to storage pressure. google.protobuf.Timestamp expire_at = 4; // Directory to associate build.bazel.remote.execution.v2.Digest root_directory_digest = 5; // Referenced blobs or directories that need to not expire before expiration // of this association, in addition to `root_directory_digest` itself. // These fields are hints - clients *MAY* omit them, and servers *SHOULD* // respect them, at the risk of increased incidents of Fetch responses // indirectly referencing unavailable blobs. repeated build.bazel.remote.execution.v2.Digest references_blobs = 6; repeated build.bazel.remote.execution.v2.Digest references_directories = 7; // The digest function that was used to compute blob digests. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the action digest hash and the digest functions announced // in the server's capabilities. build.bazel.remote.execution.v2.DigestFunction.Value digest_function = 8; } // A response message for // [Push.PushDirectory][build.bazel.remote.asset.v1.Push.PushDirectory]. message PushDirectoryResponse { /* empty */ } apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/v1/remote_asset_pb2.py000066400000000000000000000210321514607367700333520ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: build/bazel/remote/asset/v1/remote_asset.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'build/bazel/remote/asset/v1/remote_asset.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from buildstream._protos.google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.build/bazel/remote/asset/v1/remote_asset.proto\x12\x1b\x62uild.bazel.remote.asset.v1\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\"(\n\tQualifier\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\xac\x02\n\x10\x46\x65tchBlobRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12*\n\x07timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12;\n\x17oldest_content_accepted\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04uris\x18\x04 \x03(\t\x12:\n\nqualifiers\x18\x05 \x03(\x0b\x32&.build.bazel.remote.asset.v1.Qualifier\x12N\n\x0f\x64igest_function\x18\x06 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xbe\x02\n\x11\x46\x65tchBlobResponse\x12\"\n\x06status\x18\x01 \x01(\x0b\x32\x12.google.rpc.Status\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12:\n\nqualifiers\x18\x03 \x03(\x0b\x32&.build.bazel.remote.asset.v1.Qualifier\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12<\n\x0b\x62lob_digest\x18\x05 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12N\n\x0f\x64igest_function\x18\x06 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xb1\x02\n\x15\x46\x65tchDirectoryRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12*\n\x07timeout\x18\x02 \x01(\x0b\x32\x19.google.protobuf.Duration\x12;\n\x17oldest_content_accepted\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04uris\x18\x04 \x03(\t\x12:\n\nqualifiers\x18\x05 \x03(\x0b\x32&.build.bazel.remote.asset.v1.Qualifier\x12N\n\x0f\x64igest_function\x18\x06 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xcd\x02\n\x16\x46\x65tchDirectoryResponse\x12\"\n\x06status\x18\x01 \x01(\x0b\x32\x12.google.rpc.Status\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12:\n\nqualifiers\x18\x03 \x03(\x0b\x32&.build.bazel.remote.asset.v1.Qualifier\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x15root_directory_digest\x18\x05 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12N\n\x0f\x64igest_function\x18\x06 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xbb\x03\n\x0fPushBlobRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04uris\x18\x02 \x03(\t\x12:\n\nqualifiers\x18\x03 \x03(\x0b\x32&.build.bazel.remote.asset.v1.Qualifier\x12-\n\texpire_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12<\n\x0b\x62lob_digest\x18\x05 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x41\n\x10references_blobs\x18\x06 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12G\n\x16references_directories\x18\x07 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12N\n\x0f\x64igest_function\x18\x08 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\x12\n\x10PushBlobResponse\"\xca\x03\n\x14PushDirectoryRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04uris\x18\x02 \x03(\t\x12:\n\nqualifiers\x18\x03 \x03(\x0b\x32&.build.bazel.remote.asset.v1.Qualifier\x12-\n\texpire_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x15root_directory_digest\x18\x05 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x41\n\x10references_blobs\x18\x06 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12G\n\x16references_directories\x18\x07 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12N\n\x0f\x64igest_function\x18\x08 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\x17\n\x15PushDirectoryResponse2\xdd\x02\n\x05\x46\x65tch\x12\x9e\x01\n\tFetchBlob\x12-.build.bazel.remote.asset.v1.FetchBlobRequest\x1a..build.bazel.remote.asset.v1.FetchBlobResponse\"2\x82\xd3\xe4\x93\x02,\"\'/v1/{instance_name=**}/assets:fetchBlob:\x01*\x12\xb2\x01\n\x0e\x46\x65tchDirectory\x12\x32.build.bazel.remote.asset.v1.FetchDirectoryRequest\x1a\x33.build.bazel.remote.asset.v1.FetchDirectoryResponse\"7\x82\xd3\xe4\x93\x02\x31\",/v1/{instance_name=**}/assets:fetchDirectory:\x01*2\xd4\x02\n\x04Push\x12\x9a\x01\n\x08PushBlob\x12,.build.bazel.remote.asset.v1.PushBlobRequest\x1a-.build.bazel.remote.asset.v1.PushBlobResponse\"1\x82\xd3\xe4\x93\x02+\"&/v1/{instance_name=**}/assets:pushBlob:\x01*\x12\xae\x01\n\rPushDirectory\x12\x31.build.bazel.remote.asset.v1.PushDirectoryRequest\x1a\x32.build.bazel.remote.asset.v1.PushDirectoryResponse\"6\x82\xd3\xe4\x93\x02\x30\"+/v1/{instance_name=**}/assets:pushDirectory:\x01*B\x9f\x01\n\x1b\x62uild.bazel.remote.asset.v1B\x10RemoteAssetProtoP\x01ZIgithub.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1;remoteasset\xa2\x02\x02RA\xaa\x02\x1b\x42uild.Bazel.Remote.Asset.v1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'build.bazel.remote.asset.v1.remote_asset_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\033build.bazel.remote.asset.v1B\020RemoteAssetProtoP\001ZIgithub.com/bazelbuild/remote-apis/build/bazel/remote/asset/v1;remoteasset\242\002\002RA\252\002\033Build.Bazel.Remote.Asset.v1' _globals['_FETCH'].methods_by_name['FetchBlob']._loaded_options = None _globals['_FETCH'].methods_by_name['FetchBlob']._serialized_options = b'\202\323\344\223\002,\"\'/v1/{instance_name=**}/assets:fetchBlob:\001*' _globals['_FETCH'].methods_by_name['FetchDirectory']._loaded_options = None _globals['_FETCH'].methods_by_name['FetchDirectory']._serialized_options = b'\202\323\344\223\0021\",/v1/{instance_name=**}/assets:fetchDirectory:\001*' _globals['_PUSH'].methods_by_name['PushBlob']._loaded_options = None _globals['_PUSH'].methods_by_name['PushBlob']._serialized_options = b'\202\323\344\223\002+\"&/v1/{instance_name=**}/assets:pushBlob:\001*' _globals['_PUSH'].methods_by_name['PushDirectory']._loaded_options = None _globals['_PUSH'].methods_by_name['PushDirectory']._serialized_options = b'\202\323\344\223\0020\"+/v1/{instance_name=**}/assets:pushDirectory:\001*' _globals['_QUALIFIER']._serialized_start=255 _globals['_QUALIFIER']._serialized_end=295 _globals['_FETCHBLOBREQUEST']._serialized_start=298 _globals['_FETCHBLOBREQUEST']._serialized_end=598 _globals['_FETCHBLOBRESPONSE']._serialized_start=601 _globals['_FETCHBLOBRESPONSE']._serialized_end=919 _globals['_FETCHDIRECTORYREQUEST']._serialized_start=922 _globals['_FETCHDIRECTORYREQUEST']._serialized_end=1227 _globals['_FETCHDIRECTORYRESPONSE']._serialized_start=1230 _globals['_FETCHDIRECTORYRESPONSE']._serialized_end=1563 _globals['_PUSHBLOBREQUEST']._serialized_start=1566 _globals['_PUSHBLOBREQUEST']._serialized_end=2009 _globals['_PUSHBLOBRESPONSE']._serialized_start=2011 _globals['_PUSHBLOBRESPONSE']._serialized_end=2029 _globals['_PUSHDIRECTORYREQUEST']._serialized_start=2032 _globals['_PUSHDIRECTORYREQUEST']._serialized_end=2490 _globals['_PUSHDIRECTORYRESPONSE']._serialized_start=2492 _globals['_PUSHDIRECTORYRESPONSE']._serialized_end=2515 _globals['_FETCH']._serialized_start=2518 _globals['_FETCH']._serialized_end=2867 _globals['_PUSH']._serialized_start=2870 _globals['_PUSH']._serialized_end=3210 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/v1/remote_asset_pb2.pyi000066400000000000000000000216051514607367700335310ustar00rootroot00000000000000from build.bazel.remote.execution.v2 import remote_execution_pb2 as _remote_execution_pb2 from google.api import annotations_pb2 as _annotations_pb2 from google.protobuf import duration_pb2 as _duration_pb2 from google.protobuf import timestamp_pb2 as _timestamp_pb2 from google.rpc import status_pb2 as _status_pb2 from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Qualifier(_message.Message): __slots__ = ("name", "value") NAME_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] name: str value: str def __init__(self, name: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... class FetchBlobRequest(_message.Message): __slots__ = ("instance_name", "timeout", "oldest_content_accepted", "uris", "qualifiers", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] TIMEOUT_FIELD_NUMBER: _ClassVar[int] OLDEST_CONTENT_ACCEPTED_FIELD_NUMBER: _ClassVar[int] URIS_FIELD_NUMBER: _ClassVar[int] QUALIFIERS_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str timeout: _duration_pb2.Duration oldest_content_accepted: _timestamp_pb2.Timestamp uris: _containers.RepeatedScalarFieldContainer[str] qualifiers: _containers.RepeatedCompositeFieldContainer[Qualifier] digest_function: _remote_execution_pb2.DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., oldest_content_accepted: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., uris: _Optional[_Iterable[str]] = ..., qualifiers: _Optional[_Iterable[_Union[Qualifier, _Mapping]]] = ..., digest_function: _Optional[_Union[_remote_execution_pb2.DigestFunction.Value, str]] = ...) -> None: ... class FetchBlobResponse(_message.Message): __slots__ = ("status", "uri", "qualifiers", "expires_at", "blob_digest", "digest_function") STATUS_FIELD_NUMBER: _ClassVar[int] URI_FIELD_NUMBER: _ClassVar[int] QUALIFIERS_FIELD_NUMBER: _ClassVar[int] EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] BLOB_DIGEST_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] status: _status_pb2.Status uri: str qualifiers: _containers.RepeatedCompositeFieldContainer[Qualifier] expires_at: _timestamp_pb2.Timestamp blob_digest: _remote_execution_pb2.Digest digest_function: _remote_execution_pb2.DigestFunction.Value def __init__(self, status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., uri: _Optional[str] = ..., qualifiers: _Optional[_Iterable[_Union[Qualifier, _Mapping]]] = ..., expires_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., blob_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., digest_function: _Optional[_Union[_remote_execution_pb2.DigestFunction.Value, str]] = ...) -> None: ... class FetchDirectoryRequest(_message.Message): __slots__ = ("instance_name", "timeout", "oldest_content_accepted", "uris", "qualifiers", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] TIMEOUT_FIELD_NUMBER: _ClassVar[int] OLDEST_CONTENT_ACCEPTED_FIELD_NUMBER: _ClassVar[int] URIS_FIELD_NUMBER: _ClassVar[int] QUALIFIERS_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str timeout: _duration_pb2.Duration oldest_content_accepted: _timestamp_pb2.Timestamp uris: _containers.RepeatedScalarFieldContainer[str] qualifiers: _containers.RepeatedCompositeFieldContainer[Qualifier] digest_function: _remote_execution_pb2.DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., oldest_content_accepted: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., uris: _Optional[_Iterable[str]] = ..., qualifiers: _Optional[_Iterable[_Union[Qualifier, _Mapping]]] = ..., digest_function: _Optional[_Union[_remote_execution_pb2.DigestFunction.Value, str]] = ...) -> None: ... class FetchDirectoryResponse(_message.Message): __slots__ = ("status", "uri", "qualifiers", "expires_at", "root_directory_digest", "digest_function") STATUS_FIELD_NUMBER: _ClassVar[int] URI_FIELD_NUMBER: _ClassVar[int] QUALIFIERS_FIELD_NUMBER: _ClassVar[int] EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] ROOT_DIRECTORY_DIGEST_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] status: _status_pb2.Status uri: str qualifiers: _containers.RepeatedCompositeFieldContainer[Qualifier] expires_at: _timestamp_pb2.Timestamp root_directory_digest: _remote_execution_pb2.Digest digest_function: _remote_execution_pb2.DigestFunction.Value def __init__(self, status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., uri: _Optional[str] = ..., qualifiers: _Optional[_Iterable[_Union[Qualifier, _Mapping]]] = ..., expires_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., root_directory_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., digest_function: _Optional[_Union[_remote_execution_pb2.DigestFunction.Value, str]] = ...) -> None: ... class PushBlobRequest(_message.Message): __slots__ = ("instance_name", "uris", "qualifiers", "expire_at", "blob_digest", "references_blobs", "references_directories", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] URIS_FIELD_NUMBER: _ClassVar[int] QUALIFIERS_FIELD_NUMBER: _ClassVar[int] EXPIRE_AT_FIELD_NUMBER: _ClassVar[int] BLOB_DIGEST_FIELD_NUMBER: _ClassVar[int] REFERENCES_BLOBS_FIELD_NUMBER: _ClassVar[int] REFERENCES_DIRECTORIES_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str uris: _containers.RepeatedScalarFieldContainer[str] qualifiers: _containers.RepeatedCompositeFieldContainer[Qualifier] expire_at: _timestamp_pb2.Timestamp blob_digest: _remote_execution_pb2.Digest references_blobs: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] references_directories: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] digest_function: _remote_execution_pb2.DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., uris: _Optional[_Iterable[str]] = ..., qualifiers: _Optional[_Iterable[_Union[Qualifier, _Mapping]]] = ..., expire_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., blob_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., references_blobs: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., references_directories: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., digest_function: _Optional[_Union[_remote_execution_pb2.DigestFunction.Value, str]] = ...) -> None: ... class PushBlobResponse(_message.Message): __slots__ = () def __init__(self) -> None: ... class PushDirectoryRequest(_message.Message): __slots__ = ("instance_name", "uris", "qualifiers", "expire_at", "root_directory_digest", "references_blobs", "references_directories", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] URIS_FIELD_NUMBER: _ClassVar[int] QUALIFIERS_FIELD_NUMBER: _ClassVar[int] EXPIRE_AT_FIELD_NUMBER: _ClassVar[int] ROOT_DIRECTORY_DIGEST_FIELD_NUMBER: _ClassVar[int] REFERENCES_BLOBS_FIELD_NUMBER: _ClassVar[int] REFERENCES_DIRECTORIES_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str uris: _containers.RepeatedScalarFieldContainer[str] qualifiers: _containers.RepeatedCompositeFieldContainer[Qualifier] expire_at: _timestamp_pb2.Timestamp root_directory_digest: _remote_execution_pb2.Digest references_blobs: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] references_directories: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] digest_function: _remote_execution_pb2.DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., uris: _Optional[_Iterable[str]] = ..., qualifiers: _Optional[_Iterable[_Union[Qualifier, _Mapping]]] = ..., expire_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., root_directory_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., references_blobs: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., references_directories: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., digest_function: _Optional[_Union[_remote_execution_pb2.DigestFunction.Value, str]] = ...) -> None: ... class PushDirectoryResponse(_message.Message): __slots__ = () def __init__(self) -> None: ... remote_asset_pb2_grpc.py000066400000000000000000000437711514607367700343240ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/asset/v1# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings from buildstream._protos.build.bazel.remote.asset.v1 import remote_asset_pb2 as build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2 GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in build/bazel/remote/asset/v1/remote_asset_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) class FetchStub(object): """The Fetch service resolves or fetches assets referenced by URI and Qualifiers, returning a Digest for the content in [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.FetchBlob = channel.unary_unary( '/build.bazel.remote.asset.v1.Fetch/FetchBlob', request_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchBlobRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchBlobResponse.FromString, _registered_method=True) self.FetchDirectory = channel.unary_unary( '/build.bazel.remote.asset.v1.Fetch/FetchDirectory', request_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchDirectoryRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchDirectoryResponse.FromString, _registered_method=True) class FetchServicer(object): """The Fetch service resolves or fetches assets referenced by URI and Qualifiers, returning a Digest for the content in [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def FetchBlob(self, request, context): """Resolve or fetch referenced assets, making them available to the caller and other consumers in the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. Servers *MAY* fetch content that they do not already have cached, for any URLs they support. Servers *SHOULD* ensure that referenced files are present in the CAS at the time of the response, and (if supported) that they will remain available for a reasonable period of time. The lifetimes of the referenced blobs *SHOULD* be increased if necessary and applicable. In the event that a client receives a reference to content that is no longer present, it *MAY* re-issue the request with `oldest_content_accepted` set to a more recent timestamp than the original attempt, to induce a re-fetch from origin. Servers *MAY* cache fetched content and reuse it for subsequent requests, subject to `oldest_content_accepted`. Servers *MAY* support the complementary [Push][build.bazel.remote.asset.v1.Push] API and allow content to be directly inserted for use in future fetch responses. Servers *MUST* ensure Fetch'd content matches all the specified qualifiers except in the case of previously Push'd resources, for which the server *MAY* trust the pushing client to have set the qualifiers correctly, without validation. Servers not implementing the complementary [Push][build.bazel.remote.asset.v1.Push] API *MUST* reject requests containing qualifiers it does not support. Servers *MAY* transform assets as part of the fetch. For example a tarball fetched by [FetchDirectory][build.bazel.remote.asset.v1.Fetch.FetchDirectory] might be unpacked, or a Git repository fetched by [FetchBlob][build.bazel.remote.asset.v1.Fetch.FetchBlob] might be passed through `git-archive`. Errors handling the requested assets will be returned as gRPC Status errors here; errors outside the server's control will be returned inline in the `status` field of the response (see comment there for details). The possible RPC errors include: * `INVALID_ARGUMENT`: One or more arguments were invalid, such as a qualifier that is not supported by the server. * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to perform the requested operation. The client may retry after a delay. * `UNAVAILABLE`: Due to a transient condition the operation could not be completed. The client should retry. * `INTERNAL`: An internal error occurred while performing the operation. The client should retry. * `DEADLINE_EXCEEDED`: The fetch could not be completed within the given RPC deadline. The client should retry for at least as long as the value provided in `timeout` field of the request. In the case of unsupported qualifiers, the server *SHOULD* additionally send a [BadRequest][google.rpc.BadRequest] error detail where, for each unsupported qualifier, there is a `FieldViolation` with a `field` of `qualifiers.name` and a `description` of `"{qualifier}" not supported` indicating the name of the unsupported qualifier. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def FetchDirectory(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_FetchServicer_to_server(servicer, server): rpc_method_handlers = { 'FetchBlob': grpc.unary_unary_rpc_method_handler( servicer.FetchBlob, request_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchBlobRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchBlobResponse.SerializeToString, ), 'FetchDirectory': grpc.unary_unary_rpc_method_handler( servicer.FetchDirectory, request_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchDirectoryRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchDirectoryResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.bazel.remote.asset.v1.Fetch', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.bazel.remote.asset.v1.Fetch', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class Fetch(object): """The Fetch service resolves or fetches assets referenced by URI and Qualifiers, returning a Digest for the content in [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ @staticmethod def FetchBlob(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.asset.v1.Fetch/FetchBlob', build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchBlobRequest.SerializeToString, build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchBlobResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def FetchDirectory(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.asset.v1.Fetch/FetchDirectory', build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchDirectoryRequest.SerializeToString, build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.FetchDirectoryResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) class PushStub(object): """The Push service is complementary to the Fetch, and allows for associating contents of URLs to be returned in future Fetch API calls. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.PushBlob = channel.unary_unary( '/build.bazel.remote.asset.v1.Push/PushBlob', request_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushBlobRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushBlobResponse.FromString, _registered_method=True) self.PushDirectory = channel.unary_unary( '/build.bazel.remote.asset.v1.Push/PushDirectory', request_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushDirectoryRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushDirectoryResponse.FromString, _registered_method=True) class PushServicer(object): """The Push service is complementary to the Fetch, and allows for associating contents of URLs to be returned in future Fetch API calls. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def PushBlob(self, request, context): """These APIs associate the identifying information of a resource, as indicated by URI and optionally Qualifiers, with content available in the CAS. For example, associating a repository url and a commit id with a Directory Digest. Servers *SHOULD* only allow trusted clients to associate content, and *MAY* only allow certain URIs to be pushed. Clients *MUST* ensure associated content is available in CAS prior to pushing. Clients *MUST* ensure the Qualifiers listed correctly match the contents, and Servers *MAY* trust these values without validation. Fetch servers *MAY* require exact match of all qualifiers when returning content previously pushed, or allow fetching content with only a subset of the qualifiers specified on Push. Clients can specify expiration information that the server *SHOULD* respect. Subsequent requests can be used to alter the expiration time. A minimal compliant Fetch implementation may support only Push'd content and return `NOT_FOUND` for any resource that was not pushed first. Alternatively, a compliant implementation may choose to not support Push and only return resources that can be Fetch'd from origin. Errors will be returned as gRPC Status errors. The possible RPC errors include: * `INVALID_ARGUMENT`: One or more arguments to the RPC were invalid. * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to perform the requested operation. The client may retry after a delay. * `UNAVAILABLE`: Due to a transient condition the operation could not be completed. The client should retry. * `INTERNAL`: An internal error occurred while performing the operation. The client should retry. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def PushDirectory(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_PushServicer_to_server(servicer, server): rpc_method_handlers = { 'PushBlob': grpc.unary_unary_rpc_method_handler( servicer.PushBlob, request_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushBlobRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushBlobResponse.SerializeToString, ), 'PushDirectory': grpc.unary_unary_rpc_method_handler( servicer.PushDirectory, request_deserializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushDirectoryRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushDirectoryResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.bazel.remote.asset.v1.Push', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.bazel.remote.asset.v1.Push', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class Push(object): """The Push service is complementary to the Fetch, and allows for associating contents of URLs to be returned in future Fetch API calls. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ @staticmethod def PushBlob(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.asset.v1.Push/PushBlob', build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushBlobRequest.SerializeToString, build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushBlobResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def PushDirectory(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.asset.v1.Push/PushDirectory', build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushDirectoryRequest.SerializeToString, build_dot_bazel_dot_remote_dot_asset_dot_v1_dot_remote__asset__pb2.PushDirectoryResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/000077500000000000000000000000001514607367700301035ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/__init__.py000066400000000000000000000000001514607367700322020ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/v2/000077500000000000000000000000001514607367700304325ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/v2/__init__.py000066400000000000000000000000001514607367700325310ustar00rootroot00000000000000remote_execution.proto000066400000000000000000002761311514607367700350300ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/v2// Copyright 2018 The Bazel Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package build.bazel.remote.execution.v2; import "build/bazel/semver/semver.proto"; import "google/api/annotations.proto"; import "google/longrunning/operations.proto"; import "google/protobuf/any.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/timestamp.proto"; import "google/protobuf/wrappers.proto"; import "google/rpc/status.proto"; option csharp_namespace = "Build.Bazel.Remote.Execution.V2"; option go_package = "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2;remoteexecution"; option java_multiple_files = true; option java_outer_classname = "RemoteExecutionProto"; option java_package = "build.bazel.remote.execution.v2"; option objc_class_prefix = "REX"; // The Remote Execution API is used to execute an // [Action][build.bazel.remote.execution.v2.Action] on the remote // workers. // // As with other services in the Remote Execution API, any call may return an // error with a [RetryInfo][google.rpc.RetryInfo] error detail providing // information about when the client should retry the request; clients SHOULD // respect the information provided. service Execution { // Execute an action remotely. // // In order to execute an action, the client must first upload all of the // inputs, the // [Command][build.bazel.remote.execution.v2.Command] to run, and the // [Action][build.bazel.remote.execution.v2.Action] into the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. // It then calls `Execute` with an `action_digest` referring to them. The // server will run the action and eventually return the result. // // The input `Action`'s fields MUST meet the various canonicalization // requirements specified in the documentation for their types so that it has // the same digest as other logically equivalent `Action`s. The server MAY // enforce the requirements and return errors if a non-canonical input is // received. It MAY also proceed without verifying some or all of the // requirements, such as for performance reasons. If the server does not // verify the requirement, then it will treat the `Action` as distinct from // another logically equivalent action if they hash differently. // // Returns a stream of // [google.longrunning.Operation][google.longrunning.Operation] messages // describing the resulting execution, with eventual `response` // [ExecuteResponse][build.bazel.remote.execution.v2.ExecuteResponse]. The // `metadata` on the operation is of type // [ExecuteOperationMetadata][build.bazel.remote.execution.v2.ExecuteOperationMetadata]. // // If the client remains connected after the first response is returned after // the server, then updates are streamed as if the client had called // [WaitExecution][build.bazel.remote.execution.v2.Execution.WaitExecution] // until the execution completes or the request reaches an error. The // operation can also be queried using [Operations // API][google.longrunning.Operations.GetOperation]. // // The server NEED NOT implement other methods or functionality of the // Operations API. // // Errors discovered during creation of the `Operation` will be reported // as gRPC Status errors, while errors that occurred while running the // action will be reported in the `status` field of the `ExecuteResponse`. The // server MUST NOT set the `error` field of the `Operation` proto. // The possible errors include: // // * `INVALID_ARGUMENT`: One or more arguments are invalid. // * `FAILED_PRECONDITION`: One or more errors occurred in setting up the // action requested, such as a missing input or command or no worker being // available. The client may be able to fix the errors and retry. // * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to run // the action. // * `UNAVAILABLE`: Due to a transient condition, such as all workers being // occupied (and the server does not support a queue), the action could not // be started. The client should retry. // * `INTERNAL`: An internal error occurred in the execution engine or the // worker. // * `DEADLINE_EXCEEDED`: The execution timed out. // * `CANCELLED`: The operation was cancelled by the client. This status is // only possible if the server implements the Operations API CancelOperation // method, and it was called for the current execution. // // In the case of a missing input or command, the server SHOULD additionally // send a [PreconditionFailure][google.rpc.PreconditionFailure] error detail // where, for each requested blob not present in the CAS, there is a // `Violation` with a `type` of `MISSING` and a `subject` of // `"blobs/{digest_function/}{hash}/{size}"` indicating the digest of the // missing blob. The `subject` is formatted the same way as the // `resource_name` provided to // [ByteStream.Read][google.bytestream.ByteStream.Read], with the leading // instance name omitted. `digest_function` MUST thus be omitted if its value // is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, or VSO. // // The server does not need to guarantee that a call to this method leads to // at most one execution of the action. The server MAY execute the action // multiple times, potentially in parallel. These redundant executions MAY // continue to run, even if the operation is completed. rpc Execute(ExecuteRequest) returns (stream google.longrunning.Operation) { option (google.api.http) = { post: "/v2/{instance_name=**}/actions:execute" body: "*" }; } // Wait for an execution operation to complete. When the client initially // makes the request, the server immediately responds with the current status // of the execution. The server will leave the request stream open until the // operation completes, and then respond with the completed operation. The // server MAY choose to stream additional updates as execution progresses, // such as to provide an update as to the state of the execution. // // In addition to the cases describe for Execute, the WaitExecution method // may fail as follows: // // * `NOT_FOUND`: The operation no longer exists due to any of a transient // condition, an unknown operation name, or if the server implements the // Operations API DeleteOperation method and it was called for the current // execution. The client should call `Execute` to retry. rpc WaitExecution(WaitExecutionRequest) returns (stream google.longrunning.Operation) { option (google.api.http) = { post: "/v2/{name=operations/**}:waitExecution" body: "*" }; } } // The action cache API is used to query whether a given action has already been // performed and, if so, retrieve its result. Unlike the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage], // which addresses blobs by their own content, the action cache addresses the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] by a // digest of the encoded [Action][build.bazel.remote.execution.v2.Action] // which produced them. // // The lifetime of entries in the action cache is implementation-specific, but // the server SHOULD assume that more recently used entries are more likely to // be used again. // // As with other services in the Remote Execution API, any call may return an // error with a [RetryInfo][google.rpc.RetryInfo] error detail providing // information about when the client should retry the request; clients SHOULD // respect the information provided. service ActionCache { // Retrieve a cached execution result. // // Implementations SHOULD ensure that any blobs referenced from the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] // are available at the time of returning the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] and will be // for some period of time afterwards. The lifetimes of the referenced blobs SHOULD be increased // if necessary and applicable. // // Errors: // // * `NOT_FOUND`: The requested `ActionResult` is not in the cache. rpc GetActionResult(GetActionResultRequest) returns (ActionResult) { option (google.api.http) = { get: "/v2/{instance_name=**}/actionResults/{action_digest.hash}/{action_digest.size_bytes}" }; } // Upload a new execution result. // // In order to allow the server to perform access control based on the type of // action, and to assist with client debugging, the client MUST first upload // the [Action][build.bazel.remote.execution.v2.Execution] that produced the // result, along with its // [Command][build.bazel.remote.execution.v2.Command], into the // `ContentAddressableStorage`. // // Server implementations MAY modify the // `UpdateActionResultRequest.action_result` and return an equivalent value. // // Errors: // // * `INVALID_ARGUMENT`: One or more arguments are invalid. // * `FAILED_PRECONDITION`: One or more errors occurred in updating the // action result, such as a missing command or action. // * `RESOURCE_EXHAUSTED`: There is insufficient storage space to add the // entry to the cache. rpc UpdateActionResult(UpdateActionResultRequest) returns (ActionResult) { option (google.api.http) = { put: "/v2/{instance_name=**}/actionResults/{action_digest.hash}/{action_digest.size_bytes}" body: "action_result" }; } } // The CAS (content-addressable storage) is used to store the inputs to and // outputs from the execution service. Each piece of content is addressed by the // digest of its binary data. // // Most of the binary data stored in the CAS is opaque to the execution engine, // and is only used as a communication medium. In order to build an // [Action][build.bazel.remote.execution.v2.Action], // however, the client will need to also upload the // [Command][build.bazel.remote.execution.v2.Command] and input root // [Directory][build.bazel.remote.execution.v2.Directory] for the Action. // The Command and Directory messages must be marshalled to wire format and then // uploaded under the hash as with any other piece of content. In practice, the // input root directory is likely to refer to other Directories in its // hierarchy, which must also each be uploaded on their own. // // For small file uploads the client should group them together and call // [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]. // // For large uploads, the client must use the // [Write method][google.bytestream.ByteStream.Write] of the ByteStream API. // // For uncompressed data, The `WriteRequest.resource_name` is of the following form: // `{instance_name}/uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}` // // Where: // * `instance_name` is an identifier used to distinguish between the various // instances on the server. Syntax and semantics of this field are defined // by the server; Clients must not make any assumptions about it (e.g., // whether it spans multiple path segments or not). If it is the empty path, // the leading slash is omitted, so that the `resource_name` becomes // `uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}`. // To simplify parsing, a path segment cannot equal any of the following // keywords: `blobs`, `uploads`, `actions`, `actionResults`, `operations`, // `capabilities` or `compressed-blobs`. // * `uuid` is a version 4 UUID generated by the client, used to avoid // collisions between concurrent uploads of the same data. Clients MAY // reuse the same `uuid` for uploading different blobs. // * `digest_function` is a lowercase string form of a `DigestFunction.Value` // enum, indicating which digest function was used to compute `hash`. If the // digest function used is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, // or VSO, this component MUST be omitted. In that case the server SHOULD // infer the digest function using the length of the `hash` and the digest // functions announced in the server's capabilities. // * `hash` and `size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] // of the data being uploaded. // * `optional_metadata` is implementation specific data, which clients MAY omit. // Servers MAY ignore this metadata. // // Data can alternatively be uploaded in compressed form, with the following // `WriteRequest.resource_name` form: // `{instance_name}/uploads/{uuid}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}{/optional_metadata}` // // Where: // * `instance_name`, `uuid`, `digest_function` and `optional_metadata` are // defined as above. // * `compressor` is a lowercase string form of a `Compressor.Value` enum // other than `identity`, which is supported by the server and advertised in // [CacheCapabilities.supported_compressor][build.bazel.remote.execution.v2.CacheCapabilities.supported_compressor]. // * `uncompressed_hash` and `uncompressed_size` refer to the // [Digest][build.bazel.remote.execution.v2.Digest] of the data being // uploaded, once uncompressed. Servers MUST verify that these match // the uploaded data once uncompressed, and MUST return an // `INVALID_ARGUMENT` error in the case of mismatch. // // Note that when writing compressed blobs, the `WriteRequest.write_offset` in // the initial request in a stream refers to the offset in the uncompressed form // of the blob. In subsequent requests, `WriteRequest.write_offset` MUST be the // sum of the first request's 'WriteRequest.write_offset' and the total size of // all the compressed data bundles in the previous requests. // Note that this mixes an uncompressed offset with a compressed byte length, // which is nonsensical, but it is done to fit the semantics of the existing // ByteStream protocol. // // Uploads of the same data MAY occur concurrently in any form, compressed or // uncompressed. // // Clients SHOULD NOT use gRPC-level compression for ByteStream API `Write` // calls of compressed blobs, since this would compress already-compressed data. // // When attempting an upload, if another client has already completed the upload // (which may occur in the middle of a single upload if another client uploads // the same blob concurrently), the request will terminate immediately without // error, and with a response whose `committed_size` is the value `-1` if this // is a compressed upload, or with the full size of the uploaded file if this is // an uncompressed upload (regardless of how much data was transmitted by the // client). If the client completes the upload but the // [Digest][build.bazel.remote.execution.v2.Digest] does not match, an // `INVALID_ARGUMENT` error will be returned. In either case, the client should // not attempt to retry the upload. // // Small downloads can be grouped and requested in a batch via // [BatchReadBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchReadBlobs]. // // For large downloads, the client must use the // [Read method][google.bytestream.ByteStream.Read] of the ByteStream API. // // For uncompressed data, The `ReadRequest.resource_name` is of the following form: // `{instance_name}/blobs/{digest_function/}{hash}/{size}` // Where `instance_name`, `digest_function`, `hash` and `size` are defined as // for uploads. // // Data can alternatively be downloaded in compressed form, with the following // `ReadRequest.resource_name` form: // `{instance_name}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}` // // Where: // * `instance_name`, `compressor` and `digest_function` are defined as for // uploads. // * `uncompressed_hash` and `uncompressed_size` refer to the // [Digest][build.bazel.remote.execution.v2.Digest] of the data being // downloaded, once uncompressed. Clients MUST verify that these match // the downloaded data once uncompressed, and take appropriate steps in // the case of failure such as retrying a limited number of times or // surfacing an error to the user. // // When downloading compressed blobs: // * `ReadRequest.read_offset` refers to the offset in the uncompressed form // of the blob. // * Servers MUST return `INVALID_ARGUMENT` if `ReadRequest.read_limit` is // non-zero. // * Servers MAY use any compression level they choose, including different // levels for different blobs (e.g. choosing a level designed for maximum // speed for data known to be incompressible). // * Clients SHOULD NOT use gRPC-level compression, since this would compress // already-compressed data. // // Servers MUST be able to provide data for all recently advertised blobs in // each of the compression formats that the server supports, as well as in // uncompressed form. // // The lifetime of entries in the CAS is implementation specific, but it SHOULD // be long enough to allow for newly-added and recently looked-up entries to be // used in subsequent calls (e.g. to // [Execute][build.bazel.remote.execution.v2.Execution.Execute]). // // Servers MUST behave as though empty blobs are always available, even if they // have not been uploaded. Clients MAY optimize away the uploading or // downloading of empty blobs. // // As with other services in the Remote Execution API, any call may return an // error with a [RetryInfo][google.rpc.RetryInfo] error detail providing // information about when the client should retry the request; clients SHOULD // respect the information provided. service ContentAddressableStorage { // Determine if blobs are present in the CAS. // // Clients can use this API before uploading blobs to determine which ones are // already present in the CAS and do not need to be uploaded again. // // Servers SHOULD increase the lifetimes of the referenced blobs if necessary and // applicable. // // There are no method-specific errors. rpc FindMissingBlobs(FindMissingBlobsRequest) returns (FindMissingBlobsResponse) { option (google.api.http) = { post: "/v2/{instance_name=**}/blobs:findMissing" body: "*" }; } // Upload many blobs at once. // // The server may enforce a limit of the combined total size of blobs // to be uploaded using this API. This limit may be obtained using the // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. // Requests exceeding the limit should either be split into smaller // chunks or uploaded using the // [ByteStream API][google.bytestream.ByteStream], as appropriate. // // This request is equivalent to calling a Bytestream `Write` request // on each individual blob, in parallel. The requests may succeed or fail // independently. // // Errors: // // * `INVALID_ARGUMENT`: The client attempted to upload more than the // server supported limit. // // Individual requests may return the following errors, additionally: // // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the blob. // * `INVALID_ARGUMENT`: The // [Digest][build.bazel.remote.execution.v2.Digest] does not match the // provided data. rpc BatchUpdateBlobs(BatchUpdateBlobsRequest) returns (BatchUpdateBlobsResponse) { option (google.api.http) = { post: "/v2/{instance_name=**}/blobs:batchUpdate" body: "*" }; } // Download many blobs at once. // // The server may enforce a limit of the combined total size of blobs // to be downloaded using this API. This limit may be obtained using the // [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. // Requests exceeding the limit should either be split into smaller // chunks or downloaded using the // [ByteStream API][google.bytestream.ByteStream], as appropriate. // // This request is equivalent to calling a Bytestream `Read` request // on each individual blob, in parallel. The requests may succeed or fail // independently. // // Errors: // // * `INVALID_ARGUMENT`: The client attempted to read more than the // server supported limit. // // Every error on individual read will be returned in the corresponding digest // status. rpc BatchReadBlobs(BatchReadBlobsRequest) returns (BatchReadBlobsResponse) { option (google.api.http) = { post: "/v2/{instance_name=**}/blobs:batchRead" body: "*" }; } // Fetch the entire directory tree rooted at a node. // // This request must be targeted at a // [Directory][build.bazel.remote.execution.v2.Directory] stored in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] // (CAS). The server will enumerate the `Directory` tree recursively and // return every node descended from the root. // // The GetTreeRequest.page_token parameter can be used to skip ahead in // the stream (e.g. when retrying a partially completed and aborted request), // by setting it to a value taken from GetTreeResponse.next_page_token of the // last successfully processed GetTreeResponse). // // The exact traversal order is unspecified and, unless retrieving subsequent // pages from an earlier request, is not guaranteed to be stable across // multiple invocations of `GetTree`. // // If part of the tree is missing from the CAS, the server will return the // portion present and omit the rest. // // Errors: // // * `NOT_FOUND`: The requested tree root is not present in the CAS. rpc GetTree(GetTreeRequest) returns (stream GetTreeResponse) { option (google.api.http) = { get: "/v2/{instance_name=**}/blobs/{root_digest.hash}/{root_digest.size_bytes}:getTree" }; } } // The Capabilities service may be used by remote execution clients to query // various server properties, in order to self-configure or return meaningful // error messages. // // The query may include a particular `instance_name`, in which case the values // returned will pertain to that instance. service Capabilities { // GetCapabilities returns the server capabilities configuration of the // remote endpoint. // Only the capabilities of the services supported by the endpoint will // be returned: // * Execution + CAS + Action Cache endpoints should return both // CacheCapabilities and ExecutionCapabilities. // * Execution only endpoints should return ExecutionCapabilities. // * CAS + Action Cache only endpoints should return CacheCapabilities. // // There are no method-specific errors. rpc GetCapabilities(GetCapabilitiesRequest) returns (ServerCapabilities) { option (google.api.http) = { get: "/v2/{instance_name=**}/capabilities" }; } } // An `Action` captures all the information about an execution which is required // to reproduce it. // // `Action`s are the core component of the [Execution] service. A single // `Action` represents a repeatable action that can be performed by the // execution service. `Action`s can be succinctly identified by the digest of // their wire format encoding and, once an `Action` has been executed, will be // cached in the action cache. Future requests can then use the cached result // rather than needing to run afresh. // // When a server completes execution of an // [Action][build.bazel.remote.execution.v2.Action], it MAY choose to // cache the [result][build.bazel.remote.execution.v2.ActionResult] in // the [ActionCache][build.bazel.remote.execution.v2.ActionCache] unless // `do_not_cache` is `true`. Clients SHOULD expect the server to do so. By // default, future calls to // [Execute][build.bazel.remote.execution.v2.Execution.Execute] the same // `Action` will also serve their results from the cache. Clients must take care // to understand the caching behaviour. Ideally, all `Action`s will be // reproducible so that serving a result from cache is always desirable and // correct. message Action { // The digest of the [Command][build.bazel.remote.execution.v2.Command] // to run, which MUST be present in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. Digest command_digest = 1; // The digest of the root // [Directory][build.bazel.remote.execution.v2.Directory] for the input // files. The files in the directory tree are available in the correct // location on the build machine before the command is executed. The root // directory, as well as every subdirectory and content blob referred to, MUST // be in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. Digest input_root_digest = 2; reserved 3 to 5; // Used for fields moved to [Command][build.bazel.remote.execution.v2.Command]. // A timeout after which the execution should be killed. If the timeout is // absent, then the client is specifying that the execution should continue // as long as the server will let it. The server SHOULD impose a timeout if // the client does not specify one, however, if the client does specify a // timeout that is longer than the server's maximum timeout, the server MUST // reject the request. // // The timeout is only intended to cover the "execution" of the specified // action and not time in queue nor any overheads before or after execution // such as marshalling inputs/outputs. The server SHOULD avoid including time // spent the client doesn't have control over, and MAY extend or reduce the // timeout to account for delays or speedups that occur during execution // itself (e.g., lazily loading data from the Content Addressable Storage, // live migration of virtual machines, emulation overhead). // // The timeout is a part of the // [Action][build.bazel.remote.execution.v2.Action] message, and // therefore two `Actions` with different timeouts are different, even if they // are otherwise identical. This is because, if they were not, running an // `Action` with a lower timeout than is required might result in a cache hit // from an execution run with a longer timeout, hiding the fact that the // timeout is too short. By encoding it directly in the `Action`, a lower // timeout will result in a cache miss and the execution timeout will fail // immediately, rather than whenever the cache entry gets evicted. google.protobuf.Duration timeout = 6; // If true, then the `Action`'s result cannot be cached, and in-flight // requests for the same `Action` may not be merged. bool do_not_cache = 7; reserved 8; // Used for field moved to [Command][build.bazel.remote.execution.v2.Command]. // An optional additional salt value used to place this `Action` into a // separate cache namespace from other instances having the same field // contents. This salt typically comes from operational configuration // specific to sources such as repo and service configuration, // and allows disowning an entire set of ActionResults that might have been // poisoned by buggy software or tool failures. bytes salt = 9; // The optional platform requirements for the execution environment. The // server MAY choose to execute the action on any worker satisfying the // requirements, so the client SHOULD ensure that running the action on any // such worker will have the same result. A detailed lexicon for this can be // found in the accompanying platform.md. // New in version 2.2: clients SHOULD set these platform properties as well // as those in the [Command][build.bazel.remote.execution.v2.Command]. Servers // SHOULD prefer those set here. Platform platform = 10; } // A `Command` is the actual command executed by a worker running an // [Action][build.bazel.remote.execution.v2.Action] and specifications of its // environment. // // Except as otherwise required, the environment (such as which system // libraries or binaries are available, and what filesystems are mounted where) // is defined by and specific to the implementation of the remote execution API. message Command { // An `EnvironmentVariable` is one variable to set in the running program's // environment. message EnvironmentVariable { // The variable name. string name = 1; // The variable value. string value = 2; } // The arguments to the command. // // The first argument specifies the command to run, which may be either an // absolute path, a path relative to the working directory, or an unqualified // path (without path separators) which will be resolved using the operating // system's equivalent of the PATH environment variable. Path separators // native to the operating system running on the worker SHOULD be used. If the // `environment_variables` list contains an entry for the PATH environment // variable, it SHOULD be respected. If not, the resolution process is // implementation-defined. // // Changed in v2.3. v2.2 and older require that no PATH lookups are performed, // and that relative paths are resolved relative to the input root. This // behavior can, however, not be relied upon, as most implementations already // followed the rules described above. repeated string arguments = 1; // The environment variables to set when running the program. The worker may // provide its own default environment variables; these defaults can be // overridden using this field. Additional variables can also be specified. // // In order to ensure that equivalent // [Command][build.bazel.remote.execution.v2.Command]s always hash to the same // value, the environment variables MUST be lexicographically sorted by name. // Sorting of strings is done by code point, equivalently, by the UTF-8 bytes. repeated EnvironmentVariable environment_variables = 2; // A list of the output files that the client expects to retrieve from the // action. Only the listed files, as well as directories listed in // `output_directories`, will be returned to the client as output. // Other files or directories that may be created during command execution // are discarded. // // The paths are relative to the working directory of the action execution. // The paths are specified using a single forward slash (`/`) as a path // separator, even if the execution platform natively uses a different // separator. The path MUST NOT include a trailing slash, nor a leading slash, // being a relative path. // // In order to ensure consistent hashing of the same Action, the output paths // MUST be sorted lexicographically by code point (or, equivalently, by UTF-8 // bytes). // // An output file cannot be duplicated, be a parent of another output file, or // have the same path as any of the listed output directories. // // Directories leading up to the output files are created by the worker prior // to execution, even if they are not explicitly part of the input root. // // DEPRECATED since v2.1: Use `output_paths` instead. repeated string output_files = 3 [ deprecated = true ]; // A list of the output directories that the client expects to retrieve from // the action. Only the listed directories will be returned (an entire // directory structure will be returned as a // [Tree][build.bazel.remote.execution.v2.Tree] message digest, see // [OutputDirectory][build.bazel.remote.execution.v2.OutputDirectory]), as // well as files listed in `output_files`. Other files or directories that // may be created during command execution are discarded. // // The paths are relative to the working directory of the action execution. // The paths are specified using a single forward slash (`/`) as a path // separator, even if the execution platform natively uses a different // separator. The path MUST NOT include a trailing slash, nor a leading slash, // being a relative path. The special value of empty string is allowed, // although not recommended, and can be used to capture the entire working // directory tree, including inputs. // // In order to ensure consistent hashing of the same Action, the output paths // MUST be sorted lexicographically by code point (or, equivalently, by UTF-8 // bytes). // // An output directory cannot be duplicated or have the same path as any of // the listed output files. An output directory is allowed to be a parent of // another output directory. // // Directories leading up to the output directories (but not the output // directories themselves) are created by the worker prior to execution, even // if they are not explicitly part of the input root. // // DEPRECATED since 2.1: Use `output_paths` instead. repeated string output_directories = 4 [ deprecated = true ]; // A list of the output paths that the client expects to retrieve from the // action. Only the listed paths will be returned to the client as output. // The type of the output (file or directory) is not specified, and will be // determined by the server after action execution. If the resulting path is // a file, it will be returned in an // [OutputFile][build.bazel.remote.execution.v2.OutputFile] typed field. // If the path is a directory, the entire directory structure will be returned // as a [Tree][build.bazel.remote.execution.v2.Tree] message digest, see // [OutputDirectory][build.bazel.remote.execution.v2.OutputDirectory] // Other files or directories that may be created during command execution // are discarded. // // The paths are relative to the working directory of the action execution. // The paths are specified using a single forward slash (`/`) as a path // separator, even if the execution platform natively uses a different // separator. The path MUST NOT include a trailing slash, nor a leading slash, // being a relative path. // // In order to ensure consistent hashing of the same Action, the output paths // MUST be deduplicated and sorted lexicographically by code point (or, // equivalently, by UTF-8 bytes). // // Directories leading up to the output paths are created by the worker prior // to execution, even if they are not explicitly part of the input root. // // New in v2.1: this field supersedes the DEPRECATED `output_files` and // `output_directories` fields. If `output_paths` is used, `output_files` and // `output_directories` will be ignored! repeated string output_paths = 7; // The platform requirements for the execution environment. The server MAY // choose to execute the action on any worker satisfying the requirements, so // the client SHOULD ensure that running the action on any such worker will // have the same result. A detailed lexicon for this can be found in the // accompanying platform.md. // DEPRECATED as of v2.2: platform properties are now specified directly in // the action. See documentation note in the // [Action][build.bazel.remote.execution.v2.Action] for migration. Platform platform = 5 [ deprecated = true ]; // The working directory, relative to the input root, for the command to run // in. It must be a directory which exists in the input tree. If it is left // empty, then the action is run in the input root. string working_directory = 6; // A list of keys for node properties the client expects to retrieve for // output files and directories. Keys are either names of string-based // [NodeProperty][build.bazel.remote.execution.v2.NodeProperty] or // names of fields in [NodeProperties][build.bazel.remote.execution.v2.NodeProperties]. // In order to ensure that equivalent `Action`s always hash to the same // value, the node properties MUST be lexicographically sorted by name. // Sorting of strings is done by code point, equivalently, by the UTF-8 bytes. // // The interpretation of string-based properties is server-dependent. If a // property is not recognized by the server, the server will return an // `INVALID_ARGUMENT`. repeated string output_node_properties = 8; enum OutputDirectoryFormat { // The client is only interested in receiving output directories in // the form of a single Tree object, using the `tree_digest` field. TREE_ONLY = 0; // The client is only interested in receiving output directories in // the form of a hierarchy of separately stored Directory objects, // using the `root_directory_digest` field. DIRECTORY_ONLY = 1; // The client is interested in receiving output directories both in // the form of a single Tree object and a hierarchy of separately // stored Directory objects, using both the `tree_digest` and // `root_directory_digest` fields. TREE_AND_DIRECTORY = 2; } // The format that the worker should use to store the contents of // output directories. // // In case this field is set to a value that is not supported by the // worker, the worker SHOULD interpret this field as TREE_ONLY. The // worker MAY store output directories in formats that are a superset // of what was requested (e.g., interpreting DIRECTORY_ONLY as // TREE_AND_DIRECTORY). OutputDirectoryFormat output_directory_format = 9; } // A `Platform` is a set of requirements, such as hardware, operating system, or // compiler toolchain, for an // [Action][build.bazel.remote.execution.v2.Action]'s execution // environment. A `Platform` is represented as a series of key-value pairs // representing the properties that are required of the platform. message Platform { // A single property for the environment. The server is responsible for // specifying the property `name`s that it accepts. If an unknown `name` is // provided in the requirements for an // [Action][build.bazel.remote.execution.v2.Action], the server SHOULD // reject the execution request. If permitted by the server, the same `name` // may occur multiple times. // // The server is also responsible for specifying the interpretation of // property `value`s. For instance, a property describing how much RAM must be // available may be interpreted as allowing a worker with 16GB to fulfill a // request for 8GB, while a property describing the OS environment on which // the action must be performed may require an exact match with the worker's // OS. // // The server MAY use the `value` of one or more properties to determine how // it sets up the execution environment, such as by making specific system // files available to the worker. // // Both names and values are typically case-sensitive. Note that the platform // is implicitly part of the action digest, so even tiny changes in the names // or values (like changing case) may result in different action cache // entries. message Property { // The property name. string name = 1; // The property value. string value = 2; } // The properties that make up this platform. In order to ensure that // equivalent `Platform`s always hash to the same value, the properties MUST // be lexicographically sorted by name, and then by value. Sorting of strings // is done by code point, equivalently, by the UTF-8 bytes. repeated Property properties = 1; } // A `Directory` represents a directory node in a file tree, containing zero or // more children [FileNodes][build.bazel.remote.execution.v2.FileNode], // [DirectoryNodes][build.bazel.remote.execution.v2.DirectoryNode] and // [SymlinkNodes][build.bazel.remote.execution.v2.SymlinkNode]. // Each `Node` contains its name in the directory, either the digest of its // content (either a file blob or a `Directory` proto) or a symlink target, as // well as possibly some metadata about the file or directory. // // In order to ensure that two equivalent directory trees hash to the same // value, the following restrictions MUST be obeyed when constructing a // a `Directory`: // // * Every child in the directory must have a path of exactly one segment. // Multiple levels of directory hierarchy may not be collapsed. // * Each child in the directory must have a unique path segment (file name). // Note that while the API itself is case-sensitive, the environment where // the Action is executed may or may not be case-sensitive. That is, it is // legal to call the API with a Directory that has both "Foo" and "foo" as // children, but the Action may be rejected by the remote system upon // execution. // * The files, directories and symlinks in the directory must each be sorted // in lexicographical order by path. The path strings must be sorted by code // point, equivalently, by UTF-8 bytes. // * The [NodeProperties][build.bazel.remote.execution.v2.NodeProperty] of files, // directories, and symlinks must be sorted in lexicographical order by // property name. // // A `Directory` that obeys the restrictions is said to be in canonical form. // // As an example, the following could be used for a file named `bar` and a // directory named `foo` with an executable file named `baz` (hashes shortened // for readability): // // ```json // // (Directory proto) // { // files: [ // { // name: "bar", // digest: { // hash: "4a73bc9d03...", // size: 65534 // }, // node_properties: [ // { // "name": "MTime", // "value": "2017-01-15T01:30:15.01Z" // } // ] // } // ], // directories: [ // { // name: "foo", // digest: { // hash: "4cf2eda940...", // size: 43 // } // } // ] // } // // // (Directory proto with hash "4cf2eda940..." and size 43) // { // files: [ // { // name: "baz", // digest: { // hash: "b2c941073e...", // size: 1294, // }, // is_executable: true // } // ] // } // ``` message Directory { // The files in the directory. repeated FileNode files = 1; // The subdirectories in the directory. repeated DirectoryNode directories = 2; // The symlinks in the directory. repeated SymlinkNode symlinks = 3; // The node properties of the Directory. reserved 4; NodeProperties node_properties = 5; } // A single property for [FileNodes][build.bazel.remote.execution.v2.FileNode], // [DirectoryNodes][build.bazel.remote.execution.v2.DirectoryNode], and // [SymlinkNodes][build.bazel.remote.execution.v2.SymlinkNode]. The server is // responsible for specifying the property `name`s that it accepts. If // permitted by the server, the same `name` may occur multiple times. message NodeProperty { // The property name. string name = 1; // The property value. string value = 2; } // Node properties for [FileNodes][build.bazel.remote.execution.v2.FileNode], // [DirectoryNodes][build.bazel.remote.execution.v2.DirectoryNode], and // [SymlinkNodes][build.bazel.remote.execution.v2.SymlinkNode]. The server is // responsible for specifying the properties that it accepts. // message NodeProperties { // A list of string-based // [NodeProperties][build.bazel.remote.execution.v2.NodeProperty]. repeated NodeProperty properties = 1; // The file's last modification timestamp. google.protobuf.Timestamp mtime = 2; // The UNIX file mode, e.g., 0755. google.protobuf.UInt32Value unix_mode = 3; } // A `FileNode` represents a single file and associated metadata. message FileNode { // The name of the file. string name = 1; // The digest of the file's content. Digest digest = 2; reserved 3; // Reserved to ensure wire-compatibility with `OutputFile`. // True if file is executable, false otherwise. bool is_executable = 4; // The node properties of the FileNode. reserved 5; NodeProperties node_properties = 6; } // A `DirectoryNode` represents a child of a // [Directory][build.bazel.remote.execution.v2.Directory] which is itself // a `Directory` and its associated metadata. message DirectoryNode { // The name of the directory. string name = 1; // The digest of the // [Directory][build.bazel.remote.execution.v2.Directory] object // represented. See [Digest][build.bazel.remote.execution.v2.Digest] // for information about how to take the digest of a proto message. Digest digest = 2; } // A `SymlinkNode` represents a symbolic link. message SymlinkNode { // The name of the symlink. string name = 1; // The target path of the symlink. The path separator is a forward slash `/`. // The target path can be relative to the parent directory of the symlink or // it can be an absolute path starting with `/`. Support for absolute paths // can be checked using the [Capabilities][build.bazel.remote.execution.v2.Capabilities] // API. `..` components are allowed anywhere in the target path as logical // canonicalization may lead to different behavior in the presence of // directory symlinks (e.g. `foo/../bar` may not be the same as `bar`). // To reduce potential cache misses, canonicalization is still recommended // where this is possible without impacting correctness. string target = 2; // The node properties of the SymlinkNode. reserved 3; NodeProperties node_properties = 4; } // A content digest. A digest for a given blob consists of the size of the blob // and its hash. The hash algorithm to use is defined by the server. // // The size is considered to be an integral part of the digest and cannot be // separated. That is, even if the `hash` field is correctly specified but // `size_bytes` is not, the server MUST reject the request. // // The reason for including the size in the digest is as follows: in a great // many cases, the server needs to know the size of the blob it is about to work // with prior to starting an operation with it, such as flattening Merkle tree // structures or streaming it to a worker. Technically, the server could // implement a separate metadata store, but this results in a significantly more // complicated implementation as opposed to having the client specify the size // up-front (or storing the size along with the digest in every message where // digests are embedded). This does mean that the API leaks some implementation // details of (what we consider to be) a reasonable server implementation, but // we consider this to be a worthwhile tradeoff. // // When a `Digest` is used to refer to a proto message, it always refers to the // message in binary encoded form. To ensure consistent hashing, clients and // servers MUST ensure that they serialize messages according to the following // rules, even if there are alternate valid encodings for the same message: // // * Fields are serialized in tag order. // * There are no unknown fields. // * There are no duplicate fields. // * Fields are serialized according to the default semantics for their type. // // Most protocol buffer implementations will always follow these rules when // serializing, but care should be taken to avoid shortcuts. For instance, // concatenating two messages to merge them may produce duplicate fields. message Digest { // The hash, represented as a lowercase hexadecimal string, padded with // leading zeroes up to the hash function length. string hash = 1; // The size of the blob, in bytes. int64 size_bytes = 2; } // ExecutedActionMetadata contains details about a completed execution. message ExecutedActionMetadata { // The name of the worker which ran the execution. string worker = 1; // When was the action added to the queue. google.protobuf.Timestamp queued_timestamp = 2; // When the worker received the action. google.protobuf.Timestamp worker_start_timestamp = 3; // When the worker completed the action, including all stages. google.protobuf.Timestamp worker_completed_timestamp = 4; // When the worker started fetching action inputs. google.protobuf.Timestamp input_fetch_start_timestamp = 5; // When the worker finished fetching action inputs. google.protobuf.Timestamp input_fetch_completed_timestamp = 6; // When the worker started executing the action command. google.protobuf.Timestamp execution_start_timestamp = 7; // When the worker completed executing the action command. google.protobuf.Timestamp execution_completed_timestamp = 8; // New in v2.3: the amount of time the worker spent executing the action // command, potentially computed using a worker-specific virtual clock. // // The virtual execution duration is only intended to cover the "execution" of // the specified action and not time in queue nor any overheads before or // after execution such as marshalling inputs/outputs. The server SHOULD avoid // including time spent the client doesn't have control over, and MAY extend // or reduce the execution duration to account for delays or speedups that // occur during execution itself (e.g., lazily loading data from the Content // Addressable Storage, live migration of virtual machines, emulation // overhead). // // The method of timekeeping used to compute the virtual execution duration // MUST be consistent with what is used to enforce the // [Action][[build.bazel.remote.execution.v2.Action]'s `timeout`. There is no // relationship between the virtual execution duration and the values of // `execution_start_timestamp` and `execution_completed_timestamp`. google.protobuf.Duration virtual_execution_duration = 12; // When the worker started uploading action outputs. google.protobuf.Timestamp output_upload_start_timestamp = 9; // When the worker finished uploading action outputs. google.protobuf.Timestamp output_upload_completed_timestamp = 10; // Details that are specific to the kind of worker used. For example, // on POSIX-like systems this could contain a message with // getrusage(2) statistics. repeated google.protobuf.Any auxiliary_metadata = 11; } // An ActionResult represents the result of an // [Action][build.bazel.remote.execution.v2.Action] being run. // // It is advised that at least one field (for example // `ActionResult.execution_metadata.Worker`) have a non-default value, to // ensure that the serialized value is non-empty, which can then be used // as a basic data sanity check. message ActionResult { reserved 1; // Reserved for use as the resource name. // The output files of the action. For each output file requested in the // `output_files` or `output_paths` field of the Action, if the corresponding // file existed after the action completed, a single entry will be present // either in this field, or the `output_file_symlinks` field if the file was // a symbolic link to another file (`output_symlinks` field after v2.1). // // If an output listed in `output_files` was found, but was a directory rather // than a regular file, the server will return a FAILED_PRECONDITION. // If the action does not produce the requested output, then that output // will be omitted from the list. The server is free to arrange the output // list as desired; clients MUST NOT assume that the output list is sorted. repeated OutputFile output_files = 2; // The output files of the action that are symbolic links to other files. Those // may be links to other output files, or input files, or even absolute paths // outside of the working directory, if the server supports // [SymlinkAbsolutePathStrategy.ALLOWED][build.bazel.remote.execution.v2.CacheCapabilities.SymlinkAbsolutePathStrategy]. // For each output file requested in the `output_files` or `output_paths` // field of the Action, if the corresponding file existed after // the action completed, a single entry will be present either in this field, // or in the `output_files` field, if the file was not a symbolic link. // // If an output symbolic link of the same name as listed in `output_files` of // the Command was found, but its target type was not a regular file, the // server will return a FAILED_PRECONDITION. // If the action does not produce the requested output, then that output // will be omitted from the list. The server is free to arrange the output // list as desired; clients MUST NOT assume that the output list is sorted. // // DEPRECATED as of v2.1. Servers that wish to be compatible with v2.0 API // should still populate this field in addition to `output_symlinks`. repeated OutputSymlink output_file_symlinks = 10 [ deprecated = true ]; // New in v2.1: this field will only be populated if the command // `output_paths` field was used, and not the pre v2.1 `output_files` or // `output_directories` fields. // The output paths of the action that are symbolic links to other paths. Those // may be links to other outputs, or inputs, or even absolute paths // outside of the working directory, if the server supports // [SymlinkAbsolutePathStrategy.ALLOWED][build.bazel.remote.execution.v2.CacheCapabilities.SymlinkAbsolutePathStrategy]. // A single entry for each output requested in `output_paths` // field of the Action, if the corresponding path existed after // the action completed and was a symbolic link. // // If the action does not produce a requested output, then that output // will be omitted from the list. The server is free to arrange the output // list as desired; clients MUST NOT assume that the output list is sorted. repeated OutputSymlink output_symlinks = 12; // The output directories of the action. For each output directory requested // in the `output_directories` or `output_paths` field of the Action, if the // corresponding directory existed after the action completed, a single entry // will be present in the output list, which will contain the digest of a // [Tree][build.bazel.remote.execution.v2.Tree] message containing the // directory tree, and the path equal exactly to the corresponding Action // output_directories member. // // As an example, suppose the Action had an output directory `a/b/dir` and the // execution produced the following contents in `a/b/dir`: a file named `bar` // and a directory named `foo` with an executable file named `baz`. Then, // output_directory will contain (hashes shortened for readability): // // ```json // // OutputDirectory proto: // { // path: "a/b/dir" // tree_digest: { // hash: "4a73bc9d03...", // size: 55 // } // } // // Tree proto with hash "4a73bc9d03..." and size 55: // { // root: { // files: [ // { // name: "bar", // digest: { // hash: "4a73bc9d03...", // size: 65534 // } // } // ], // directories: [ // { // name: "foo", // digest: { // hash: "4cf2eda940...", // size: 43 // } // } // ] // } // children : { // // (Directory proto with hash "4cf2eda940..." and size 43) // files: [ // { // name: "baz", // digest: { // hash: "b2c941073e...", // size: 1294, // }, // is_executable: true // } // ] // } // } // ``` // If an output of the same name as listed in `output_files` of // the Command was found in `output_directories`, but was not a directory, the // server will return a FAILED_PRECONDITION. repeated OutputDirectory output_directories = 3; // The output directories of the action that are symbolic links to other // directories. Those may be links to other output directories, or input // directories, or even absolute paths outside of the working directory, // if the server supports // [SymlinkAbsolutePathStrategy.ALLOWED][build.bazel.remote.execution.v2.CacheCapabilities.SymlinkAbsolutePathStrategy]. // For each output directory requested in the `output_directories` field of // the Action, if the directory existed after the action completed, a // single entry will be present either in this field, or in the // `output_directories` field, if the directory was not a symbolic link. // // If an output of the same name was found, but was a symbolic link to a file // instead of a directory, the server will return a FAILED_PRECONDITION. // If the action does not produce the requested output, then that output // will be omitted from the list. The server is free to arrange the output // list as desired; clients MUST NOT assume that the output list is sorted. // // DEPRECATED as of v2.1. Servers that wish to be compatible with v2.0 API // should still populate this field in addition to `output_symlinks`. repeated OutputSymlink output_directory_symlinks = 11 [ deprecated = true ]; // The exit code of the command. int32 exit_code = 4; // The standard output buffer of the action. The server SHOULD NOT inline // stdout unless requested by the client in the // [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest] // message. The server MAY omit inlining, even if requested, and MUST do so if inlining // would cause the response to exceed message size limits. // Clients SHOULD NOT populate this field when uploading to the cache. bytes stdout_raw = 5; // The digest for a blob containing the standard output of the action, which // can be retrieved from the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. Digest stdout_digest = 6; // The standard error buffer of the action. The server SHOULD NOT inline // stderr unless requested by the client in the // [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest] // message. The server MAY omit inlining, even if requested, and MUST do so if inlining // would cause the response to exceed message size limits. // Clients SHOULD NOT populate this field when uploading to the cache. bytes stderr_raw = 7; // The digest for a blob containing the standard error of the action, which // can be retrieved from the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. Digest stderr_digest = 8; // The details of the execution that originally produced this result. ExecutedActionMetadata execution_metadata = 9; } // An `OutputFile` is similar to a // [FileNode][build.bazel.remote.execution.v2.FileNode], but it is used as an // output in an `ActionResult`. It allows a full file path rather than // only a name. message OutputFile { // The full path of the file relative to the working directory, including the // filename. The path separator is a forward slash `/`. Since this is a // relative path, it MUST NOT begin with a leading forward slash. string path = 1; // The digest of the file's content. Digest digest = 2; reserved 3; // Used for a removed field in an earlier version of the API. // True if file is executable, false otherwise. bool is_executable = 4; // The contents of the file if inlining was requested. The server SHOULD NOT inline // file contents unless requested by the client in the // [GetActionResultRequest][build.bazel.remote.execution.v2.GetActionResultRequest] // message. The server MAY omit inlining, even if requested, and MUST do so if inlining // would cause the response to exceed message size limits. // Clients SHOULD NOT populate this field when uploading to the cache. bytes contents = 5; // The supported node properties of the OutputFile, if requested by the Action. reserved 6; NodeProperties node_properties = 7; } // A `Tree` contains all the // [Directory][build.bazel.remote.execution.v2.Directory] protos in a // single directory Merkle tree, compressed into one message. message Tree { // The root directory in the tree. Directory root = 1; // All the child directories: the directories referred to by the root and, // recursively, all its children. In order to reconstruct the directory tree, // the client must take the digests of each of the child directories and then // build up a tree starting from the `root`. // Servers SHOULD ensure that these are ordered consistently such that two // actions producing equivalent output directories on the same server // implementation also produce Tree messages with matching digests. repeated Directory children = 2; } // An `OutputDirectory` is the output in an `ActionResult` corresponding to a // directory's full contents rather than a single file. message OutputDirectory { // The full path of the directory relative to the working directory. The path // separator is a forward slash `/`. Since this is a relative path, it MUST // NOT begin with a leading forward slash. The empty string value is allowed, // and it denotes the entire working directory. string path = 1; reserved 2; // Used for a removed field in an earlier version of the API. // The digest of the encoded // [Tree][build.bazel.remote.execution.v2.Tree] proto containing the // directory's contents. Digest tree_digest = 3; // If set, consumers MAY make the following assumptions about the // directories contained in the the Tree, so that it may be // instantiated on a local file system by scanning through it // sequentially: // // - All directories with the same binary representation are stored // exactly once. // - All directories, apart from the root directory, are referenced by // at least one parent directory. // - Directories are stored in topological order, with parents being // stored before the child. The root directory is thus the first to // be stored. // // Additionally, the Tree MUST be encoded as a stream of records, // where each record has the following format: // // - A tag byte, having one of the following two values: // - (1 << 3) | 2 == 0x0a: First record (the root directory). // - (2 << 3) | 2 == 0x12: Any subsequent records (child directories). // - The size of the directory, encoded as a base 128 varint. // - The contents of the directory, encoded as a binary serialized // Protobuf message. // // This encoding is a subset of the Protobuf wire format of the Tree // message. As it is only permitted to store data associated with // field numbers 1 and 2, the tag MUST be encoded as a single byte. // More details on the Protobuf wire format can be found here: // https://developers.google.com/protocol-buffers/docs/encoding // // It is recommended that implementations using this feature construct // Tree objects manually using the specification given above, as // opposed to using a Protobuf library to marshal a full Tree message. // As individual Directory messages already need to be marshaled to // compute their digests, constructing the Tree object manually avoids // redundant marshaling. bool is_topologically_sorted = 4; // The digest of the encoded // [Directory][build.bazel.remote.execution.v2.Directory] proto // containing the contents the directory's root. // // If both `tree_digest` and `root_directory_digest` are set, this // field MUST match the digest of the root directory contained in the // Tree message. Digest root_directory_digest = 5; } // An `OutputSymlink` is similar to a // [Symlink][build.bazel.remote.execution.v2.SymlinkNode], but it is used as an // output in an `ActionResult`. // // `OutputSymlink` is binary-compatible with `SymlinkNode`. message OutputSymlink { // The full path of the symlink relative to the working directory, including the // filename. The path separator is a forward slash `/`. Since this is a // relative path, it MUST NOT begin with a leading forward slash. string path = 1; // The target path of the symlink. The path separator is a forward slash `/`. // The target path can be relative to the parent directory of the symlink or // it can be an absolute path starting with `/`. Support for absolute paths // can be checked using the [Capabilities][build.bazel.remote.execution.v2.Capabilities] // API. `..` components are allowed anywhere in the target path. string target = 2; // The supported node properties of the OutputSymlink, if requested by the // Action. reserved 3; NodeProperties node_properties = 4; } // An `ExecutionPolicy` can be used to control the scheduling of the action. message ExecutionPolicy { // The priority (relative importance) of this action. Generally, a lower value // means that the action should be run sooner than actions having a greater // priority value, but the interpretation of a given value is server- // dependent. A priority of 0 means the *default* priority. Priorities may be // positive or negative, and such actions should run later or sooner than // actions having the default priority, respectively. The particular semantics // of this field is up to the server. In particular, every server will have // their own supported range of priorities, and will decide how these map into // scheduling policy. int32 priority = 1; } // A `ResultsCachePolicy` is used for fine-grained control over how action // outputs are stored in the CAS and Action Cache. message ResultsCachePolicy { // The priority (relative importance) of this content in the overall cache. // Generally, a lower value means a longer retention time or other advantage, // but the interpretation of a given value is server-dependent. A priority of // 0 means a *default* value, decided by the server. // // The particular semantics of this field is up to the server. In particular, // every server will have their own supported range of priorities, and will // decide how these map into retention/eviction policy. int32 priority = 1; } // A request message for // [Execution.Execute][build.bazel.remote.execution.v2.Execution.Execute]. message ExecuteRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // If true, the action will be executed even if its result is already // present in the [ActionCache][build.bazel.remote.execution.v2.ActionCache]. // The execution is still allowed to be merged with other in-flight executions // of the same action, however - semantically, the service MUST only guarantee // that the results of an execution with this field set were not visible // before the corresponding execution request was sent. // Note that actions from execution requests setting this field set are still // eligible to be entered into the action cache upon completion, and services // SHOULD overwrite any existing entries that may exist. This allows // skip_cache_lookup requests to be used as a mechanism for replacing action // cache entries that reference outputs no longer available or that are // poisoned in any way. // If false, the result may be served from the action cache. bool skip_cache_lookup = 3; reserved 2, 4, 5; // Used for removed fields in an earlier version of the API. // The digest of the [Action][build.bazel.remote.execution.v2.Action] to // execute. Digest action_digest = 6; // An optional policy for execution of the action. // The server will have a default policy if this is not provided. ExecutionPolicy execution_policy = 7; // An optional policy for the results of this execution in the remote cache. // The server will have a default policy if this is not provided. // This may be applied to both the ActionResult and the associated blobs. ResultsCachePolicy results_cache_policy = 8; // The digest function that was used to compute the action digest. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the action digest hash and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 9; // A hint to the server to request inlining stdout in the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] message. bool inline_stdout = 10; // A hint to the server to request inlining stderr in the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] message. bool inline_stderr = 11; // A hint to the server to inline the contents of the listed output files. // Each path needs to exactly match one file path in either `output_paths` or // `output_files` (DEPRECATED since v2.1) in the // [Command][build.bazel.remote.execution.v2.Command] message. repeated string inline_output_files = 12; } // A `LogFile` is a log stored in the CAS. message LogFile { // The digest of the log contents. Digest digest = 1; // This is a hint as to the purpose of the log, and is set to true if the log // is human-readable text that can be usefully displayed to a user, and false // otherwise. For instance, if a command-line client wishes to print the // server logs to the terminal for a failed action, this allows it to avoid // displaying a binary file. bool human_readable = 2; } // The response message for // [Execution.Execute][build.bazel.remote.execution.v2.Execution.Execute], // which will be contained in the [response // field][google.longrunning.Operation.response] of the // [Operation][google.longrunning.Operation]. message ExecuteResponse { // The result of the action. ActionResult result = 1; // True if the result was served from cache, false if it was executed. bool cached_result = 2; // If the status has a code other than `OK`, it indicates that the action did // not finish execution. For example, if the operation times out during // execution, the status will have a `DEADLINE_EXCEEDED` code. Servers MUST // use this field for errors in execution, rather than the error field on the // `Operation` object. // // If the status code is other than `OK`, then the result MUST NOT be cached. // For an error status, the `result` field is optional; the server may // populate the output-, stdout-, and stderr-related fields if it has any // information available, such as the stdout and stderr of a timed-out action. google.rpc.Status status = 3; // An optional list of additional log outputs the server wishes to provide. A // server can use this to return execution-specific logs however it wishes. // This is intended primarily to make it easier for users to debug issues that // may be outside of the actual job execution, such as by identifying the // worker executing the action or by providing logs from the worker's setup // phase. The keys SHOULD be human readable so that a client can display them // to a user. map server_logs = 4; // Freeform informational message with details on the execution of the action // that may be displayed to the user upon failure or when requested explicitly. string message = 5; } // The current stage of action execution. // // Even though these stages are numbered according to the order in which // they generally occur, there is no requirement that the remote // execution system reports events along this order. For example, an // operation MAY transition from the EXECUTING stage back to QUEUED // in case the hardware on which the operation executes fails. // // If and only if the remote execution system reports that an operation // has reached the COMPLETED stage, it MUST set the [done // field][google.longrunning.Operation.done] of the // [Operation][google.longrunning.Operation] and terminate the stream. message ExecutionStage { enum Value { // Invalid value. UNKNOWN = 0; // Checking the result against the cache. CACHE_CHECK = 1; // Currently idle, awaiting a free machine to execute. QUEUED = 2; // Currently being executed by a worker. EXECUTING = 3; // Finished execution. COMPLETED = 4; } } // Metadata about an ongoing // [execution][build.bazel.remote.execution.v2.Execution.Execute], which // will be contained in the [metadata // field][google.longrunning.Operation.response] of the // [Operation][google.longrunning.Operation]. message ExecuteOperationMetadata { // The current stage of execution. ExecutionStage.Value stage = 1; // The digest of the [Action][build.bazel.remote.execution.v2.Action] // being executed. Digest action_digest = 2; // If set, the client can use this resource name with // [ByteStream.Read][google.bytestream.ByteStream.Read] to stream the // standard output from the endpoint hosting streamed responses. string stdout_stream_name = 3; // If set, the client can use this resource name with // [ByteStream.Read][google.bytestream.ByteStream.Read] to stream the // standard error from the endpoint hosting streamed responses. string stderr_stream_name = 4; // The client can read this field to view details about the ongoing // execution. ExecutedActionMetadata partial_execution_metadata = 5; } // A request message for // [WaitExecution][build.bazel.remote.execution.v2.Execution.WaitExecution]. message WaitExecutionRequest { // The name of the [Operation][google.longrunning.Operation] // returned by [Execute][build.bazel.remote.execution.v2.Execution.Execute]. string name = 1; } // A request message for // [ActionCache.GetActionResult][build.bazel.remote.execution.v2.ActionCache.GetActionResult]. message GetActionResultRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The digest of the [Action][build.bazel.remote.execution.v2.Action] // whose result is requested. Digest action_digest = 2; // A hint to the server to request inlining stdout in the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] message. bool inline_stdout = 3; // A hint to the server to request inlining stderr in the // [ActionResult][build.bazel.remote.execution.v2.ActionResult] message. bool inline_stderr = 4; // A hint to the server to inline the contents of the listed output files. // Each path needs to exactly match one file path in either `output_paths` or // `output_files` (DEPRECATED since v2.1) in the // [Command][build.bazel.remote.execution.v2.Command] message. repeated string inline_output_files = 5; // The digest function that was used to compute the action digest. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the action digest hash and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 6; } // A request message for // [ActionCache.UpdateActionResult][build.bazel.remote.execution.v2.ActionCache.UpdateActionResult]. message UpdateActionResultRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The digest of the [Action][build.bazel.remote.execution.v2.Action] // whose result is being uploaded. Digest action_digest = 2; // The [ActionResult][build.bazel.remote.execution.v2.ActionResult] // to store in the cache. ActionResult action_result = 3; // An optional policy for the results of this execution in the remote cache. // The server will have a default policy if this is not provided. // This may be applied to both the ActionResult and the associated blobs. ResultsCachePolicy results_cache_policy = 4; // The digest function that was used to compute the action digest. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the action digest hash and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 5; } // A request message for // [ContentAddressableStorage.FindMissingBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.FindMissingBlobs]. message FindMissingBlobsRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // A list of the blobs to check. All digests MUST use the same digest // function. repeated Digest blob_digests = 2; // The digest function of the blobs whose existence is checked. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the blob digest hashes and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 3; } // A response message for // [ContentAddressableStorage.FindMissingBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.FindMissingBlobs]. message FindMissingBlobsResponse { // A list of the blobs requested *not* present in the storage. repeated Digest missing_blob_digests = 2; } // A request message for // [ContentAddressableStorage.BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]. message BatchUpdateBlobsRequest { // A request corresponding to a single blob that the client wants to upload. message Request { // The digest of the blob. This MUST be the digest of `data`. All // digests MUST use the same digest function. Digest digest = 1; // The raw binary data. bytes data = 2; // The format of `data`. Must be `IDENTITY`/unspecified, or one of the // compressors advertised by the // [CacheCapabilities.supported_batch_compressors][build.bazel.remote.execution.v2.CacheCapabilities.supported_batch_compressors] // field. Compressor.Value compressor = 3; } // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The individual upload requests. repeated Request requests = 2; // The digest function that was used to compute the digests of the // blobs being uploaded. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the blob digest hashes and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 5; } // A response message for // [ContentAddressableStorage.BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]. message BatchUpdateBlobsResponse { // A response corresponding to a single blob that the client tried to upload. message Response { // The blob digest to which this response corresponds. Digest digest = 1; // The result of attempting to upload that blob. google.rpc.Status status = 2; } // The responses to the requests. repeated Response responses = 1; } // A request message for // [ContentAddressableStorage.BatchReadBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchReadBlobs]. message BatchReadBlobsRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The individual blob digests. All digests MUST use the same digest // function. repeated Digest digests = 2; // A list of acceptable encodings for the returned inlined data, in no // particular order. `IDENTITY` is always allowed even if not specified here. repeated Compressor.Value acceptable_compressors = 3; // The digest function of the blobs being requested. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the blob digest hashes and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 4; } // A response message for // [ContentAddressableStorage.BatchReadBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchReadBlobs]. message BatchReadBlobsResponse { // A response corresponding to a single blob that the client tried to download. message Response { // The digest to which this response corresponds. Digest digest = 1; // The raw binary data. bytes data = 2; // The format the data is encoded in. MUST be `IDENTITY`/unspecified, // or one of the acceptable compressors specified in the `BatchReadBlobsRequest`. Compressor.Value compressor = 4; // The result of attempting to download that blob. google.rpc.Status status = 3; } // The responses to the requests. repeated Response responses = 1; } // A request message for // [ContentAddressableStorage.GetTree][build.bazel.remote.execution.v2.ContentAddressableStorage.GetTree]. message GetTreeRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The digest of the root, which must be an encoded // [Directory][build.bazel.remote.execution.v2.Directory] message // stored in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. Digest root_digest = 2; // A maximum page size to request. If present, the server will request no more // than this many items. Regardless of whether a page size is specified, the // server may place its own limit on the number of items to be returned and // require the client to retrieve more items using a subsequent request. int32 page_size = 3; // A page token, which must be a value received in a previous // [GetTreeResponse][build.bazel.remote.execution.v2.GetTreeResponse]. // If present, the server will use that token as an offset, returning only // that page and the ones that succeed it. string page_token = 4; // The digest function that was used to compute the digest of the root // directory. // // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, // SHA384, SHA512, or VSO, the client MAY leave this field unset. In // that case the server SHOULD infer the digest function using the // length of the root digest hash and the digest functions announced // in the server's capabilities. DigestFunction.Value digest_function = 5; } // A response message for // [ContentAddressableStorage.GetTree][build.bazel.remote.execution.v2.ContentAddressableStorage.GetTree]. message GetTreeResponse { // The directories descended from the requested root. repeated Directory directories = 1; // If present, signifies that there are more results which the client can // retrieve by passing this as the page_token in a subsequent // [request][build.bazel.remote.execution.v2.GetTreeRequest]. // If empty, signifies that this is the last page of results. string next_page_token = 2; } // A request message for // [Capabilities.GetCapabilities][build.bazel.remote.execution.v2.Capabilities.GetCapabilities]. message GetCapabilitiesRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; } // A response message for // [Capabilities.GetCapabilities][build.bazel.remote.execution.v2.Capabilities.GetCapabilities]. message ServerCapabilities { // Capabilities of the remote cache system. CacheCapabilities cache_capabilities = 1; // Capabilities of the remote execution system. ExecutionCapabilities execution_capabilities = 2; // Earliest RE API version supported, including deprecated versions. build.bazel.semver.SemVer deprecated_api_version = 3; // Earliest non-deprecated RE API version supported. build.bazel.semver.SemVer low_api_version = 4; // Latest RE API version supported. build.bazel.semver.SemVer high_api_version = 5; } // The digest function used for converting values into keys for CAS and Action // Cache. message DigestFunction { enum Value { // It is an error for the server to return this value. UNKNOWN = 0; // The SHA-256 digest function. SHA256 = 1; // The SHA-1 digest function. SHA1 = 2; // The MD5 digest function. MD5 = 3; // The Microsoft "VSO-Hash" paged SHA256 digest function. // See https://github.com/microsoft/BuildXL/blob/master/Documentation/Specs/PagedHash.md . VSO = 4; // The SHA-384 digest function. SHA384 = 5; // The SHA-512 digest function. SHA512 = 6; // Murmur3 128-bit digest function, x64 variant. Note that this is not a // cryptographic hash function and its collision properties are not strongly guaranteed. // See https://github.com/aappleby/smhasher/wiki/MurmurHash3 . MURMUR3 = 7; // The SHA-256 digest function, modified to use a Merkle tree for // large objects. This permits implementations to store large blobs // as a decomposed sequence of 2^j sized chunks, where j >= 10, // while being able to validate integrity at the chunk level. // // Furthermore, on systems that do not offer dedicated instructions // for computing SHA-256 hashes (e.g., the Intel SHA and ARMv8 // cryptographic extensions), SHA256TREE hashes can be computed more // efficiently than plain SHA-256 hashes by using generic SIMD // extensions, such as Intel AVX2 or ARM NEON. // // SHA256TREE hashes are computed as follows: // // - For blobs that are 1024 bytes or smaller, the hash is computed // using the regular SHA-256 digest function. // // - For blobs that are more than 1024 bytes in size, the hash is // computed as follows: // // 1. The blob is partitioned into a left (leading) and right // (trailing) blob. These blobs have lengths m and n // respectively, where m = 2^k and 0 < n <= m. // // 2. Hashes of the left and right blob, Hash(left) and // Hash(right) respectively, are computed by recursively // applying the SHA256TREE algorithm. // // 3. A single invocation is made to the SHA-256 block cipher with // the following parameters: // // M = Hash(left) || Hash(right) // H = { // 0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8, // 0x67332667, 0x8eb44a87, 0xdb0c2e0d, 0x47b5481d, // } // // The values of H are the leading fractional parts of the // square roots of the 9th to the 16th prime number (23 to 53). // This differs from plain SHA-256, where the first eight prime // numbers (2 to 19) are used, thereby preventing trivial hash // collisions between small and large objects. // // 4. The hash of the full blob can then be obtained by // concatenating the outputs of the block cipher: // // Hash(blob) = a || b || c || d || e || f || g || h // // Addition of the original values of H, as normally done // through the use of the Davies-Meyer structure, is not // performed. This isn't necessary, as the block cipher is only // invoked once. // // Test vectors of this digest function can be found in the // accompanying sha256tree_test_vectors.txt file. SHA256TREE = 8; // The BLAKE3 hash function. // See https://github.com/BLAKE3-team/BLAKE3. BLAKE3 = 9; } } // Describes the server/instance capabilities for updating the action cache. message ActionCacheUpdateCapabilities { bool update_enabled = 1; } // Allowed values for priority in // [ResultsCachePolicy][build.bazel.remoteexecution.v2.ResultsCachePolicy] and // [ExecutionPolicy][build.bazel.remoteexecution.v2.ResultsCachePolicy] // Used for querying both cache and execution valid priority ranges. message PriorityCapabilities { // Supported range of priorities, including boundaries. message PriorityRange { // The minimum numeric value for this priority range, which represents the // most urgent task or longest retained item. int32 min_priority = 1; // The maximum numeric value for this priority range, which represents the // least urgent task or shortest retained item. int32 max_priority = 2; } repeated PriorityRange priorities = 1; } // Describes how the server treats absolute symlink targets. message SymlinkAbsolutePathStrategy { enum Value { // Invalid value. UNKNOWN = 0; // Server will return an `INVALID_ARGUMENT` on input symlinks with absolute // targets. // If an action tries to create an output symlink with an absolute target, a // `FAILED_PRECONDITION` will be returned. DISALLOWED = 1; // Server will allow symlink targets to escape the input root tree, possibly // resulting in non-hermetic builds. ALLOWED = 2; } } // Compression formats which may be supported. message Compressor { enum Value { // No compression. Servers and clients MUST always support this, and do // not need to advertise it. IDENTITY = 0; // Zstandard compression. ZSTD = 1; // RFC 1951 Deflate. This format is identical to what is used by ZIP // files. Headers such as the one generated by gzip are not // included. // // It is advised to use algorithms such as Zstandard instead, as // those are faster and/or provide a better compression ratio. DEFLATE = 2; // Brotli compression. BROTLI = 3; } } // Capabilities of the remote cache system. message CacheCapabilities { // All the digest functions supported by the remote cache. // Remote cache may support multiple digest functions simultaneously. repeated DigestFunction.Value digest_functions = 1; // Capabilities for updating the action cache. ActionCacheUpdateCapabilities action_cache_update_capabilities = 2; // Supported cache priority range for both CAS and ActionCache. PriorityCapabilities cache_priority_capabilities = 3; // Maximum total size of blobs to be uploaded/downloaded using // batch methods. A value of 0 means no limit is set, although // in practice there will always be a message size limitation // of the protocol in use, e.g. GRPC. int64 max_batch_total_size_bytes = 4; // Whether absolute symlink targets are supported. SymlinkAbsolutePathStrategy.Value symlink_absolute_path_strategy = 5; // Compressors supported by the "compressed-blobs" bytestream resources. // Servers MUST support identity/no-compression, even if it is not listed // here. // // Note that this does not imply which if any compressors are supported by // the server at the gRPC level. repeated Compressor.Value supported_compressors = 6; // Compressors supported for inlined data in // [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs] // requests. repeated Compressor.Value supported_batch_update_compressors = 7; } // Capabilities of the remote execution system. message ExecutionCapabilities { // Legacy field for indicating which digest function is supported by the // remote execution system. It MUST be set to a value other than UNKNOWN. // Implementations should consider the repeated digest_functions field // first, falling back to this singular field if digest_functions is unset. DigestFunction.Value digest_function = 1; // Whether remote execution is enabled for the particular server/instance. bool exec_enabled = 2; // Supported execution priority range. PriorityCapabilities execution_priority_capabilities = 3; // Supported node properties. repeated string supported_node_properties = 4; // All the digest functions supported by the remote execution system. // If this field is set, it MUST also contain digest_function. // // Even if the remote execution system announces support for multiple // digest functions, individual execution requests may only reference // CAS objects using a single digest function. For example, it is not // permitted to execute actions having both MD5 and SHA-256 hashed // files in their input root. // // The CAS objects referenced by action results generated by the // remote execution system MUST use the same digest function as the // one used to construct the action. repeated DigestFunction.Value digest_functions = 5; } // Details for the tool used to call the API. message ToolDetails { // Name of the tool, e.g. bazel. string tool_name = 1; // Version of the tool used for the request, e.g. 5.0.3. string tool_version = 2; } // An optional Metadata to attach to any RPC request to tell the server about an // external context of the request. The server may use this for logging or other // purposes. To use it, the client attaches the header to the call using the // canonical proto serialization: // // * name: `build.bazel.remote.execution.v2.requestmetadata-bin` // * contents: the base64 encoded binary `RequestMetadata` message. // Note: the gRPC library serializes binary headers encoded in base64 by // default (https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests). // Therefore, if the gRPC library is used to pass/retrieve this // metadata, the user may ignore the base64 encoding and assume it is simply // serialized as a binary message. message RequestMetadata { // The details for the tool invoking the requests. ToolDetails tool_details = 1; // An identifier that ties multiple requests to the same action. // For example, multiple requests to the CAS, Action Cache, and Execution // API are used in order to compile foo.cc. string action_id = 2; // An identifier that ties multiple actions together to a final result. // For example, multiple actions are required to build and run foo_test. string tool_invocation_id = 3; // An identifier to tie multiple tool invocations together. For example, // runs of foo_test, bar_test and baz_test on a post-submit of a given patch. string correlated_invocations_id = 4; // A brief description of the kind of action, for example, CppCompile or GoLink. // There is no standard agreed set of values for this, and they are expected to vary between different client tools. string action_mnemonic = 5; // An identifier for the target which produced this action. // No guarantees are made around how many actions may relate to a single target. string target_id = 6; // An identifier for the configuration in which the target was built, // e.g. for differentiating building host tools or different target platforms. // There is no expectation that this value will have any particular structure, // or equality across invocations, though some client tools may offer these guarantees. string configuration_id = 7; } remote_execution_pb2.py000066400000000000000000000736651514607367700350670ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/v2# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: build/bazel/remote/execution/v2/remote_execution.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'build/bazel/remote/execution/v2/remote_execution.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.build.bazel.semver import semver_pb2 as build_dot_bazel_dot_semver_dot_semver__pb2 from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from buildstream._protos.google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 from buildstream._protos.google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n6build/bazel/remote/execution/v2/remote_execution.proto\x12\x1f\x62uild.bazel.remote.execution.v2\x1a\x1f\x62uild/bazel/semver/semver.proto\x1a\x1cgoogle/api/annotations.proto\x1a#google/longrunning/operations.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17google/rpc/status.proto\"\xa6\x02\n\x06\x41\x63tion\x12?\n\x0e\x63ommand_digest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x42\n\x11input_root_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12*\n\x07timeout\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0c\x64o_not_cache\x18\x07 \x01(\x08\x12\x0c\n\x04salt\x18\t \x01(\x0c\x12;\n\x08platform\x18\n \x01(\x0b\x32).build.bazel.remote.execution.v2.PlatformJ\x04\x08\x03\x10\x06J\x04\x08\x08\x10\t\"\xae\x04\n\x07\x43ommand\x12\x11\n\targuments\x18\x01 \x03(\t\x12[\n\x15\x65nvironment_variables\x18\x02 \x03(\x0b\x32<.build.bazel.remote.execution.v2.Command.EnvironmentVariable\x12\x18\n\x0coutput_files\x18\x03 \x03(\tB\x02\x18\x01\x12\x1e\n\x12output_directories\x18\x04 \x03(\tB\x02\x18\x01\x12\x14\n\x0coutput_paths\x18\x07 \x03(\t\x12?\n\x08platform\x18\x05 \x01(\x0b\x32).build.bazel.remote.execution.v2.PlatformB\x02\x18\x01\x12\x19\n\x11working_directory\x18\x06 \x01(\t\x12\x1e\n\x16output_node_properties\x18\x08 \x03(\t\x12_\n\x17output_directory_format\x18\t \x01(\x0e\x32>.build.bazel.remote.execution.v2.Command.OutputDirectoryFormat\x1a\x32\n\x13\x45nvironmentVariable\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"R\n\x15OutputDirectoryFormat\x12\r\n\tTREE_ONLY\x10\x00\x12\x12\n\x0e\x44IRECTORY_ONLY\x10\x01\x12\x16\n\x12TREE_AND_DIRECTORY\x10\x02\"{\n\x08Platform\x12\x46\n\nproperties\x18\x01 \x03(\x0b\x32\x32.build.bazel.remote.execution.v2.Platform.Property\x1a\'\n\x08Property\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x9a\x02\n\tDirectory\x12\x38\n\x05\x66iles\x18\x01 \x03(\x0b\x32).build.bazel.remote.execution.v2.FileNode\x12\x43\n\x0b\x64irectories\x18\x02 \x03(\x0b\x32..build.bazel.remote.execution.v2.DirectoryNode\x12>\n\x08symlinks\x18\x03 \x03(\x0b\x32,.build.bazel.remote.execution.v2.SymlinkNode\x12H\n\x0fnode_properties\x18\x05 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x04\x10\x05\"+\n\x0cNodeProperty\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\xaf\x01\n\x0eNodeProperties\x12\x41\n\nproperties\x18\x01 \x03(\x0b\x32-.build.bazel.remote.execution.v2.NodeProperty\x12)\n\x05mtime\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\tunix_mode\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\"\xbe\x01\n\x08\x46ileNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x15\n\ris_executable\x18\x04 \x01(\x08\x12H\n\x0fnode_properties\x18\x06 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x03\x10\x04J\x04\x08\x05\x10\x06\"V\n\rDirectoryNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"{\n\x0bSymlinkNode\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06target\x18\x02 \x01(\t\x12H\n\x0fnode_properties\x18\x04 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x03\x10\x04\"*\n\x06\x44igest\x12\x0c\n\x04hash\x18\x01 \x01(\t\x12\x12\n\nsize_bytes\x18\x02 \x01(\x03\"\xdd\x05\n\x16\x45xecutedActionMetadata\x12\x0e\n\x06worker\x18\x01 \x01(\t\x12\x34\n\x10queued_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16worker_start_timestamp\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12>\n\x1aworker_completed_timestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12?\n\x1binput_fetch_start_timestamp\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x43\n\x1finput_fetch_completed_timestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x19\x65xecution_start_timestamp\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x41\n\x1d\x65xecution_completed_timestamp\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12=\n\x1avirtual_execution_duration\x18\x0c \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x41\n\x1doutput_upload_start_timestamp\x18\t \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x45\n!output_upload_completed_timestamp\x18\n \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x30\n\x12\x61uxiliary_metadata\x18\x0b \x03(\x0b\x32\x14.google.protobuf.Any\"\xa7\x05\n\x0c\x41\x63tionResult\x12\x41\n\x0coutput_files\x18\x02 \x03(\x0b\x32+.build.bazel.remote.execution.v2.OutputFile\x12P\n\x14output_file_symlinks\x18\n \x03(\x0b\x32..build.bazel.remote.execution.v2.OutputSymlinkB\x02\x18\x01\x12G\n\x0foutput_symlinks\x18\x0c \x03(\x0b\x32..build.bazel.remote.execution.v2.OutputSymlink\x12L\n\x12output_directories\x18\x03 \x03(\x0b\x32\x30.build.bazel.remote.execution.v2.OutputDirectory\x12U\n\x19output_directory_symlinks\x18\x0b \x03(\x0b\x32..build.bazel.remote.execution.v2.OutputSymlinkB\x02\x18\x01\x12\x11\n\texit_code\x18\x04 \x01(\x05\x12\x12\n\nstdout_raw\x18\x05 \x01(\x0c\x12>\n\rstdout_digest\x18\x06 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x12\n\nstderr_raw\x18\x07 \x01(\x0c\x12>\n\rstderr_digest\x18\x08 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12S\n\x12\x65xecution_metadata\x18\t \x01(\x0b\x32\x37.build.bazel.remote.execution.v2.ExecutedActionMetadataJ\x04\x08\x01\x10\x02\"\xd2\x01\n\nOutputFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x15\n\ris_executable\x18\x04 \x01(\x08\x12\x10\n\x08\x63ontents\x18\x05 \x01(\x0c\x12H\n\x0fnode_properties\x18\x07 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x03\x10\x04J\x04\x08\x06\x10\x07\"~\n\x04Tree\x12\x38\n\x04root\x18\x01 \x01(\x0b\x32*.build.bazel.remote.execution.v2.Directory\x12<\n\x08\x63hildren\x18\x02 \x03(\x0b\x32*.build.bazel.remote.execution.v2.Directory\"\xcc\x01\n\x0fOutputDirectory\x12\x0c\n\x04path\x18\x01 \x01(\t\x12<\n\x0btree_digest\x18\x03 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x1f\n\x17is_topologically_sorted\x18\x04 \x01(\x08\x12\x46\n\x15root_directory_digest\x18\x05 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.DigestJ\x04\x08\x02\x10\x03\"}\n\rOutputSymlink\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0e\n\x06target\x18\x02 \x01(\t\x12H\n\x0fnode_properties\x18\x04 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x03\x10\x04\"#\n\x0f\x45xecutionPolicy\x12\x10\n\x08priority\x18\x01 \x01(\x05\"&\n\x12ResultsCachePolicy\x12\x10\n\x08priority\x18\x01 \x01(\x05\"\xce\x03\n\x0e\x45xecuteRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x19\n\x11skip_cache_lookup\x18\x03 \x01(\x08\x12>\n\raction_digest\x18\x06 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12J\n\x10\x65xecution_policy\x18\x07 \x01(\x0b\x32\x30.build.bazel.remote.execution.v2.ExecutionPolicy\x12Q\n\x14results_cache_policy\x18\x08 \x01(\x0b\x32\x33.build.bazel.remote.execution.v2.ResultsCachePolicy\x12N\n\x0f\x64igest_function\x18\t \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\x12\x15\n\rinline_stdout\x18\n \x01(\x08\x12\x15\n\rinline_stderr\x18\x0b \x01(\x08\x12\x1b\n\x13inline_output_files\x18\x0c \x03(\tJ\x04\x08\x02\x10\x03J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06\"Z\n\x07LogFile\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x16\n\x0ehuman_readable\x18\x02 \x01(\x08\"\xd0\x02\n\x0f\x45xecuteResponse\x12=\n\x06result\x18\x01 \x01(\x0b\x32-.build.bazel.remote.execution.v2.ActionResult\x12\x15\n\rcached_result\x18\x02 \x01(\x08\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12U\n\x0bserver_logs\x18\x04 \x03(\x0b\x32@.build.bazel.remote.execution.v2.ExecuteResponse.ServerLogsEntry\x12\x0f\n\x07message\x18\x05 \x01(\t\x1a[\n\x0fServerLogsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32(.build.bazel.remote.execution.v2.LogFile:\x02\x38\x01\"a\n\x0e\x45xecutionStage\"O\n\x05Value\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0f\n\x0b\x43\x41\x43HE_CHECK\x10\x01\x12\n\n\x06QUEUED\x10\x02\x12\r\n\tEXECUTING\x10\x03\x12\r\n\tCOMPLETED\x10\x04\"\xb5\x02\n\x18\x45xecuteOperationMetadata\x12\x44\n\x05stage\x18\x01 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.ExecutionStage.Value\x12>\n\raction_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x1a\n\x12stdout_stream_name\x18\x03 \x01(\t\x12\x1a\n\x12stderr_stream_name\x18\x04 \x01(\t\x12[\n\x1apartial_execution_metadata\x18\x05 \x01(\x0b\x32\x37.build.bazel.remote.execution.v2.ExecutedActionMetadata\"$\n\x14WaitExecutionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x8a\x02\n\x16GetActionResultRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12>\n\raction_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x15\n\rinline_stdout\x18\x03 \x01(\x08\x12\x15\n\rinline_stderr\x18\x04 \x01(\x08\x12\x1b\n\x13inline_output_files\x18\x05 \x03(\t\x12N\n\x0f\x64igest_function\x18\x06 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xdb\x02\n\x19UpdateActionResultRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12>\n\raction_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x44\n\raction_result\x18\x03 \x01(\x0b\x32-.build.bazel.remote.execution.v2.ActionResult\x12Q\n\x14results_cache_policy\x18\x04 \x01(\x0b\x32\x33.build.bazel.remote.execution.v2.ResultsCachePolicy\x12N\n\x0f\x64igest_function\x18\x05 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xbf\x01\n\x17\x46indMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12N\n\x0f\x64igest_function\x18\x03 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"a\n\x18\x46indMissingBlobsResponse\x12\x45\n\x14missing_blob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xee\x02\n\x17\x42\x61tchUpdateBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12R\n\x08requests\x18\x02 \x03(\x0b\x32@.build.bazel.remote.execution.v2.BatchUpdateBlobsRequest.Request\x12N\n\x0f\x64igest_function\x18\x05 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\x1a\x97\x01\n\x07Request\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x45\n\ncompressor\x18\x03 \x01(\x0e\x32\x31.build.bazel.remote.execution.v2.Compressor.Value\"\xda\x01\n\x18\x42\x61tchUpdateBlobsResponse\x12U\n\tresponses\x18\x01 \x03(\x0b\x32\x42.build.bazel.remote.execution.v2.BatchUpdateBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"\x8b\x02\n\x15\x42\x61tchReadBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x38\n\x07\x64igests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12Q\n\x16\x61\x63\x63\x65ptable_compressors\x18\x03 \x03(\x0e\x32\x31.build.bazel.remote.execution.v2.Compressor.Value\x12N\n\x0f\x64igest_function\x18\x04 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"\xac\x02\n\x16\x42\x61tchReadBlobsResponse\x12S\n\tresponses\x18\x01 \x03(\x0b\x32@.build.bazel.remote.execution.v2.BatchReadBlobsResponse.Response\x1a\xbc\x01\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\x0c\x12\x45\n\ncompressor\x18\x04 \x01(\x0e\x32\x31.build.bazel.remote.execution.v2.Compressor.Value\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\"\xdc\x01\n\x0eGetTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x11\n\tpage_size\x18\x03 \x01(\x05\x12\x12\n\npage_token\x18\x04 \x01(\t\x12N\n\x0f\x64igest_function\x18\x05 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"k\n\x0fGetTreeResponse\x12?\n\x0b\x64irectories\x18\x01 \x03(\x0b\x32*.build.bazel.remote.execution.v2.Directory\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"/\n\x16GetCapabilitiesRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\xe3\x02\n\x12ServerCapabilities\x12N\n\x12\x63\x61\x63he_capabilities\x18\x01 \x01(\x0b\x32\x32.build.bazel.remote.execution.v2.CacheCapabilities\x12V\n\x16\x65xecution_capabilities\x18\x02 \x01(\x0b\x32\x36.build.bazel.remote.execution.v2.ExecutionCapabilities\x12:\n\x16\x64\x65precated_api_version\x18\x03 \x01(\x0b\x32\x1a.build.bazel.semver.SemVer\x12\x33\n\x0flow_api_version\x18\x04 \x01(\x0b\x32\x1a.build.bazel.semver.SemVer\x12\x34\n\x10high_api_version\x18\x05 \x01(\x0b\x32\x1a.build.bazel.semver.SemVer\"\x8f\x01\n\x0e\x44igestFunction\"}\n\x05Value\x12\x0b\n\x07UNKNOWN\x10\x00\x12\n\n\x06SHA256\x10\x01\x12\x08\n\x04SHA1\x10\x02\x12\x07\n\x03MD5\x10\x03\x12\x07\n\x03VSO\x10\x04\x12\n\n\x06SHA384\x10\x05\x12\n\n\x06SHA512\x10\x06\x12\x0b\n\x07MURMUR3\x10\x07\x12\x0e\n\nSHA256TREE\x10\x08\x12\n\n\x06\x42LAKE3\x10\t\"7\n\x1d\x41\x63tionCacheUpdateCapabilities\x12\x16\n\x0eupdate_enabled\x18\x01 \x01(\x08\"\xac\x01\n\x14PriorityCapabilities\x12W\n\npriorities\x18\x01 \x03(\x0b\x32\x43.build.bazel.remote.execution.v2.PriorityCapabilities.PriorityRange\x1a;\n\rPriorityRange\x12\x14\n\x0cmin_priority\x18\x01 \x01(\x05\x12\x14\n\x0cmax_priority\x18\x02 \x01(\x05\"P\n\x1bSymlinkAbsolutePathStrategy\"1\n\x05Value\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x0e\n\nDISALLOWED\x10\x01\x12\x0b\n\x07\x41LLOWED\x10\x02\"F\n\nCompressor\"8\n\x05Value\x12\x0c\n\x08IDENTITY\x10\x00\x12\x08\n\x04ZSTD\x10\x01\x12\x0b\n\x07\x44\x45\x46LATE\x10\x02\x12\n\n\x06\x42ROTLI\x10\x03\"\xeb\x04\n\x11\x43\x61\x63heCapabilities\x12O\n\x10\x64igest_functions\x18\x01 \x03(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\x12h\n action_cache_update_capabilities\x18\x02 \x01(\x0b\x32>.build.bazel.remote.execution.v2.ActionCacheUpdateCapabilities\x12Z\n\x1b\x63\x61\x63he_priority_capabilities\x18\x03 \x01(\x0b\x32\x35.build.bazel.remote.execution.v2.PriorityCapabilities\x12\"\n\x1amax_batch_total_size_bytes\x18\x04 \x01(\x03\x12j\n\x1esymlink_absolute_path_strategy\x18\x05 \x01(\x0e\x32\x42.build.bazel.remote.execution.v2.SymlinkAbsolutePathStrategy.Value\x12P\n\x15supported_compressors\x18\x06 \x03(\x0e\x32\x31.build.bazel.remote.execution.v2.Compressor.Value\x12]\n\"supported_batch_update_compressors\x18\x07 \x03(\x0e\x32\x31.build.bazel.remote.execution.v2.Compressor.Value\"\xd1\x02\n\x15\x45xecutionCapabilities\x12N\n\x0f\x64igest_function\x18\x01 \x01(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\x12\x14\n\x0c\x65xec_enabled\x18\x02 \x01(\x08\x12^\n\x1f\x65xecution_priority_capabilities\x18\x03 \x01(\x0b\x32\x35.build.bazel.remote.execution.v2.PriorityCapabilities\x12!\n\x19supported_node_properties\x18\x04 \x03(\t\x12O\n\x10\x64igest_functions\x18\x05 \x03(\x0e\x32\x35.build.bazel.remote.execution.v2.DigestFunction.Value\"6\n\x0bToolDetails\x12\x11\n\ttool_name\x18\x01 \x01(\t\x12\x14\n\x0ctool_version\x18\x02 \x01(\t\"\xed\x01\n\x0fRequestMetadata\x12\x42\n\x0ctool_details\x18\x01 \x01(\x0b\x32,.build.bazel.remote.execution.v2.ToolDetails\x12\x11\n\taction_id\x18\x02 \x01(\t\x12\x1a\n\x12tool_invocation_id\x18\x03 \x01(\t\x12!\n\x19\x63orrelated_invocations_id\x18\x04 \x01(\t\x12\x17\n\x0f\x61\x63tion_mnemonic\x18\x05 \x01(\t\x12\x11\n\ttarget_id\x18\x06 \x01(\t\x12\x18\n\x10\x63onfiguration_id\x18\x07 \x01(\t2\xb9\x02\n\tExecution\x12\x8e\x01\n\x07\x45xecute\x12/.build.bazel.remote.execution.v2.ExecuteRequest\x1a\x1d.google.longrunning.Operation\"1\x82\xd3\xe4\x93\x02+\"&/v2/{instance_name=**}/actions:execute:\x01*0\x01\x12\x9a\x01\n\rWaitExecution\x12\x35.build.bazel.remote.execution.v2.WaitExecutionRequest\x1a\x1d.google.longrunning.Operation\"1\x82\xd3\xe4\x93\x02+\"&/v2/{name=operations/**}:waitExecution:\x01*0\x01\x32\xd6\x03\n\x0b\x41\x63tionCache\x12\xd7\x01\n\x0fGetActionResult\x12\x37.build.bazel.remote.execution.v2.GetActionResultRequest\x1a-.build.bazel.remote.execution.v2.ActionResult\"\\\x82\xd3\xe4\x93\x02V\x12T/v2/{instance_name=**}/actionResults/{action_digest.hash}/{action_digest.size_bytes}\x12\xec\x01\n\x12UpdateActionResult\x12:.build.bazel.remote.execution.v2.UpdateActionResultRequest\x1a-.build.bazel.remote.execution.v2.ActionResult\"k\x82\xd3\xe4\x93\x02\x65\x1aT/v2/{instance_name=**}/actionResults/{action_digest.hash}/{action_digest.size_bytes}:\raction_result2\x9b\x06\n\x19\x43ontentAddressableStorage\x12\xbc\x01\n\x10\x46indMissingBlobs\x12\x38.build.bazel.remote.execution.v2.FindMissingBlobsRequest\x1a\x39.build.bazel.remote.execution.v2.FindMissingBlobsResponse\"3\x82\xd3\xe4\x93\x02-\"(/v2/{instance_name=**}/blobs:findMissing:\x01*\x12\xbc\x01\n\x10\x42\x61tchUpdateBlobs\x12\x38.build.bazel.remote.execution.v2.BatchUpdateBlobsRequest\x1a\x39.build.bazel.remote.execution.v2.BatchUpdateBlobsResponse\"3\x82\xd3\xe4\x93\x02-\"(/v2/{instance_name=**}/blobs:batchUpdate:\x01*\x12\xb4\x01\n\x0e\x42\x61tchReadBlobs\x12\x36.build.bazel.remote.execution.v2.BatchReadBlobsRequest\x1a\x37.build.bazel.remote.execution.v2.BatchReadBlobsResponse\"1\x82\xd3\xe4\x93\x02+\"&/v2/{instance_name=**}/blobs:batchRead:\x01*\x12\xc8\x01\n\x07GetTree\x12/.build.bazel.remote.execution.v2.GetTreeRequest\x1a\x30.build.bazel.remote.execution.v2.GetTreeResponse\"X\x82\xd3\xe4\x93\x02R\x12P/v2/{instance_name=**}/blobs/{root_digest.hash}/{root_digest.size_bytes}:getTree0\x01\x32\xbd\x01\n\x0c\x43\x61pabilities\x12\xac\x01\n\x0fGetCapabilities\x12\x37.build.bazel.remote.execution.v2.GetCapabilitiesRequest\x1a\x33.build.bazel.remote.execution.v2.ServerCapabilities\"+\x82\xd3\xe4\x93\x02%\x12#/v2/{instance_name=**}/capabilitiesB\xb4\x01\n\x1f\x62uild.bazel.remote.execution.v2B\x14RemoteExecutionProtoP\x01ZQgithub.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2;remoteexecution\xa2\x02\x03REX\xaa\x02\x1f\x42uild.Bazel.Remote.Execution.V2b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'build.bazel.remote.execution.v2.remote_execution_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\037build.bazel.remote.execution.v2B\024RemoteExecutionProtoP\001ZQgithub.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2;remoteexecution\242\002\003REX\252\002\037Build.Bazel.Remote.Execution.V2' _globals['_COMMAND'].fields_by_name['output_files']._loaded_options = None _globals['_COMMAND'].fields_by_name['output_files']._serialized_options = b'\030\001' _globals['_COMMAND'].fields_by_name['output_directories']._loaded_options = None _globals['_COMMAND'].fields_by_name['output_directories']._serialized_options = b'\030\001' _globals['_COMMAND'].fields_by_name['platform']._loaded_options = None _globals['_COMMAND'].fields_by_name['platform']._serialized_options = b'\030\001' _globals['_ACTIONRESULT'].fields_by_name['output_file_symlinks']._loaded_options = None _globals['_ACTIONRESULT'].fields_by_name['output_file_symlinks']._serialized_options = b'\030\001' _globals['_ACTIONRESULT'].fields_by_name['output_directory_symlinks']._loaded_options = None _globals['_ACTIONRESULT'].fields_by_name['output_directory_symlinks']._serialized_options = b'\030\001' _globals['_EXECUTERESPONSE_SERVERLOGSENTRY']._loaded_options = None _globals['_EXECUTERESPONSE_SERVERLOGSENTRY']._serialized_options = b'8\001' _globals['_EXECUTION'].methods_by_name['Execute']._loaded_options = None _globals['_EXECUTION'].methods_by_name['Execute']._serialized_options = b'\202\323\344\223\002+\"&/v2/{instance_name=**}/actions:execute:\001*' _globals['_EXECUTION'].methods_by_name['WaitExecution']._loaded_options = None _globals['_EXECUTION'].methods_by_name['WaitExecution']._serialized_options = b'\202\323\344\223\002+\"&/v2/{name=operations/**}:waitExecution:\001*' _globals['_ACTIONCACHE'].methods_by_name['GetActionResult']._loaded_options = None _globals['_ACTIONCACHE'].methods_by_name['GetActionResult']._serialized_options = b'\202\323\344\223\002V\022T/v2/{instance_name=**}/actionResults/{action_digest.hash}/{action_digest.size_bytes}' _globals['_ACTIONCACHE'].methods_by_name['UpdateActionResult']._loaded_options = None _globals['_ACTIONCACHE'].methods_by_name['UpdateActionResult']._serialized_options = b'\202\323\344\223\002e\032T/v2/{instance_name=**}/actionResults/{action_digest.hash}/{action_digest.size_bytes}:\raction_result' _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['FindMissingBlobs']._loaded_options = None _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['FindMissingBlobs']._serialized_options = b'\202\323\344\223\002-\"(/v2/{instance_name=**}/blobs:findMissing:\001*' _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['BatchUpdateBlobs']._loaded_options = None _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['BatchUpdateBlobs']._serialized_options = b'\202\323\344\223\002-\"(/v2/{instance_name=**}/blobs:batchUpdate:\001*' _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['BatchReadBlobs']._loaded_options = None _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['BatchReadBlobs']._serialized_options = b'\202\323\344\223\002+\"&/v2/{instance_name=**}/blobs:batchRead:\001*' _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['GetTree']._loaded_options = None _globals['_CONTENTADDRESSABLESTORAGE'].methods_by_name['GetTree']._serialized_options = b'\202\323\344\223\002R\022P/v2/{instance_name=**}/blobs/{root_digest.hash}/{root_digest.size_bytes}:getTree' _globals['_CAPABILITIES'].methods_by_name['GetCapabilities']._loaded_options = None _globals['_CAPABILITIES'].methods_by_name['GetCapabilities']._serialized_options = b'\202\323\344\223\002%\022#/v2/{instance_name=**}/capabilities' _globals['_ACTION']._serialized_start=341 _globals['_ACTION']._serialized_end=635 _globals['_COMMAND']._serialized_start=638 _globals['_COMMAND']._serialized_end=1196 _globals['_COMMAND_ENVIRONMENTVARIABLE']._serialized_start=1062 _globals['_COMMAND_ENVIRONMENTVARIABLE']._serialized_end=1112 _globals['_COMMAND_OUTPUTDIRECTORYFORMAT']._serialized_start=1114 _globals['_COMMAND_OUTPUTDIRECTORYFORMAT']._serialized_end=1196 _globals['_PLATFORM']._serialized_start=1198 _globals['_PLATFORM']._serialized_end=1321 _globals['_PLATFORM_PROPERTY']._serialized_start=1282 _globals['_PLATFORM_PROPERTY']._serialized_end=1321 _globals['_DIRECTORY']._serialized_start=1324 _globals['_DIRECTORY']._serialized_end=1606 _globals['_NODEPROPERTY']._serialized_start=1608 _globals['_NODEPROPERTY']._serialized_end=1651 _globals['_NODEPROPERTIES']._serialized_start=1654 _globals['_NODEPROPERTIES']._serialized_end=1829 _globals['_FILENODE']._serialized_start=1832 _globals['_FILENODE']._serialized_end=2022 _globals['_DIRECTORYNODE']._serialized_start=2024 _globals['_DIRECTORYNODE']._serialized_end=2110 _globals['_SYMLINKNODE']._serialized_start=2112 _globals['_SYMLINKNODE']._serialized_end=2235 _globals['_DIGEST']._serialized_start=2237 _globals['_DIGEST']._serialized_end=2279 _globals['_EXECUTEDACTIONMETADATA']._serialized_start=2282 _globals['_EXECUTEDACTIONMETADATA']._serialized_end=3015 _globals['_ACTIONRESULT']._serialized_start=3018 _globals['_ACTIONRESULT']._serialized_end=3697 _globals['_OUTPUTFILE']._serialized_start=3700 _globals['_OUTPUTFILE']._serialized_end=3910 _globals['_TREE']._serialized_start=3912 _globals['_TREE']._serialized_end=4038 _globals['_OUTPUTDIRECTORY']._serialized_start=4041 _globals['_OUTPUTDIRECTORY']._serialized_end=4245 _globals['_OUTPUTSYMLINK']._serialized_start=4247 _globals['_OUTPUTSYMLINK']._serialized_end=4372 _globals['_EXECUTIONPOLICY']._serialized_start=4374 _globals['_EXECUTIONPOLICY']._serialized_end=4409 _globals['_RESULTSCACHEPOLICY']._serialized_start=4411 _globals['_RESULTSCACHEPOLICY']._serialized_end=4449 _globals['_EXECUTEREQUEST']._serialized_start=4452 _globals['_EXECUTEREQUEST']._serialized_end=4914 _globals['_LOGFILE']._serialized_start=4916 _globals['_LOGFILE']._serialized_end=5006 _globals['_EXECUTERESPONSE']._serialized_start=5009 _globals['_EXECUTERESPONSE']._serialized_end=5345 _globals['_EXECUTERESPONSE_SERVERLOGSENTRY']._serialized_start=5254 _globals['_EXECUTERESPONSE_SERVERLOGSENTRY']._serialized_end=5345 _globals['_EXECUTIONSTAGE']._serialized_start=5347 _globals['_EXECUTIONSTAGE']._serialized_end=5444 _globals['_EXECUTIONSTAGE_VALUE']._serialized_start=5365 _globals['_EXECUTIONSTAGE_VALUE']._serialized_end=5444 _globals['_EXECUTEOPERATIONMETADATA']._serialized_start=5447 _globals['_EXECUTEOPERATIONMETADATA']._serialized_end=5756 _globals['_WAITEXECUTIONREQUEST']._serialized_start=5758 _globals['_WAITEXECUTIONREQUEST']._serialized_end=5794 _globals['_GETACTIONRESULTREQUEST']._serialized_start=5797 _globals['_GETACTIONRESULTREQUEST']._serialized_end=6063 _globals['_UPDATEACTIONRESULTREQUEST']._serialized_start=6066 _globals['_UPDATEACTIONRESULTREQUEST']._serialized_end=6413 _globals['_FINDMISSINGBLOBSREQUEST']._serialized_start=6416 _globals['_FINDMISSINGBLOBSREQUEST']._serialized_end=6607 _globals['_FINDMISSINGBLOBSRESPONSE']._serialized_start=6609 _globals['_FINDMISSINGBLOBSRESPONSE']._serialized_end=6706 _globals['_BATCHUPDATEBLOBSREQUEST']._serialized_start=6709 _globals['_BATCHUPDATEBLOBSREQUEST']._serialized_end=7075 _globals['_BATCHUPDATEBLOBSREQUEST_REQUEST']._serialized_start=6924 _globals['_BATCHUPDATEBLOBSREQUEST_REQUEST']._serialized_end=7075 _globals['_BATCHUPDATEBLOBSRESPONSE']._serialized_start=7078 _globals['_BATCHUPDATEBLOBSRESPONSE']._serialized_end=7296 _globals['_BATCHUPDATEBLOBSRESPONSE_RESPONSE']._serialized_start=7193 _globals['_BATCHUPDATEBLOBSRESPONSE_RESPONSE']._serialized_end=7296 _globals['_BATCHREADBLOBSREQUEST']._serialized_start=7299 _globals['_BATCHREADBLOBSREQUEST']._serialized_end=7566 _globals['_BATCHREADBLOBSRESPONSE']._serialized_start=7569 _globals['_BATCHREADBLOBSRESPONSE']._serialized_end=7869 _globals['_BATCHREADBLOBSRESPONSE_RESPONSE']._serialized_start=7681 _globals['_BATCHREADBLOBSRESPONSE_RESPONSE']._serialized_end=7869 _globals['_GETTREEREQUEST']._serialized_start=7872 _globals['_GETTREEREQUEST']._serialized_end=8092 _globals['_GETTREERESPONSE']._serialized_start=8094 _globals['_GETTREERESPONSE']._serialized_end=8201 _globals['_GETCAPABILITIESREQUEST']._serialized_start=8203 _globals['_GETCAPABILITIESREQUEST']._serialized_end=8250 _globals['_SERVERCAPABILITIES']._serialized_start=8253 _globals['_SERVERCAPABILITIES']._serialized_end=8608 _globals['_DIGESTFUNCTION']._serialized_start=8611 _globals['_DIGESTFUNCTION']._serialized_end=8754 _globals['_DIGESTFUNCTION_VALUE']._serialized_start=8629 _globals['_DIGESTFUNCTION_VALUE']._serialized_end=8754 _globals['_ACTIONCACHEUPDATECAPABILITIES']._serialized_start=8756 _globals['_ACTIONCACHEUPDATECAPABILITIES']._serialized_end=8811 _globals['_PRIORITYCAPABILITIES']._serialized_start=8814 _globals['_PRIORITYCAPABILITIES']._serialized_end=8986 _globals['_PRIORITYCAPABILITIES_PRIORITYRANGE']._serialized_start=8927 _globals['_PRIORITYCAPABILITIES_PRIORITYRANGE']._serialized_end=8986 _globals['_SYMLINKABSOLUTEPATHSTRATEGY']._serialized_start=8988 _globals['_SYMLINKABSOLUTEPATHSTRATEGY']._serialized_end=9068 _globals['_SYMLINKABSOLUTEPATHSTRATEGY_VALUE']._serialized_start=9019 _globals['_SYMLINKABSOLUTEPATHSTRATEGY_VALUE']._serialized_end=9068 _globals['_COMPRESSOR']._serialized_start=9070 _globals['_COMPRESSOR']._serialized_end=9140 _globals['_COMPRESSOR_VALUE']._serialized_start=9084 _globals['_COMPRESSOR_VALUE']._serialized_end=9140 _globals['_CACHECAPABILITIES']._serialized_start=9143 _globals['_CACHECAPABILITIES']._serialized_end=9762 _globals['_EXECUTIONCAPABILITIES']._serialized_start=9765 _globals['_EXECUTIONCAPABILITIES']._serialized_end=10102 _globals['_TOOLDETAILS']._serialized_start=10104 _globals['_TOOLDETAILS']._serialized_end=10158 _globals['_REQUESTMETADATA']._serialized_start=10161 _globals['_REQUESTMETADATA']._serialized_end=10398 _globals['_EXECUTION']._serialized_start=10401 _globals['_EXECUTION']._serialized_end=10714 _globals['_ACTIONCACHE']._serialized_start=10717 _globals['_ACTIONCACHE']._serialized_end=11187 _globals['_CONTENTADDRESSABLESTORAGE']._serialized_start=11190 _globals['_CONTENTADDRESSABLESTORAGE']._serialized_end=11985 _globals['_CAPABILITIES']._serialized_start=11988 _globals['_CAPABILITIES']._serialized_end=12177 # @@protoc_insertion_point(module_scope) remote_execution_pb2.pyi000066400000000000000000001073601514607367700352260ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/v2from build.bazel.semver import semver_pb2 as _semver_pb2 from google.api import annotations_pb2 as _annotations_pb2 from google.longrunning import operations_pb2 as _operations_pb2 from google.protobuf import any_pb2 as _any_pb2 from google.protobuf import duration_pb2 as _duration_pb2 from google.protobuf import timestamp_pb2 as _timestamp_pb2 from google.protobuf import wrappers_pb2 as _wrappers_pb2 from google.rpc import status_pb2 as _status_pb2 from google.protobuf.internal import containers as _containers from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Action(_message.Message): __slots__ = ("command_digest", "input_root_digest", "timeout", "do_not_cache", "salt", "platform") COMMAND_DIGEST_FIELD_NUMBER: _ClassVar[int] INPUT_ROOT_DIGEST_FIELD_NUMBER: _ClassVar[int] TIMEOUT_FIELD_NUMBER: _ClassVar[int] DO_NOT_CACHE_FIELD_NUMBER: _ClassVar[int] SALT_FIELD_NUMBER: _ClassVar[int] PLATFORM_FIELD_NUMBER: _ClassVar[int] command_digest: Digest input_root_digest: Digest timeout: _duration_pb2.Duration do_not_cache: bool salt: bytes platform: Platform def __init__(self, command_digest: _Optional[_Union[Digest, _Mapping]] = ..., input_root_digest: _Optional[_Union[Digest, _Mapping]] = ..., timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., do_not_cache: bool = ..., salt: _Optional[bytes] = ..., platform: _Optional[_Union[Platform, _Mapping]] = ...) -> None: ... class Command(_message.Message): __slots__ = ("arguments", "environment_variables", "output_files", "output_directories", "output_paths", "platform", "working_directory", "output_node_properties", "output_directory_format") class OutputDirectoryFormat(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () TREE_ONLY: _ClassVar[Command.OutputDirectoryFormat] DIRECTORY_ONLY: _ClassVar[Command.OutputDirectoryFormat] TREE_AND_DIRECTORY: _ClassVar[Command.OutputDirectoryFormat] TREE_ONLY: Command.OutputDirectoryFormat DIRECTORY_ONLY: Command.OutputDirectoryFormat TREE_AND_DIRECTORY: Command.OutputDirectoryFormat class EnvironmentVariable(_message.Message): __slots__ = ("name", "value") NAME_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] name: str value: str def __init__(self, name: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... ARGUMENTS_FIELD_NUMBER: _ClassVar[int] ENVIRONMENT_VARIABLES_FIELD_NUMBER: _ClassVar[int] OUTPUT_FILES_FIELD_NUMBER: _ClassVar[int] OUTPUT_DIRECTORIES_FIELD_NUMBER: _ClassVar[int] OUTPUT_PATHS_FIELD_NUMBER: _ClassVar[int] PLATFORM_FIELD_NUMBER: _ClassVar[int] WORKING_DIRECTORY_FIELD_NUMBER: _ClassVar[int] OUTPUT_NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] OUTPUT_DIRECTORY_FORMAT_FIELD_NUMBER: _ClassVar[int] arguments: _containers.RepeatedScalarFieldContainer[str] environment_variables: _containers.RepeatedCompositeFieldContainer[Command.EnvironmentVariable] output_files: _containers.RepeatedScalarFieldContainer[str] output_directories: _containers.RepeatedScalarFieldContainer[str] output_paths: _containers.RepeatedScalarFieldContainer[str] platform: Platform working_directory: str output_node_properties: _containers.RepeatedScalarFieldContainer[str] output_directory_format: Command.OutputDirectoryFormat def __init__(self, arguments: _Optional[_Iterable[str]] = ..., environment_variables: _Optional[_Iterable[_Union[Command.EnvironmentVariable, _Mapping]]] = ..., output_files: _Optional[_Iterable[str]] = ..., output_directories: _Optional[_Iterable[str]] = ..., output_paths: _Optional[_Iterable[str]] = ..., platform: _Optional[_Union[Platform, _Mapping]] = ..., working_directory: _Optional[str] = ..., output_node_properties: _Optional[_Iterable[str]] = ..., output_directory_format: _Optional[_Union[Command.OutputDirectoryFormat, str]] = ...) -> None: ... class Platform(_message.Message): __slots__ = ("properties",) class Property(_message.Message): __slots__ = ("name", "value") NAME_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] name: str value: str def __init__(self, name: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... PROPERTIES_FIELD_NUMBER: _ClassVar[int] properties: _containers.RepeatedCompositeFieldContainer[Platform.Property] def __init__(self, properties: _Optional[_Iterable[_Union[Platform.Property, _Mapping]]] = ...) -> None: ... class Directory(_message.Message): __slots__ = ("files", "directories", "symlinks", "node_properties") FILES_FIELD_NUMBER: _ClassVar[int] DIRECTORIES_FIELD_NUMBER: _ClassVar[int] SYMLINKS_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] files: _containers.RepeatedCompositeFieldContainer[FileNode] directories: _containers.RepeatedCompositeFieldContainer[DirectoryNode] symlinks: _containers.RepeatedCompositeFieldContainer[SymlinkNode] node_properties: NodeProperties def __init__(self, files: _Optional[_Iterable[_Union[FileNode, _Mapping]]] = ..., directories: _Optional[_Iterable[_Union[DirectoryNode, _Mapping]]] = ..., symlinks: _Optional[_Iterable[_Union[SymlinkNode, _Mapping]]] = ..., node_properties: _Optional[_Union[NodeProperties, _Mapping]] = ...) -> None: ... class NodeProperty(_message.Message): __slots__ = ("name", "value") NAME_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] name: str value: str def __init__(self, name: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... class NodeProperties(_message.Message): __slots__ = ("properties", "mtime", "unix_mode") PROPERTIES_FIELD_NUMBER: _ClassVar[int] MTIME_FIELD_NUMBER: _ClassVar[int] UNIX_MODE_FIELD_NUMBER: _ClassVar[int] properties: _containers.RepeatedCompositeFieldContainer[NodeProperty] mtime: _timestamp_pb2.Timestamp unix_mode: _wrappers_pb2.UInt32Value def __init__(self, properties: _Optional[_Iterable[_Union[NodeProperty, _Mapping]]] = ..., mtime: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., unix_mode: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ...) -> None: ... class FileNode(_message.Message): __slots__ = ("name", "digest", "is_executable", "node_properties") NAME_FIELD_NUMBER: _ClassVar[int] DIGEST_FIELD_NUMBER: _ClassVar[int] IS_EXECUTABLE_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] name: str digest: Digest is_executable: bool node_properties: NodeProperties def __init__(self, name: _Optional[str] = ..., digest: _Optional[_Union[Digest, _Mapping]] = ..., is_executable: bool = ..., node_properties: _Optional[_Union[NodeProperties, _Mapping]] = ...) -> None: ... class DirectoryNode(_message.Message): __slots__ = ("name", "digest") NAME_FIELD_NUMBER: _ClassVar[int] DIGEST_FIELD_NUMBER: _ClassVar[int] name: str digest: Digest def __init__(self, name: _Optional[str] = ..., digest: _Optional[_Union[Digest, _Mapping]] = ...) -> None: ... class SymlinkNode(_message.Message): __slots__ = ("name", "target", "node_properties") NAME_FIELD_NUMBER: _ClassVar[int] TARGET_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] name: str target: str node_properties: NodeProperties def __init__(self, name: _Optional[str] = ..., target: _Optional[str] = ..., node_properties: _Optional[_Union[NodeProperties, _Mapping]] = ...) -> None: ... class Digest(_message.Message): __slots__ = ("hash", "size_bytes") HASH_FIELD_NUMBER: _ClassVar[int] SIZE_BYTES_FIELD_NUMBER: _ClassVar[int] hash: str size_bytes: int def __init__(self, hash: _Optional[str] = ..., size_bytes: _Optional[int] = ...) -> None: ... class ExecutedActionMetadata(_message.Message): __slots__ = ("worker", "queued_timestamp", "worker_start_timestamp", "worker_completed_timestamp", "input_fetch_start_timestamp", "input_fetch_completed_timestamp", "execution_start_timestamp", "execution_completed_timestamp", "virtual_execution_duration", "output_upload_start_timestamp", "output_upload_completed_timestamp", "auxiliary_metadata") WORKER_FIELD_NUMBER: _ClassVar[int] QUEUED_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] WORKER_START_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] WORKER_COMPLETED_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] INPUT_FETCH_START_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] INPUT_FETCH_COMPLETED_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] EXECUTION_START_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] EXECUTION_COMPLETED_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] VIRTUAL_EXECUTION_DURATION_FIELD_NUMBER: _ClassVar[int] OUTPUT_UPLOAD_START_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] OUTPUT_UPLOAD_COMPLETED_TIMESTAMP_FIELD_NUMBER: _ClassVar[int] AUXILIARY_METADATA_FIELD_NUMBER: _ClassVar[int] worker: str queued_timestamp: _timestamp_pb2.Timestamp worker_start_timestamp: _timestamp_pb2.Timestamp worker_completed_timestamp: _timestamp_pb2.Timestamp input_fetch_start_timestamp: _timestamp_pb2.Timestamp input_fetch_completed_timestamp: _timestamp_pb2.Timestamp execution_start_timestamp: _timestamp_pb2.Timestamp execution_completed_timestamp: _timestamp_pb2.Timestamp virtual_execution_duration: _duration_pb2.Duration output_upload_start_timestamp: _timestamp_pb2.Timestamp output_upload_completed_timestamp: _timestamp_pb2.Timestamp auxiliary_metadata: _containers.RepeatedCompositeFieldContainer[_any_pb2.Any] def __init__(self, worker: _Optional[str] = ..., queued_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., worker_start_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., worker_completed_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., input_fetch_start_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., input_fetch_completed_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., execution_start_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., execution_completed_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., virtual_execution_duration: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., output_upload_start_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., output_upload_completed_timestamp: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., auxiliary_metadata: _Optional[_Iterable[_Union[_any_pb2.Any, _Mapping]]] = ...) -> None: ... class ActionResult(_message.Message): __slots__ = ("output_files", "output_file_symlinks", "output_symlinks", "output_directories", "output_directory_symlinks", "exit_code", "stdout_raw", "stdout_digest", "stderr_raw", "stderr_digest", "execution_metadata") OUTPUT_FILES_FIELD_NUMBER: _ClassVar[int] OUTPUT_FILE_SYMLINKS_FIELD_NUMBER: _ClassVar[int] OUTPUT_SYMLINKS_FIELD_NUMBER: _ClassVar[int] OUTPUT_DIRECTORIES_FIELD_NUMBER: _ClassVar[int] OUTPUT_DIRECTORY_SYMLINKS_FIELD_NUMBER: _ClassVar[int] EXIT_CODE_FIELD_NUMBER: _ClassVar[int] STDOUT_RAW_FIELD_NUMBER: _ClassVar[int] STDOUT_DIGEST_FIELD_NUMBER: _ClassVar[int] STDERR_RAW_FIELD_NUMBER: _ClassVar[int] STDERR_DIGEST_FIELD_NUMBER: _ClassVar[int] EXECUTION_METADATA_FIELD_NUMBER: _ClassVar[int] output_files: _containers.RepeatedCompositeFieldContainer[OutputFile] output_file_symlinks: _containers.RepeatedCompositeFieldContainer[OutputSymlink] output_symlinks: _containers.RepeatedCompositeFieldContainer[OutputSymlink] output_directories: _containers.RepeatedCompositeFieldContainer[OutputDirectory] output_directory_symlinks: _containers.RepeatedCompositeFieldContainer[OutputSymlink] exit_code: int stdout_raw: bytes stdout_digest: Digest stderr_raw: bytes stderr_digest: Digest execution_metadata: ExecutedActionMetadata def __init__(self, output_files: _Optional[_Iterable[_Union[OutputFile, _Mapping]]] = ..., output_file_symlinks: _Optional[_Iterable[_Union[OutputSymlink, _Mapping]]] = ..., output_symlinks: _Optional[_Iterable[_Union[OutputSymlink, _Mapping]]] = ..., output_directories: _Optional[_Iterable[_Union[OutputDirectory, _Mapping]]] = ..., output_directory_symlinks: _Optional[_Iterable[_Union[OutputSymlink, _Mapping]]] = ..., exit_code: _Optional[int] = ..., stdout_raw: _Optional[bytes] = ..., stdout_digest: _Optional[_Union[Digest, _Mapping]] = ..., stderr_raw: _Optional[bytes] = ..., stderr_digest: _Optional[_Union[Digest, _Mapping]] = ..., execution_metadata: _Optional[_Union[ExecutedActionMetadata, _Mapping]] = ...) -> None: ... class OutputFile(_message.Message): __slots__ = ("path", "digest", "is_executable", "contents", "node_properties") PATH_FIELD_NUMBER: _ClassVar[int] DIGEST_FIELD_NUMBER: _ClassVar[int] IS_EXECUTABLE_FIELD_NUMBER: _ClassVar[int] CONTENTS_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] path: str digest: Digest is_executable: bool contents: bytes node_properties: NodeProperties def __init__(self, path: _Optional[str] = ..., digest: _Optional[_Union[Digest, _Mapping]] = ..., is_executable: bool = ..., contents: _Optional[bytes] = ..., node_properties: _Optional[_Union[NodeProperties, _Mapping]] = ...) -> None: ... class Tree(_message.Message): __slots__ = ("root", "children") ROOT_FIELD_NUMBER: _ClassVar[int] CHILDREN_FIELD_NUMBER: _ClassVar[int] root: Directory children: _containers.RepeatedCompositeFieldContainer[Directory] def __init__(self, root: _Optional[_Union[Directory, _Mapping]] = ..., children: _Optional[_Iterable[_Union[Directory, _Mapping]]] = ...) -> None: ... class OutputDirectory(_message.Message): __slots__ = ("path", "tree_digest", "is_topologically_sorted", "root_directory_digest") PATH_FIELD_NUMBER: _ClassVar[int] TREE_DIGEST_FIELD_NUMBER: _ClassVar[int] IS_TOPOLOGICALLY_SORTED_FIELD_NUMBER: _ClassVar[int] ROOT_DIRECTORY_DIGEST_FIELD_NUMBER: _ClassVar[int] path: str tree_digest: Digest is_topologically_sorted: bool root_directory_digest: Digest def __init__(self, path: _Optional[str] = ..., tree_digest: _Optional[_Union[Digest, _Mapping]] = ..., is_topologically_sorted: bool = ..., root_directory_digest: _Optional[_Union[Digest, _Mapping]] = ...) -> None: ... class OutputSymlink(_message.Message): __slots__ = ("path", "target", "node_properties") PATH_FIELD_NUMBER: _ClassVar[int] TARGET_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] path: str target: str node_properties: NodeProperties def __init__(self, path: _Optional[str] = ..., target: _Optional[str] = ..., node_properties: _Optional[_Union[NodeProperties, _Mapping]] = ...) -> None: ... class ExecutionPolicy(_message.Message): __slots__ = ("priority",) PRIORITY_FIELD_NUMBER: _ClassVar[int] priority: int def __init__(self, priority: _Optional[int] = ...) -> None: ... class ResultsCachePolicy(_message.Message): __slots__ = ("priority",) PRIORITY_FIELD_NUMBER: _ClassVar[int] priority: int def __init__(self, priority: _Optional[int] = ...) -> None: ... class ExecuteRequest(_message.Message): __slots__ = ("instance_name", "skip_cache_lookup", "action_digest", "execution_policy", "results_cache_policy", "digest_function", "inline_stdout", "inline_stderr", "inline_output_files") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] SKIP_CACHE_LOOKUP_FIELD_NUMBER: _ClassVar[int] ACTION_DIGEST_FIELD_NUMBER: _ClassVar[int] EXECUTION_POLICY_FIELD_NUMBER: _ClassVar[int] RESULTS_CACHE_POLICY_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] INLINE_STDOUT_FIELD_NUMBER: _ClassVar[int] INLINE_STDERR_FIELD_NUMBER: _ClassVar[int] INLINE_OUTPUT_FILES_FIELD_NUMBER: _ClassVar[int] instance_name: str skip_cache_lookup: bool action_digest: Digest execution_policy: ExecutionPolicy results_cache_policy: ResultsCachePolicy digest_function: DigestFunction.Value inline_stdout: bool inline_stderr: bool inline_output_files: _containers.RepeatedScalarFieldContainer[str] def __init__(self, instance_name: _Optional[str] = ..., skip_cache_lookup: bool = ..., action_digest: _Optional[_Union[Digest, _Mapping]] = ..., execution_policy: _Optional[_Union[ExecutionPolicy, _Mapping]] = ..., results_cache_policy: _Optional[_Union[ResultsCachePolicy, _Mapping]] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ..., inline_stdout: bool = ..., inline_stderr: bool = ..., inline_output_files: _Optional[_Iterable[str]] = ...) -> None: ... class LogFile(_message.Message): __slots__ = ("digest", "human_readable") DIGEST_FIELD_NUMBER: _ClassVar[int] HUMAN_READABLE_FIELD_NUMBER: _ClassVar[int] digest: Digest human_readable: bool def __init__(self, digest: _Optional[_Union[Digest, _Mapping]] = ..., human_readable: bool = ...) -> None: ... class ExecuteResponse(_message.Message): __slots__ = ("result", "cached_result", "status", "server_logs", "message") class ServerLogsEntry(_message.Message): __slots__ = ("key", "value") KEY_FIELD_NUMBER: _ClassVar[int] VALUE_FIELD_NUMBER: _ClassVar[int] key: str value: LogFile def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[LogFile, _Mapping]] = ...) -> None: ... RESULT_FIELD_NUMBER: _ClassVar[int] CACHED_RESULT_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] SERVER_LOGS_FIELD_NUMBER: _ClassVar[int] MESSAGE_FIELD_NUMBER: _ClassVar[int] result: ActionResult cached_result: bool status: _status_pb2.Status server_logs: _containers.MessageMap[str, LogFile] message: str def __init__(self, result: _Optional[_Union[ActionResult, _Mapping]] = ..., cached_result: bool = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., server_logs: _Optional[_Mapping[str, LogFile]] = ..., message: _Optional[str] = ...) -> None: ... class ExecutionStage(_message.Message): __slots__ = () class Value(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () UNKNOWN: _ClassVar[ExecutionStage.Value] CACHE_CHECK: _ClassVar[ExecutionStage.Value] QUEUED: _ClassVar[ExecutionStage.Value] EXECUTING: _ClassVar[ExecutionStage.Value] COMPLETED: _ClassVar[ExecutionStage.Value] UNKNOWN: ExecutionStage.Value CACHE_CHECK: ExecutionStage.Value QUEUED: ExecutionStage.Value EXECUTING: ExecutionStage.Value COMPLETED: ExecutionStage.Value def __init__(self) -> None: ... class ExecuteOperationMetadata(_message.Message): __slots__ = ("stage", "action_digest", "stdout_stream_name", "stderr_stream_name", "partial_execution_metadata") STAGE_FIELD_NUMBER: _ClassVar[int] ACTION_DIGEST_FIELD_NUMBER: _ClassVar[int] STDOUT_STREAM_NAME_FIELD_NUMBER: _ClassVar[int] STDERR_STREAM_NAME_FIELD_NUMBER: _ClassVar[int] PARTIAL_EXECUTION_METADATA_FIELD_NUMBER: _ClassVar[int] stage: ExecutionStage.Value action_digest: Digest stdout_stream_name: str stderr_stream_name: str partial_execution_metadata: ExecutedActionMetadata def __init__(self, stage: _Optional[_Union[ExecutionStage.Value, str]] = ..., action_digest: _Optional[_Union[Digest, _Mapping]] = ..., stdout_stream_name: _Optional[str] = ..., stderr_stream_name: _Optional[str] = ..., partial_execution_metadata: _Optional[_Union[ExecutedActionMetadata, _Mapping]] = ...) -> None: ... class WaitExecutionRequest(_message.Message): __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] name: str def __init__(self, name: _Optional[str] = ...) -> None: ... class GetActionResultRequest(_message.Message): __slots__ = ("instance_name", "action_digest", "inline_stdout", "inline_stderr", "inline_output_files", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ACTION_DIGEST_FIELD_NUMBER: _ClassVar[int] INLINE_STDOUT_FIELD_NUMBER: _ClassVar[int] INLINE_STDERR_FIELD_NUMBER: _ClassVar[int] INLINE_OUTPUT_FILES_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str action_digest: Digest inline_stdout: bool inline_stderr: bool inline_output_files: _containers.RepeatedScalarFieldContainer[str] digest_function: DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., action_digest: _Optional[_Union[Digest, _Mapping]] = ..., inline_stdout: bool = ..., inline_stderr: bool = ..., inline_output_files: _Optional[_Iterable[str]] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ...) -> None: ... class UpdateActionResultRequest(_message.Message): __slots__ = ("instance_name", "action_digest", "action_result", "results_cache_policy", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ACTION_DIGEST_FIELD_NUMBER: _ClassVar[int] ACTION_RESULT_FIELD_NUMBER: _ClassVar[int] RESULTS_CACHE_POLICY_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str action_digest: Digest action_result: ActionResult results_cache_policy: ResultsCachePolicy digest_function: DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., action_digest: _Optional[_Union[Digest, _Mapping]] = ..., action_result: _Optional[_Union[ActionResult, _Mapping]] = ..., results_cache_policy: _Optional[_Union[ResultsCachePolicy, _Mapping]] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ...) -> None: ... class FindMissingBlobsRequest(_message.Message): __slots__ = ("instance_name", "blob_digests", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] BLOB_DIGESTS_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str blob_digests: _containers.RepeatedCompositeFieldContainer[Digest] digest_function: DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., blob_digests: _Optional[_Iterable[_Union[Digest, _Mapping]]] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ...) -> None: ... class FindMissingBlobsResponse(_message.Message): __slots__ = ("missing_blob_digests",) MISSING_BLOB_DIGESTS_FIELD_NUMBER: _ClassVar[int] missing_blob_digests: _containers.RepeatedCompositeFieldContainer[Digest] def __init__(self, missing_blob_digests: _Optional[_Iterable[_Union[Digest, _Mapping]]] = ...) -> None: ... class BatchUpdateBlobsRequest(_message.Message): __slots__ = ("instance_name", "requests", "digest_function") class Request(_message.Message): __slots__ = ("digest", "data", "compressor") DIGEST_FIELD_NUMBER: _ClassVar[int] DATA_FIELD_NUMBER: _ClassVar[int] COMPRESSOR_FIELD_NUMBER: _ClassVar[int] digest: Digest data: bytes compressor: Compressor.Value def __init__(self, digest: _Optional[_Union[Digest, _Mapping]] = ..., data: _Optional[bytes] = ..., compressor: _Optional[_Union[Compressor.Value, str]] = ...) -> None: ... INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] REQUESTS_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str requests: _containers.RepeatedCompositeFieldContainer[BatchUpdateBlobsRequest.Request] digest_function: DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., requests: _Optional[_Iterable[_Union[BatchUpdateBlobsRequest.Request, _Mapping]]] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ...) -> None: ... class BatchUpdateBlobsResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): __slots__ = ("digest", "status") DIGEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] digest: Digest status: _status_pb2.Status def __init__(self, digest: _Optional[_Union[Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[BatchUpdateBlobsResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[BatchUpdateBlobsResponse.Response, _Mapping]]] = ...) -> None: ... class BatchReadBlobsRequest(_message.Message): __slots__ = ("instance_name", "digests", "acceptable_compressors", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] DIGESTS_FIELD_NUMBER: _ClassVar[int] ACCEPTABLE_COMPRESSORS_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str digests: _containers.RepeatedCompositeFieldContainer[Digest] acceptable_compressors: _containers.RepeatedScalarFieldContainer[Compressor.Value] digest_function: DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., digests: _Optional[_Iterable[_Union[Digest, _Mapping]]] = ..., acceptable_compressors: _Optional[_Iterable[_Union[Compressor.Value, str]]] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ...) -> None: ... class BatchReadBlobsResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): __slots__ = ("digest", "data", "compressor", "status") DIGEST_FIELD_NUMBER: _ClassVar[int] DATA_FIELD_NUMBER: _ClassVar[int] COMPRESSOR_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] digest: Digest data: bytes compressor: Compressor.Value status: _status_pb2.Status def __init__(self, digest: _Optional[_Union[Digest, _Mapping]] = ..., data: _Optional[bytes] = ..., compressor: _Optional[_Union[Compressor.Value, str]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[BatchReadBlobsResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[BatchReadBlobsResponse.Response, _Mapping]]] = ...) -> None: ... class GetTreeRequest(_message.Message): __slots__ = ("instance_name", "root_digest", "page_size", "page_token", "digest_function") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_DIGEST_FIELD_NUMBER: _ClassVar[int] PAGE_SIZE_FIELD_NUMBER: _ClassVar[int] PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] instance_name: str root_digest: Digest page_size: int page_token: str digest_function: DigestFunction.Value def __init__(self, instance_name: _Optional[str] = ..., root_digest: _Optional[_Union[Digest, _Mapping]] = ..., page_size: _Optional[int] = ..., page_token: _Optional[str] = ..., digest_function: _Optional[_Union[DigestFunction.Value, str]] = ...) -> None: ... class GetTreeResponse(_message.Message): __slots__ = ("directories", "next_page_token") DIRECTORIES_FIELD_NUMBER: _ClassVar[int] NEXT_PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] directories: _containers.RepeatedCompositeFieldContainer[Directory] next_page_token: str def __init__(self, directories: _Optional[_Iterable[_Union[Directory, _Mapping]]] = ..., next_page_token: _Optional[str] = ...) -> None: ... class GetCapabilitiesRequest(_message.Message): __slots__ = ("instance_name",) INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] instance_name: str def __init__(self, instance_name: _Optional[str] = ...) -> None: ... class ServerCapabilities(_message.Message): __slots__ = ("cache_capabilities", "execution_capabilities", "deprecated_api_version", "low_api_version", "high_api_version") CACHE_CAPABILITIES_FIELD_NUMBER: _ClassVar[int] EXECUTION_CAPABILITIES_FIELD_NUMBER: _ClassVar[int] DEPRECATED_API_VERSION_FIELD_NUMBER: _ClassVar[int] LOW_API_VERSION_FIELD_NUMBER: _ClassVar[int] HIGH_API_VERSION_FIELD_NUMBER: _ClassVar[int] cache_capabilities: CacheCapabilities execution_capabilities: ExecutionCapabilities deprecated_api_version: _semver_pb2.SemVer low_api_version: _semver_pb2.SemVer high_api_version: _semver_pb2.SemVer def __init__(self, cache_capabilities: _Optional[_Union[CacheCapabilities, _Mapping]] = ..., execution_capabilities: _Optional[_Union[ExecutionCapabilities, _Mapping]] = ..., deprecated_api_version: _Optional[_Union[_semver_pb2.SemVer, _Mapping]] = ..., low_api_version: _Optional[_Union[_semver_pb2.SemVer, _Mapping]] = ..., high_api_version: _Optional[_Union[_semver_pb2.SemVer, _Mapping]] = ...) -> None: ... class DigestFunction(_message.Message): __slots__ = () class Value(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () UNKNOWN: _ClassVar[DigestFunction.Value] SHA256: _ClassVar[DigestFunction.Value] SHA1: _ClassVar[DigestFunction.Value] MD5: _ClassVar[DigestFunction.Value] VSO: _ClassVar[DigestFunction.Value] SHA384: _ClassVar[DigestFunction.Value] SHA512: _ClassVar[DigestFunction.Value] MURMUR3: _ClassVar[DigestFunction.Value] SHA256TREE: _ClassVar[DigestFunction.Value] BLAKE3: _ClassVar[DigestFunction.Value] UNKNOWN: DigestFunction.Value SHA256: DigestFunction.Value SHA1: DigestFunction.Value MD5: DigestFunction.Value VSO: DigestFunction.Value SHA384: DigestFunction.Value SHA512: DigestFunction.Value MURMUR3: DigestFunction.Value SHA256TREE: DigestFunction.Value BLAKE3: DigestFunction.Value def __init__(self) -> None: ... class ActionCacheUpdateCapabilities(_message.Message): __slots__ = ("update_enabled",) UPDATE_ENABLED_FIELD_NUMBER: _ClassVar[int] update_enabled: bool def __init__(self, update_enabled: bool = ...) -> None: ... class PriorityCapabilities(_message.Message): __slots__ = ("priorities",) class PriorityRange(_message.Message): __slots__ = ("min_priority", "max_priority") MIN_PRIORITY_FIELD_NUMBER: _ClassVar[int] MAX_PRIORITY_FIELD_NUMBER: _ClassVar[int] min_priority: int max_priority: int def __init__(self, min_priority: _Optional[int] = ..., max_priority: _Optional[int] = ...) -> None: ... PRIORITIES_FIELD_NUMBER: _ClassVar[int] priorities: _containers.RepeatedCompositeFieldContainer[PriorityCapabilities.PriorityRange] def __init__(self, priorities: _Optional[_Iterable[_Union[PriorityCapabilities.PriorityRange, _Mapping]]] = ...) -> None: ... class SymlinkAbsolutePathStrategy(_message.Message): __slots__ = () class Value(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () UNKNOWN: _ClassVar[SymlinkAbsolutePathStrategy.Value] DISALLOWED: _ClassVar[SymlinkAbsolutePathStrategy.Value] ALLOWED: _ClassVar[SymlinkAbsolutePathStrategy.Value] UNKNOWN: SymlinkAbsolutePathStrategy.Value DISALLOWED: SymlinkAbsolutePathStrategy.Value ALLOWED: SymlinkAbsolutePathStrategy.Value def __init__(self) -> None: ... class Compressor(_message.Message): __slots__ = () class Value(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () IDENTITY: _ClassVar[Compressor.Value] ZSTD: _ClassVar[Compressor.Value] DEFLATE: _ClassVar[Compressor.Value] BROTLI: _ClassVar[Compressor.Value] IDENTITY: Compressor.Value ZSTD: Compressor.Value DEFLATE: Compressor.Value BROTLI: Compressor.Value def __init__(self) -> None: ... class CacheCapabilities(_message.Message): __slots__ = ("digest_functions", "action_cache_update_capabilities", "cache_priority_capabilities", "max_batch_total_size_bytes", "symlink_absolute_path_strategy", "supported_compressors", "supported_batch_update_compressors") DIGEST_FUNCTIONS_FIELD_NUMBER: _ClassVar[int] ACTION_CACHE_UPDATE_CAPABILITIES_FIELD_NUMBER: _ClassVar[int] CACHE_PRIORITY_CAPABILITIES_FIELD_NUMBER: _ClassVar[int] MAX_BATCH_TOTAL_SIZE_BYTES_FIELD_NUMBER: _ClassVar[int] SYMLINK_ABSOLUTE_PATH_STRATEGY_FIELD_NUMBER: _ClassVar[int] SUPPORTED_COMPRESSORS_FIELD_NUMBER: _ClassVar[int] SUPPORTED_BATCH_UPDATE_COMPRESSORS_FIELD_NUMBER: _ClassVar[int] digest_functions: _containers.RepeatedScalarFieldContainer[DigestFunction.Value] action_cache_update_capabilities: ActionCacheUpdateCapabilities cache_priority_capabilities: PriorityCapabilities max_batch_total_size_bytes: int symlink_absolute_path_strategy: SymlinkAbsolutePathStrategy.Value supported_compressors: _containers.RepeatedScalarFieldContainer[Compressor.Value] supported_batch_update_compressors: _containers.RepeatedScalarFieldContainer[Compressor.Value] def __init__(self, digest_functions: _Optional[_Iterable[_Union[DigestFunction.Value, str]]] = ..., action_cache_update_capabilities: _Optional[_Union[ActionCacheUpdateCapabilities, _Mapping]] = ..., cache_priority_capabilities: _Optional[_Union[PriorityCapabilities, _Mapping]] = ..., max_batch_total_size_bytes: _Optional[int] = ..., symlink_absolute_path_strategy: _Optional[_Union[SymlinkAbsolutePathStrategy.Value, str]] = ..., supported_compressors: _Optional[_Iterable[_Union[Compressor.Value, str]]] = ..., supported_batch_update_compressors: _Optional[_Iterable[_Union[Compressor.Value, str]]] = ...) -> None: ... class ExecutionCapabilities(_message.Message): __slots__ = ("digest_function", "exec_enabled", "execution_priority_capabilities", "supported_node_properties", "digest_functions") DIGEST_FUNCTION_FIELD_NUMBER: _ClassVar[int] EXEC_ENABLED_FIELD_NUMBER: _ClassVar[int] EXECUTION_PRIORITY_CAPABILITIES_FIELD_NUMBER: _ClassVar[int] SUPPORTED_NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] DIGEST_FUNCTIONS_FIELD_NUMBER: _ClassVar[int] digest_function: DigestFunction.Value exec_enabled: bool execution_priority_capabilities: PriorityCapabilities supported_node_properties: _containers.RepeatedScalarFieldContainer[str] digest_functions: _containers.RepeatedScalarFieldContainer[DigestFunction.Value] def __init__(self, digest_function: _Optional[_Union[DigestFunction.Value, str]] = ..., exec_enabled: bool = ..., execution_priority_capabilities: _Optional[_Union[PriorityCapabilities, _Mapping]] = ..., supported_node_properties: _Optional[_Iterable[str]] = ..., digest_functions: _Optional[_Iterable[_Union[DigestFunction.Value, str]]] = ...) -> None: ... class ToolDetails(_message.Message): __slots__ = ("tool_name", "tool_version") TOOL_NAME_FIELD_NUMBER: _ClassVar[int] TOOL_VERSION_FIELD_NUMBER: _ClassVar[int] tool_name: str tool_version: str def __init__(self, tool_name: _Optional[str] = ..., tool_version: _Optional[str] = ...) -> None: ... class RequestMetadata(_message.Message): __slots__ = ("tool_details", "action_id", "tool_invocation_id", "correlated_invocations_id", "action_mnemonic", "target_id", "configuration_id") TOOL_DETAILS_FIELD_NUMBER: _ClassVar[int] ACTION_ID_FIELD_NUMBER: _ClassVar[int] TOOL_INVOCATION_ID_FIELD_NUMBER: _ClassVar[int] CORRELATED_INVOCATIONS_ID_FIELD_NUMBER: _ClassVar[int] ACTION_MNEMONIC_FIELD_NUMBER: _ClassVar[int] TARGET_ID_FIELD_NUMBER: _ClassVar[int] CONFIGURATION_ID_FIELD_NUMBER: _ClassVar[int] tool_details: ToolDetails action_id: str tool_invocation_id: str correlated_invocations_id: str action_mnemonic: str target_id: str configuration_id: str def __init__(self, tool_details: _Optional[_Union[ToolDetails, _Mapping]] = ..., action_id: _Optional[str] = ..., tool_invocation_id: _Optional[str] = ..., correlated_invocations_id: _Optional[str] = ..., action_mnemonic: _Optional[str] = ..., target_id: _Optional[str] = ..., configuration_id: _Optional[str] = ...) -> None: ... remote_execution_pb2_grpc.py000066400000000000000000001722621514607367700360730ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/remote/execution/v2# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 from buildstream._protos.google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in build/bazel/remote/execution/v2/remote_execution_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) class ExecutionStub(object): """The Remote Execution API is used to execute an [Action][build.bazel.remote.execution.v2.Action] on the remote workers. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.Execute = channel.unary_stream( '/build.bazel.remote.execution.v2.Execution/Execute', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ExecuteRequest.SerializeToString, response_deserializer=google_dot_longrunning_dot_operations__pb2.Operation.FromString, _registered_method=True) self.WaitExecution = channel.unary_stream( '/build.bazel.remote.execution.v2.Execution/WaitExecution', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.WaitExecutionRequest.SerializeToString, response_deserializer=google_dot_longrunning_dot_operations__pb2.Operation.FromString, _registered_method=True) class ExecutionServicer(object): """The Remote Execution API is used to execute an [Action][build.bazel.remote.execution.v2.Action] on the remote workers. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def Execute(self, request, context): """Execute an action remotely. In order to execute an action, the client must first upload all of the inputs, the [Command][build.bazel.remote.execution.v2.Command] to run, and the [Action][build.bazel.remote.execution.v2.Action] into the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. It then calls `Execute` with an `action_digest` referring to them. The server will run the action and eventually return the result. The input `Action`'s fields MUST meet the various canonicalization requirements specified in the documentation for their types so that it has the same digest as other logically equivalent `Action`s. The server MAY enforce the requirements and return errors if a non-canonical input is received. It MAY also proceed without verifying some or all of the requirements, such as for performance reasons. If the server does not verify the requirement, then it will treat the `Action` as distinct from another logically equivalent action if they hash differently. Returns a stream of [google.longrunning.Operation][google.longrunning.Operation] messages describing the resulting execution, with eventual `response` [ExecuteResponse][build.bazel.remote.execution.v2.ExecuteResponse]. The `metadata` on the operation is of type [ExecuteOperationMetadata][build.bazel.remote.execution.v2.ExecuteOperationMetadata]. If the client remains connected after the first response is returned after the server, then updates are streamed as if the client had called [WaitExecution][build.bazel.remote.execution.v2.Execution.WaitExecution] until the execution completes or the request reaches an error. The operation can also be queried using [Operations API][google.longrunning.Operations.GetOperation]. The server NEED NOT implement other methods or functionality of the Operations API. Errors discovered during creation of the `Operation` will be reported as gRPC Status errors, while errors that occurred while running the action will be reported in the `status` field of the `ExecuteResponse`. The server MUST NOT set the `error` field of the `Operation` proto. The possible errors include: * `INVALID_ARGUMENT`: One or more arguments are invalid. * `FAILED_PRECONDITION`: One or more errors occurred in setting up the action requested, such as a missing input or command or no worker being available. The client may be able to fix the errors and retry. * `RESOURCE_EXHAUSTED`: There is insufficient quota of some resource to run the action. * `UNAVAILABLE`: Due to a transient condition, such as all workers being occupied (and the server does not support a queue), the action could not be started. The client should retry. * `INTERNAL`: An internal error occurred in the execution engine or the worker. * `DEADLINE_EXCEEDED`: The execution timed out. * `CANCELLED`: The operation was cancelled by the client. This status is only possible if the server implements the Operations API CancelOperation method, and it was called for the current execution. In the case of a missing input or command, the server SHOULD additionally send a [PreconditionFailure][google.rpc.PreconditionFailure] error detail where, for each requested blob not present in the CAS, there is a `Violation` with a `type` of `MISSING` and a `subject` of `"blobs/{digest_function/}{hash}/{size}"` indicating the digest of the missing blob. The `subject` is formatted the same way as the `resource_name` provided to [ByteStream.Read][google.bytestream.ByteStream.Read], with the leading instance name omitted. `digest_function` MUST thus be omitted if its value is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, or VSO. The server does not need to guarantee that a call to this method leads to at most one execution of the action. The server MAY execute the action multiple times, potentially in parallel. These redundant executions MAY continue to run, even if the operation is completed. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def WaitExecution(self, request, context): """Wait for an execution operation to complete. When the client initially makes the request, the server immediately responds with the current status of the execution. The server will leave the request stream open until the operation completes, and then respond with the completed operation. The server MAY choose to stream additional updates as execution progresses, such as to provide an update as to the state of the execution. In addition to the cases describe for Execute, the WaitExecution method may fail as follows: * `NOT_FOUND`: The operation no longer exists due to any of a transient condition, an unknown operation name, or if the server implements the Operations API DeleteOperation method and it was called for the current execution. The client should call `Execute` to retry. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_ExecutionServicer_to_server(servicer, server): rpc_method_handlers = { 'Execute': grpc.unary_stream_rpc_method_handler( servicer.Execute, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ExecuteRequest.FromString, response_serializer=google_dot_longrunning_dot_operations__pb2.Operation.SerializeToString, ), 'WaitExecution': grpc.unary_stream_rpc_method_handler( servicer.WaitExecution, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.WaitExecutionRequest.FromString, response_serializer=google_dot_longrunning_dot_operations__pb2.Operation.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.bazel.remote.execution.v2.Execution', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.bazel.remote.execution.v2.Execution', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class Execution(object): """The Remote Execution API is used to execute an [Action][build.bazel.remote.execution.v2.Action] on the remote workers. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ @staticmethod def Execute(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_stream( request, target, '/build.bazel.remote.execution.v2.Execution/Execute', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ExecuteRequest.SerializeToString, google_dot_longrunning_dot_operations__pb2.Operation.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def WaitExecution(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_stream( request, target, '/build.bazel.remote.execution.v2.Execution/WaitExecution', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.WaitExecutionRequest.SerializeToString, google_dot_longrunning_dot_operations__pb2.Operation.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) class ActionCacheStub(object): """The action cache API is used to query whether a given action has already been performed and, if so, retrieve its result. Unlike the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage], which addresses blobs by their own content, the action cache addresses the [ActionResult][build.bazel.remote.execution.v2.ActionResult] by a digest of the encoded [Action][build.bazel.remote.execution.v2.Action] which produced them. The lifetime of entries in the action cache is implementation-specific, but the server SHOULD assume that more recently used entries are more likely to be used again. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.GetActionResult = channel.unary_unary( '/build.bazel.remote.execution.v2.ActionCache/GetActionResult', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetActionResultRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ActionResult.FromString, _registered_method=True) self.UpdateActionResult = channel.unary_unary( '/build.bazel.remote.execution.v2.ActionCache/UpdateActionResult', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.UpdateActionResultRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ActionResult.FromString, _registered_method=True) class ActionCacheServicer(object): """The action cache API is used to query whether a given action has already been performed and, if so, retrieve its result. Unlike the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage], which addresses blobs by their own content, the action cache addresses the [ActionResult][build.bazel.remote.execution.v2.ActionResult] by a digest of the encoded [Action][build.bazel.remote.execution.v2.Action] which produced them. The lifetime of entries in the action cache is implementation-specific, but the server SHOULD assume that more recently used entries are more likely to be used again. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def GetActionResult(self, request, context): """Retrieve a cached execution result. Implementations SHOULD ensure that any blobs referenced from the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] are available at the time of returning the [ActionResult][build.bazel.remote.execution.v2.ActionResult] and will be for some period of time afterwards. The lifetimes of the referenced blobs SHOULD be increased if necessary and applicable. Errors: * `NOT_FOUND`: The requested `ActionResult` is not in the cache. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def UpdateActionResult(self, request, context): """Upload a new execution result. In order to allow the server to perform access control based on the type of action, and to assist with client debugging, the client MUST first upload the [Action][build.bazel.remote.execution.v2.Execution] that produced the result, along with its [Command][build.bazel.remote.execution.v2.Command], into the `ContentAddressableStorage`. Server implementations MAY modify the `UpdateActionResultRequest.action_result` and return an equivalent value. Errors: * `INVALID_ARGUMENT`: One or more arguments are invalid. * `FAILED_PRECONDITION`: One or more errors occurred in updating the action result, such as a missing command or action. * `RESOURCE_EXHAUSTED`: There is insufficient storage space to add the entry to the cache. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_ActionCacheServicer_to_server(servicer, server): rpc_method_handlers = { 'GetActionResult': grpc.unary_unary_rpc_method_handler( servicer.GetActionResult, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetActionResultRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ActionResult.SerializeToString, ), 'UpdateActionResult': grpc.unary_unary_rpc_method_handler( servicer.UpdateActionResult, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.UpdateActionResultRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ActionResult.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.bazel.remote.execution.v2.ActionCache', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.bazel.remote.execution.v2.ActionCache', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class ActionCache(object): """The action cache API is used to query whether a given action has already been performed and, if so, retrieve its result. Unlike the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage], which addresses blobs by their own content, the action cache addresses the [ActionResult][build.bazel.remote.execution.v2.ActionResult] by a digest of the encoded [Action][build.bazel.remote.execution.v2.Action] which produced them. The lifetime of entries in the action cache is implementation-specific, but the server SHOULD assume that more recently used entries are more likely to be used again. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ @staticmethod def GetActionResult(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.execution.v2.ActionCache/GetActionResult', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetActionResultRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ActionResult.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def UpdateActionResult(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.execution.v2.ActionCache/UpdateActionResult', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.UpdateActionResultRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ActionResult.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) class ContentAddressableStorageStub(object): """The CAS (content-addressable storage) is used to store the inputs to and outputs from the execution service. Each piece of content is addressed by the digest of its binary data. Most of the binary data stored in the CAS is opaque to the execution engine, and is only used as a communication medium. In order to build an [Action][build.bazel.remote.execution.v2.Action], however, the client will need to also upload the [Command][build.bazel.remote.execution.v2.Command] and input root [Directory][build.bazel.remote.execution.v2.Directory] for the Action. The Command and Directory messages must be marshalled to wire format and then uploaded under the hash as with any other piece of content. In practice, the input root directory is likely to refer to other Directories in its hierarchy, which must also each be uploaded on their own. For small file uploads the client should group them together and call [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]. For large uploads, the client must use the [Write method][google.bytestream.ByteStream.Write] of the ByteStream API. For uncompressed data, The `WriteRequest.resource_name` is of the following form: `{instance_name}/uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}` Where: * `instance_name` is an identifier used to distinguish between the various instances on the server. Syntax and semantics of this field are defined by the server; Clients must not make any assumptions about it (e.g., whether it spans multiple path segments or not). If it is the empty path, the leading slash is omitted, so that the `resource_name` becomes `uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}`. To simplify parsing, a path segment cannot equal any of the following keywords: `blobs`, `uploads`, `actions`, `actionResults`, `operations`, `capabilities` or `compressed-blobs`. * `uuid` is a version 4 UUID generated by the client, used to avoid collisions between concurrent uploads of the same data. Clients MAY reuse the same `uuid` for uploading different blobs. * `digest_function` is a lowercase string form of a `DigestFunction.Value` enum, indicating which digest function was used to compute `hash`. If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, or VSO, this component MUST be omitted. In that case the server SHOULD infer the digest function using the length of the `hash` and the digest functions announced in the server's capabilities. * `hash` and `size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being uploaded. * `optional_metadata` is implementation specific data, which clients MAY omit. Servers MAY ignore this metadata. Data can alternatively be uploaded in compressed form, with the following `WriteRequest.resource_name` form: `{instance_name}/uploads/{uuid}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}{/optional_metadata}` Where: * `instance_name`, `uuid`, `digest_function` and `optional_metadata` are defined as above. * `compressor` is a lowercase string form of a `Compressor.Value` enum other than `identity`, which is supported by the server and advertised in [CacheCapabilities.supported_compressor][build.bazel.remote.execution.v2.CacheCapabilities.supported_compressor]. * `uncompressed_hash` and `uncompressed_size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being uploaded, once uncompressed. Servers MUST verify that these match the uploaded data once uncompressed, and MUST return an `INVALID_ARGUMENT` error in the case of mismatch. Note that when writing compressed blobs, the `WriteRequest.write_offset` in the initial request in a stream refers to the offset in the uncompressed form of the blob. In subsequent requests, `WriteRequest.write_offset` MUST be the sum of the first request's 'WriteRequest.write_offset' and the total size of all the compressed data bundles in the previous requests. Note that this mixes an uncompressed offset with a compressed byte length, which is nonsensical, but it is done to fit the semantics of the existing ByteStream protocol. Uploads of the same data MAY occur concurrently in any form, compressed or uncompressed. Clients SHOULD NOT use gRPC-level compression for ByteStream API `Write` calls of compressed blobs, since this would compress already-compressed data. When attempting an upload, if another client has already completed the upload (which may occur in the middle of a single upload if another client uploads the same blob concurrently), the request will terminate immediately without error, and with a response whose `committed_size` is the value `-1` if this is a compressed upload, or with the full size of the uploaded file if this is an uncompressed upload (regardless of how much data was transmitted by the client). If the client completes the upload but the [Digest][build.bazel.remote.execution.v2.Digest] does not match, an `INVALID_ARGUMENT` error will be returned. In either case, the client should not attempt to retry the upload. Small downloads can be grouped and requested in a batch via [BatchReadBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchReadBlobs]. For large downloads, the client must use the [Read method][google.bytestream.ByteStream.Read] of the ByteStream API. For uncompressed data, The `ReadRequest.resource_name` is of the following form: `{instance_name}/blobs/{digest_function/}{hash}/{size}` Where `instance_name`, `digest_function`, `hash` and `size` are defined as for uploads. Data can alternatively be downloaded in compressed form, with the following `ReadRequest.resource_name` form: `{instance_name}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}` Where: * `instance_name`, `compressor` and `digest_function` are defined as for uploads. * `uncompressed_hash` and `uncompressed_size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being downloaded, once uncompressed. Clients MUST verify that these match the downloaded data once uncompressed, and take appropriate steps in the case of failure such as retrying a limited number of times or surfacing an error to the user. When downloading compressed blobs: * `ReadRequest.read_offset` refers to the offset in the uncompressed form of the blob. * Servers MUST return `INVALID_ARGUMENT` if `ReadRequest.read_limit` is non-zero. * Servers MAY use any compression level they choose, including different levels for different blobs (e.g. choosing a level designed for maximum speed for data known to be incompressible). * Clients SHOULD NOT use gRPC-level compression, since this would compress already-compressed data. Servers MUST be able to provide data for all recently advertised blobs in each of the compression formats that the server supports, as well as in uncompressed form. The lifetime of entries in the CAS is implementation specific, but it SHOULD be long enough to allow for newly-added and recently looked-up entries to be used in subsequent calls (e.g. to [Execute][build.bazel.remote.execution.v2.Execution.Execute]). Servers MUST behave as though empty blobs are always available, even if they have not been uploaded. Clients MAY optimize away the uploading or downloading of empty blobs. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.FindMissingBlobs = channel.unary_unary( '/build.bazel.remote.execution.v2.ContentAddressableStorage/FindMissingBlobs', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.FindMissingBlobsRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.FindMissingBlobsResponse.FromString, _registered_method=True) self.BatchUpdateBlobs = channel.unary_unary( '/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchUpdateBlobs', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchUpdateBlobsRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchUpdateBlobsResponse.FromString, _registered_method=True) self.BatchReadBlobs = channel.unary_unary( '/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchReadBlobs', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchReadBlobsRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchReadBlobsResponse.FromString, _registered_method=True) self.GetTree = channel.unary_stream( '/build.bazel.remote.execution.v2.ContentAddressableStorage/GetTree', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetTreeRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetTreeResponse.FromString, _registered_method=True) class ContentAddressableStorageServicer(object): """The CAS (content-addressable storage) is used to store the inputs to and outputs from the execution service. Each piece of content is addressed by the digest of its binary data. Most of the binary data stored in the CAS is opaque to the execution engine, and is only used as a communication medium. In order to build an [Action][build.bazel.remote.execution.v2.Action], however, the client will need to also upload the [Command][build.bazel.remote.execution.v2.Command] and input root [Directory][build.bazel.remote.execution.v2.Directory] for the Action. The Command and Directory messages must be marshalled to wire format and then uploaded under the hash as with any other piece of content. In practice, the input root directory is likely to refer to other Directories in its hierarchy, which must also each be uploaded on their own. For small file uploads the client should group them together and call [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]. For large uploads, the client must use the [Write method][google.bytestream.ByteStream.Write] of the ByteStream API. For uncompressed data, The `WriteRequest.resource_name` is of the following form: `{instance_name}/uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}` Where: * `instance_name` is an identifier used to distinguish between the various instances on the server. Syntax and semantics of this field are defined by the server; Clients must not make any assumptions about it (e.g., whether it spans multiple path segments or not). If it is the empty path, the leading slash is omitted, so that the `resource_name` becomes `uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}`. To simplify parsing, a path segment cannot equal any of the following keywords: `blobs`, `uploads`, `actions`, `actionResults`, `operations`, `capabilities` or `compressed-blobs`. * `uuid` is a version 4 UUID generated by the client, used to avoid collisions between concurrent uploads of the same data. Clients MAY reuse the same `uuid` for uploading different blobs. * `digest_function` is a lowercase string form of a `DigestFunction.Value` enum, indicating which digest function was used to compute `hash`. If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, or VSO, this component MUST be omitted. In that case the server SHOULD infer the digest function using the length of the `hash` and the digest functions announced in the server's capabilities. * `hash` and `size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being uploaded. * `optional_metadata` is implementation specific data, which clients MAY omit. Servers MAY ignore this metadata. Data can alternatively be uploaded in compressed form, with the following `WriteRequest.resource_name` form: `{instance_name}/uploads/{uuid}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}{/optional_metadata}` Where: * `instance_name`, `uuid`, `digest_function` and `optional_metadata` are defined as above. * `compressor` is a lowercase string form of a `Compressor.Value` enum other than `identity`, which is supported by the server and advertised in [CacheCapabilities.supported_compressor][build.bazel.remote.execution.v2.CacheCapabilities.supported_compressor]. * `uncompressed_hash` and `uncompressed_size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being uploaded, once uncompressed. Servers MUST verify that these match the uploaded data once uncompressed, and MUST return an `INVALID_ARGUMENT` error in the case of mismatch. Note that when writing compressed blobs, the `WriteRequest.write_offset` in the initial request in a stream refers to the offset in the uncompressed form of the blob. In subsequent requests, `WriteRequest.write_offset` MUST be the sum of the first request's 'WriteRequest.write_offset' and the total size of all the compressed data bundles in the previous requests. Note that this mixes an uncompressed offset with a compressed byte length, which is nonsensical, but it is done to fit the semantics of the existing ByteStream protocol. Uploads of the same data MAY occur concurrently in any form, compressed or uncompressed. Clients SHOULD NOT use gRPC-level compression for ByteStream API `Write` calls of compressed blobs, since this would compress already-compressed data. When attempting an upload, if another client has already completed the upload (which may occur in the middle of a single upload if another client uploads the same blob concurrently), the request will terminate immediately without error, and with a response whose `committed_size` is the value `-1` if this is a compressed upload, or with the full size of the uploaded file if this is an uncompressed upload (regardless of how much data was transmitted by the client). If the client completes the upload but the [Digest][build.bazel.remote.execution.v2.Digest] does not match, an `INVALID_ARGUMENT` error will be returned. In either case, the client should not attempt to retry the upload. Small downloads can be grouped and requested in a batch via [BatchReadBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchReadBlobs]. For large downloads, the client must use the [Read method][google.bytestream.ByteStream.Read] of the ByteStream API. For uncompressed data, The `ReadRequest.resource_name` is of the following form: `{instance_name}/blobs/{digest_function/}{hash}/{size}` Where `instance_name`, `digest_function`, `hash` and `size` are defined as for uploads. Data can alternatively be downloaded in compressed form, with the following `ReadRequest.resource_name` form: `{instance_name}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}` Where: * `instance_name`, `compressor` and `digest_function` are defined as for uploads. * `uncompressed_hash` and `uncompressed_size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being downloaded, once uncompressed. Clients MUST verify that these match the downloaded data once uncompressed, and take appropriate steps in the case of failure such as retrying a limited number of times or surfacing an error to the user. When downloading compressed blobs: * `ReadRequest.read_offset` refers to the offset in the uncompressed form of the blob. * Servers MUST return `INVALID_ARGUMENT` if `ReadRequest.read_limit` is non-zero. * Servers MAY use any compression level they choose, including different levels for different blobs (e.g. choosing a level designed for maximum speed for data known to be incompressible). * Clients SHOULD NOT use gRPC-level compression, since this would compress already-compressed data. Servers MUST be able to provide data for all recently advertised blobs in each of the compression formats that the server supports, as well as in uncompressed form. The lifetime of entries in the CAS is implementation specific, but it SHOULD be long enough to allow for newly-added and recently looked-up entries to be used in subsequent calls (e.g. to [Execute][build.bazel.remote.execution.v2.Execution.Execute]). Servers MUST behave as though empty blobs are always available, even if they have not been uploaded. Clients MAY optimize away the uploading or downloading of empty blobs. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ def FindMissingBlobs(self, request, context): """Determine if blobs are present in the CAS. Clients can use this API before uploading blobs to determine which ones are already present in the CAS and do not need to be uploaded again. Servers SHOULD increase the lifetimes of the referenced blobs if necessary and applicable. There are no method-specific errors. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def BatchUpdateBlobs(self, request, context): """Upload many blobs at once. The server may enforce a limit of the combined total size of blobs to be uploaded using this API. This limit may be obtained using the [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. Requests exceeding the limit should either be split into smaller chunks or uploaded using the [ByteStream API][google.bytestream.ByteStream], as appropriate. This request is equivalent to calling a Bytestream `Write` request on each individual blob, in parallel. The requests may succeed or fail independently. Errors: * `INVALID_ARGUMENT`: The client attempted to upload more than the server supported limit. Individual requests may return the following errors, additionally: * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the blob. * `INVALID_ARGUMENT`: The [Digest][build.bazel.remote.execution.v2.Digest] does not match the provided data. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def BatchReadBlobs(self, request, context): """Download many blobs at once. The server may enforce a limit of the combined total size of blobs to be downloaded using this API. This limit may be obtained using the [Capabilities][build.bazel.remote.execution.v2.Capabilities] API. Requests exceeding the limit should either be split into smaller chunks or downloaded using the [ByteStream API][google.bytestream.ByteStream], as appropriate. This request is equivalent to calling a Bytestream `Read` request on each individual blob, in parallel. The requests may succeed or fail independently. Errors: * `INVALID_ARGUMENT`: The client attempted to read more than the server supported limit. Every error on individual read will be returned in the corresponding digest status. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetTree(self, request, context): """Fetch the entire directory tree rooted at a node. This request must be targeted at a [Directory][build.bazel.remote.execution.v2.Directory] stored in the [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage] (CAS). The server will enumerate the `Directory` tree recursively and return every node descended from the root. The GetTreeRequest.page_token parameter can be used to skip ahead in the stream (e.g. when retrying a partially completed and aborted request), by setting it to a value taken from GetTreeResponse.next_page_token of the last successfully processed GetTreeResponse). The exact traversal order is unspecified and, unless retrieving subsequent pages from an earlier request, is not guaranteed to be stable across multiple invocations of `GetTree`. If part of the tree is missing from the CAS, the server will return the portion present and omit the rest. Errors: * `NOT_FOUND`: The requested tree root is not present in the CAS. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_ContentAddressableStorageServicer_to_server(servicer, server): rpc_method_handlers = { 'FindMissingBlobs': grpc.unary_unary_rpc_method_handler( servicer.FindMissingBlobs, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.FindMissingBlobsRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.FindMissingBlobsResponse.SerializeToString, ), 'BatchUpdateBlobs': grpc.unary_unary_rpc_method_handler( servicer.BatchUpdateBlobs, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchUpdateBlobsRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchUpdateBlobsResponse.SerializeToString, ), 'BatchReadBlobs': grpc.unary_unary_rpc_method_handler( servicer.BatchReadBlobs, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchReadBlobsRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchReadBlobsResponse.SerializeToString, ), 'GetTree': grpc.unary_stream_rpc_method_handler( servicer.GetTree, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetTreeRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetTreeResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.bazel.remote.execution.v2.ContentAddressableStorage', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.bazel.remote.execution.v2.ContentAddressableStorage', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class ContentAddressableStorage(object): """The CAS (content-addressable storage) is used to store the inputs to and outputs from the execution service. Each piece of content is addressed by the digest of its binary data. Most of the binary data stored in the CAS is opaque to the execution engine, and is only used as a communication medium. In order to build an [Action][build.bazel.remote.execution.v2.Action], however, the client will need to also upload the [Command][build.bazel.remote.execution.v2.Command] and input root [Directory][build.bazel.remote.execution.v2.Directory] for the Action. The Command and Directory messages must be marshalled to wire format and then uploaded under the hash as with any other piece of content. In practice, the input root directory is likely to refer to other Directories in its hierarchy, which must also each be uploaded on their own. For small file uploads the client should group them together and call [BatchUpdateBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchUpdateBlobs]. For large uploads, the client must use the [Write method][google.bytestream.ByteStream.Write] of the ByteStream API. For uncompressed data, The `WriteRequest.resource_name` is of the following form: `{instance_name}/uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}` Where: * `instance_name` is an identifier used to distinguish between the various instances on the server. Syntax and semantics of this field are defined by the server; Clients must not make any assumptions about it (e.g., whether it spans multiple path segments or not). If it is the empty path, the leading slash is omitted, so that the `resource_name` becomes `uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}`. To simplify parsing, a path segment cannot equal any of the following keywords: `blobs`, `uploads`, `actions`, `actionResults`, `operations`, `capabilities` or `compressed-blobs`. * `uuid` is a version 4 UUID generated by the client, used to avoid collisions between concurrent uploads of the same data. Clients MAY reuse the same `uuid` for uploading different blobs. * `digest_function` is a lowercase string form of a `DigestFunction.Value` enum, indicating which digest function was used to compute `hash`. If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, or VSO, this component MUST be omitted. In that case the server SHOULD infer the digest function using the length of the `hash` and the digest functions announced in the server's capabilities. * `hash` and `size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being uploaded. * `optional_metadata` is implementation specific data, which clients MAY omit. Servers MAY ignore this metadata. Data can alternatively be uploaded in compressed form, with the following `WriteRequest.resource_name` form: `{instance_name}/uploads/{uuid}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}{/optional_metadata}` Where: * `instance_name`, `uuid`, `digest_function` and `optional_metadata` are defined as above. * `compressor` is a lowercase string form of a `Compressor.Value` enum other than `identity`, which is supported by the server and advertised in [CacheCapabilities.supported_compressor][build.bazel.remote.execution.v2.CacheCapabilities.supported_compressor]. * `uncompressed_hash` and `uncompressed_size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being uploaded, once uncompressed. Servers MUST verify that these match the uploaded data once uncompressed, and MUST return an `INVALID_ARGUMENT` error in the case of mismatch. Note that when writing compressed blobs, the `WriteRequest.write_offset` in the initial request in a stream refers to the offset in the uncompressed form of the blob. In subsequent requests, `WriteRequest.write_offset` MUST be the sum of the first request's 'WriteRequest.write_offset' and the total size of all the compressed data bundles in the previous requests. Note that this mixes an uncompressed offset with a compressed byte length, which is nonsensical, but it is done to fit the semantics of the existing ByteStream protocol. Uploads of the same data MAY occur concurrently in any form, compressed or uncompressed. Clients SHOULD NOT use gRPC-level compression for ByteStream API `Write` calls of compressed blobs, since this would compress already-compressed data. When attempting an upload, if another client has already completed the upload (which may occur in the middle of a single upload if another client uploads the same blob concurrently), the request will terminate immediately without error, and with a response whose `committed_size` is the value `-1` if this is a compressed upload, or with the full size of the uploaded file if this is an uncompressed upload (regardless of how much data was transmitted by the client). If the client completes the upload but the [Digest][build.bazel.remote.execution.v2.Digest] does not match, an `INVALID_ARGUMENT` error will be returned. In either case, the client should not attempt to retry the upload. Small downloads can be grouped and requested in a batch via [BatchReadBlobs][build.bazel.remote.execution.v2.ContentAddressableStorage.BatchReadBlobs]. For large downloads, the client must use the [Read method][google.bytestream.ByteStream.Read] of the ByteStream API. For uncompressed data, The `ReadRequest.resource_name` is of the following form: `{instance_name}/blobs/{digest_function/}{hash}/{size}` Where `instance_name`, `digest_function`, `hash` and `size` are defined as for uploads. Data can alternatively be downloaded in compressed form, with the following `ReadRequest.resource_name` form: `{instance_name}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}` Where: * `instance_name`, `compressor` and `digest_function` are defined as for uploads. * `uncompressed_hash` and `uncompressed_size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] of the data being downloaded, once uncompressed. Clients MUST verify that these match the downloaded data once uncompressed, and take appropriate steps in the case of failure such as retrying a limited number of times or surfacing an error to the user. When downloading compressed blobs: * `ReadRequest.read_offset` refers to the offset in the uncompressed form of the blob. * Servers MUST return `INVALID_ARGUMENT` if `ReadRequest.read_limit` is non-zero. * Servers MAY use any compression level they choose, including different levels for different blobs (e.g. choosing a level designed for maximum speed for data known to be incompressible). * Clients SHOULD NOT use gRPC-level compression, since this would compress already-compressed data. Servers MUST be able to provide data for all recently advertised blobs in each of the compression formats that the server supports, as well as in uncompressed form. The lifetime of entries in the CAS is implementation specific, but it SHOULD be long enough to allow for newly-added and recently looked-up entries to be used in subsequent calls (e.g. to [Execute][build.bazel.remote.execution.v2.Execution.Execute]). Servers MUST behave as though empty blobs are always available, even if they have not been uploaded. Clients MAY optimize away the uploading or downloading of empty blobs. As with other services in the Remote Execution API, any call may return an error with a [RetryInfo][google.rpc.RetryInfo] error detail providing information about when the client should retry the request; clients SHOULD respect the information provided. """ @staticmethod def FindMissingBlobs(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.execution.v2.ContentAddressableStorage/FindMissingBlobs', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.FindMissingBlobsRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.FindMissingBlobsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def BatchUpdateBlobs(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchUpdateBlobs', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchUpdateBlobsRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchUpdateBlobsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def BatchReadBlobs(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.execution.v2.ContentAddressableStorage/BatchReadBlobs', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchReadBlobsRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.BatchReadBlobsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def GetTree(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_stream( request, target, '/build.bazel.remote.execution.v2.ContentAddressableStorage/GetTree', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetTreeRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetTreeResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) class CapabilitiesStub(object): """The Capabilities service may be used by remote execution clients to query various server properties, in order to self-configure or return meaningful error messages. The query may include a particular `instance_name`, in which case the values returned will pertain to that instance. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.GetCapabilities = channel.unary_unary( '/build.bazel.remote.execution.v2.Capabilities/GetCapabilities', request_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetCapabilitiesRequest.SerializeToString, response_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ServerCapabilities.FromString, _registered_method=True) class CapabilitiesServicer(object): """The Capabilities service may be used by remote execution clients to query various server properties, in order to self-configure or return meaningful error messages. The query may include a particular `instance_name`, in which case the values returned will pertain to that instance. """ def GetCapabilities(self, request, context): """GetCapabilities returns the server capabilities configuration of the remote endpoint. Only the capabilities of the services supported by the endpoint will be returned: * Execution + CAS + Action Cache endpoints should return both CacheCapabilities and ExecutionCapabilities. * Execution only endpoints should return ExecutionCapabilities. * CAS + Action Cache only endpoints should return CacheCapabilities. There are no method-specific errors. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_CapabilitiesServicer_to_server(servicer, server): rpc_method_handlers = { 'GetCapabilities': grpc.unary_unary_rpc_method_handler( servicer.GetCapabilities, request_deserializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetCapabilitiesRequest.FromString, response_serializer=build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ServerCapabilities.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.bazel.remote.execution.v2.Capabilities', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.bazel.remote.execution.v2.Capabilities', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class Capabilities(object): """The Capabilities service may be used by remote execution clients to query various server properties, in order to self-configure or return meaningful error messages. The query may include a particular `instance_name`, in which case the values returned will pertain to that instance. """ @staticmethod def GetCapabilities(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.bazel.remote.execution.v2.Capabilities/GetCapabilities', build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.GetCapabilitiesRequest.SerializeToString, build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2.ServerCapabilities.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/semver/000077500000000000000000000000001514607367700261065ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/semver/__init__.py000066400000000000000000000000001514607367700302050ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/semver/semver.proto000066400000000000000000000013521514607367700304750ustar00rootroot00000000000000// Copyright 2018 The Bazel Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package build.bazel.semver; message SemVer { int32 major = 1; int32 minor = 2; int32 patch = 3; string prerelease = 4; } apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/semver/semver_pb2.py000066400000000000000000000026471514607367700305350ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: build/bazel/semver/semver.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'build/bazel/semver/semver.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x62uild/bazel/semver/semver.proto\x12\x12\x62uild.bazel.semver\"I\n\x06SemVer\x12\r\n\x05major\x18\x01 \x01(\x05\x12\r\n\x05minor\x18\x02 \x01(\x05\x12\r\n\x05patch\x18\x03 \x01(\x05\x12\x12\n\nprerelease\x18\x04 \x01(\tb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'build.bazel.semver.semver_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_SEMVER']._serialized_start=55 _globals['_SEMVER']._serialized_end=128 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/semver/semver_pb2.pyi000066400000000000000000000012501514607367700306730ustar00rootroot00000000000000from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Optional as _Optional DESCRIPTOR: _descriptor.FileDescriptor class SemVer(_message.Message): __slots__ = ("major", "minor", "patch", "prerelease") MAJOR_FIELD_NUMBER: _ClassVar[int] MINOR_FIELD_NUMBER: _ClassVar[int] PATCH_FIELD_NUMBER: _ClassVar[int] PRERELEASE_FIELD_NUMBER: _ClassVar[int] major: int minor: int patch: int prerelease: str def __init__(self, major: _Optional[int] = ..., minor: _Optional[int] = ..., patch: _Optional[int] = ..., prerelease: _Optional[str] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/build/bazel/semver/semver_pb2_grpc.py000066400000000000000000000016121514607367700315370ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in build/bazel/semver/semver_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_protos/build/buildgrid/000077500000000000000000000000001514607367700254555ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/buildgrid/__init__.py000066400000000000000000000000001514607367700275540ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/build/buildgrid/local_cas.proto000066400000000000000000000674741514607367700305040ustar00rootroot00000000000000// Copyright (C) 2018-2019 Bloomberg LP // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package build.buildgrid; import "build/bazel/remote/execution/v2/remote_execution.proto"; import "google/rpc/status.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; service LocalContentAddressableStorage { // Retrieves the configuration details for the local CAS server. // This RPC allows clients to obtain information such as the hostname, // user ID, and storage root of the local CAS server. rpc GetLocalServerDetails(GetLocalServerDetailsRequest) returns (LocalServerDetails) {} // Fetch blobs from a remote CAS to the local cache. // // This request is equivalent to ByteStream `Read` or `BatchReadBlobs` // requests, storing the downloaded blobs in the local cache. // // Requested blobs that failed to be downloaded will be listed in the // response. // // Errors: // * `INVALID_ARGUMENT`: The client attempted to download more than the // server supported limit. // // Individual requests may return the following error, additionally: // * `NOT_FOUND`: The requested blob is not present in the remote CAS. rpc FetchMissingBlobs(FetchMissingBlobsRequest) returns (FetchMissingBlobsResponse) {} // Upload blobs from the local cache to a remote CAS. // // This request is equivalent to `FindMissingBlobs` followed by // ByteStream `Write` or `BatchUpdateBlobs` requests. // // Blobs that failed to be uploaded will be listed in the response. // // Errors: // * `INVALID_ARGUMENT`: The client attempted to upload more than the // server supported limit. // // Individual requests may return the following error, additionally: // * `NOT_FOUND`: The requested blob is not present in the local cache. // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the blob. rpc UploadMissingBlobs(UploadMissingBlobsRequest) returns (UploadMissingBlobsResponse) {} // Fetch the entire directory tree rooted at a node from a remote CAS to the // local cache. // // This request is equivalent to `GetTree`, storing the `Directory` objects // in the local cache. Optionally, this will also fetch all blobs referenced // by the `Directory` objects, equivalent to `FetchMissingBlobs`. // // If no remote CAS is available, this will check presence of the entire // directory tree (and optionally also file blobs) in the local cache. // // * `NOT_FOUND`: The requested tree is not present in the CAS or incomplete. rpc FetchTree(FetchTreeRequest) returns (FetchTreeResponse) {} // Upload the entire directory tree from the local cache to a remote CAS. // // This request is equivalent to `UploadMissingBlobs` for all blobs // referenced by the specified tree (recursively). // // Errors: // * `NOT_FOUND`: The requested tree root is not present in the local cache. // * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the tree. rpc UploadTree(UploadTreeRequest) returns (UploadTreeResponse) {} // Stage a directory tree in the local filesystem. // // This makes the specified directory tree temporarily available for local // filesystem access. It is implementation-defined whether this uses a // userspace filesystem such as FUSE, hardlinking or a full copy. // // Missing blobs are fetched, if a CAS remote is configured. // // The staging starts when the server receives the initial request and // it is ready to be used on the initial (non-error) response from the // server. // // The server will clean up the staged directory when it either // receives an additional request (with all fields unset) or when the // stream is closed. The server will send an additional response after // cleanup is complete. rpc StageTree(stream StageTreeRequest) returns (stream StageTreeResponse) {} // Capture a directory tree from the local filesystem. // // This imports the specified path from the local filesystem into CAS. // // If a CAS remote is configured, the blobs are uploaded. // The `bypass_local_cache` parameter is a hint to indicate whether the blobs // shall be uploaded without first storing them in the local cache. // // The `move_files` parameter is a hint to indicate that files could be // moved into the storage. This can make capturing more efficient by // avoiding copies when it is known that files will not be needed after they // are imported. If a server chooses not to move them, the source files will // still exist after this request. // // The `skip_upload` parameter is a hint to indicate that the files do not // need to be uploaded to the remote CAS. This can be useful for local-only // workflows where an upload step adds unnecessary latency. rpc CaptureTree(CaptureTreeRequest) returns (CaptureTreeResponse) {} // Capture files from the local filesystem. // // This imports the specified paths from the local filesystem into CAS. // // If a CAS remote is configured, the blobs are uploaded. // The `bypass_local_cache` parameter is a hint to indicate whether the blobs // shall be uploaded without first storing them in the local cache. // // The `move_files` parameter is a hint to indicate that the files could be // moved into the storage. This can make capturing more efficient by // avoiding copies when it is known that files will not be needed after they // are imported. If a server chooses not to move them, the source files will // still exist after this request. // // The `skip_upload` parameter is a hint to indicate that the files do not // need to be uploaded to the remote CAS. This can be useful for local-only // workflows where an upload step adds unnecessary latency. rpc CaptureFiles(CaptureFilesRequest) returns (CaptureFilesResponse) {} // Configure remote CAS endpoint. // // This returns a string that can be used as instance_name to access the // specified endpoint in further requests. // // DEPRECATED: Use `content_addressable_storage` in `GetInstanceNameForRemotes()` // instead. rpc GetInstanceNameForRemote(GetInstanceNameForRemoteRequest) returns (GetInstanceNameForRemoteResponse) {} // Configure remote endpoints. // // This returns a string that can be used as instance_name to access the // specified endpoints in further requests. rpc GetInstanceNameForRemotes(GetInstanceNameForRemotesRequest) returns (GetInstanceNameForRemotesResponse) {} // Configure sandboxed clients. // // This returns a string that can be used as instance_name to access // this service from clients running in the specified filesystem/mount // namespace or chroot environment rpc GetInstanceNameForNamespace(GetInstanceNameForNamespaceRequest) returns (GetInstanceNameForNamespaceResponse) {} // Query total space used by the local cache. rpc GetLocalDiskUsage(GetLocalDiskUsageRequest) returns (GetLocalDiskUsageResponse) {} } // A request message for // [LocalContentAddressableStorage.GetLocalServerDetails][build.buildgrid.v2.LocalContentAddressableStorage.GetLocalServerDetails]. message GetLocalServerDetailsRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own // workers, storage, caches, etc.). The server MAY require use of this // field to select between them in an implementation-defined fashion, // otherwise it can be omitted. string instance_name = 1; } // A response message for // [LocalContentAddressableStorage.GetLocalServerDetails][build.buildgrid.v2.LocalContentAddressableStorage.GetLocalServerDetails]. message LocalServerDetails { // The hostname where the local CAS server is running. // This is used to determine if the client is running on the same host as // the server. When running on the same host, you may access the // storage_root directly. string hostname = 1; // The user ID (UID) which owns the local CAS process. // This is used to determine if the client is running as the same user as // the server. When running as the same user, hardlinking from the // storage_root is dangerous. int64 process_uid = 2; // The absolute path to the local CAS cache directory. // The storage_root contains locally stored blobs for digests in CAS. // The local path to a blob has the following format: // storage_root/objects/{hash[:2]}/{hash[2:]} string storage_root = 3; } // A request message for // [LocalContentAddressableStorage.FetchMissingBlobs][build.buildgrid.v2.LocalContentAddressableStorage.FetchMissingBlobs]. message FetchMissingBlobsRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // A list of the blobs to fetch. repeated build.bazel.remote.execution.v2.Digest blob_digests = 2; } // A response message for // [LocalContentAddressableStorage.FetchMissingBlobs][build.buildgrid.v2.LocalContentAddressableStorage.FetchMissingBlobs]. message FetchMissingBlobsResponse { // A response corresponding to a single blob that the client tried to download. message Response { // The digest to which this response corresponds. build.bazel.remote.execution.v2.Digest digest = 1; // The result of attempting to download that blob. google.rpc.Status status = 2; } // The responses to the failed requests. repeated Response responses = 1; } // A request message for // [LocalContentAddressableStorage.UploadMissingBlobs][build.buildgrid.v2.LocalContentAddressableStorage.UploadMissingBlobs]. message UploadMissingBlobsRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // A list of the blobs to fetch. repeated build.bazel.remote.execution.v2.Digest blob_digests = 2; } // A response message for // [LocalContentAddressableStorage.UploadMissingBlobs][build.buildgrid.v2.LocalContentAddressableStorage.UploadMissingBlobs]. message UploadMissingBlobsResponse { // A response corresponding to a single blob that the client tried to upload. message Response { // The digest to which this response corresponds. build.bazel.remote.execution.v2.Digest digest = 1; // The result of attempting to upload that blob. google.rpc.Status status = 2; } // The responses to the failed requests. repeated Response responses = 1; } // A request message for // [LocalContentAddressableStorage.FetchTree][build.buildgrid.v2.LocalContentAddressableStorage.FetchTree]. message FetchTreeRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The digest of the root, which must be an encoded // [Directory][build.bazel.remote.execution.v2.Directory] message // stored in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. build.bazel.remote.execution.v2.Digest root_digest = 2; // Whether to fetch blobs of files in the directory tree. bool fetch_file_blobs = 3; } // A response message for // [LocalContentAddressableStorage.FetchTree][build.buildgrid.v2.LocalContentAddressableStorage.FetchTree]. message FetchTreeResponse { } // A request message for // [LocalContentAddressableStorage.UploadTree][build.buildgrid.v2.LocalContentAddressableStorage.UploadTree]. message UploadTreeRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The digest of the root, which must be an encoded // [Directory][build.bazel.remote.execution.v2.Directory] message // stored in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. build.bazel.remote.execution.v2.Digest root_digest = 2; } // A response message for // [LocalContentAddressableStorage.UploadTree][build.buildgrid.v2.LocalContentAddressableStorage.UploadTree]. message UploadTreeResponse { } // A request message for // [LocalContentAddressableStorage.StageTree][build.buildgrid.v2.LocalContentAddressableStorage.StageTree]. message StageTreeRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The digest of the root, which must be an encoded // [Directory][build.bazel.remote.execution.v2.Directory] message // stored in the // [ContentAddressableStorage][build.bazel.remote.execution.v2.ContentAddressableStorage]. build.bazel.remote.execution.v2.Digest root_digest = 2; // The path in the local filesystem where the specified tree should be // staged. It must either point to an empty directory or not exist yet. // // This is optional. If not specified, the server will choose a location. // The server may require the path to be on the same filesystem as the local // cache to support hardlinking. // // The path may be a subdirectory of another staged tree. The lifetime of // this staged tree will in that case be limited to the lifetime of the // parent. string path = 3; message Credentials { int64 uid = 1; int64 gid = 2; } // The UNIX credentials of the processes that will access the staged tree. // This will be used to ensure that the files have the right permissions // for access without risking corruption of files in the local cache. Credentials access_credentials = 4; // The path, relative to the staging directory, where a server socket should // be created for access to the Remote Execution API. string remote_apis_socket_path = 7; // The commands to run against the tree before it is cleaned. // Each command must be an absolute path to an executable which can be run as // `/path/to/command `. // The working directory of launched commands will be the same as the LocalCAS // server process. // The server SHOULD maintain an allowlist of commands that can be launched. repeated string pre_unstage_commands = 5; enum StagingMode { // The staging mode is not specified. The LocalCAS server will use the default // staging mode chosen from its configuration and environment. DEFAULT = 0; // The tree will be staged using libfuse. FUSE = 1; // The tree will be staged by copying or linking files from the local // cache. The exact behavior depends on the filesystem, contents of the // tree and the credentials of the client. COPY_OR_LINK = 2; } // The staging mode to use for the tree. // `INVALID_ARGUMENT` is returned if the server does not support the requested mode. StagingMode staging_mode = 6; } // A response message for // [LocalContentAddressableStorage.StageTree][build.buildgrid.v2.LocalContentAddressableStorage.StageTree]. message StageTreeResponse { // The path in the local filesystem where the specified tree has been staged. string path = 1; } // A request message for // [LocalContentAddressableStorage.CaptureTree][build.buildgrid.v2.LocalContentAddressableStorage.CaptureTree]. message CaptureTreeRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The optional root path to restrict capture to a subtree. // If specified, `path` will be resolved inside this root. // No files outside the root will be captured. string root = 6; // The path(s) in the local filesystem to capture. repeated string path = 2; // This is a hint whether the blobs shall be uploaded to the remote CAS // without first storing them in the local cache. bool bypass_local_cache = 3; // The properties of path(s) in the local filesystem to capture. repeated string node_properties = 4; // Hints whether files can be moved into the storage. // If enabled, they MUST NOT be modified after issuing this request in order // to guarantee consistency. bool move_files = 5; // The format that the localcas should use to capture the contents of // output directories. // // In case this field is set to a value that is not supported by the // localcas, the worker SHOULD interpret this field as TREE_ONLY. The // localcas MAY store output directories in formats that are a superset // of what was requested (e.g., interpreting DIRECTORY_ONLY as // TREE_AND_DIRECTORY). build.bazel.remote.execution.v2.Command.OutputDirectoryFormat output_directory_format = 7; // Hints whether files should be uploaded to the remote CAS. // This is mutually exclusive with `bypass_local_cache`. If both `skip_upload` // and `bypass_local_cache` are set, the server MUST return an InvalidArgument // error. bool skip_upload = 8; // The mask to apply to the files and directories being captured. // For example, a mask of `0222` will result in contents being captured as read-only. // This is only effective when the unix_mode property is captured. google.protobuf.UInt32Value unix_mode_mask = 9; } // A response message for // [LocalContentAddressableStorage.CaptureTree][build.buildgrid.v2.LocalContentAddressableStorage.CaptureTree]. message CaptureTreeResponse { // A response corresponding to a single blob that the client tried to upload. message Response { // The path to which this response corresponds. string path = 1; // The digest of the captured tree as an encoded // [Tree][build.bazel.remote.execution.v2.Tree] proto containing the // directory's contents, if successful. build.bazel.remote.execution.v2.Digest tree_digest = 2; // The result of attempting to capture and upload the tree. google.rpc.Status status = 3; // The digest of the captured directory as an encoded // [Tree][build.bazel.remote.execution.v2.Directory] proto containing the // directory's contents, if successful. build.bazel.remote.execution.v2.Digest root_directory_digest = 4; } // The responses to the requests. repeated Response responses = 1; } // A request message for // [LocalContentAddressableStorage.CaptureFiles][build.buildgrid.v2.LocalContentAddressableStorage.CaptureFiles]. message CaptureFilesRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The optional root path to restrict capture to a subtree. // If specified, `path` will be resolved inside this root. // No files outside the root will be captured. string root = 6; // The path(s) in the local filesystem to capture. repeated string path = 2; // This is a hint whether the blobs shall be uploaded to the remote CAS // without first storing them in the local cache. bool bypass_local_cache = 3; // The properties of path(s) in the local filesystem to capture. repeated string node_properties = 4; // Hints whether the files can be moved into the storage. // If enabled, they MUST NOT be modified after issuing this request in order // to guarantee consistency. bool move_files = 5; // Hints whether files should be uploaded to the remote CAS. // This is mutually exclusive with `bypass_local_cache`. If both `skip_upload` // and `bypass_local_cache` are set, the server MUST return an InvalidArgument // error. bool skip_upload = 7; // The mask to apply to the files being captured. // For example, a mask of `0222` will result in contents being captured as read-only. // This is only effective when the unix_mode property is captured. google.protobuf.UInt32Value unix_mode_mask = 8; } // A response message for // [LocalContentAddressableStorage.CaptureFiles][build.buildgrid.v2.LocalContentAddressableStorage.CaptureFiles]. message CaptureFilesResponse { // A response corresponding to a single blob that the client tried to upload. message Response { // The path to which this response corresponds. string path = 1; // The digest of the captured file's content, if successful. build.bazel.remote.execution.v2.Digest digest = 2; // The result of attempting to capture and upload the file. google.rpc.Status status = 3; // True if the captured file was executable, false otherwise. bool is_executable = 4; // The node properties of the captured file. reserved 5; build.bazel.remote.execution.v2.NodeProperties node_properties = 6; } // The responses to the requests. repeated Response responses = 1; } // A request message for // [LocalContentAddressableStorage.GetInstanceNameForRemote][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForRemote]. message GetInstanceNameForRemoteRequest { // The URL for the remote CAS server. string url = 1; // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 2; // PEM-encoded public server certificate for https connections to the remote // CAS server. bytes server_cert = 3; // PEM-encoded private client key for https with certificate-based client // authentication. If this is specified, `client_cert` must be specified // as well. bytes client_key = 4; // PEM-encoded public client certificate for https with certificate-based // client authentication. If this is specified, `client_key` must be // specified as well. bytes client_cert = 5; } // A response message for // [LocalContentAddressableStorage.GetInstanceNameForRemote][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForRemote]. message GetInstanceNameForRemoteResponse { string instance_name = 1; } message Remote { // The URL for the remote server. string url = 1; // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 2; // PEM-encoded public server certificate for https connections to the remote // server. bytes server_cert = 3; // PEM-encoded private client key for https with certificate-based client // authentication. If this is specified, `client_cert` must be specified // as well. bytes client_key = 4; // PEM-encoded public client certificate for https with certificate-based // client authentication. If this is specified, `client_key` must be // specified as well. bytes client_cert = 5; // The path to a token for optional HTTP bearer authentication. string access_token_path = 7; // Access token reload interval. google.protobuf.Duration access_token_reload_interval = 8; // Remote keepalive time. Unset or empty is treated as disabling keepalive. google.protobuf.Duration keepalive_time = 6; // The maximum number of time a request is retried on this remote // A `retry_limit` of 0 indicates that // there is no limit and the server can pick a retry_limit int64 retry_limit = 9; // The initial recommended retry delay before the first retry, the // server might implement a backoff startegy. // A `retry_delay` of unset or empty indicates // that there is no delay recommended and the server can pick a retry_delay google.protobuf.Duration retry_delay = 10; // How long to wait for a response from the remote. google.protobuf.Duration request_timeout = 11; // If set, don't upload any blobs, action results or assets to the remote. bool read_only = 12; } // A request message for // [LocalContentAddressableStorage.GetInstanceNameForRemotes][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForRemotes]. message GetInstanceNameForRemotesRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 3; Remote content_addressable_storage = 1; Remote remote_asset = 2; Remote action_cache = 4; Remote execution = 5; } // A response message for // [LocalContentAddressableStorage.GetInstanceNameForRemotes][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForRemotes]. message GetInstanceNameForRemotesResponse { string instance_name = 1; } // A request message for // [LocalContentAddressableStorage.GetInstanceNameForRemote][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForNamespace]. message GetInstanceNameForNamespaceRequest { // The instance of the execution system to operate against. A server may // support multiple instances of the execution system (with their own workers, // storage, caches, etc.). The server MAY require use of this field to select // between them in an implementation-defined fashion, otherwise it can be // omitted. string instance_name = 1; // The root path of the mount namespace to restrict capture and staging. // All paths in requests to the new instance will be resolved inside this root. string root = 2; } // A response message for // [LocalContentAddressableStorage.GetInstanceNameForRemote][build.buildgrid.v2.LocalContentAddressableStorage.GetInstanceNameForNamespace]. message GetInstanceNameForNamespaceResponse { string instance_name = 1; } // A request message for // [LocalContentAddressableStorage.GetLocalDiskUsage][build.buildgrid.v2.LocalContentAddressableStorage.GetLocalDiskUsage]. message GetLocalDiskUsageRequest { } // A response message for // [LocalContentAddressableStorage.GetLocalDiskUsage][build.buildgrid.v2.LocalContentAddressableStorage.GetLocalDiskUsage]. message GetLocalDiskUsageResponse { // Total size used by the local cache, in bytes. int64 size_bytes = 1; // Disk quota for the local cache, in bytes. A value of 0 means no quota is set. int64 quota_bytes = 2; } apache-buildstream-27ae392/src/buildstream/_protos/build/buildgrid/local_cas_pb2.py000066400000000000000000000323071514607367700305170ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: build/buildgrid/local_cas.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'build/buildgrid/local_cas.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 from buildstream._protos.google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 from google.protobuf import duration_pb2 as google_dot_protobuf_dot_duration__pb2 from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x62uild/buildgrid/local_cas.proto\x12\x0f\x62uild.buildgrid\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x17google/rpc/status.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1egoogle/protobuf/wrappers.proto\"5\n\x1cGetLocalServerDetailsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"Q\n\x12LocalServerDetails\x12\x10\n\x08hostname\x18\x01 \x01(\t\x12\x13\n\x0bprocess_uid\x18\x02 \x01(\x03\x12\x14\n\x0cstorage_root\x18\x03 \x01(\t\"p\n\x18\x46\x65tchMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xcc\x01\n\x19\x46\x65tchMissingBlobsResponse\x12\x46\n\tresponses\x18\x01 \x03(\x0b\x32\x33.build.buildgrid.FetchMissingBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"q\n\x19UploadMissingBlobsRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12=\n\x0c\x62lob_digests\x18\x02 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xce\x01\n\x1aUploadMissingBlobsResponse\x12G\n\tresponses\x18\x01 \x03(\x0b\x32\x34.build.buildgrid.UploadMissingBlobsResponse.Response\x1ag\n\x08Response\x12\x37\n\x06\x64igest\x18\x01 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x02 \x01(\x0b\x32\x12.google.rpc.Status\"\x81\x01\n\x10\x46\x65tchTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x18\n\x10\x66\x65tch_file_blobs\x18\x03 \x01(\x08\"\x13\n\x11\x46\x65tchTreeResponse\"h\n\x11UploadTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\x14\n\x12UploadTreeResponse\"\xa5\x03\n\x10StageTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12<\n\x0broot_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x0c\n\x04path\x18\x03 \x01(\t\x12I\n\x12\x61\x63\x63\x65ss_credentials\x18\x04 \x01(\x0b\x32-.build.buildgrid.StageTreeRequest.Credentials\x12\x1f\n\x17remote_apis_socket_path\x18\x07 \x01(\t\x12\x1c\n\x14pre_unstage_commands\x18\x05 \x03(\t\x12\x43\n\x0cstaging_mode\x18\x06 \x01(\x0e\x32-.build.buildgrid.StageTreeRequest.StagingMode\x1a\'\n\x0b\x43redentials\x12\x0b\n\x03uid\x18\x01 \x01(\x03\x12\x0b\n\x03gid\x18\x02 \x01(\x03\"6\n\x0bStagingMode\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x08\n\x04\x46USE\x10\x01\x12\x10\n\x0c\x43OPY_OR_LINK\x10\x02\"!\n\x11StageTreeResponse\x12\x0c\n\x04path\x18\x01 \x01(\t\"\xbc\x02\n\x12\x43\x61ptureTreeRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x06 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x1a\n\x12\x62ypass_local_cache\x18\x03 \x01(\x08\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x12\n\nmove_files\x18\x05 \x01(\x08\x12_\n\x17output_directory_format\x18\x07 \x01(\x0e\x32>.build.bazel.remote.execution.v2.Command.OutputDirectoryFormat\x12\x13\n\x0bskip_upload\x18\x08 \x01(\x08\x12\x34\n\x0eunix_mode_mask\x18\t \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\"\x9c\x02\n\x13\x43\x61ptureTreeResponse\x12@\n\tresponses\x18\x01 \x03(\x0b\x32-.build.buildgrid.CaptureTreeResponse.Response\x1a\xc2\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12<\n\x0btree_digest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x46\n\x15root_directory_digest\x18\x04 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\"\xdc\x01\n\x13\x43\x61ptureFilesRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x06 \x01(\t\x12\x0c\n\x04path\x18\x02 \x03(\t\x12\x1a\n\x12\x62ypass_local_cache\x18\x03 \x01(\x08\x12\x17\n\x0fnode_properties\x18\x04 \x03(\t\x12\x12\n\nmove_files\x18\x05 \x01(\x08\x12\x13\n\x0bskip_upload\x18\x07 \x01(\x08\x12\x34\n\x0eunix_mode_mask\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.UInt32Value\"\xb8\x02\n\x14\x43\x61ptureFilesResponse\x12\x41\n\tresponses\x18\x01 \x03(\x0b\x32..build.buildgrid.CaptureFilesResponse.Response\x1a\xdc\x01\n\x08Response\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\"\n\x06status\x18\x03 \x01(\x0b\x32\x12.google.rpc.Status\x12\x15\n\ris_executable\x18\x04 \x01(\x08\x12H\n\x0fnode_properties\x18\x06 \x01(\x0b\x32/.build.bazel.remote.execution.v2.NodePropertiesJ\x04\x08\x05\x10\x06\"\x83\x01\n\x1fGetInstanceNameForRemoteRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rinstance_name\x18\x02 \x01(\t\x12\x13\n\x0bserver_cert\x18\x03 \x01(\x0c\x12\x12\n\nclient_key\x18\x04 \x01(\x0c\x12\x13\n\x0b\x63lient_cert\x18\x05 \x01(\x0c\"9\n GetInstanceNameForRemoteResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\x85\x03\n\x06Remote\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x15\n\rinstance_name\x18\x02 \x01(\t\x12\x13\n\x0bserver_cert\x18\x03 \x01(\x0c\x12\x12\n\nclient_key\x18\x04 \x01(\x0c\x12\x13\n\x0b\x63lient_cert\x18\x05 \x01(\x0c\x12\x19\n\x11\x61\x63\x63\x65ss_token_path\x18\x07 \x01(\t\x12?\n\x1c\x61\x63\x63\x65ss_token_reload_interval\x18\x08 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x31\n\x0ekeepalive_time\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x13\n\x0bretry_limit\x18\t \x01(\x03\x12.\n\x0bretry_delay\x18\n \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x32\n\x0frequest_timeout\x18\x0b \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tread_only\x18\x0c \x01(\x08\"\x81\x02\n GetInstanceNameForRemotesRequest\x12\x15\n\rinstance_name\x18\x03 \x01(\t\x12<\n\x1b\x63ontent_addressable_storage\x18\x01 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12-\n\x0cremote_asset\x18\x02 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12-\n\x0c\x61\x63tion_cache\x18\x04 \x01(\x0b\x32\x17.build.buildgrid.Remote\x12*\n\texecution\x18\x05 \x01(\x0b\x32\x17.build.buildgrid.Remote\":\n!GetInstanceNameForRemotesResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"I\n\"GetInstanceNameForNamespaceRequest\x12\x15\n\rinstance_name\x18\x01 \x01(\t\x12\x0c\n\x04root\x18\x02 \x01(\t\"<\n#GetInstanceNameForNamespaceResponse\x12\x15\n\rinstance_name\x18\x01 \x01(\t\"\x1a\n\x18GetLocalDiskUsageRequest\"D\n\x19GetLocalDiskUsageResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x03\x12\x13\n\x0bquota_bytes\x18\x02 \x01(\x03\x32\xb8\n\n\x1eLocalContentAddressableStorage\x12m\n\x15GetLocalServerDetails\x12-.build.buildgrid.GetLocalServerDetailsRequest\x1a#.build.buildgrid.LocalServerDetails\"\x00\x12l\n\x11\x46\x65tchMissingBlobs\x12).build.buildgrid.FetchMissingBlobsRequest\x1a*.build.buildgrid.FetchMissingBlobsResponse\"\x00\x12o\n\x12UploadMissingBlobs\x12*.build.buildgrid.UploadMissingBlobsRequest\x1a+.build.buildgrid.UploadMissingBlobsResponse\"\x00\x12T\n\tFetchTree\x12!.build.buildgrid.FetchTreeRequest\x1a\".build.buildgrid.FetchTreeResponse\"\x00\x12W\n\nUploadTree\x12\".build.buildgrid.UploadTreeRequest\x1a#.build.buildgrid.UploadTreeResponse\"\x00\x12X\n\tStageTree\x12!.build.buildgrid.StageTreeRequest\x1a\".build.buildgrid.StageTreeResponse\"\x00(\x01\x30\x01\x12Z\n\x0b\x43\x61ptureTree\x12#.build.buildgrid.CaptureTreeRequest\x1a$.build.buildgrid.CaptureTreeResponse\"\x00\x12]\n\x0c\x43\x61ptureFiles\x12$.build.buildgrid.CaptureFilesRequest\x1a%.build.buildgrid.CaptureFilesResponse\"\x00\x12\x81\x01\n\x18GetInstanceNameForRemote\x12\x30.build.buildgrid.GetInstanceNameForRemoteRequest\x1a\x31.build.buildgrid.GetInstanceNameForRemoteResponse\"\x00\x12\x84\x01\n\x19GetInstanceNameForRemotes\x12\x31.build.buildgrid.GetInstanceNameForRemotesRequest\x1a\x32.build.buildgrid.GetInstanceNameForRemotesResponse\"\x00\x12\x8a\x01\n\x1bGetInstanceNameForNamespace\x12\x33.build.buildgrid.GetInstanceNameForNamespaceRequest\x1a\x34.build.buildgrid.GetInstanceNameForNamespaceResponse\"\x00\x12l\n\x11GetLocalDiskUsage\x12).build.buildgrid.GetLocalDiskUsageRequest\x1a*.build.buildgrid.GetLocalDiskUsageResponse\"\x00\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'build.buildgrid.local_cas_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_GETLOCALSERVERDETAILSREQUEST']._serialized_start=197 _globals['_GETLOCALSERVERDETAILSREQUEST']._serialized_end=250 _globals['_LOCALSERVERDETAILS']._serialized_start=252 _globals['_LOCALSERVERDETAILS']._serialized_end=333 _globals['_FETCHMISSINGBLOBSREQUEST']._serialized_start=335 _globals['_FETCHMISSINGBLOBSREQUEST']._serialized_end=447 _globals['_FETCHMISSINGBLOBSRESPONSE']._serialized_start=450 _globals['_FETCHMISSINGBLOBSRESPONSE']._serialized_end=654 _globals['_FETCHMISSINGBLOBSRESPONSE_RESPONSE']._serialized_start=551 _globals['_FETCHMISSINGBLOBSRESPONSE_RESPONSE']._serialized_end=654 _globals['_UPLOADMISSINGBLOBSREQUEST']._serialized_start=656 _globals['_UPLOADMISSINGBLOBSREQUEST']._serialized_end=769 _globals['_UPLOADMISSINGBLOBSRESPONSE']._serialized_start=772 _globals['_UPLOADMISSINGBLOBSRESPONSE']._serialized_end=978 _globals['_UPLOADMISSINGBLOBSRESPONSE_RESPONSE']._serialized_start=551 _globals['_UPLOADMISSINGBLOBSRESPONSE_RESPONSE']._serialized_end=654 _globals['_FETCHTREEREQUEST']._serialized_start=981 _globals['_FETCHTREEREQUEST']._serialized_end=1110 _globals['_FETCHTREERESPONSE']._serialized_start=1112 _globals['_FETCHTREERESPONSE']._serialized_end=1131 _globals['_UPLOADTREEREQUEST']._serialized_start=1133 _globals['_UPLOADTREEREQUEST']._serialized_end=1237 _globals['_UPLOADTREERESPONSE']._serialized_start=1239 _globals['_UPLOADTREERESPONSE']._serialized_end=1259 _globals['_STAGETREEREQUEST']._serialized_start=1262 _globals['_STAGETREEREQUEST']._serialized_end=1683 _globals['_STAGETREEREQUEST_CREDENTIALS']._serialized_start=1588 _globals['_STAGETREEREQUEST_CREDENTIALS']._serialized_end=1627 _globals['_STAGETREEREQUEST_STAGINGMODE']._serialized_start=1629 _globals['_STAGETREEREQUEST_STAGINGMODE']._serialized_end=1683 _globals['_STAGETREERESPONSE']._serialized_start=1685 _globals['_STAGETREERESPONSE']._serialized_end=1718 _globals['_CAPTURETREEREQUEST']._serialized_start=1721 _globals['_CAPTURETREEREQUEST']._serialized_end=2037 _globals['_CAPTURETREERESPONSE']._serialized_start=2040 _globals['_CAPTURETREERESPONSE']._serialized_end=2324 _globals['_CAPTURETREERESPONSE_RESPONSE']._serialized_start=2130 _globals['_CAPTURETREERESPONSE_RESPONSE']._serialized_end=2324 _globals['_CAPTUREFILESREQUEST']._serialized_start=2327 _globals['_CAPTUREFILESREQUEST']._serialized_end=2547 _globals['_CAPTUREFILESRESPONSE']._serialized_start=2550 _globals['_CAPTUREFILESRESPONSE']._serialized_end=2862 _globals['_CAPTUREFILESRESPONSE_RESPONSE']._serialized_start=2642 _globals['_CAPTUREFILESRESPONSE_RESPONSE']._serialized_end=2862 _globals['_GETINSTANCENAMEFORREMOTEREQUEST']._serialized_start=2865 _globals['_GETINSTANCENAMEFORREMOTEREQUEST']._serialized_end=2996 _globals['_GETINSTANCENAMEFORREMOTERESPONSE']._serialized_start=2998 _globals['_GETINSTANCENAMEFORREMOTERESPONSE']._serialized_end=3055 _globals['_REMOTE']._serialized_start=3058 _globals['_REMOTE']._serialized_end=3447 _globals['_GETINSTANCENAMEFORREMOTESREQUEST']._serialized_start=3450 _globals['_GETINSTANCENAMEFORREMOTESREQUEST']._serialized_end=3707 _globals['_GETINSTANCENAMEFORREMOTESRESPONSE']._serialized_start=3709 _globals['_GETINSTANCENAMEFORREMOTESRESPONSE']._serialized_end=3767 _globals['_GETINSTANCENAMEFORNAMESPACEREQUEST']._serialized_start=3769 _globals['_GETINSTANCENAMEFORNAMESPACEREQUEST']._serialized_end=3842 _globals['_GETINSTANCENAMEFORNAMESPACERESPONSE']._serialized_start=3844 _globals['_GETINSTANCENAMEFORNAMESPACERESPONSE']._serialized_end=3904 _globals['_GETLOCALDISKUSAGEREQUEST']._serialized_start=3906 _globals['_GETLOCALDISKUSAGEREQUEST']._serialized_end=3932 _globals['_GETLOCALDISKUSAGERESPONSE']._serialized_start=3934 _globals['_GETLOCALDISKUSAGERESPONSE']._serialized_end=4002 _globals['_LOCALCONTENTADDRESSABLESTORAGE']._serialized_start=4005 _globals['_LOCALCONTENTADDRESSABLESTORAGE']._serialized_end=5341 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/build/buildgrid/local_cas_pb2.pyi000066400000000000000000000406541514607367700306740ustar00rootroot00000000000000from build.bazel.remote.execution.v2 import remote_execution_pb2 as _remote_execution_pb2 from google.rpc import status_pb2 as _status_pb2 from google.protobuf import duration_pb2 as _duration_pb2 from google.protobuf import wrappers_pb2 as _wrappers_pb2 from google.protobuf.internal import containers as _containers from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class GetLocalServerDetailsRequest(_message.Message): __slots__ = ("instance_name",) INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] instance_name: str def __init__(self, instance_name: _Optional[str] = ...) -> None: ... class LocalServerDetails(_message.Message): __slots__ = ("hostname", "process_uid", "storage_root") HOSTNAME_FIELD_NUMBER: _ClassVar[int] PROCESS_UID_FIELD_NUMBER: _ClassVar[int] STORAGE_ROOT_FIELD_NUMBER: _ClassVar[int] hostname: str process_uid: int storage_root: str def __init__(self, hostname: _Optional[str] = ..., process_uid: _Optional[int] = ..., storage_root: _Optional[str] = ...) -> None: ... class FetchMissingBlobsRequest(_message.Message): __slots__ = ("instance_name", "blob_digests") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] BLOB_DIGESTS_FIELD_NUMBER: _ClassVar[int] instance_name: str blob_digests: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] def __init__(self, instance_name: _Optional[str] = ..., blob_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ...) -> None: ... class FetchMissingBlobsResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): __slots__ = ("digest", "status") DIGEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] digest: _remote_execution_pb2.Digest status: _status_pb2.Status def __init__(self, digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[FetchMissingBlobsResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[FetchMissingBlobsResponse.Response, _Mapping]]] = ...) -> None: ... class UploadMissingBlobsRequest(_message.Message): __slots__ = ("instance_name", "blob_digests") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] BLOB_DIGESTS_FIELD_NUMBER: _ClassVar[int] instance_name: str blob_digests: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] def __init__(self, instance_name: _Optional[str] = ..., blob_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ...) -> None: ... class UploadMissingBlobsResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): __slots__ = ("digest", "status") DIGEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] digest: _remote_execution_pb2.Digest status: _status_pb2.Status def __init__(self, digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[UploadMissingBlobsResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[UploadMissingBlobsResponse.Response, _Mapping]]] = ...) -> None: ... class FetchTreeRequest(_message.Message): __slots__ = ("instance_name", "root_digest", "fetch_file_blobs") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_DIGEST_FIELD_NUMBER: _ClassVar[int] FETCH_FILE_BLOBS_FIELD_NUMBER: _ClassVar[int] instance_name: str root_digest: _remote_execution_pb2.Digest fetch_file_blobs: bool def __init__(self, instance_name: _Optional[str] = ..., root_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., fetch_file_blobs: bool = ...) -> None: ... class FetchTreeResponse(_message.Message): __slots__ = () def __init__(self) -> None: ... class UploadTreeRequest(_message.Message): __slots__ = ("instance_name", "root_digest") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_DIGEST_FIELD_NUMBER: _ClassVar[int] instance_name: str root_digest: _remote_execution_pb2.Digest def __init__(self, instance_name: _Optional[str] = ..., root_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... class UploadTreeResponse(_message.Message): __slots__ = () def __init__(self) -> None: ... class StageTreeRequest(_message.Message): __slots__ = ("instance_name", "root_digest", "path", "access_credentials", "remote_apis_socket_path", "pre_unstage_commands", "staging_mode") class StagingMode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () DEFAULT: _ClassVar[StageTreeRequest.StagingMode] FUSE: _ClassVar[StageTreeRequest.StagingMode] COPY_OR_LINK: _ClassVar[StageTreeRequest.StagingMode] DEFAULT: StageTreeRequest.StagingMode FUSE: StageTreeRequest.StagingMode COPY_OR_LINK: StageTreeRequest.StagingMode class Credentials(_message.Message): __slots__ = ("uid", "gid") UID_FIELD_NUMBER: _ClassVar[int] GID_FIELD_NUMBER: _ClassVar[int] uid: int gid: int def __init__(self, uid: _Optional[int] = ..., gid: _Optional[int] = ...) -> None: ... INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_DIGEST_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] ACCESS_CREDENTIALS_FIELD_NUMBER: _ClassVar[int] REMOTE_APIS_SOCKET_PATH_FIELD_NUMBER: _ClassVar[int] PRE_UNSTAGE_COMMANDS_FIELD_NUMBER: _ClassVar[int] STAGING_MODE_FIELD_NUMBER: _ClassVar[int] instance_name: str root_digest: _remote_execution_pb2.Digest path: str access_credentials: StageTreeRequest.Credentials remote_apis_socket_path: str pre_unstage_commands: _containers.RepeatedScalarFieldContainer[str] staging_mode: StageTreeRequest.StagingMode def __init__(self, instance_name: _Optional[str] = ..., root_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., path: _Optional[str] = ..., access_credentials: _Optional[_Union[StageTreeRequest.Credentials, _Mapping]] = ..., remote_apis_socket_path: _Optional[str] = ..., pre_unstage_commands: _Optional[_Iterable[str]] = ..., staging_mode: _Optional[_Union[StageTreeRequest.StagingMode, str]] = ...) -> None: ... class StageTreeResponse(_message.Message): __slots__ = ("path",) PATH_FIELD_NUMBER: _ClassVar[int] path: str def __init__(self, path: _Optional[str] = ...) -> None: ... class CaptureTreeRequest(_message.Message): __slots__ = ("instance_name", "root", "path", "bypass_local_cache", "node_properties", "move_files", "output_directory_format", "skip_upload", "unix_mode_mask") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] BYPASS_LOCAL_CACHE_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] MOVE_FILES_FIELD_NUMBER: _ClassVar[int] OUTPUT_DIRECTORY_FORMAT_FIELD_NUMBER: _ClassVar[int] SKIP_UPLOAD_FIELD_NUMBER: _ClassVar[int] UNIX_MODE_MASK_FIELD_NUMBER: _ClassVar[int] instance_name: str root: str path: _containers.RepeatedScalarFieldContainer[str] bypass_local_cache: bool node_properties: _containers.RepeatedScalarFieldContainer[str] move_files: bool output_directory_format: _remote_execution_pb2.Command.OutputDirectoryFormat skip_upload: bool unix_mode_mask: _wrappers_pb2.UInt32Value def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., bypass_local_cache: bool = ..., node_properties: _Optional[_Iterable[str]] = ..., move_files: bool = ..., output_directory_format: _Optional[_Union[_remote_execution_pb2.Command.OutputDirectoryFormat, str]] = ..., skip_upload: bool = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ...) -> None: ... class CaptureTreeResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): __slots__ = ("path", "tree_digest", "status", "root_directory_digest") PATH_FIELD_NUMBER: _ClassVar[int] TREE_DIGEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] ROOT_DIRECTORY_DIGEST_FIELD_NUMBER: _ClassVar[int] path: str tree_digest: _remote_execution_pb2.Digest status: _status_pb2.Status root_directory_digest: _remote_execution_pb2.Digest def __init__(self, path: _Optional[str] = ..., tree_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., root_directory_digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[CaptureTreeResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[CaptureTreeResponse.Response, _Mapping]]] = ...) -> None: ... class CaptureFilesRequest(_message.Message): __slots__ = ("instance_name", "root", "path", "bypass_local_cache", "node_properties", "move_files", "skip_upload", "unix_mode_mask") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] BYPASS_LOCAL_CACHE_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] MOVE_FILES_FIELD_NUMBER: _ClassVar[int] SKIP_UPLOAD_FIELD_NUMBER: _ClassVar[int] UNIX_MODE_MASK_FIELD_NUMBER: _ClassVar[int] instance_name: str root: str path: _containers.RepeatedScalarFieldContainer[str] bypass_local_cache: bool node_properties: _containers.RepeatedScalarFieldContainer[str] move_files: bool skip_upload: bool unix_mode_mask: _wrappers_pb2.UInt32Value def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ..., path: _Optional[_Iterable[str]] = ..., bypass_local_cache: bool = ..., node_properties: _Optional[_Iterable[str]] = ..., move_files: bool = ..., skip_upload: bool = ..., unix_mode_mask: _Optional[_Union[_wrappers_pb2.UInt32Value, _Mapping]] = ...) -> None: ... class CaptureFilesResponse(_message.Message): __slots__ = ("responses",) class Response(_message.Message): __slots__ = ("path", "digest", "status", "is_executable", "node_properties") PATH_FIELD_NUMBER: _ClassVar[int] DIGEST_FIELD_NUMBER: _ClassVar[int] STATUS_FIELD_NUMBER: _ClassVar[int] IS_EXECUTABLE_FIELD_NUMBER: _ClassVar[int] NODE_PROPERTIES_FIELD_NUMBER: _ClassVar[int] path: str digest: _remote_execution_pb2.Digest status: _status_pb2.Status is_executable: bool node_properties: _remote_execution_pb2.NodeProperties def __init__(self, path: _Optional[str] = ..., digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., status: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., is_executable: bool = ..., node_properties: _Optional[_Union[_remote_execution_pb2.NodeProperties, _Mapping]] = ...) -> None: ... RESPONSES_FIELD_NUMBER: _ClassVar[int] responses: _containers.RepeatedCompositeFieldContainer[CaptureFilesResponse.Response] def __init__(self, responses: _Optional[_Iterable[_Union[CaptureFilesResponse.Response, _Mapping]]] = ...) -> None: ... class GetInstanceNameForRemoteRequest(_message.Message): __slots__ = ("url", "instance_name", "server_cert", "client_key", "client_cert") URL_FIELD_NUMBER: _ClassVar[int] INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] SERVER_CERT_FIELD_NUMBER: _ClassVar[int] CLIENT_KEY_FIELD_NUMBER: _ClassVar[int] CLIENT_CERT_FIELD_NUMBER: _ClassVar[int] url: str instance_name: str server_cert: bytes client_key: bytes client_cert: bytes def __init__(self, url: _Optional[str] = ..., instance_name: _Optional[str] = ..., server_cert: _Optional[bytes] = ..., client_key: _Optional[bytes] = ..., client_cert: _Optional[bytes] = ...) -> None: ... class GetInstanceNameForRemoteResponse(_message.Message): __slots__ = ("instance_name",) INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] instance_name: str def __init__(self, instance_name: _Optional[str] = ...) -> None: ... class Remote(_message.Message): __slots__ = ("url", "instance_name", "server_cert", "client_key", "client_cert", "access_token_path", "access_token_reload_interval", "keepalive_time", "retry_limit", "retry_delay", "request_timeout", "read_only") URL_FIELD_NUMBER: _ClassVar[int] INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] SERVER_CERT_FIELD_NUMBER: _ClassVar[int] CLIENT_KEY_FIELD_NUMBER: _ClassVar[int] CLIENT_CERT_FIELD_NUMBER: _ClassVar[int] ACCESS_TOKEN_PATH_FIELD_NUMBER: _ClassVar[int] ACCESS_TOKEN_RELOAD_INTERVAL_FIELD_NUMBER: _ClassVar[int] KEEPALIVE_TIME_FIELD_NUMBER: _ClassVar[int] RETRY_LIMIT_FIELD_NUMBER: _ClassVar[int] RETRY_DELAY_FIELD_NUMBER: _ClassVar[int] REQUEST_TIMEOUT_FIELD_NUMBER: _ClassVar[int] READ_ONLY_FIELD_NUMBER: _ClassVar[int] url: str instance_name: str server_cert: bytes client_key: bytes client_cert: bytes access_token_path: str access_token_reload_interval: _duration_pb2.Duration keepalive_time: _duration_pb2.Duration retry_limit: int retry_delay: _duration_pb2.Duration request_timeout: _duration_pb2.Duration read_only: bool def __init__(self, url: _Optional[str] = ..., instance_name: _Optional[str] = ..., server_cert: _Optional[bytes] = ..., client_key: _Optional[bytes] = ..., client_cert: _Optional[bytes] = ..., access_token_path: _Optional[str] = ..., access_token_reload_interval: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., keepalive_time: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., retry_limit: _Optional[int] = ..., retry_delay: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., request_timeout: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., read_only: bool = ...) -> None: ... class GetInstanceNameForRemotesRequest(_message.Message): __slots__ = ("instance_name", "content_addressable_storage", "remote_asset", "action_cache", "execution") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] CONTENT_ADDRESSABLE_STORAGE_FIELD_NUMBER: _ClassVar[int] REMOTE_ASSET_FIELD_NUMBER: _ClassVar[int] ACTION_CACHE_FIELD_NUMBER: _ClassVar[int] EXECUTION_FIELD_NUMBER: _ClassVar[int] instance_name: str content_addressable_storage: Remote remote_asset: Remote action_cache: Remote execution: Remote def __init__(self, instance_name: _Optional[str] = ..., content_addressable_storage: _Optional[_Union[Remote, _Mapping]] = ..., remote_asset: _Optional[_Union[Remote, _Mapping]] = ..., action_cache: _Optional[_Union[Remote, _Mapping]] = ..., execution: _Optional[_Union[Remote, _Mapping]] = ...) -> None: ... class GetInstanceNameForRemotesResponse(_message.Message): __slots__ = ("instance_name",) INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] instance_name: str def __init__(self, instance_name: _Optional[str] = ...) -> None: ... class GetInstanceNameForNamespaceRequest(_message.Message): __slots__ = ("instance_name", "root") INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] ROOT_FIELD_NUMBER: _ClassVar[int] instance_name: str root: str def __init__(self, instance_name: _Optional[str] = ..., root: _Optional[str] = ...) -> None: ... class GetInstanceNameForNamespaceResponse(_message.Message): __slots__ = ("instance_name",) INSTANCE_NAME_FIELD_NUMBER: _ClassVar[int] instance_name: str def __init__(self, instance_name: _Optional[str] = ...) -> None: ... class GetLocalDiskUsageRequest(_message.Message): __slots__ = () def __init__(self) -> None: ... class GetLocalDiskUsageResponse(_message.Message): __slots__ = ("size_bytes", "quota_bytes") SIZE_BYTES_FIELD_NUMBER: _ClassVar[int] QUOTA_BYTES_FIELD_NUMBER: _ClassVar[int] size_bytes: int quota_bytes: int def __init__(self, size_bytes: _Optional[int] = ..., quota_bytes: _Optional[int] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/build/buildgrid/local_cas_pb2_grpc.py000066400000000000000000000742671514607367700315450ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings from buildstream._protos.build.buildgrid import local_cas_pb2 as build_dot_buildgrid_dot_local__cas__pb2 GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in build/buildgrid/local_cas_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) class LocalContentAddressableStorageStub(object): """Missing associated documentation comment in .proto file.""" def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.GetLocalServerDetails = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetLocalServerDetails', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalServerDetailsRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.LocalServerDetails.FromString, _registered_method=True) self.FetchMissingBlobs = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/FetchMissingBlobs', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.FetchMissingBlobsRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.FetchMissingBlobsResponse.FromString, _registered_method=True) self.UploadMissingBlobs = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/UploadMissingBlobs', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.UploadMissingBlobsRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.UploadMissingBlobsResponse.FromString, _registered_method=True) self.FetchTree = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/FetchTree', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.FetchTreeRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.FetchTreeResponse.FromString, _registered_method=True) self.UploadTree = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/UploadTree', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.UploadTreeRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.UploadTreeResponse.FromString, _registered_method=True) self.StageTree = channel.stream_stream( '/build.buildgrid.LocalContentAddressableStorage/StageTree', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeResponse.FromString, _registered_method=True) self.CaptureTree = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/CaptureTree', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeResponse.FromString, _registered_method=True) self.CaptureFiles = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/CaptureFiles', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesResponse.FromString, _registered_method=True) self.GetInstanceNameForRemote = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForRemote', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteResponse.FromString, _registered_method=True) self.GetInstanceNameForRemotes = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForRemotes', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesResponse.FromString, _registered_method=True) self.GetInstanceNameForNamespace = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForNamespace', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceResponse.FromString, _registered_method=True) self.GetLocalDiskUsage = channel.unary_unary( '/build.buildgrid.LocalContentAddressableStorage/GetLocalDiskUsage', request_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.SerializeToString, response_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.FromString, _registered_method=True) class LocalContentAddressableStorageServicer(object): """Missing associated documentation comment in .proto file.""" def GetLocalServerDetails(self, request, context): """Retrieves the configuration details for the local CAS server. This RPC allows clients to obtain information such as the hostname, user ID, and storage root of the local CAS server. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def FetchMissingBlobs(self, request, context): """Fetch blobs from a remote CAS to the local cache. This request is equivalent to ByteStream `Read` or `BatchReadBlobs` requests, storing the downloaded blobs in the local cache. Requested blobs that failed to be downloaded will be listed in the response. Errors: * `INVALID_ARGUMENT`: The client attempted to download more than the server supported limit. Individual requests may return the following error, additionally: * `NOT_FOUND`: The requested blob is not present in the remote CAS. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def UploadMissingBlobs(self, request, context): """Upload blobs from the local cache to a remote CAS. This request is equivalent to `FindMissingBlobs` followed by ByteStream `Write` or `BatchUpdateBlobs` requests. Blobs that failed to be uploaded will be listed in the response. Errors: * `INVALID_ARGUMENT`: The client attempted to upload more than the server supported limit. Individual requests may return the following error, additionally: * `NOT_FOUND`: The requested blob is not present in the local cache. * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the blob. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def FetchTree(self, request, context): """Fetch the entire directory tree rooted at a node from a remote CAS to the local cache. This request is equivalent to `GetTree`, storing the `Directory` objects in the local cache. Optionally, this will also fetch all blobs referenced by the `Directory` objects, equivalent to `FetchMissingBlobs`. If no remote CAS is available, this will check presence of the entire directory tree (and optionally also file blobs) in the local cache. * `NOT_FOUND`: The requested tree is not present in the CAS or incomplete. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def UploadTree(self, request, context): """Upload the entire directory tree from the local cache to a remote CAS. This request is equivalent to `UploadMissingBlobs` for all blobs referenced by the specified tree (recursively). Errors: * `NOT_FOUND`: The requested tree root is not present in the local cache. * `RESOURCE_EXHAUSTED`: There is insufficient disk quota to store the tree. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def StageTree(self, request_iterator, context): """Stage a directory tree in the local filesystem. This makes the specified directory tree temporarily available for local filesystem access. It is implementation-defined whether this uses a userspace filesystem such as FUSE, hardlinking or a full copy. Missing blobs are fetched, if a CAS remote is configured. The staging starts when the server receives the initial request and it is ready to be used on the initial (non-error) response from the server. The server will clean up the staged directory when it either receives an additional request (with all fields unset) or when the stream is closed. The server will send an additional response after cleanup is complete. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CaptureTree(self, request, context): """Capture a directory tree from the local filesystem. This imports the specified path from the local filesystem into CAS. If a CAS remote is configured, the blobs are uploaded. The `bypass_local_cache` parameter is a hint to indicate whether the blobs shall be uploaded without first storing them in the local cache. The `move_files` parameter is a hint to indicate that files could be moved into the storage. This can make capturing more efficient by avoiding copies when it is known that files will not be needed after they are imported. If a server chooses not to move them, the source files will still exist after this request. The `skip_upload` parameter is a hint to indicate that the files do not need to be uploaded to the remote CAS. This can be useful for local-only workflows where an upload step adds unnecessary latency. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CaptureFiles(self, request, context): """Capture files from the local filesystem. This imports the specified paths from the local filesystem into CAS. If a CAS remote is configured, the blobs are uploaded. The `bypass_local_cache` parameter is a hint to indicate whether the blobs shall be uploaded without first storing them in the local cache. The `move_files` parameter is a hint to indicate that the files could be moved into the storage. This can make capturing more efficient by avoiding copies when it is known that files will not be needed after they are imported. If a server chooses not to move them, the source files will still exist after this request. The `skip_upload` parameter is a hint to indicate that the files do not need to be uploaded to the remote CAS. This can be useful for local-only workflows where an upload step adds unnecessary latency. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetInstanceNameForRemote(self, request, context): """Configure remote CAS endpoint. This returns a string that can be used as instance_name to access the specified endpoint in further requests. DEPRECATED: Use `content_addressable_storage` in `GetInstanceNameForRemotes()` instead. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetInstanceNameForRemotes(self, request, context): """Configure remote endpoints. This returns a string that can be used as instance_name to access the specified endpoints in further requests. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetInstanceNameForNamespace(self, request, context): """Configure sandboxed clients. This returns a string that can be used as instance_name to access this service from clients running in the specified filesystem/mount namespace or chroot environment """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetLocalDiskUsage(self, request, context): """Query total space used by the local cache. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_LocalContentAddressableStorageServicer_to_server(servicer, server): rpc_method_handlers = { 'GetLocalServerDetails': grpc.unary_unary_rpc_method_handler( servicer.GetLocalServerDetails, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalServerDetailsRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.LocalServerDetails.SerializeToString, ), 'FetchMissingBlobs': grpc.unary_unary_rpc_method_handler( servicer.FetchMissingBlobs, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.FetchMissingBlobsRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.FetchMissingBlobsResponse.SerializeToString, ), 'UploadMissingBlobs': grpc.unary_unary_rpc_method_handler( servicer.UploadMissingBlobs, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.UploadMissingBlobsRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.UploadMissingBlobsResponse.SerializeToString, ), 'FetchTree': grpc.unary_unary_rpc_method_handler( servicer.FetchTree, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.FetchTreeRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.FetchTreeResponse.SerializeToString, ), 'UploadTree': grpc.unary_unary_rpc_method_handler( servicer.UploadTree, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.UploadTreeRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.UploadTreeResponse.SerializeToString, ), 'StageTree': grpc.stream_stream_rpc_method_handler( servicer.StageTree, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.StageTreeResponse.SerializeToString, ), 'CaptureTree': grpc.unary_unary_rpc_method_handler( servicer.CaptureTree, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeResponse.SerializeToString, ), 'CaptureFiles': grpc.unary_unary_rpc_method_handler( servicer.CaptureFiles, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesResponse.SerializeToString, ), 'GetInstanceNameForRemote': grpc.unary_unary_rpc_method_handler( servicer.GetInstanceNameForRemote, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteResponse.SerializeToString, ), 'GetInstanceNameForRemotes': grpc.unary_unary_rpc_method_handler( servicer.GetInstanceNameForRemotes, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesResponse.SerializeToString, ), 'GetInstanceNameForNamespace': grpc.unary_unary_rpc_method_handler( servicer.GetInstanceNameForNamespace, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceResponse.SerializeToString, ), 'GetLocalDiskUsage': grpc.unary_unary_rpc_method_handler( servicer.GetLocalDiskUsage, request_deserializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.FromString, response_serializer=build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'build.buildgrid.LocalContentAddressableStorage', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('build.buildgrid.LocalContentAddressableStorage', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class LocalContentAddressableStorage(object): """Missing associated documentation comment in .proto file.""" @staticmethod def GetLocalServerDetails(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/GetLocalServerDetails', build_dot_buildgrid_dot_local__cas__pb2.GetLocalServerDetailsRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.LocalServerDetails.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def FetchMissingBlobs(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/FetchMissingBlobs', build_dot_buildgrid_dot_local__cas__pb2.FetchMissingBlobsRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.FetchMissingBlobsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def UploadMissingBlobs(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/UploadMissingBlobs', build_dot_buildgrid_dot_local__cas__pb2.UploadMissingBlobsRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.UploadMissingBlobsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def FetchTree(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/FetchTree', build_dot_buildgrid_dot_local__cas__pb2.FetchTreeRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.FetchTreeResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def UploadTree(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/UploadTree', build_dot_buildgrid_dot_local__cas__pb2.UploadTreeRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.UploadTreeResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def StageTree(request_iterator, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.stream_stream( request_iterator, target, '/build.buildgrid.LocalContentAddressableStorage/StageTree', build_dot_buildgrid_dot_local__cas__pb2.StageTreeRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.StageTreeResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def CaptureTree(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/CaptureTree', build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.CaptureTreeResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def CaptureFiles(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/CaptureFiles', build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.CaptureFilesResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def GetInstanceNameForRemote(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForRemote', build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemoteResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def GetInstanceNameForRemotes(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForRemotes', build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForRemotesResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def GetInstanceNameForNamespace(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/GetInstanceNameForNamespace', build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.GetInstanceNameForNamespaceResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def GetLocalDiskUsage(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/build.buildgrid.LocalContentAddressableStorage/GetLocalDiskUsage', build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageRequest.SerializeToString, build_dot_buildgrid_dot_local__cas__pb2.GetLocalDiskUsageResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) apache-buildstream-27ae392/src/buildstream/_protos/buildstream/000077500000000000000000000000001514607367700247245ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/buildstream/__init__.py000066400000000000000000000000001514607367700270230ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/000077500000000000000000000000001514607367700252535ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/__init__.py000066400000000000000000000000001514607367700273520ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/artifact.proto000066400000000000000000000065441514607367700301460ustar00rootroot00000000000000// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // // Authors // Raoul Hidalgo Charman syntax = "proto3"; package buildstream.v2; import "build/bazel/remote/execution/v2/remote_execution.proto"; import "google/api/annotations.proto"; message Artifact { // This version number must always be present and can be used to // further indicate presence or absence of parts of the proto at a // later date. It only needs incrementing if a change to what is // *mandatory* changes. int32 version = 1; // Core metadata bool build_success = 2; string build_error = 3; // optional string build_error_details = 4; string strong_key = 5; string weak_key = 6; bool was_workspaced = 7; // digest of a Directory build.bazel.remote.execution.v2.Digest files = 8; // Information about the build dependencies message Dependency { string project_name = 1; string element_name = 2; string cache_key = 3; bool was_workspaced = 4; }; repeated Dependency build_deps = 9; // The public data is a yaml file which is stored into the CAS // Digest is of a directory build.bazel.remote.execution.v2.Digest public_data = 10; // The logs are stored in the CAS message LogFile { string name = 1; // digest of a file build.bazel.remote.execution.v2.Digest digest = 2; }; repeated LogFile logs = 11; // Zero or more log files here // digest of a directory build.bazel.remote.execution.v2.Digest buildtree = 12; // optional // digest of a directory build.bazel.remote.execution.v2.Digest sources = 13; // optional // The low/high diversity meta fields are digests of YAML files // in which we store detatched metadata related to the artifact. // // metadata found in the low diversity file is expected to be // frequently reused in the context of a given BuildStream project, // ergo there is a very high chance that the entire content of the // file can be shared across many artifacts. // // metadata found in the high diversity file is expected to diverge // from one artifact to another. This still consists of metadata // which does not directly affect an artifact's cache key, and as // such cannot be stored directly in the proto itself. // build.bazel.remote.execution.v2.Digest low_diversity_meta = 14; build.bazel.remote.execution.v2.Digest high_diversity_meta = 15; // Strict key is a later addition to the core metadata string strict_key = 16; // digest of a directory build.bazel.remote.execution.v2.Digest buildroot = 17; // optional message SandboxState { repeated build.bazel.remote.execution.v2.Command.EnvironmentVariable environment = 1; string working_directory = 2; repeated build.bazel.remote.execution.v2.Digest subsandbox_digests = 3; repeated string marked_directories = 4; }; SandboxState buildsandbox = 18; // optional } apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/artifact_pb2.py000066400000000000000000000074021514607367700301700ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: buildstream/v2/artifact.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'buildstream/v2/artifact.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1d\x62uildstream/v2/artifact.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"\xa6\t\n\x08\x41rtifact\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x15\n\rbuild_success\x18\x02 \x01(\x08\x12\x13\n\x0b\x62uild_error\x18\x03 \x01(\t\x12\x1b\n\x13\x62uild_error_details\x18\x04 \x01(\t\x12\x12\n\nstrong_key\x18\x05 \x01(\t\x12\x10\n\x08weak_key\x18\x06 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x07 \x01(\x08\x12\x36\n\x05\x66iles\x18\x08 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x37\n\nbuild_deps\x18\t \x03(\x0b\x32#.buildstream.v2.Artifact.Dependency\x12<\n\x0bpublic_data\x18\n \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12.\n\x04logs\x18\x0b \x03(\x0b\x32 .buildstream.v2.Artifact.LogFile\x12:\n\tbuildtree\x18\x0c \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x38\n\x07sources\x18\r \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x43\n\x12low_diversity_meta\x18\x0e \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x44\n\x13high_diversity_meta\x18\x0f \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x12\n\nstrict_key\x18\x10 \x01(\t\x12:\n\tbuildroot\x18\x11 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12;\n\x0c\x62uildsandbox\x18\x12 \x01(\x0b\x32%.buildstream.v2.Artifact.SandboxState\x1a\x63\n\nDependency\x12\x14\n\x0cproject_name\x18\x01 \x01(\t\x12\x14\n\x0c\x65lement_name\x18\x02 \x01(\t\x12\x11\n\tcache_key\x18\x03 \x01(\t\x12\x16\n\x0ewas_workspaced\x18\x04 \x01(\x08\x1aP\n\x07LogFile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x37\n\x06\x64igest\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x1a\xdd\x01\n\x0cSandboxState\x12Q\n\x0b\x65nvironment\x18\x01 \x03(\x0b\x32<.build.bazel.remote.execution.v2.Command.EnvironmentVariable\x12\x19\n\x11working_directory\x18\x02 \x01(\t\x12\x43\n\x12subsandbox_digests\x18\x03 \x03(\x0b\x32\'.build.bazel.remote.execution.v2.Digest\x12\x1a\n\x12marked_directories\x18\x04 \x03(\tb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'buildstream.v2.artifact_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_ARTIFACT']._serialized_start=136 _globals['_ARTIFACT']._serialized_end=1326 _globals['_ARTIFACT_DEPENDENCY']._serialized_start=921 _globals['_ARTIFACT_DEPENDENCY']._serialized_end=1020 _globals['_ARTIFACT_LOGFILE']._serialized_start=1022 _globals['_ARTIFACT_LOGFILE']._serialized_end=1102 _globals['_ARTIFACT_SANDBOXSTATE']._serialized_start=1105 _globals['_ARTIFACT_SANDBOXSTATE']._serialized_end=1326 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/artifact_pb2.pyi000066400000000000000000000123511514607367700303400ustar00rootroot00000000000000from build.bazel.remote.execution.v2 import remote_execution_pb2 as _remote_execution_pb2 from google.api import annotations_pb2 as _annotations_pb2 from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Artifact(_message.Message): __slots__ = ("version", "build_success", "build_error", "build_error_details", "strong_key", "weak_key", "was_workspaced", "files", "build_deps", "public_data", "logs", "buildtree", "sources", "low_diversity_meta", "high_diversity_meta", "strict_key", "buildroot", "buildsandbox") class Dependency(_message.Message): __slots__ = ("project_name", "element_name", "cache_key", "was_workspaced") PROJECT_NAME_FIELD_NUMBER: _ClassVar[int] ELEMENT_NAME_FIELD_NUMBER: _ClassVar[int] CACHE_KEY_FIELD_NUMBER: _ClassVar[int] WAS_WORKSPACED_FIELD_NUMBER: _ClassVar[int] project_name: str element_name: str cache_key: str was_workspaced: bool def __init__(self, project_name: _Optional[str] = ..., element_name: _Optional[str] = ..., cache_key: _Optional[str] = ..., was_workspaced: bool = ...) -> None: ... class LogFile(_message.Message): __slots__ = ("name", "digest") NAME_FIELD_NUMBER: _ClassVar[int] DIGEST_FIELD_NUMBER: _ClassVar[int] name: str digest: _remote_execution_pb2.Digest def __init__(self, name: _Optional[str] = ..., digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... class SandboxState(_message.Message): __slots__ = ("environment", "working_directory", "subsandbox_digests", "marked_directories") ENVIRONMENT_FIELD_NUMBER: _ClassVar[int] WORKING_DIRECTORY_FIELD_NUMBER: _ClassVar[int] SUBSANDBOX_DIGESTS_FIELD_NUMBER: _ClassVar[int] MARKED_DIRECTORIES_FIELD_NUMBER: _ClassVar[int] environment: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Command.EnvironmentVariable] working_directory: str subsandbox_digests: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest] marked_directories: _containers.RepeatedScalarFieldContainer[str] def __init__(self, environment: _Optional[_Iterable[_Union[_remote_execution_pb2.Command.EnvironmentVariable, _Mapping]]] = ..., working_directory: _Optional[str] = ..., subsandbox_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., marked_directories: _Optional[_Iterable[str]] = ...) -> None: ... VERSION_FIELD_NUMBER: _ClassVar[int] BUILD_SUCCESS_FIELD_NUMBER: _ClassVar[int] BUILD_ERROR_FIELD_NUMBER: _ClassVar[int] BUILD_ERROR_DETAILS_FIELD_NUMBER: _ClassVar[int] STRONG_KEY_FIELD_NUMBER: _ClassVar[int] WEAK_KEY_FIELD_NUMBER: _ClassVar[int] WAS_WORKSPACED_FIELD_NUMBER: _ClassVar[int] FILES_FIELD_NUMBER: _ClassVar[int] BUILD_DEPS_FIELD_NUMBER: _ClassVar[int] PUBLIC_DATA_FIELD_NUMBER: _ClassVar[int] LOGS_FIELD_NUMBER: _ClassVar[int] BUILDTREE_FIELD_NUMBER: _ClassVar[int] SOURCES_FIELD_NUMBER: _ClassVar[int] LOW_DIVERSITY_META_FIELD_NUMBER: _ClassVar[int] HIGH_DIVERSITY_META_FIELD_NUMBER: _ClassVar[int] STRICT_KEY_FIELD_NUMBER: _ClassVar[int] BUILDROOT_FIELD_NUMBER: _ClassVar[int] BUILDSANDBOX_FIELD_NUMBER: _ClassVar[int] version: int build_success: bool build_error: str build_error_details: str strong_key: str weak_key: str was_workspaced: bool files: _remote_execution_pb2.Digest build_deps: _containers.RepeatedCompositeFieldContainer[Artifact.Dependency] public_data: _remote_execution_pb2.Digest logs: _containers.RepeatedCompositeFieldContainer[Artifact.LogFile] buildtree: _remote_execution_pb2.Digest sources: _remote_execution_pb2.Digest low_diversity_meta: _remote_execution_pb2.Digest high_diversity_meta: _remote_execution_pb2.Digest strict_key: str buildroot: _remote_execution_pb2.Digest buildsandbox: Artifact.SandboxState def __init__(self, version: _Optional[int] = ..., build_success: bool = ..., build_error: _Optional[str] = ..., build_error_details: _Optional[str] = ..., strong_key: _Optional[str] = ..., weak_key: _Optional[str] = ..., was_workspaced: bool = ..., files: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., build_deps: _Optional[_Iterable[_Union[Artifact.Dependency, _Mapping]]] = ..., public_data: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., logs: _Optional[_Iterable[_Union[Artifact.LogFile, _Mapping]]] = ..., buildtree: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., sources: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., low_diversity_meta: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., high_diversity_meta: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., strict_key: _Optional[str] = ..., buildroot: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ..., buildsandbox: _Optional[_Union[Artifact.SandboxState, _Mapping]] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/artifact_pb2_grpc.py000066400000000000000000000016101514607367700311760ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in buildstream/v2/artifact_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/source.proto000066400000000000000000000020571514607367700276440ustar00rootroot00000000000000// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package buildstream.v2; import "build/bazel/remote/execution/v2/remote_execution.proto"; import "google/api/annotations.proto"; message Source { // This version number must always be present and can be used to // further indicate presence or absence of parts of the proto at a // later date. It only needs incrementing if a change to what is // *mandatory* changes. int32 version = 1; // root directory digest of the files build.bazel.remote.execution.v2.Digest files = 2; } apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/source_pb2.py000066400000000000000000000033531514607367700276740ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: buildstream/v2/source.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'buildstream/v2/source.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as build_dot_bazel_dot_remote_dot_execution_dot_v2_dot_remote__execution__pb2 from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x62uildstream/v2/source.proto\x12\x0e\x62uildstream.v2\x1a\x36\x62uild/bazel/remote/execution/v2/remote_execution.proto\x1a\x1cgoogle/api/annotations.proto\"Q\n\x06Source\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x36\n\x05\x66iles\x18\x02 \x01(\x0b\x32\'.build.bazel.remote.execution.v2.Digestb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'buildstream.v2.source_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_SOURCE']._serialized_start=133 _globals['_SOURCE']._serialized_end=214 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/source_pb2.pyi000066400000000000000000000013431514607367700300420ustar00rootroot00000000000000from build.bazel.remote.execution.v2 import remote_execution_pb2 as _remote_execution_pb2 from google.api import annotations_pb2 as _annotations_pb2 from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Source(_message.Message): __slots__ = ("version", "files") VERSION_FIELD_NUMBER: _ClassVar[int] FILES_FIELD_NUMBER: _ClassVar[int] version: int files: _remote_execution_pb2.Digest def __init__(self, version: _Optional[int] = ..., files: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/buildstream/v2/source_pb2_grpc.py000066400000000000000000000016061514607367700307060ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in buildstream/v2/source_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_protos/google/000077500000000000000000000000001514607367700236655ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/__init__.py000066400000000000000000000000001514607367700257640ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/api/000077500000000000000000000000001514607367700244365ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/api/__init__.py000066400000000000000000000000001514607367700265350ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/api/annotations.proto000066400000000000000000000020331514607367700300560ustar00rootroot00000000000000// Copyright (c) 2015, Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.api; import "google/api/http.proto"; import "google/protobuf/descriptor.proto"; option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; option java_multiple_files = true; option java_outer_classname = "AnnotationsProto"; option java_package = "com.google.api"; option objc_class_prefix = "GAPI"; extend google.protobuf.MethodOptions { // See `HttpRule`. HttpRule http = 72295728; } apache-buildstream-27ae392/src/buildstream/_protos/google/api/annotations_pb2.py000066400000000000000000000034671514607367700301220ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: google/api/annotations.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'google/api/annotations.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.google.api import http_pb2 as google_dot_api_dot_http__pb2 from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cgoogle/api/annotations.proto\x12\ngoogle.api\x1a\x15google/api/http.proto\x1a google/protobuf/descriptor.proto:E\n\x04http\x12\x1e.google.protobuf.MethodOptions\x18\xb0\xca\xbc\" \x01(\x0b\x32\x14.google.api.HttpRuleBn\n\x0e\x63om.google.apiB\x10\x41nnotationsProtoP\x01ZAgoogle.golang.org/genproto/googleapis/api/annotations;annotations\xa2\x02\x04GAPIb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.api.annotations_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\016com.google.apiB\020AnnotationsProtoP\001ZAgoogle.golang.org/genproto/googleapis/api/annotations;annotations\242\002\004GAPI' # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/google/api/annotations_pb2.pyi000066400000000000000000000004661514607367700302670ustar00rootroot00000000000000from google.api import http_pb2 as _http_pb2 from google.protobuf import descriptor_pb2 as _descriptor_pb2 from google.protobuf import descriptor as _descriptor from typing import ClassVar as _ClassVar DESCRIPTOR: _descriptor.FileDescriptor HTTP_FIELD_NUMBER: _ClassVar[int] http: _descriptor.FieldDescriptor apache-buildstream-27ae392/src/buildstream/_protos/google/api/annotations_pb2_grpc.py000066400000000000000000000016071514607367700311270ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in google/api/annotations_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_protos/google/api/http.proto000066400000000000000000000271141514607367700265070ustar00rootroot00000000000000// Copyright 2018 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.api; option cc_enable_arenas = true; option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; option java_multiple_files = true; option java_outer_classname = "HttpProto"; option java_package = "com.google.api"; option objc_class_prefix = "GAPI"; // Defines the HTTP configuration for an API service. It contains a list of // [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method // to one or more HTTP REST API methods. message Http { // A list of HTTP configuration rules that apply to individual API methods. // // **NOTE:** All service configuration rules follow "last one wins" order. repeated HttpRule rules = 1; // When set to true, URL path parmeters will be fully URI-decoded except in // cases of single segment matches in reserved expansion, where "%2F" will be // left encoded. // // The default behavior is to not decode RFC 6570 reserved characters in multi // segment matches. bool fully_decode_reserved_expansion = 2; } // `HttpRule` defines the mapping of an RPC method to one or more HTTP // REST API methods. The mapping specifies how different portions of the RPC // request message are mapped to URL path, URL query parameters, and // HTTP request body. The mapping is typically specified as an // `google.api.http` annotation on the RPC method, // see "google/api/annotations.proto" for details. // // The mapping consists of a field specifying the path template and // method kind. The path template can refer to fields in the request // message, as in the example below which describes a REST GET // operation on a resource collection of messages: // // // service Messaging { // rpc GetMessage(GetMessageRequest) returns (Message) { // option (google.api.http).get = "/v1/messages/{message_id}/{sub.subfield}"; // } // } // message GetMessageRequest { // message SubMessage { // string subfield = 1; // } // string message_id = 1; // mapped to the URL // SubMessage sub = 2; // `sub.subfield` is url-mapped // } // message Message { // string text = 1; // content of the resource // } // // The same http annotation can alternatively be expressed inside the // `GRPC API Configuration` YAML file. // // http: // rules: // - selector: .Messaging.GetMessage // get: /v1/messages/{message_id}/{sub.subfield} // // This definition enables an automatic, bidrectional mapping of HTTP // JSON to RPC. Example: // // HTTP | RPC // -----|----- // `GET /v1/messages/123456/foo` | `GetMessage(message_id: "123456" sub: SubMessage(subfield: "foo"))` // // In general, not only fields but also field paths can be referenced // from a path pattern. Fields mapped to the path pattern cannot be // repeated and must have a primitive (non-message) type. // // Any fields in the request message which are not bound by the path // pattern automatically become (optional) HTTP query // parameters. Assume the following definition of the request message: // // // service Messaging { // rpc GetMessage(GetMessageRequest) returns (Message) { // option (google.api.http).get = "/v1/messages/{message_id}"; // } // } // message GetMessageRequest { // message SubMessage { // string subfield = 1; // } // string message_id = 1; // mapped to the URL // int64 revision = 2; // becomes a parameter // SubMessage sub = 3; // `sub.subfield` becomes a parameter // } // // // This enables a HTTP JSON to RPC mapping as below: // // HTTP | RPC // -----|----- // `GET /v1/messages/123456?revision=2&sub.subfield=foo` | `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: "foo"))` // // Note that fields which are mapped to HTTP parameters must have a // primitive type or a repeated primitive type. Message types are not // allowed. In the case of a repeated type, the parameter can be // repeated in the URL, as in `...?param=A¶m=B`. // // For HTTP method kinds which allow a request body, the `body` field // specifies the mapping. Consider a REST update method on the // message resource collection: // // // service Messaging { // rpc UpdateMessage(UpdateMessageRequest) returns (Message) { // option (google.api.http) = { // put: "/v1/messages/{message_id}" // body: "message" // }; // } // } // message UpdateMessageRequest { // string message_id = 1; // mapped to the URL // Message message = 2; // mapped to the body // } // // // The following HTTP JSON to RPC mapping is enabled, where the // representation of the JSON in the request body is determined by // protos JSON encoding: // // HTTP | RPC // -----|----- // `PUT /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: "123456" message { text: "Hi!" })` // // The special name `*` can be used in the body mapping to define that // every field not bound by the path template should be mapped to the // request body. This enables the following alternative definition of // the update method: // // service Messaging { // rpc UpdateMessage(Message) returns (Message) { // option (google.api.http) = { // put: "/v1/messages/{message_id}" // body: "*" // }; // } // } // message Message { // string message_id = 1; // string text = 2; // } // // // The following HTTP JSON to RPC mapping is enabled: // // HTTP | RPC // -----|----- // `PUT /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: "123456" text: "Hi!")` // // Note that when using `*` in the body mapping, it is not possible to // have HTTP parameters, as all fields not bound by the path end in // the body. This makes this option more rarely used in practice of // defining REST APIs. The common usage of `*` is in custom methods // which don't use the URL at all for transferring data. // // It is possible to define multiple HTTP methods for one RPC by using // the `additional_bindings` option. Example: // // service Messaging { // rpc GetMessage(GetMessageRequest) returns (Message) { // option (google.api.http) = { // get: "/v1/messages/{message_id}" // additional_bindings { // get: "/v1/users/{user_id}/messages/{message_id}" // } // }; // } // } // message GetMessageRequest { // string message_id = 1; // string user_id = 2; // } // // // This enables the following two alternative HTTP JSON to RPC // mappings: // // HTTP | RPC // -----|----- // `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` // `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: "123456")` // // # Rules for HTTP mapping // // The rules for mapping HTTP path, query parameters, and body fields // to the request message are as follows: // // 1. The `body` field specifies either `*` or a field path, or is // omitted. If omitted, it indicates there is no HTTP request body. // 2. Leaf fields (recursive expansion of nested messages in the // request) can be classified into three types: // (a) Matched in the URL template. // (b) Covered by body (if body is `*`, everything except (a) fields; // else everything under the body field) // (c) All other fields. // 3. URL query parameters found in the HTTP request are mapped to (c) fields. // 4. Any body sent with an HTTP request can contain only (b) fields. // // The syntax of the path template is as follows: // // Template = "/" Segments [ Verb ] ; // Segments = Segment { "/" Segment } ; // Segment = "*" | "**" | LITERAL | Variable ; // Variable = "{" FieldPath [ "=" Segments ] "}" ; // FieldPath = IDENT { "." IDENT } ; // Verb = ":" LITERAL ; // // The syntax `*` matches a single path segment. The syntax `**` matches zero // or more path segments, which must be the last part of the path except the // `Verb`. The syntax `LITERAL` matches literal text in the path. // // The syntax `Variable` matches part of the URL path as specified by its // template. A variable template must not contain other variables. If a variable // matches a single path segment, its template may be omitted, e.g. `{var}` // is equivalent to `{var=*}`. // // If a variable contains exactly one path segment, such as `"{var}"` or // `"{var=*}"`, when such a variable is expanded into a URL path, all characters // except `[-_.~0-9a-zA-Z]` are percent-encoded. Such variables show up in the // Discovery Document as `{var}`. // // If a variable contains one or more path segments, such as `"{var=foo/*}"` // or `"{var=**}"`, when such a variable is expanded into a URL path, all // characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. Such variables // show up in the Discovery Document as `{+var}`. // // NOTE: While the single segment variable matches the semantics of // [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 // Simple String Expansion, the multi segment variable **does not** match // RFC 6570 Reserved Expansion. The reason is that the Reserved Expansion // does not expand special characters like `?` and `#`, which would lead // to invalid URLs. // // NOTE: the field paths in variables and in the `body` must not refer to // repeated fields or map fields. message HttpRule { // Selects methods to which this rule applies. // // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. string selector = 1; // Determines the URL pattern is matched by this rules. This pattern can be // used with any of the {get|put|post|delete|patch} methods. A custom method // can be defined using the 'custom' field. oneof pattern { // Used for listing and getting information about resources. string get = 2; // Used for updating a resource. string put = 3; // Used for creating a resource. string post = 4; // Used for deleting a resource. string delete = 5; // Used for updating a resource. string patch = 6; // The custom pattern is used for specifying an HTTP method that is not // included in the `pattern` field, such as HEAD, or "*" to leave the // HTTP method unspecified for this rule. The wild-card rule is useful // for services that provide content to Web (HTML) clients. CustomHttpPattern custom = 8; } // The name of the request field whose value is mapped to the HTTP body, or // `*` for mapping all fields not captured by the path pattern to the HTTP // body. NOTE: the referred field must not be a repeated field and must be // present at the top-level of request message type. string body = 7; // Additional HTTP bindings for the selector. Nested bindings must // not contain an `additional_bindings` field themselves (that is, // the nesting may only be one level deep). repeated HttpRule additional_bindings = 11; } // A custom pattern is used for defining custom HTTP verb. message CustomHttpPattern { // The name of this custom HTTP verb. string kind = 1; // The path matched by this custom verb. string path = 2; } apache-buildstream-27ae392/src/buildstream/_protos/google/api/http_pb2.py000066400000000000000000000046541514607367700265430ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: google/api/http.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'google/api/http.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15google/api/http.proto\x12\ngoogle.api\"T\n\x04Http\x12#\n\x05rules\x18\x01 \x03(\x0b\x32\x14.google.api.HttpRule\x12\'\n\x1f\x66ully_decode_reserved_expansion\x18\x02 \x01(\x08\"\xea\x01\n\x08HttpRule\x12\x10\n\x08selector\x18\x01 \x01(\t\x12\r\n\x03get\x18\x02 \x01(\tH\x00\x12\r\n\x03put\x18\x03 \x01(\tH\x00\x12\x0e\n\x04post\x18\x04 \x01(\tH\x00\x12\x10\n\x06\x64\x65lete\x18\x05 \x01(\tH\x00\x12\x0f\n\x05patch\x18\x06 \x01(\tH\x00\x12/\n\x06\x63ustom\x18\x08 \x01(\x0b\x32\x1d.google.api.CustomHttpPatternH\x00\x12\x0c\n\x04\x62ody\x18\x07 \x01(\t\x12\x31\n\x13\x61\x64\x64itional_bindings\x18\x0b \x03(\x0b\x32\x14.google.api.HttpRuleB\t\n\x07pattern\"/\n\x11\x43ustomHttpPattern\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\tBj\n\x0e\x63om.google.apiB\tHttpProtoP\x01ZAgoogle.golang.org/genproto/googleapis/api/annotations;annotations\xf8\x01\x01\xa2\x02\x04GAPIb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.api.http_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\016com.google.apiB\tHttpProtoP\001ZAgoogle.golang.org/genproto/googleapis/api/annotations;annotations\370\001\001\242\002\004GAPI' _globals['_HTTP']._serialized_start=37 _globals['_HTTP']._serialized_end=121 _globals['_HTTPRULE']._serialized_start=124 _globals['_HTTPRULE']._serialized_end=358 _globals['_CUSTOMHTTPPATTERN']._serialized_start=360 _globals['_CUSTOMHTTPPATTERN']._serialized_end=407 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/google/api/http_pb2.pyi000066400000000000000000000041441514607367700267060ustar00rootroot00000000000000from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Http(_message.Message): __slots__ = ("rules", "fully_decode_reserved_expansion") RULES_FIELD_NUMBER: _ClassVar[int] FULLY_DECODE_RESERVED_EXPANSION_FIELD_NUMBER: _ClassVar[int] rules: _containers.RepeatedCompositeFieldContainer[HttpRule] fully_decode_reserved_expansion: bool def __init__(self, rules: _Optional[_Iterable[_Union[HttpRule, _Mapping]]] = ..., fully_decode_reserved_expansion: bool = ...) -> None: ... class HttpRule(_message.Message): __slots__ = ("selector", "get", "put", "post", "delete", "patch", "custom", "body", "additional_bindings") SELECTOR_FIELD_NUMBER: _ClassVar[int] GET_FIELD_NUMBER: _ClassVar[int] PUT_FIELD_NUMBER: _ClassVar[int] POST_FIELD_NUMBER: _ClassVar[int] DELETE_FIELD_NUMBER: _ClassVar[int] PATCH_FIELD_NUMBER: _ClassVar[int] CUSTOM_FIELD_NUMBER: _ClassVar[int] BODY_FIELD_NUMBER: _ClassVar[int] ADDITIONAL_BINDINGS_FIELD_NUMBER: _ClassVar[int] selector: str get: str put: str post: str delete: str patch: str custom: CustomHttpPattern body: str additional_bindings: _containers.RepeatedCompositeFieldContainer[HttpRule] def __init__(self, selector: _Optional[str] = ..., get: _Optional[str] = ..., put: _Optional[str] = ..., post: _Optional[str] = ..., delete: _Optional[str] = ..., patch: _Optional[str] = ..., custom: _Optional[_Union[CustomHttpPattern, _Mapping]] = ..., body: _Optional[str] = ..., additional_bindings: _Optional[_Iterable[_Union[HttpRule, _Mapping]]] = ...) -> None: ... class CustomHttpPattern(_message.Message): __slots__ = ("kind", "path") KIND_FIELD_NUMBER: _ClassVar[int] PATH_FIELD_NUMBER: _ClassVar[int] kind: str path: str def __init__(self, kind: _Optional[str] = ..., path: _Optional[str] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/google/api/http_pb2_grpc.py000066400000000000000000000016001514607367700275420ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in google/api/http_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_protos/google/bytestream/000077500000000000000000000000001514607367700260445ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/bytestream/__init__.py000066400000000000000000000000001514607367700301430ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/bytestream/bytestream.proto000066400000000000000000000166611514607367700313220ustar00rootroot00000000000000// Copyright 2016 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.bytestream; import "google/api/annotations.proto"; import "google/protobuf/wrappers.proto"; option go_package = "google.golang.org/genproto/googleapis/bytestream;bytestream"; option java_outer_classname = "ByteStreamProto"; option java_package = "com.google.bytestream"; // #### Introduction // // The Byte Stream API enables a client to read and write a stream of bytes to // and from a resource. Resources have names, and these names are supplied in // the API calls below to identify the resource that is being read from or // written to. // // All implementations of the Byte Stream API export the interface defined here: // // * `Read()`: Reads the contents of a resource. // // * `Write()`: Writes the contents of a resource. The client can call `Write()` // multiple times with the same resource and can check the status of the write // by calling `QueryWriteStatus()`. // // #### Service parameters and metadata // // The ByteStream API provides no direct way to access/modify any metadata // associated with the resource. // // #### Errors // // The errors returned by the service are in the Google canonical error space. service ByteStream { // `Read()` is used to retrieve the contents of a resource as a sequence // of bytes. The bytes are returned in a sequence of responses, and the // responses are delivered as the results of a server-side streaming RPC. rpc Read(ReadRequest) returns (stream ReadResponse); // `Write()` is used to send the contents of a resource as a sequence of // bytes. The bytes are sent in a sequence of request protos of a client-side // streaming RPC. // // A `Write()` action is resumable. If there is an error or the connection is // broken during the `Write()`, the client should check the status of the // `Write()` by calling `QueryWriteStatus()` and continue writing from the // returned `committed_size`. This may be less than the amount of data the // client previously sent. // // Calling `Write()` on a resource name that was previously written and // finalized could cause an error, depending on whether the underlying service // allows over-writing of previously written resources. // // When the client closes the request channel, the service will respond with // a `WriteResponse`. The service will not view the resource as `complete` // until the client has sent a `WriteRequest` with `finish_write` set to // `true`. Sending any requests on a stream after sending a request with // `finish_write` set to `true` will cause an error. The client **should** // check the `WriteResponse` it receives to determine how much data the // service was able to commit and whether the service views the resource as // `complete` or not. rpc Write(stream WriteRequest) returns (WriteResponse); // `QueryWriteStatus()` is used to find the `committed_size` for a resource // that is being written, which can then be used as the `write_offset` for // the next `Write()` call. // // If the resource does not exist (i.e., the resource has been deleted, or the // first `Write()` has not yet reached the service), this method returns the // error `NOT_FOUND`. // // The client **may** call `QueryWriteStatus()` at any time to determine how // much data has been processed for this resource. This is useful if the // client is buffering data and needs to know which data can be safely // evicted. For any sequence of `QueryWriteStatus()` calls for a given // resource name, the sequence of returned `committed_size` values will be // non-decreasing. rpc QueryWriteStatus(QueryWriteStatusRequest) returns (QueryWriteStatusResponse); } // Request object for ByteStream.Read. message ReadRequest { // The name of the resource to read. string resource_name = 1; // The offset for the first byte to return in the read, relative to the start // of the resource. // // A `read_offset` that is negative or greater than the size of the resource // will cause an `OUT_OF_RANGE` error. int64 read_offset = 2; // The maximum number of `data` bytes the server is allowed to return in the // sum of all `ReadResponse` messages. A `read_limit` of zero indicates that // there is no limit, and a negative `read_limit` will cause an error. // // If the stream returns fewer bytes than allowed by the `read_limit` and no // error occurred, the stream includes all data from the `read_offset` to the // end of the resource. int64 read_limit = 3; } // Response object for ByteStream.Read. message ReadResponse { // A portion of the data for the resource. The service **may** leave `data` // empty for any given `ReadResponse`. This enables the service to inform the // client that the request is still live while it is running an operation to // generate more data. bytes data = 10; } // Request object for ByteStream.Write. message WriteRequest { // The name of the resource to write. This **must** be set on the first // `WriteRequest` of each `Write()` action. If it is set on subsequent calls, // it **must** match the value of the first request. string resource_name = 1; // The offset from the beginning of the resource at which the data should be // written. It is required on all `WriteRequest`s. // // In the first `WriteRequest` of a `Write()` action, it indicates // the initial offset for the `Write()` call. The value **must** be equal to // the `committed_size` that a call to `QueryWriteStatus()` would return. // // On subsequent calls, this value **must** be set and **must** be equal to // the sum of the first `write_offset` and the sizes of all `data` bundles // sent previously on this stream. // // An incorrect value will cause an error. int64 write_offset = 2; // If `true`, this indicates that the write is complete. Sending any // `WriteRequest`s subsequent to one in which `finish_write` is `true` will // cause an error. bool finish_write = 3; // A portion of the data for the resource. The client **may** leave `data` // empty for any given `WriteRequest`. This enables the client to inform the // service that the request is still live while it is running an operation to // generate more data. bytes data = 10; } // Response object for ByteStream.Write. message WriteResponse { // The number of bytes that have been processed for the given resource. int64 committed_size = 1; } // Request object for ByteStream.QueryWriteStatus. message QueryWriteStatusRequest { // The name of the resource whose write status is being requested. string resource_name = 1; } // Response object for ByteStream.QueryWriteStatus. message QueryWriteStatusResponse { // The number of bytes that have been processed for the given resource. int64 committed_size = 1; // `complete` is `true` only if the client has sent a `WriteRequest` with // `finish_write` set to true, and the server has processed that request. bool complete = 2; } apache-buildstream-27ae392/src/buildstream/_protos/google/bytestream/bytestream_pb2.py000066400000000000000000000066371514607367700313540ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: google/bytestream/bytestream.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'google/bytestream/bytestream.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from google.protobuf import wrappers_pb2 as google_dot_protobuf_dot_wrappers__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\"google/bytestream/bytestream.proto\x12\x11google.bytestream\x1a\x1cgoogle/api/annotations.proto\x1a\x1egoogle/protobuf/wrappers.proto\"M\n\x0bReadRequest\x12\x15\n\rresource_name\x18\x01 \x01(\t\x12\x13\n\x0bread_offset\x18\x02 \x01(\x03\x12\x12\n\nread_limit\x18\x03 \x01(\x03\"\x1c\n\x0cReadResponse\x12\x0c\n\x04\x64\x61ta\x18\n \x01(\x0c\"_\n\x0cWriteRequest\x12\x15\n\rresource_name\x18\x01 \x01(\t\x12\x14\n\x0cwrite_offset\x18\x02 \x01(\x03\x12\x14\n\x0c\x66inish_write\x18\x03 \x01(\x08\x12\x0c\n\x04\x64\x61ta\x18\n \x01(\x0c\"\'\n\rWriteResponse\x12\x16\n\x0e\x63ommitted_size\x18\x01 \x01(\x03\"0\n\x17QueryWriteStatusRequest\x12\x15\n\rresource_name\x18\x01 \x01(\t\"D\n\x18QueryWriteStatusResponse\x12\x16\n\x0e\x63ommitted_size\x18\x01 \x01(\x03\x12\x10\n\x08\x63omplete\x18\x02 \x01(\x08\x32\x92\x02\n\nByteStream\x12I\n\x04Read\x12\x1e.google.bytestream.ReadRequest\x1a\x1f.google.bytestream.ReadResponse0\x01\x12L\n\x05Write\x12\x1f.google.bytestream.WriteRequest\x1a .google.bytestream.WriteResponse(\x01\x12k\n\x10QueryWriteStatus\x12*.google.bytestream.QueryWriteStatusRequest\x1a+.google.bytestream.QueryWriteStatusResponseBe\n\x15\x63om.google.bytestreamB\x0f\x42yteStreamProtoZ;google.golang.org/genproto/googleapis/bytestream;bytestreamb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.bytestream.bytestream_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\025com.google.bytestreamB\017ByteStreamProtoZ;google.golang.org/genproto/googleapis/bytestream;bytestream' _globals['_READREQUEST']._serialized_start=119 _globals['_READREQUEST']._serialized_end=196 _globals['_READRESPONSE']._serialized_start=198 _globals['_READRESPONSE']._serialized_end=226 _globals['_WRITEREQUEST']._serialized_start=228 _globals['_WRITEREQUEST']._serialized_end=323 _globals['_WRITERESPONSE']._serialized_start=325 _globals['_WRITERESPONSE']._serialized_end=364 _globals['_QUERYWRITESTATUSREQUEST']._serialized_start=366 _globals['_QUERYWRITESTATUSREQUEST']._serialized_end=414 _globals['_QUERYWRITESTATUSRESPONSE']._serialized_start=416 _globals['_QUERYWRITESTATUSRESPONSE']._serialized_end=484 _globals['_BYTESTREAM']._serialized_start=487 _globals['_BYTESTREAM']._serialized_end=761 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/google/bytestream/bytestream_pb2.pyi000066400000000000000000000043331514607367700315140ustar00rootroot00000000000000from google.api import annotations_pb2 as _annotations_pb2 from google.protobuf import wrappers_pb2 as _wrappers_pb2 from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Optional as _Optional DESCRIPTOR: _descriptor.FileDescriptor class ReadRequest(_message.Message): __slots__ = ("resource_name", "read_offset", "read_limit") RESOURCE_NAME_FIELD_NUMBER: _ClassVar[int] READ_OFFSET_FIELD_NUMBER: _ClassVar[int] READ_LIMIT_FIELD_NUMBER: _ClassVar[int] resource_name: str read_offset: int read_limit: int def __init__(self, resource_name: _Optional[str] = ..., read_offset: _Optional[int] = ..., read_limit: _Optional[int] = ...) -> None: ... class ReadResponse(_message.Message): __slots__ = ("data",) DATA_FIELD_NUMBER: _ClassVar[int] data: bytes def __init__(self, data: _Optional[bytes] = ...) -> None: ... class WriteRequest(_message.Message): __slots__ = ("resource_name", "write_offset", "finish_write", "data") RESOURCE_NAME_FIELD_NUMBER: _ClassVar[int] WRITE_OFFSET_FIELD_NUMBER: _ClassVar[int] FINISH_WRITE_FIELD_NUMBER: _ClassVar[int] DATA_FIELD_NUMBER: _ClassVar[int] resource_name: str write_offset: int finish_write: bool data: bytes def __init__(self, resource_name: _Optional[str] = ..., write_offset: _Optional[int] = ..., finish_write: bool = ..., data: _Optional[bytes] = ...) -> None: ... class WriteResponse(_message.Message): __slots__ = ("committed_size",) COMMITTED_SIZE_FIELD_NUMBER: _ClassVar[int] committed_size: int def __init__(self, committed_size: _Optional[int] = ...) -> None: ... class QueryWriteStatusRequest(_message.Message): __slots__ = ("resource_name",) RESOURCE_NAME_FIELD_NUMBER: _ClassVar[int] resource_name: str def __init__(self, resource_name: _Optional[str] = ...) -> None: ... class QueryWriteStatusResponse(_message.Message): __slots__ = ("committed_size", "complete") COMMITTED_SIZE_FIELD_NUMBER: _ClassVar[int] COMPLETE_FIELD_NUMBER: _ClassVar[int] committed_size: int complete: bool def __init__(self, committed_size: _Optional[int] = ..., complete: bool = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/google/bytestream/bytestream_pb2_grpc.py000066400000000000000000000274661514607367700323720ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings from buildstream._protos.google.bytestream import bytestream_pb2 as google_dot_bytestream_dot_bytestream__pb2 GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in google/bytestream/bytestream_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) class ByteStreamStub(object): """#### Introduction The Byte Stream API enables a client to read and write a stream of bytes to and from a resource. Resources have names, and these names are supplied in the API calls below to identify the resource that is being read from or written to. All implementations of the Byte Stream API export the interface defined here: * `Read()`: Reads the contents of a resource. * `Write()`: Writes the contents of a resource. The client can call `Write()` multiple times with the same resource and can check the status of the write by calling `QueryWriteStatus()`. #### Service parameters and metadata The ByteStream API provides no direct way to access/modify any metadata associated with the resource. #### Errors The errors returned by the service are in the Google canonical error space. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.Read = channel.unary_stream( '/google.bytestream.ByteStream/Read', request_serializer=google_dot_bytestream_dot_bytestream__pb2.ReadRequest.SerializeToString, response_deserializer=google_dot_bytestream_dot_bytestream__pb2.ReadResponse.FromString, _registered_method=True) self.Write = channel.stream_unary( '/google.bytestream.ByteStream/Write', request_serializer=google_dot_bytestream_dot_bytestream__pb2.WriteRequest.SerializeToString, response_deserializer=google_dot_bytestream_dot_bytestream__pb2.WriteResponse.FromString, _registered_method=True) self.QueryWriteStatus = channel.unary_unary( '/google.bytestream.ByteStream/QueryWriteStatus', request_serializer=google_dot_bytestream_dot_bytestream__pb2.QueryWriteStatusRequest.SerializeToString, response_deserializer=google_dot_bytestream_dot_bytestream__pb2.QueryWriteStatusResponse.FromString, _registered_method=True) class ByteStreamServicer(object): """#### Introduction The Byte Stream API enables a client to read and write a stream of bytes to and from a resource. Resources have names, and these names are supplied in the API calls below to identify the resource that is being read from or written to. All implementations of the Byte Stream API export the interface defined here: * `Read()`: Reads the contents of a resource. * `Write()`: Writes the contents of a resource. The client can call `Write()` multiple times with the same resource and can check the status of the write by calling `QueryWriteStatus()`. #### Service parameters and metadata The ByteStream API provides no direct way to access/modify any metadata associated with the resource. #### Errors The errors returned by the service are in the Google canonical error space. """ def Read(self, request, context): """`Read()` is used to retrieve the contents of a resource as a sequence of bytes. The bytes are returned in a sequence of responses, and the responses are delivered as the results of a server-side streaming RPC. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def Write(self, request_iterator, context): """`Write()` is used to send the contents of a resource as a sequence of bytes. The bytes are sent in a sequence of request protos of a client-side streaming RPC. A `Write()` action is resumable. If there is an error or the connection is broken during the `Write()`, the client should check the status of the `Write()` by calling `QueryWriteStatus()` and continue writing from the returned `committed_size`. This may be less than the amount of data the client previously sent. Calling `Write()` on a resource name that was previously written and finalized could cause an error, depending on whether the underlying service allows over-writing of previously written resources. When the client closes the request channel, the service will respond with a `WriteResponse`. The service will not view the resource as `complete` until the client has sent a `WriteRequest` with `finish_write` set to `true`. Sending any requests on a stream after sending a request with `finish_write` set to `true` will cause an error. The client **should** check the `WriteResponse` it receives to determine how much data the service was able to commit and whether the service views the resource as `complete` or not. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def QueryWriteStatus(self, request, context): """`QueryWriteStatus()` is used to find the `committed_size` for a resource that is being written, which can then be used as the `write_offset` for the next `Write()` call. If the resource does not exist (i.e., the resource has been deleted, or the first `Write()` has not yet reached the service), this method returns the error `NOT_FOUND`. The client **may** call `QueryWriteStatus()` at any time to determine how much data has been processed for this resource. This is useful if the client is buffering data and needs to know which data can be safely evicted. For any sequence of `QueryWriteStatus()` calls for a given resource name, the sequence of returned `committed_size` values will be non-decreasing. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_ByteStreamServicer_to_server(servicer, server): rpc_method_handlers = { 'Read': grpc.unary_stream_rpc_method_handler( servicer.Read, request_deserializer=google_dot_bytestream_dot_bytestream__pb2.ReadRequest.FromString, response_serializer=google_dot_bytestream_dot_bytestream__pb2.ReadResponse.SerializeToString, ), 'Write': grpc.stream_unary_rpc_method_handler( servicer.Write, request_deserializer=google_dot_bytestream_dot_bytestream__pb2.WriteRequest.FromString, response_serializer=google_dot_bytestream_dot_bytestream__pb2.WriteResponse.SerializeToString, ), 'QueryWriteStatus': grpc.unary_unary_rpc_method_handler( servicer.QueryWriteStatus, request_deserializer=google_dot_bytestream_dot_bytestream__pb2.QueryWriteStatusRequest.FromString, response_serializer=google_dot_bytestream_dot_bytestream__pb2.QueryWriteStatusResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'google.bytestream.ByteStream', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('google.bytestream.ByteStream', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class ByteStream(object): """#### Introduction The Byte Stream API enables a client to read and write a stream of bytes to and from a resource. Resources have names, and these names are supplied in the API calls below to identify the resource that is being read from or written to. All implementations of the Byte Stream API export the interface defined here: * `Read()`: Reads the contents of a resource. * `Write()`: Writes the contents of a resource. The client can call `Write()` multiple times with the same resource and can check the status of the write by calling `QueryWriteStatus()`. #### Service parameters and metadata The ByteStream API provides no direct way to access/modify any metadata associated with the resource. #### Errors The errors returned by the service are in the Google canonical error space. """ @staticmethod def Read(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_stream( request, target, '/google.bytestream.ByteStream/Read', google_dot_bytestream_dot_bytestream__pb2.ReadRequest.SerializeToString, google_dot_bytestream_dot_bytestream__pb2.ReadResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def Write(request_iterator, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.stream_unary( request_iterator, target, '/google.bytestream.ByteStream/Write', google_dot_bytestream_dot_bytestream__pb2.WriteRequest.SerializeToString, google_dot_bytestream_dot_bytestream__pb2.WriteResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def QueryWriteStatus(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/google.bytestream.ByteStream/QueryWriteStatus', google_dot_bytestream_dot_bytestream__pb2.QueryWriteStatusRequest.SerializeToString, google_dot_bytestream_dot_bytestream__pb2.QueryWriteStatusResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) apache-buildstream-27ae392/src/buildstream/_protos/google/longrunning/000077500000000000000000000000001514607367700262255ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/longrunning/__init__.py000066400000000000000000000000001514607367700303240ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/longrunning/operations.proto000066400000000000000000000154571514607367700315110ustar00rootroot00000000000000// Copyright 2016 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.longrunning; import "google/api/annotations.proto"; import "google/protobuf/any.proto"; import "google/protobuf/empty.proto"; import "google/rpc/status.proto"; option csharp_namespace = "Google.LongRunning"; option go_package = "google.golang.org/genproto/googleapis/longrunning;longrunning"; option java_multiple_files = true; option java_outer_classname = "OperationsProto"; option java_package = "com.google.longrunning"; option php_namespace = "Google\\LongRunning"; // Manages long-running operations with an API service. // // When an API method normally takes long time to complete, it can be designed // to return [Operation][google.longrunning.Operation] to the client, and the client can use this // interface to receive the real response asynchronously by polling the // operation resource, or pass the operation resource to another API (such as // Google Cloud Pub/Sub API) to receive the response. Any API service that // returns long-running operations should implement the `Operations` interface // so developers can have a consistent client experience. service Operations { // Lists operations that match the specified filter in the request. If the // server doesn't support this method, it returns `UNIMPLEMENTED`. // // NOTE: the `name` binding below allows API services to override the binding // to use different resource name schemes, such as `users/*/operations`. rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { option (google.api.http) = { get: "/v1/{name=operations}" }; } // Gets the latest state of a long-running operation. Clients can use this // method to poll the operation result at intervals as recommended by the API // service. rpc GetOperation(GetOperationRequest) returns (Operation) { option (google.api.http) = { get: "/v1/{name=operations/**}" }; } // Deletes a long-running operation. This method indicates that the client is // no longer interested in the operation result. It does not cancel the // operation. If the server doesn't support this method, it returns // `google.rpc.Code.UNIMPLEMENTED`. rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v1/{name=operations/**}" }; } // Starts asynchronous cancellation on a long-running operation. The server // makes a best effort to cancel the operation, but success is not // guaranteed. If the server doesn't support this method, it returns // `google.rpc.Code.UNIMPLEMENTED`. Clients can use // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or // other methods to check whether the cancellation succeeded or whether the // operation completed despite cancellation. On successful cancellation, // the operation is not deleted; instead, it becomes an operation with // an [Operation.error][google.longrunning.Operation.error] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1, // corresponding to `Code.CANCELLED`. rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { option (google.api.http) = { post: "/v1/{name=operations/**}:cancel" body: "*" }; } } // This resource represents a long-running operation that is the result of a // network API call. message Operation { // The server-assigned name, which is only unique within the same service that // originally returns it. If you use the default HTTP mapping, the // `name` should have the format of `operations/some/unique/name`. string name = 1; // Service-specific metadata associated with the operation. It typically // contains progress information and common metadata such as create time. // Some services might not provide such metadata. Any method that returns a // long-running operation should document the metadata type, if any. google.protobuf.Any metadata = 2; // If the value is `false`, it means the operation is still in progress. // If true, the operation is completed, and either `error` or `response` is // available. bool done = 3; // The operation result, which can be either an `error` or a valid `response`. // If `done` == `false`, neither `error` nor `response` is set. // If `done` == `true`, exactly one of `error` or `response` is set. oneof result { // The error result of the operation in case of failure or cancellation. google.rpc.Status error = 4; // The normal response of the operation in case of success. If the original // method returns no data on success, such as `Delete`, the response is // `google.protobuf.Empty`. If the original method is standard // `Get`/`Create`/`Update`, the response should be the resource. For other // methods, the response should have the type `XxxResponse`, where `Xxx` // is the original method name. For example, if the original method name // is `TakeSnapshot()`, the inferred response type is // `TakeSnapshotResponse`. google.protobuf.Any response = 5; } } // The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation]. message GetOperationRequest { // The name of the operation resource. string name = 1; } // The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. message ListOperationsRequest { // The name of the operation collection. string name = 4; // The standard list filter. string filter = 1; // The standard list page size. int32 page_size = 2; // The standard list page token. string page_token = 3; } // The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. message ListOperationsResponse { // A list of operations that matches the specified filter in the request. repeated Operation operations = 1; // The standard List next-page token. string next_page_token = 2; } // The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. message CancelOperationRequest { // The name of the operation resource to be cancelled. string name = 1; } // The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. message DeleteOperationRequest { // The name of the operation resource to be deleted. string name = 1; } apache-buildstream-27ae392/src/buildstream/_protos/google/longrunning/operations_pb2.py000066400000000000000000000123541514607367700315320ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: google/longrunning/operations.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'google/longrunning/operations.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from buildstream._protos.google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2 from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 from buildstream._protos.google.rpc import status_pb2 as google_dot_rpc_dot_status__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#google/longrunning/operations.proto\x12\x12google.longrunning\x1a\x1cgoogle/api/annotations.proto\x1a\x19google/protobuf/any.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x17google/rpc/status.proto\"\xa8\x01\n\tOperation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x08metadata\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12#\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x12.google.rpc.StatusH\x00\x12(\n\x08response\x18\x05 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00\x42\x08\n\x06result\"#\n\x13GetOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\\\n\x15ListOperationsRequest\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"d\n\x16ListOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"&\n\x16\x43\x61ncelOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n\x16\x44\x65leteOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t2\x8c\x04\n\nOperations\x12\x86\x01\n\x0eListOperations\x12).google.longrunning.ListOperationsRequest\x1a*.google.longrunning.ListOperationsResponse\"\x1d\x82\xd3\xe4\x93\x02\x17\x12\x15/v1/{name=operations}\x12x\n\x0cGetOperation\x12\'.google.longrunning.GetOperationRequest\x1a\x1d.google.longrunning.Operation\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/v1/{name=operations/**}\x12w\n\x0f\x44\x65leteOperation\x12*.google.longrunning.DeleteOperationRequest\x1a\x16.google.protobuf.Empty\" \x82\xd3\xe4\x93\x02\x1a*\x18/v1/{name=operations/**}\x12\x81\x01\n\x0f\x43\x61ncelOperation\x12*.google.longrunning.CancelOperationRequest\x1a\x16.google.protobuf.Empty\"*\x82\xd3\xe4\x93\x02$\"\x1f/v1/{name=operations/**}:cancel:\x01*B\x94\x01\n\x16\x63om.google.longrunningB\x0fOperationsProtoP\x01Z=google.golang.org/genproto/googleapis/longrunning;longrunning\xaa\x02\x12Google.LongRunning\xca\x02\x12Google\\LongRunningb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.longrunning.operations_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\026com.google.longrunningB\017OperationsProtoP\001Z=google.golang.org/genproto/googleapis/longrunning;longrunning\252\002\022Google.LongRunning\312\002\022Google\\LongRunning' _globals['_OPERATIONS'].methods_by_name['ListOperations']._loaded_options = None _globals['_OPERATIONS'].methods_by_name['ListOperations']._serialized_options = b'\202\323\344\223\002\027\022\025/v1/{name=operations}' _globals['_OPERATIONS'].methods_by_name['GetOperation']._loaded_options = None _globals['_OPERATIONS'].methods_by_name['GetOperation']._serialized_options = b'\202\323\344\223\002\032\022\030/v1/{name=operations/**}' _globals['_OPERATIONS'].methods_by_name['DeleteOperation']._loaded_options = None _globals['_OPERATIONS'].methods_by_name['DeleteOperation']._serialized_options = b'\202\323\344\223\002\032*\030/v1/{name=operations/**}' _globals['_OPERATIONS'].methods_by_name['CancelOperation']._loaded_options = None _globals['_OPERATIONS'].methods_by_name['CancelOperation']._serialized_options = b'\202\323\344\223\002$\"\037/v1/{name=operations/**}:cancel:\001*' _globals['_OPERATION']._serialized_start=171 _globals['_OPERATION']._serialized_end=339 _globals['_GETOPERATIONREQUEST']._serialized_start=341 _globals['_GETOPERATIONREQUEST']._serialized_end=376 _globals['_LISTOPERATIONSREQUEST']._serialized_start=378 _globals['_LISTOPERATIONSREQUEST']._serialized_end=470 _globals['_LISTOPERATIONSRESPONSE']._serialized_start=472 _globals['_LISTOPERATIONSRESPONSE']._serialized_end=572 _globals['_CANCELOPERATIONREQUEST']._serialized_start=574 _globals['_CANCELOPERATIONREQUEST']._serialized_end=612 _globals['_DELETEOPERATIONREQUEST']._serialized_start=614 _globals['_DELETEOPERATIONREQUEST']._serialized_end=652 _globals['_OPERATIONS']._serialized_start=655 _globals['_OPERATIONS']._serialized_end=1179 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/google/longrunning/operations_pb2.pyi000066400000000000000000000052411514607367700317000ustar00rootroot00000000000000from google.api import annotations_pb2 as _annotations_pb2 from google.protobuf import any_pb2 as _any_pb2 from google.protobuf import empty_pb2 as _empty_pb2 from google.rpc import status_pb2 as _status_pb2 from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Operation(_message.Message): __slots__ = ("name", "metadata", "done", "error", "response") NAME_FIELD_NUMBER: _ClassVar[int] METADATA_FIELD_NUMBER: _ClassVar[int] DONE_FIELD_NUMBER: _ClassVar[int] ERROR_FIELD_NUMBER: _ClassVar[int] RESPONSE_FIELD_NUMBER: _ClassVar[int] name: str metadata: _any_pb2.Any done: bool error: _status_pb2.Status response: _any_pb2.Any def __init__(self, name: _Optional[str] = ..., metadata: _Optional[_Union[_any_pb2.Any, _Mapping]] = ..., done: bool = ..., error: _Optional[_Union[_status_pb2.Status, _Mapping]] = ..., response: _Optional[_Union[_any_pb2.Any, _Mapping]] = ...) -> None: ... class GetOperationRequest(_message.Message): __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] name: str def __init__(self, name: _Optional[str] = ...) -> None: ... class ListOperationsRequest(_message.Message): __slots__ = ("name", "filter", "page_size", "page_token") NAME_FIELD_NUMBER: _ClassVar[int] FILTER_FIELD_NUMBER: _ClassVar[int] PAGE_SIZE_FIELD_NUMBER: _ClassVar[int] PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] name: str filter: str page_size: int page_token: str def __init__(self, name: _Optional[str] = ..., filter: _Optional[str] = ..., page_size: _Optional[int] = ..., page_token: _Optional[str] = ...) -> None: ... class ListOperationsResponse(_message.Message): __slots__ = ("operations", "next_page_token") OPERATIONS_FIELD_NUMBER: _ClassVar[int] NEXT_PAGE_TOKEN_FIELD_NUMBER: _ClassVar[int] operations: _containers.RepeatedCompositeFieldContainer[Operation] next_page_token: str def __init__(self, operations: _Optional[_Iterable[_Union[Operation, _Mapping]]] = ..., next_page_token: _Optional[str] = ...) -> None: ... class CancelOperationRequest(_message.Message): __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] name: str def __init__(self, name: _Optional[str] = ...) -> None: ... class DeleteOperationRequest(_message.Message): __slots__ = ("name",) NAME_FIELD_NUMBER: _ClassVar[int] name: str def __init__(self, name: _Optional[str] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/google/longrunning/operations_pb2_grpc.py000066400000000000000000000302731514607367700325450ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings from buildstream._protos.google.longrunning import operations_pb2 as google_dot_longrunning_dot_operations__pb2 from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in google/longrunning/operations_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) class OperationsStub(object): """Manages long-running operations with an API service. When an API method normally takes long time to complete, it can be designed to return [Operation][google.longrunning.Operation] to the client, and the client can use this interface to receive the real response asynchronously by polling the operation resource, or pass the operation resource to another API (such as Google Cloud Pub/Sub API) to receive the response. Any API service that returns long-running operations should implement the `Operations` interface so developers can have a consistent client experience. """ def __init__(self, channel): """Constructor. Args: channel: A grpc.Channel. """ self.ListOperations = channel.unary_unary( '/google.longrunning.Operations/ListOperations', request_serializer=google_dot_longrunning_dot_operations__pb2.ListOperationsRequest.SerializeToString, response_deserializer=google_dot_longrunning_dot_operations__pb2.ListOperationsResponse.FromString, _registered_method=True) self.GetOperation = channel.unary_unary( '/google.longrunning.Operations/GetOperation', request_serializer=google_dot_longrunning_dot_operations__pb2.GetOperationRequest.SerializeToString, response_deserializer=google_dot_longrunning_dot_operations__pb2.Operation.FromString, _registered_method=True) self.DeleteOperation = channel.unary_unary( '/google.longrunning.Operations/DeleteOperation', request_serializer=google_dot_longrunning_dot_operations__pb2.DeleteOperationRequest.SerializeToString, response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, _registered_method=True) self.CancelOperation = channel.unary_unary( '/google.longrunning.Operations/CancelOperation', request_serializer=google_dot_longrunning_dot_operations__pb2.CancelOperationRequest.SerializeToString, response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, _registered_method=True) class OperationsServicer(object): """Manages long-running operations with an API service. When an API method normally takes long time to complete, it can be designed to return [Operation][google.longrunning.Operation] to the client, and the client can use this interface to receive the real response asynchronously by polling the operation resource, or pass the operation resource to another API (such as Google Cloud Pub/Sub API) to receive the response. Any API service that returns long-running operations should implement the `Operations` interface so developers can have a consistent client experience. """ def ListOperations(self, request, context): """Lists operations that match the specified filter in the request. If the server doesn't support this method, it returns `UNIMPLEMENTED`. NOTE: the `name` binding below allows API services to override the binding to use different resource name schemes, such as `users/*/operations`. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def GetOperation(self, request, context): """Gets the latest state of a long-running operation. Clients can use this method to poll the operation result at intervals as recommended by the API service. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def DeleteOperation(self, request, context): """Deletes a long-running operation. This method indicates that the client is no longer interested in the operation result. It does not cancel the operation. If the server doesn't support this method, it returns `google.rpc.Code.UNIMPLEMENTED`. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def CancelOperation(self, request, context): """Starts asynchronous cancellation on a long-running operation. The server makes a best effort to cancel the operation, but success is not guaranteed. If the server doesn't support this method, it returns `google.rpc.Code.UNIMPLEMENTED`. Clients can use [Operations.GetOperation][google.longrunning.Operations.GetOperation] or other methods to check whether the cancellation succeeded or whether the operation completed despite cancellation. On successful cancellation, the operation is not deleted; instead, it becomes an operation with an [Operation.error][google.longrunning.Operation.error] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1, corresponding to `Code.CANCELLED`. """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') def add_OperationsServicer_to_server(servicer, server): rpc_method_handlers = { 'ListOperations': grpc.unary_unary_rpc_method_handler( servicer.ListOperations, request_deserializer=google_dot_longrunning_dot_operations__pb2.ListOperationsRequest.FromString, response_serializer=google_dot_longrunning_dot_operations__pb2.ListOperationsResponse.SerializeToString, ), 'GetOperation': grpc.unary_unary_rpc_method_handler( servicer.GetOperation, request_deserializer=google_dot_longrunning_dot_operations__pb2.GetOperationRequest.FromString, response_serializer=google_dot_longrunning_dot_operations__pb2.Operation.SerializeToString, ), 'DeleteOperation': grpc.unary_unary_rpc_method_handler( servicer.DeleteOperation, request_deserializer=google_dot_longrunning_dot_operations__pb2.DeleteOperationRequest.FromString, response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, ), 'CancelOperation': grpc.unary_unary_rpc_method_handler( servicer.CancelOperation, request_deserializer=google_dot_longrunning_dot_operations__pb2.CancelOperationRequest.FromString, response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( 'google.longrunning.Operations', rpc_method_handlers) server.add_generic_rpc_handlers((generic_handler,)) server.add_registered_method_handlers('google.longrunning.Operations', rpc_method_handlers) # This class is part of an EXPERIMENTAL API. class Operations(object): """Manages long-running operations with an API service. When an API method normally takes long time to complete, it can be designed to return [Operation][google.longrunning.Operation] to the client, and the client can use this interface to receive the real response asynchronously by polling the operation resource, or pass the operation resource to another API (such as Google Cloud Pub/Sub API) to receive the response. Any API service that returns long-running operations should implement the `Operations` interface so developers can have a consistent client experience. """ @staticmethod def ListOperations(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/google.longrunning.Operations/ListOperations', google_dot_longrunning_dot_operations__pb2.ListOperationsRequest.SerializeToString, google_dot_longrunning_dot_operations__pb2.ListOperationsResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def GetOperation(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/google.longrunning.Operations/GetOperation', google_dot_longrunning_dot_operations__pb2.GetOperationRequest.SerializeToString, google_dot_longrunning_dot_operations__pb2.Operation.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def DeleteOperation(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/google.longrunning.Operations/DeleteOperation', google_dot_longrunning_dot_operations__pb2.DeleteOperationRequest.SerializeToString, google_dot_protobuf_dot_empty__pb2.Empty.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) @staticmethod def CancelOperation(request, target, options=(), channel_credentials=None, call_credentials=None, insecure=False, compression=None, wait_for_ready=None, timeout=None, metadata=None): return grpc.experimental.unary_unary( request, target, '/google.longrunning.Operations/CancelOperation', google_dot_longrunning_dot_operations__pb2.CancelOperationRequest.SerializeToString, google_dot_protobuf_dot_empty__pb2.Empty.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata, _registered_method=True) apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/000077500000000000000000000000001514607367700244515ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/__init__.py000066400000000000000000000000001514607367700265500ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/code.proto000066400000000000000000000156171514607367700264620ustar00rootroot00000000000000// Copyright 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.rpc; option go_package = "google.golang.org/genproto/googleapis/rpc/code;code"; option java_multiple_files = true; option java_outer_classname = "CodeProto"; option java_package = "com.google.rpc"; option objc_class_prefix = "RPC"; // The canonical error codes for Google APIs. // // // Sometimes multiple error codes may apply. Services should return // the most specific error code that applies. For example, prefer // `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply. // Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`. enum Code { // Not an error; returned on success // // HTTP Mapping: 200 OK OK = 0; // The operation was cancelled, typically by the caller. // // HTTP Mapping: 499 Client Closed Request CANCELLED = 1; // Unknown error. For example, this error may be returned when // a `Status` value received from another address space belongs to // an error space that is not known in this address space. Also // errors raised by APIs that do not return enough error information // may be converted to this error. // // HTTP Mapping: 500 Internal Server Error UNKNOWN = 2; // The client specified an invalid argument. Note that this differs // from `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments // that are problematic regardless of the state of the system // (e.g., a malformed file name). // // HTTP Mapping: 400 Bad Request INVALID_ARGUMENT = 3; // The deadline expired before the operation could complete. For operations // that change the state of the system, this error may be returned // even if the operation has completed successfully. For example, a // successful response from a server could have been delayed long // enough for the deadline to expire. // // HTTP Mapping: 504 Gateway Timeout DEADLINE_EXCEEDED = 4; // Some requested entity (e.g., file or directory) was not found. // // Note to server developers: if a request is denied for an entire class // of users, such as gradual feature rollout or undocumented whitelist, // `NOT_FOUND` may be used. If a request is denied for some users within // a class of users, such as user-based access control, `PERMISSION_DENIED` // must be used. // // HTTP Mapping: 404 Not Found NOT_FOUND = 5; // The entity that a client attempted to create (e.g., file or directory) // already exists. // // HTTP Mapping: 409 Conflict ALREADY_EXISTS = 6; // The caller does not have permission to execute the specified // operation. `PERMISSION_DENIED` must not be used for rejections // caused by exhausting some resource (use `RESOURCE_EXHAUSTED` // instead for those errors). `PERMISSION_DENIED` must not be // used if the caller can not be identified (use `UNAUTHENTICATED` // instead for those errors). This error code does not imply the // request is valid or the requested entity exists or satisfies // other pre-conditions. // // HTTP Mapping: 403 Forbidden PERMISSION_DENIED = 7; // The request does not have valid authentication credentials for the // operation. // // HTTP Mapping: 401 Unauthorized UNAUTHENTICATED = 16; // Some resource has been exhausted, perhaps a per-user quota, or // perhaps the entire file system is out of space. // // HTTP Mapping: 429 Too Many Requests RESOURCE_EXHAUSTED = 8; // The operation was rejected because the system is not in a state // required for the operation's execution. For example, the directory // to be deleted is non-empty, an rmdir operation is applied to // a non-directory, etc. // // Service implementors can use the following guidelines to decide // between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`: // (a) Use `UNAVAILABLE` if the client can retry just the failing call. // (b) Use `ABORTED` if the client should retry at a higher level // (e.g., when a client-specified test-and-set fails, indicating the // client should restart a read-modify-write sequence). // (c) Use `FAILED_PRECONDITION` if the client should not retry until // the system state has been explicitly fixed. E.g., if an "rmdir" // fails because the directory is non-empty, `FAILED_PRECONDITION` // should be returned since the client should not retry unless // the files are deleted from the directory. // // HTTP Mapping: 400 Bad Request FAILED_PRECONDITION = 9; // The operation was aborted, typically due to a concurrency issue such as // a sequencer check failure or transaction abort. // // See the guidelines above for deciding between `FAILED_PRECONDITION`, // `ABORTED`, and `UNAVAILABLE`. // // HTTP Mapping: 409 Conflict ABORTED = 10; // The operation was attempted past the valid range. E.g., seeking or // reading past end-of-file. // // Unlike `INVALID_ARGUMENT`, this error indicates a problem that may // be fixed if the system state changes. For example, a 32-bit file // system will generate `INVALID_ARGUMENT` if asked to read at an // offset that is not in the range [0,2^32-1], but it will generate // `OUT_OF_RANGE` if asked to read from an offset past the current // file size. // // There is a fair bit of overlap between `FAILED_PRECONDITION` and // `OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific // error) when it applies so that callers who are iterating through // a space can easily look for an `OUT_OF_RANGE` error to detect when // they are done. // // HTTP Mapping: 400 Bad Request OUT_OF_RANGE = 11; // The operation is not implemented or is not supported/enabled in this // service. // // HTTP Mapping: 501 Not Implemented UNIMPLEMENTED = 12; // Internal errors. This means that some invariants expected by the // underlying system have been broken. This error code is reserved // for serious errors. // // HTTP Mapping: 500 Internal Server Error INTERNAL = 13; // The service is currently unavailable. This is most likely a // transient condition, which can be corrected by retrying with // a backoff. // // See the guidelines above for deciding between `FAILED_PRECONDITION`, // `ABORTED`, and `UNAVAILABLE`. // // HTTP Mapping: 503 Service Unavailable UNAVAILABLE = 14; // Unrecoverable data loss or corruption. // // HTTP Mapping: 500 Internal Server Error DATA_LOSS = 15; }apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/code_pb2.py000066400000000000000000000041041514607367700264770ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: google/rpc/code.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'google/rpc/code.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15google/rpc/code.proto\x12\ngoogle.rpc*\xb7\x02\n\x04\x43ode\x12\x06\n\x02OK\x10\x00\x12\r\n\tCANCELLED\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x14\n\x10INVALID_ARGUMENT\x10\x03\x12\x15\n\x11\x44\x45\x41\x44LINE_EXCEEDED\x10\x04\x12\r\n\tNOT_FOUND\x10\x05\x12\x12\n\x0e\x41LREADY_EXISTS\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x13\n\x0fUNAUTHENTICATED\x10\x10\x12\x16\n\x12RESOURCE_EXHAUSTED\x10\x08\x12\x17\n\x13\x46\x41ILED_PRECONDITION\x10\t\x12\x0b\n\x07\x41\x42ORTED\x10\n\x12\x10\n\x0cOUT_OF_RANGE\x10\x0b\x12\x11\n\rUNIMPLEMENTED\x10\x0c\x12\x0c\n\x08INTERNAL\x10\r\x12\x0f\n\x0bUNAVAILABLE\x10\x0e\x12\r\n\tDATA_LOSS\x10\x0f\x42X\n\x0e\x63om.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPCb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.rpc.code_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\016com.google.rpcB\tCodeProtoP\001Z3google.golang.org/genproto/googleapis/rpc/code;code\242\002\003RPC' _globals['_CODE']._serialized_start=38 _globals['_CODE']._serialized_end=349 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/code_pb2.pyi000066400000000000000000000022521514607367700266520ustar00rootroot00000000000000from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper from google.protobuf import descriptor as _descriptor from typing import ClassVar as _ClassVar DESCRIPTOR: _descriptor.FileDescriptor class Code(int, metaclass=_enum_type_wrapper.EnumTypeWrapper): __slots__ = () OK: _ClassVar[Code] CANCELLED: _ClassVar[Code] UNKNOWN: _ClassVar[Code] INVALID_ARGUMENT: _ClassVar[Code] DEADLINE_EXCEEDED: _ClassVar[Code] NOT_FOUND: _ClassVar[Code] ALREADY_EXISTS: _ClassVar[Code] PERMISSION_DENIED: _ClassVar[Code] UNAUTHENTICATED: _ClassVar[Code] RESOURCE_EXHAUSTED: _ClassVar[Code] FAILED_PRECONDITION: _ClassVar[Code] ABORTED: _ClassVar[Code] OUT_OF_RANGE: _ClassVar[Code] UNIMPLEMENTED: _ClassVar[Code] INTERNAL: _ClassVar[Code] UNAVAILABLE: _ClassVar[Code] DATA_LOSS: _ClassVar[Code] OK: Code CANCELLED: Code UNKNOWN: Code INVALID_ARGUMENT: Code DEADLINE_EXCEEDED: Code NOT_FOUND: Code ALREADY_EXISTS: Code PERMISSION_DENIED: Code UNAUTHENTICATED: Code RESOURCE_EXHAUSTED: Code FAILED_PRECONDITION: Code ABORTED: Code OUT_OF_RANGE: Code UNIMPLEMENTED: Code INTERNAL: Code UNAVAILABLE: Code DATA_LOSS: Code apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/code_pb2_grpc.py000066400000000000000000000016001514607367700275100ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in google/rpc/code_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/status.proto000066400000000000000000000077171514607367700270750ustar00rootroot00000000000000// Copyright 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. syntax = "proto3"; package google.rpc; import "google/protobuf/any.proto"; option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; option java_multiple_files = true; option java_outer_classname = "StatusProto"; option java_package = "com.google.rpc"; option objc_class_prefix = "RPC"; // The `Status` type defines a logical error model that is suitable for different // programming environments, including REST APIs and RPC APIs. It is used by // [gRPC](https://github.com/grpc). The error model is designed to be: // // - Simple to use and understand for most users // - Flexible enough to meet unexpected needs // // # Overview // // The `Status` message contains three pieces of data: error code, error message, // and error details. The error code should be an enum value of // [google.rpc.Code][google.rpc.Code], but it may accept additional error codes if needed. The // error message should be a developer-facing English message that helps // developers *understand* and *resolve* the error. If a localized user-facing // error message is needed, put the localized message in the error details or // localize it in the client. The optional error details may contain arbitrary // information about the error. There is a predefined set of error detail types // in the package `google.rpc` that can be used for common error conditions. // // # Language mapping // // The `Status` message is the logical representation of the error model, but it // is not necessarily the actual wire format. When the `Status` message is // exposed in different client libraries and different wire protocols, it can be // mapped differently. For example, it will likely be mapped to some exceptions // in Java, but more likely mapped to some error codes in C. // // # Other uses // // The error model and the `Status` message can be used in a variety of // environments, either with or without APIs, to provide a // consistent developer experience across different environments. // // Example uses of this error model include: // // - Partial errors. If a service needs to return partial errors to the client, // it may embed the `Status` in the normal response to indicate the partial // errors. // // - Workflow errors. A typical workflow has multiple steps. Each step may // have a `Status` message for error reporting. // // - Batch operations. If a client uses batch request and batch response, the // `Status` message should be used directly inside batch response, one for // each error sub-response. // // - Asynchronous operations. If an API call embeds asynchronous operation // results in its response, the status of those operations should be // represented directly using the `Status` message. // // - Logging. If some API errors are stored in logs, the message `Status` could // be used directly after any stripping needed for security/privacy reasons. message Status { // The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]. int32 code = 1; // A developer-facing error message, which should be in English. Any // user-facing error message should be localized and sent in the // [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client. string message = 2; // A list of messages that carry the error details. There is a common set of // message types for APIs to use. repeated google.protobuf.Any details = 3; } apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/status_pb2.py000066400000000000000000000034111514607367700271100ustar00rootroot00000000000000# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: google/rpc/status.proto # Protobuf Python Version: 5.29.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, 29, 0, '', 'google/rpc/status.proto' ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17google/rpc/status.proto\x12\ngoogle.rpc\x1a\x19google/protobuf/any.proto\"N\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\x12%\n\x07\x64\x65tails\x18\x03 \x03(\x0b\x32\x14.google.protobuf.AnyB^\n\x0e\x63om.google.rpcB\x0bStatusProtoP\x01Z7google.golang.org/genproto/googleapis/rpc/status;status\xa2\x02\x03RPCb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.rpc.status_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: _globals['DESCRIPTOR']._loaded_options = None _globals['DESCRIPTOR']._serialized_options = b'\n\016com.google.rpcB\013StatusProtoP\001Z7google.golang.org/genproto/googleapis/rpc/status;status\242\002\003RPC' _globals['_STATUS']._serialized_start=66 _globals['_STATUS']._serialized_end=144 # @@protoc_insertion_point(module_scope) apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/status_pb2.pyi000066400000000000000000000015141514607367700272630ustar00rootroot00000000000000from google.protobuf import any_pb2 as _any_pb2 from google.protobuf.internal import containers as _containers from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union DESCRIPTOR: _descriptor.FileDescriptor class Status(_message.Message): __slots__ = ("code", "message", "details") CODE_FIELD_NUMBER: _ClassVar[int] MESSAGE_FIELD_NUMBER: _ClassVar[int] DETAILS_FIELD_NUMBER: _ClassVar[int] code: int message: str details: _containers.RepeatedCompositeFieldContainer[_any_pb2.Any] def __init__(self, code: _Optional[int] = ..., message: _Optional[str] = ..., details: _Optional[_Iterable[_Union[_any_pb2.Any, _Mapping]]] = ...) -> None: ... apache-buildstream-27ae392/src/buildstream/_protos/google/rpc/status_pb2_grpc.py000066400000000000000000000016021514607367700301230ustar00rootroot00000000000000# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! """Client and server classes corresponding to protobuf-defined services.""" import grpc import warnings GRPC_GENERATED_VERSION = '1.69.0' GRPC_VERSION = grpc.__version__ _version_not_supported = False try: from grpc._utilities import first_version_is_lower _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) except ImportError: _version_not_supported = True if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' + f' but the generated code in google/rpc/status_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' ) apache-buildstream-27ae392/src/buildstream/_remote.py000066400000000000000000000072051514607367700227340ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import threading import grpc from ._exceptions import ImplError, RemoteError # BaseRemote(): # # Provides the basic functionality required to set up remote # interaction via GRPC. In particular, this will set up a # grpc.insecure_channel, or a grpc.secure_channel, based on the given # spec. # # Customization for the particular protocol is expected to be # performed in children. # class BaseRemote: key_name = None def __init__(self, spec): self.spec = spec self._initialized = False self._lock = threading.Lock() #################################################### # Dunder methods # #################################################### def __enter__(self): return self def __exit__(self, _exc_type, _exc_value, traceback): return False def __str__(self): if self.spec: return self.spec.url else: return "(default remote)" #################################################### # Remote API # #################################################### # init(): # # Initialize the given remote. This function must be called before # any communication is performed, since such will otherwise fail. # def init(self): with self._lock: if self._initialized: return self._configure_protocols() self._initialized = True # check(): # # Check if the remote is functional and has all the required # capabilities. This should be used somewhat like an assertion, # expecting a RemoteError. # # Note that this method runs the calls on a separate process, so # that we can use grpc calls even if we are on the main process. # # Raises: # RemoteError: If the grpc call fails. # def check(self): try: self.init() self._check() except grpc.RpcError as e: # str(e) is too verbose for errors reported to the user raise RemoteError("{}: {}".format(e.code().name, e.details())) #################################################### # Abstract methods # #################################################### # _check(): # # Check if this remote provides everything required for the # particular kind of remote. This is expected to be called as part # of check(), and must be called in a non-main process. # # Raises: # RemoteError: when the remote isn't compatible or another error happened. # def _check(self): pass # _configure_protocols(): # # An abstract method to configure remote-specific protocols. This # is *not* done as super().init() because we want to be able to # set self._initialized *after* initialization completes in the # parent class. # # This method should *never* be called outside of init(). # def _configure_protocols(self): raise ImplError("An implementation of a Remote must configure its protocols.") apache-buildstream-27ae392/src/buildstream/_remotespec.py000066400000000000000000000476321514607367700236170ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from typing import Optional, Tuple, List, cast from ._exceptions import LoadError, RemoteError from .exceptions import LoadErrorReason from .types import FastEnum from .node import MappingNode # RemoteType(): # # Defines the different types of remote. # class RemoteType(FastEnum): INDEX = "index" STORAGE = "storage" ENDPOINT = "endpoint" ALL = "all" def __str__(self) -> str: if self.name: return self.name.lower().replace("_", "-") return "" # RemoteSpecPurpose(): # # What a RemoteSpec is going to be used for. # # This is currently only used to control the behavior # of RemoteSpec.new_from_string(), after that, a RemoteSpec # has a `push` attribute which is either True or False. # class RemoteSpecPurpose(FastEnum): ALL = 0 # Pushing and pulling PUSH = 1 # Only pushing PULL = 2 # Only pulling # RemoteSpec(): # # This data structure holds all of the details required to # connect to and communicate with the various grpc remote # services, like the artifact cache, source cache and remote # execution service. # class RemoteSpec: def __init__( self, remote_type: str, url: str, *, push: bool = False, server_cert: Optional[str] = None, client_key: Optional[str] = None, client_cert: Optional[str] = None, access_token: Optional[str] = None, access_token_reload_interval: Optional[int] = None, instance_name: Optional[str] = None, connection_config: Optional[MappingNode] = None, spec_node: Optional[MappingNode] = None, ) -> None: # # Public members # # The remote type self.remote_type: str = remote_type # Whether we are allowed to push (for asset caches only) self.push: bool = push # The url of the remote, this may contain a port number self.url: str = url # The name of the grpc service to talk to at this remote url self.instance_name: Optional[str] = instance_name # The credentials self.server_cert_file: Optional[str] = server_cert self.client_key_file: Optional[str] = client_key self.client_cert_file: Optional[str] = client_cert self.access_token_file: Optional[str] = access_token self.access_token_reload_interval: Optional[int] = access_token_reload_interval # # Private members # # The provenance node for error reporting self._spec_node: Optional[MappingNode] = spec_node # The credentials loaded from disk, and whether they were loaded self._server_cert: Optional[bytes] = None self._client_key: Optional[bytes] = None self._client_cert: Optional[bytes] = None self._cred_files_loaded: bool = False # Various connection parameters for grpc connection self._connection_config: Optional[MappingNode] = connection_config # # Implement dunder methods to support hashing and # comparisons. # def __eq__(self, other: object) -> bool: return hash(self) == hash(other) def __hash__(self) -> int: return hash( ( self.remote_type, self.push, self.url, self.instance_name, self.server_cert_file, self.client_key_file, self.client_cert_file, self.access_token_file, self.access_token_reload_interval, self.keepalive_time, self.retry_limit, self.retry_delay, self.request_timeout, ) ) def __str__(self) -> str: string = self.url + "\n" string += "push: {} type: {} instance: {}\n".format(self.push, self.remote_type, self.instance_name) if self._spec_node: provenance = str(self._spec_node.get_provenance()) else: provenance = "command line" string += "loaded from: {}".format(provenance) return string # server_cert() # @property def server_cert(self) -> Optional[bytes]: self._load_credential_files() return self._server_cert # client_key() # @property def client_key(self) -> Optional[bytes]: self._load_credential_files() return self._client_key # client_cert() # @property def client_cert(self) -> Optional[bytes]: self._load_credential_files() return self._client_cert # grpc keepalive time # @property def keepalive_time(self) -> Optional[int]: if self._connection_config: return self._connection_config.get_int("keepalive-time", None) return None # grpc retry limit # @property def retry_limit(self) -> Optional[int]: if self._connection_config: return self._connection_config.get_int("retry-limit", None) return None # grpc retry delay in milliseconds # @property def retry_delay(self) -> Optional[int]: if self._connection_config: return self._connection_config.get_int("retry-delay", None) return None # grpc request timeout in seconds # @property def request_timeout(self) -> Optional[int]: if self._connection_config: return self._connection_config.get_int("request-timeout", None) return None # to_localcas_remote() # # Create a `LocalContentAddressableStorage.Remote` proto from the `RemoteSpec` object. # def to_localcas_remote(self, remote): remote.url = self.url if self.instance_name: remote.instance_name = self.instance_name if self.server_cert: remote.server_cert = self.server_cert if self.client_key: remote.client_key = self.client_key if self.client_cert: remote.client_cert = self.client_cert if self.access_token_file: remote.access_token_path = self.access_token_file if self.access_token_reload_interval is not None: remote.access_token_reload_interval.FromSeconds(self.access_token_reload_interval * 60) if self.keepalive_time is not None: remote.keepalive_time.FromSeconds(self.keepalive_time) if self.retry_limit is not None: remote.retry_limit = self.retry_limit if self.retry_delay is not None: remote.retry_delay.FromMilliseconds(self.retry_delay) if self.request_timeout is not None: remote.request_timeout.FromSeconds(self.request_timeout) # new_from_node(): # # Creates a RemoteSpec() from a YAML loaded node. # # Args: # spec_node: The configuration node describing the spec. # basedir: The base directory from which to find certificates. # remote_execution: Whether this spec is used for remote execution (some keys are invalid) # action_cache: Whether this spec is used for remote execution action cache # # Returns: # The described RemoteSpec instance. # # Raises: # LoadError: If the node is malformed. # @classmethod def new_from_node( cls, spec_node: MappingNode, basedir: Optional[str] = None, *, remote_execution: bool = False, action_cache: bool = False, ) -> "RemoteSpec": server_cert: Optional[str] = None client_key: Optional[str] = None client_cert: Optional[str] = None access_token: Optional[str] = None access_token_reload_interval: Optional[int] = None push: bool = False remote_type: str = RemoteType.ENDPOINT valid_keys: List[str] = ["url", "instance-name", "auth", "connection-config"] if not remote_execution: remote_type = cast(str, spec_node.get_enum("type", RemoteType, default=RemoteType.ALL)) valid_keys += ["type"] if not remote_execution or action_cache: push = spec_node.get_bool("push", default=False) valid_keys += ["push"] spec_node.validate_keys(valid_keys) # FIXME: This explicit error message should not be necessary, instead # we should be able to inform Node.get_str() that an empty string # is not acceptable, and have Node do the work of raising this error. # url = spec_node.get_str("url") if not url: provenance = spec_node.get_node("url").get_provenance() raise LoadError("{}: empty artifact cache URL".format(provenance), LoadErrorReason.INVALID_DATA) instance_name = spec_node.get_str("instance-name", default=None) auth_node = spec_node.get_mapping("auth", None) if auth_node: server_cert, client_key, client_cert, access_token, access_token_reload_interval = cls._parse_auth( auth_node, basedir ) connection_config = spec_node.get_mapping("connection-config", None) return cls( remote_type, url, push=push, server_cert=server_cert, client_key=client_key, client_cert=client_cert, access_token=access_token, access_token_reload_interval=access_token_reload_interval, instance_name=instance_name, connection_config=connection_config, spec_node=spec_node, ) # new_from_string(): # # Creates a RemoteSpec() from a string, used to parse CLI parameters # # If certificates are passed, they are interpreted as relative to the # current working directory. # # Args: # string: The user provided string # purpose: The purpose this RemoteSpec is intended for (RemoteSpecPurpose) # # Returns: # The described RemoteSpec instance. # # Raises: # RemoteError: In case parsing the string fails # @classmethod def new_from_string(cls, string: str, purpose: int = RemoteSpecPurpose.ALL) -> "RemoteSpec": url: Optional[str] = None instance_name: Optional[str] = None remote_type: str = RemoteType.ALL push: bool = True server_cert: Optional[str] = None client_key: Optional[str] = None client_cert: Optional[str] = None access_token: Optional[str] = None if purpose == RemoteSpecPurpose.PULL: push = False split = string.split(",") if len(split) > 1: for split_string in split: subsplit = split_string.split("=") if len(subsplit) != 2: raise RemoteError( "Invalid format '{}' found in remote specification: {}".format(split_string, string) ) key: str = subsplit[0] val: str = subsplit[1] if key == "url": url = val elif key == "instance-name": instance_name = val elif key == "type": remote_type = cast(str, RemoteType(val)) allowed_types = [RemoteType.INDEX, RemoteType.STORAGE, RemoteType.ALL] if remote_type not in allowed_types: raise RemoteError( "Value for remote 'type' must be one of: {}".format( ", ".join([str(_type) for _type in allowed_types]) ) ) elif key == "push": # Provide a sensible error for `bst artifact push --remote url=http://pony.com,push=False ...` if purpose != RemoteSpecPurpose.ALL: raise RemoteError("The 'push' key is invalid and assumed to be {}".format(push)) if val in ("True", "true"): push = True elif val in ("False", "false"): push = False else: raise RemoteError("Value for 'push' must be 'True' or 'False'") elif key == "server-cert": server_cert = cls._resolve_path(val, os.getcwd()) elif key == "client-key": client_key = cls._resolve_path(val, os.getcwd()) elif key == "client-cert": client_cert = cls._resolve_path(val, os.getcwd()) elif key == "access-token": access_token = cls._resolve_path(val, os.getcwd()) else: raise RemoteError("Unexpected key '{}' encountered".format(key)) else: # No commas, only the URL was specified url = string if not url: raise RemoteError("No URL specified in remote") return cls( remote_type, url, push=push, server_cert=server_cert, client_key=client_key, client_cert=client_cert, access_token=access_token, instance_name=instance_name, ) # _resolve_path() # # Resolve a path relative to the base directory # # Args: # path: The path # basedir: The base directory # # Returns: # The resolved path # @classmethod def _resolve_path(cls, path: str, basedir: Optional[str]) -> str: path = os.path.expanduser(path) if basedir: path = os.path.join(basedir, path) return path # _parse_auth(): # # Parse the "auth" data # # Args: # auth_node: The auth node # basedir: The base directory which cert files are relative to, or None # # Returns: # A 5 tuple containing the filenames for the server-cert, the client-key, # the client-cert, the access-token and the access-token-reload-interval # @classmethod def _parse_auth( cls, auth_node: MappingNode, basedir: Optional[str] = None ) -> Tuple[Optional[str], Optional[str], Optional[str], Optional[str], Optional[int]]: auth_path_keys = ["server-cert", "client-key", "client-cert", "access-token"] auth_int_keys = ["access-token-reload-interval"] auth_values = {} auth_int_values = {} auth_node.validate_keys(auth_path_keys + auth_int_keys) for key in auth_path_keys: value = auth_node.get_str(key, None) if value: value = cls._resolve_path(value, basedir) auth_values[key] = value for key in auth_int_keys: auth_int_values[key] = auth_node.get_int(key, None) server_cert = auth_values["server-cert"] client_key = auth_values["client-key"] client_cert = auth_values["client-cert"] access_token = auth_values["access-token"] access_token_reload_interval = auth_int_values["access-token-reload-interval"] if client_key and not client_cert: provenance = auth_node.get_node("client-key").get_provenance() raise LoadError( "{}: 'client-key' was specified without 'client-cert'".format(provenance), LoadErrorReason.INVALID_DATA ) if client_cert and not client_key: provenance = auth_node.get_node("client-cert").get_provenance() raise LoadError( "{}: 'client-cert' was specified without 'client-key'".format(provenance), LoadErrorReason.INVALID_DATA ) return server_cert, client_key, client_cert, access_token, access_token_reload_interval # _load_credential_files(): # # A helper method to load the credentials files, ignoring any input # arguments that are None. # def _load_credential_files(self) -> None: def maybe_read_file(filename: Optional[str]) -> Optional[bytes]: if filename: try: with open(filename, "rb") as f: return f.read() except IOError as e: message = "Failed to load credentials file: {}".format(filename) if self._spec_node: message = "{}: {}".format(self._spec_node.get_provenance(), message) raise RemoteError(message, detail=str(e), reason="load-remote-creds-failed") from e return None if not self._cred_files_loaded: self._server_cert = maybe_read_file(self.server_cert_file) self._client_key = maybe_read_file(self.client_key_file) self._client_cert = maybe_read_file(self.client_cert_file) self._cred_files_loaded = True # RemoteExecutionSpec(): # # This data structure holds all of the details required to # connect to a remote execution cluster, it is essentially # comprised of 3 RemoteSpec objects which are used to # communicate with various components of an RE build cluster. # class RemoteExecutionSpec: def __init__( self, exec_spec: Optional[RemoteSpec], storage_spec: Optional[RemoteSpec], action_spec: Optional[RemoteSpec] ) -> None: self.exec_spec: Optional[RemoteSpec] = exec_spec self.storage_spec: Optional[RemoteSpec] = storage_spec self.action_spec: Optional[RemoteSpec] = action_spec # new_from_node(): # # Creates a RemoteExecutionSpec() from a YAML loaded node. # # Args: # node: The node to parse # basedir: The base directory from which to find certificates. # # Returns: # The described RemoteSpec instance. # # Raises: # LoadError: If the node is malformed. # @classmethod def new_from_node( cls, node: MappingNode, basedir: Optional[str] = None, *, remote_cache: bool = False ) -> "RemoteExecutionSpec": node.validate_keys(["execution-service", "storage-service", "action-cache-service"]) exec_node = node.get_mapping("execution-service", default=None) storage_node = node.get_mapping("storage-service", default=None) if not storage_node and not remote_cache: provenance = node.get_provenance() raise LoadError( "{}: Remote execution requires 'storage-service' to be specified in the 'remote-execution' section if not already specified globally in the 'cache' section".format( provenance ), LoadErrorReason.INVALID_DATA, ) action_node = node.get_mapping("action-cache-service", default=None) if not exec_node and not action_node: provenance = node.get_provenance() raise LoadError( "{}: At least one of `execution-service` or `action-cache-service` need to be specified in the 'remote-execution' section".format( provenance ), LoadErrorReason.INVALID_DATA, ) exec_spec: Optional[RemoteSpec] if exec_node: exec_spec = RemoteSpec.new_from_node(exec_node, basedir, remote_execution=True) else: exec_spec = None storage_spec: Optional[RemoteSpec] if storage_node: storage_spec = RemoteSpec.new_from_node(storage_node, basedir, remote_execution=True) else: storage_spec = None action_spec: Optional[RemoteSpec] if action_node: action_spec = RemoteSpec.new_from_node(action_node, basedir, remote_execution=True, action_cache=True) else: action_spec = None return cls(exec_spec, storage_spec, action_spec) apache-buildstream-27ae392/src/buildstream/_scheduler/000077500000000000000000000000001514607367700230415ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_scheduler/__init__.py000066400000000000000000000020771514607367700251600ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom from .queues import Queue, QueueStatus from .queues.fetchqueue import FetchQueue from .queues.sourcepushqueue import SourcePushQueue from .queues.trackqueue import TrackQueue from .queues.buildqueue import BuildQueue from .queues.artifactpushqueue import ArtifactPushQueue from .queues.pullqueue import PullQueue from .queues.cachequeryqueue import CacheQueryQueue from .scheduler import Scheduler, SchedStatus from .jobs import ElementJob, JobStatus apache-buildstream-27ae392/src/buildstream/_scheduler/jobs/000077500000000000000000000000001514607367700237765ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_scheduler/jobs/__init__.py000066400000000000000000000012551514607367700261120ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat from .elementjob import ElementJob from .job import JobStatus apache-buildstream-27ae392/src/buildstream/_scheduler/jobs/elementjob.py000066400000000000000000000056231514607367700265020ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Author: # Tristan Daniël Maat # from .job import Job # ElementJob() # # A job to run an element's commands. When this job is started # `action_cb` will be called, and when it completes `complete_cb` will # be called. # # Args: # scheduler (Scheduler): The scheduler # action_name (str): The queue action name # max_retries (int): The maximum number of retries # action_cb (callable): The function to execute on the child # complete_cb (callable): The function to execute when the job completes # element (Element): The element to work on # kwargs: Remaining Job() constructor arguments # # Here is the calling signature of the action_cb: # # action_cb(): # # This function will be called in the child task # # Args: # element (Element): The element passed to the Job() constructor # # Returns: # (object): Any abstract simple python object, including a string, int, # bool, list or dict, this must be a simple serializable object. # # Here is the calling signature of the complete_cb: # # complete_cb(): # # This function will be called in the main thread when the child task completes # # Args: # job (Job): The job object which completed # element (Element): The element passed to the Job() constructor # status (JobStatus): The status of whether the workload raised an exception # result (object): The deserialized object returned by the `action_cb`, or None # if `success` is False # class ElementJob(Job): def __init__(self, *args, element, queue, action_cb, complete_cb, **kwargs): super().__init__(*args, **kwargs) self.set_name(element._get_full_name()) self.queue = queue self._element = element # Set the Element pertaining to the job self._action_cb = action_cb # The action callable function self._complete_cb = complete_cb # The complete callable function # Set the plugin element name & key for logging purposes self.set_message_element_name(self.name) self.set_message_element_key(self._element._get_display_key()) def parent_complete(self, status, result): self._complete_cb(self, self._element, status, self._result) def child_process(self): # Run the action return self._action_cb(self._element) apache-buildstream-27ae392/src/buildstream/_scheduler/jobs/job.py000066400000000000000000000350511514607367700251260ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # Tristan Maat # System imports import asyncio import datetime import itertools import threading import traceback from contextlib import ExitStack # BuildStream toplevel imports from ... import utils from ..._utils import terminate_thread from ..._exceptions import ImplError, BstError, set_last_task_error, SkipJob from ..._message import Message, MessageType from ...types import FastEnum from ..._signals import TerminateException # Return code values of child tasks of a job # class _ReturnCode(FastEnum): OK = 0 FAIL = 1 PERM_FAIL = 2 SKIPPED = 3 TERMINATED = 4 # JobStatus: # # The job completion status, passed back through the # complete callbacks. # class JobStatus(FastEnum): # Job succeeded OK = 0 # A temporary BstError was raised FAIL = 1 # A SkipJob was raised SKIPPED = 3 # Job() # # The Job object represents a task that will run in parallel to the main # thread. It has some methods that are not implemented - they are meant for # you to implement in a subclass. # # Args: # scheduler (Scheduler): The scheduler # action_name (str): The queue action name # logfile (str): A template string that points to the logfile # that should be used - should contain {pid}. # max_retries (int): The maximum number of retries # class Job: # Unique id generator for jobs # # This is used to identify tasks in the `State` class _id_generator = itertools.count(1) def __init__(self, scheduler, action_name, logfile, *, max_retries=0): # # Public members # self.id = "{}-{}".format(action_name, next(Job._id_generator)) self.name = None # The name of the job, set by the job's subclass self.action_name = action_name # The action name for the Queue # # Private members # self._scheduler = scheduler # The scheduler self._messenger = self._scheduler.context.messenger self._suspended = False # Whether this job is currently suspended self._max_retries = max_retries # Maximum number of automatic retries self._result = None # Return value of child action in the parent self._tries = 0 # Try count, for retryable jobs self._terminated = False # Whether this job has been explicitly terminated self._logfile = logfile self._message_element_name = None # The task-wide element name self._message_element_key = None # The task-wide element cache key self._element = None # The Element() passed to the Job() constructor, if applicable self._task = None # The task that is run self._thread_id = None # Thread in which the child executes its action self._should_terminate = False self._terminate_lock = threading.Lock() # set_name() # # Sets the name of this job def set_name(self, name): self.name = name # start() # # Starts the job. # def start(self): assert not self._terminated, "Attempted to start a job which was already terminated" self._tries += 1 loop = asyncio.get_event_loop() async def execute(): ret_code, self._result = await loop.run_in_executor(None, self.child_action) await self._parent_child_completed(ret_code) self._task = loop.create_task(execute()) # terminate() # # Politely request that an ongoing job terminate soon. # # This will raise an exception in the child to ask it to exit. # def terminate(self): self.message(MessageType.STATUS, "{} terminating".format(self.action_name)) if self._task: assert utils._is_in_main_thread(), "Terminating the job's thread should only be done from the scheduler" if self._should_terminate: return with self._terminate_lock: self._should_terminate = True if self._thread_id is None: return terminate_thread(self._thread_id) self._terminated = True # get_terminated() # # Check if a job has been terminated. # # Returns: # (bool): True in the main thread if Job.terminate() was called. # def get_terminated(self): return self._terminated # set_message_element_name() # # This is called by Job subclasses to set the plugin instance element # name issuing the message (if an element is related to the Job). # # Args: # element_name (int): The element_name to be supplied to the Message() constructor # def set_message_element_name(self, element_name): self._message_element_name = element_name # set_message_element_key() # # This is called by Job subclasses to set the element # key for for the issuing message (if an element is related to the Job). # # Args: # element_key (_DisplayKey): The element_key tuple to be supplied to the Message() constructor # def set_message_element_key(self, element_key): self._message_element_key = element_key # message(): # # Logs a message, this will be logged in the task's logfile and # conditionally also be sent to the frontend. # # Args: # message_type (MessageType): The type of message to send # message (str): The message # kwargs: Remaining Message() constructor arguments, note that you can # override 'element_name' and 'element_key' this way. # def message(self, message_type, message, **kwargs): kwargs["scheduler"] = True message = Message( message_type, message, element_name=self._message_element_name, element_key=self._message_element_key, **kwargs ) self._messenger.message(message) # get_element() # # Get the Element() related to the job, if jobtype (i.e ElementJob) is # applicable, default None. # # Returns: # (Element): The Element() instance pertaining to the Job, else None. # def get_element(self): return self._element ####################################################### # Abstract Methods # ####################################################### # child_process() # # This will be executed in a thread, and is intended to perform the job's task. # # Returns: # (any): A simple object (must be pickle-able, i.e. strings, lists, # dicts, numbers, but not Element instances). It is returned to # the parent Job running in the main process. This is taken as # the result of the Job. # def child_process(self): raise ImplError("Job '{kind}' does not implement child_process()".format(kind=type(self).__name__)) # parent_complete() # # This will be executed in the main thread after the job finishes, and is # expected to pass the result to the main thread. # # Args: # status (JobStatus): The job exit status # result (any): The result returned by child_process(). # def parent_complete(self, status, result): raise ImplError("Job '{kind}' does not implement parent_complete()".format(kind=type(self).__name__)) ####################################################### # Local Private Methods # ####################################################### # _parent_child_completed() # # Called in the main process courtesy of asyncio's ChildWatcher.add_child_handler() # # Args: # returncode (int): The return code of the child process # async def _parent_child_completed(self, returncode): try: returncode = _ReturnCode(returncode) except ValueError: # An unexpected return code was returned; fail permanently and report self.message( MessageType.ERROR, "Internal job process unexpectedly died with exit code {}".format(returncode), logfile=self._logfile, ) returncode = _ReturnCode.PERM_FAIL # We don't want to retry if we got OK or a permanent fail. retry_flag = returncode == _ReturnCode.FAIL if retry_flag and (self._tries <= self._max_retries) and not self._scheduler.terminated: self.start() return # Resolve the outward facing overall job completion status # if returncode == _ReturnCode.OK: status = JobStatus.OK elif returncode == _ReturnCode.SKIPPED: status = JobStatus.SKIPPED elif returncode in (_ReturnCode.FAIL, _ReturnCode.PERM_FAIL): status = JobStatus.FAIL elif returncode == _ReturnCode.TERMINATED: if self._terminated: self.message(MessageType.INFO, "Job terminated") else: self.message(MessageType.ERROR, "Job was terminated unexpectedly") status = JobStatus.FAIL else: status = JobStatus.FAIL self.parent_complete(status, self._result) self._scheduler.job_completed(self, status) self._task = None # child_action() # # Perform the action in the child process, this calls the action_cb. # def child_action(self): # Set the global message handler in this child # process to forward messages to the parent process self._messenger.setup_new_action_context( self.action_name, self._message_element_name, self._message_element_key ) with ExitStack() as stack: # Time, log and and run the action function # timeinfo = stack.enter_context(self._messenger.timed_suspendable()) try: filename = ( stack.enter_context( self._messenger.recorded_messages(self._logfile, self._scheduler.context.logdir) ) if self._logfile else None ) except Exception as e: # pylint: disable=broad-except elapsed = datetime.datetime.now() - timeinfo.start_time self.message( MessageType.ERROR, "Error opening log file: {}".format(e), elapsed=elapsed, detail=traceback.format_exc(), ) self._thread_id = None return _ReturnCode.PERM_FAIL, None try: self.message(MessageType.START, self.action_name, logfile=filename) with self._terminate_lock: self._thread_id = threading.current_thread().ident if self._should_terminate: return _ReturnCode.TERMINATED, None try: # Try the task action result = self.child_process() # pylint: disable=assignment-from-no-return except SkipJob as e: elapsed = datetime.datetime.now() - timeinfo.start_time self.message(MessageType.SKIPPED, str(e), elapsed=elapsed, logfile=filename) # Alert parent of skip by return code return _ReturnCode.SKIPPED, None except BstError as e: elapsed = datetime.datetime.now() - timeinfo.start_time retry_flag = e.temporary if retry_flag and (self._tries <= self._max_retries): self.message( MessageType.FAIL, "Try #{} failed, retrying".format(self._tries), elapsed=elapsed, logfile=filename, ) else: self.message( MessageType.FAIL, str(e), elapsed=elapsed, detail=e.detail, logfile=filename, sandbox=e.sandbox, ) # Report the exception to the parent (for internal testing purposes) set_last_task_error(e.domain, e.reason) # Set return code based on whether or not the error was temporary. # return _ReturnCode.FAIL if retry_flag else _ReturnCode.PERM_FAIL, None except Exception: # pylint: disable=broad-except # If an unhandled (not normalized to BstError) occurs, that's a bug, # send the traceback and formatted exception back to the frontend # and print it to the log file. # elapsed = datetime.datetime.now() - timeinfo.start_time detail = "An unhandled exception occured:\n\n{}".format(traceback.format_exc()) self.message(MessageType.BUG, self.action_name, elapsed=elapsed, detail=detail, logfile=filename) # Unhandled exceptions should permenantly fail return _ReturnCode.PERM_FAIL, None else: # No exception occurred in the action elapsed = datetime.datetime.now() - timeinfo.start_time self.message(MessageType.SUCCESS, self.action_name, elapsed=elapsed, logfile=filename) # Shutdown needs to stay outside of the above context manager, # make sure we dont try to handle SIGTERM while the process # is already busy in sys.exit() return _ReturnCode.OK, result finally: self._thread_id = None except TerminateException: self._thread_id = None return _ReturnCode.TERMINATED, None apache-buildstream-27ae392/src/buildstream/_scheduler/queues/000077500000000000000000000000001514607367700243505ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_scheduler/queues/__init__.py000066400000000000000000000011231514607367700264560ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from .queue import Queue, QueueStatus apache-buildstream-27ae392/src/buildstream/_scheduler/queues/artifactpushqueue.py000066400000000000000000000030321514607367700304620ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # Local imports from . import Queue, QueueStatus from ..resources import ResourceType from ..._exceptions import SkipJob # A queue which pushes element artifacts # class ArtifactPushQueue(Queue): action_name = "Push" complete_name = "Artifacts Pushed" resources = [ResourceType.UPLOAD] def __init__(self, scheduler, *, imperative=False, skip_uncached=False): super().__init__(scheduler, imperative=imperative) self._skip_uncached = skip_uncached def get_process_func(self): return ArtifactPushQueue._push_or_skip def status(self, element): if element._skip_push(skip_uncached=self._skip_uncached): return QueueStatus.SKIP return QueueStatus.READY @staticmethod def _push_or_skip(element): if not element._push(): raise SkipJob(ArtifactPushQueue.action_name) apache-buildstream-27ae392/src/buildstream/_scheduler/queues/buildqueue.py000066400000000000000000000033141514607367700270670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter from . import Queue, QueueStatus from ..resources import ResourceType from ..jobs import JobStatus # A queue which assembles elements # class BuildQueue(Queue): action_name = "Build" complete_name = "Built" resources = [ResourceType.PROCESS, ResourceType.CACHE] def get_process_func(self): return BuildQueue._assemble_element def status(self, element): if element._cached_success(): return QueueStatus.SKIP if not element._buildable(): return QueueStatus.PENDING return QueueStatus.READY def done(self, job, element, result, status): # Inform element in main process that assembly is done element._assemble_done(status is JobStatus.OK) def register_pending_element(self, element): # Set a "buildable" callback for an element not yet ready # to be processed in the build queue. element._set_buildable_callback(self._enqueue_element) @staticmethod def _assemble_element(element): element._assemble() apache-buildstream-27ae392/src/buildstream/_scheduler/queues/cachequeryqueue.py000066400000000000000000000051161514607367700301230ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from . import Queue, QueueStatus from ..resources import ResourceType from ..jobs import JobStatus from ...types import _KeyStrength # A queue which queries the cache for artifacts and sources # class CacheQueryQueue(Queue): action_name = "Cache-query" complete_name = "Cache queried" resources = [ResourceType.PROCESS, ResourceType.CACHE] log_to_file = False def __init__(self, scheduler, *, sources=False, sources_if_cached=False): super().__init__(scheduler) self._sources = sources self._sources_if_cached = sources_if_cached def get_process_func(self): if self._sources_if_cached: return CacheQueryQueue._query_artifacts_and_sources elif not self._sources: return CacheQueryQueue._query_artifacts_or_sources else: return CacheQueryQueue._query_sources def status(self, element): if element._can_query_cache(): # Cache status already available. # This is the case for artifact elements, which load the # artifact early on. return QueueStatus.SKIP if not element._get_cache_key(strength=_KeyStrength.WEAK): # Strict and weak cache keys are unavailable if the element or # a dependency has an unresolved source return QueueStatus.SKIP return QueueStatus.READY def done(self, _, element, result, status): if status is JobStatus.FAIL: return if not self._sources: if not element._pull_pending(): element._load_artifact_done() @staticmethod def _query_artifacts_or_sources(element): element._load_artifact(pull=False) if not element._can_query_cache() or not element._cached_success(): element._query_source_cache() @staticmethod def _query_artifacts_and_sources(element): element._load_artifact(pull=False) element._query_source_cache() @staticmethod def _query_sources(element): element._query_source_cache() apache-buildstream-27ae392/src/buildstream/_scheduler/queues/fetchqueue.py000066400000000000000000000047511514607367700270670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # Local imports from . import Queue, QueueStatus from ..resources import ResourceType from ..jobs import JobStatus # A queue which fetches element sources # class FetchQueue(Queue): action_name = "Fetch" complete_name = "Sources Fetched" resources = [ResourceType.DOWNLOAD] def __init__(self, scheduler, skip_cached=False, fetch_original=False): super().__init__(scheduler) self._skip_cached = skip_cached self._should_fetch_original = fetch_original def get_process_func(self): if self._should_fetch_original: return FetchQueue._fetch_original else: return FetchQueue._fetch_not_original def status(self, element): # Optionally skip elements that are already in the artifact cache if self._skip_cached: if not element._can_query_cache(): return QueueStatus.PENDING if element._cached_success(): return QueueStatus.SKIP # This will automatically skip elements which # have no sources. if element._can_query_source_cache() and not element._should_fetch(self._should_fetch_original): return QueueStatus.SKIP return QueueStatus.READY def done(self, _, element, result, status): if status is JobStatus.FAIL: return element._fetch_done(self._should_fetch_original) def register_pending_element(self, element): # Set a "can_query_cache" callback for an element not yet ready # to be processed in the fetch queue. element._set_can_query_cache_callback(self._enqueue_element) @staticmethod def _fetch_not_original(element): element._fetch(fetch_original=False) @staticmethod def _fetch_original(element): element._fetch(fetch_original=True) apache-buildstream-27ae392/src/buildstream/_scheduler/queues/pullqueue.py000066400000000000000000000030241514607367700267420ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # Local imports from . import Queue, QueueStatus from ..resources import ResourceType from ..jobs import JobStatus from ..._exceptions import SkipJob # A queue which pulls element artifacts # class PullQueue(Queue): action_name = "Pull" complete_name = "Artifacts Pulled" resources = [ResourceType.DOWNLOAD, ResourceType.CACHE] def get_process_func(self): return PullQueue._pull_or_skip def status(self, element): if element._pull_pending(): return QueueStatus.READY else: return QueueStatus.SKIP def done(self, _, element, result, status): if status is JobStatus.FAIL: return element._load_artifact_done() @staticmethod def _pull_or_skip(element): if not element._load_artifact(pull=True): raise SkipJob(PullQueue.action_name) apache-buildstream-27ae392/src/buildstream/_scheduler/queues/queue.py000066400000000000000000000326461514607367700260610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # System imports import os from collections import deque import heapq import traceback from typing import TYPE_CHECKING # Local imports from ..jobs import ElementJob, JobStatus from ..resources import ResourceType # BuildStream toplevel imports from ..._exceptions import BstError, ImplError, set_last_task_error from ..._message import Message, MessageType from ...types import FastEnum if TYPE_CHECKING: from typing import List, Optional # Queue status for a given element # # class QueueStatus(FastEnum): # The element is not yet ready to be processed in the queue. PENDING = 1 # The element can skip this queue. SKIP = 2 # The element is ready for processing in this queue. READY = 3 # Queue() # # Args: # scheduler (Scheduler): The Scheduler # class Queue: # These should be overridden on class data of of concrete Queue implementations action_name = None # type: Optional[str] complete_name = None # type: Optional[str] # Resources this queues' jobs want resources = [] # type: List[int] log_to_file = True def __init__(self, scheduler, *, imperative=False): # # Private members # self._scheduler = scheduler self._resources = scheduler.resources # Shared resource pool self._ready_queue = [] # Ready elements self._done_queue = deque() # Processed / Skipped elements self._max_retries = 0 self._queued_elements = 0 # Number of elements queued self._required_element_check = False # Whether we should check that elements are required before enqueuing # # Public members # self.imperative = imperative # Assert the subclass has setup class data assert self.action_name is not None assert self.complete_name is not None if ResourceType.UPLOAD in self.resources or ResourceType.DOWNLOAD in self.resources: self._max_retries = scheduler.context.sched_network_retries self._task_group = self._scheduler._state.add_task_group(self.action_name, self.complete_name) # destroy() # # Explicitly performs all cleanup tasks for this queue # # Note: Doing this explicitly is preferred to a __del__ method because # it is not at the mercy of the garbage collector def destroy(self): self._scheduler._state.remove_task_group(self.action_name) ##################################################### # Abstract Methods for Queue implementations # ##################################################### # get_process_func() # # Abstract method, returns a callable for processing an element. # # The callable should fit the signature `process(element: Element) -> any`. # # Note that the callable may be executed in a child process, so the return # value should be a simple object (must be pickle-able, i.e. strings, # lists, dicts, numbers, but not Element instances). This is sent to back # to the main process. # # This method is the only way for a queue to affect elements, and so is # not optional to implement. # # Returns: # (Callable[[Element], Any]): The callable for processing elements. # def get_process_func(self): raise NotImplementedError() # status() # # Abstract method for reporting the immediate status of an element. The status # determines whether an element can/cannot be pushed into the queue, or even # skip the queue entirely, when called. # # Args: # element (Element): An element to process # # Returns: # (QueueStatus): The element status # def status(self, element): return QueueStatus.READY # done() # # Abstract method for handling a successful job completion. # # Args: # job (Job): The job which completed processing # element (Element): The element which completed processing # result (any): The return value of the process() implementation # status (JobStatus): The return status of the Job # def done(self, job, element, result, status): pass ##################################################### # Virtual Methods for Queue implementations # ##################################################### # register_pending_element() # # Virtual method for registering a queue specific callback # to an Element which is not immediately ready to advance # to the next queue # # Args: # element (Element): The element waiting to be pushed into the queue # def register_pending_element(self, element): raise ImplError("Queue type: {} does not implement register_pending_element()".format(self.action_name)) ##################################################### # Scheduler / Pipeline facing APIs # ##################################################### # enqueue() # # Enqueues some elements # # Args: # elts (list): A list of Elements # def enqueue(self, elts): if not elts: return # Obtain immediate element status for elt in elts: if self._required_element_check and not elt._is_required(): elt._set_required_callback(self._enqueue_element) else: self._enqueue_element(elt) # dequeue() # # A generator which dequeues the elements which # are ready to exit the queue. # # Yields: # (Element): Elements being dequeued # def dequeue(self): while self._done_queue: yield self._done_queue.popleft() # dequeue_ready() # # Reports whether any elements can be promoted to other queues # # Returns: # (bool): Whether there are elements ready # def dequeue_ready(self): return any(self._done_queue) # harvest_jobs() # # Spawn as many jobs from the ready queue for which resources # can be reserved. # # Priority is first given to elements which have been assigned a lower # depth (see Element._set_depth()), and then to elements which have # been enqueued earlier. # # Returns: # ([Job]): A list of jobs which can be run now # def harvest_jobs(self): ready = [] while self._ready_queue: # Now reserve them reserved = self._resources.reserve(self.resources) if not reserved: break _, _, element = heapq.heappop(self._ready_queue) ready.append(element) return [ ElementJob( self._scheduler, self.action_name, self._element_log_path(element) if self.log_to_file else None, element=element, queue=self, action_cb=self.get_process_func(), complete_cb=self._job_done, max_retries=self._max_retries, ) for element in ready ] # set_required_element_check() # # This ensures that, for the first non-track queue, we must check # whether elements are required before enqueuing them def set_required_element_check(self): self._required_element_check = True # any_failed_elements() # # Returns whether any elements in this queue have failed their jobs def any_failed_elements(self): return any(self._task_group.failed_tasks) ##################################################### # Private Methods # ##################################################### # _update_workspaces() # # Updates and possibly saves the workspaces in the # main data model in the main process after a job completes. # # Args: # element (Element): The element which completed # def _update_workspaces(self, element): # FIXME: Does this really needs to be done for every job or only some? # If some, we should only run it for those. workspace = element._get_workspace() # Handle any workspace modifications now # if workspace: context = element._get_context() workspaces = context.get_workspaces() if workspaces.update_workspace(element._get_full_name(), workspace.to_dict()): try: workspaces.save_config() except BstError as e: self._message(element, MessageType.ERROR, "Error saving workspaces", detail=str(e)) except Exception: # pylint: disable=broad-except self._message( element, MessageType.BUG, "Unhandled exception while saving workspaces", detail=traceback.format_exc(), ) # _job_done() # # A callback reported by the Job() when a job completes # # This will call the Queue implementation specific Queue.done() # implementation and trigger the scheduler to reschedule. # # See the Job object for an explanation of the call signature # def _job_done(self, job, element, status, result): # Now release the resources we reserved # self._resources.release(self.resources) # Update values that need to be synchronized in the main task # before calling any queue implementation self._update_workspaces(element) # Give the result of the job to the Queue implementor, # and determine if it should be considered as processed # or skipped. try: self.done(job, element, result, status) except BstError as e: # Report error and mark as failed # self._message(element, MessageType.ERROR, "Post processing error", detail=str(e)) self._task_group.add_failed_task(element._get_full_name()) # Treat this as a task error as it's related to a task # even though it did not occur in the task context # # This just allows us stronger testing capability # set_last_task_error(e.domain, e.reason) except Exception: # pylint: disable=broad-except # Report unhandled exceptions and mark as failed # self._message( element, MessageType.BUG, "Unhandled exception in post processing", detail=traceback.format_exc() ) self._task_group.add_failed_task(element._get_full_name()) else: # All elements get placed on the done queue for later processing. self._done_queue.append(element) # These lists are for bookkeeping purposes for the UI and logging. if status == JobStatus.SKIPPED or job.get_terminated(): self._task_group.add_skipped_task() elif status == JobStatus.OK: self._task_group.add_processed_task() else: self._task_group.add_failed_task(element._get_full_name()) # Convenience wrapper for Queue implementations to send # a message for the element they are processing def _message(self, element, message_type, brief, **kwargs): message = Message(message_type, brief, element_name=element._get_full_name(), **kwargs) self._scheduler.context.messenger.message(message) def _element_log_path(self, element): project = element._get_project() key = element._get_display_key() action = self.action_name.lower() logfile = "{key}-{action}".format(key=key.brief, action=action) return os.path.join(project.name, element.normal_name, logfile) # _enqueue_element() # # Enqueue an Element upon a callback to a specific queue # Here we check whether an element is either immediately ready to be processed # in the current queue or whether it can skip the queue. Element's which are # not yet ready to be processed or cannot skip will have the appropriate # callback registered # # Args: # element (Element): The Element to enqueue # def _enqueue_element(self, element): status = self.status(element) if status == QueueStatus.SKIP: # Place skipped elements into the done queue immediately self._task_group.add_skipped_task() self._done_queue.append(element) # Elements to proceed to the next queue elif status == QueueStatus.READY: # Push elements which are ready to be processed immediately into the queue heapq.heappush(self._ready_queue, (element._depth, self._queued_elements, element)) self._queued_elements += 1 else: # Register a queue specific callback for pending elements self.register_pending_element(element) apache-buildstream-27ae392/src/buildstream/_scheduler/queues/sourcepushqueue.py000066400000000000000000000024071514607367700301720ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman from . import Queue, QueueStatus from ..resources import ResourceType from ..._exceptions import SkipJob # A queue which pushes staged sources # class SourcePushQueue(Queue): action_name = "Src-push" complete_name = "Sources Pushed" resources = [ResourceType.UPLOAD] def get_process_func(self): return SourcePushQueue._push_or_skip def status(self, element): if element._skip_source_push(): return QueueStatus.SKIP return QueueStatus.READY @staticmethod def _push_or_skip(element): if not element._source_push(): raise SkipJob(SourcePushQueue.action_name) apache-buildstream-27ae392/src/buildstream/_scheduler/queues/trackqueue.py000066400000000000000000000036571514607367700271060ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # BuildStream toplevel imports from ...plugin import Plugin # Local imports from . import Queue, QueueStatus from ..resources import ResourceType from ..jobs import JobStatus # A queue which tracks sources # class TrackQueue(Queue): action_name = "Track" complete_name = "Sources Tracked" resources = [ResourceType.DOWNLOAD] def get_process_func(self): return TrackQueue._track_element def status(self, element): # We can skip elements without any sources if not any(element.sources()): # But we still have to mark them as tracked element._tracking_done() return QueueStatus.SKIP return QueueStatus.READY def done(self, _, element, result, status): if status is JobStatus.FAIL: return # Set the new refs in the main process one by one as they complete, # writing to bst files this time if result is not None: for unique_id, new_ref, ref_changed in result: if ref_changed: source = Plugin._lookup(unique_id) source._set_ref(new_ref, save=True) element._tracking_done() @staticmethod def _track_element(element): return element._track() apache-buildstream-27ae392/src/buildstream/_scheduler/resources.py000066400000000000000000000107641514607367700254350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # class ResourceType: CACHE = 0 DOWNLOAD = 1 PROCESS = 2 UPLOAD = 3 class Resources: def __init__(self, num_builders, num_fetchers, num_pushers): self._max_resources = { ResourceType.CACHE: 0, ResourceType.DOWNLOAD: num_fetchers, ResourceType.PROCESS: num_builders, ResourceType.UPLOAD: num_pushers, } # Resources jobs are currently using. self._used_resources = { ResourceType.CACHE: 0, ResourceType.DOWNLOAD: 0, ResourceType.PROCESS: 0, ResourceType.UPLOAD: 0, } # Resources jobs currently want exclusive access to. The set # of jobs that have asked for exclusive access is the value - # this is so that we can avoid scheduling any other jobs until # *all* exclusive jobs that "register interest" have finished # - which avoids starving them of scheduling time. self._exclusive_resources = { ResourceType.CACHE: set(), ResourceType.DOWNLOAD: set(), ResourceType.PROCESS: set(), ResourceType.UPLOAD: set(), } # reserve() # # Reserves a set of resources # # Args: # resources (set): A set of ResourceTypes # exclusive (set): Another set of ResourceTypes # peek (bool): Whether to only peek at whether the resource is available # # Returns: # (bool): True if the resources could be reserved # def reserve(self, resources, exclusive=None, *, peek=False): if exclusive is None: exclusive = set() resources = set(resources) exclusive = set(exclusive) # First, we check if the job wants to access a resource that # another job wants exclusive access to. If so, it cannot be # scheduled. # # Note that if *both* jobs want this exclusively, we don't # fail yet. # # FIXME: I *think* we can deadlock if two jobs want disjoint # sets of exclusive and non-exclusive resources. This # is currently not possible, but may be worth thinking # about. # for resource in resources - exclusive: # If our job wants this resource exclusively, we never # check this, so we can get away with not (temporarily) # removing it from the set. if self._exclusive_resources[resource]: return False # Now we check if anything is currently using any resources # this job wants exclusively. If so, the job cannot be # scheduled. # # Since jobs that use a resource exclusively are also using # it, this means only one exclusive job can ever be scheduled # at a time, despite being allowed to be part of the exclusive # set. # for resource in exclusive: if self._used_resources[resource] != 0: return False # Finally, we check if we have enough of each resource # available. If we don't have enough, the job cannot be # scheduled. for resource in resources: if self._max_resources[resource] > 0 and self._used_resources[resource] >= self._max_resources[resource]: return False # Now we register the fact that our job is using the resources # it asked for, and tell the scheduler that it is allowed to # continue. if not peek: for resource in resources: self._used_resources[resource] += 1 return True # release() # # Release resources previously reserved with Resources.reserve() # # Args: # resources (set): A set of resources to release # def release(self, resources): for resource in resources: assert self._used_resources[resource] > 0, "Scheduler resource imbalance" self._used_resources[resource] -= 1 apache-buildstream-27ae392/src/buildstream/_scheduler/scheduler.py000066400000000000000000000457171514607367700254070ustar00rootroot00000000000000# # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # System imports import functools import os import asyncio from itertools import chain import signal import datetime import multiprocessing.forkserver import sys import threading from concurrent.futures import ThreadPoolExecutor # Local imports from .resources import Resources from .jobs import JobStatus from ..types import FastEnum from .._profile import Topics, PROFILER from .. import _signals # A decent return code for Scheduler.run() class SchedStatus(FastEnum): SUCCESS = 0 ERROR = -1 TERMINATED = 1 def reset_signals_on_exit(func): @functools.wraps(func) def wrapper(*args, **kwargs): orig_sigint = signal.getsignal(signal.SIGINT) orig_sigterm = signal.getsignal(signal.SIGTERM) orig_sigtstp = signal.getsignal(signal.SIGTSTP) try: return func(*args, **kwargs) finally: signal.signal(signal.SIGINT, orig_sigint) signal.signal(signal.SIGTERM, orig_sigterm) signal.signal(signal.SIGTSTP, orig_sigtstp) return wrapper # Scheduler() # # The scheduler operates on a list queues, each of which is meant to accomplish # a specific task. Elements enter the first queue when Scheduler.run() is called # and into the next queue when complete. Scheduler.run() returns when all of the # elements have been traversed or when an error occurs. # # Using the scheduler is a matter of: # a.) Deriving the Queue class and implementing its abstract methods # b.) Instantiating a Scheduler with one or more queues # c.) Calling Scheduler.run(elements) with a list of elements # d.) Fetching results from your queues # # Args: # context: The Context in the parent scheduling process # start_time: The time at which the session started # state: The state that can be made available to the frontend # interrupt_callback: A callback to handle ^C # ticker_callback: A callback call once per second # class Scheduler: def __init__(self, context, start_time, state, interrupt_callback, ticker_callback): # # Public members # self.queues = None # Exposed for the frontend to print summaries self.context = context # The Context object shared with Queues self.terminated = False # Whether the scheduler was asked to terminate or has terminated self.suspended = False # Whether the scheduler is currently suspended # These are shared with the Job, but should probably be removed or made private in some way. self.loop = None # Shared for Job access to observe the message queue # # Private members # self._active_jobs = [] # Jobs currently being run in the scheduler self._suspendtime = None # Session time compensation for suspended state self._queue_jobs = True # Whether we should continue to queue jobs self._state = state self._casd_process = None # handle to the casd process for monitoring purpose self._sched_handle = None # Whether a scheduling job is already scheduled or not self._ticker_callback = ticker_callback self._interrupt_callback = interrupt_callback self.resources = Resources(context.sched_builders, context.sched_fetchers, context.sched_pushers) # Ensure that the forkserver is started before we start. # This is best run before we do any GRPC connections to casd or have # other background threads started. # We ignore signals here, as this is the state all the python child # processes from now on will have when starting with _signals.blocked([signal.SIGINT, signal.SIGTSTP], ignore=True): multiprocessing.forkserver.ensure_running() # run() # # Args: # queues (list): A list of Queue objects # casd_process_manager (cascache.CASDProcessManager): The subprocess which runs casd, in order to be notified # of failures. # # Returns: # (SchedStatus): How the scheduling terminated # # Elements in the 'plan' will be processed by each # queue in order. Processing will complete when all # elements have been processed by each queue or when # an error arises # @reset_signals_on_exit def run(self, queues, casd_process_manager): # Hold on to the queues to process self.queues = queues # Ensure that we have a fresh new event loop, in case we want # to run another test in this thread. self.loop = asyncio.new_event_loop() asyncio.set_event_loop(self.loop) # Add timeouts self.loop.call_later(1, self._tick) # Add exception handler self.loop.set_exception_handler(self._handle_exception) # Handle unix signals while running self._connect_signals() # Watch casd while running to ensure it doesn't die self._casd_process = casd_process_manager.process threading.Thread(target=self._watch_casd, name="watch-casd", daemon=True).start() # Start the profiler with PROFILER.profile(Topics.SCHEDULER, "_".join(queue.action_name for queue in self.queues)): # This is not a no-op. Since it is the first signal registration # that is set, it allows then other threads to register signal # handling routines, which would not be possible if the main thread # hadn't set it before. # FIXME: this should be done in a cleaner way with _signals.suspendable(lambda: None, lambda: None), _signals.terminator(lambda: None): with ThreadPoolExecutor(max_workers=sum(self.resources._max_resources.values())) as pool: self.loop.set_default_executor(pool) # Run the queues self._sched() self.loop.run_forever() self.loop.close() # Invoke the ticker callback a final time to render pending messages self._ticker_callback() self._casd_process = None # Stop handling unix signals self._disconnect_signals() failed = any(queue.any_failed_elements() for queue in self.queues) self.loop = None if failed: status = SchedStatus.ERROR elif self.terminated: status = SchedStatus.TERMINATED else: status = SchedStatus.SUCCESS return status # clear_queues() # # Forcibly destroys all the scheduler's queues # This is needed because Queues register TaskGroups with State, # which must be unique. As there is not yet any reason to have multiple # Queues of the same type, old ones should be deleted. # def clear_queues(self): if self.queues: for queue in self.queues: queue.destroy() self.queues.clear() # terminate() # # Forcefully terminates all ongoing jobs. # # For this to be effective, one needs to return to # the scheduler loop first and allow the scheduler # to complete gracefully. # # NOTE: This will block SIGINT so that graceful process # termination is not interrupted, and SIGINT will # remain blocked after Scheduler.run() returns. # def terminate(self): # Set this right away, the frontend will check this # attribute to decide whether or not to print status info # etc and the following code block will trigger some callbacks. self.terminated = True # Notify the frontend that we're terminated as it might be # from an interactive prompt callback or SIGTERM self.loop.call_soon(self._terminate_jobs_real) # Block this until we're finished terminating jobs, # this will remain blocked forever. signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGINT]) # suspend() # # Suspend the scheduler # def suspend(self): self._disconnect_signals() self._suspend_jobs() # resume() # # Restart the scheduler # def resume(self): self._connect_signals() self._resume_jobs() # stop() # # Stop queueing additional imperative jobs, causes Scheduler.run() # to return once all post-imperative jobs are finished. # def stop(self): self._queue_jobs = False # job_completed(): # # Called when a Job completes # # Args: # queue (Queue): The Queue holding a complete job # job (Job): The completed Job # status (JobStatus): The status of the completed job # def job_completed(self, job, status): # Remove from the active jobs list self._active_jobs.remove(job) if status == JobStatus.FAIL: # If it's an elementjob, we want to compare against the failure messages # and send the unique_id and display key tuple of the Element. This can then # be used to load the element instance relative to the process it is running in. element = job.get_element() if element: element_info = element._unique_id, element._get_display_key() else: element_info = None self._state.fail_task(job.id, element_info) self._state.remove_task(job.id) self._sched() ####################################################### # Local Private Methods # ####################################################### # _abort_on_casd_failure() # # Abort if casd failed while running. # # This will terminate immediately all jobs, since buildbox-casd is dead, # we can't do anything with them anymore. # def _abort_on_casd_failure(self): self.context.messenger.bug("buildbox-casd died while the pipeline was active.") self.terminate() # _watch_casd() # # This runs in a separate thread to detect casd exiting while the loop is # still running. # def _watch_casd(self): loop = self.loop proc = self._casd_process if loop and proc: # This sets the `returncode` attribute proc.wait() if not loop.is_closed(): loop.call_soon_threadsafe(self._abort_on_casd_failure) # _start_job() # # Spanws a job # # Args: # job (Job): The job to start # def _start_job(self, job): # From the scheduler perspective, the following # is considered atomic; started jobs are always in the # active_jobs list, and jobs in the active_jobs list # are always started. # self._active_jobs.append(job) job.start() self._state.add_task(job.id, job.action_name, job.name, self._state.elapsed_time()) # _sched_queue_jobs() # # Ask the queues what jobs they want to schedule and schedule # them. This is done here so we can ask for new jobs when jobs # from previous queues become available. # # This will process the Queues, pull elements through the Queues # and process anything that is ready. # def _sched_queue_jobs(self): ready = [] process_queues = True if self._queue_jobs: # Scheduler.stop() was not called, consider all queues. queues_to_process = self.queues else: # Limit processing to post-imperative queues for queue in self.queues: if queue.imperative: # Here the `queues_to_process` list will consists of all the queues after # the imperative queue. This means only post-imperative jobs will be processed. queues_to_process = self.queues[self.queues.index(queue) + 1 :] break else: # No imperative queue was marked, stop queueing any jobs queues_to_process = [] while process_queues: # Pull elements forward through all queues, regardless of whether we're processing those # queues. The main reason to do this is to ensure we propagate finished jobs from the # imperative queue. elements = [] for queue in self.queues: queue.enqueue(elements) elements = list(queue.dequeue()) # Kickoff whatever processes can be processed at this time # # We start by queuing from the last queue first, because # we want to give priority to queues later in the # scheduling process in the case that multiple queues # share the same token type. # # This avoids starvation situations where we dont move on # to fetch tasks for elements which failed to pull, and # thus need all the pulls to complete before ever starting # a build ready.extend(chain.from_iterable(q.harvest_jobs() for q in reversed(queues_to_process))) # harvest_jobs() may have decided to skip some jobs, making # them eligible for promotion to the next queue as a side effect. # # If that happens, do another round. process_queues = any(q.dequeue_ready() for q in queues_to_process) # Start the jobs # for job in ready: self._start_job(job) # _sched() # # Run any jobs which are ready to run, or quit the main loop # when nothing is running or is ready to run. # # This is the main driving function of the scheduler, it is called # initially when we enter Scheduler.run(), and at the end of whenever # any job completes, after any bussiness logic has occurred and before # going back to sleep. # def _sched(self): def real_schedule(): # Reset the scheduling handle before queuing any jobs. # # We do this right away because starting jobs can result # in their being terminated and completed during the body # of this function, and we want to be sure that we get # called again in this case. # # This can happen if jobs are explicitly killed as a result, # which might happen as a side effect of a crash in an # abstracted frontend implementation handling notifications # about jobs starting. # self._sched_handle = None if not self.terminated: # # Run as many jobs as the queues can handle for the # available resources # self._sched_queue_jobs() # # If nothing is ticking then bail out # if not self._active_jobs: self.loop.stop() if self._sched_handle is None: self._sched_handle = self.loop.call_soon(real_schedule) # _suspend_jobs() # # Suspend all ongoing jobs. # def _suspend_jobs(self): if not self.suspended: self._suspendtime = datetime.datetime.now() self.suspended = True _signals.is_not_suspended.clear() for suspender in reversed(_signals.suspendable_stack): suspender.suspend() # _resume_jobs() # # Resume suspended jobs. # def _resume_jobs(self): if self.suspended: for suspender in _signals.suspendable_stack: suspender.resume() _signals.is_not_suspended.set() self.suspended = False # Notify that we're unsuspended self._state.offset_start_time(datetime.datetime.now() - self._suspendtime) self._suspendtime = None # _interrupt_event(): # # A loop registered event callback for keyboard interrupts # def _interrupt_event(self): # The event loop receives a copy of all signals that are sent while it is running # This means that even though we catch the SIGINT in the question to the user, # the loop will receive it too, and thus we need to skip it here. if self.terminated: return self._interrupt_callback() # _terminate_event(): # # A loop registered event callback for SIGTERM # def _terminate_event(self): self.terminate() # _suspend_event(): # # A loop registered event callback for SIGTSTP # def _suspend_event(self): # No need to care if jobs were suspended or not, we _only_ handle this # while we know jobs are not suspended. self._suspend_jobs() os.kill(os.getpid(), signal.SIGSTOP) self._resume_jobs() # _connect_signals(): # # Connects our signal handler event callbacks to the mainloop # def _connect_signals(self): self.loop.add_signal_handler(signal.SIGINT, self._interrupt_event) self.loop.add_signal_handler(signal.SIGTERM, self._terminate_event) self.loop.add_signal_handler(signal.SIGTSTP, self._suspend_event) def _disconnect_signals(self): self.loop.remove_signal_handler(signal.SIGINT) self.loop.remove_signal_handler(signal.SIGTSTP) self.loop.remove_signal_handler(signal.SIGTERM) def _terminate_jobs_real(self): for job in self._active_jobs: job.terminate() # Regular timeout for driving status in the UI def _tick(self): self._ticker_callback() self.loop.call_later(1, self._tick) def _handle_exception(self, loop, context: dict) -> None: e = context.get("exception") exc = bool(e) if e is None: # https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_exception_handler # If no optional Exception generate a generic exception with message value. # exc will be False, instructing the global handler to skip formatting the # assumed exception & related traceback. e = Exception(str(context.get("message")) + " asyncio exception handler called, but no Exception() given") # Call the sys global exception handler directly, as to avoid the default # async handler raising an unhandled exception here. App will treat this # as a 'BUG', format it appropriately & exit. mypy needs to ignore parameter # types here as we're overriding sys globally in App._global_exception_handler() sys.excepthook(type(e), e, e.__traceback__, exc) # type: ignore apache-buildstream-27ae392/src/buildstream/_signals.py000066400000000000000000000200731514607367700230770ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os import signal import sys import threading import traceback from contextlib import contextmanager, ExitStack from collections import deque from typing import Callable, Deque # Global per process state for handling of sigterm/sigtstp/sigcont, # note that it is expected that this only ever be used by new processes # the scheduler starts, not the main process. # terminator_stack: Deque[Callable] = deque() suspendable_stack: Deque[Callable] = deque() terminator_lock = threading.Lock() suspendable_lock = threading.Lock() # This event is used to block all the threads while we wait for user # interaction. This is because we can't stop all the pythin threads but the # one easily when waiting for user input. However, most performance intensive # tasks will pass through a subprocess or a multiprocess.Process and all of # those are guarded by the signal handling. Thus, by setting and unsetting this # event in the scheduler, we can enable and disable the launching of processes # and ensure we don't do anything resource intensive while being interrupted. is_not_suspended = threading.Event() is_not_suspended.set() class TerminateException(BaseException): pass # Per process SIGTERM handler def terminator_handler(signal_, frame): exit_code = -1 while terminator_stack: terminator_ = terminator_stack.pop() try: terminator_() except SystemExit as e: exit_code = e.code or 0 except: # noqa pylint: disable=bare-except # Ensure we print something if there's an exception raised when # processing the handlers. Note that the default exception # handler won't be called because we os._exit next, so we must # catch all possible exceptions with the unqualified 'except' # clause. traceback.print_exc(file=sys.stderr) print( "Error encountered in BuildStream while processing custom SIGTERM handler:", terminator_, file=sys.stderr, ) # Use special exit here, terminate immediately, recommended # for precisely this situation where child processes are teminated. os._exit(exit_code) # terminator() # # A context manager for interruptable tasks, this guarantees # that while the code block is running, the supplied function # will be called upon process termination. # # /!\ The callbacks passed must only contain code that does not acces thread # local variables. Those will run in the main thread. # # Note that after handlers are called, the termination will be handled by # terminating immediately with os._exit(). This means that SystemExit will not # be raised and 'finally' clauses will not be executed. # # Args: # terminate_func (callable): A function to call when aborting # the nested code block. # @contextmanager def terminator(terminate_func): global terminator_stack # pylint: disable=global-statement,global-variable-not-assigned outermost = bool(not terminator_stack) assert threading.current_thread() == threading.main_thread() or not outermost with terminator_lock: terminator_stack.append(terminate_func) if outermost: original_handler = signal.signal(signal.SIGTERM, terminator_handler) try: yield except TerminateException: terminate_func() raise finally: if outermost: signal.signal(signal.SIGTERM, original_handler) with terminator_lock: terminator_stack.remove(terminate_func) # Just a simple object for holding on to two callbacks class Suspender: def __init__(self, suspend_callback, resume_callback): self.suspend = suspend_callback self.resume = resume_callback # Per process SIGTSTP handler def suspend_handler(sig, frame): is_not_suspended.clear() # Suspend callbacks from innermost frame first with suspendable_lock: for suspender in reversed(suspendable_stack): suspender.suspend() # Use SIGSTOP directly now on self, dont introduce more SIGTSTP # # Here the process sleeps until SIGCONT, which we simply # dont handle. We know we'll pickup execution right here # when we wake up. os.kill(os.getpid(), signal.SIGSTOP) # Resume callbacks from outermost frame inwards for suspender in suspendable_stack: suspender.resume() is_not_suspended.set() # suspendable() # # A context manager for handling process suspending and resumeing # # Args: # suspend_callback (callable): A function to call as process suspend time. # resume_callback (callable): A function to call as process resume time. # # /!\ The callbacks passed must only contain code that does not acces thread # local variables. Those will run in the main thread. # # This must be used in code blocks which start processes that become # their own session leader. In these cases, SIGSTOP and SIGCONT need # to be propagated to the child process group. # # This context manager can also be used recursively, so multiple # things can happen at suspend/resume time (such as tracking timers # and ensuring durations do not count suspended time). # @contextmanager def suspendable(suspend_callback, resume_callback): global suspendable_stack # pylint: disable=global-statement,global-variable-not-assigned outermost = bool(not suspendable_stack) assert threading.current_thread() == threading.main_thread() or not outermost # If we are not in the main thread, ensure that we are not suspended # before running. # If we are in the main thread, never block on this, to ensure we # don't deadlock. if threading.current_thread() != threading.main_thread(): is_not_suspended.wait() suspender = Suspender(suspend_callback, resume_callback) with suspendable_lock: suspendable_stack.append(suspender) if outermost: original_stop = signal.signal(signal.SIGTSTP, suspend_handler) try: yield finally: if outermost: signal.signal(signal.SIGTSTP, original_stop) with suspendable_lock: suspendable_stack.remove(suspender) # blocked() # # A context manager for running a code block with blocked signals # # Args: # signals (list): A list of unix signals to block # ignore (bool): Whether to ignore entirely the signals which were # received and pending while the process had blocked them # @contextmanager def blocked(signal_list, ignore=True): with ExitStack() as stack: # Optionally add the ignored() context manager to this context if ignore: stack.enter_context(ignored(signal_list)) # Set and save the sigprocmask blocked_signals = signal.pthread_sigmask(signal.SIG_BLOCK, signal_list) try: yield finally: # If we have discarded the signals completely, this line will cause # the discard_handler() to trigger for each signal in the list signal.pthread_sigmask(signal.SIG_SETMASK, blocked_signals) # ignored() # # A context manager for running a code block with ignored signals # # Args: # signals (list): A list of unix signals to ignore # @contextmanager def ignored(signal_list): orig_handlers = {} for sig in signal_list: orig_handlers[sig] = signal.signal(sig, signal.SIG_IGN) try: yield finally: for sig in signal_list: signal.signal(sig, orig_handlers[sig]) apache-buildstream-27ae392/src/buildstream/_site.py000066400000000000000000000032411514607367700224010ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom import os # # Private module declaring some info about where the buildstream # is installed so we can lookup package relative resources easily # # The package root, wherever we are running the package from root = os.path.dirname(os.path.abspath(__file__)) # The Element plugin directory element_plugins = os.path.join(root, "plugins", "elements") # The Source plugin directory source_plugins = os.path.join(root, "plugins", "sources") # The SourceMirror plugin directory source_mirror_plugins = os.path.join(root, "plugins", "sourcemirrors") # Default user configuration default_user_config = os.path.join(root, "data", "userconfig.yaml") # Default project configuration default_project_config = os.path.join(root, "data", "projectconfig.yaml") # Script template to call module building scripts build_all_template = os.path.join(root, "data", "build-all.sh.in") # Module building script template build_module_template = os.path.join(root, "data", "build-module.sh.in") # The bundled subprojects directory subprojects = os.path.join(root, "subprojects") apache-buildstream-27ae392/src/buildstream/_sourcecache.py000066400000000000000000000201631514607367700237230ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # import os from ._cas.casremote import BlobNotFound from .storage._casbaseddirectory import CasBasedDirectory from ._assetcache import AssetCache from ._exceptions import CASError, CASRemoteError, SourceCacheError, AssetCacheError from . import utils from ._protos.buildstream.v2 import source_pb2 REMOTE_ASSET_SOURCE_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:source:{}" # Class that keeps config of remotes and deals with caching of sources. # # Args: # context (Context): The Buildstream context # class SourceCache(AssetCache): def __init__(self, context): super().__init__(context) self._basedir = os.path.join(context.cachedir, "source_protos") os.makedirs(self._basedir, exist_ok=True) # contains() # # Given a source, gets the ref name and checks whether the local CAS # contains it. # # Args: # source (Source): Source to check # # Returns: # (bool): whether the CAS contains this source or not # def contains(self, source): ref = source._get_source_name() path = self._source_path(ref) if not os.path.exists(path): return False # check files source_proto = self._get_source(ref) return self.cas.contains_directory(source_proto.files) # commit() # # Given a source, it stages and commits it to the local CAS. # # Args: # source: last source def commit(self, source): ref = source._get_source_name() vdir = CasBasedDirectory(self.cas) if not source.BST_STAGE_VIRTUAL_DIRECTORY: with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir: source._stage(tmpdir) vdir.import_files(tmpdir, collect_result=False) else: source._stage(vdir) self._store_source(ref, vdir._get_digest()) # export() # # Exports a source in the CAS to a virtual directory # # Args: # source (Source): source we want to export # # Returns: # CASBasedDirectory def export(self, source): ref = source._get_source_name() source = self._get_source(ref) return CasBasedDirectory(self.cas, digest=source.files) # pull() # # Attempts to pull sources from configure remote source caches. # # Args: # source (Source): The source we want to fetch # progress (callable|None): The progress callback # # Returns: # (bool): True if pull successful, False if not def pull(self, source): ref = source._get_source_name() project = source._get_project() display_key = source._get_brief_display_key() index_remotes, storage_remotes = self.get_remotes(project.name, False) # First fetch the source directory digest so we know what to pull source_digest = None for remote in index_remotes: remote.init() source.status("Pulling source {} <- {}".format(display_key, remote)) source_digest = self._pull_source(ref, remote) if source_digest is None: source.info("Remote source service ({}) does not have source {} cached".format(remote, display_key)) continue if not source_digest: return False for remote in storage_remotes: remote.init() source.status("Pulling data for source {} <- {}".format(display_key, remote)) try: # Fetch source blobs self.cas.fetch_directory(remote, source_digest) source.info("Pulled source {} <- {}".format(display_key, remote)) return True except BlobNotFound as e: # Not all blobs are available on this remote source.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) continue except CASError as e: raise SourceCacheError("Failed to pull source {}: {}".format(display_key, e), temporary=True) from e return False # push() # # Push a source to configured remote source caches # # Args: # source (Source): source to push # # Returns: # (Bool): whether it pushed to a remote source cache # def push(self, source): ref = source._get_source_name() project = source._get_project() index_remotes, storage_remotes = self.get_remotes(project.name, True) pushed_storage = False pushed_index = False display_key = source._get_brief_display_key() for remote in storage_remotes: remote.init() source.status("Pushing data for source {} -> {}".format(display_key, remote)) source_proto = self._get_source(ref) try: self.cas._send_directory(remote, source_proto.files) pushed_storage = True except CASRemoteError: source.info("Failed to push source files {} -> {}".format(display_key, remote)) continue for remote in index_remotes: remote.init() source.status("Pushing source {} -> {}".format(display_key, remote)) # check whether cache has files already if self._pull_source(ref, remote) is not None: source.info("Remote ({}) already has source {} cached".format(remote, display_key)) continue if not self._push_source(ref, remote): source.info("Failed to push source metadata {} -> {}".format(display_key, remote)) continue source.info("Pushed source {} -> {}".format(display_key, remote)) pushed_index = True return pushed_index and pushed_storage def _store_source(self, ref, digest): source_proto = source_pb2.Source() source_proto.files.CopyFrom(digest) self._store_proto(source_proto, ref) def _store_proto(self, proto, ref): path = self._source_path(ref) os.makedirs(os.path.dirname(path), exist_ok=True) with utils.save_file_atomic(path, "w+b") as f: f.write(proto.SerializeToString()) def _get_source(self, ref): path = self._source_path(ref) source_proto = source_pb2.Source() try: with open(path, "r+b") as f: source_proto.ParseFromString(f.read()) return source_proto except FileNotFoundError as e: raise SourceCacheError("Attempted to access unavailable source: {}".format(e)) from e def _source_path(self, ref): return os.path.join(self._basedir, ref) def _pull_source(self, source_ref, remote): uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(source_ref) remote.init() try: response = remote.fetch_directory([uri]) except AssetCacheError as e: raise SourceCacheError("Failed to pull source: {}".format(e), temporary=True) from e if response: self._store_source(source_ref, response.root_directory_digest) return response.root_directory_digest return None def _push_source(self, source_ref, remote): uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(source_ref) remote.init() source_proto = self._get_source(source_ref) try: remote.push_directory([uri], source_proto.files) except AssetCacheError as e: raise SourceCacheError("Failed to push source: {}".format(e), temporary=True) from e return True apache-buildstream-27ae392/src/buildstream/_state.py000066400000000000000000000403541514607367700225630ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import datetime from typing import Optional, Tuple, List, Dict, Callable from .types import _DisplayKey # TaskGroup # # The state data stored for a group of tasks (usually scheduler queues) # # Args: # name: The name of the Task Group, e.g. 'build' # state: The state object # complete_name: Optional name for frontend status rendering, e.g. 'built' # class TaskGroup: def __init__(self, name: str, state: "State", complete_name: Optional[str] = None) -> None: # # Public members # self.name: str = name # The name of tasks in this group self.complete_name: Optional[str] = complete_name # Optional name for frontend status rendering, e.g. 'built' self.processed_tasks: int = 0 # Number of processed tasks self.skipped_tasks: int = 0 # Number of skipped tasks self.failed_tasks: List[str] = [] # List of element full names which failed # # Private members # self._state: "State" = state ########################################### # Core-facing APIs to drive notifications # ########################################### # add_processed_task() # # Update the TaskGroup's count of processed tasks and notify of changes # # This is a core-facing API and should not be called from the frontend # def add_processed_task(self) -> None: self.processed_tasks += 1 for cb in self._state._task_groups_changed_cbs: cb() # add_skipped_task() # # Update the TaskGroup's count of skipped tasks and notify of changes # # This is a core-facing API and should not be called from the frontend # def add_skipped_task(self) -> None: self.skipped_tasks += 1 for cb in self._state._task_groups_changed_cbs: cb() # add_failed_task() # # Update the TaskGroup's list of failed tasks and notify of changes # # Args: # full_name: The full name of the task, distinguishing # it from other tasks with the same action name # e.g. an element's name. # # This is a core-facing API and should not be called from the frontend # def add_failed_task(self, full_name: str) -> None: self.failed_tasks.append(full_name) for cb in self._state._task_groups_changed_cbs: cb() # Task # # The state data stored for an individual task # # Args: # state: The State object # task_id: The unique identifier of the task # action_name: The name of the action, e.g. 'build' # full_name: The full name of the task, distinguishing # it from other tasks with the same action name # e.g. an element's name. # elapsed_offset: The time the task started, relative to # buildstream's start time. class Task: def __init__( self, state: "State", task_id: str, action_name: str, full_name: str, elapsed_offset: datetime.timedelta ) -> None: # # Public members # self.id: str = task_id self.action_name: str = action_name self.full_name: str = full_name self.elapsed_offset: datetime.timedelta = elapsed_offset self.current_progress: Optional[int] = None self.maximum_progress: Optional[int] = None # # Private members # self._state: "State" = state self._task_changed_cb: Optional[Callable[[], None]] = None # Callback to call when something could be rendered ############################################## # Core-facing APIs for driving notifications # ############################################## # set_task_changed_callback() # # Sets the callback to be called when this task has # changed. # # This is just a convenience codepath for the Messenger object # run simple tasks outside of the scheduler context, rather # than connecting to the State callbacks which are there for the # purpose of the frontend to get notifications of task progress. # # Args: # callback: The callback to call when progress changed # def set_task_changed_callback(self, callback: Optional[Callable[[], None]]) -> None: self._task_changed_cb = callback # set_maximum_progress() # # Sets the maximum progress possible for this task. # # Args: # progress: The maximum progress possible for this task # def set_maximum_progress(self, progress: int) -> None: self.maximum_progress = progress self._notify_task_changed() # set_current_progress() # # Sets the current progress of the task, this should # be a number between 0 and the maximum progress, if a # maximum progress has been set. # # Args: # progress: The current progress # def set_current_progress(self, progress: int) -> None: self.current_progress = progress self._notify_task_changed() # add_current_progress() # # A convenience function for incrementing the current # progress of this task by 1. # def add_current_progress(self) -> None: if self.current_progress is None: new_progress = 1 else: new_progress = self.current_progress + 1 self.set_current_progress(new_progress) ############################################## # Private methods # ############################################## def _notify_task_changed(self) -> None: for cb in self._state._task_changed_cbs: cb(self.id) if self._task_changed_cb: self._task_changed_cb() # State # # The state data that is stored for the purpose of sharing with the frontend. # # BuildStream's Core is responsible for making changes to this data. # BuildStream's Frontend may register callbacks with State to be notified # when parts of State change, and read State to know what has changed. # # Args: # session_start: The time the session started # class State: def __init__(self, session_start: datetime.datetime) -> None: # # Public members # self.task_groups: Dict[str, TaskGroup] = {} # Dictionary of active task groups by group name self.tasks: Dict[str, Task] = {} # Dictionary of active tasks by unique task ID # # Private members # self._session_start: datetime.datetime = session_start self._task_added_cbs: List[Callable[[str], None]] = [] self._task_removed_cbs: List[Callable[[str], None]] = [] self._task_changed_cbs: List[Callable[[str], None]] = [] self._task_failed_cbs: List[Callable[[str, Optional[Tuple[int, _DisplayKey]]], None]] = [] self._task_groups_changed_cbs: List[Callable[[], None]] = [] ##################################### # Frontend-facing notification APIs # ##################################### # register_task_added_callback() # # Registers a callback to be notified when a task is added # # Args: # callback: The callback to be notified # # Callback Args: # task_id: The unique identifier of the task # def register_task_added_callback(self, callback: Callable[[str], None]) -> None: self._task_added_cbs.append(callback) # unregister_task_added_callback() # # Unregisters a callback previously registered by # register_task_added_callback() # # Args: # callback: The callback to be removed # def unregister_task_added_callback(self, callback: Callable[[str], None]) -> None: self._task_added_cbs.remove(callback) # register_task_removed_callback() # # Registers a callback to be notified when a task is removed # # Args: # callback: The callback to be notified # # Callback Args: # task_id: The unique identifier of the task # def register_task_removed_callback(self, callback: Callable[[str], None]) -> None: self._task_removed_cbs.append(callback) # unregister_task_removed_callback() # # Unregisters a callback previously registered by # register_task_removed_callback() # # Args: # callback: The callback to be notified # def unregister_task_removed_callback(self, callback: Callable[[str], None]) -> None: self._task_removed_cbs.remove(callback) # register_task_changed_callback() # # Register a callback to be notified when a task has changed # # Args: # callback: The callback to be notified # # Callback Args: # task_id: The unique identifier of the task # def register_task_changed_callback(self, callback: Callable[[str], None]) -> None: self._task_changed_cbs.append(callback) # unregister_task_changed_callback() # # Unregisters a callback previously registered by # register_task_changed_callback() # # Args: # callback: The callback to be notified # def unregister_task_changed_callback(self, callback: Callable[[str], None]) -> None: self._task_changed_cbs.remove(callback) # register_task_failed_callback() # # Registers a callback to be notified when a task has failed # # Args: # callback (function): The callback to be notified # # Callback Args: # task_id: The unique identifier of the task # element: (optionally) The element unique_id and DisplayKey of an element job # def register_task_failed_callback( self, callback: Callable[[str, Optional[Tuple[int, _DisplayKey]]], None] ) -> None: self._task_failed_cbs.append(callback) # unregister_task_failed_callback() # # Unregisters a callback previously registered by # register_task_failed_callback() # # Args: # callback (function): The callback to be removed # def unregister_task_failed_callback( self, callback: Callable[[str, Optional[Tuple[int, _DisplayKey]]], None] ) -> None: self._task_failed_cbs.remove(callback) # register_task_groups_changed_callback() # # Registers a callback to be notified whenever the task groups info has changed # # Args: # callback: The callback to be notified # # Callback Args: # task_id: The unique identifier of the task # element: (optionally) The element unique_id and DisplayKey of an element job # def register_task_groups_changed_callback(self, callback: Callable[[], None]) -> None: self._task_groups_changed_cbs.append(callback) # unregister_task_groups_changed_callback() # # Unregisters a callback previously registered by register_task_groups_changed_callback() # # Args: # callback (function): The callback to be removed # def unregister_task_groups_changed_callback(self, callback: Callable[[], None]) -> None: self._task_groups_changed_cbs.remove(callback) ############################################## # Core-facing APIs for driving notifications # ############################################## # add_task_group() # # Notification that a new task group has been added # # This is a core-facing API and should not be called from the frontend # # Args: # name (str): The name of the task group, e.g. 'build' # complete_name (str): Optional name to be used for frontend status rendering, e.g. 'built' # # Returns: # TaskGroup: The task group created # def add_task_group(self, name, complete_name=None) -> TaskGroup: assert name not in self.task_groups, "Trying to add task group '{}' to '{}'".format(name, self.task_groups) group = TaskGroup(name, self, complete_name) self.task_groups[name] = group return group # remove_task_group() # # Notification that a task group has been removed # # This is a core-facing API and should not be called from the frontend # # Args: # name (str): The name of the task group, e.g. 'build' # def remove_task_group(self, name) -> None: # Rely on 'del' to raise an error when removing nonexistent task groups del self.task_groups[name] # add_task() # # Add a task and send appropriate notifications # # This is a core-facing API and should not be called from the frontend # # Args: # task_id: The unique identifier of the task # action_name: The name of the action, e.g. 'build' # full_name: The full name of the task, distinguishing # it from other tasks with the same action name # e.g. an element's name. # elapsed_offset (timedelta): (Optional) The time the task started, relative # to buildstream's start time. Note scheduler tasks # use this as they don't report relative to wallclock time # if the Scheduler has been suspended. # # Returns: # The new task # def add_task( self, task_id: str, action_name: str, full_name: str, elapsed_offset: Optional[datetime.timedelta] = None ) -> Task: assert task_id not in self.tasks, "Trying to add task '{}:{}' with ID '{}' to '{}'".format( action_name, full_name, task_id, self.tasks ) if not elapsed_offset: elapsed_offset = self.elapsed_time() task = Task(self, task_id, action_name, full_name, elapsed_offset) self.tasks[task_id] = task for cb in self._task_added_cbs: cb(task_id) return task # remove_task() # # Remove the task and send appropriate notifications # # This is a core-facing API and should not be called from the frontend # # Args: # task_id: The unique identifier of the task # def remove_task(self, task_id: str) -> None: # Rely on 'del' to raise an error when removing nonexistent tasks del self.tasks[task_id] for cb in self._task_removed_cbs: cb(task_id) # fail_task() # # Notify all registered callbacks that a task has failed. # # This is separate from the tasks changed callbacks because a failed task # requires the frontend to intervene to decide what happens next. # # This is a core-facing API and should not be called from the frontend # # Args: # task_id: The unique identifier of the task # element: (optionally) The element unique_id and display keys if an # element job # def fail_task(self, task_id: str, element: Optional[Tuple[int, _DisplayKey]] = None) -> None: for cb in self._task_failed_cbs: cb(task_id, element) # elapsed_time() # # Fetches the current session elapsed time # # Args: # start_time: Optional explicit start time, relative to caller. # # Returns: # The amount of time since the start of the session, # discounting any time spent while jobs were suspended if # start_time given relative to the Scheduler # def elapsed_time(self, start_time: Optional[datetime.datetime] = None) -> datetime.timedelta: time_now = datetime.datetime.now() if start_time is None: start_time = self._session_start or time_now return time_now - start_time # offset_start_time() # # Update the 'start' time of the application by a given offset # # This allows modifying the time spent building when BuildStream # gets paused then restarted, to give an accurate view of the real # time spend building. # # Args: # offset: the offset to add to the start time # def offset_start_time(self, offset: datetime.timedelta) -> None: self._session_start += offset apache-buildstream-27ae392/src/buildstream/_stream.py000066400000000000000000002435341514607367700227430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jürg Billeter # Tristan Maat import itertools import os import sys import stat import shlex import shutil import tarfile import tempfile from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError from ._scheduler import ( Scheduler, SchedStatus, TrackQueue, CacheQueryQueue, FetchQueue, SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue, ) from .element import Element from ._profile import Topics, PROFILER from ._project import ProjectRefStorage from ._remotespec import RemoteSpec from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount from .plugin import Plugin from . import utils, node, _yaml, _site, _pipeline # Stream() # # This is the main, toplevel calling interface in BuildStream core. # # Args: # context (Context): The Context object # session_start (datetime): The time when the session started # session_start_callback (callable): A callback to invoke when the session starts # interrupt_callback (callable): A callback to invoke when we get interrupted # ticker_callback (callable): Invoked every second while running the scheduler # class Stream: def __init__( self, context, session_start, *, session_start_callback=None, interrupt_callback=None, ticker_callback=None ): # # Public members # self.targets = [] # Resolved target elements self.session_elements = [] # List of elements being processed this session self.total_elements = [] # Total list of elements based on targets self.queues = [] # Queue objects # # Private members # self._context = context self._artifacts = None self._elementsourcescache = None self._sourcecache = None self._project = None self._state = State(session_start) # Owned by Stream, used by Core to set state self._notification_queue = deque() context.messenger.set_state(self._state) self._scheduler = Scheduler(context, session_start, self._state, interrupt_callback, ticker_callback) self._session_start_callback = session_start_callback self._running = False self._terminated = False self._suspended = False # init() # # Initialization of Stream that has side-effects that require it to be # performed after the Stream is created. # def init(self): self._artifacts = self._context.artifactcache self._elementsourcescache = self._context.elementsourcescache self._sourcecache = self._context.sourcecache # cleanup() # # Cleans up application state # def cleanup(self): # Reset the element loader state Element._reset_load_state() # Reset global state in node.pyx, this is for the sake of # test isolation. node._reset_global_state() # set_project() # # Set the top-level project. # # Args: # project (Project): The Project object # def set_project(self, project): assert self._project is None self._project = project # load_selection() # # An all purpose method for loading a selection of elements, this # is primarily useful for the frontend to implement `bst show` # and `bst shell`. # # Args: # targets: Targets to pull # selection: The selection mode for the specified targets (_PipelineSelection) # except_targets: Specified targets to except from fetching # load_artifacts (bool): Whether to load artifacts with artifact names # connect_artifact_cache: Whether to try to contact remote artifact caches # connect_source_cache: Whether to try to contact remote source caches # artifact_remotes: Artifact cache remotes specified on the commmand line # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # need_state: Whether resolving element state is required # # Returns: # (list of Element): The selected elements def load_selection( self, targets: Iterable[str], *, selection: str = _PipelineSelection.NONE, except_targets: Iterable[str] = (), load_artifacts: bool = False, connect_artifact_cache: bool = False, connect_source_cache: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, need_state: bool = True, ): with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( targets, selection=selection, except_targets=except_targets, load_artifacts=load_artifacts, connect_artifact_cache=connect_artifact_cache, connect_source_cache=connect_source_cache, artifact_remotes=artifact_remotes, source_remotes=source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, need_state=need_state, ) return target_objects # query_cache() # # Query the artifact and source caches to determine the cache status # of the specified elements. # # Args: # elements (list of Element): The elements to check # sources_of_cached_elements (bool): True to query the source cache for elements with a cached artifact # only_sources (bool): True to only query the source cache # def query_cache(self, elements, *, sources_of_cached_elements=False, only_sources=False, need_state=False): # It doesn't make sense to combine these flags assert not sources_of_cached_elements or not only_sources with self._context.messenger.simple_task("Query cache", silent_nested=True) as task: if need_state: # Enqueue complete build plan as this is required to determine `buildable` status. plan = list(_pipeline.dependencies(elements, _Scope.ALL)) else: plan = elements if self._context.remote_cache_spec: # Parallelize cache queries if a remote cache is configured self._reset() self._add_queue( CacheQueryQueue( self._scheduler, sources=only_sources, sources_if_cached=sources_of_cached_elements ), track=True, ) self._enqueue_plan(plan) self._run() else: task.set_maximum_progress(len(plan)) for element in plan: if element._can_query_cache(): # Cache status already available. # This is the case for artifact elements, which load the # artifact early on. pass elif not only_sources and element._get_cache_key(strength=_KeyStrength.WEAK): element._load_artifact(pull=False) if ( sources_of_cached_elements or not element._can_query_cache() or not element._cached_success() ): element._query_source_cache() if not element._pull_pending(): element._load_artifact_done() elif element._has_all_sources_resolved(): element._query_source_cache() task.add_current_progress() # shell() # # Run a shell # # Args: # target: The name of the element to run the shell for # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) # prompt: A function to return the prompt to display in the shell # unique_id: (str): A unique_id to use to lookup an Element instance # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds # command (list): An argv to launch in the sandbox, or None # usebuildtree (bool): Whether to use a buildtree as the source, given cli option # artifact_remotes: Artifact cache remotes specified on the commmand line # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # Returns: # (int): The exit code of the launched shell # def shell( self, target: str, scope: int, prompt: Callable[[Element], str], *, unique_id: Optional[str] = None, mounts: Optional[List[_HostMount]] = None, isolate: bool = False, command: Optional[List[str]] = None, usebuildtree: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, ): element: Element # Load the Element via the unique_id if given if unique_id and target is None: element = Plugin._lookup(unique_id) else: if usebuildtree: selection = _PipelineSelection.NONE elif scope == _Scope.BUILD: selection = _PipelineSelection.BUILD else: selection = _PipelineSelection.RUN try: elements = self.load_selection( (target,), selection=selection, load_artifacts=True, connect_artifact_cache=True, connect_source_cache=True, artifact_remotes=artifact_remotes, source_remotes=source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) except StreamError as e: if e.reason == "deps-not-supported": raise StreamError( "Only buildtrees are supported with artifact names", detail="Use the --build and --use-buildtree options to shell into a cached build tree", reason="only-buildtrees-supported", ) from e raise # Get element to stage from `targets` list. # If scope is BUILD, it will not be in the `elements` list. assert len(self.targets) == 1 element = self.targets[0] element._set_required(scope) if scope == _Scope.BUILD and not usebuildtree: pull_elements = [element] + elements else: pull_elements = elements # Check whether the required elements are cached, and then # try to pull them if they are not already cached. # self.query_cache(pull_elements) self._pull_missing_artifacts(pull_elements) # We dont need dependency artifacts to shell into a cached build tree if not usebuildtree: missing_deps = [dep for dep in _pipeline.dependencies([element], scope) if not dep._cached()] if missing_deps: raise StreamError( "Elements need to be built or downloaded before staging a shell environment", detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))), reason="shell-missing-deps", ) # Check if we require a pull queue attempt, with given artifact state and context if usebuildtree: if not element._cached_buildroot(): if not element._cached(): message = "Artifact not cached locally or in available remotes" reason = "missing-buildtree-artifact-not-cached" elif element._buildroot_exists(): message = "Buildtree is not cached locally or in available remotes" reason = "missing-buildtree-artifact-buildtree-not-cached" else: message = "Artifact was created without buildtree" reason = "missing-buildtree-artifact-created-without-buildtree" raise StreamError(message, reason=reason) # Raise warning if the element is cached in a failed state if element._cached_failure(): self._context.messenger.warn("using a buildtree from a failed build.") # Ensure we have our sources if we are launching a build shell if scope == _Scope.BUILD and not usebuildtree: self.query_cache([element], only_sources=True) self._fetch([element]) _pipeline.assert_sources_cached(self._context, [element]) return element._shell( scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree ) # build() # # Builds (assembles) elements in the pipeline. # # Args: # targets: Targets to build # selection: The selection mode for the specified targets (_PipelineSelection) # ignore_junction_targets: Whether junction targets should be filtered out # artifact_remotes: Artifact cache remotes specified on the commmand line # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # retry_failed: Try to build elements for which a failed build artifact is found # # If `remote` specified as None, then regular configuration will be used # to determine where to push artifacts to. # def build( self, targets: Iterable[str], *, selection: str = _PipelineSelection.NONE, ignore_junction_targets: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, retry_failed: bool = False, ): # Flag the build state self._context.build = True # Override user configuration if --retry-failed is specified if retry_failed: self._context.build_retry_failed = True elements = self._load( targets, selection=selection, ignore_junction_targets=ignore_junction_targets, dynamic_plan=True, connect_artifact_cache=True, connect_source_cache=True, artifact_remotes=artifact_remotes, source_remotes=source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) # Assert that the elements are consistent _pipeline.assert_consistent(self._context, elements) source_push_enabled = self._sourcecache.has_push_remotes() # If source push is enabled, the source cache status of all elements # is required, independent of whether the artifact is already available. self.query_cache(elements, sources_of_cached_elements=source_push_enabled) # Now construct the queues # self._reset() if self._artifacts.has_fetch_remotes(): self._add_queue(PullQueue(self._scheduler)) self._add_queue(FetchQueue(self._scheduler, skip_cached=True)) self._add_queue(BuildQueue(self._scheduler, imperative=True)) if self._artifacts.has_push_remotes(): self._add_queue(ArtifactPushQueue(self._scheduler, skip_uncached=True)) if source_push_enabled: self._add_queue(SourcePushQueue(self._scheduler)) # Enqueue elements self._enqueue_plan(elements) self._run(announce_session=True) # fetch() # # Fetches sources on the pipeline. # # Args: # targets: Targets to fetch # selection: The selection mode for the specified targets (_PipelineSelection) # except_targets: Specified targets to except from fetching # source_remotes: Source cache remotes specified on the commmand line # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # def fetch( self, targets: Iterable[str], *, selection: str = _PipelineSelection.NONE, except_targets: Iterable[str] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_source_remotes: bool = False, ): if self._context.remote_cache_spec: self._context.messenger.warn( "Cache Storage Service is configured, fetched sources may not be available in the local cache" ) elements = self._load( targets, selection=selection, except_targets=except_targets, connect_source_cache=True, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) self.query_cache(elements, only_sources=True) # Delegated to a shared fetch method self._fetch(elements, announce_session=True) # track() # # Tracks all the sources of the selected elements. # # Args: # targets (list of str): Targets to track # selection (_PipelineSelection): The selection mode for the specified targets # except_targets (list of str): Specified targets to except from tracking # cross_junctions (bool): Whether tracking should cross junction boundaries # # If no error is encountered while tracking, then the project files # are rewritten inline. # def track(self, targets, *, selection=_PipelineSelection.REDIRECT, except_targets=None, cross_junctions=False): elements = self._load_tracking( targets, selection=selection, except_targets=except_targets, cross_junctions=cross_junctions ) # Note: We do not currently need to initialize the state of an # element before it is tracked, since tracking can be done # irrespective of source/artifact condition. Once an element # is tracked, its state must be fully updated in either case, # and we anyway don't do anything else with it. self._reset() track_queue = TrackQueue(self._scheduler) self._add_queue(track_queue, track=True) self._enqueue_plan(elements, queue=track_queue) self._run(announce_session=True) # source_push() # # Push sources. # # Args: # targets (list of str): Targets to push # selection (_PipelineSelection): The selection mode for the specified targets # except_targets: Specified targets to except from pushing # source_remotes: Source cache remotes specified on the commmand line # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # # If `remote` specified as None, then regular configuration will be used # to determine where to push sources to. # # If any of the given targets are missing their expected sources, # a fetch queue will be created if user context and available remotes allow for # attempting to fetch them. # def source_push( self, targets, *, selection=_PipelineSelection.NONE, except_targets: Iterable[str] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_source_remotes: bool = False, ): elements = self._load( targets, selection=selection, except_targets=except_targets, load_artifacts=True, connect_source_cache=True, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) self.query_cache(elements, only_sources=True) if not self._sourcecache.has_push_remotes(): raise StreamError("No source caches available for pushing sources") _pipeline.assert_consistent(self._context, elements) self._add_queue(FetchQueue(self._scheduler)) self._add_queue(SourcePushQueue(self._scheduler, imperative=True)) self._enqueue_plan(elements) self._run(announce_session=True) # pull() # # Pulls artifacts from remote artifact server(s) # # Args: # targets: Targets to pull # selection: The selection mode for the specified targets (_PipelineSelection) # ignore_junction_targets: Whether junction targets should be filtered out # artifact_remotes: Artifact cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # def pull( self, targets: Iterable[str], *, selection: str = _PipelineSelection.NONE, ignore_junction_targets: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ): if self._context.remote_cache_spec: self._context.messenger.warn( "Cache Storage Service is configured, pulled artifacts may not be available in the local cache" ) elements = self._load( targets, selection=selection, ignore_junction_targets=ignore_junction_targets, load_artifacts=True, attempt_artifact_metadata=True, connect_artifact_cache=True, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) if not self._artifacts.has_fetch_remotes(): raise StreamError("No artifact caches available for pulling artifacts") _pipeline.assert_consistent(self._context, elements) self.query_cache(elements) self._reset() self._add_queue(PullQueue(self._scheduler)) self._enqueue_plan(elements) self._run(announce_session=True) # push() # # Pushes artifacts to remote artifact server(s), pulling them first if necessary, # possibly from different remotes. # # Args: # targets (list of str): Targets to push # selection (_PipelineSelection): The selection mode for the specified targets # ignore_junction_targets (bool): Whether junction targets should be filtered out # artifact_remotes: Artifact cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # # If any of the given targets are missing their expected buildtree artifact, # a pull queue will be created if user context and available remotes allow for # attempting to fetch them. # def push( self, targets: Iterable[str], *, selection: str = _PipelineSelection.NONE, ignore_junction_targets: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ): elements = self._load( targets, selection=selection, ignore_junction_targets=ignore_junction_targets, load_artifacts=True, connect_artifact_cache=True, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) if not self._artifacts.has_push_remotes(): raise StreamError("No artifact caches available for pushing artifacts") _pipeline.assert_consistent(self._context, elements) self.query_cache(elements) self._reset() self._add_queue(PullQueue(self._scheduler)) self._add_queue(ArtifactPushQueue(self._scheduler, imperative=True)) self._enqueue_plan(elements) self._run(announce_session=True) # checkout() # # Checkout target artifact to the specified location # # Args: # target: Target to checkout # location: Location to checkout the artifact to # force: Whether files can be overwritten if necessary # selection: The selection mode for the specified targets (_PipelineSelection) # integrate: Whether to run integration commands # hardlinks: Whether checking out files hardlinked to # their artifacts is acceptable # tar: If true, a tarball from the artifact contents will # be created, otherwise the file tree of the artifact # will be placed at the given location. If true and # location is '-', the tarball will be dumped on the # standard output. # artifact_remotes: Artifact cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # def checkout( self, target: str, *, location: Optional[str] = None, force: bool = False, selection: str = _PipelineSelection.RUN, integrate: bool = True, hardlinks: bool = False, compression: str = "", tar: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ): elements = self._load( (target,), selection=selection, load_artifacts=True, attempt_artifact_metadata=True, connect_artifact_cache=True, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) # self.targets contains a list of the loaded target objects # if we specify --deps build, Stream._load() will return a list # of build dependency objects, however, we need to prepare a sandbox # with the target (which has had its appropriate dependencies loaded) element: Element = self.targets[0] self._check_location_writable(location, force=force, tar=tar) # Check whether the required elements are cached, and then # try to pull them if they are not already cached. # self.query_cache(elements) self._pull_missing_artifacts(elements) try: scope = { _PipelineSelection.RUN: _Scope.RUN, _PipelineSelection.BUILD: _Scope.BUILD, _PipelineSelection.NONE: _Scope.NONE, _PipelineSelection.ALL: _Scope.ALL, } with element._prepare_sandbox(scope=scope[selection], integrate=integrate) as sandbox: # Copy or move the sandbox to the target directory virdir = sandbox.get_virtual_directory() self._export_artifact(tar, location, compression, element, hardlinks, virdir) except BstError as e: raise StreamError( "Error while staging dependencies into a sandbox" ": '{}'".format(e), detail=e.detail, reason=e.reason ) from e # _export_artifact() # # Export the files of the artifact/a tarball to a virtual directory # # Args: # tar (bool): Whether we want to create a tarfile # location (str): The name of the directory/the tarfile we want to export to/create # compression (str): The type of compression for the tarball # target (Element/ArtifactElement): The Element/ArtifactElement we want to checkout # hardlinks (bool): Whether to checkout hardlinks instead of copying # virdir (Directory): The sandbox's root directory as a virtual directory # def _export_artifact(self, tar, location, compression, target, hardlinks, virdir): if not tar: with target.timed_activity("Checking out files in '{}'".format(location)): try: if hardlinks: try: utils.safe_remove(location) except OSError as e: raise StreamError("Failed to remove checkout directory: {}".format(e)) from e virdir._export_files(location, can_link=True, can_destroy=True) else: virdir._export_files(location) except OSError as e: raise StreamError("Failed to checkout files: '{}'".format(e)) from e else: to_stdout = location == "-" mode = _handle_compression(compression, to_stream=to_stdout) with target.timed_activity("Creating tarball"): if to_stdout: # Save the stdout FD to restore later saved_fd = os.dup(sys.stdout.fileno()) try: with os.fdopen(sys.stdout.fileno(), "wb") as fo: with tarfile.open(fileobj=fo, mode=mode) as tf: virdir.export_to_tar(tf, ".") finally: # No matter what, restore stdout for further use os.dup2(saved_fd, sys.stdout.fileno()) os.close(saved_fd) else: with tarfile.open(location, mode=mode) as tf: virdir.export_to_tar(tf, ".") # artifact_show() # # Show cached artifacts # # Args: # targets (str): Targets to show the cached state of # artifact_remotes: Artifact cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # def artifact_show( self, targets, *, selection=_PipelineSelection.NONE, artifact_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ): # Obtain list of Element and/or ArtifactElement objects target_objects = self.load_selection( targets, selection=selection, connect_artifact_cache=True, load_artifacts=True, artifact_remotes=artifact_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) self.query_cache(target_objects) not_cached_locally = [element for element in target_objects if not element._cached()] if self._artifacts.has_fetch_remotes(): self._resolve_cached_remotely(not_cached_locally) return target_objects # artifact_log() # # Show the full log of an artifact # # Args: # targets (str): Targets to view the logs of # # Returns: # logsdir (list): A list of CasBasedDirectory objects containing artifact logs # def artifact_log(self, targets): # Return list of Element and/or ArtifactElement objects target_objects = self.load_selection(targets, selection=_PipelineSelection.NONE, load_artifacts=True) self.query_cache(target_objects) artifact_logs = {} for obj in target_objects: ref = obj.get_artifact_name() if not obj._cached(): self._context.messenger.warn("{} is not cached".format(ref)) continue if not obj._cached_logs(): self._context.messenger.warn("{} is cached without log files".format(ref)) continue artifact_logs[obj.name] = obj._get_logs() return artifact_logs # artifact_list_contents() # # Show a list of content of an artifact # # Args: # targets (str): Targets to view the contents of # # Returns: # elements_to_files (Dict[str, Directory]): A list of tuples of the artifact name and it's contents # def artifact_list_contents(self, targets): # Return list of Element and/or ArtifactElement objects target_objects = self.load_selection(targets, selection=_PipelineSelection.NONE, load_artifacts=True) self.query_cache(target_objects) elements_to_files = {} for obj in target_objects: ref = obj.get_artifact_name() if not obj._cached(): self._context.messenger.warn("{} is not cached".format(ref)) obj.name = {ref: "No artifact cached"} continue if isinstance(obj, ArtifactElement): obj.name = ref # Just hand over a Directory here artifact = obj._get_artifact() files = artifact.get_files() elements_to_files[obj.name] = files return elements_to_files # artifact_delete() # # Remove artifacts from the local cache # # Args: # targets (str): Targets to remove # def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE): # Return list of Element and/or ArtifactElement objects target_objects = self.load_selection(targets, selection=selection, load_artifacts=True) self.query_cache(target_objects) # Some of the targets may refer to the same key, so first obtain a # set of the refs to be removed. remove_refs = set() for obj in target_objects: for key_strength in [_KeyStrength.STRONG, _KeyStrength.WEAK]: key = obj._get_cache_key(strength=key_strength) remove_refs.add(obj.get_artifact_name(key=key)) ref_removed = False for ref in remove_refs: try: self._artifacts.remove(ref) except ArtifactError as e: self._context.messenger.warn(str(e)) continue self._context.messenger.info("Removed: {}".format(ref)) ref_removed = True if not ref_removed: self._context.messenger.info("No artifacts were removed") # source_checkout() # # Checkout sources of the target element to the specified location # # Args: # target: The target element whose sources to checkout # location: Location to checkout the sources to # force: Whether to overwrite existing directories/tarfiles # deps: The selection mode for the specified targets (_PipelineSelection) # except_targets: List of targets to except from staging # tar: Whether to write a tarfile holding the checkout contents # compression: The type of compression for tarball # include_build_scripts: Whether to include build scripts in the checkout # source_remotes: Source cache remotes specified on the commmand line # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # def source_checkout( self, target: str, *, location: Optional[str] = None, force: bool = False, deps=_PipelineSelection.NONE, except_targets: Iterable[str] = (), tar: bool = False, compression: Optional[str] = None, include_build_scripts: bool = False, source_remotes: Iterable[RemoteSpec] = (), ignore_project_source_remotes: bool = False, ): self._check_location_writable(location, force=force, tar=tar) elements = self._load( (target,), selection=deps, except_targets=except_targets, connect_source_cache=True, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) # Assert all sources are cached in the source dir self.query_cache(elements, only_sources=True) self._fetch(elements) _pipeline.assert_sources_cached(self._context, elements) # Stage all sources determined by scope try: self._source_checkout(elements, location, force, deps, tar, compression, include_build_scripts) except BstError as e: raise StreamError( "Error while writing sources" ": '{}'".format(e), detail=e.detail, reason=e.reason ) from e self._context.messenger.info("Checked out sources to '{}'".format(location)) # workspace_open # # Open a project workspace # # Args: # targets (list): List of target elements to open workspaces for # no_checkout (bool): Whether to skip checking out the source # force (bool): Whether to ignore contents in an existing directory # custom_dir (str): Custom location to create a workspace or false to use default location. # source_remotes: Source cache remotes specified on the commmand line # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # def workspace_open( self, targets: Iterable[str], *, no_checkout: bool = False, force: bool = False, custom_dir: Optional[str] = None, source_remotes: Iterable[RemoteSpec] = (), ignore_project_source_remotes: bool = False, ): # This function is a little funny but it is trying to be as atomic as possible. elements = self._load( targets, selection=_PipelineSelection.REDIRECT, connect_source_cache=True, source_remotes=source_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) workspaces = self._context.get_workspaces() # If we're going to checkout, we need at least a fetch, # if not no_checkout: self.query_cache(elements, only_sources=True) self._fetch(elements, fetch_original=True) expanded_directories = [] # To try to be more atomic, loop through the elements and raise any errors we can early for target in elements: if not list(target.sources()): build_depends = [x.name for x in target._dependencies(_Scope.BUILD, recurse=False)] if not build_depends: raise StreamError("The element {} has no sources".format(target.name)) detail = "Try opening a workspace on one of its dependencies instead:\n" detail += " \n".join(build_depends) raise StreamError("The element {} has no sources".format(target.name), detail=detail) # Check for workspace config workspace = workspaces.get_workspace(target._get_full_name()) if workspace: if not force: raise StreamError( "Element '{}' already has an open workspace defined at: {}".format( target.name, workspace.get_absolute_path() ) ) if not no_checkout: target.warn( "Replacing existing workspace for element '{}' defined at: {}".format( target.name, workspace.get_absolute_path() ) ) self.workspace_close(target._get_full_name(), remove_dir=not no_checkout) if not custom_dir: directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name)) if directory[-4:] == ".bst": directory = directory[:-4] expanded_directories.append(directory) if custom_dir: if len(elements) != 1: raise StreamError( "Exactly one element can be given if --directory is used", reason="directory-with-multiple-elements", ) directory = os.path.abspath(custom_dir) expanded_directories = [ directory, ] else: # If this fails it is a bug in what ever calls this, usually cli.py and so can not be tested for via the # run bst test mechanism. assert len(elements) == len(expanded_directories) for target, directory in zip(elements, expanded_directories): if os.path.exists(directory): if not os.path.isdir(directory): raise StreamError( "For element '{}', Directory path is not a directory: {}".format(target.name, directory), reason="bad-directory", ) if not (no_checkout or force) and os.listdir(directory): raise StreamError( "For element '{}', Directory path is not empty: {}".format(target.name, directory), reason="bad-directory", ) if os.listdir(directory): if force and not no_checkout: utils._force_rmtree(directory) # So far this function has tried to catch as many issues as possible with out making any changes # Now it does the bits that can not be made atomic. targetGenerator = zip(elements, expanded_directories) for target, directory in targetGenerator: self._context.messenger.status("Creating workspace for element {}".format(target.name)) workspace = workspaces.get_workspace(target._get_full_name()) if workspace and not no_checkout: workspaces.delete_workspace(target._get_full_name()) workspaces.save_config() utils._force_rmtree(directory) try: os.makedirs(directory, exist_ok=True) except OSError as e: todo_elements = " ".join([str(target.name) for target, directory_dict in targetGenerator]) if todo_elements: # This output should make creating the remaining workspaces as easy as possible. todo_elements = "\nDid not try to create workspaces for " + todo_elements raise StreamError( "Failed to create workspace directory: {}".format(e), reason="workspace-directory-failure", detail=todo_elements, ) from e workspaces.create_workspace(target, directory, checkout=not no_checkout) self._context.messenger.info("Created a workspace for element {}".format(target._get_full_name())) # workspace_close # # Close a project workspace # # Args: # element_name (str): The element name to close the workspace for # remove_dir (bool): Whether to remove the associated directory # def workspace_close(self, element_name, *, remove_dir): self._assert_project("Unable to locate workspaces") workspaces = self._context.get_workspaces() workspace = workspaces.get_workspace(element_name) # Remove workspace directory if prompted if remove_dir: with self._context.messenger.timed_activity( "Removing workspace directory {}".format(workspace.get_absolute_path()) ): try: shutil.rmtree(workspace.get_absolute_path()) except OSError as e: raise StreamError("Could not remove '{}': {}".format(workspace.get_absolute_path(), e)) from e # Delete the workspace and save the configuration workspaces.delete_workspace(element_name) workspaces.save_config() self._context.messenger.info("Closed workspace for {}".format(element_name)) # workspace_reset # # Reset a workspace to its original state, discarding any user # changes. # # Args: # targets (list of str): The target elements to reset the workspace for # soft (bool): Only set the workspace state to not prepared # def workspace_reset(self, targets, *, soft): self._assert_project("Unable to locate workspaces") elements = self._load(targets, selection=_PipelineSelection.REDIRECT) nonexisting = [] for element in elements: if not self.workspace_exists(element.name): nonexisting.append(element.name) if nonexisting: raise StreamError("Workspace does not exist", detail="\n".join(nonexisting)) workspaces = self._context.get_workspaces() for element in elements: workspace = workspaces.get_workspace(element._get_full_name()) workspace_path = workspace.get_absolute_path() if soft: workspace.last_build = None self._context.messenger.info( "Reset workspace state for {} at: {}".format(element.name, workspace_path) ) continue self.workspace_close(element._get_full_name(), remove_dir=True) workspaces.save_config() self.workspace_open([element._get_full_name()], no_checkout=False, force=True, custom_dir=workspace_path) # workspace_exists # # Check if a workspace exists # # Args: # element_name (str): The element name to close the workspace for, or None # # Returns: # (bool): True if the workspace exists # # If None is specified for `element_name`, then this will return # True if there are any existing workspaces. # def workspace_exists(self, element_name=None): self._assert_project("Unable to locate workspaces") workspaces = self._context.get_workspaces() if element_name: workspace = workspaces.get_workspace(element_name) if workspace: return True elif any(workspaces.list()): return True return False # workspace_list # # Serializes the workspaces and dumps them in YAML to stdout. # def workspace_list(self): self._assert_project("Unable to locate workspaces") workspaces = [] for element_name, workspace_ in self._context.get_workspaces().list(): workspace_detail = { "element": element_name, "directory": workspace_.get_absolute_path(), } workspaces.append(workspace_detail) _yaml.roundtrip_dump({"workspaces": workspaces}) # redirect_element_names() # # Takes a list of element names and returns a list where elements have been # redirected to their source elements if the element file exists, and just # the name, if not. # # Args: # elements (list of str): The element names to redirect # # Returns: # (list of str): The element names after redirecting # def redirect_element_names(self, elements): element_dir = self._project.element_path load_elements = [] output_elements = set() for e in elements: element_path = os.path.join(element_dir, e) if os.path.exists(element_path): load_elements.append(e) else: output_elements.add(e) if load_elements: loaded_elements = self._load(load_elements, selection=_PipelineSelection.REDIRECT) for e in loaded_elements: output_elements.add(e.name) return list(output_elements) # get_state() # # Get the State object owned by Stream # # Returns: # State: The State object def get_state(self): return self._state # get_default_target() # # Attempts to interpret which element the user intended to run a command on. # This is for commands that only accept a single target element and thus, # this only uses the workspace element (if invoked from workspace directory) # and does not use the project default targets. # def get_default_target(self): return self._project.get_default_target() if self._project else None # get_default_targets() # # Attempts to interpret which elements the user intended to run a command on. # This is for commands that accept multiple target elements. # def get_default_targets(self): self._assert_project("Unable to determine default targets") return self._project.get_default_targets() ############################################################# # Scheduler API forwarding # ############################################################# # running # # Whether the scheduler is running # @property def running(self): return self._running # suspended # # Whether the scheduler is currently suspended # @property def suspended(self): return self._suspended # terminated # # Whether the scheduler is currently terminated # @property def terminated(self): return self._terminated # terminate() # # Terminate jobs # def terminate(self): self._scheduler.terminate() self._terminated = True # quit() # # Quit the session, this will continue with any ongoing # jobs, use Stream.terminate() instead for cancellation # of ongoing jobs # def quit(self): self._scheduler.stop() # suspend() # # Context manager to suspend ongoing jobs # @contextmanager def suspend(self): self._scheduler.suspend() self._suspended = True yield self._suspended = False self._scheduler.resume() # retry_job() # # Retry the indicated job # # Args: # action_name: The unique identifier of the task # unique_id: A unique_id to load an Element instance # def retry_job(self, action_name: str, unique_id: str) -> None: element = Plugin._lookup(unique_id) # # Update the state task group, remove the failed element # group = self._state.task_groups[action_name] group.failed_tasks.remove(element._get_full_name()) # # Find the queue for this action name and requeue the element # queue = None for q in self.queues: if q.action_name == action_name: queue = q assert queue queue.enqueue([element]) # fetch_subprojects() # # Fetch subprojects as part of the project and element loading process. # # This is passed to the Project in order to handle loading of subprojects # # Args: # junctions (list of Element): The junctions to fetch # def fetch_subprojects(self, junctions): self._reset() queue = FetchQueue(self._scheduler) queue.enqueue(junctions) self.queues = [queue] self._run() ############################################################# # Private Methods # ############################################################# # _assert_project() # # Raises an assertion of a project was not loaded # # Args: # message: The user facing error message, e.g. "Unable to load elements" # # Raises: # A StreamError with reason "project-not-loaded" is raised if no project was loaded # def _assert_project(self, message: str) -> None: if not self._project: raise StreamError( message, detail="No project.conf or active workspace was located", reason="project-not-loaded" ) # _load_artifacts() # # Loads artifacts from target artifact refs # # Args: # artifact_names (list): List of target artifact names to load # # Returns: # (list): A list of loaded ArtifactElement # def _load_artifacts(self, artifact_names): with self._context.messenger.simple_task("Loading artifacts") as task: # Use a set here to avoid duplicates. # # ArtifactElement.new_from_artifact_name() will take care of ensuring # uniqueness of multiple artifact names which refer to the same artifact # (e.g., if both weak and strong names happen to be requested), here we # still need to ensure we generate a list that does not contain duplicates. # artifacts = set() for artifact_name in artifact_names: artifact = ArtifactElement.new_from_artifact_name(artifact_name, self._context, task) artifacts.add(artifact) ArtifactElement.clear_artifact_name_cache() ArtifactProject.clear_project_cache() return list(artifacts) # _load_elements() # # Loads elements from target names. # # This function is called with a list of lists, such that multiple # target groups may be specified. Element names specified in `targets` # are allowed to be redundant. # # Args: # target_groups (list of lists): Groups of toplevel targets to load # # Returns: # (tuple of lists): A tuple of Element object lists, grouped corresponding to target_groups # def _load_elements(self, target_groups): # First concatenate all the lists for the loader's sake targets = list(itertools.chain(*target_groups)) with PROFILER.profile(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, "-") for t in targets)): elements = self._project.load_elements(targets) # Now create element groups to match the input target groups elt_iter = iter(elements) element_groups = [[next(elt_iter) for i in range(len(group))] for group in target_groups] return tuple(element_groups) # _load_elements_from_targets # # Given the usual set of target element names/artifact refs, load # the `Element` objects required to describe the selection. # # The result is returned as a tuple - firstly the loaded normal # elements, secondly the loaded "excepting" elements and lastly # the loaded artifact elements. # # Args: # targets - The target element names/artifact refs # except_targets - The names of elements to except # rewritable - Whether to load the elements in re-writable mode # valid_artifact_names: Whether artifact names are valid # # Returns: # ([elements], [except_elements], [artifact_elements]) # def _load_elements_from_targets( self, targets: Iterable[str], except_targets: Iterable[str], *, rewritable: bool = False, valid_artifact_names: bool = False, ) -> Tuple[List[Element], List[Element], List[Element]]: # First determine which of the user specified targets are artifact # names and which are element names. element_names, artifact_names = self._expand_and_classify_targets( targets, valid_artifact_names=valid_artifact_names ) # We need a project in order to load elements if element_names: self._assert_project("Unable to load elements: {}".format(", ".join(element_names))) if self._project: self._project.load_context.set_rewritable(rewritable) # Load elements and except elements if element_names: elements, except_elements = self._load_elements([element_names, except_targets]) else: elements, except_elements = [], [] # Load artifacts if artifact_names: artifacts = self._load_artifacts(artifact_names) else: artifacts = [] return elements, except_elements, artifacts # _resolve_cached_remotely() # # Checks whether the listed elements are currently cached in # any of their respectively configured remotes. # # Args: # targets (list [Element]): The list of element targets # def _resolve_cached_remotely(self, targets): with self._context.messenger.simple_task("Querying remotes for cached status", silent_nested=True) as task: task.set_maximum_progress(len(targets)) for element in targets: element._cached_remotely() task.add_current_progress() # _pull_missing_artifacts() # # Pull missing artifacts from available remotes, this runs the scheduler # just to pull the artifacts if any of the artifacts are missing locally, # and is used in commands which need to use the artifacts. # # This function requires Stream.query_cache() to be called in advance # in order to determine which artifacts to try and pull. # # Args: # elements (list [Element]): The selected list of required elements # def _pull_missing_artifacts(self, elements): uncached_elts = [elt for elt in elements if elt._pull_pending()] if uncached_elts: self._context.messenger.info("Attempting to fetch missing or incomplete artifact(s)") self._reset() self._add_queue(PullQueue(self._scheduler)) self._enqueue_plan(uncached_elts) self._run(announce_session=True) # _load_tracking() # # A variant of _load() to be used when the elements should be used # for tracking # # If `targets` is not empty used project configuration will be # fully loaded. # # Args: # targets (list of str): Targets to load # selection (_PipelineSelection): The selection mode for the specified targets # except_targets (list of str): Specified targets to except # cross_junctions (bool): Whether tracking should cross junction boundaries # # Returns: # (list of Element): The tracking element selection # def _load_tracking(self, targets, *, selection=_PipelineSelection.NONE, except_targets=(), cross_junctions=False): elements, except_elements, artifacts = self._load_elements_from_targets( targets, except_targets, rewritable=True ) # We can't track artifact refs, since they have no underlying # elements or sources to interact with. Abort if the user asks # us to do that. if artifacts: detail = "\n".join(artifact.get_artifact_name() for artifact in artifacts) raise ArtifactElementError("Cannot perform this operation with artifact refs:", detail=detail) # Hold on to the targets self.targets = elements track_projects = {} for element in elements: project = element._get_project() if project not in track_projects: track_projects[project] = [element] else: track_projects[project].append(element) track_selected = [] for project, project_elements in track_projects.items(): selected = _pipeline.get_selection(self._context, project_elements, selection) selected = self._track_cross_junction_filter(project, selected, cross_junctions) track_selected.extend(selected) return _pipeline.except_elements(elements, track_selected, except_elements) # _track_cross_junction_filter() # # Filters out elements which are across junction boundaries, # otherwise asserts that there are no such elements. # # This is currently assumed to be only relevant for element # lists targetted at tracking. # # Args: # project (Project): Project used for cross_junction filtering. # All elements are expected to belong to that project. # elements (list of Element): The list of elements to filter # cross_junction_requested (bool): Whether the user requested # cross junction tracking # # Returns: # (list of Element): The filtered or asserted result # def _track_cross_junction_filter(self, project, elements, cross_junction_requested): # First filter out cross junctioned elements if not cross_junction_requested: elements = [element for element in elements if element._get_project() is project] # We can track anything if the toplevel project uses project.refs # if self._project.ref_storage == ProjectRefStorage.PROJECT_REFS: return elements # Ideally, we would want to report every cross junction element but not # their dependencies, unless those cross junction elements dependencies # were also explicitly requested on the command line. # # But this is too hard, lets shoot for a simple error. for element in elements: element_project = element._get_project() if element_project is not self._project: detail = ( "Requested to track sources across junction boundaries\n" + "in a project which does not use project.refs ref-storage." ) raise StreamError("Untrackable sources", detail=detail, reason="untrackable-sources") return elements # _load() # # A convenience method for loading element lists # # If `targets` is not empty used project configuration will be # fully loaded. # # Args: # targets: Main targets to load # selection: The selection mode for the specified targets (_PipelineSelection) # except_targets: Specified targets to except from fetching # ignore_junction_targets (bool): Whether junction targets should be filtered out # dynamic_plan: Require artifacts as needed during the build # load_artifacts: Whether to load artifacts with artifact names # attempt_artifact_metadata: Whether to attempt to download artifact metadata in # order to deduce build dependencies and reload. # connect_artifact_cache: Whether to try to contact remote artifact caches # connect_source_cache: Whether to try to contact remote source caches # artifact_remotes: Artifact cache remotes specified on the commmand line # source_remotes: Source cache remotes specified on the commmand line # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects # ignore_project_source_remotes: Whether to ignore source remotes specified by projects # need_state: Whether resolving element state is required # # Returns: # (list of Element): The primary element selection # def _load( self, targets: Iterable[str], *, selection: str = _PipelineSelection.NONE, except_targets: Iterable[str] = (), ignore_junction_targets: bool = False, dynamic_plan: bool = False, load_artifacts: bool = False, attempt_artifact_metadata: bool = False, connect_artifact_cache: bool = False, connect_source_cache: bool = False, artifact_remotes: Iterable[RemoteSpec] = (), source_remotes: Iterable[RemoteSpec] = (), ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, need_state: bool = True, ): elements, except_elements, artifacts = self._load_elements_from_targets( targets, except_targets, rewritable=False, valid_artifact_names=load_artifacts ) if artifacts: if selection in (_PipelineSelection.ALL, _PipelineSelection.RUN): raise StreamError( "Error: '--deps {}' is not supported for artifact names".format(selection), reason="deps-not-supported", ) if ignore_junction_targets: elements = [e for e in elements if e.get_kind() != "junction"] # Hold on to the targets self.targets = elements # Connect to remote caches, this needs to be done before resolving element state if need_state: self._context.initialize_remotes( connect_artifact_cache, connect_source_cache, artifact_remotes, source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) # In some cases we need to have an actualized artifact, with all of # it's metadata, such that we can derive attributes about the artifact # like it's build dependencies. if artifacts and attempt_artifact_metadata: # # FIXME: We need a semantic here to download only the metadata # for element in artifacts: element._set_required(_Scope.NONE) self.query_cache(artifacts) self._reset() self._add_queue(PullQueue(self._scheduler)) self._enqueue_plan(artifacts) self._run() # # After obtaining the metadata for the toplevel specified artifact # targets, we need to reload just the artifacts. # artifact_targets = [e.get_artifact_name() for e in artifacts] _, _, artifacts = self._load_elements_from_targets( artifact_targets, [], rewritable=False, valid_artifact_names=True ) # It can be that new remotes have been added by way of loading new # projects referenced by the new artifact elements, so we need to # ensure those remotes are also initialized. # self._context.initialize_remotes( connect_artifact_cache, connect_source_cache, artifact_remotes, source_remotes, ignore_project_artifact_remotes=ignore_project_artifact_remotes, ignore_project_source_remotes=ignore_project_source_remotes, ) self.targets += artifacts # Now move on to loading primary selection. # selected = _pipeline.get_selection( self._context, self.targets, selection, silent=False, depth_sort=dynamic_plan ) selected = _pipeline.except_elements(self.targets, selected, except_elements) # Mark the appropriate required elements # required_elements: List[Element] = [] if dynamic_plan: # # In a dynamic build plan, we only require the top-level targets and # rely on state changes during processing to determine which elements # must be processed. # if selection in (_PipelineSelection.NONE, _PipelineSelection.RUN): required_elements = elements elif selection == _PipelineSelection.BUILD: required_elements = list(_pipeline.dependencies(elements, _Scope.BUILD, recurse=False)) # Without a dynamic build plan, or if `all` selection was made, then everything is required if not required_elements: required_elements = selected if selection == _PipelineSelection.NONE: scope = _Scope.NONE else: scope = _Scope.RUN for element in required_elements: element._set_required(scope) return selected # _reset() # # Resets the internal state related to a given scheduler run. # # Invocations to the scheduler should start with a _reset() and end # with _run() like so: # # self._reset() # self._add_queue(...) # self._add_queue(...) # self._enqueue_plan(...) # self._run() # def _reset(self): self._scheduler.clear_queues() self.session_elements = [] self.total_elements = [] # _add_queue() # # Adds a queue to the stream # # Args: # queue (Queue): Queue to add to the pipeline # def _add_queue(self, queue, *, track=False): if not track and not self.queues: # First non-track queue queue.set_required_element_check() self.queues.append(queue) # _enqueue_plan() # # Enqueues planned elements to the specified queue. # # Args: # plan (list of Element): The list of elements to be enqueued # queue (Queue): The target queue, defaults to the first queue # def _enqueue_plan(self, plan, *, queue=None): queue = queue or self.queues[0] queue.enqueue(plan) self.session_elements += plan # _run() # # Common function for running the scheduler # # Args: # announce_session (bool): Whether to announce the session in the frontend. # def _run(self, *, announce_session: bool = False): # Inform the frontend of the full list of elements # and the list of elements which will be processed in this run # self.total_elements = list(_pipeline.dependencies(self.targets, _Scope.ALL)) if announce_session and self._session_start_callback is not None: self._session_start_callback() self._running = True status = self._scheduler.run(self.queues, self._context.get_casd()) self._running = False if status == SchedStatus.ERROR: raise StreamError() if status == SchedStatus.TERMINATED: raise StreamError(terminated=True) # _fetch() # # Performs the fetch job, the body of this function is here because # it is shared between a few internals. # # Args: # elements (list of Element): Elements to fetch # fetch_original (bool): Whether to fetch original unstaged # announce_session (bool): Whether to announce the session in the frontend # def _fetch(self, elements: List[Element], *, fetch_original: bool = False, announce_session: bool = False): # Assert consistency for the fetch elements _pipeline.assert_consistent(self._context, elements) # Construct queues, enqueue and run # self._reset() self._add_queue(FetchQueue(self._scheduler, fetch_original=fetch_original)) self._enqueue_plan(elements) self._run(announce_session=announce_session) # _check_location_writable() # # Check if given location is writable. # # Args: # location (str): Destination path # force (bool): Allow files to be overwritten # tar (bool): Whether destination is a tarball # # Raises: # (StreamError): If the destination is not writable # def _check_location_writable(self, location, force=False, tar=False): if not tar: try: os.makedirs(location, exist_ok=True) except OSError as e: raise StreamError("Failed to create destination directory: '{}'".format(e)) from e if not os.access(location, os.W_OK): raise StreamError("Destination directory '{}' not writable".format(location)) if not force and os.listdir(location): raise StreamError("Destination directory '{}' not empty".format(location)) elif os.path.exists(location) and location != "-": if not os.access(location, os.W_OK): raise StreamError("Output file '{}' not writable".format(location)) if not force and os.path.exists(location): raise StreamError("Output file '{}' already exists".format(location)) # Helper function for source_checkout() def _source_checkout( self, elements, location=None, force=False, deps="none", tar=False, compression=None, include_build_scripts=False, ): location = os.path.abspath(location) # Stage all our sources in a temporary directory. The this # directory can be used to either construct a tarball or moved # to the final desired location. temp_source_dir = tempfile.TemporaryDirectory(dir=self._context.tmpdir) # pylint: disable=consider-using-with try: self._write_element_sources(temp_source_dir.name, elements) if include_build_scripts: self._write_build_scripts(temp_source_dir.name, elements) if tar: self._create_tarball(temp_source_dir.name, location, compression) else: self._move_directory(temp_source_dir.name, location, force) except OSError as e: raise StreamError("Failed to checkout sources to {}: {}".format(location, e)) from e finally: with suppress(FileNotFoundError): temp_source_dir.cleanup() # Move a directory src to dest. This will work across devices and # may optionaly overwrite existing files. def _move_directory(self, src, dest, force=False): def is_empty_dir(path): return os.path.isdir(dest) and not os.listdir(dest) try: os.rename(src, dest) return except OSError: pass if force or is_empty_dir(dest): try: utils.link_files(src, dest) except utils.UtilError as e: raise StreamError("Failed to move directory: {}".format(e)) from e # Write the element build script to the given directory def _write_element_script(self, directory, element): try: element._write_script(directory) except ImplError: return False return True # Write all source elements to the given directory def _write_element_sources(self, directory, elements): for element in elements: element_source_dir = self._get_element_dirname(directory, element) if list(element.sources()): os.makedirs(element_source_dir) element._stage_sources_at(element_source_dir) # Create a tarball from the content of directory def _create_tarball(self, directory, tar_name, compression): if compression is None: compression = "" mode = _handle_compression(compression) try: with utils.save_file_atomic(tar_name, mode="wb") as f, tarfile.open(fileobj=f, mode=mode) as tarball: for item in os.listdir(str(directory)): file_to_add = os.path.join(directory, item) tarball.add(file_to_add, arcname=item) except OSError as e: raise StreamError("Failed to create tar archive: {}".format(e)) from e # Write all the build_scripts for elements in the directory location def _write_build_scripts(self, location, elements): for element in elements: self._write_element_script(location, element) self._write_master_build_script(location, elements) # Write a master build script to the sandbox def _write_master_build_script(self, directory, elements): module_string = "" for element in elements: module_string += shlex.quote(element.normal_name) + " " script_path = os.path.join(directory, "build.sh") with open(_site.build_all_template, "r", encoding="utf-8") as f: script_template = f.read() with utils.save_file_atomic(script_path, "w") as script: script.write(script_template.format(modules=module_string)) os.chmod(script_path, stat.S_IEXEC | stat.S_IREAD) # _get_element_dirname() # # Get path to directory for an element based on its normal name. # # For cross-junction elements, the path will be prefixed with the name # of the junction element. # # Args: # directory (str): path to base directory # element (Element): the element # # Returns: # (str): Path to directory for this element # def _get_element_dirname(self, directory, element): parts = [element.normal_name] while element._get_project() != self._project: element = element._get_project().junction parts.append(element.normal_name) return os.path.join(directory, *reversed(parts)) # _expand_and_classify_targets() # # Takes the user provided targets, expand any glob patterns, and # return a new list of targets. # # If valid_artifact_names is specified, then glob patterns will # also be checked for locally existing artifact names, and the # targets will be classified into separate lists, any targets # which are found to be an artifact name will be returned in # the list of artifact names. # # Args: # targets: A list of targets # valid_artifact_names: Whether artifact names are valid # # Returns: # (list): element names present in the targets # (list): artifact names present in the targets # def _expand_and_classify_targets( self, targets: Iterable[str], valid_artifact_names: bool = False ) -> Tuple[List[str], List[str]]: # # We use dicts here instead of sets, in order to deduplicate any possibly duplicate # entries, while also retaining the original order of element specification/discovery, # (which we cannot do with sets). # element_names = {} artifact_names = {} element_globs = {} artifact_globs = {} # First sort out globs and targets for target in targets: if any(c in "*?[" for c in target): if target.endswith(".bst"): element_globs[target] = True else: artifact_globs[target] = True elif target.endswith(".bst"): element_names[target] = True else: artifact_names[target] = True # Bail out in commands which don't support artifacts if any of the targets # or globs did not end with the expected '.bst' suffix. # if (artifact_names or artifact_globs) and not valid_artifact_names: raise StreamError( "Invalid element names or element glob patterns were specified: {}".format( ", ".join(list(artifact_names) + list(artifact_globs)) ), reason="invalid-element-names", detail="Element names and element glob expressions must end in '.bst'", ) # Verify targets which were not classified as elements for artifact_name in artifact_names: try: verify_artifact_ref(artifact_name) except ArtifactElementError as e: raise StreamError( "Specified target does not appear to be an artifact or element name: {}".format(artifact_name), reason="unrecognized-target-format", detail="Element names and element glob expressions must end in '.bst'", ) from e # Expand globs for elements if element_globs: # Bail out if an element glob is specified without providing a project directory if not self._project: raise StreamError( "Element glob expressions were specified without any project directory: {}".format( ", ".join(element_globs) ), reason="glob-elements-without-project", ) # Collect a list of `all_elements` in the project, stripping out the leading # project directory and element path prefix, to produce only element names. # all_elements = [] element_path_length = len(self._project.element_path) + 1 for dirpath, _, filenames in os.walk(self._project.element_path): for filename in filenames: if filename.endswith(".bst"): element_name = os.path.join(dirpath, filename) element_name = element_name[element_path_length:] all_elements.append(element_name) # Glob the elements and add the results to the set # for glob in element_globs: glob_results = list(utils.glob(all_elements, glob)) for element_name in glob_results: element_names[element_name] = True if not glob_results: self._context.messenger.warn("No elements matched the glob expression: {}".format(glob)) # Glob the artifact names and add the results to the set # for glob in artifact_globs: glob_results = self._artifacts.list_artifacts(glob=glob) for artifact_name in glob_results: artifact_names[artifact_name] = True if not glob_results: self._context.messenger.warn("No artifact names matched the glob expression: {}".format(glob)) return list(element_names), list(artifact_names) # _handle_compression() # # Return the tarfile mode str to be used when creating a tarball # # Args: # compression (str): The type of compression (either 'gz', 'xz' or 'bz2') # to_stdout (bool): Whether we want to open a stream for writing # # Returns: # (str): The tarfile mode string # def _handle_compression(compression, *, to_stream=False): mode_prefix = "w|" if to_stream else "w:" return mode_prefix + compression apache-buildstream-27ae392/src/buildstream/_testing/000077500000000000000000000000001514607367700225405ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/__init__.py000066400000000000000000000103531514607367700246530ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ This package contains various utilities which make it easier to test plugins. """ import os from collections import OrderedDict from typing import Tuple from buildstream.exceptions import ErrorDomain, LoadErrorReason from ._yaml import generate_project, generate_element, load_yaml from .repo import Repo from .runcli import cli, cli_integration, cli_remote_execution, Cli from .integration import integration_cache from ._cachekeys import check_cache_key_stability __all__ = [ "check_cache_key_stability", "create_repo", "register_repo_kind", "sourcetests_collection_hook", ] # To make use of these test utilities it is necessary to have pytest # available. However, we don't want to have a hard dependency on # pytest. try: import pytest except ImportError: module_name = globals()["__name__"] msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name) raise ImportError(msg) # Of the form plugin_name -> (repo_class, plugin_package) ALL_REPO_KINDS = OrderedDict() # type: OrderedDict[str, Tuple[Repo, str]] def create_repo(kind, directory, subdir="repo"): """Convenience method for creating a Repo Args: kind (str): The kind of repo to create (a source plugin basename). This must have previously been registered using `register_repo_kind` directory (str): The path where the repo will keep a cache Returns: (Repo): A new Repo object """ try: constructor = ALL_REPO_KINDS[kind] except KeyError as e: raise AssertionError("Unsupported repo kind {}".format(kind)) from e return constructor[0](directory, subdir=subdir) def register_repo_kind(kind, cls, plugin_package): """Register a new repo kind. Registering a repo kind will allow the use of the `create_repo` method for that kind and include that repo kind in ALL_REPO_KINDS In addition, repo_kinds registred prior to `sourcetests_collection_hook` being called will be automatically used to test the basic behaviour of their associated source plugins using the tests in `testing._sourcetests`. Args: kind (str): The kind of repo to create (a source plugin basename) cls (cls) : A class derived from Repo. plugin_package (str): The name of the python package containing the plugin """ ALL_REPO_KINDS[kind] = (cls, plugin_package) def sourcetests_collection_hook(session): """Used to hook the templated source plugin tests into a pyest test suite. This should be called via the `pytest_sessionstart hook `_. The tests in the _sourcetests package will be collected as part of whichever test package this hook is called from. Args: session (pytest.Session): The current pytest session """ def should_collect_tests(config): # When no args are supplied, pytest defaults the arg list to the # value of the `testpaths` configuration option. We want to collect # tests as part of the default collection return config.args_source != pytest.Config.ArgsSource.ARGS from . import _sourcetests source_test_path = os.path.dirname(_sourcetests.__file__) # Add the location of the source tests to the session's # python_files config. Without this, pytest may filter out these # tests during collection. session.config.addinivalue_line("python_files", os.path.join(source_test_path, "*.py")) # If test invocation has specified specic tests, don't # automatically collect templated tests. if should_collect_tests(session.config): session.config.args.append(source_test_path) apache-buildstream-27ae392/src/buildstream/_testing/_cachekeys.py000066400000000000000000000113201514607367700252050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from collections import OrderedDict from .runcli import Cli class CacheKeyTestError(Exception): pass def check_cache_key_stability(project_path: os.PathLike, cli: Cli) -> None: """ Check that the cache key of various elements has not changed. This ensures that elements do not break cache keys unexpectedly. The format of the project is expected to be: .. code-block:: ./ ./project.conf ./elem1.bst ./elem1.expected ./elem2.bst ./elem2.expected # Or in sub-directories ./mydir/elem3.bst ./mydir/elem3.expected The ``.expected`` file should contain the expected cache key. In order to automatically created the ``.expected`` files, or updated them, you can run ``python3 -m buildstream._testing._update_cachekeys`` in the project's directory. :param project_path: Path to a project :param cli: a `cli` object as provided by the fixture :func:`buildstream._testing.runcli.cli` """ result = cli.run( project=project_path, silent=True, args=["show", "--format", "%{name}::%{full-key}", "target.bst"] ) result.assert_success() _assert_cache_keys(project_path, result.output) ############################### ## Internal Helper functions ## ############################### # Those functions are for internal use only and are not part of the public API def _element_filename(project_dir, element_name, alt_suffix=None): # Get whole filename in the temp project with # the option of changing the .bst suffix to something else # if alt_suffix: # Just in case... assert element_name.endswith(".bst") # Chop off the 'bst' in '.bst' and add the new suffix element_name = element_name[:-3] element_name = element_name + alt_suffix return os.path.join(project_dir, element_name) def _parse_output_keys(output): # Returns an OrderedDict of element names # and their cache keys # actual_keys = OrderedDict() lines = output.splitlines() for line in lines: split = line.split("::") name = split[0] key = split[1] actual_keys[name] = key return actual_keys def _load_expected_keys(project_dir, actual_keys, raise_error=True): # Returns an OrderedDict of element names # and their cache keys # expected_keys = OrderedDict() for element_name in actual_keys: expected = _element_filename(project_dir, element_name, "expected") try: with open(expected, "r", encoding="utf-8") as f: expected_key = f.read() expected_key = expected_key.strip() except FileNotFoundError: expected_key = None if raise_error: raise CacheKeyTestError( "Cache key test needs update, " + "expected file {} not found.\n\n".format(expected) + "Use python3 -m buildstream._testing._update_cachekeys in the" + " project's directory to automatically update this test case" ) expected_keys[element_name] = expected_key return expected_keys def _assert_cache_keys(project_dir, output): # Read in the expected keys from the cache key test directory # and parse the actual keys from the `bst show` output # actual_keys = _parse_output_keys(output) expected_keys = _load_expected_keys(project_dir, actual_keys) mismatches = [] for element_name in actual_keys: if actual_keys[element_name] != expected_keys[element_name]: mismatches.append(element_name) if mismatches: info = "" for element_name in mismatches: info += ( " Element: {}\n".format(element_name) + " Expected: {}\n".format(expected_keys[element_name]) + " Actual: {}\n".format(actual_keys[element_name]) ) raise AssertionError( "Cache key mismatches occurred:\n{}\n".format(info) + "Use python3 -m buildstream._testing._update_cachekeys in the project's " + "directory to automatically update this test case" ) apache-buildstream-27ae392/src/buildstream/_testing/_fixtures.py000066400000000000000000000034061514607367700251250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # pylint: disable=redefined-outer-name import time import psutil import pytest # Number of seconds to wait for background threads to exit. _AWAIT_THREADS_TIMEOUT_SECONDS = 5 def has_no_unexpected_background_threads(expected_num_threads): # Use psutil as threading.active_count() doesn't include gRPC threads. # # If background gRPC threads are lingering, there is a good chance that # this is due to BuildStream failing to close an open grpc channel. # process = psutil.Process() wait = 0.1 for _ in range(0, int(_AWAIT_THREADS_TIMEOUT_SECONDS / wait)): if process.num_threads() == expected_num_threads: return True time.sleep(wait) return False @pytest.fixture(autouse=True, scope="session") def default_thread_number(): # xdist/execnet has its own helper thread. return psutil.Process().num_threads() # Catch tests that don't shut down background threads, which could then lead # to other tests hanging when BuildStream uses fork(). @pytest.fixture(autouse=True) def thread_check(default_thread_number): assert has_no_unexpected_background_threads(default_thread_number) yield assert has_no_unexpected_background_threads(default_thread_number) apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/000077500000000000000000000000001514607367700252625ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/__init__.py000066400000000000000000000000001514607367700273610ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/build_checkout.py000066400000000000000000000047361514607367700306320ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from .. import create_repo from .. import cli # pylint: disable=unused-import from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def strict_args(args, strict): if strict != "strict": return ["--no-strict", *args] return args @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", ["strict", "non-strict"]) def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind): checkout = os.path.join(cli.directory, "checkout") project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_name = "build-test-{}.bst".format(kind) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir)) ref = repo.create(dev_files_path) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) assert cli.get_element_state(project, element_name) == "fetch needed" result = cli.run(project=project, args=strict_args(["build", element_name], strict)) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" # Now check it out result = cli.run( project=project, args=strict_args(["artifact", "checkout", element_name, "--directory", checkout], strict) ) result.assert_success() # Check that the pony.h include from files/dev-files exists filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/conftest.py000066400000000000000000000012111514607367700274540ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from .._fixtures import default_thread_number, thread_check # pylint: disable=unused-import apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/fetch.py000066400000000000000000000066401514607367700267330ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from .._utils import generate_junction from .. import create_repo from .. import cli # pylint: disable=unused-import from .utils import update_project_configuration from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") @pytest.mark.datafiles(DATA_DIR) def test_fetch(cli, tmpdir, datafiles, kind): project = str(datafiles) bin_files_path = os.path.join(project, "files", "bin-files") element_path = os.path.join(project, "elements") element_name = "fetch-test-{}.bst".format(kind) # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == "fetch needed" # Now try to fetch it result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. assert cli.get_element_state(project, element_name) == "buildable" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") import_etc_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst") etc_files_path = os.path.join(subproject_path, "files", "etc-files") repo = create_repo(kind, str(tmpdir.join("import-etc"))) ref = repo.create(etc_files_path) element = {"kind": "import", "sources": [repo.source_config(ref=(ref if ref_storage == "inline" else None))]} _yaml.roundtrip_dump(element, import_etc_path) update_project_configuration(project, {"ref-storage": ref_storage}) generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) if ref_storage == "project.refs": result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "junction.bst:import-etc.bst"]) result.assert_success() apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/mirror.py000066400000000000000000000256571514607367700271650ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from .._utils import generate_junction from .. import create_repo from .. import cli # pylint: disable=unused-import from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def _set_project_mirrors_and_aliases(project_path, mirrors, aliases): project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf["mirrors"] = mirrors project_conf["aliases"].update(aliases) _yaml.roundtrip_dump(project_conf, project_conf_path) def _set_project_includes_and_aliases(project_path, includes, aliases): project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf["aliases"].update(aliases) project_conf["(@)"] = includes _yaml.roundtrip_dump(project_conf, project_conf_path) @pytest.mark.datafiles(DATA_DIR) def test_mirror_fetch(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") upstream_repodir = os.path.join(str(tmpdir), "upstream") mirror_repodir = os.path.join(str(tmpdir), "mirror") element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) upstream_ref = upstream_repo.create(dev_files_path) element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) alias = "foo-" + kind aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, }, ], {alias: upstream_map + "/"}, ) # No obvious ways of checking that the mirror has been fetched # But at least we can be sure it succeeds result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") upstream_repodir = os.path.join(project_dir, "upstream") mirror_repodir = os.path.join(str(tmpdir), "mirror") element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) ref = upstream_repo.create(dev_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) element = {"kind": "import", "sources": [upstream_repo.source_config(ref=ref)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) full_repo = element["sources"][0]["url"] _, repo_name = os.path.split(full_repo) alias = "foo-" + kind aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, [ { "name": "middle-earth", "aliases": {alias: [mirror_map + "/"]}, }, ], {alias: "http://www.example.com"}, ) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_from_includes(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") upstream_repodir = os.path.join(str(tmpdir), "upstream") mirror_repodir = os.path.join(str(tmpdir), "mirror") element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) upstream_ref = upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) alias = "foo-" + kind aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) config_project_dir = str(tmpdir.join("config")) os.makedirs(config_project_dir, exist_ok=True) config_project = {"name": "config", "min-version": "2.0"} _yaml.roundtrip_dump(config_project, os.path.join(config_project_dir, "project.conf")) extra_mirrors = { "mirrors": [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, } ] } _yaml.roundtrip_dump(extra_mirrors, os.path.join(config_project_dir, "mirrors.yml")) generate_junction(str(tmpdir.join("config_repo")), config_project_dir, os.path.join(element_dir, "config.bst")) _set_project_includes_and_aliases( project_dir, ["config.bst:mirrors.yml"], {alias: upstream_map + "/"}, ) # Now make the upstream unavailable. os.rename(upstream_repo.repo, "{}.bak".format(upstream_repo.repo)) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") upstream_repodir = os.path.join(str(tmpdir), "upstream") mirror_repodir = os.path.join(str(tmpdir), "mirror") element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) upstream_ref = upstream_repo.create(dev_files_path) element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) alias = "foo-" + kind aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, }, ], {alias: upstream_map + "/"}, ) result = cli.run(project=project_dir, args=["source", "track", element_name]) result.assert_success() # Tracking tries upstream first. Check the ref is from upstream. new_element = _yaml.load(element_path, shortname=element_name) source = new_element.get_sequence("sources").mapping_at(0) if "ref" in source: assert source.get_str("ref") == upstream_ref @pytest.mark.datafiles(DATA_DIR) def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): project_dir = str(datafiles) bin_files_path = os.path.join(project_dir, "files", "bin-files", "usr") dev_files_path = os.path.join(project_dir, "files", "dev-files", "usr") upstream_repodir = os.path.join(str(tmpdir), "upstream") mirror_repodir = os.path.join(str(tmpdir), "mirror") element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo(kind, upstream_repodir) upstream_ref = upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) mirror_ref = upstream_ref upstream_ref = upstream_repo.create(dev_files_path) # Configure to allow tracking mirrors as well as upstream cli.configure({"track": {"source": "all"}}) element = {"kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) full_repo = element["sources"][0]["url"] _, repo_name = os.path.split(full_repo) alias = "foo-" + kind aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) _yaml.roundtrip_dump(element, element_path) _set_project_mirrors_and_aliases( project_dir, [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, }, ], {alias: "http://www.example.com"}, ) result = cli.run(project=project_dir, args=["source", "track", element_name]) result.assert_success() # Check that tracking fell back to the mirror new_element = _yaml.load(element_path, shortname=element_name) source = new_element.get_sequence("sources").mapping_at(0) if "ref" in source: assert source.get_str("ref") == mirror_ref apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/000077500000000000000000000000001514607367700267305ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/000077500000000000000000000000001514607367700305445ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/base.bst000066400000000000000000000001031514607367700321620ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/base/000077500000000000000000000000001514607367700314565ustar00rootroot00000000000000base-alpine.bst000066400000000000000000000010061514607367700342660ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/basekind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/import-bin.bst000066400000000000000000000000741514607367700333370ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/import-dev.bst000066400000000000000000000000741514607367700333450ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/000077500000000000000000000000001514607367700341305ustar00rootroot00000000000000dependency/000077500000000000000000000000001514607367700361675ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targetshorsey.bst000066400000000000000000000000771514607367700402160ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/dependencykind: manual depends: - multiple_targets/dependency/pony.bst pony.bst000066400000000000000000000000151514607367700376620ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/dependencykind: manual zebry.bst000066400000000000000000000001011514607367700400240ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/dependencykind: manual depends: - multiple_targets/dependency/horsey.bst order/000077500000000000000000000000001514607367700351645ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets0.bst000066400000000000000000000002561514607367700360400ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Root node depends: - multiple_targets/order/2.bst - multiple_targets/order/3.bst - filename: multiple_targets/order/run.bst type: runtime 1.bst000066400000000000000000000001161514607367700360340ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Root node depends: - multiple_targets/order/9.bst 2.bst000066400000000000000000000001331514607367700360340ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: First dependency level depends: - multiple_targets/order/3.bst 3.bst000066400000000000000000000002361514607367700360410ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Second dependency level depends: - multiple_targets/order/4.bst - multiple_targets/order/5.bst - multiple_targets/order/6.bst 4.bst000066400000000000000000000000611514607367700360360ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Third level dependency 5.bst000066400000000000000000000000611514607367700360370ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Fifth level dependency 6.bst000066400000000000000000000001341514607367700360410ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Fourth level dependency depends: - multiple_targets/order/5.bst 7.bst000066400000000000000000000001331514607367700360410ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Third level dependency depends: - multiple_targets/order/6.bst 8.bst000066400000000000000000000001341514607367700360430ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Second level dependency depends: - multiple_targets/order/7.bst 9.bst000066400000000000000000000001331514607367700360430ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: First level dependency depends: - multiple_targets/order/8.bst run.bst000066400000000000000000000001241514607367700364770ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/elements/multiple_targets/orderkind: manual description: Not a root node, yet built at the same time as root nodes apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/000077500000000000000000000000001514607367700300325ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/bar000066400000000000000000000000001514607367700305070ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/bin-files/000077500000000000000000000000001514607367700317025ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/bin-files/usr/000077500000000000000000000000001514607367700325135ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/bin-files/usr/bin/000077500000000000000000000000001514607367700332635ustar00rootroot00000000000000hello000077500000000000000000000000341514607367700342320ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/bin-files/usr/bin#!/bin/bash echo "Hello !" apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/dev-files/000077500000000000000000000000001514607367700317105ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/dev-files/usr/000077500000000000000000000000001514607367700325215ustar00rootroot00000000000000include/000077500000000000000000000000001514607367700340655ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/dev-files/usrpony.h000066400000000000000000000003711514607367700352240ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/dev-files/usr/include#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/etc-files/000077500000000000000000000000001514607367700317055ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/etc-files/etc/000077500000000000000000000000001514607367700324605ustar00rootroot00000000000000buildstream/000077500000000000000000000000001514607367700347145ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/etc-files/etcconfig000066400000000000000000000000071514607367700361010ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/etc-files/etc/buildstreamconfig apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/foo000066400000000000000000000000001514607367700305260ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/source-bundle/000077500000000000000000000000001514607367700326015ustar00rootroot00000000000000llamas.txt000066400000000000000000000000071514607367700345310ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/source-bundlellamas apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/000077500000000000000000000000001514607367700322675ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/elements/000077500000000000000000000000001514607367700341035ustar00rootroot00000000000000import-etc.bst000066400000000000000000000000741514607367700366220ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/elementskind: import sources: - kind: local path: files/etc-files apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/files/000077500000000000000000000000001514607367700333715ustar00rootroot00000000000000etc-files/000077500000000000000000000000001514607367700351655ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/filesetc/000077500000000000000000000000001514607367700357405ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/files/etc-filesanimal.conf000066400000000000000000000000141514607367700400430ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project/files/etc-files/etcanimal=Pony project.conf000066400000000000000000000001371514607367700345260ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/files/sub-project# Project config for frontend build test name: subtest min-version: 2.0 element-path: elements apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/project/project.conf000066400000000000000000000010471514607367700312470ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ project_dir: file://{project_dir} options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture values: - x86-64 - aarch64 split-rules: test: - | /tests - | /tests/* fatal-warnings: - bad-element-suffix - bad-characters-in-name apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/source_determinism.py000066400000000000000000000102361514607367700315360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from .._utils.site import HAVE_SANDBOX, CASD_SEPARATE_USER from .. import create_repo from .. import cli # pylint: disable=unused-import from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def create_test_file(*path, mode=0o644, content="content\n"): path = os.path.join(*path) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write(content) os.fchmod(f.fileno(), mode) def create_test_directory(*path, mode=0o644): create_test_file(*path, ".keep", content="") path = os.path.join(*path) os.chmod(path, mode) @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_deterministic_source_umask(cli, tmpdir, datafiles, kind): if CASD_SEPARATE_USER and kind == "ostree": pytest.xfail("The ostree plugin ignores the umask") project = str(datafiles) element_name = "list.bst" element_path = os.path.join(project, "elements", element_name) repodir = os.path.join(str(tmpdir), "repo") sourcedir = os.path.join(project, "source") create_test_file(sourcedir, "a.txt", mode=0o700) create_test_file(sourcedir, "b.txt", mode=0o755) create_test_file(sourcedir, "c.txt", mode=0o600) create_test_file(sourcedir, "d.txt", mode=0o400) create_test_file(sourcedir, "e.txt", mode=0o644) create_test_file(sourcedir, "f.txt", mode=0o4755) create_test_file(sourcedir, "g.txt", mode=0o2755) create_test_file(sourcedir, "h.txt", mode=0o1755) create_test_directory(sourcedir, "dir-a", mode=0o0700) create_test_directory(sourcedir, "dir-c", mode=0o0755) create_test_directory(sourcedir, "dir-d", mode=0o4755) create_test_directory(sourcedir, "dir-e", mode=0o2755) create_test_directory(sourcedir, "dir-f", mode=0o1755) repo = create_repo(kind, repodir) ref = repo.create(sourcedir) source = repo.source_config(ref=ref) element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "sources": [source], "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']}, } _yaml.roundtrip_dump(element, element_path) def get_value_for_umask(umask): checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(umask)) old_umask = os.umask(umask) try: test_values = [] result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir]) result.assert_success() with open(os.path.join(checkoutdir, "ls-l"), "r", encoding="utf-8") as f: for line in f.readlines(): test_values.append(line.split()[0] + " " + line.split()[-1]) return test_values finally: os.umask(old_umask) cli.remove_artifact_from_cache(project, element_name) if CASD_SEPARATE_USER: # buildbox-casd running as separate user of the same group can't # function in a test environment with a too restrictive umask. assert get_value_for_umask(0o002) == get_value_for_umask(0o007) else: assert get_value_for_umask(0o022) == get_value_for_umask(0o077) apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/track.py000066400000000000000000000352331514607367700267460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain from .._utils import generate_junction from .. import create_repo from .. import cli # pylint: disable=unused-import from .utils import update_project_configuration from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(repo, element_path, dep_name=None): element = {"kind": "import", "sources": [repo.source_config()]} if dep_name: element["depends"] = [dep_name] _yaml.roundtrip_dump(element, element_path) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_name = "track-test-{}.bst".format(kind) update_project_configuration(project, {"ref-storage": ref_storage}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir)) repo.create(dev_files_path) # Generate the element generate_element(repo, os.path.join(element_path, element_name)) # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == "no reference" # Now first try to track it result = cli.run(project=project, args=["source", "track", element_name]) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. assert cli.get_element_state(project, element_name) == "buildable" # Assert there was a project.refs created, depending on the configuration if ref_storage == "project.refs": assert os.path.exists(os.path.join(project, "project.refs")) else: assert not os.path.exists(os.path.join(project, "project.refs")) # NOTE: # # This test checks that recursive tracking works by observing # element states after running a recursive tracking operation. # # However, this test is ALSO valuable as it stresses the source # plugins in a situation where many source plugins are operating # at once on the same backing repository. # # Do not change this test to use a separate 'Repo' per element # as that would defeat the purpose of the stress test, otherwise # please refactor that aspect into another test. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("amount", [1, 10]) def test_track_recurse(cli, tmpdir, datafiles, kind, amount): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") # Try to actually launch as many fetch jobs as possible at the same time # # This stresses the Source plugins and helps to ensure that # they handle concurrent access to the store correctly. cli.configure( { "scheduler": { "fetchers": amount, } } ) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir)) repo.create(dev_files_path) # Write out our test targets element_names = [] last_element_name = None for i in range(amount + 1): element_name = "track-test-{}-{}.bst".format(kind, i + 1) filename = os.path.join(element_path, element_name) element_names.append(element_name) generate_element(repo, filename, dep_name=last_element_name) last_element_name = element_name # Assert that a fetch is needed states = cli.get_element_states(project, [last_element_name]) for element_name in element_names: assert states[element_name] == "no reference" # Now first try to track it result = cli.run(project=project, args=["source", "track", "--deps", "all", last_element_name]) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. result = cli.run(project=project, args=["source", "fetch", "--deps", "all", last_element_name]) result.assert_success() # Assert that the base is buildable and the rest are waiting states = cli.get_element_states(project, [last_element_name]) for element_name in element_names: if element_name == element_names[0]: assert states[element_name] == "buildable" else: assert states[element_name] == "waiting" @pytest.mark.datafiles(DATA_DIR) def test_track_recurse_except(cli, tmpdir, datafiles, kind): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_dep_name = "track-test-dep-{}.bst".format(kind) element_target_name = "track-test-target-{}.bst".format(kind) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir)) repo.create(dev_files_path) # Write out our test targets generate_element(repo, os.path.join(element_path, element_dep_name)) generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name) # Assert that a fetch is needed states = cli.get_element_states(project, [element_target_name]) assert states[element_dep_name] == "no reference" assert states[element_target_name] == "no reference" # Now first try to track it result = cli.run( project=project, args=["source", "track", "--deps", "all", "--except", element_dep_name, element_target_name] ) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name]) result.assert_success() # Assert that the dependency is buildable and the target is waiting states = cli.get_element_states(project, [element_target_name]) assert states[element_dep_name] == "no reference" assert states[element_target_name] == "waiting" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") etc_files = os.path.join(subproject_path, "files", "etc-files") repo_element_path = os.path.join(subproject_path, "elements", "import-etc-repo.bst") update_project_configuration(project, {"ref-storage": ref_storage}) repo = create_repo(kind, str(tmpdir.join("element_repo"))) repo.create(etc_files) generate_element(repo, repo_element_path) generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False) # Track the junction itself first. result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "no reference" # Track the cross junction element. -J is not given, it is implied. result = cli.run(project=project, args=["source", "track", "junction.bst:import-etc-repo.bst"]) if ref_storage == "inline": # This is not allowed to track cross junction without project.refs. result.assert_main_error(ErrorDomain.STREAM, "untrackable-sources") else: result.assert_success() assert cli.get_element_state(project, "junction.bst:import-etc-repo.bst") == "buildable" assert os.path.exists(os.path.join(project, "project.refs")) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track_include(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_name = "track-test-{}.bst".format(kind) update_project_configuration(project, {"ref-storage": ref_storage}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir)) ref = repo.create(dev_files_path) # Generate the element element = {"kind": "import", "(@)": ["elements/sources.yml"]} sources = {"sources": [repo.source_config()]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) _yaml.roundtrip_dump(sources, os.path.join(element_path, "sources.yml")) # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == "no reference" # Now first try to track it result = cli.run(project=project, args=["source", "track", element_name]) result.assert_success() # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. assert cli.get_element_state(project, element_name) == "buildable" # Assert there was a project.refs created, depending on the configuration if ref_storage == "project.refs": assert os.path.exists(os.path.join(project, "project.refs")) else: assert not os.path.exists(os.path.join(project, "project.refs")) new_sources = _yaml.load(os.path.join(element_path, "sources.yml"), shortname="sources.yml") # Get all of the sources assert "sources" in new_sources sources_list = new_sources.get_sequence("sources") assert len(sources_list) == 1 # Get the first source from the sources list new_source = sources_list.mapping_at(0) assert "ref" in new_source assert ref == new_source.get_str("ref") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_name = "track-test-{}.bst".format(kind) subproject_path = os.path.join(project, "files", "sub-project") sub_element_path = os.path.join(subproject_path, "elements") junction_path = os.path.join(element_path, "junction.bst") update_project_configuration(project, {"ref-storage": ref_storage}) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = create_repo(kind, str(tmpdir.join("element_repo"))) repo.create(dev_files_path) # Generate the element element = {"kind": "import", "(@)": ["junction.bst:elements/sources.yml"]} sources = {"sources": [repo.source_config()]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) _yaml.roundtrip_dump(sources, os.path.join(sub_element_path, "sources.yml")) generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=True) result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # Assert that a fetch is needed assert cli.get_element_state(project, element_name) == "no reference" # Now first try to track it result = cli.run(project=project, args=["source", "track", element_name]) # Assert there was a project.refs created, depending on the configuration if ref_storage == "inline": # FIXME: We should expect an error. But only a warning is emitted # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment') assert "junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction" in result.stderr else: assert os.path.exists(os.path.join(project, "project.refs")) # And now fetch it: The Source has probably already cached the # latest ref locally, but it is not required to have cached # the associated content of the latest ref at track time, that # is the job of fetch. result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() # Assert that we are now buildable because the source is # now cached. assert cli.get_element_state(project, element_name) == "buildable" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", ["inline", "project.refs"]) def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind): project = str(datafiles) element_path = os.path.join(project, "elements") subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(element_path, "junction.bst") update_project_configuration(project, {"ref-storage": ref_storage, "(@)": ["junction.bst:test.yml"]}) generate_junction(str(tmpdir.join("junction_repo")), subproject_path, junction_path, store_ref=False) result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/track_cross_junction.py000066400000000000000000000152311514607367700320640ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from .._utils import generate_junction from .. import create_repo, ALL_REPO_KINDS from .. import cli # pylint: disable=unused-import from .utils import add_plugins_conf # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(repo, element_path, dep_name=None): element = {"kind": "import", "sources": [repo.source_config()]} if dep_name: element["depends"] = [dep_name] _yaml.roundtrip_dump(element, element_path) def generate_import_element(tmpdir, kind, project, name): element_name = "import-{}.bst".format(name) repo_element_path = os.path.join(project, "elements", element_name) files = str(tmpdir.join("imported_files_{}".format(name))) os.makedirs(files) with open(os.path.join(files, "{}.txt".format(name)), "w", encoding="utf-8") as f: f.write(name) repo = create_repo(kind, str(tmpdir.join("element_{}_repo".format(name)))) repo.create(files) generate_element(repo, repo_element_path) return element_name def generate_project(tmpdir, name, kind, config=None): if config is None: config = {} project_name = "project-{}".format(name) subproject_path = os.path.join(str(tmpdir.join(project_name))) os.makedirs(os.path.join(subproject_path, "elements")) project_conf = {"name": name, "min-version": "2.0", "element-path": "elements"} project_conf.update(config) _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, "project.conf")) add_plugins_conf(subproject_path, kind) return project_name, subproject_path def generate_simple_stack(project, name, dependencies): element_name = "{}.bst".format(name) element_path = os.path.join(project, "elements", element_name) element = {"kind": "stack", "depends": dependencies} _yaml.roundtrip_dump(element, element_path) return element_name def generate_cross_element(project, subproject_name, import_name): basename, _ = os.path.splitext(import_name) return generate_simple_stack( project, "import-{}-{}".format(subproject_name, basename), [{"junction": "{}.bst".format(subproject_name), "filename": import_name}], ) @pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys()) def test_cross_junction_multiple_projects(cli, tmpdir, kind): tmpdir = tmpdir.join(kind) # Generate 3 projects: main, a, b _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"}) project_a, project_a_path = generate_project(tmpdir, "a", kind) project_b, project_b_path = generate_project(tmpdir, "b", kind) # Generate an element with a trackable source for each project element_a = generate_import_element(tmpdir, kind, project_a_path, "a") element_b = generate_import_element(tmpdir, kind, project_b_path, "b") element_c = generate_import_element(tmpdir, kind, project, "c") # Create some indirections to the elements with dependencies to test --deps stack_a = generate_simple_stack(project_a_path, "stack-a", [element_a]) stack_b = generate_simple_stack(project_b_path, "stack-b", [element_b]) # Create junctions for projects a and b in main. junction_a = "{}.bst".format(project_a) junction_a_path = os.path.join(project, "elements", junction_a) generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False) junction_b = "{}.bst".format(project_b) junction_b_path = os.path.join(project, "elements", junction_b) generate_junction(tmpdir.join("repo_b"), project_b_path, junction_b_path, store_ref=False) # Track the junctions. result = cli.run(project=project, args=["source", "track", junction_a, junction_b]) result.assert_success() # Import elements from a and b in to main. imported_a = generate_cross_element(project, project_a, stack_a) imported_b = generate_cross_element(project, project_b, stack_b) # Generate a top level stack depending on everything all_bst = generate_simple_stack(project, "all", [imported_a, imported_b, element_c]) # Track without following junctions. But explicitly also track the elements in project a. result = cli.run( project=project, args=["source", "track", "--deps", "all", all_bst, "{}:{}".format(junction_a, stack_a)] ) result.assert_success() # Elements in project b should not be tracked. But elements in project a and main should. expected = [element_c, "{}:{}".format(junction_a, element_a)] assert set(result.get_tracked_elements()) == set(expected) @pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys()) def test_track_exceptions(cli, tmpdir, kind): tmpdir = tmpdir.join(kind) _, project = generate_project(tmpdir, "main", kind, {"ref-storage": "project.refs"}) project_a, project_a_path = generate_project(tmpdir, "a", kind) element_a = generate_import_element(tmpdir, kind, project_a_path, "a") element_b = generate_import_element(tmpdir, kind, project_a_path, "b") all_bst = generate_simple_stack(project_a_path, "all", [element_a, element_b]) junction_a = "{}.bst".format(project_a) junction_a_path = os.path.join(project, "elements", junction_a) generate_junction(tmpdir.join("repo_a"), project_a_path, junction_a_path, store_ref=False) result = cli.run(project=project, args=["source", "track", junction_a]) result.assert_success() imported_b = generate_cross_element(project, project_a, element_b) indirection = generate_simple_stack(project, "indirection", [imported_b]) result = cli.run( project=project, args=[ "source", "track", "--deps", "all", "--except", indirection, "{}:{}".format(junction_a, all_bst), imported_b, ], ) result.assert_success() expected = ["{}:{}".format(junction_a, element_a), "{}:{}".format(junction_a, element_b)] assert set(result.get_tracked_elements()) == set(expected) apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/utils.py000066400000000000000000000060601514607367700267760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Benjamin Schubert (bschubert15@bloomberg.net) # import os # To make use of these test utilities it is necessary to have pytest # available. However, we don't want to have a hard dependency on # pytest. try: import pytest except ImportError: module_name = globals()["__name__"] msg = "Could not import pytest:\n" "To use the {} module, you must have pytest installed.".format(module_name) raise ImportError(msg) from buildstream import _yaml from .. import ALL_REPO_KINDS # kind() # # Pytest fixture to get all the registered source plugins. # # This assumes the usage of the standard `project` and will automatically # register the plugin in the project configuration, in addition to its junction # configuration # # Yields: # the plugin kind name # @pytest.fixture(params=ALL_REPO_KINDS.keys()) def kind(request, datafiles): # Register plugins both on the toplevel project and on its junctions for project_dir in [str(datafiles), os.path.join(str(datafiles), "files", "sub-project")]: add_plugins_conf(project_dir, request.param) yield request.param # add_plugins_conf() # # Add the given plugin to the configuration of the given project. # # Args: # project (str): path to the project on which to register the plugin # plugin_kind (str): name of the plugin kind to register # def add_plugins_conf(project, plugin_kind): _scaffolder, plugin_package = ALL_REPO_KINDS[plugin_kind] project_conf_file = os.path.join(project, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_file) if plugin_package is not None: project_conf["plugins"] = [ { "origin": "pip", "package-name": plugin_package, "sources": [plugin_kind], }, ] _yaml.roundtrip_dump(project_conf, project_conf_file) # update_project_configuration() # # Update the project configuration with the given updated configuration. # # Note: This does a simple `dict.update()` call, which will not merge recursively # but will replace every defined key. # # Args: # project_path (str): the path to the root of the project # updated_configuration (dict): configuration to merge into the existing one # def update_project_configuration(project_path, updated_configuration): project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf.update(updated_configuration) _yaml.roundtrip_dump(project_conf, project_conf_path) apache-buildstream-27ae392/src/buildstream/_testing/_sourcetests/workspace.py000066400000000000000000000047171514607367700276430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from .. import create_repo from .. import cli # pylint: disable=unused-import from .utils import kind # pylint: disable=unused-import # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") @pytest.mark.datafiles(DATA_DIR) def test_open(cli, tmpdir_factory, datafiles, kind): project_path = str(datafiles) bin_files_path = os.path.join(project_path, "files", "bin-files") element_name = "workspace-test-{}.bst".format(kind) element_path = os.path.join(project_path, "elements") # Create our repo object of the given source type with # the bin files, and then collect the initial ref. repo = create_repo(kind, str(tmpdir_factory.mktemp("repo-{}".format(kind)))) ref = repo.create(bin_files_path) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Assert that there is no reference, a fetch is needed assert cli.get_element_state(project_path, element_name) == "fetch needed" workspace_dir = os.path.join(tmpdir_factory.mktemp("opened_workspace")) # Now open the workspace, this should have the effect of automatically # fetching the source from the repo. result = cli.run(project=project_path, args=["workspace", "open", "--directory", workspace_dir, element_name]) result.assert_success() # Assert that we are now buildable because the source is now cached. assert cli.get_element_state(project_path, element_name) == "buildable" # Check that the executable hello file is found in each workspace filename = os.path.join(workspace_dir, "usr", "bin", "hello") assert os.path.exists(filename) apache-buildstream-27ae392/src/buildstream/_testing/_update_cachekeys.py000077500000000000000000000055561514607367700265700ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Automatically create or update the .expected files in the # cache key test directory. # # Simply run without any arguments, from the directory containing the project, e.g.: # # python3 -m buildstream.testing._update_cachekeys # # After this, add any files which were newly created and commit # the result in order to adjust the cache key test to changed # keys. # import os import tempfile from unittest import mock from ._cachekeys import _element_filename, _parse_output_keys, _load_expected_keys from .runcli import Cli def write_expected_key(project_dir, element_name, actual_key): expected_file = _element_filename(project_dir, element_name, "expected") with open(expected_file, "w", encoding="utf-8") as f: f.write(actual_key) def update_keys(): project_dir = os.getcwd() with tempfile.TemporaryDirectory(dir=project_dir) as cache_dir: directory = os.path.join(str(cache_dir), "cache") # Run bst show cli = Cli(directory, verbose=True) result = cli.run( project=project_dir, silent=True, args=["--no-colors", "show", "--format", "%{name}::%{full-key}"], ) # Load the actual keys, and the expected ones if they exist if not result.output: print("No results from parsing {}".format(project_dir)) return actual_keys = _parse_output_keys(result.output) expected_keys = _load_expected_keys(project_dir, actual_keys, raise_error=False) for element_name in actual_keys: expected = _element_filename(project_dir, element_name, "expected") if actual_keys[element_name] != expected_keys[element_name]: if not expected_keys[element_name]: print("Creating new expected file: {}".format(expected)) else: print("Updating expected file: {}".format(expected)) write_expected_key(project_dir, element_name, actual_keys[element_name]) if __name__ == "__main__": # patch the environment BST_TEST_SUITE value to something if it's not # present. This avoids an exception thrown at the cli level bst = "BST_TEST_SUITE" mock_bst = os.environ.get(bst, "True") with mock.patch.dict(os.environ, {**os.environ, bst: mock_bst}): update_keys() apache-buildstream-27ae392/src/buildstream/_testing/_utils/000077500000000000000000000000001514607367700240375ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/_testing/_utils/__init__.py000066400000000000000000000011251514607367700261470ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from .junction import generate_junction apache-buildstream-27ae392/src/buildstream/_testing/_utils/junction.py000066400000000000000000000043001514607367700262370ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import tarfile from buildstream import _yaml, utils from .. import Repo # generate_junction() # # Generates a junction element with a git repository # # Args: # tmpdir: The tmpdir fixture, for storing the generated git repo # subproject_path: The path for the subproject, to add to the git repo # junction_path: The location to store the generated junction element # store_ref: Whether to store the ref in the junction.bst file # # Returns: # (str): The ref # def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True, options=None): # Create a repo to hold the subproject and generate # a junction element for it # repo = _SimpleTar(str(tmpdir)) source_ref = ref = repo.create(subproject_path) if not store_ref: source_ref = None element = {"kind": "junction", "sources": [repo.source_config(ref=source_ref)]} if options: element["config"] = {"options": options} _yaml.roundtrip_dump(element, junction_path) return ref # A barebones Tar Repo class to use for generating junctions class _SimpleTar(Repo): def create(self, directory): tarball = os.path.join(self.repo, "file.tar.gz") old_dir = os.getcwd() os.chdir(directory) with tarfile.open(tarball, "w:gz") as tar: tar.add(".") os.chdir(old_dir) return utils.sha256sum(tarball) def source_config(self, ref=None): tarball = os.path.join(self.repo, "file.tar.gz") config = {"kind": "tar", "url": "file://" + tarball, "directory": "", "base-dir": ""} if ref is not None: config["ref"] = ref return config apache-buildstream-27ae392/src/buildstream/_testing/_utils/site.py000066400000000000000000000072131514607367700253600ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Some things resolved about the execution site, # so we dont have to repeat this everywhere # import os import stat import subprocess import sys import tempfile from typing import Optional # pylint: disable=unused-import from buildstream import utils, ProgramNotFoundError from buildstream._platform import Platform try: GIT = utils.get_host_tool("git") # type: Optional[str] HAVE_GIT = True out = str(subprocess.check_output(["git", "--version"]), "utf-8") # e.g. on Git for Windows we get "git version 2.21.0.windows.1". # e.g. on Mac via Homebrew we get "git version 2.19.0". version = tuple(int(x) for x in out.split(" ")[2].split(".")[:3]) HAVE_OLD_GIT = version < (1, 8, 5) GIT_ENV = { "GIT_AUTHOR_DATE": "1320966000 +0200", "GIT_AUTHOR_NAME": "tomjon", "GIT_AUTHOR_EMAIL": "tom@jon.com", "GIT_COMMITTER_DATE": "1320966000 +0200", "GIT_COMMITTER_NAME": "tomjon", "GIT_COMMITTER_EMAIL": "tom@jon.com", } except ProgramNotFoundError: GIT = None HAVE_GIT = False HAVE_OLD_GIT = False GIT_ENV = {} try: BZR = utils.get_host_tool("bzr") # type: Optional[str] HAVE_BZR = True # Breezy 3.0 supports `BRZ_EMAIL` but not `BZR_EMAIL` BZR_ENV = { "BZR_EMAIL": "Testy McTesterson ", "BRZ_EMAIL": "Testy McTesterson ", } except ProgramNotFoundError: BZR = None HAVE_BZR = False BZR_ENV = {} try: utils.get_host_tool("lzip") HAVE_LZIP = True except ProgramNotFoundError: HAVE_LZIP = False casd_path = utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox") CASD_SEPARATE_USER = bool(os.stat(casd_path).st_mode & stat.S_ISUID) del casd_path IS_LINUX = sys.platform.startswith("linux") IS_WINDOWS = os.name == "nt" MACHINE_ARCH = Platform.get_host_arch() HAVE_SANDBOX = None BUILDBOX_RUN = None try: path = utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox") subprocess.run([path, "--capabilities"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) BUILDBOX_RUN = os.path.basename(os.readlink(path)) HAVE_SANDBOX = "buildbox-run" except (ProgramNotFoundError, OSError, subprocess.CalledProcessError): pass # Check if we have subsecond mtime support on the # filesystem where @directory is located. # def have_subsecond_mtime(directory): try: test_file, test_filename = tempfile.mkstemp(dir=directory) os.close(test_file) except OSError: # If we can't create a temp file, lets just say this is False return False try: os.utime(test_filename, times=None, ns=(int(12345), int(12345))) except OSError: # If we can't set the mtime, lets just say this is False os.unlink(test_filename) return False try: stat_result = os.stat(test_filename) except OSError: # If we can't stat the file, lets just say this is False os.unlink(test_filename) return False os.unlink(test_filename) return stat_result.st_mtime_ns == 12345 apache-buildstream-27ae392/src/buildstream/_testing/_yaml.py000066400000000000000000000023621514607367700242160ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from buildstream import _yaml def load_yaml(filename): return _yaml.load(filename, shortname=os.path.basename(filename)) def generate_project(project_dir, config=None): if config is None: config = {} project_file = os.path.join(project_dir, "project.conf") if "name" not in config: config["name"] = os.path.basename(project_dir) if "min-version" not in config: config["min-version"] = "2.0" _yaml.roundtrip_dump(config, project_file) def generate_element(element_dir, element_name, config=None): if config is None: config = {} element_path = os.path.join(element_dir, element_name) _yaml.roundtrip_dump(config, element_path) apache-buildstream-27ae392/src/buildstream/_testing/integration.py000066400000000000000000000066241514607367700254450ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Integration - tools for inspecting the output of plugin integration tests ========================================================================= This module contains utilities for inspecting the artifacts produced during integration tests. """ import os import shutil import tempfile import pytest from buildstream import utils # Return a list of files relative to the given directory def walk_dir(root): for dirname, dirnames, filenames in os.walk(root): # ensure consistent traversal order, needed for consistent # handling of symlinks. dirnames.sort() filenames.sort() # print path to all subdirectories first. for subdirname in dirnames: yield os.path.join(dirname, subdirname)[len(root) :] # print path to all filenames. for filename in filenames: yield os.path.join(dirname, filename)[len(root) :] # Ensure that a directory contains the given filenames. # If `strict` is `True` then no additional filenames are allowed. def assert_contains(directory, expected, strict=False): expected = set(expected) missing = set(expected) found = set(walk_dir(directory)) # elements expected but not found missing.difference_update(found) if missing: msg = "Missing {} expected elements from list: {}".format(len(missing), missing) raise AssertionError(msg) if strict: # elements found but not expected found.difference_update(expected) msg = "{} additional elements were present in the directory: {}".format(len(found), found) if found: raise AssertionError(msg) class IntegrationCache: def __init__(self, cache): self.root = os.path.abspath(cache) os.makedirs(cache, exist_ok=True) # Use the same sources every time self.sources = os.path.join(self.root, "sources") # Create a temp directory for the duration of the test for # the artifacts directory try: self.cachedir = tempfile.mkdtemp(dir=self.root, prefix="cache-") # Apply mode allowed by umask os.chmod(self.cachedir, 0o777 & ~utils.get_umask()) except OSError as e: raise AssertionError("Unable to create test directory !") from e @pytest.fixture(scope="module") def integration_cache(request): # Set the cache dir to the INTEGRATION_CACHE variable, or the # default if that is not set. if "INTEGRATION_CACHE" in os.environ: cache_dir = os.environ["INTEGRATION_CACHE"] else: cache_dir = os.path.abspath("./integration-cache") cache = IntegrationCache(cache_dir) yield cache # Clean up the artifacts after each test session - we only want to # cache sources between tests shutil.rmtree(cache.cachedir, ignore_errors=True) shutil.rmtree(os.path.join(cache.root, "cas"), ignore_errors=True) apache-buildstream-27ae392/src/buildstream/_testing/repo.py000066400000000000000000000065101514607367700240610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Repo - Utility class for testing source plugins =============================================== """ import os import shutil class Repo: """Repo() Abstract class providing scaffolding for generating data to be used with various sources. Subclasses of Repo may be registered to run through the suite of generic source plugin tests provided in buildstream.testing. Args: directory (str): The base temp directory for the test subdir (str): The subdir for the repo, in case there is more than one """ def __init__(self, directory, subdir="repo"): # The working directory for the repo object # self.directory = os.path.abspath(directory) # The directory the actual repo will be stored in self.repo = os.path.join(self.directory, subdir) os.makedirs(self.repo, exist_ok=True) def create(self, directory): """Create a repository in self.directory and add the initial content Args: directory: A directory with content to commit Returns: (smth): A new ref corresponding to this commit, which can be passed as the ref in the Repo.source_config() API. """ raise NotImplementedError("create method has not been implemeted") def source_config(self, ref=None): """ Args: ref (smth): An optional abstract ref object, usually a string. Returns: (dict): A configuration which can be serialized as a source when generating an element file on the fly """ raise NotImplementedError("source_config method has not been implemeted") def copy_directory(self, src, dest): """Copies the content of src to the directory dest Like shutil.copytree(), except dest is expected to exist. Args: src (str): The source directory dest (str): The destination directory """ for filename in os.listdir(src): src_path = os.path.join(src, filename) dest_path = os.path.join(dest, filename) if os.path.isdir(src_path): shutil.copytree(src_path, dest_path) else: shutil.copy2(src_path, dest_path) def copy(self, dest): """Creates a copy of this repository in the specified destination. Args: dest (str): The destination directory Returns: (Repo): A Repo object for the new repository. """ subdir = self.repo[len(self.directory) :].lstrip(os.sep) new_dir = os.path.join(dest, subdir) os.makedirs(new_dir, exist_ok=True) self.copy_directory(self.repo, new_dir) repo_type = type(self) new_repo = repo_type(dest, subdir) return new_repo apache-buildstream-27ae392/src/buildstream/_testing/runcli.py000066400000000000000000000744551514607367700244250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ runcli - Test fixtures used for running BuildStream commands ============================================================ :function:'cli' Use result = cli.run([arg1, arg2]) to run buildstream commands :function:'cli_integration' A variant of the main fixture that keeps persistent artifact and source caches. It also does not use the click test runner to avoid deadlock issues when running `bst shell`, but unfortunately cannot produce nice stacktraces. """ import os import re import sys import shutil import tempfile import itertools import traceback from contextlib import contextmanager, ExitStack from ruamel import yaml import pytest # XXX Using pytest private internals here # # We use pytest internals to capture the stdout/stderr during # a run of the buildstream CLI. We do this because click's # CliRunner convenience API (click.testing module) does not support # separation of stdout/stderr. # from _pytest.capture import MultiCapture, FDCapture, FDCaptureBinary # Import the main cli entrypoint from buildstream._frontend import cli as bst_cli from buildstream import _yaml, node from buildstream._cas import CASCache from buildstream.element import _get_normal_name, _compose_artifact_name # Special private exception accessor, for test case purposes from buildstream._exceptions import BstError, get_last_exception, get_last_task_error from buildstream._protos.buildstream.v2 import artifact_pb2 # Wrapper for the click.testing result class Result: def __init__(self, exit_code=None, exception=None, exc_info=None, output=None, stderr=None): self.exit_code = exit_code self.exc = exception self.exc_info = exc_info self.output = output self.stderr = stderr self.unhandled_exception = False # The last exception/error state is stored at exception # creation time in BstError(), but this breaks down with # recoverable errors where code blocks ignore some errors # and fallback to alternative branches. # # For this reason, we just ignore the exception and errors # in the case that the exit code reported is 0 (success). # if self.exit_code != 0: # Check if buildstream failed to handle an # exception, topevel CLI exit should always # be a SystemExit exception. # if not isinstance(exception, SystemExit): self.unhandled_exception = True self.exception = get_last_exception() self.task_error_domain, self.task_error_reason = get_last_task_error() else: self.exception = None self.task_error_domain = None self.task_error_reason = None # assert_success() # # Asserts that the buildstream session completed successfully # # Args: # fail_message (str): An optional message to override the automatic # assertion error messages # Raises: # (AssertionError): If the session did not complete successfully # def assert_success(self, fail_message=""): assert self.exit_code == 0, fail_message assert self.exc is None, fail_message assert self.exception is None, fail_message assert self.unhandled_exception is False # assert_main_error() # # Asserts that the buildstream session failed, and that # the main process error report is as expected # # Args: # error_domain (ErrorDomain): The domain of the error which occurred # error_reason (any): The reason field of the error which occurred # fail_message (str): An optional message to override the automatic # assertion error messages # debug (bool): If true, prints information regarding the exit state of the result() # Raises: # (AssertionError): If any of the assertions fail # def assert_main_error(self, error_domain, error_reason, fail_message="", *, debug=False): if debug: print( """ Exit code: {} Exception: {} Domain: {} Reason: {} """.format( self.exit_code, self.exception, self.exception.domain, self.exception.reason ) ) assert self.exit_code == -1, fail_message assert self.exc is not None, fail_message assert self.exception is not None, fail_message assert isinstance(self.exception, BstError), fail_message assert self.unhandled_exception is False assert self.exception.domain == error_domain, fail_message assert self.exception.reason == error_reason, fail_message # assert_task_error() # # Asserts that the buildstream session failed, and that # the child task error which caused buildstream to exit # is as expected. # # Args: # error_domain (ErrorDomain): The domain of the error which occurred # error_reason (any): The reason field of the error which occurred # fail_message (str): An optional message to override the automatic # assertion error messages # Raises: # (AssertionError): If any of the assertions fail # def assert_task_error(self, error_domain, error_reason, fail_message=""): assert self.exit_code == -1, fail_message assert self.exc is not None, fail_message assert self.exception is not None, fail_message assert isinstance(self.exception, BstError), fail_message assert self.unhandled_exception is False assert self.task_error_domain == error_domain, fail_message assert self.task_error_reason == error_reason, fail_message # assert_shell_error() # # Asserts that the buildstream created a shell and that the task in the # shell failed. # # Args: # fail_message (str): An optional message to override the automatic # assertion error messages # Raises: # (AssertionError): If any of the assertions fail # def assert_shell_error(self, fail_message=""): assert self.exit_code == 1, fail_message # get_start_order() # # Gets the list of elements processed in a given queue, in the # order of their first appearances in the session. # # Args: # activity (str): The queue activity name (like 'fetch') # # Returns: # (list): A list of element names in the order which they first appeared in the result # def get_start_order(self, activity): results = re.findall(r"\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log".format(activity), self.stderr) if results is None: return [] return list(results) # get_tracked_elements() # # Produces a list of element names on which tracking occurred # during the session. # # This is done by parsing the buildstream stderr log # # Returns: # (list): A list of element names # def get_tracked_elements(self): tracked = re.findall(r"\[\s*track:(\S+)\s*]", self.stderr) if tracked is None: return [] return list(tracked) def get_built_elements(self): built = re.findall(r"\[\s*build:(\S+)\s*\]\s*SUCCESS\s*Caching artifact", self.stderr) if built is None: return [] return list(built) def get_pushed_elements(self): pushed = re.findall(r"\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed artifact", self.stderr) if pushed is None: return [] return list(pushed) def get_pulled_elements(self): pulled = re.findall(r"\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact", self.stderr) if pulled is None: return [] return list(pulled) def get_discarded_elements(self): discarded = re.findall(r"\[\s*(?:main|pull):(\S+)\s*\]\s*INFO\s*Discarded failed build", self.stderr) if discarded is None: return [] return list(discarded) class Cli: def __init__(self, directory, verbose=True, default_options=None): self.directory = directory self.config = None self.verbose = verbose self.artifact = TestArtifact() os.makedirs(directory) if default_options is None: default_options = [] self.default_options = default_options # configure(): # # Serializes a user configuration into a buildstream.conf # to use for this test cli. # # Args: # config (dict): The user configuration to use # def configure(self, config): if self.config is None: self.config = {} for key, val in config.items(): self.config[key] = val # remove_artifact_from_cache(): # # Remove given element artifact from artifact cache # # Args: # project (str): The project path under test # element_name (str): The name of the element artifact # cache_dir (str): Specific cache dir to remove artifact from # def remove_artifact_from_cache(self, project, element_name, *, cache_dir=None): # Read configuration to figure out where artifacts are stored if not cache_dir: default = os.path.join(project, "cache") if self.config is not None: cache_dir = self.config.get("cachedir", default) else: cache_dir = default self.artifact.remove_artifact_from_cache(cache_dir, element_name) # run(): # # Runs buildstream with the given arguments, additionally # also passes some global options to buildstream in order # to stay contained in the testing environment. # # Args: # project (str): An optional path to a project # silent (bool): Whether to pass --no-verbose # env (dict): Environment variables to temporarily set during the test # args (list): A list of arguments to pass buildstream # binary_capture (bool): Whether to capture the stdout/stderr as binary # def run(self, project=None, silent=False, env=None, cwd=None, options=None, args=None, binary_capture=False): # We don't want to carry the state of one bst invocation into another # bst invocation. Since node _FileInfo objects hold onto BuildStream # projects, this means that they would be also carried forward. This # becomes a problem when spawning new processes - when pickling the # state of the node module we will also be pickling elements from # previous bst invocations. node._reset_global_state() if args is None: args = [] if options is None: options = [] # We may have been passed e.g. pathlib.Path or py.path args = [str(x) for x in args] options = self.default_options + options with ExitStack() as stack: bst_args = ["--no-colors"] if silent: bst_args += ["--no-verbose"] config_file = stack.enter_context(configured(self.directory, self.config)) bst_args += ["--config", config_file] if project: bst_args += ["--directory", str(project)] for option, value in options: bst_args += ["--option", option, value] bst_args += args if cwd is not None: stack.enter_context(chdir(cwd)) if env is not None: stack.enter_context(environment(env)) result = self._invoke(bst_cli, bst_args, binary_capture=binary_capture) # Some informative stdout we can observe when anything fails if self.verbose: command = "bst " + " ".join(bst_args) print("BuildStream exited with code {} for invocation:\n\t{}".format(result.exit_code, command)) if result.output: print("Program output was:\n{}".format(result.output)) if result.stderr: print("Program stderr was:\n{}".format(result.stderr)) if result.exc_info and result.exc_info[0] != SystemExit: traceback.print_exception(*result.exc_info) return result def _invoke(self, cli_object, args=None, binary_capture=False): exc_info = None exception = None exit_code = 0 # Temporarily redirect sys.stdin to /dev/null to ensure that # Popen doesn't attempt to read pytest's dummy stdin. old_stdin = sys.stdin with open(os.devnull, "rb") as devnull: sys.stdin = devnull capture_kind = FDCaptureBinary if binary_capture else FDCapture capture = MultiCapture(out=capture_kind(1), err=capture_kind(2), in_=None) capture.start_capturing() try: cli_object.main(args=args or (), prog_name=cli_object.name) except SystemExit as e: if e.code != 0: exception = e exc_info = sys.exc_info() exit_code = e.code if not isinstance(exit_code, int): sys.stdout.write("Program exit code was not an integer: ") sys.stdout.write(str(exit_code)) sys.stdout.write("\n") exit_code = 1 except Exception as e: # pylint: disable=broad-except exception = e exit_code = -1 exc_info = sys.exc_info() finally: sys.stdout.flush() sys.stdin = old_stdin out, err = capture.readouterr() capture.stop_capturing() return Result(exit_code=exit_code, exception=exception, exc_info=exc_info, output=out, stderr=err) # Fetch an element state by name by # invoking bst show on the project with the CLI # # If you need to get the states of multiple elements, # then use get_element_states(s) instead. # def get_element_state(self, project, element_name): result = self.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{state}", element_name] ) result.assert_success() return result.output.strip() # Fetch the states of elements for a given target / deps # # Returns a dictionary with the element names as keys # def get_element_states(self, project, targets, deps="all"): result = self.run( project=project, silent=True, args=["show", "--deps", deps, "--format", "%{name}||%{state}", *targets] ) result.assert_success() lines = result.output.splitlines() states = {} for line in lines: split = line.split(sep="||") states[split[0]] = split[1] return states # Fetch an element's cache key by invoking bst show # on the project with the CLI # def get_element_key(self, project, element_name): result = self.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{full-key}", element_name] ) result.assert_success() return result.output.strip() # Get the decoded config of an element. # def get_element_config(self, project, element_name): result = self.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element_name] ) result.assert_success() return yaml.safe_load(result.output) # Fetch the elements that would be in the pipeline with the given # arguments. # def get_pipeline(self, project, elements, except_=None, scope="all"): if except_ is None: except_ = [] args = ["show", "--deps", scope, "--format", "%{name}"] args += list(itertools.chain.from_iterable(zip(itertools.repeat("--except"), except_))) result = self.run(project=project, silent=True, args=args + elements) result.assert_success() return result.output.splitlines() # Fetch an element's complete artifact name, cache_key will be generated # if not given. # def get_artifact_name(self, project, project_name, element_name, cache_key=None): if not cache_key: cache_key = self.get_element_key(project, element_name) # Replace path separator and chop off the .bst suffix for normal name normal_name = _get_normal_name(element_name) return _compose_artifact_name(project_name, normal_name, cache_key) class CliIntegration(Cli): # run() # # This supports the same arguments as Cli.run(), see run_project_config(). # def run(self, project=None, silent=False, env=None, cwd=None, options=None, args=None, binary_capture=False): return self.run_project_config( project=project, silent=silent, env=env, cwd=cwd, options=options, args=args, binary_capture=binary_capture ) # run_project_config() # # This supports the same arguments as Cli.run() and additionally # it supports the project_config keyword argument. # # This will first load the project.conf file from the specified # project directory ('project' keyword argument) and perform substitutions # of any {project_dir} specified in the existing project.conf. # # If the project_config parameter is specified, it is expected to # be a dictionary of additional project configuration options, and # will be composited on top of the already loaded project.conf # def run_project_config(self, *, project_config=None, **kwargs): # First load the project.conf and substitute {project_dir} # # Save the original project.conf, because we will run more than # once in the same temp directory # project_directory = kwargs["project"] project_filename = os.path.join(project_directory, "project.conf") project_backup = os.path.join(project_directory, "project.conf.backup") project_load_filename = project_filename if not os.path.exists(project_backup): shutil.copy(project_filename, project_backup) else: project_load_filename = project_backup with open(project_load_filename, encoding="utf-8") as f: config = f.read() config = config.format(project_dir=project_directory) if project_config is not None: # If a custom project configuration dictionary was # specified, composite it on top of the already # substituted base project configuration # base_config = _yaml.load_data(config) # In order to leverage _yaml.composite_dict(), both # dictionaries need to be loaded via _yaml.load_data() first # with tempfile.TemporaryDirectory(dir=project_directory) as scratchdir: temp_project = os.path.join(scratchdir, "project.conf") with open(temp_project, "w", encoding="utf-8") as f: _yaml.roundtrip_dump(project_config, f) project_config = _yaml.load(temp_project, shortname="project.conf") project_config._composite(base_config) _yaml.roundtrip_dump(base_config, project_filename) else: # Otherwise, just dump it as is with open(project_filename, "w", encoding="utf-8") as f: f.write(config) return super().run(**kwargs) class CliRemote(CliIntegration): # ensure_services(): # # Make sure that required services are configured and that # non-required ones are not. # # Args: # actions (bool): Whether to use the 'action-cache' service # artifacts (bool): Whether to use the 'artifact-cache' service # execution (bool): Whether to use the 'execution' service # sources (bool): Whether to use the 'source-cache' service # storage (bool): Whether to use the 'storage' service # # Returns a list of configured services (by names). # def ensure_services(self, actions=True, execution=True, storage=True, artifacts=False, sources=False): # Build a list of configured services by name: configured_services = [] if not self.config: return configured_services if "remote-execution" in self.config: rexec_config = self.config["remote-execution"] if "action-cache-service" in rexec_config: if actions: configured_services.append("action-cache") else: rexec_config.pop("action-cache-service") if "execution-service" in rexec_config: if execution: configured_services.append("execution") else: rexec_config.pop("execution-service") if "storage-service" in rexec_config: if storage: configured_services.append("storage") else: rexec_config.pop("storage-service") if "artifacts" in self.config: if artifacts: configured_services.append("artifact-cache") else: self.config.pop("artifacts") if "source-caches" in self.config: if sources: configured_services.append("source-cache") else: self.config.pop("source-caches") return configured_services class TestArtifact: # remove_artifact_from_cache(): # # Remove given element artifact from artifact cache # # Args: # cache_dir (str): Specific cache dir to remove artifact from # element_name (str): The name of the element artifact # def remove_artifact_from_cache(self, cache_dir, element_name): cache_dir = os.path.join(cache_dir, "artifacts", "refs") normal_name = element_name.replace(os.sep, "-") cache_dir = os.path.splitext(os.path.join(cache_dir, "test", normal_name))[0] # Don't cause a failure here if the artifact is not there, we use this in integration # tests where the cache is shared and we just want to clean up some artifacts # regardless of whether they have already been built or not. try: shutil.rmtree(cache_dir) except FileNotFoundError: pass # is_cached(): # # Check if given element has a cached artifact # # Args: # cache_dir (str): Specific cache dir to check # element (Element): The element object # element_key (str): The element's cache key # # Returns: # (bool): If the cache contains the element's artifact # def is_cached(self, cache_dir, element, element_key): # cas = CASCache(str(cache_dir)) artifact_ref = element.get_artifact_name(element_key) return os.path.exists(os.path.join(cache_dir, "artifacts", "refs", artifact_ref)) # get_digest(): # # Get the digest for a given element's artifact files # # Args: # cache_dir (str): Specific cache dir to check # element (Element): The element object # element_key (str): The element's cache key # # Returns: # (Digest): The digest stored in the ref # def get_digest(self, cache_dir, element, element_key): artifact_ref = element.get_artifact_name(element_key) artifact_dir = os.path.join(cache_dir, "artifacts", "refs") artifact_proto = artifact_pb2.Artifact() with open(os.path.join(artifact_dir, artifact_ref), "rb") as f: artifact_proto.ParseFromString(f.read()) return artifact_proto.files # extract_buildtree(): # # Context manager for extracting an elements artifact buildtree for # inspection. # # Args: # tmpdir (LocalPath): pytest fixture for the tests tmp dir # digest (Digest): The element directory digest to extract # # Yields: # (str): path to extracted buildtree directory, does not guarantee # existence. @contextmanager def extract_buildtree(self, cache_dir, tmpdir, ref): artifact = artifact_pb2.Artifact() try: with open(os.path.join(cache_dir, "artifacts", "refs", ref), "rb") as f: artifact.ParseFromString(f.read()) except FileNotFoundError: yield None else: if str(artifact.buildtree): # pylint: disable-next=contextmanager-generator-missing-cleanup with self._extract_subdirectory(tmpdir, artifact.buildtree) as f: yield f else: yield None # _extract_subdirectory(): # # Context manager for extracting an element artifact for inspection, # providing an expected path for a given subdirectory # # Args: # tmpdir (LocalPath): pytest fixture for the tests tmp dir # digest (Digest): The element directory digest to extract # subdir (str): Subdirectory to path # # Yields: # (str): path to extracted subdir directory, does not guarantee # existence. @contextmanager def _extract_subdirectory(self, tmpdir, digest): with tempfile.TemporaryDirectory() as extractdir: try: cas = CASCache(str(tmpdir), casd=None) cas.checkout(extractdir, digest) yield extractdir except FileNotFoundError: yield None # Main fixture # # Use result = cli.run([arg1, arg2]) to run buildstream commands # @pytest.fixture() def cli(tmpdir): directory = os.path.join(str(tmpdir), "cache") return Cli(directory) # A variant of the main fixture that keeps persistent artifact and # source caches. # # It also does not use the click test runner to avoid deadlock issues # when running `bst shell`, but unfortunately cannot produce nice # stacktraces. @pytest.fixture() def cli_integration(tmpdir, integration_cache): directory = os.path.join(str(tmpdir), "cache") fixture = CliIntegration(directory) # We want to cache sources for integration tests more permanently, # to avoid downloading the huge base-sdk repeatedly fixture.configure( { "cachedir": integration_cache.cachedir, "sourcedir": integration_cache.sources, } ) yield fixture # remove following folders if necessary try: shutil.rmtree(os.path.join(integration_cache.cachedir, "build")) except FileNotFoundError: pass try: shutil.rmtree(os.path.join(integration_cache.cachedir, "tmp")) except FileNotFoundError: pass # A variant of the main fixture that is configured for remote-execution. # # It also does not use the click test runner to avoid deadlock issues # when running `bst shell`, but unfortunately cannot produce nice # stacktraces. @pytest.fixture() def cli_remote_execution(tmpdir, remote_services): directory = os.path.join(str(tmpdir), "cache") fixture = CliRemote(directory) artifacts = [] if remote_services.artifact_service: artifacts.append({"url": remote_services.artifact_service, "push": True}) if remote_services.artifact_index_service: artifacts.append({"url": remote_services.artifact_index_service, "push": True, "type": "index"}) if remote_services.artifact_storage_service: artifacts.append({"url": remote_services.artifact_storage_service, "push": True, "type": "storage"}) if artifacts: fixture.configure({"artifacts": {"servers": artifacts}}) remote_execution = {} if remote_services.action_service: remote_execution["action-cache-service"] = { "url": remote_services.action_service, } if remote_services.exec_service: remote_execution["execution-service"] = { "url": remote_services.exec_service, } if remote_services.storage_service: remote_execution["storage-service"] = { "url": remote_services.storage_service, } if remote_execution: fixture.configure({"remote-execution": remote_execution}) if remote_services.source_service: fixture.configure( { "source-caches": { "servers": [ { "url": remote_services.source_service, } ] } } ) return fixture @contextmanager def chdir(directory): old_dir = os.getcwd() os.chdir(directory) yield os.chdir(old_dir) @contextmanager def environment(env): old_env = {} for key, value in env.items(): old_env[key] = os.environ.get(key) if value is None: os.environ.pop(key, None) else: os.environ[key] = value yield for key, value in old_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @contextmanager def configured(directory, config=None): # Ensure we've at least relocated the caches to a temp directory if not config: config = {} if not config.get("sourcedir", False): config["sourcedir"] = os.path.join(directory, "sources") if not config.get("cachedir", False): config["cachedir"] = directory if not config.get("logdir", False): config["logdir"] = os.path.join(directory, "logs") cas_stage_root = os.environ.get("BST_CAS_STAGING_ROOT") if cas_stage_root: symlink_path = os.path.join(config["cachedir"], "cas", "staging") if not os.path.lexists(symlink_path): os.makedirs(os.path.join(config["cachedir"], "cas"), exist_ok=True) os.symlink(cas_stage_root, symlink_path) # Dump it and yield the filename for test scripts to feed it # to buildstream as an artument filename = os.path.join(directory, "buildstream.conf") _yaml.roundtrip_dump(config, filename) yield filename apache-buildstream-27ae392/src/buildstream/_types.pyi000066400000000000000000000011131514607367700227460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # class MetaFastEnum(type): ... apache-buildstream-27ae392/src/buildstream/_types.pyx000066400000000000000000000060441514607367700227750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jim MacArthur # Benjamin Schubert # MetaFastEnum() # # This is a reemplementation of MetaEnum, in order to get a faster implementation of Enum. # # Enum turns out to be very slow and would add a noticeable slowdown when we try to add them to the codebase. # We therefore reimplement a subset of the `Enum` functionality that keeps it compatible with normal `Enum`. # That way, any place in the code base that access a `FastEnum`, can normally also accept an `Enum`. The reverse # is not correct, since we only implement a subset of `Enum`. class MetaFastEnum(type): def __new__(mcs, name, bases, dct): if name == "FastEnum": return type.__new__(mcs, name, bases, dct) assert len(bases) == 1, "Multiple inheritance with Fast enums is not currently supported." dunder_values = {} normal_values = {} parent_keys = bases[0].__dict__.keys() assert "__class__" not in dct.keys(), "Overriding '__class__' is not allowed on 'FastEnum' classes" for key, value in dct.items(): if key.startswith("__") and key.endswith("__"): dunder_values[key] = value else: assert key not in parent_keys, "Overriding 'FastEnum.{}' is not allowed. ".format(key) normal_values[key] = value kls = type.__new__(mcs, name, bases, dunder_values) mcs.set_values(kls, normal_values) return kls @classmethod def set_values(mcs, kls, data): value_to_entry = {} assert len(set(data.values())) == len(data.values()), "Values for {} are not unique".format(kls) assert len(set(type(value) for value in data.values())) <= 1, \ "Values of {} are of heterogeneous types".format(kls) for key, value in data.items(): new_value = object.__new__(kls) object.__setattr__(new_value, "value", value) object.__setattr__(new_value, "name", key) type.__setattr__(kls, key, new_value) value_to_entry[value] = new_value type.__setattr__(kls, "_value_to_entry", value_to_entry) def __repr__(self): return "".format(self.__name__) def __setattr__(self, key, value): raise AttributeError("Adding new values dynamically is not supported") def __iter__(self): return iter(self._value_to_entry.values()) apache-buildstream-27ae392/src/buildstream/_utils.pyi000066400000000000000000000012711514607367700227470ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # def url_directory_name(url: str) -> str: ... def valid_chars_name(name: str) -> bool: ... def terminate_thread(thread_id: int) -> None: ... apache-buildstream-27ae392/src/buildstream/_utils.pyx000066400000000000000000000056261514607367700227760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Benjamin Schubert # """ This module contains utilities that have been optimized in Cython """ from cpython.pystate cimport PyThreadState_SetAsyncExc from cpython.ref cimport PyObject from ._signals import TerminateException def url_directory_name(str url): """Normalizes a url into a directory name Args: url (str): A url string Returns: A string which can be used as a directory name """ return ''.join([_transl(x) for x in url]) # terminate_thread() # # Ask a given a given thread to terminate by raising an exception in it. # # Args: # thread_id (int): the thread id in which to throw the exception # def terminate_thread(long thread_id): res = PyThreadState_SetAsyncExc(thread_id, TerminateException) assert res == 1 # Check if given filename containers valid characters. # # Args: # name (str): Name of the file # # Returns: # (bool): True if all characters are valid, False otherwise. # def valid_chars_name(str name): cdef int char_value cdef int forbidden_char for char_value in name: # 0-31 are control chars, 127 is DEL, and >127 means non-ASCII if char_value <= 31 or char_value >= 127: return False # Disallow characters that are invalid on Windows. The list can be # found at https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file # # Note that although : (colon) is not allowed, we do not raise # warnings because of that, since we use it as a separator for # junctioned elements. # # We also do not raise warnings on slashes since they are used as # path separators. for forbidden_char in '<>"|?*': if char_value == forbidden_char: return False return True ############################################################# # Module local helper Methods # ############################################################# # _transl(x) # # Helper for `url_directory_name` # # This transforms the value to "_" if is it not a ascii letter, a digit or "%" or "_" # cdef Py_UNICODE _transl(Py_UNICODE x): if ("a" <= x <= "z") or ("A" <= x <= "Z") or ("0" <= x <= "9") or x == "%": return x return "_" # get_mirror_directory in configure of _downlaodablefilesource apache-buildstream-27ae392/src/buildstream/_variables.pyi000066400000000000000000000014711514607367700235610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Optional from .node import MappingNode, Node class Variables: def __init__(self, node: MappingNode) -> None: ... def check(self) -> None: ... def expand(self, node: Node) -> None: ... def get(self, name: str) -> Optional[str]: ... apache-buildstream-27ae392/src/buildstream/_variables.pyx000066400000000000000000000615301514607367700236020ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Daniel Silverstone # Benjamin Schubert import re import sys import itertools from ._exceptions import LoadError from .exceptions import LoadErrorReason from .node cimport MappingNode, Node, ScalarNode, SequenceNode, ProvenanceInformation ######################################################## # Understanding Value Expressions # ######################################################## # # This code uses the term "value expression" a lot to refer to `str` objects # which have references to variables in them, and also to `list` objects which # are effectively broken down strings. # # Ideally we would have a ValueExpression type in order to make this more # comprehensive, but this would unfortunately introduce unnecessary overhead, # making the code measurably slower. # # Value Expression Strings # ------------------------ # Strings which contain variables in them, such as: # # "My name is %{username}, good day." # # # Parsed Value Expression Lists # ----------------------------- # Using `re.split()` from python's regular expression implementation, we # parse the list using our locally defined VALUE_EXPRESSION_REGEX, which # breaks down the string into a list of "literal" and "variable" components. # # The "literal" components are literal portions of the string which need # no substitution, while the "variable" components represent variable names # which need to be substituted with their corresponding resolved values. # # The parsed variable expressions have the following properties: # # * They are sparse, some of the "literal" values contain zero length # strings which can be ignored. # # * Literal values are found only at even indices of the parsed # variable expression # # * Variable names are found only at odd indices # # The above example "My name is %{username}, good day." is broken down # into a parsed value expression as follows: # # [ # "My name is ", # <- Index 0, literal value # "username", # <- Index 1, variable name, '%{ ... }' discarded # ", good day." # <- Index 2, literal value # ] # # Maximum recursion depth using the fast (recursive) variable resolution # algorithm. # cdef Py_ssize_t MAX_RECURSION_DEPTH = 200 # Regular expression used to parse %{variables} in value expressions # # Note that variables are allowed to have dashes # VALUE_EXPRESSION_REGEX = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}") # Cache for the parsed expansion strings. # cdef dict VALUE_EXPRESSION_CACHE = { # Prime the cache with the empty string since otherwise that can # cause issues with the parser, complications to which cause slowdown "": [""], } # Variables() # # The Variables object resolves the variable references in the given MappingNode, # expecting that any dictionary values which contain variable references can be # resolved from the same dictionary. # # Each Element creates its own Variables instance to track the configured # variable settings for the element. # # Notably, this object is delegated the responsibility of expanding # variables in yaml Node hierarchies and substituting variables in strings # in the context of a given Element's variable configuration. # # Args: # node (Node): A node loaded and composited with yaml tools # # Raises: # LoadError, if unresolved variables, or cycles in resolution, occur. # cdef class Variables: cdef MappingNode _original cdef dict _values ################################################################# # Dunder Methods # ################################################################# def __init__(self, MappingNode node): # The original MappingNode, we need to keep this # around for proper error reporting. # self._original = node # The value map, this dictionary contains either unresolved # value expressions, or resolved values. # # Each mapping value is a list, in the case that the value # is resolved, then the list is only 1 element long. # self._values = self._init_values(node) # __getitem__() # # Fetches a resolved variable by it's name, allows # addressing the Variables instance like a dictionary. # # Args: # name (str): The name of the variable # # Returns: # (str): The resolved variable value # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # def __getitem__(self, str name): if name not in self._values: raise KeyError(name) return self._expand_var(name) # __contains__() # # Checks whether a given variable exists, allows # supporting `if 'foo' in variables` expressions. # # Args: # name (str): The name of the variable to check for # # Returns: # (bool): True if `name` is a valid variable # def __contains__(self, str name): return name in self._values # __iter__() # # Provide an iterator for all variables effective values # # Returns: # (Iterator[Tuple[str, str]]) # def __iter__(self): return _VariablesIterator(self) ################################################################# # Public API # ################################################################# # check() # # Assert that all variables declared on this Variables # instance have been resolved properly, and reports errors # for undefined references and circular references. # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cpdef check(self): cdef object key # Just resolve all variables. for key in self._values.keys(): self._expand_var( key) # get() # # Expand definition of variable by name. If the variable is not # defined, it will return None instead of failing. # # Args: # name (str): Name of the variable to expand # # Returns: # (str|None): The expanded value for the variable or None variable was not defined. # cpdef str get(self, str name): if name not in self._values: return None return self[name] # expand() # # Expand all the variables found in the given Node, recursively. # This does the change in place, modifying the node. If you want to keep # the node untouched, you should use `node.clone()` beforehand # # Args: # (Node): A node for which to substitute the values # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cpdef expand(self, Node node): if isinstance(node, ScalarNode): ( node).value = self.subst( node) elif isinstance(node, SequenceNode): for entry in ( node).value: self.expand(entry) elif isinstance(node, MappingNode): for entry in ( node).value.values(): self.expand(entry) else: assert False, "Unknown 'Node' type" # subst(): # # Substitutes any variables in 'string' and returns the result. # # Args: # node (ScalarNode): The ScalarNode to substitute variables in # # Returns: # (str): The new string with any substitutions made # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cpdef str subst(self, ScalarNode node): value_expression = _parse_value_expression(node.as_str()) return self._expand_value_expression(value_expression, node) ################################################################# # Private API # ################################################################# # _init_values() # # Initialize the table of values. # # The value table is a dictionary keyed by the variable names where # the values are value expressions (lists) which are initially unresolved. # # Value expressions are later resolved on demand and replaced in this # table with single element lists. # # Args: # node (MappingNode): The original variables mapping node # # Returns: # (dict): A dictionary of value expressions (lists) # cdef dict _init_values(self, MappingNode node): # Special case, if notparallel is specified in the variables for this # element, then override max-jobs to be 1. # Initialize it as a string as all variables are processed as strings. # if node.get_bool('notparallel', False): node['max-jobs'] = str(1) cdef dict ret = {} cdef object key_object cdef str key cdef str value for key_object in node.keys(): key = key_object value = node.get_str(key) ret[sys.intern(key)] = _parse_value_expression(value) return ret # _expand_var() # # Expand and cache a variable definition. # # This will try the fast, recursive path first and fallback to # the slower iterative codepath. # # Args: # name (str): Name of the variable to expand # # Returns: # (str): The expanded value of variable # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cdef str _expand_var(self, str name): try: return self._fast_expand_var(name) except (KeyError, RecursionError): return self._slow_expand_var(name) # _expand_value_expression() # # Expands a value expression # # This will try the fast, recursive path first and fallback to # the slower iterative codepath. # # Args: # value_expression (list): The parsed value expression to be expanded # node (ScalarNode): The toplevel ScalarNode who is asking for an expansion # # Returns: # (str): The expanded value expression # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cdef str _expand_value_expression(self, list value_expression, ScalarNode node): try: return self._fast_expand_value_expression(value_expression) except (KeyError, RecursionError): return self._slow_expand_value_expression(None, value_expression, node) ################################################################# # Resolution algorithm: fast path # ################################################################# # _fast_expand_var() # # Fast, recursive path for variable expansion # # Args: # name (str): Name of the variable to expand # counter (int): Number of recursion cycles (used only in recursion) # # Returns: # (str): The expanded value of variable # # Raises: # (KeyError): If a reference to an undefined variable is encountered # (RecursionError): If MAX_RECURSION_DEPTH recursion cycles is reached # cdef str _fast_expand_var(self, str name, int counter = 0): cdef str sub cdef list value_expression value_expression = self._values[name] if len(value_expression) > 1: sub = self._fast_expand_value_expression(value_expression, counter) value_expression = [sys.intern(sub)] self._values[name] = value_expression return value_expression[0] # _fast_expand_value_expression() # # Fast, recursive path for value expression expansion. # # Args: # value_expression (list): The parsed value expression to be expanded # counter (int): Number of recursion cycles (used only in recursion) # # Returns: # (str): The expanded value expression # # Raises: # (KeyError): If a reference to an undefined variable is encountered # (RecursionError): If MAX_RECURSION_DEPTH recursion cycles is reached # cdef str _fast_expand_value_expression(self, list value_expression, int counter = 0): if counter > MAX_RECURSION_DEPTH: raise RecursionError() cdef Py_ssize_t idx cdef object value cdef list acc = [] for idx, value in enumerate(value_expression): if (idx % 2) == 0: acc.append(value) else: acc.append(self._fast_expand_var( value, counter + 1)) return "".join(acc) ################################################################# # Resolution algorithm: slow path # ################################################################# # _slow_expand_var() # # Slow, iterative path for variable expansion with full error reporting # # Args: # name (str): Name of the variable to expand # # Returns: # (str): The expanded value of variable # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cdef str _slow_expand_var(self, str name): cdef list value_expression cdef str expanded value_expression = self._get_checked_value_expression(name, None, None) if len(value_expression) > 1: expanded = self._slow_expand_value_expression(name, value_expression, None) value_expression = [sys.intern(expanded)] self._values[name] = value_expression return value_expression[0] # _slow_expand_value_expression() # # Slow, iterative path for value expression expansion with full error reporting # # Note that either `varname` or `node` must be provided, these are used to # identify the provenance of this value expression (which might be the value # of a variable, or a value expression found elswhere in project YAML which # needs to be substituted). # # Args: # varname (str|None): The variable name associated with this value expression, if any # value_expression (list): The parsed value expression to be expanded # node (ScalarNode|None): The ScalarNode who is asking for an expansion # # Returns: # (str): The expanded value expression # # Raises: # (LoadError): In the case of an undefined variable or # a cyclic variable reference # cdef str _slow_expand_value_expression(self, str varname, list value_expression, ScalarNode node): cdef ResolutionStep step cdef ResolutionStep new_step cdef ResolutionStep this_step cdef list iter_value_expression cdef Py_ssize_t idx = 0 cdef object value cdef str resolved_varname cdef str resolved_value = None # We will collect the varnames and value expressions which need # to be resolved in the loop, sorted by dependency, and then # finally reverse through them resolving them one at a time # cdef list resolved_varnames = [] cdef list resolved_values = [] step = ResolutionStep() step.init(varname, value_expression, None) while step: # Keep a hold of the current overall step this_step = step step = step.prev # Check for circular dependencies this_step.check_circular(self._original) for idx, value in enumerate(this_step.value_expression): # Skip literal parts of the value expression if (idx % 2) == 0: continue iter_value_expression = self._get_checked_value_expression( value, this_step.referee, node) # Queue up this value. # # Even if the value was already resolved, we need it in context to resolve # previously enqueued variables resolved_values.append(iter_value_expression) resolved_varnames.append(value) # Queue up the values dependencies. # if len(iter_value_expression) > 1: new_step = ResolutionStep() new_step.init( value, iter_value_expression, this_step) # Link it to the end of the stack new_step.prev = step step = new_step # We've now constructed the dependencies queue such that # later dependencies are on the right, we can now safely peddle # backwards and the last (leftmost) resolved value is the one # we want to return. # for iter_value_expression, resolved_varname in zip(reversed(resolved_values), reversed(resolved_varnames)): # Resolve variable expressions as needed # if len(iter_value_expression) > 1: resolved_value = self._resolve_value_expression(iter_value_expression) iter_value_expression = [resolved_value] if resolved_varname is not None: self._values[resolved_varname] = iter_value_expression return resolved_value # _get_checked_value_expression() # # Fetches a value expression from the value table and raises a user # facing error if the value is undefined. # # Args: # varname (str): The variable name to fetch # referee (str): The variable name referring to `varname`, or None # node (ScalarNode): The ScalarNode for which we need to resolve `name` # # Returns: # (list): The value expression for varname # # Raises: # (LoadError): An appropriate error in case of undefined variables # cdef list _get_checked_value_expression(self, str varname, str referee, ScalarNode node): cdef ProvenanceInformation provenance = None cdef Node referee_node cdef str error_message # # Fetch the value and detect undefined references # try: return self._values[varname] except KeyError as e: # Either the provenance is the toplevel calling provenance, # or it is the provenance of the direct referee referee_node = self._original.get_node(referee, allowed_types=None, allow_none=True) if referee_node is not None: provenance = referee_node.get_provenance() elif node: provenance = node.get_provenance() error_message = "Reference to undefined variable '{}'".format(varname) if provenance: error_message = "{}: {}".format(provenance, error_message) raise LoadError(error_message, LoadErrorReason.UNRESOLVED_VARIABLE) from e # _resolve_value_expression() # # Resolves a value expression with the expectation that all # variables within this value expression have already been # resolved and updated in the Variables._values table. # # This is used as a part of the iterative resolution codepath, # where value expressions are first sorted by dependency before # being resolved in one go. # # Args: # value_expression (list): The value expression to resolve # # Returns: # (str): The resolved value expression # cdef str _resolve_value_expression(self, list value_expression): cdef Py_ssize_t idx cdef object value cdef list acc = [] for idx, value in enumerate(value_expression): if (idx % 2) == 0: acc.append(value) else: acc.append(self._values[value][0]) return "".join(acc) # ResolutionStep() # # The context for a single iteration in variable resolution. # cdef class ResolutionStep: cdef str referee cdef list value_expression cdef ResolutionStep parent cdef ResolutionStep prev # init() # # Initialize this ResolutionStep # # Args: # referee (str): The name of the referring variable # value_expression (list): The parsed value expression to be expanded # parent (ResolutionStep): The parent ResolutionStep # cdef init(self, str referee, list value_expression, ResolutionStep parent): self.referee = referee self.value_expression = value_expression self.parent = parent self.prev = None # check_circular() # # Check for circular references in this step. # # Args: # original_values (MappingNode): The original MappingNode for the Variables # # Raises: # (LoadError): Will raise a user facing LoadError with # LoadErrorReason.CIRCULAR_REFERENCE_VARIABLE in case # circular references were encountered. # cdef check_circular(self, MappingNode original_values): cdef ResolutionStep step = self.parent while step: if self.referee is step.referee: self._raise_circular_reference_error(step, original_values) step = step.parent # _raise_circular_reference_error() # # Helper function to construct a full report and raise the LoadError # with LoadErrorReason.CIRCULAR_REFERENCE_VARIABLE. # # Args: # conflict (ResolutionStep): The resolution step which conflicts with this step # original_values (MappingNode): The original node to extract provenances from # # Raises: # (LoadError): Unconditionally # cdef _raise_circular_reference_error(self, ResolutionStep conflict, MappingNode original_values): cdef list error_lines = [] cdef ResolutionStep step = self cdef ScalarNode node cdef str referee while step is not conflict: if step.parent: referee = step.parent.referee else: referee = self.referee node = original_values.get_scalar(referee) error_lines.append("{}: Variable '{}' refers to variable '{}'".format(node.get_provenance(), referee, step.referee)) step = step.parent raise LoadError("Circular dependency detected on variable '{}'".format(self.referee), LoadErrorReason.CIRCULAR_REFERENCE_VARIABLE, detail="\n".join(reversed(error_lines))) # _parse_value_expression() # # Tries to fetch the parsed value expression from the cache, parsing and # caching value expressions on demand and returns the parsed value expression. # # Args: # value_expression (str): The value expression in string form to parse # # Returns: # (list): The parsed value expression in list form. # cdef list _parse_value_expression(str value_expression): cdef list ret try: return VALUE_EXPRESSION_CACHE[value_expression] except KeyError: # This use of the regex turns a string like "foo %{bar} baz" into # a list ["foo ", "bar", " baz"] # # The result is a parsed value expression, where even indicies # contain literal parts of the value and odd indices contain # variable names which need to be replaced by resolved variables. # splits = VALUE_EXPRESSION_REGEX.split(value_expression) # Optimize later routines by discarding any unnecessary trailing # empty strings. # if splits[-1] == '': del splits[-1] # We intern the string parts to try and reduce the memory impact # of the cache. # ret = [sys.intern( s) for s in splits] # Cache and return the value expression # VALUE_EXPRESSION_CACHE[value_expression] = ret return ret # Iterator for all flatten variables. # Used by Variables.__iter__ cdef class _VariablesIterator: cdef Variables _variables cdef object _iter def __init__(self, Variables variables): self._variables = variables self._iter = iter(variables._values) def __iter__(self): return self def __next__(self): name = next(self._iter) return name, self._variables._expand_var(name) apache-buildstream-27ae392/src/buildstream/_version.py000066400000000000000000000563151514607367700231340ustar00rootroot00000000000000# This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. # Generated by versioneer-0.28 # https://github.com/python-versioneer/python-versioneer # # Downstream edit: This file is from versioneer and presents some linting # errors we are not concerned with, disable linting here. # # pylint: skip-file """Git implementation of _version.py.""" import errno import os import re import subprocess import sys from typing import Callable, Dict import functools def get_keywords(): """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = " (tag: 2.7.0)" git_full = "e4dcda0551f59e3a9e397aaabd620d25b71fc95a" git_date = "2026-02-20 16:03:27 +0100" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: """Container for Versioneer configuration parameters.""" def get_config(): """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "pep440" cfg.tag_prefix = "" cfg.parentdir_prefix = "BuildStream-" cfg.versionfile_source = "src/buildstream/_version.py" cfg.verbose = False return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen( [command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs, ) break except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue if verbose: print("unable to run %s" % dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) return None, process.returncode return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return { "version": dirname[len(parentdir_prefix) :], "full-revisionid": None, "dirty": False, "error": None, "date": None, } rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix) :] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r"\d", r): continue if verbose: print("picking %s" % r) return { "version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date, } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return { "version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None, } @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner( GITS, ["describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*"], cwd=root ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def plus_or_dot(pieces): """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces): """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver): """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces): """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%d" % (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%d" % pieces["distance"] return rendered def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%s" % pieces["short"] return rendered def render_pep440_post_branch(pieces): """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces): """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces): """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: return { "version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None, } if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%s'" % style) return { "version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date"), } def get_versions(): """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which # case we can only use expanded keywords. cfg = get_config() verbose = cfg.verbose try: return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass try: root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. for _ in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: return { "version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to find root of source tree", "date": None, } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) return render(pieces, cfg.style) except NotThisMethod: pass try: if cfg.parentdir_prefix: return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) except NotThisMethod: pass return { "version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None, } apache-buildstream-27ae392/src/buildstream/_versions.py000066400000000000000000000016611514607367700233110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # The base BuildStream artifact version # # The artifact version changes whenever the cache key # calculation algorithm changes in an incompatible way # or if buildstream was changed in a way which can cause # the same cache key to produce something that is no longer # the same. BST_CORE_ARTIFACT_VERSION = 11 apache-buildstream-27ae392/src/buildstream/_workspaces.py000066400000000000000000000417621514607367700236300ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat import os from . import utils from . import _yaml from ._exceptions import LoadError from .exceptions import LoadErrorReason BST_WORKSPACE_FORMAT_VERSION = 4 BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1 WORKSPACE_PROJECT_FILE = ".bstproject.yaml" # WorkspaceProject() # # An object to contain various helper functions and data required for # referring from a workspace back to buildstream. # # Args: # directory (str): The directory that the workspace exists in. # class WorkspaceProject: def __init__(self, directory): self._projects = [] self._directory = directory # get_default_project_path() # # Retrieves the default path to a project. # # Returns: # (str): The path to a project # def get_default_project_path(self): return self._projects[0]["project-path"] # get_default_element() # # Retrieves the name of the element that owns this workspace. # # Returns: # (str): The name of an element # def get_default_element(self): return self._projects[0]["element-name"] # to_dict() # # Turn the members data into a dict for serialization purposes # # Returns: # (dict): A dict representation of the WorkspaceProject # def to_dict(self): ret = { "projects": self._projects, "format-version": BST_WORKSPACE_PROJECT_FORMAT_VERSION, } return ret # from_dict() # # Loads a new WorkspaceProject from a simple dictionary # # Args: # directory (str): The directory that the workspace exists in # dictionary (dict): The dict to generate a WorkspaceProject from # # Returns: # (WorkspaceProject): A newly instantiated WorkspaceProject # @classmethod def from_dict(cls, directory, dictionary): # Only know how to handle one format-version at the moment. format_version = int(dictionary["format-version"]) assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, "Format version {} not found in {}".format( BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary ) workspace_project = cls(directory) for item in dictionary["projects"]: workspace_project.add_project(item["project-path"], item["element-name"]) return workspace_project # load() # # Loads the WorkspaceProject for a given directory. # # Args: # directory (str): The directory # Returns: # (WorkspaceProject): The created WorkspaceProject, if in a workspace, or # (NoneType): None, if the directory is not inside a workspace. # @classmethod def load(cls, directory): workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE) if os.path.exists(workspace_file): data_dict = _yaml.roundtrip_load(workspace_file) return cls.from_dict(directory, data_dict) else: return None # write() # # Writes the WorkspaceProject to disk # def write(self): os.makedirs(self._directory, exist_ok=True) _yaml.roundtrip_dump(self.to_dict(), self.get_filename()) # get_filename() # # Returns the full path to the workspace local project file # def get_filename(self): return os.path.join(self._directory, WORKSPACE_PROJECT_FILE) # add_project() # # Adds an entry containing the project's path and element's name. # # Args: # project_path (str): The path to the project that opened the workspace. # element_name (str): The name of the element that the workspace belongs to. # def add_project(self, project_path, element_name): assert project_path and element_name self._projects.append({"project-path": project_path, "element-name": element_name}) # WorkspaceProjectCache() # # A class to manage workspace project data for multiple workspaces. # class WorkspaceProjectCache: def __init__(self): self._projects = {} # Mapping of a workspace directory to its WorkspaceProject # get() # # Returns a WorkspaceProject for a given directory, retrieving from the cache if # present. # # Args: # directory (str): The directory to search for a WorkspaceProject. # # Returns: # (WorkspaceProject): The WorkspaceProject that was found for that directory. # or (NoneType): None, if no WorkspaceProject can be found. # def get(self, directory): try: workspace_project = self._projects[directory] except KeyError: workspace_project = WorkspaceProject.load(directory) if workspace_project: self._projects[directory] = workspace_project return workspace_project # add() # # Adds the project path and element name to the WorkspaceProject that exists # for that directory # # Args: # directory (str): The directory to search for a WorkspaceProject. # project_path (str): The path to the project that refers to this workspace # element_name (str): The element in the project that was refers to this workspace # # Returns: # (WorkspaceProject): The WorkspaceProject that was found for that directory. # def add(self, directory, project_path, element_name): workspace_project = self.get(directory) if not workspace_project: workspace_project = WorkspaceProject(directory) self._projects[directory] = workspace_project workspace_project.add_project(project_path, element_name) return workspace_project # remove() # # Removes the project path and element name from the WorkspaceProject that exists # for that directory. # # NOTE: This currently just deletes the file, but with support for multiple # projects opening the same workspace, this will involve decreasing the count # and deleting the file if there are no more projects. # # Args: # directory (str): The directory to search for a WorkspaceProject. # def remove(self, directory): workspace_project = self.get(directory) if not workspace_project: raise LoadError( "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE), LoadErrorReason.MISSING_FILE ) path = workspace_project.get_filename() try: os.unlink(path) except FileNotFoundError: pass # Workspace() # # An object to contain various helper functions and data required for # workspaces. # # last_build and path are intended to be public # properties, but may be best accessed using this classes' helper # methods. # # Args: # toplevel_project (Project): Top project. Will be used for resolving relative workspace paths. # path (str): The path that should host this workspace # last_build (str): The key of the last attempted build of this workspace # class Workspace: def __init__(self, toplevel_project, *, last_build=None, path=None): self.last_build = last_build self._path = path self._toplevel_project = toplevel_project self._key = None # to_dict() # # Convert a list of members which get serialized to a dict for serialization purposes # # Returns: # (dict) A dict representation of the workspace # def to_dict(self): ret = {"path": self._path} if self.last_build is not None: ret["last_build"] = self.last_build return ret # from_dict(): # # Loads a new workspace from a simple dictionary, the dictionary # is expected to be generated from Workspace.to_dict(), or manually # when loading from a YAML file. # # Args: # toplevel_project (Project): Top project. Will be used for resolving relative workspace paths. # dictionary: A simple dictionary object # # Returns: # (Workspace): A newly instantiated Workspace # @classmethod def from_dict(cls, toplevel_project, dictionary): # Just pass the dictionary as kwargs return cls(toplevel_project, **dictionary) # differs() # # Checks if two workspaces are different in any way. # # Args: # other (Workspace): Another workspace instance # # Returns: # True if the workspace differs from 'other', otherwise False # def differs(self, other): return self.to_dict() != other.to_dict() # get_absolute_path(): # # Returns: The absolute path of the element's workspace. # def get_absolute_path(self): return os.path.join(self._toplevel_project.directory, self._path) # Workspaces() # # A class to manage Workspaces for multiple elements. # # Args: # toplevel_project (Project): Top project used to resolve paths. # workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects # class Workspaces: def __init__(self, toplevel_project, workspace_project_cache): self._toplevel_project = toplevel_project self._workspace_project_cache = workspace_project_cache # A project without a directory can happen if toplevel_project.directory: self._bst_directory = os.path.join(toplevel_project.directory, ".bst2") self._workspaces = self._load_config() else: self._bst_directory = None self._workspaces = {} # list() # # Generator function to enumerate workspaces. # # Yields: # A tuple in the following format: (str, Workspace), where the # first element is the name of the workspaced element. def list(self): for element in self._workspaces.keys(): yield (element, self._workspaces[element]) # create_workspace() # # Create a workspace in the given path for the given element, and potentially # checks-out the target into it. # # Args: # target (Element) - The element to create a workspace for # path (str) - The path in which the workspace should be kept # checkout (bool): Whether to check-out the element's sources into the directory # def create_workspace(self, target, path, *, checkout): element_name = target._get_full_name() project_dir = self._toplevel_project.directory if path.startswith(project_dir): workspace_path = os.path.relpath(path, project_dir) else: workspace_path = path self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path) if checkout: with target.timed_activity("Staging sources to {}".format(path)): target._open_workspace() workspace_project = self._workspace_project_cache.add(path, project_dir, element_name) project_file_path = workspace_project.get_filename() if os.path.exists(project_file_path): target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE)) workspace_project.write() self.save_config() # get_workspace() # # Get the path of the workspace source associated with the given # element's source at the given index # # Args: # element_name (str) - The element name whose workspace to return # # Returns: # (None|Workspace) # def get_workspace(self, element_name): if element_name not in self._workspaces: return None return self._workspaces[element_name] # update_workspace() # # Update the datamodel with a new Workspace instance # # Args: # element_name (str): The name of the element to update a workspace for # workspace_dict (Workspace): A serialized workspace dictionary # # Returns: # (bool): Whether the workspace has changed as a result # def update_workspace(self, element_name, workspace_dict): assert element_name in self._workspaces workspace = Workspace.from_dict(self._toplevel_project, workspace_dict) if self._workspaces[element_name].differs(workspace): self._workspaces[element_name] = workspace return True return False # delete_workspace() # # Remove the workspace from the workspace element. Note that this # does *not* remove the workspace from the stored yaml # configuration, call save_config() afterwards. # # Args: # element_name (str) - The element name whose workspace to delete # def delete_workspace(self, element_name): workspace = self.get_workspace(element_name) del self._workspaces[element_name] # Remove from the cache if it exists try: self._workspace_project_cache.remove(workspace.get_absolute_path()) except LoadError as e: # We might be closing a workspace with a deleted directory if e.reason == LoadErrorReason.MISSING_FILE: pass else: raise # save_config() # # Dump the current workspace element to the project configuration # file. This makes any changes performed with delete_workspace or # create_workspace permanent # def save_config(self): assert utils._is_in_main_thread() config = { "format-version": BST_WORKSPACE_FORMAT_VERSION, "workspaces": {element: workspace.to_dict() for element, workspace in self._workspaces.items()}, } os.makedirs(self._bst_directory, exist_ok=True) _yaml.roundtrip_dump(config, self._get_filename()) # _load_config() # # Loads and parses the workspace configuration # # Returns: # (dict) The extracted workspaces # # Raises: LoadError if there was a problem with the workspace config # def _load_config(self): workspace_file = self._get_filename() try: node = _yaml.load(workspace_file, shortname="workspaces.yml") except LoadError as e: if e.reason == LoadErrorReason.MISSING_FILE: # Return an empty dict if there was no workspace file return {} raise return self._parse_workspace_config(node) # _parse_workspace_config_format() # # If workspace config is in old-style format, i.e. it is using # source-specific workspaces, try to convert it to element-specific # workspaces. # # Args: # workspaces (dict): current workspace config, usually output of _load_workspace_config() # # Returns: # (dict) The extracted workspaces # # Raises: LoadError if there was a problem with the workspace config # def _parse_workspace_config(self, workspaces): try: version = workspaces.get_int("format-version", default=0) except ValueError: raise LoadError( "Format version is not an integer in workspace configuration", LoadErrorReason.INVALID_DATA ) if version < 4: # bst 1.x workspaces do not separate source and build files. raise LoadError( "Workspace configuration format version {} not supported." "Please recreate this workspace.".format(version), LoadErrorReason.INVALID_DATA, ) if 4 <= version <= BST_WORKSPACE_FORMAT_VERSION: workspaces = workspaces.get_mapping("workspaces", default={}) res = {element: self._load_workspace(node) for element, node in workspaces.items()} else: raise LoadError( "Workspace configuration format version {} not supported." "Your version of buildstream may be too old. Max supported version: {}".format( version, BST_WORKSPACE_FORMAT_VERSION ), LoadErrorReason.INVALID_DATA, ) return res # _load_workspace(): # # Loads a new workspace from a YAML node # # Args: # node: A YAML dict # # Returns: # (Workspace): A newly instantiated Workspace # def _load_workspace(self, node): dictionary = { "path": node.get_str("path"), "last_build": node.get_str("last_build", default=None), } return Workspace.from_dict(self._toplevel_project, dictionary) # _get_filename(): # # Get the workspaces.yml file path. # # Returns: # (str): The path to workspaces.yml file. def _get_filename(self): return os.path.join(self._bst_directory, "workspaces.yml") apache-buildstream-27ae392/src/buildstream/_yaml.pyi000066400000000000000000000013401514607367700225460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import Optional from .node import MappingNode def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ... apache-buildstream-27ae392/src/buildstream/_yaml.pyx000066400000000000000000000462041514607367700225750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Daniel Silverstone # James Ennis # Benjamin Schubert import datetime import sys from io import StringIO from contextlib import ExitStack from collections import OrderedDict from collections.abc import Mapping from ruamel import yaml from ._exceptions import LoadError from .exceptions import LoadErrorReason from . cimport node from .node cimport MappingNode, ScalarNode, SequenceNode # These exceptions are intended to be caught entirely within # the BuildStream framework, hence they do not reside in the # public exceptions.py class YAMLLoadError(Exception): pass # Represents the various states in which the Representer can be # while parsing yaml. cdef enum RepresenterState: doc init stream wait_key wait_list_item wait_value ctypedef RepresenterState (*representer_action)(Representer, object) # Representer for YAML events comprising input to the BuildStream format. # # All streams MUST represent a single document which must be a Mapping. # Anything else is considered an error. # # Mappings must only have string keys, values are always represented as # strings if they are scalar, or else as simple dictionaries and lists. # cdef class Representer: cdef int _file_index cdef RepresenterState state cdef list output, keys # Initialise a new representer # # The file index is used to store into the Node instances so that the # provenance of the YAML can be tracked. # # Args: # file_index (int): The index of this YAML file def __init__(self, int file_index): self._file_index = file_index self.state = RepresenterState.init self.output = [] self.keys = [] # Handle a YAML parse event # # Args: # event (YAML Event): The event to be handled # # Raises: # YAMLLoadError: Something went wrong. cdef void handle_event(self, event) except *: if getattr(event, "anchor", None) is not None: raise YAMLLoadError("Anchors are disallowed in BuildStream at line {} column {}" .format(event.start_mark.line, event.start_mark.column)) cdef str event_name = event.__class__.__name__ if event_name == "ScalarEvent": if event.tag is not None: if not event.tag.startswith("tag:yaml.org,2002:"): raise YAMLLoadError( "Non-core tag expressed in input. " + "This is disallowed in BuildStream. At line {} column {}" .format(event.start_mark.line, event.start_mark.column)) cdef representer_action handler = self._get_handler_for_event(event_name) if not handler: raise YAMLLoadError( "Invalid input detected. No handler for {} in state {} at line {} column {}" .format(event, self.state, event.start_mark.line, event.start_mark.column)) # Cython weirdness here, we need to pass self to the function self.state = handler(self, event) # pylint: disable=not-callable # Get the output of the YAML parse # # Returns: # (Node or None): Return the Node instance of the top level mapping or # None if there wasn't one. cdef MappingNode get_output(self): if len(self.output): return self.output[0] return None cdef representer_action _get_handler_for_event(self, str event_name): if self.state == RepresenterState.wait_list_item: if event_name == "ScalarEvent": return self._handle_wait_list_item_ScalarEvent elif event_name == "MappingStartEvent": return self._handle_wait_list_item_MappingStartEvent elif event_name == "SequenceStartEvent": return self._handle_wait_list_item_SequenceStartEvent elif event_name == "SequenceEndEvent": return self._handle_wait_list_item_SequenceEndEvent elif self.state == RepresenterState.wait_value: if event_name == "ScalarEvent": return self._handle_wait_value_ScalarEvent elif event_name == "MappingStartEvent": return self._handle_wait_value_MappingStartEvent elif event_name == "SequenceStartEvent": return self._handle_wait_value_SequenceStartEvent elif self.state == RepresenterState.wait_key: if event_name == "ScalarEvent": return self._handle_wait_key_ScalarEvent elif event_name == "MappingEndEvent": return self._handle_wait_key_MappingEndEvent elif self.state == RepresenterState.stream: if event_name == "DocumentStartEvent": return self._handle_stream_DocumentStartEvent elif event_name == "StreamEndEvent": return self._handle_stream_StreamEndEvent elif self.state == RepresenterState.doc: if event_name == "MappingStartEvent": return self._handle_doc_MappingStartEvent elif event_name == "DocumentEndEvent": return self._handle_doc_DocumentEndEvent elif self.state == RepresenterState.init and event_name == "StreamStartEvent": return self._handle_init_StreamStartEvent return NULL cdef RepresenterState _handle_init_StreamStartEvent(self, object ev): return RepresenterState.stream cdef RepresenterState _handle_stream_DocumentStartEvent(self, object ev): return RepresenterState.doc cdef RepresenterState _handle_doc_MappingStartEvent(self, object ev): newmap = MappingNode.__new__(MappingNode, self._file_index, ev.start_mark.line, ev.start_mark.column, {}) self.output.append(newmap) return RepresenterState.wait_key cdef RepresenterState _handle_wait_key_ScalarEvent(self, object ev): if ev.value in self.output[-1]: raise YAMLLoadError(f"Duplicate key {ev.value} at line {ev.start_mark.line} column {ev.start_mark.column}") self.keys.append(ev.value) return RepresenterState.wait_value cdef RepresenterState _handle_wait_value_ScalarEvent(self, object ev): key = self.keys.pop() ( self.output[-1]).value[key] = \ ScalarNode.__new__(ScalarNode, self._file_index, ev.start_mark.line, ev.start_mark.column, ev.value) return RepresenterState.wait_key cdef RepresenterState _handle_wait_value_MappingStartEvent(self, object ev): cdef RepresenterState new_state = self._handle_doc_MappingStartEvent(ev) key = self.keys.pop() ( self.output[-2]).value[key] = self.output[-1] return new_state cdef RepresenterState _handle_wait_key_MappingEndEvent(self, object ev): # We've finished a mapping, so pop it off the output stack # unless it's the last one in which case we leave it if len(self.output) > 1: self.output.pop() if type(self.output[-1]) is SequenceNode: return RepresenterState.wait_list_item else: return RepresenterState.wait_key else: return RepresenterState.doc cdef RepresenterState _handle_wait_value_SequenceStartEvent(self, object ev): self.output.append(SequenceNode.__new__( SequenceNode, self._file_index, ev.start_mark.line, ev.start_mark.column, [])) ( self.output[-2]).value[self.keys[-1]] = self.output[-1] return RepresenterState.wait_list_item cdef RepresenterState _handle_wait_list_item_SequenceStartEvent(self, object ev): self.keys.append(len(( self.output[-1]).value)) self.output.append(SequenceNode.__new__( SequenceNode, self._file_index, ev.start_mark.line, ev.start_mark.column, [])) ( self.output[-2]).value.append(self.output[-1]) return RepresenterState.wait_list_item cdef RepresenterState _handle_wait_list_item_SequenceEndEvent(self, object ev): # When ending a sequence, we need to pop a key because we retain the # key until the end so that if we need to mutate the underlying entry # we can. key = self.keys.pop() self.output.pop() if type(key) is int: return RepresenterState.wait_list_item else: return RepresenterState.wait_key cdef RepresenterState _handle_wait_list_item_ScalarEvent(self, object ev): ( self.output[-1]).value.append( ScalarNode.__new__(ScalarNode, self._file_index, ev.start_mark.line, ev.start_mark.column, ev.value)) return RepresenterState.wait_list_item cdef RepresenterState _handle_wait_list_item_MappingStartEvent(self, object ev): cdef RepresenterState new_state = self._handle_doc_MappingStartEvent(ev) ( self.output[-2]).value.append(self.output[-1]) return new_state cdef RepresenterState _handle_doc_DocumentEndEvent(self, object ev): if len(self.output) != 1: raise YAMLLoadError("Zero, or more than one document found in YAML stream") return RepresenterState.stream cdef RepresenterState _handle_stream_StreamEndEvent(self, object ev): return RepresenterState.init # Loads a dictionary from some YAML # # Args: # filename (str): The YAML file to load # shortname (str): The filename in shorthand for error reporting (or None) # copy_tree (bool): Whether to make a copy, preserving the original toplevels # for later serialization # project (Project): The (optional) project to associate the parsed YAML with # # Returns (dict): A loaded copy of the YAML file with provenance information # # Raises: LoadError # cpdef MappingNode load(str filename, str shortname, bint copy_tree=False, object project=None): cdef MappingNode data if not shortname: shortname = filename cdef str displayname if (project is not None) and (project.junction is not None): displayname = "{}:{}".format(project.junction.name, shortname) else: displayname = shortname cdef Py_ssize_t file_number = node._create_new_file(filename, shortname, displayname, project) try: with open(filename) as f: contents = f.read() data = load_data(contents, file_index=file_number, file_name=filename, copy_tree=copy_tree) return data except FileNotFoundError as e: raise LoadError("Could not find file at {}".format(filename), LoadErrorReason.MISSING_FILE) from e except IsADirectoryError as e: raise LoadError("{} is a directory".format(filename), LoadErrorReason.LOADING_DIRECTORY) from e except LoadError as e: raise LoadError("{}: {}".format(displayname, e), e.reason) from e # Like load(), but doesnt require the data to be in a file # cpdef MappingNode load_data(str data, int file_index=node._SYNTHETIC_FILE_INDEX, str file_name=None, bint copy_tree=False): cdef Representer rep try: rep = Representer(file_index) parser = yaml.CParser(data) try: while parser.check_event(): rep.handle_event(parser.get_event()) finally: parser.dispose() contents = rep.get_output() except YAMLLoadError as e: raise LoadError("Malformed YAML:\n\n{}\n\n".format(e), LoadErrorReason.INVALID_YAML) from e except Exception as e: raise LoadError("Severely malformed YAML:\n\n{}\n\n".format(e), LoadErrorReason.INVALID_YAML) from e if type(contents) != MappingNode: # Special case allowance for None, when the loaded file has only comments in it. if contents is None: contents = MappingNode.__new__(MappingNode, file_index, 0, 0, {}) else: raise LoadError("YAML file has content of type '{}' instead of expected type 'dict': {}" .format(type(contents[0]).__name__, file_name), LoadErrorReason.INVALID_YAML) # Store this away because we'll use it later for "top level" provenance node._set_root_node_for_file(file_index, contents) if copy_tree: contents = contents.clone() return contents ############################################################################### # Roundtrip code # Represent Nodes automatically def represent_mapping(self, MappingNode mapping): return self.represent_dict(mapping.value) def represent_scalar(self, ScalarNode scalar): # We load None values as strings, and also save them as strings if scalar.value is None: return self.represent_str("") return self.represent_str(scalar.value) def represent_sequence(self, SequenceNode sequence): return self.represent_list(sequence.value) yaml.RoundTripRepresenter.add_representer(MappingNode, represent_mapping) yaml.RoundTripRepresenter.add_representer(ScalarNode, represent_scalar) yaml.RoundTripRepresenter.add_representer(SequenceNode, represent_sequence) # Represent simple types as strings def represent_as_str(self, value): return self.represent_str(str(value)) yaml.RoundTripRepresenter.add_representer(type(None), represent_as_str) yaml.RoundTripRepresenter.add_representer(int, represent_as_str) yaml.RoundTripRepresenter.add_representer(float, represent_as_str) yaml.RoundTripRepresenter.add_representer(bool, represent_as_str) yaml.RoundTripRepresenter.add_representer(datetime.datetime, represent_as_str) yaml.RoundTripRepresenter.add_representer(datetime.date, represent_as_str) # Always represent things consistently: yaml.RoundTripRepresenter.add_representer(OrderedDict, yaml.SafeRepresenter.represent_dict) # Always parse things consistently yaml.RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:int', yaml.RoundTripConstructor.construct_yaml_str) yaml.RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:float', yaml.RoundTripConstructor.construct_yaml_str) yaml.RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:bool', yaml.RoundTripConstructor.construct_yaml_str) yaml.RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:null', yaml.RoundTripConstructor.construct_yaml_str) yaml.RoundTripConstructor.add_constructor(u'tag:yaml.org,2002:timestamp', yaml.RoundTripConstructor.construct_yaml_str) # This is a dumper used during roundtrip_dump which forces every scalar to be # a plain string, in order to match the output format to the input format. # # If you discover something is broken, please add a test case to the roundtrip # test in tests/internals/yaml/roundtrip-test.yaml def prepare_roundtrip_yaml(): yml = yaml.YAML() yml.preserve_quotes=True # defaults to 80 if we don't set it yml.width=9999 # For each of YAML 1.1 and 1.2, force everything to be a plain string for version in [(1, 1), (1, 2), None]: yml.resolver.add_version_implicit_resolver( version, u'tag:yaml.org,2002:str', yaml.util.RegExp(r'.*'), None) return yml # roundtrip_load() # # Load a YAML file into memory in a form which allows roundtripping as best # as ruamel permits. # # Note, the returned objects can be treated as Mappings and Lists and Strings # but replacing content wholesale with plain dicts and lists may result # in a loss of comments and formatting. # # Args: # filename (str): The file to load in # allow_missing (bool): Optionally set this to True to allow missing files # # Returns: # (Mapping): The loaded YAML mapping. # # Raises: # (LoadError): If the file is missing, or a directory, this is raised. # Also if the YAML is malformed. # def roundtrip_load(filename, *, allow_missing=False): yml = prepare_roundtrip_yaml() try: with open(filename, "r") as fh: try: contents = yml.load(fh) except (yaml.scanner.ScannerError, yaml.composer.ComposerError, yaml.parser.ParserError) as e: raise LoadError("Malformed YAML:\n\n{}\n\n{}\n".format(e.problem, e.problem_mark), LoadErrorReason.INVALID_YAML) from e # Special case empty files at this point if contents is None: # We'll make them empty mappings like the main Node loader contents = {} if not isinstance(contents, Mapping): raise LoadError("YAML file has content of type '{}' instead of expected type 'dict': {}" .format(type(contents).__name__, filename), LoadErrorReason.INVALID_YAML) except FileNotFoundError as e: if allow_missing: # Missing files are always empty dictionaries return {} else: raise LoadError("Could not find file at {}".format(filename), LoadErrorReason.MISSING_FILE) from e except IsADirectoryError as e: raise LoadError("{} is a directory.".format(filename), LoadErrorReason.LOADING_DIRECTORY) from e return contents # roundtrip_dump() # # Dumps the given contents as a YAML file. Ideally the contents came from # parsing with `roundtrip_load` or `roundtrip_load_data` so that they will be # dumped in the same form as they came from. # # If `file` is a string, it is the filename to write to, if `file` has a # `write` method, it's treated as a stream, otherwise output is to stdout. # # Args: # contents (Mapping or list): The content to write out as YAML. # file (any): The file to write to # def roundtrip_dump(contents, file=None): yml = prepare_roundtrip_yaml() with ExitStack() as stack: if type(file) is str: from . import utils f = stack.enter_context(utils.save_file_atomic(file, 'w')) elif hasattr(file, 'write'): f = file else: f = sys.stdout yml.dump(contents, f) # roundtrip_dump_string() # # Helper to call roundtrip_dump() but get the content in a string. # # Args: # contents (Mapping or list): The content to write out as YAML. # # Returns: # (str): The generated string # def roundtrip_dump_string(node): with StringIO() as f: roundtrip_dump(node, f) return f.getvalue() apache-buildstream-27ae392/src/buildstream/buildelement.py000066400000000000000000000420051514607367700237500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ BuildElement - Abstract class for build elements ================================================ The BuildElement class is a convenience element one can derive from for implementing the most common case of element. .. _core_buildelement_builtins: Built-in functionality ---------------------- The BuildElement base class provides built in functionality that could be overridden by the individual plugins. This section will give a brief summary of how some of the common features work, some of them or the variables they use will be further detailed in the following sections. The `strip-binaries` variable ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The `strip-binaries` variable is by default **empty**. You need to use the appropiate commands depending of the system you are building. If you are targetting Linux, ones known to work are the ones used by the `freedesktop-sdk `_, you can take a look to them in their `project.conf `_ Location for staging dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The BuildElement supports the "location" :term:`dependency configuration `, which means you can use this configuration for any BuildElement class. The "location" configuration defines where the dependency will be staged in the build sandbox. **Example:** Here is an example of how one might stage some dependencies into an alternative location while staging some elements in the sandbox root. .. code:: yaml # Stage these build dependencies in /opt # build-depends: - baseproject.bst:opt-dependencies.bst config: location: /opt # Stage these tools in "/" and require them as # runtime dependencies. depends: - baseproject.bst:base-tools.bst .. note:: The order of dependencies specified is not significant. The staging locations will be sorted so that elements are staged in parent directories before subdirectories. `digest-environment` for dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The BuildElement supports the ``digest-environment`` :term:`dependency configuration `, which sets the specified environment variable in the build sandbox to the CAS digest corresponding to a directory that contains all dependencies that are configured with the same ``digest-environment``. This is useful for REAPI clients in the sandbox such as `recc `_, see ``remote-apis-socket`` in the :ref:`sandbox configuration `. **Example:** Here is an example of how to set the environment variable `GCC_DIGEST` to the CAS digest of a directory that contains ``gcc.bst`` and its runtime dependencies. The ``libpony.bst`` dependency will not be included in that CAS directory. .. code:: yaml build-depends: - baseproject.bst:gcc.bst config: digest-environment: GCC_DIGEST - libpony.bst Location for running commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``command-subdir`` variable sets where commands will be executed, and the directory will be created automatically if it does not exist. The ``command-subdir`` is a relative path from ``%{build-root}``, and cannot be a parent or adjacent directory, it must expand to a subdirectory of ``${build-root}``. Location for configuring the project ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The ``conf-root`` is the location that specific build elements can use to look for build configuration files. This is used by elements such as `autotools `_, `cmake `_, `meson `_, `setuptools `_ and `pip `_. The default value of ``conf-root`` is defined by default as ``.``. This means that if the ``conf-root`` is not explicitly set to another directory, the configuration files are expected to be found in ``command-subdir``. Separating source and build directories ''''''''''''''''''''''''''''''''''''''' A typical example of using ``conf-root`` is when performing `autotools `_ builds where your source directory is separate from your build directory. This can be achieved in build elements which use ``conf-root`` as follows: .. code:: yaml variables: # Specify that build configuration scripts are found in %{build-root} conf-root: "%{build-root}" # The build will run in the `_build` subdirectory command-subdir: _build Install Location ~~~~~~~~~~~~~~~~ Build elements must install the build output to the directory defined by ``install-root``. You need not set or change the ``install-root`` variable as it will be defined automatically on your behalf, and it is used to collect build output when creating the resulting artifacts. It is important to know about ``install-root`` in order to write your own custom install instructions, for example the `cmake `_ element will use it to specify the ``DESTDIR``. Abstract method implementations ------------------------------- Element.configure_sandbox() ~~~~~~~~~~~~~~~~~~~~~~~~~~~ In :func:`Element.configure_sandbox() `, the BuildElement will ensure that the sandbox locations described by the ``%{build-root}`` and ``%{install-root}`` variables are marked and will be mounted read-write for the :func:`assemble phase`. The working directory for the sandbox will be configured to be the ``%{build-root}``, unless the ``%{command-subdir}`` variable is specified for the element in question, in which case the working directory will be configured as ``%{build-root}/%{command-subdir}``. Element.stage() ~~~~~~~~~~~~~~~ In :func:`Element.stage() `, the BuildElement will do the following operations: * Stage all of the build dependencies into the sandbox root. * Run the integration commands for all staged dependencies using :func:`Element.integrate() ` * Stage any Source on the given element to the ``%{build-root}`` location inside the sandbox, using :func:`Element.stage_sources() ` Element.assemble() ~~~~~~~~~~~~~~~~~~ In :func:`Element.assemble() `, the BuildElement will proceed to run sandboxed commands which are expected to be found in the element configuration. Commands are run in the following order: * ``configure-commands``: Commands to configure the build scripts * ``build-commands``: Commands to build the element * ``install-commands``: Commands to install the results into ``%{install-root}`` * ``strip-commands``: Commands to strip debugging symbols installed binaries The result of the build is expected to end up in ``%{install-root}``, and as such; Element.assemble() method will return the ``%{install-root}`` for artifact collection purposes. .. note:: In the case that the element is currently workspaced, the ``configure-commands`` will only be run in subsequent builds until they succeed at least once, unless :ref:`bst workspace reset --soft ` is called on the workspace to explicitly avoid an incremental build. """ import os from .element import Element _command_steps = ["configure-commands", "build-commands", "install-commands", "strip-commands"] class BuildElement(Element): ############################################################# # Abstract Method Implementations # ############################################################# def configure(self, node): self.__commands = {} # pylint: disable=attribute-defined-outside-init # FIXME: Currently this forcefully validates configurations # for all BuildElement subclasses so they are unable to # extend the configuration node.validate_keys(_command_steps) self._command_subdir = self.get_variable("command-subdir") # pylint: disable=attribute-defined-outside-init for command_name in _command_steps: self.__commands[command_name] = node.get_str_list(command_name, []) def configure_dependencies(self, dependencies): self.__layout = {} # pylint: disable=attribute-defined-outside-init self.__digest_environment = {} # pylint: disable=attribute-defined-outside-init # FIXME: Currently this forcefully validates configurations # for all BuildElement subclasses so they are unable to # extend the configuration for dep in dependencies: # Determine the location to stage each element, default is "/" location = "/" if dep.config: dep.config.validate_keys(["digest-environment", "location"]) location = dep.config.get_str("location", "/") digest_var_name = dep.config.get_str("digest-environment", None) if digest_var_name is not None: element_list = self.__digest_environment.setdefault(digest_var_name, []) element_list.append((dep.element, dep.path)) try: element_list = self.__layout[location] except KeyError: element_list = [] self.__layout[location] = element_list element_list.append((dep.element, dep.path)) def preflight(self): pass def get_unique_key(self): dictionary = {} for command_name, command_list in self.__commands.items(): dictionary[command_name] = command_list if self._command_subdir: dictionary["command-subdir"] = self._command_subdir # Specifying notparallel for a given element effects the # cache key, while having the side effect of setting max-jobs to 1, # which is normally automatically resolved and does not affect # the cache key. if self.get_variable("notparallel"): dictionary["notparallel"] = True # Specify the layout in the key, if any of the elements are not going to # be staged in "/" # if any(location for location in self.__layout if location != "/"): sorted_locations = sorted(self.__layout) layout_key = { location: [dependency_path for _, dependency_path in self.__layout[location]] for location in sorted_locations } dictionary["layout"] = layout_key # Specify the layout in the key, if buildstream is to generate an environment # variable with the digest # if self.__digest_environment: sorted_envs = sorted(self.__digest_environment) digest_key = { env: [dependency_path for _, dependency_path in self.__digest_environment[env]] for env in sorted_envs } dictionary["digest-enviornment"] = digest_key return dictionary def configure_sandbox(self, sandbox): build_root = self.get_variable("build-root") install_root = self.get_variable("install-root") # Tell the sandbox to mount the build root and install root sandbox.mark_directory(build_root) sandbox.mark_directory(install_root) # Allow running all commands in a specified subdirectory if self._command_subdir: command_dir = os.path.join(build_root, self._command_subdir) else: command_dir = build_root sandbox.set_work_directory(command_dir) # Setup environment env = self.get_environment() # Add "CAS digest" environment variables sorted_envs = sorted(self.__digest_environment) for digest_variable in sorted_envs: element_list = [element for element, _ in self.__digest_environment[digest_variable]] with self.timed_activity( f"Staging dependencies for '{digest_variable}' in subsandbox", silent_nested=True ), self.subsandbox(sandbox) as subsandbox: self.stage_dependency_artifacts(subsandbox, element_list) digest = subsandbox.get_virtual_directory()._get_digest() env[digest_variable] = "{}/{}".format(digest.hash, digest.size_bytes) sandbox.set_environment(env) def stage(self, sandbox): # First stage it all # sorted_locations = sorted(self.__layout) for location in sorted_locations: with self.timed_activity("Staging dependencies at: {}".format(location), silent_nested=True): element_list = [element for element, _ in self.__layout[location]] self.stage_dependency_artifacts(sandbox, element_list, path=location) # Now integrate any elements staged in the root # root_list = self.__layout.get("/", None) if root_list: element_list = [element for element, _ in root_list] with self.timed_activity("Integrating sandbox", silent_nested=True), sandbox.batch(): for dep in self.dependencies(element_list): dep.integrate(sandbox) # Stage sources in the build root self.stage_sources(sandbox, self.get_variable("build-root")) def assemble(self, sandbox): with sandbox.batch(root_read_only=True, label="Running commands"): # We need to ensure that configure-commands are only called # once in workspaces, because the changes will persist across # incremental builds - not desirable, for example, in the case # of autotools, we don't want to run `./configure` a second time # in an incremental build if it has succeeded at least once. # # Here we use an empty file `.bst-prepared` as a marker of whether # configure-commands have already completed successfully in a previous build. # needs_configure = True marker_filename = ".bst-prepared" commands = self.__commands["configure-commands"] if commands: if self._get_workspace(): vdir = sandbox.get_virtual_directory() buildroot = self.get_variable("build-root") buildroot_vdir = vdir.open_directory(buildroot.lstrip(os.sep)) # Marker found, no need to configure if buildroot_vdir.exists(marker_filename): needs_configure = False if needs_configure: for cmd in commands: self.__run_command(sandbox, cmd) # This will serialize a command to create the marker file # in the sandbox batch after running configure if self._get_workspace(): sandbox._create_empty_file(marker_filename) # Run commands for command_name in _command_steps: commands = self.__commands[command_name] if not commands or command_name == "configure-commands": continue for cmd in commands: self.__run_command(sandbox, cmd) # Empty the build directory after a successful build to avoid the # overhead of capturing the build directory. self.run_cleanup_commands(sandbox) # Return the payload, this is configurable but is generally # always the /buildstream-install directory return self.get_variable("install-root") def generate_script(self): script = "" for command_name in _command_steps: commands = self.__commands[command_name] for cmd in commands: script += "(set -ex; {}\n) || exit 1\n".format(cmd) return script ############################################################# # Private Local Methods # ############################################################# def __run_command(self, sandbox, cmd): # Note the -e switch to 'sh' means to exit with an error # if any untested command fails. # sandbox.run(["sh", "-c", "-e", cmd + "\n"], root_read_only=True, label=cmd) apache-buildstream-27ae392/src/buildstream/data/000077500000000000000000000000001514607367700216355ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/data/bst000066400000000000000000000022441514607367700223520ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # BuildStream bash completion scriptlet. # # On systems which use the bash-completion module for # completion discovery with bash, this can be installed at: # # pkg-config --variable=completionsdir bash-completion # # If BuildStream is not installed system wide, you can # simply source this script to enable completions or append # this script to your ~/.bash_completion file. # _bst_completion() { local IFS=$' ' COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ COMP_CWORD=$COMP_CWORD \ _BST_COMPLETION=complete $1 ) ) return 0 } complete -F _bst_completion -o nospace bst; apache-buildstream-27ae392/src/buildstream/data/build-all.sh.in000066400000000000000000000023711514607367700244460ustar00rootroot00000000000000#!/bin/sh # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # DO NOT EDIT THIS FILE # # This is a build script generated by BuildStream # # Builds all given modules using their respective scripts. set -eu echo "BuildStream native bootstrap script" export PATH='/usr/bin:/usr/sbin/:/sbin:/bin:/tools/bin:/tools/sbin' export SRCDIR='./source' SUCCESS=false CURRENT_MODULE='None' echo 'Setting up build environment...' except() {{ if [ "$SUCCESS" = true ]; then echo "Done!" else echo "Error building module ${{CURRENT_MODULE}}." fi }} trap "except" EXIT for module in {modules}; do CURRENT_MODULE="$module" "$SRCDIR/build-$module" if [ -e /sbin/ldconfig ]; then /sbin/ldconfig || true; fi done SUCCESS=true apache-buildstream-27ae392/src/buildstream/data/build-module.sh.in000066400000000000000000000022751514607367700251660ustar00rootroot00000000000000#!/bin/sh # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # DO NOT EDIT THIS FILE # # This is a build script generated by BuildStream # # Builds the module {name}. set -e # Prepare the build environment echo 'Building {name}' if [ -d '{build_root}' ]; then rm -rf '{build_root}' fi if [ -d '{install_root}' ]; then rm -rf '{install_root}' fi mkdir -p '{build_root}' mkdir -p '{install_root}' if [ -d "$SRCDIR/{name}/" ]; then cp -a "$SRCDIR/{name}/." '{build_root}' fi cd '{build_root}' export PREFIX='{install_root}' export {variables} # Build the module {commands} rm -rf '{build_root}' # Install the module echo 'Installing {name}' (cd '{install_root}'; find . | cpio -umdp /) apache-buildstream-27ae392/src/buildstream/data/projectconfig.yaml000066400000000000000000000103601514607367700253550ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Default BuildStream project configuration. # General configuration defaults # # Elements are found at the project root element-path: . # Store source references in element files ref-storage: inline # Variable Configuration # variables: # Path configuration, to be used in build instructions. prefix: "/usr" exec_prefix: "%{prefix}" bindir: "%{exec_prefix}/bin" sbindir: "%{exec_prefix}/sbin" libexecdir: "%{exec_prefix}/libexec" datadir: "%{prefix}/share" sysconfdir: "/etc" sharedstatedir: "%{prefix}/com" localstatedir: "/var" lib: "lib" libdir: "%{prefix}/%{lib}" debugdir: "%{libdir}/debug" includedir: "%{prefix}/include" docdir: "%{datadir}/doc" infodir: "%{datadir}/info" mandir: "%{datadir}/man" # Indicates the default build directory where input is # normally staged build-root: /buildstream/%{project-name}/%{element-name} # Indicates where the build system should look for configuration files conf-root: . # Indicates the build installation directory in the sandbox install-root: /buildstream-install # You need to override this with the commands specific for your system strip-binaries: "" # Base sandbox environment, can be overridden by plugins environment: PATH: /usr/bin:/bin:/usr/sbin:/sbin SHELL: /bin/sh TERM: dumb USER: tomjon USERNAME: tomjon LOGNAME: tomjon LC_ALL: C HOME: /tmp TZ: UTC # For reproducible builds we use 2011-11-11 11:11:11 UTC as a constant SOURCE_DATE_EPOCH: 1321009871 # List of environment variables which should not be taken into # account when calculating a cache key for a given element. # environment-nocache: [] # Configuration for the sandbox other than environment variables # should go in 'sandbox'. sandbox: {} # Defaults for the 'split-rules' public data found on elements # in the 'bst' domain. # split-rules: # The runtime domain includes whatever is needed for the # built element to run, this includes stripped executables # and shared libraries by default. runtime: - | %{bindir} - | %{bindir}/* - | %{sbindir} - | %{sbindir}/* - | %{libexecdir} - | %{libexecdir}/* - | %{libdir}/lib*.so* # The devel domain includes additional things which # you may need for development. # # By default this includes header files, static libraries # and other metadata such as pkgconfig files, m4 macros and # libtool archives. devel: - | %{includedir} - | %{includedir}/** - | %{libdir}/lib*.a - | %{libdir}/lib*.la - | %{libdir}/pkgconfig/*.pc - | %{datadir}/pkgconfig/*.pc - | %{datadir}/aclocal/*.m4 # The debug domain includes debugging information stripped # away from libraries and executables debug: - | %{debugdir} - | %{debugdir}/** # The doc domain includes documentation doc: - | %{docdir} - | %{docdir}/** - | %{infodir} - | %{infodir}/** - | %{mandir} - | %{mandir}/** # The locale domain includes translations etc locale: - | %{datadir}/locale - | %{datadir}/locale/** - | %{datadir}/i18n - | %{datadir}/i18n/** - | %{datadir}/zoneinfo - | %{datadir}/zoneinfo/** # Default behavior for `bst shell` # shell: # Command to run when `bst shell` does not provide a command # command: [ 'sh', '-i' ] # Define the set of fields accepted in `provenance` dictionaries of sources. # source-provenance-attributes: homepage: "The project homepage URL" issue-tracker: "The project's issue tracking URL" # Defaults for bst commands # defaults: # Set default target elements to use when none are passed on the command line. # If none are configured in the project, default to all project elements. targets: [] apache-buildstream-27ae392/src/buildstream/data/userconfig.yaml000066400000000000000000000061651514607367700246750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Default BuildStream user configuration. # # Work Directories # # Location to store sources sourcedir: ${XDG_CACHE_HOME}/buildstream/sources # Root location for other directories in the cache cachedir: ${XDG_CACHE_HOME}/buildstream # Location to store build logs logdir: ${XDG_CACHE_HOME}/buildstream/logs # Default root location for workspaces, blank for no default set. workspacedir: . # # Cache # cache: # Use as much space as possible quota: infinity # Keep 5% of disk space available reserved-disk-space: 5% # Retain 80% of the cache on cleanup low-watermark: 80% # Whether to pull build trees when downloading element artifacts pull-buildtrees: False # Whether to cache build trees on artifact creation # cache-buildtrees: auto # # Scheduler # scheduler: # Maximum number of simultaneous downloading tasks. fetchers: 10 # Maximum number of simultaneous build tasks. builders: 4 # Maximum number of simultaneous uploading tasks. pushers: 4 # Maximum number of retries for network tasks. network-retries: 2 # Control what to do when a task fails, if not running in # interactive mode # on-error: quit # # Build related configuration # build: # # Maximum number of jobs to run per build task. # max-jobs: 0 # # Try to build elements for which a failed build artifact is found # retry-failed: False # # Control which dependencies to build # dependencies: run # # Source fetch related configuration # fetch: # # Which URIs are allowed to be accessed # source: all # # Source track related configuration # track: # # Which URIs are allowed to be accessed # source: aliases # # Logging # logging: # The abbreviated cache key length to display in the UI key-length: 8 # Whether to show extra detailed messages verbose: True # Maximum number of lines to print from the # end of a failing build log error-lines: 20 # Maximum number of lines to print in a detailed # message on the console or in the master log (the full # messages are always recorded in the individual build # logs) message-lines: 20 # Whether to enable debugging messages debug: False # Format string for printing the pipeline at startup, this # also determines the default display format for `bst show` # element-format: | %{state: >12} %{full-key} %{name} %{workspace-dirs} # Format string for log messages. # message-format: | [%{elapsed}][%{key}][%{element}] %{action} %{message} # Limit bst console output update rate to 1Hz where applicable throttle-ui-updates: True apache-buildstream-27ae392/src/buildstream/data/zsh/000077500000000000000000000000001514607367700224415ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/data/zsh/_bst000066400000000000000000000010541514607367700233130ustar00rootroot00000000000000#compdef bst _bst_cmpl() { local idx completions name idx="${#words[@]}" # ZSH arrays start at 1 let idx=idx-1 completions=( $( env COMP_WORDS="$words" \ COMP_CWORD=$idx \ _BST_COMPLETION=complete bst ) ) for name in ${completions[@]}; do # For items that are an incomplete path, do not add trailing space if [[ $name = */ ]]; then compadd -S '' $name else A=($name) compadd -a A fi done return 0 } _bst_cmpl "$@" apache-buildstream-27ae392/src/buildstream/downloadablefilesource.py000066400000000000000000000370151514607367700260200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """ DownloadableFileSource - Abstract class for sources downloaded from a URI ========================================================================= This DownloadableFileSource class is a convenience class on can derive for implementing sources that get downloaded from a URI. It provides utilities around handling mirrors, tracking and fetching the source. Any derived classes must write their own stage() and get_unique_key() implementation. .. _core_downloadable_source_builtins: Built-in functionality ---------------------- The DownloadableFileSource class provides built in keys which can be set when intantiating any Source which derives from DownloadableFileSource * Guess pattern The ``version-guess-pattern`` sets the regular expression which will be used to attempt to guess the version of a source when parsing the source's URI. The DownloadableFileSource provides a default implementation of :func:`Source.collect_source_info() `, which will use the ``version-guess-pattern`` to attempt to extract a human readable version string from the specified URI, in order to fill out the reported :attr:`~buildstream.source.SourceInfo.version_guess`. This is done using the :func:`utils.guess_version() ` utility function, please refer to that function documentation to understand how the guessing mechanics works, and what kind of string you should provide here. .. note: The version guessing mechanism will not be observed if ``version`` is specified. **Since: 2.5**. * Version The ``version`` explicitly sets the :attr:`~buildstream.source.SourceInfo.version_guess` attribute of the :class:`SourceInfo ` reported for this source, overriding any guessing. This is useful for remote files which do not express their version in their filenames. **Since: 2.5**. SourceMirror extra data "http-auth" -------------------------------------------- The DownloadableFileSource, and consequently any :class:`Source ` implementations which derive from DownloadableFileSource, support the "http-auth" extra data returned by :class:`SourceMirror ` plugins through :func:`Source.translate_url() `. This functionality is available **Since: 2.2**. This allows one to use :class:`SourceMirror ` plugins to add an authorization header to the ``GET`` requests. **Example:** .. code:: python class MySourceMirror(SourceMirror): def translate_url( self, *, alias: str, alias_url: str, source_url: str, extra_data: Optional[Dict[str, Any]], ) -> str: # # Set the "http-auth" extra data # if extra_data is not None: extra_data["http-auth"] = "bearer" # ... Only the "http-auth" value ``bearer`` is supported. **Example:** If the URL reported by :func:`SourceMirror.translate_url() ` is ``http://flying-ponies.com/downloads/pony.tgz``, then a corresponding entry will be expected in the user's ``~/.netrc``: .. code:: flying-ponies.com password 1234 DownloadableFileSource will add the following header to the ``GET`` request to download the file: .. code:: Authorization: Bearer 1234 .. _core_downloadable_source_info: Default reporting of :class:`.SourceInfo` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Source plugins which derive from the DownloadableFileSource, unless overridden and specified in the documentation of the specific source plugin, will behave as described here. The DownloadableFileSource reports the URL of the remote file as the *url*. Further, the DownloadableFileSourcebzr source reports the :attr:`SourceInfoMedium.REMOTE_FILE ` *medium* and the :attr:`SourceVersionType.SHA256 ` *version_type*, for which it reports the sha256 checksum of the remote file content as the *version*. An attempt to guess the version based on the remote filename will be made for the reporting of the *version_guess*. Control over how the guess is made or overridden is explained above in the :ref:`built-in functionality documentation `. """ import os import re import urllib.request import urllib.error import contextlib import shutil import netrc from .source import Source, SourceError, SourceInfoMedium, SourceVersionType from . import utils class _NetrcFTPOpener(urllib.request.FTPHandler): def __init__(self, netrc_config): self.netrc = netrc_config def _unsplit(self, host, port, user, passwd): if port: host = "{}:{}".format(host, port) if user: if passwd: user = "{}:{}".format(user, passwd) host = "{}@{}".format(user, host) return host def ftp_open(self, req): uri = urllib.parse.urlparse(req.full_url) username = uri.username password = uri.password if uri.username is None and self.netrc: entry = self.netrc.authenticators(uri.hostname) if entry: username, _, password = entry req.host = self._unsplit(uri.hostname, uri.port, username, password) return super().ftp_open(req) class _NetrcPasswordManager: def __init__(self, netrc_config): self.netrc = netrc_config def add_password(self, realm, uri, user, passwd): pass def find_user_password(self, realm, authuri): if not self.netrc: return None, None parts = urllib.parse.urlsplit(authuri) entry = self.netrc.authenticators(parts.hostname) if not entry: return None, None else: login, _, password = entry return login, password def _download_file(opener_creator, url, etag, directory, bearer_auth): opener = opener_creator.get_url_opener(bearer_auth) default_name = os.path.basename(url) request = urllib.request.Request(url) request.add_header("Accept", "*/*") request.add_header("User-Agent", "BuildStream/2") if opener_creator.netrc_config and bearer_auth: parts = urllib.parse.urlsplit(url) entry = opener_creator.netrc_config.authenticators(parts.hostname) if entry: _, _, password = entry auth_header = "Bearer " + password request.add_header("Authorization", auth_header) if etag is not None: request.add_header("If-None-Match", etag) try: with contextlib.closing(opener.open(request, timeout=10 * 60)) as response: info = response.info() # some servers don't honor the 'If-None-Match' header if etag and info["ETag"] == etag: return None, None, None etag = info["ETag"] length = info.get("Content-Length") filename = info.get_filename(default_name) filename = os.path.basename(filename) local_file = os.path.join(directory, filename) with open(local_file, "wb") as dest: shutil.copyfileobj(response, dest) actual_length = dest.tell() if length and actual_length < int(length): raise ValueError(f"Partial file {actual_length}/{length}") except urllib.error.HTTPError as e: if e.code == 304: # 304 Not Modified. # Because we use etag only for matching ref, currently specified ref is what # we would have downloaded. return None, None, None return None, None, str(e) except (urllib.error.URLError, OSError, ValueError) as e: # Note that urllib.request.Request in the try block may throw a # ValueError for unknown url types, so we handle it here. return None, None, str(e) return local_file, etag, None class DownloadableFileSource(Source): # pylint: disable=attribute-defined-outside-init COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ["url", "ref", "version-guess-pattern", "version"] __default_mirror_file = None def configure(self, node): self.original_url = node.get_str("url") self.ref = node.get_str("ref", None) extra_data = {} self.url = self.translate_url(self.original_url, extra_data=extra_data) self.bearer_auth = extra_data.get("http-auth") == "bearer" self._mirror_dir = os.path.join(self.get_mirror_directory(), utils.url_directory_name(self.original_url)) self._guess_pattern_string = node.get_str("version-guess-pattern", None) self._guess_pattern = None if self._guess_pattern_string is not None: self._guess_pattern = re.compile(self._guess_pattern_string) self._version = node.get_str("version", None) def preflight(self): return def get_unique_key(self): unique_key = [self.original_url, self.ref] # Backwards compatible method of supporting configuration # attributes which affect SourceInfo generation. if self._version is not None: unique_key.append(self._version) elif self._guess_pattern_string is not None: unique_key.append(self._guess_pattern_string) return unique_key def is_cached(self) -> bool: return os.path.isfile(self._get_mirror_file()) def load_ref(self, node): self.ref = node.get_str("ref", None) def get_ref(self): return self.ref def set_ref(self, ref, node): node["ref"] = self.ref = ref def track(self): # pylint: disable=arguments-differ # there is no 'track' field in the source to determine what/whether # or not to update refs, because tracking a ref is always a conscious # decision by the user. new_ref = self._ensure_mirror("Tracking {}".format(self.url)) if self.ref and self.ref != new_ref: detail = ( "When tracking, new ref differs from current ref:\n" + " Tracked URL: {}\n".format(self.url) + " Current ref: {}\n".format(self.ref) + " New ref: {}\n".format(new_ref) ) self.warn("Potential man-in-the-middle attack!", detail=detail) return new_ref def fetch(self): # pylint: disable=arguments-differ # Just a defensive check, it is impossible for the # file to be already cached because Source.fetch() will # not be called if the source is already cached. # if os.path.isfile(self._get_mirror_file()): return # pragma: nocover # Download the file, raise hell if the sha256sums don't match, # and mirror the file otherwise. sha256 = self._ensure_mirror( "Fetching {}".format(self.url), ) if sha256 != self.ref: raise SourceError( "File downloaded from {} has sha256sum '{}', not '{}'!".format(self.url, sha256, self.ref) ) def collect_source_info(self): version_guess = self._version if version_guess is None: version_guess = utils.guess_version(self.original_url, pattern=self._guess_pattern) return [ self.create_source_info( self.url, SourceInfoMedium.REMOTE_FILE, SourceVersionType.SHA256, self.ref, version_guess=version_guess ) ] def _get_etag(self, ref): etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) if os.path.exists(etagfilename): with open(etagfilename, "r", encoding="utf-8") as etagfile: return etagfile.read() return None def _store_etag(self, ref, etag): etagfilename = os.path.join(self._mirror_dir, "{}.etag".format(ref)) with utils.save_file_atomic(etagfilename) as etagfile: etagfile.write(etag) def _ensure_mirror(self, activity_name: str): # Downloads from the url and caches it according to its sha256sum. with self.tempdir() as td: # We do not use etag in case what we have in cache is # not matching ref in order to be able to recover from # corrupted download. if self.ref and self.is_cached(): # Do not re-download the file if the ETag matches. etag = self._get_etag(self.ref) else: etag = None url_opener_creator = _UrlOpenerCreator(self._parse_netrc()) local_file, new_etag, error = self.blocking_activity( _download_file, (url_opener_creator, self.url, etag, td, self.bearer_auth), activity_name ) if error: raise SourceError("{}: Error mirroring {}: {}".format(self, self.url, error), temporary=True) if local_file is None: return self.ref # Make sure url-specific mirror dir exists. try: os.makedirs(self._mirror_dir, exist_ok=True) except FileExistsError as e: raise SourceError( "{}: Mirror directory exists but is not a directory: {}".format(self, self._mirror_dir) ) from e # Store by sha256sum sha256 = utils.sha256sum(local_file) # Even if the file already exists, move the new file over. # In case the old file was corrupted somehow. os.rename(local_file, self._get_mirror_file(sha256)) if new_etag: self._store_etag(sha256, new_etag) return sha256 def _parse_netrc(self): netrc_config = None try: netrc_config = netrc.netrc() except OSError: # If the .netrc file was not found, FileNotFoundError will be # raised, but OSError will be raised directly by the netrc package # in the case that $HOME is not set. # # This will catch both cases. pass except netrc.NetrcParseError as e: self.warn("{}: While reading .netrc: {}".format(self, e)) return netrc_config def _get_mirror_file(self, sha=None): if sha is not None: return os.path.join(self._mirror_dir, sha) if self.__default_mirror_file is None: self.__default_mirror_file = os.path.join(self._mirror_dir, self.ref) return self.__default_mirror_file class _UrlOpenerCreator: def __init__(self, netrc_config): self.netrc_config = netrc_config def get_url_opener(self, bearer_auth): if self.netrc_config and not bearer_auth: netrc_pw_mgr = _NetrcPasswordManager(self.netrc_config) http_auth = urllib.request.HTTPBasicAuthHandler(netrc_pw_mgr) ftp_handler = _NetrcFTPOpener(self.netrc_config) return urllib.request.build_opener(http_auth, ftp_handler) return urllib.request.build_opener() apache-buildstream-27ae392/src/buildstream/element.py000066400000000000000000004012461514607367700227360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ Element - Base element class ============================ .. _core_element_abstract_methods: Abstract Methods ---------------- For loading and configuration purposes, Elements must implement the :ref:`Plugin base class abstract methods `. .. _core_element_build_phase: Build Phase ~~~~~~~~~~~ The following methods are the foundation of the element's *build phase*, they must be implemented by all Element classes, unless explicitly stated otherwise. * :func:`Element.configure_sandbox() ` Configures the :class:`.Sandbox`. This is called before anything else * :func:`Element.stage() ` Stage dependencies and :class:`Sources ` into the sandbox. * :func:`Element.assemble() ` Perform the actual assembly of the element Miscellaneous ~~~~~~~~~~~~~ Miscellaneous abstract methods also exist: * :func:`Element.generate_script() ` For the purpose of ``bst source checkout --include-build-scripts``, an Element may optionally implement this. Class Reference --------------- """ import os import re import stat import copy import warnings from contextlib import contextmanager, suppress from functools import partial from itertools import chain import string from threading import Lock from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION from ._exceptions import BstError, LoadError, ImplError, SourceCacheError, CachedFailure from .exceptions import ErrorDomain, LoadErrorReason from .utils import FileListResult, BST_ARBITRARY_TIMESTAMP from . import utils from . import _cachekey from . import _site from .node import Node, MappingNode, ScalarNode from .plugin import Plugin from .sandbox import _SandboxFlags, SandboxCommandError from .sandbox._config import SandboxConfig from .sandbox._sandboxremote import SandboxRemote from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey from ._artifact import Artifact from ._elementproxy import ElementProxy from ._elementsources import ElementSources from ._loader import Symbol, DependencyType, MetaSource from ._overlapcollector import OverlapCollector from .storage import Directory, DirectoryError from .storage._filebaseddirectory import FileBasedDirectory if TYPE_CHECKING: from typing import Tuple from .node import SequenceNode from .types import SourceRef # pylint: disable=cyclic-import from .sandbox import Sandbox from .source import Source from ._context import Context from ._loader import LoadElement from ._project import Project # pylint: enable=cyclic-import class ElementError(BstError): """This exception should be raised by :class:`.Element` implementations to report errors to the user. Args: message: The error message to report to the user detail: A possibly multiline, more detailed error message reason: An optional machine readable reason string, used for test cases collect: An optional directory containing partial install contents temporary: An indicator to whether the error may occur if the operation was run again. """ def __init__( self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, collect: Optional[str] = None, temporary: bool = False, ): super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason, temporary=temporary) self.collect = collect class DependencyConfiguration: """An object representing the configuration of a dependency This is used to provide dependency configurations for elements which implement :func:`Element.configure_dependencies() ` """ def __init__(self, element: "Element", path: str, config: Optional["MappingNode"]): self.element = element # type: Element """The dependency Element""" self.path = path # type: str """The path used to refer to this dependency""" self.config = config # type: Optional[MappingNode] """The custom :term:`dependency configuration `, or ``None`` if no custom configuration was provided""" class Element(Plugin): """Element() Base Element class. All elements derive from this class, this interface defines how the core will be interacting with Elements. """ # The defaults from the yaml file and project __defaults = None # A hash of Element by LoadElement __instantiated_elements = {} # type: Dict[LoadElement, Element] # A list of (source, ref) tuples which were redundantly specified __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]] BST_ARTIFACT_VERSION = 0 """The element plugin's artifact version Elements must first set this to 1 if they change their unique key structure in a way that would produce a different key for the same input, or introduce a change in the build output for the same unique key. Further changes of this nature require bumping the artifact version. """ BST_STRICT_REBUILD = False """Whether to rebuild this element in non strict mode if any of the dependencies have changed. """ BST_FORBID_RDEPENDS = False """Whether to raise exceptions if an element has runtime dependencies. """ BST_FORBID_BDEPENDS = False """Whether to raise exceptions if an element has build dependencies. """ BST_FORBID_SOURCES = False """Whether to raise exceptions if an element has sources. """ BST_RUN_COMMANDS = True """Whether the element may run commands using Sandbox.run. """ BST_ELEMENT_HAS_ARTIFACT = True """Whether the element produces an artifact when built. """ def __init__( self, context: "Context", project: "Project", load_element: "LoadElement", plugin_conf: Optional[str], *, artifact_key: Optional[str] = None, ): self.__cache_key_dict = None # Dict for cache key calculation self.__cache_key: Optional[str] = None # Our cached cache key super().__init__(load_element.name, context, project, load_element.node, "element") # Ensure the project is fully loaded here rather than later on if not load_element.first_pass: project.ensure_fully_loaded() self.project_name = self._get_project().name """The :ref:`name ` of the owning project .. attention:: Combining this attribute with :attr:`Plugin.name ` does not provide a unique identifier for an element within a project, this is because multiple :mod:`junction ` elements can be used specify the same project as a subproject. """ self.normal_name = _get_normal_name(self.name) """A normalized element name This is the original element without path separators or the extension, it's used mainly for composing log file names and creating directory names and such. """ # # Internal instance properties # self._depth = None # Depth of Element in its current dependency graph self._overlap_collectors: Dict[Sandbox, OverlapCollector] = {} # Active overlap collector per sandbox self._description = load_element.description or "" # type: str # # Private instance properties # # Cache of proxies instantiated, indexed by the proxy owner self.__proxies = {} # type: Dict[Element, ElementProxy] # Direct runtime dependency Elements self.__runtime_dependencies = [] # type: List[Element] # Direct build dependency Elements self.__build_dependencies = [] # type: List[Element] # Direct build dependency subset which require strict rebuilds self.__strict_dependencies = [] # type: List[Element] # Direct reverse build dependency Elements self.__reverse_build_deps = set() # type: Set[Element] # Direct reverse runtime dependency Elements self.__reverse_runtime_deps = set() # type: Set[Element] self.__build_deps_uncached = None # Build dependencies which are not yet cached self.__runtime_deps_uncached = None # Runtime dependencies which are not yet cached self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element self.__cached_remotely = None # Whether the element is cached remotely self.__sources = ElementSources(context, project, self) # The element sources self.__weak_cache_key: Optional[str] = None # Our cached weak cache key self.__strict_cache_key: Optional[str] = None # Our cached cache key for strict builds self.__artifacts = context.artifactcache # Artifact cache self.__sourcecache = context.sourcecache # Source cache self.__assemble_scheduled = False # Element is scheduled to be assembled self.__assemble_done = False # Element is assembled self.__pull_pending = False # Whether pull is pending self.__cached_successfully = None # If the Element is known to be successfully cached self.__splits = None # Resolved regex objects for computing split domains self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap self.__tainted = None # Whether the artifact is tainted and should not be shared self.__required = False # Whether the artifact is required in the current session self.__build_result = None # The result of assembling this Element (success, description, detail) # Artifact class for direct artifact composite interaction self.__artifact = None # type: Optional[Artifact] self.__dynamic_public = None self.__sandbox_config = None # type: Optional[SandboxConfig] # Callbacks self.__required_callback = None # Callback to Queues self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue self.__buildable_callback = None # Callback to BuildQueue self.__resolved_initial_state = False # Whether the initial state of the Element has been resolved self.__environment: Dict[str, str] = {} self.__variables: Optional[Variables] = None self.__dynamic_public_guard = Lock() if artifact_key: self.__initialize_from_artifact_key(artifact_key) else: self.__initialize_from_yaml(load_element, plugin_conf) def __lt__(self, other): return self.name < other.name ############################################################# # Abstract Methods # ############################################################# def configure_dependencies(self, dependencies: Iterable[DependencyConfiguration]) -> None: """Configure the Element with regards to it's build dependencies Elements can use this method to parse custom configuration which define their relationship to their build dependencies. If this method is implemented, then it will be called with all direct build dependencies specified in their :ref:`element declaration ` in a list. If the dependency was declared with custom configuration, it will be provided along with the dependency element, otherwise `None` will be passed with dependencies which do not have any additional configuration. If the user has specified the same build dependency multiple times with differing configurations, then those build dependencies will be provided multiple times in the ``dependencies`` list. Args: dependencies (list): A list of :class:`DependencyConfiguration ` objects Raises: :class:`.ElementError`: When the element raises an error The format of the :class:`MappingNode ` provided as :attr:`DependencyConfiguration.config belongs to the implementing element, and as such the format should be documented by the plugin, and the :func:`MappingNode.validate_keys() ` method should be called by the implementing plugin in order to validate it. .. note:: It is unnecessary to implement this method if the plugin does not support any custom :term:`dependency configuration `. """ # This method is not called on plugins which do not implement it, so it would # be a bug if this accidentally gets called. # assert False, "Code should not be reached" def configure_sandbox(self, sandbox: "Sandbox") -> None: """Configures the the sandbox for execution Args: sandbox: The build sandbox Raises: (:class:`.ElementError`): When the element raises an error Elements must implement this method to configure the sandbox object for execution. """ raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format(kind=self.get_kind())) def stage(self, sandbox: "Sandbox") -> None: """Stage inputs into the sandbox directories Args: sandbox: The build sandbox Raises: (:class:`.ElementError`): When the element raises an error Elements must implement this method to populate the sandbox directory with data. This is done either by staging :class:`.Source` objects, by staging the artifacts of the elements this element depends on, or both. """ raise ImplError("element plugin '{kind}' does not implement stage()".format(kind=self.get_kind())) def assemble(self, sandbox: "Sandbox") -> str: """Assemble the output artifact Args: sandbox: The build sandbox Returns: An absolute path within the sandbox to collect the artifact from Raises: (:class:`.ElementError`): When the element raises an error Elements must implement this method to create an output artifact from its sources and dependencies. """ raise ImplError("element plugin '{kind}' does not implement assemble()".format(kind=self.get_kind())) def generate_script(self) -> str: """Generate a build (sh) script to build this element Returns: A string containing the shell commands required to build the element BuildStream guarantees the following environment when the generated script is run: - All element variables have been exported. - The cwd is `self.get_variable('build-root')/self.normal_name`. - $PREFIX is set to `self.get_variable('install-root')`. - The directory indicated by $PREFIX is an empty directory. Files are expected to be installed to $PREFIX. If the script fails, it is expected to return with an exit code != 0. """ raise ImplError("element plugin '{kind}' does not implement write_script()".format(kind=self.get_kind())) ############################################################# # Public Methods # ############################################################# def sources(self) -> Iterator["Source"]: """A generator function to enumerate the element sources Yields: The sources of this element """ return self.__sources.sources() def dependencies( self, selection: Optional[Sequence["Element"]] = None, *, recurse: bool = True ) -> Iterator["Element"]: """A generator function which yields the build dependencies of the given element. This generator gives the Element access to all of the dependencies which it is has access to at build time. As explained in :ref:`the dependency type documentation `, this includes the direct build dependencies of the element being built, along with any transient runtime dependencies of those build dependencies. Subsets of the dependency graph can be selected using the `selection` argument,, which must consist of dependencies of this element. If the `selection` argument is specified as `None`, then the `self` element on which this is called is used as the `selection`. If `recurse` is specified (the default), the full dependencies will be listed in deterministic staging order, starting with the basemost elements. Otherwise, if `recurse` is not specified then only the direct dependencies will be traversed. Args: selection (Sequence[Element]): A list of dependencies to select, or None recurse (bool): Whether to recurse Yields: The dependencies of the selection, in deterministic staging order """ # # In this public API, we ensure the invariant that an element can only # ever see elements in it's own _Scope.BUILD scope. # # - Yield ElementProxy objects for every element except for the self element # - When a selection is provided, ensure that we call the real _dependencies() # method using _Scope.RUNTIME # - When iterating over the self element, use _Scope.BUILD # visited = (BitMap(), BitMap()) if selection is None: selection = [self] for element in selection: if element is self: scope = _Scope.BUILD else: scope = _Scope.RUN # Elements in the `selection` will actually be `ElementProxy` objects, but # those calls will be forwarded to their actual internal `_dependencies()` # methods. # for dep in element._dependencies(scope, recurse=recurse, visited=visited): yield cast("Element", dep.__get_proxy(self)) def search(self, name: str) -> Optional["Element"]: """Search for a dependency by name Args: name: The dependency to search for Returns: The dependency element, or None if not found. """ search = self._search(_Scope.BUILD, name) if search is self: return self elif search: return cast("Element", search.__get_proxy(self)) return None def node_subst_vars(self, node: "ScalarNode") -> str: """Replace any variables in the string contained in the node and returns it. **Warning**: The method is deprecated and will get removed in the next version Args: node: A ScalarNode loaded from YAML Returns: The value with all variables replaced Raises: :class:`.LoadError`: When the node doesn't contain a string or a variable was not found. **Example:** .. code:: python # Expect a string 'name' in 'node', substituting any # variables in the returned string name = self.node_subst_vars(node.get_scalar('name')) """ # FIXME: remove this warnings.warn( "configuration is now automatically expanded, this is a no-op and will be removed.", DeprecationWarning ) return node.as_str() def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str]: """Substitute any variables in the given sequence **Warning**: The method is deprecated and will get removed in the next version Args: node: A SequenceNode loaded from YAML Returns: The list with every variable replaced Raises: :class:`.LoadError` """ # FIXME: remove this warnings.warn( "configuration is now automatically expanded, this is a no-op and will be removed.", DeprecationWarning ) return node.as_str_list() def compute_manifest( self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True ) -> str: """Compute and return this element's selective manifest The manifest consists on the list of file paths in the artifact. The files in the manifest are selected according to `include`, `exclude` and `orphans` parameters. If `include` is not specified then all files spoken for by any domain are included unless explicitly excluded with an `exclude` domain. Args: include: An optional list of domains to include files from exclude: An optional list of domains to exclude files from orphans: Whether to include files not spoken for by split domains Yields: The paths of the files in manifest """ self.__assert_cached() return self.__compute_splits(include, exclude, orphans) def get_artifact_name(self, key: Optional[str] = None) -> str: """Compute and return this element's full artifact name Generate a full name for an artifact, including the project namespace, element name and :ref:`cache key `. This can also be used as a relative path safely, and will normalize parts of the element name such that only digits, letters and some select characters are allowed. Args: key: The element's :ref:`cache key `. Defaults to None Returns: The relative path for the artifact """ if key is None: key = self._get_cache_key() assert key is not None return _compose_artifact_name(self.project_name, self.normal_name, key) def stage_artifact( self, sandbox: "Sandbox", *, path: Optional[str] = None, action: str = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True, ) -> FileListResult: """Stage this element's output artifact in the sandbox This will stage the files from the artifact to the sandbox at specified location. The files are selected for staging according to the `include`, `exclude` and `orphans` parameters; if `include` is not specified then all files spoken for by any domain are included unless explicitly excluded with an `exclude` domain. Args: sandbox: The build sandbox path: An optional sandbox relative path action (OverlapAction): The action to take when overlapping with previous invocations include: An optional list of domains to include files from exclude: An optional list of domains to exclude files from orphans: Whether to include files not spoken for by split domains Raises: (:class:`.ElementError`): If the element has not yet produced an artifact. Returns: The result describing what happened while staging .. note:: Directories in `dest` are replaced with files from `src`, unless the existing directory in `dest` is not empty in which case the path will be reported in the return value. .. attention:: When staging artifacts with their dependencies, use :func:`Element.stage_dependency_artifacts() ` instead. """ overlap_collector = self._overlap_collectors.get(sandbox) assert overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()" # # The public API can only be called on the implementing plugin itself. # # ElementProxy calls to stage_artifact() are routed directly to _stage_artifact(), # and the ElementProxy takes care of starting and ending the OverlapCollector session. # with overlap_collector.session(action, path): result = self._stage_artifact( sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans ) return result def stage_dependency_artifacts( self, sandbox: "Sandbox", selection: Optional[Sequence["Element"]] = None, *, path: Optional[str] = None, action: str = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True, ) -> None: """Stage element dependencies in scope This is primarily a convenience wrapper around :func:`Element.stage_artifact() ` which takes care of staging all the dependencies in staging order and issueing the appropriate warnings. The `selection` argument will behave in the same was as specified by :func:`Element.dependencies() `, If the `selection` argument is specified as `None`, then the `self` element on which this is called is used as the `selection`. Args: sandbox (Sandbox): The build sandbox selection (Sequence[Element]): A list of dependencies to select, or None path (str): An optional sandbox relative path action (OverlapAction): The action to take when overlapping with previous invocations include (List[str]): An optional list of domains to include files from exclude (List[str]): An optional list of domains to exclude files from orphans (bool): Whether to include files not spoken for by split domains Raises: (:class:`.ElementError`): if forbidden overlaps occur. """ overlap_collector = self._overlap_collectors.get(sandbox) assert overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()" with overlap_collector.session(action, path): for dep in self.dependencies(selection): dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self) def integrate(self, sandbox: "Sandbox") -> None: """Integrate currently staged filesystem against this artifact. Args: sandbox: The build sandbox This modifies the sysroot staged inside the sandbox so that the sysroot is *integrated*. Only an *integrated* sandbox may be trusted for running the software therein, as the integration commands will create and update important system cache files required for running the installed software (such as the ld.so.cache). """ bstdata = self.get_public_data("bst") environment = self.get_environment() if bstdata is not None: with sandbox.batch(): for command in bstdata.get_str_list("integration-commands", []): sandbox.run(["sh", "-e", "-c", command], env=environment, cwd="/", label=command) def stage_sources(self, sandbox: "Sandbox", directory: str) -> None: """Stage this element's sources to a directory in the sandbox Args: sandbox: The build sandbox directory: An absolute path within the sandbox to stage the sources at """ self._stage_sources_in_sandbox(sandbox, directory) def get_public_data(self, domain: str) -> "MappingNode[Node]": """Fetch public data on this element Args: domain: A public domain name to fetch data for Returns: .. note:: This can only be called the abstract methods which are called as a part of the :ref:`build phase ` and never before. """ with self.__dynamic_public_guard: if self.__dynamic_public is None: self.__load_public_data() # Disable type-checking since we can't easily tell mypy that # `self.__dynamic_public` can't be None here. data = self.__dynamic_public.get_mapping(domain, default=None) # type: ignore if data is not None: data = data.clone() return data def set_public_data(self, domain: str, data: "MappingNode[Node]") -> None: """Set public data on this element Args: domain: A public domain name to fetch data for data: The public data dictionary for the given domain This allows an element to dynamically mutate public data of elements or add new domains as the result of success completion of the :func:`Element.assemble() ` method. """ with self.__dynamic_public_guard: if self.__dynamic_public is None: self.__load_public_data() if data is not None: data = data.clone() self.__dynamic_public[domain] = data # type: ignore def get_environment(self) -> Dict[str, str]: """Fetch the environment suitable for running in the sandbox Returns: A dictionary of string key/values suitable for passing to :func:`Sandbox.run() ` """ return self.__environment def get_variable(self, varname: str) -> Optional[str]: """Fetch the value of a variable resolved for this element. Args: varname: The name of the variable to fetch Returns: The resolved value for *varname*, or None if no variable was declared with the given name. """ assert self.__variables return self.__variables.get(varname) def run_cleanup_commands(self, sandbox: "Sandbox") -> None: """Run commands to cleanup the build directory. Args: sandbox: The build sandbox This may be called at the end of a command batch in :func:`Element.assemble() ` to avoid the costs of capturing the build directory after a successful build. This will have no effect if the build tree is required after the build. """ context = self._get_context() if self._get_workspace() or context.cache_buildtrees == _CacheBuildTrees.ALWAYS: # Buildtree must be preserved even after a success build if this is a # workspace build or the user has configured to always cache buildtrees. return build_root = self.get_variable("build-root") install_root = self.get_variable("install-root") assert build_root if install_root and (build_root.startswith(install_root) or install_root.startswith(build_root)): # Preserve the build directory if cleaning would affect the install directory return sandbox._clean_directory(build_root) @contextmanager def subsandbox(self, sandbox: "Sandbox") -> Iterator["Sandbox"]: """A context manager for a subsandbox. Args: sandbox: The main build sandbox This allows an element to use a secondary sandbox for manipulating artifacts without affecting the main build sandbox. The subsandbox is initially empty. """ subsandbox = sandbox._create_subsandbox() with self.__collect_overlaps(subsandbox): yield subsandbox ############################################################# # Private Methods used in BuildStream # ############################################################# # _dependencies() # # A generator function which yields the dependencies of the given element. # # If `recurse` is specified (the default), the full dependencies will be listed # in deterministic staging order, starting with the basemost elements in the # given `scope`. Otherwise, if `recurse` is not specified then only the direct # dependencies in the given `scope` will be traversed, and the element itself # will be omitted. # # Args: # scope (_Scope): The scope to iterate in # recurse (bool): Whether to recurse # # Yields: # (Element): The dependencies in `scope`, in deterministic staging order # def _dependencies(self, scope, *, recurse=True, visited=None): # The format of visited is (BitMap(), BitMap()), with the first BitMap # containing element that have been visited for the `_Scope.BUILD` case # and the second one relating to the `_Scope.RUN` case. if not recurse: result: Set["Element"] = set() if scope in (_Scope.BUILD, _Scope.ALL): for dep in self.__build_dependencies: if dep not in result: result.add(dep) yield dep if scope in (_Scope.RUN, _Scope.ALL): for dep in self.__runtime_dependencies: if dep not in result: result.add(dep) yield dep else: def visit(element, scope, visited): if scope == _Scope.ALL: visited[0].add(element._unique_id) visited[1].add(element._unique_id) for dep in chain(element.__build_dependencies, element.__runtime_dependencies): if dep._unique_id not in visited[0] and dep._unique_id not in visited[1]: yield from visit(dep, _Scope.ALL, visited) yield element elif scope == _Scope.BUILD: visited[0].add(element._unique_id) for dep in element.__build_dependencies: if dep._unique_id not in visited[1]: yield from visit(dep, _Scope.RUN, visited) elif scope == _Scope.RUN: visited[1].add(element._unique_id) for dep in element.__runtime_dependencies: if dep._unique_id not in visited[1]: yield from visit(dep, _Scope.RUN, visited) yield element else: yield element if visited is None: # Visited is of the form (Visited for _Scope.BUILD, Visited for _Scope.RUN) visited = (BitMap(), BitMap()) else: # We have already a visited set passed. we might be able to short-circuit if scope in (_Scope.BUILD, _Scope.ALL) and self._unique_id in visited[0]: return if scope in (_Scope.RUN, _Scope.ALL) and self._unique_id in visited[1]: return yield from visit(self, scope, visited) # _search() # # Search for a dependency by name # # Args: # scope (_Scope): The scope to search # name (str): The dependency to search for # # Returns: # (Element): The dependency element, or None if not found. # def _search(self, scope, name): for dep in self._dependencies(scope): if dep.name == name: return dep return None # _stage_artifact() # # Stage this element's output artifact in the sandbox # # This will stage the files from the artifact to the sandbox at specified location. # The files are selected for staging according to the `include`, `exclude` and `orphans` # parameters; if `include` is not specified then all files spoken for by any domain # are included unless explicitly excluded with an `exclude` domain. # # Args: # sandbox: The build sandbox # path: An optional sandbox relative path # action (OverlapAction): The action to take when overlapping with previous invocations # include: An optional list of domains to include files from # exclude: An optional list of domains to exclude files from # orphans: Whether to include files not spoken for by split domains # owner: The session element currently running Element.stage() # # Raises: # (:class:`.ElementError`): If the element has not yet produced an artifact. # # Returns: # The result describing what happened while staging # def _stage_artifact( self, sandbox: "Sandbox", *, path: Optional[str] = None, action: str = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True, owner: Optional["Element"] = None, ) -> FileListResult: owner = owner or self overlap_collector = owner._overlap_collectors.get(sandbox) assert overlap_collector is not None, "Attempted to stage artifacts outside of Element.stage()" if not self._cached(): detail = ( "No artifacts have been cached yet for that element\n" + "Try building the element first with `bst build`\n" ) raise ElementError("No artifacts to stage", detail=detail, reason="uncached-checkout-attempt") # Time to use the artifact, check once more that it's there self.__assert_cached() self.status("Staging {}/{}".format(self.name, self._get_display_key().brief)) # Disable type checking since we can't easily tell mypy that # `self.__artifact` can't be None at this stage. files_vdir = self.__artifact.get_files() # type: ignore # Import files into the staging area # vbasedir = sandbox.get_virtual_directory() vstagedir = vbasedir if path is None else vbasedir.open_directory(path.lstrip(os.sep), create=True) split_filter = self.__split_filter_func(include, exclude, orphans) result = vstagedir._import_files_internal(files_vdir, filter_callback=split_filter) assert result is not None overlap_collector.collect_stage_result(self, result) return result # _stage_dependency_artifacts() # # Stage element dependencies in scope, this is used for core # functionality especially in the CLI which wants to stage specifically # build or runtime dependencies. # # Args: # sandbox: The build sandbox # scope (_Scope): The scope of artifacts to stage # path An optional sandbox relative path # include: An optional list of domains to include files from # exclude: An optional list of domains to exclude files from # orphans: Whether to include files not spoken for by split domains # # Raises: # (:class:`.ElementError`): If any of the dependencies in `scope` have not # yet produced artifacts, or if forbidden overlaps # occur. # def _stage_dependency_artifacts(self, sandbox, scope, *, path=None, include=None, exclude=None, orphans=True): with self._overlap_collectors[sandbox].session(OverlapAction.WARNING, path): for dep in self._dependencies(scope): dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self) # _new_from_load_element(): # # Recursively instantiate a new Element instance, its sources # and its dependencies from a LoadElement. # # FIXME: Need to use an iterative algorithm here since recursion # will limit project dependency depth. # # Args: # load_element (LoadElement): The LoadElement # task (Task): A task object to report progress to # # Returns: # (Element): A newly created Element instance # @classmethod def _new_from_load_element(cls, load_element, task=None): if not load_element.first_pass: load_element.project.ensure_fully_loaded() with suppress(KeyError): return cls.__instantiated_elements[load_element] element = load_element.project.create_element(load_element) cls.__instantiated_elements[load_element] = element # If the element implements configure_dependencies(), we will collect # the dependency configurations for it, otherwise we will consider # it an error to specify `config` on dependencies. # if element.configure_dependencies.__func__ is not Element.configure_dependencies: custom_configurations = [] else: custom_configurations = None # Load the sources from the LoadElement element.__load_sources(load_element) # Instantiate dependencies for dep in load_element.dependencies: dependency = Element._new_from_load_element(dep.element, task) if dep.dep_type & DependencyType.BUILD: element.__build_dependencies.append(dependency) dependency.__reverse_build_deps.add(element) # Configuration data is only collected for build dependencies, # if configuration data is specified on a runtime dependency # then the assertion will be raised by the LoadElement. # if custom_configurations is not None: # Create a proxy for the dependency dep_proxy = cast("Element", ElementProxy(element, dependency)) # Class supports dependency configuration if dep.config_nodes: # Ensure variables are substituted first # for config in dep.config_nodes: element.__variables.expand(config) custom_configurations.extend( [DependencyConfiguration(dep_proxy, dep.path, config) for config in dep.config_nodes] ) else: custom_configurations.append(DependencyConfiguration(dep_proxy, dep.path, None)) elif dep.config_nodes: # Class does not support dependency configuration provenance = dep.config_nodes[0].get_provenance() raise LoadError( "{}: Custom dependency configuration is not supported by element plugin '{}'".format( provenance, element.get_kind() ), LoadErrorReason.INVALID_DEPENDENCY_CONFIG, ) if dep.dep_type & DependencyType.RUNTIME: element.__runtime_dependencies.append(dependency) dependency.__reverse_runtime_deps.add(element) if dep.strict: element.__strict_dependencies.append(dependency) no_of_runtime_deps = len(element.__runtime_dependencies) element.__runtime_deps_uncached = no_of_runtime_deps no_of_build_deps = len(element.__build_dependencies) element.__build_deps_uncached = no_of_build_deps if custom_configurations is not None: element.configure_dependencies(custom_configurations) element.__preflight() element._initialize_state() if task: task.add_current_progress() return element # _clear_meta_elements_cache() # # Clear the internal meta elements cache. # # When loading elements from meta, we cache already instantiated elements # in order to not have to load the same elements twice. # This clears the cache. # # It should be called whenever we are done loading all elements in order # to save memory. # @classmethod def _clear_meta_elements_cache(cls): cls.__instantiated_elements = {} # _get_redundant_source_refs() # # Fetches a list of (Source, ref) tuples of all the Sources # which were loaded with a ref specified in the element declaration # for projects which use project.refs ref-storage. # # This is used to produce a warning @classmethod def _get_redundant_source_refs(cls): return cls.__redundant_source_refs # _reset_load_state() # # This is used to reset the loader state across multiple load sessions. # @classmethod def _reset_load_state(cls): cls.__instantiated_elements = {} cls.__redundant_source_refs = [] # _cached(): # # Returns: # (bool): Whether this element is already present in # the artifact cache # def _cached(self): return self.__artifact.cached() # _cached_remotely(): # # Returns: # (bool): Whether this element is present in a remote cache # def _cached_remotely(self): if self.__cached_remotely is None: self.__cached_remotely = self.__artifacts.check_remotes_for_element(self) return self.__cached_remotely # _get_build_result(): # # Returns: # (bool): Whether the artifact of this element present in the artifact cache is of a success # (str): Short description of the result # (str): Detailed description of the result # def _get_build_result(self): if self.__build_result is None: self.__load_build_result() return self.__build_result # __set_build_result(): # # Sets the assembly result # # Args: # success (bool): Whether the result is a success # description (str): Short description of the result # detail (str): Detailed description of the result # def __set_build_result(self, success, description, detail=None): self.__build_result = (success, description, detail) # _cached_success(): # # Returns: # (bool): Whether this element is already present in # the artifact cache and the element assembled successfully # def _cached_success(self): # FIXME: _cache() and _cached_success() should be converted to # push based functions where we only update __cached_successfully # once we know this has changed. This will allow us to cheaply check # __cached_successfully instead of calling _cached_success() if self.__cached_successfully: return True if not self._cached(): return False success, _, _ = self._get_build_result() if success: self.__cached_successfully = True return True else: return False # _cached_failure(): # # Returns: # (bool): Whether this element is already present in # the artifact cache and the element did not assemble successfully # def _cached_failure(self): if not self._cached(): return False success, _, _ = self._get_build_result() return not success # _buildable(): # # Returns: # (bool): Whether this element can currently be built # def _buildable(self): # This check must be before `_fetch_needed()` as source cache status # is not always available for non-build pipelines. if not self.__assemble_scheduled: return False if self._fetch_needed(): return False return self.__build_deps_uncached == 0 # _get_cache_key(): # # Returns the cache key # # Args: # strength (_KeyStrength): Either STRONG or WEAK key strength # # Returns: # (str): A hex digest cache key for this Element, or None # # None is returned if information for the cache key is missing. # def _get_cache_key(self, strength=_KeyStrength.STRONG): if strength == _KeyStrength.STRONG: return self.__cache_key else: return self.__weak_cache_key # _can_query_cache(): # # Returns whether the cache key required for cache queries is available. # # Returns: # (bool): True if cache can be queried # def _can_query_cache(self): # cache cannot be queried until strict cache key is available return self.__artifact is not None # _can_query_source_cache(): # # Returns whether the source cache status is available. # # Returns: # (bool): True if source cache can be queried # def _can_query_source_cache(self): return self.__sources.can_query_cache() # _initialize_state() # # Compute up the elment's initial state. Element state contains # the following mutable sub-states: # # - Source state in `ElementSources` # - Artifact cache key # - Source key in `ElementSources` # - Integral component of the cache key # - Computed as part of the source state # - Artifact state # - Cache key # - Must be known to compute this state # - Build status # - Artifact state # - Must be known before we can decide whether to build # # Note that sub-states are dependent on each other, and changes to # one state will effect changes in the next. # # Changes to these states can be caused by numerous things, # notably jobs executed in sub-processes. Changes are performed by # invocations of the following methods: # # - __update_cache_keys() # - Computes the strong and weak cache keys. # - __schedule_assembly_when_necessary() # - Schedules assembly of an element, iff its current state # allows/necessitates it # - __update_cache_key_non_strict() # - Sets strict cache keys in non-strict builds # - Some non-strict build actions can create artifacts # compatible with strict mode (such as pulling), so # this needs to be done # # When any one of these methods are called and cause a change, # they will invoke methods that have a potential dependency on # them, causing the state change to bubble through all potential # side effects. # # After initializing the source state via `ElementSources`, # *this* method starts the process by invoking # `__update_cache_keys()`, which will cause all necessary state # changes. Other functions should use the appropriate methods and # only update what they expect to change - this will ensure that # the minimum amount of work is done. # def _initialize_state(self): if self.__resolved_initial_state: return self.__resolved_initial_state = True # This will initialize source state. self.__sources.update_resolved_state() # This will calculate the cache keys, and for un-initialized # elements recursively initialize anything else (because it # will become considered outdated after cache keys are # updated). self.__update_cache_keys() # _get_display_key(): # # Returns cache keys for display purposes # # Returns: # (_DisplayKey): The display key # # Question marks are returned if information for the cache key is missing. # def _get_display_key(self): context = self._get_context() strict = False cache_key = self._get_cache_key() if not cache_key: cache_key = "{:?<64}".format("") elif cache_key == self.__strict_cache_key: # Strong cache key used in this session matches cache key # that would be used in strict build mode strict = True length = min(len(cache_key), context.log_key_length) return _DisplayKey(cache_key, cache_key[0:length], strict) # _tracking_done(): # # This is called in the main process after the element has been tracked # def _tracking_done(self): # Tracking may change the sources' refs, and therefore the # source state. We need to update source state. self.__sources.update_resolved_state() self.__update_cache_keys() # _track(): # # Calls track() on the Element sources # # Raises: # SourceError: If one of the element sources has an error # # Returns: # (list): A list of Source object ids and their new references # def _track(self): return self.__sources.track(self._get_workspace()) # _prepare_sandbox(): # # This stages things for either _shell() (below) or also # is used to stage things by the `bst artifact checkout` codepath # @contextmanager def _prepare_sandbox(self, scope, shell=False, integrate=True, usebuildtree=False): # Assert first that we have a sandbox configuration if not self.__sandbox_config: raise ElementError( "Error preparing sandbox for element: {}".format(self.name), detail="This is most likely an artifact that is not yet cached, try building or pulling the artifact first", reason="missing-sandbox-config", ) # bst shell and bst artifact checkout require a local sandbox. # pylint: disable-next=contextmanager-generator-missing-cleanup with self.__sandbox(config=self.__sandbox_config, allow_remote=False) as sandbox: if usebuildtree: # Configure the sandbox from artifact metadata self.__artifact.configure_sandbox(sandbox) # Use the cached buildroot directly buildrootvdir = self.__artifact.get_buildroot() sandbox_vroot = sandbox.get_virtual_directory() sandbox_vroot._import_files_internal(buildrootvdir, collect_result=False) elif shell and scope == _Scope.BUILD: self.__configure_sandbox(sandbox) # Stage what we need self.__stage(sandbox) else: # Runtime shell or `bst artifact checkout` # Don't call `configure_sandbox()` as that may attempt to construct subsandboxes # with build dependencies and we're not setting up a build sandbox. sandbox.set_environment(self.get_environment()) # Stage deps in the sandbox root with self.timed_activity("Staging dependencies", silent_nested=True), self.__collect_overlaps(sandbox): self._stage_dependency_artifacts(sandbox, scope) # Run any integration commands provided by the dependencies # once they are all staged and ready if integrate: with self.timed_activity("Integrating sandbox"), sandbox.batch(): for dep in self._dependencies(scope): dep.integrate(sandbox) yield sandbox # _stage_sources_in_sandbox(): # # Stage this element's sources to a directory inside sandbox # # Args: # sandbox (:class:`.Sandbox`): The build sandbox # directory (str): An absolute path to stage the sources at # def _stage_sources_in_sandbox(self, sandbox, directory): # Stage all sources that need to be copied sandbox_vroot = sandbox.get_virtual_directory() host_vdirectory = sandbox_vroot.open_directory(directory.lstrip(os.sep), create=True) self._stage_sources_at(host_vdirectory) # _stage_sources_at(): # # Stage this element's sources to a directory # # Args: # vdirectory (Union[str, Directory]): A virtual directory object or local path to stage sources to. # def _stage_sources_at(self, vdirectory): # It's advantageous to have this temporary directory on # the same file system as the rest of our cache. with self.timed_activity("Staging sources", silent_nested=True): if not isinstance(vdirectory, Directory): vdirectory = FileBasedDirectory(vdirectory) if vdirectory: raise ElementError("Staging directory '{}' is not empty".format(vdirectory)) # stage sources from source cache staged_sources = self.__sources.get_files() # incremental builds should merge the source into the last artifact before staging last_build_artifact = self.__get_last_build_artifact() if last_build_artifact: self.info("Incremental build") last_sources = last_build_artifact.get_sources() import_dir = last_build_artifact.get_buildtree() import_dir._apply_changes(last_sources, staged_sources) else: import_dir = staged_sources # Set update_mtime to ensure deterministic mtime of sources at build time vdirectory._import_files_internal(import_dir, update_mtime=BST_ARBITRARY_TIMESTAMP, collect_result=False) # Ensure deterministic owners of sources at build time vdirectory._set_deterministic_user() # _set_required(): # # Mark this element and its dependencies as required. # This unblocks pull/fetch/build. # # Args: # scope (_Scope): The scope of dependencies to mark as required # def _set_required(self, scope=_Scope.RUN): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" if self.__required: # Already done return self.__required = True # Request artifacts of dependencies for dep in self._dependencies(scope, recurse=False): dep._set_required(scope=_Scope.RUN) # When an element becomes required, it must be assembled for # the current pipeline. `__schedule_assembly_when_necessary()` # will abort if some other state prevents it from being built, # and changes to such states will cause re-scheduling, so this # is safe. self.__schedule_assembly_when_necessary() # Callback to the Queue if self.__required_callback is not None: self.__required_callback(self) self.__required_callback = None # _is_required(): # # Returns whether this element has been marked as required. # def _is_required(self): return self.__required # __should_schedule() # # Returns: # bool - Whether the element can be scheduled for a build. # def __should_schedule(self): # We're processing if we're already scheduled, we've # finished assembling or if we're waiting to pull. processing = self.__assemble_scheduled or self.__assemble_done or self._pull_pending() # We should schedule a build when return ( # We're not processing not processing and # We're required for the current build self._is_required() and # We have figured out the state of our artifact self.__artifact and # And we're not cached yet not self._cached_success() ) # __schedule_assembly_when_necessary(): # # This is called in the main process before the element is assembled # in a subprocess. # def __schedule_assembly_when_necessary(self): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" # FIXME: We could reduce the number of function calls a bit by # factoring this out of this method (and checking whether we # should schedule at the calling end). # # This would make the code less pretty, but it's a possible # optimization if we get desperate enough (and we will ;)). if not self.__should_schedule(): return self.__assemble_scheduled = True # Requests artifacts of build dependencies for dep in self._dependencies(_Scope.BUILD, recurse=False): dep._set_required() # Once we schedule an element for assembly, we know that our # build dependencies have strong cache keys, so we can update # our own strong cache key. self.__update_cache_key_non_strict() # _assemble_done(): # # This is called in the main process after the element has been assembled. # # This will result in updating the element state. # # Args: # successful (bool): Whether the build was successful # def _assemble_done(self, successful): assert self.__assemble_scheduled assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" self.__assemble_done = True if successful: # Directly set known cached status as optimization to avoid # querying buildbox-casd and the filesystem. self.__artifact.set_cached() self.__cached_successfully = True else: self.__artifact.query_cache() # When we're building in non-strict mode, we may have # assembled everything to this point without a strong cache # key. Once the element has been assembled, a strong cache key # can be set, so we do so. self.__update_cache_key_non_strict() self._update_ready_for_runtime_and_cached() if self._get_workspace() and self._cached(): # Note that this block can only happen in the # main process, since `self._cached_success()` cannot # be true when assembly is successful in the task. # # For this reason, it is safe to update and # save the workspaces configuration # key = self._get_cache_key() workspace = self._get_workspace() workspace.last_build = key self._get_context().get_workspaces().save_config() # _assemble(): # # Internal method for running the entire build phase. # # This will: # - Prepare a sandbox for the build # - Call the public abstract methods for the build phase # - Cache the resulting artifact # def _assemble(self): # Only do this the first time around (i.e. __assemble_done is False) # to allow for retrying the job if self._cached_failure() and not self.__assemble_done: with self._output_file() as output_file: for log_path in self.__artifact.get_logs(): with open(log_path, encoding="utf-8") as log_file: output_file.write(log_file.read()) _, description, detail = self._get_build_result() e = CachedFailure(description, detail=detail) # Shelling into a sandbox is useful to debug this error e.sandbox = True raise e # Assert call ordering assert not self._cached_success() context = self._get_context() with self._output_file() as output_file: # Explicitly clean it up, keep the build dir around if exceptions are raised os.makedirs(context.builddir, exist_ok=True) with self.__sandbox(output_file, output_file, self.__sandbox_config) as sandbox: # Ensure that the plugin does not run commands if it said that it wouldn't # # We only disable commands here in _assemble() instead of __sandbox() because # the user might still run a shell on an element which itself does not run commands. # if not self.BST_RUN_COMMANDS: sandbox._disable_run() # By default, the dynamic public data is the same as the static public data. # The plugin's assemble() method may modify this, though. with self.__dynamic_public_guard: self.__dynamic_public = self.__public.clone() # Call the abstract plugin methods # Step 1 - Configure self.__configure_sandbox(sandbox) # Print the environment at the beginning of the log file. env_dump = _yaml.roundtrip_dump_string(sandbox._get_configured_environment() or self.get_environment()) self.log("Build environment for element {}".format(self.name), detail=env_dump) # Step 2 - Stage self.__stage(sandbox) try: # Step 3 - Assemble collect = self.assemble(sandbox) # pylint: disable=assignment-from-no-return self.__set_build_result(success=True, description="succeeded") except (ElementError, SandboxCommandError) as e: # Shelling into a sandbox is useful to debug this error e.sandbox = True self.__set_build_result(success=False, description=str(e), detail=e.detail) self._cache_artifact(sandbox, e.collect) raise else: self._cache_artifact(sandbox, collect) def _cache_artifact(self, sandbox, collect): context = self._get_context() buildresult = self.__build_result with self.__dynamic_public_guard: publicdata = self.__dynamic_public sandbox_vroot = sandbox.get_virtual_directory() collectvdir = None sandbox_build_dir = None sourcesvdir = None buildrootvdir = None cache_buildtrees = context.cache_buildtrees build_success = buildresult[0] # cache_buildtrees defaults to 'auto', only caching buildtrees # when necessary, which includes failed builds. # If only caching failed artifact buildtrees, then query the build # result. Element types without a build-root dir will be cached # with an empty buildtreedir regardless of this configuration. if cache_buildtrees == _CacheBuildTrees.ALWAYS or ( cache_buildtrees == _CacheBuildTrees.AUTO and (not build_success or self._get_workspace()) ): try: sandbox_build_dir = sandbox_vroot.open_directory(self.get_variable("build-root").lstrip(os.sep)) sandbox._fetch_missing_blobs(sandbox_build_dir) except DirectoryError: # Directory could not be found. Pre-virtual # directory behaviour was to continue silently # if the directory could not be found. pass buildrootvdir = sandbox_vroot sourcesvdir = self.__sources.get_files() if collect is not None: try: collectvdir = sandbox_vroot.open_directory(collect.lstrip(os.sep)) sandbox._fetch_missing_blobs(collectvdir) except DirectoryError: pass # We should always have cache keys already set when caching an artifact assert self.__cache_key is not None assert self.__artifact._cache_key is not None with self.timed_activity("Caching artifact"): self.__artifact.cache( buildrootvdir=buildrootvdir, sandbox_build_dir=sandbox_build_dir, collectvdir=collectvdir, sourcesvdir=sourcesvdir, buildresult=buildresult, publicdata=publicdata, variables=self.__variables, environment=self.__environment, sandboxconfig=self.__sandbox_config, buildsandbox=sandbox if buildrootvdir else None, ) if collect is not None and collectvdir is None: raise ElementError( "Directory '{}' was not found inside the sandbox, " "unable to collect artifact contents".format(collect) ) # _fetch_done() # # Indicates that fetching the sources for this element has been done. # # Args: # fetched_original (bool): Whether the original sources had been asked (and fetched) or not # def _fetch_done(self, fetched_original): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" self.__sources.fetch_done(fetched_original) # _pull_pending() # # Check whether the artifact will be pulled. If the pull operation is to # include a specific subdir of the element artifact (from cli or user conf) # then the local cache is queried for the subdirs existence. # # Returns: # (bool): Whether a pull operation is pending # def _pull_pending(self): return self.__pull_pending # _load_artifact_done() # # Indicate that `_load_artifact()` has completed. # # This needs to be called in the main process after `_load_artifact()` # succeeds or fails so that we properly update the main # process data model # # This will result in updating the element state. # def _load_artifact_done(self): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" assert self.__artifact context = self._get_context() if not context.get_strict() and self.__artifact.cached(): # In non-strict mode, strong cache key becomes available when # the artifact is cached self.__update_cache_key_non_strict() self._update_ready_for_runtime_and_cached() self.__schedule_assembly_when_necessary() if self.__can_query_cache_callback is not None: self.__can_query_cache_callback(self) self.__can_query_cache_callback = None # _load_artifact(): # # Load artifact from cache or pull it from remote artifact repository. # # Args: # pull (bool): Whether to attempt to pull the artifact # strict (bool|None): Force strict/non-strict operation # # Returns: True if the artifact has been downloaded, False otherwise # def _load_artifact(self, *, pull, strict=None): context = self._get_context() if strict is None: strict = context.get_strict() pull_buildtrees = context.pull_buildtrees and not self._get_workspace() # First check whether we already have the strict artifact in the local cache artifact = Artifact( self, context, strict_key=self.__strict_cache_key, strong_key=self.__strict_cache_key, weak_key=self.__weak_cache_key, ) artifact.query_cache() self.__pull_pending = False if not pull and not artifact.cached(buildtree=pull_buildtrees): if self.__artifacts.has_fetch_remotes(plugin=self) and not self._get_workspace(): # Artifact is not completely available in cache and artifact remote server is available. # Stop artifact loading here as pull is required to proceed. self.__pull_pending = True # Attempt to pull artifact with the strict cache key pulled = pull and artifact.pull(pull_buildtrees=pull_buildtrees) # Ignore failed build artifacts if a retry was requested ignore_failed_artifact = context.build and context.build_retry_failed if artifact.cached() or strict: if artifact.cached() and ignore_failed_artifact: success, _, _ = artifact.load_build_result() if not success: self.info( "Discarded failed build", detail="Discarded '{}'\n".format(artifact.strong_key) + "because retrying failed builds is enabled.", ) artifact = Artifact( self, context, strong_key=self.__cache_key, strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key, ) artifact._cached = False pulled = False self.__artifact = artifact return pulled elif self.__pull_pending: return False # In non-strict mode retry with weak cache key artifact = Artifact(self, context, strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key) artifact.query_cache() # Attempt to pull artifact with the weak cache key pulled = pull and artifact.pull(pull_buildtrees=pull_buildtrees) # Automatically retry building failed builds in non-strict mode, because # dependencies may have changed since the last build which might cause this # failed build to succeed. # # When not building (e.g. `bst show`, `bst artifact push` etc), we do not drop # the failed artifact, the retry only occurs at build time. # if context.build and artifact.cached(): success, _, _ = artifact.load_build_result() if not success: # # If we could resolve the stong cache key for this element at this time, # we could compare the artifact key against the resolved strong key. # # If we could assert that artifact state is never consulted in advance # of resolving the strong key, then we could discard the loaded artifact # at that time instead. # # Since neither of these are true, we settle for always retrying a failed # build in non-strict mode unless the failed artifact's strong key is # equal to the resolved strict key. # if ignore_failed_artifact or artifact.strong_key != self.__strict_cache_key: if ignore_failed_artifact: reason = "because retrying failed builds is enabled." else: reason = "in non strict mode because intermediate dependencies may have changed." self.info( "Discarded failed build", detail="Discarded '{}'\n{}".format(artifact.strong_key, reason), ) artifact = Artifact( self, context, strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key, ) artifact._cached = False pulled = False self.__artifact = artifact return pulled def _query_source_cache(self): self.__sources.query_cache() def _skip_source_push(self): if not self.sources() or self._get_workspace(): return True return not (self.__sourcecache.has_push_remotes(plugin=self) and self._cached_sources()) def _source_push(self): return self.__sources.push() # _skip_push(): # # Determine whether we should create a push job for this element. # # Args: # skip_uncached (bool): Whether to skip elements that aren't cached # # Returns: # (bool): True if this element does not need a push job to be created # def _skip_push(self, *, skip_uncached): if not self.__artifacts.has_push_remotes(plugin=self): # No push remotes for this element's project return True # Do not push elements that aren't cached, or that are cached with a dangling buildtree # ref unless element type is expected to have an an empty buildtree directory if skip_uncached: if not self._cached(): return True if not self._cached_buildtree() and self._buildtree_exists(): return True if not self._cached_buildroot() and self._buildroot_exists(): return True return False # _push(): # # Push locally cached artifact to remote artifact repository. # # Returns: # (bool): True if the remote was updated, False if it already existed # and no updated was required # def _push(self): if not self._cached(): raise ElementError("Push failed: {} is not cached".format(self.name)) # Do not push elements that are cached with a dangling buildtree ref # unless element type is expected to have an an empty buildtree directory if not self._cached_buildtree() and self._buildtree_exists(): raise ElementError("Push failed: buildtree of {} is not cached".format(self.name)) if not self._cached_buildroot() and self._buildroot_exists(): raise ElementError("Push failed: buildroot of {} is not cached".format(self.name)) if self.__get_tainted(): self.warn("Not pushing tainted artifact.") return False # Push all keys used for local commit via the Artifact member pushed = self.__artifacts.push(self, self.__artifact) if not pushed: return False # Notify successful upload return True # _shell(): # # Connects the terminal with a shell running in a staged # environment # # Args: # scope (_Scope): Either BUILD or RUN scopes are valid, or None # mounts (list): A list of (str, str) tuples, representing host/target paths to mount # isolate (bool): Whether to isolate the environment like we do in builds # prompt (str): A suitable prompt string for PS1 # command (list): An argv to launch in the sandbox # usebuildtree (bool): Use the buildtree as its source # # Returns: Exit code def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False): with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox: environment = sandbox._get_configured_environment() or self.get_environment() environment = copy.copy(environment) flags = _SandboxFlags.INTERACTIVE | _SandboxFlags.ROOT_READ_ONLY # Fetch the main toplevel project, in case this is a junctioned # subproject, we want to use the rules defined by the main one. context = self._get_context() project = context.get_toplevel_project() shell_command, shell_environment, shell_host_files = project.get_shell_config() if prompt is not None: environment["PS1"] = prompt # Special configurations for non-isolated sandboxes if not isolate: # Open the network, and reuse calling uid/gid # flags |= _SandboxFlags.NETWORK_ENABLED | _SandboxFlags.INHERIT_UID # Apply project defined environment vars to set for a shell for key, value in shell_environment.items(): environment[key] = value # Setup any requested bind mounts if mounts is None: mounts = [] for mount in shell_host_files + mounts: if not os.path.exists(mount.host_path): if not mount.optional: self.warn("Not mounting non-existing host file: {}".format(mount.host_path)) else: sandbox.mark_directory(mount.path) sandbox._set_mount_source(mount.path, mount.host_path) if command: argv = command else: argv = shell_command self.status("Running command", detail=" ".join(argv)) # Run shells with network enabled and readonly root. return sandbox._run_with_flags(argv, flags=flags, env=environment) # _open_workspace(): # # "Open" a workspace for this element # # This requires that a workspace already be created in # the workspaces metadata first. # def _open_workspace(self): assert utils._is_in_main_thread(), "This writes to a global file and therefore must be run in the main thread" context = self._get_context() workspace = self._get_workspace() assert workspace is not None # First lets get a temp dir in our build directory # and stage there, then link the files over to the desired # path. # # We do this so that force opening workspaces which overwrites # files in the target directory actually works without any # additional support from Source implementations. # os.makedirs(context.builddir, exist_ok=True) with utils._tempdir(dir=context.builddir, prefix="workspace-{}".format(self.normal_name)) as temp: self.__sources.init_workspace(temp) # Now hardlink the files into the workspace target. utils.link_files(temp, workspace.get_absolute_path()) # _get_workspace(): # # Returns: # (Workspace|None): A workspace associated with this element # def _get_workspace(self): workspaces = self._get_context().get_workspaces() return workspaces.get_workspace(self._get_full_name()) # _write_script(): # # Writes a script to the given directory. def _write_script(self, directory): with open(_site.build_module_template, "r", encoding="utf-8") as f: script_template = f.read() variable_string = "" for var, val in self.get_environment().items(): variable_string += "{0}={1} ".format(var, val) script = script_template.format( name=self.normal_name, build_root=self.get_variable("build-root"), install_root=self.get_variable("install-root"), variables=variable_string, commands=self.generate_script(), ) os.makedirs(directory, exist_ok=True) script_path = os.path.join(directory, "build-" + self.normal_name) with self.timed_activity("Writing build script", silent_nested=True): with utils.save_file_atomic(script_path, "w") as script_file: script_file.write(script) os.chmod(script_path, stat.S_IEXEC | stat.S_IREAD) # Returns the element whose sources this element is ultimately derived from. # # This is intended for being used to redirect commands that operate on an # element to the element whose sources it is ultimately derived from. # # For example, element A is a build element depending on source foo, # element B is a filter element that depends on element A. The source # element of B is A, since B depends on A, and A has sources. # def _get_source_element(self): return self # _cached_buildtree() # # Check if element artifact contains expected buildtree. An # element's buildtree artifact will not be present if the rest # of the partial artifact is not cached. # # Returns: # (bool): True if artifact cached with buildtree, False if # element not cached or missing expected buildtree. # Note this only confirms if a buildtree is present, # not its contents. # def _cached_buildtree(self): if not self._cached(): return False return self.__artifact.cached_buildtree() # _buildtree_exists() # # Check if artifact was created with a buildtree. This does not check # whether the buildtree is present in the local cache. # # Returns: # (bool): True if artifact was created with buildtree, False if # element not cached or not created with a buildtree. # def _buildtree_exists(self): if not self._cached(): return False return self.__artifact.buildtree_exists() # _cached_buildroot() # # Check if element artifact contains expected buildroot. An # element's buildroot artifact will not be present if the rest # of the partial artifact is not cached. # # Returns: # (bool): True if artifact cached with buildroot, False if # element not cached or missing expected buildroot. # Note this only confirms if a buildroot is present, # not its contents. # def _cached_buildroot(self): if not self._cached(): return False return self.__artifact.cached_buildroot() # _buildroot_exists() # # Check if artifact was created with a buildroot. This does not check # whether the buildroot is present in the local cache. # # Returns: # (bool): True if artifact was created with buildroot, False if # element not cached or not created with a buildroot. # def _buildroot_exists(self): if not self._cached(): return False return self.__artifact.buildroot_exists() # _cached_logs() # # Check if the artifact is cached with log files. # # Returns: # (bool): True if artifact is cached with logs, False if # element not cached or missing logs. # def _cached_logs(self): return self.__artifact.cached_logs() # _fetch() # # Fetch the element's sources. # # Raises: # SourceError: If one of the element sources has an error # def _fetch(self, fetch_original=False): if fetch_original: self.__sources.fetch_sources(fetch_original=True) self.__sources.fetch() if not self.__sources.cached(): try: # Stage all element sources into CAS self.__sources.stage_and_cache() except (SourceCacheError, DirectoryError) as e: raise ElementError( "Error trying to stage sources for {}: {}".format(self.name, e), reason="stage-sources-fail" ) # _calculate_cache_key(): # # Calculates the cache key # # Args: # dependencies (List[List[str]]): list of dependencies with project name, # element name and optional cache key # weak_cache_key (Optional[str]): the weak cache key, required for calculating the # strict and strong cache keys # # Returns: # (str): A hex digest cache key for this Element, or None # # None is returned if information for the cache key is missing. # def _calculate_cache_key(self, dependencies, weak_cache_key=None): # No cache keys for dependencies which have no cache keys if any(not all(dep) for dep in dependencies): return None # Generate dict that is used as base for all cache keys if self.__cache_key_dict is None: project = self._get_project() self.__cache_key_dict = { "core-artifact-version": BST_CORE_ARTIFACT_VERSION, "element-base-key": self.__get_base_key(), "element-plugin-key": self.get_unique_key(), "element-plugin-name": self.get_kind(), "element-plugin-version": self.BST_ARTIFACT_VERSION, "public": self.__public.strip_node_info(), } self.__cache_key_dict["sources"] = self.__sources.get_unique_key() self.__cache_key_dict["fatal-warnings"] = sorted(project._fatal_warnings) # Calculate sandbox related factors if this element runs the sandbox at assemble time. if self.BST_RUN_COMMANDS: # Filter out nocache variables from the element's environment cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache} self.__cache_key_dict["sandbox"] = self.__sandbox_config.to_dict() self.__cache_key_dict["environment"] = cache_env cache_key_dict = self.__cache_key_dict.copy() cache_key_dict["dependencies"] = dependencies if weak_cache_key is not None: cache_key_dict["weak-cache-key"] = weak_cache_key return _cachekey.generate_key(cache_key_dict) # _cached_sources() # # Get whether the staged element sources are cached in CAS # # Returns: # (bool): True if the element sources are in CAS # def _cached_sources(self): return self.__sources.cached() # _has_all_sources_resolved() # # Get whether all sources of the element are resolved # # Returns: # (bool): True if all element sources are resolved # def _has_all_sources_resolved(self): return self.__sources.is_resolved() # _fetch_needed(): # # Return whether sources need to be fetched from a remote # # Returns: # (bool): True if one or more element sources need to be fetched # def _fetch_needed(self): return not self.__sources.cached() and not self.__sources.cached_original() # _should_fetch(): # # Return whether we need to run the fetch stage for this element # # Args: # fetch_original (bool): whether we need the original unstaged source # # Returns: # (bool): True if a fetch job is required # def _should_fetch(self, fetch_original=False): if fetch_original: return not self.__sources.cached_original() return not self.__sources.cached() # _set_required_callback() # # # Notify the pull/fetch/build queue that the element is potentially # ready to be processed. # # _Set the _required_callback - the _required_callback is invoked when an # element is marked as required. This informs us that the element needs to # either be pulled or fetched + built. # # Args: # callback (callable) - The callback function # def _set_required_callback(self, callback): self.__required_callback = callback # _set_can_query_cache_callback() # # Notify the pull/fetch queue that the element is potentially # ready to be processed. # # Set the _can_query_cache_callback - the _can_query_cache_callback is # invoked when an element becomes able to query the cache. That is, # the (non-workspaced) element's strict cache key has been calculated. # However, if the element is workspaced, we also invoke this callback # once its build has been scheduled - this ensures that the workspaced # element does not get blocked in the pull queue. # # Args: # callback (callable) - The callback function # def _set_can_query_cache_callback(self, callback): self.__can_query_cache_callback = callback # _set_buildable_callback() # # Notifiy the build queue that the element is potentially ready # to be processed # # Set the _buildable_callback - the _buildable_callback is invoked when # an element is marked as "buildable". That is, its sources are consistent, # its been scheduled to be built and all of its build dependencies have # had their cache key's calculated and are cached. # # Args: # callback (callable) - The callback function # def _set_buildable_callback(self, callback): self.__buildable_callback = callback # _set_depth() # # Set the depth of the Element. # # The depth represents the position of the Element within the current # session's dependency graph. A depth of zero represents the bottommost element. # def _set_depth(self, depth): self._depth = depth # _update_ready_for_runtime_and_cached() # # An Element becomes ready for runtime and cached once the following criteria # are met: # 1. The Element has a strong cache key # 2. The Element is cached (locally) # 3. The runtime dependencies of the Element are ready for runtime and cached. # # These criteria serve as potential trigger points as to when an Element may have # become ready for runtime and cached. # # Once an Element becomes ready for runtime and cached, we notify the reverse # runtime dependencies and the reverse build dependencies of the element, decrementing # the appropriate counters. # def _update_ready_for_runtime_and_cached(self): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" if not self.__ready_for_runtime_and_cached: if self.__runtime_deps_uncached == 0 and self.__artifact and self.__cache_key and self._cached_success(): self.__ready_for_runtime_and_cached = True # Notify reverse dependencies for rdep in self.__reverse_runtime_deps: rdep.__runtime_deps_uncached -= 1 assert not rdep.__runtime_deps_uncached < 0 # Try to notify reverse dependencies if all runtime deps are ready if rdep.__runtime_deps_uncached == 0: rdep._update_ready_for_runtime_and_cached() for rdep in self.__reverse_build_deps: rdep.__build_deps_uncached -= 1 assert not rdep.__build_deps_uncached < 0 if rdep._buildable(): rdep.__update_cache_key_non_strict() if rdep.__buildable_callback is not None: rdep.__buildable_callback(rdep) rdep.__buildable_callback = None # _get_artifact() # # Return the Element's Artifact object # # Returns: # (Artifact): The Artifact object of the Element # def _get_artifact(self): assert self.__artifact, "{}: has no Artifact object".format(self.name) return self.__artifact # _mimic_artifact() # # Assume the state dictated by the currently set artifact. # # This is used both when initializing an Element's state # from a loaded artifact, or after pulling the artifact from # a remote. # def _mimic_artifact(self): artifact = self._get_artifact() # Load bits which have been stored on the artifact # if artifact.cached(): self.__environment = artifact.load_environment() self.__sandbox_config = artifact.load_sandbox_config() self.__variables = artifact.load_variables() self.__cache_key = artifact.strong_key self.__strict_cache_key = artifact.strict_key self.__weak_cache_key = artifact.weak_key # _add_build_dependency() # # Add a build dependency to the Element # # Args: # (Element): The Element to add as a build dependency # def _add_build_dependency(self, dependency): self.__build_dependencies.append(dependency) # _file_is_whitelisted() # # Checks if a file is whitelisted in the overlap whitelist # # This is only internal (one underscore) and not locally private # because it needs to be proxied through ElementProxy. # # Args: # path (str): The path to check # # Returns: # (bool): True of the specified `path` is whitelisted # def _file_is_whitelisted(self, path): # Considered storing the whitelist regex for re-use, but public data # can be altered mid-build. # Public data is not guaranteed to stay the same for the duration of # the build, but I can think of no reason to change it mid-build. # If this ever changes, things will go wrong unexpectedly. if not self.__whitelist_regex: bstdata = self.get_public_data("bst") whitelist = bstdata.get_sequence("overlap-whitelist", default=[]) whitelist_expressions = [utils._glob2re(self.__variables.subst(node)) for node in whitelist] expression = "^(?:" + "|".join(whitelist_expressions) + ")$" self.__whitelist_regex = re.compile(expression, re.MULTILINE | re.DOTALL) return self.__whitelist_regex.match(os.path.join(os.sep, path)) # _get_logs() # # Obtain a list of log file paths # # Returns: # A list of log file paths # def _get_logs(self) -> List[str]: return cast(Artifact, self.__artifact).get_logs() ############################################################# # Private Local Methods # ############################################################# # __get_proxy() # # Obtain a proxy for this element for the specified `owner`. # # We cache the proxies for plugin convenience, this allows plugins # compare proxies to other proxies returned to them, so they # can run valid statements such as `proxy_a is `proxy_b` or # `proxy_a in list_of_proxies`. # # Args: # owner (Element): The owning element # # Returns: # (ElementProxy): An ElementProxy to self, for owner. # def __get_proxy(self, owner: "Element") -> ElementProxy: with suppress(KeyError): return self.__proxies[owner] proxy = ElementProxy(owner, self) self.__proxies[owner] = proxy return proxy # __load_sources() # # Load the Source objects from the LoadElement # def __load_sources(self, load_element): project = self._get_project() workspace = self._get_workspace() meta_sources = [] # If there's a workspace for this element then we just load a workspace # source plugin instead of the real plugins if workspace: workspace_node = {"kind": "workspace"} workspace_node["path"] = workspace.get_absolute_path() workspace_node["last_build"] = str(workspace.to_dict().get("last_build", "")) meta = MetaSource( self.name, 0, self.get_kind(), "workspace", None, None, Node.from_dict(workspace_node), load_element.first_pass, ) meta_sources.append(meta) else: sources = load_element.node.get_sequence(Symbol.SOURCES, default=[]) for index, source in enumerate(sources): kind = source.get_scalar(Symbol.KIND) # The workspace source plugin is only valid for internal use if kind.as_str() == "workspace": raise LoadError( "{}: Invalid usage of workspace source kind".format(kind.get_provenance()), LoadErrorReason.INVALID_DATA, ) del source[Symbol.KIND] # Directory is optional directory = source.get_str(Symbol.DIRECTORY, default=None) if directory: del source[Symbol.DIRECTORY] # Provenance is optional provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None) if provenance_node: del source[Symbol.PROVENANCE] try: provenance_node.validate_keys(project.source_provenance_attributes.keys()) except LoadError as E: raise LoadError( f"Specified source provenance attribute not defined in project config\n {E}", LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE, ) # make sure everything is a string for key, value in provenance_node.items(): if not isinstance(value, ScalarNode): raise LoadError( f"{value}: Expected string for the value of provenance attribute '{key}'", LoadErrorReason.INVALID_DATA, ) meta_source = MetaSource( self.name, index, self.get_kind(), kind.as_str(), directory, provenance_node, source, load_element.first_pass, ) meta_sources.append(meta_source) for meta_source in meta_sources: source = project.create_source(meta_source, variables=self.__variables) redundant_ref = source._load_ref() self.__sources.add_source(source) # Collect redundant refs which occurred at load time if redundant_ref is not None: self.__redundant_source_refs.append((source, redundant_ref)) # __get_dependency_artifact_names() # # Retrieve the artifact names of all of the dependencies in _Scope.BUILD # # Returns: # (list [str]): A list of refs of all dependencies in staging order. # def __get_dependency_artifact_names(self): return [ os.path.join(dep.project_name, _get_normal_name(dep.name), dep._get_cache_key()) for dep in self._dependencies(_Scope.BUILD) ] # __get_last_build_artifact() # # Return the Artifact of the previous build of this element, # if incremental build is available. # # Returns: # (Artifact): The Artifact of the previous build or None # def __get_last_build_artifact(self): workspace = self._get_workspace() if not workspace: # Currently incremental builds are only supported for workspaces return None if not workspace.last_build: return None artifact = Artifact(self, self._get_context(), strong_key=workspace.last_build) artifact.query_cache() if not artifact.cached(): return None if not artifact.cached_buildtree(): return None if not artifact.cached_sources(): return None # Don't perform an incremental build if there has been a change in # build dependencies. old_dep_refs = artifact.get_dependency_artifact_names() new_dep_refs = self.__get_dependency_artifact_names() if old_dep_refs != new_dep_refs: return None return artifact # __configure_sandbox(): # # Internal method for calling public abstract configure_sandbox() method. # def __configure_sandbox(self, sandbox): self.configure_sandbox(sandbox) # __stage(): # # Internal method for calling public abstract stage() method. # def __stage(self, sandbox): # Enable the overlap collector during the staging process with self.__collect_overlaps(sandbox): self.stage(sandbox) # __preflight(): # # A internal wrapper for calling the abstract preflight() method on # the element and its sources. # def __preflight(self): if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS: if any(self._dependencies(_Scope.RUN, recurse=False)) or any( self._dependencies(_Scope.BUILD, recurse=False) ): raise ElementError( "{}: Dependencies are forbidden for '{}' elements".format(self, self.get_kind()), reason="element-forbidden-depends", ) if self.BST_FORBID_RDEPENDS: if any(self._dependencies(_Scope.RUN, recurse=False)): raise ElementError( "{}: Runtime dependencies are forbidden for '{}' elements".format(self, self.get_kind()), reason="element-forbidden-rdepends", ) if self.BST_FORBID_BDEPENDS: if any(self._dependencies(_Scope.BUILD, recurse=False)): raise ElementError( "{}: Build dependencies are forbidden for '{}' elements".format(self, self.get_kind()), reason="element-forbidden-bdepends", ) if self.BST_FORBID_SOURCES: if any(self.sources()): raise ElementError( "{}: Sources are forbidden for '{}' elements".format(self, self.get_kind()), reason="element-forbidden-sources", ) try: self.preflight() except BstError as e: # Prepend provenance to the error raise ElementError("{}: {}".format(self, e), reason=e.reason, detail=e.detail) from e self.__sources.preflight() # __get_base_key() # # Gets the base key for this element, the base key # is the part of the cache key which is element instance # specific and automatically generated by BuildStream core. # def __get_base_key(self): return { "build-root": self.get_variable("build-root"), } # __assert_cached() # # Raises an error if the artifact is not cached. # def __assert_cached(self): assert self._cached(), "{}: Missing artifact {}".format(self, self._get_display_key().brief) # __get_tainted(): # # Checkes whether this artifact should be pushed to an artifact cache. # # Args: # recalculate (bool) - Whether to force recalculation # # Returns: # (bool) False if this artifact should be excluded from pushing. # # Note: # This method should only be called after the element's # artifact is present in the local artifact cache. # def __get_tainted(self, recalculate=False): if recalculate or self.__tainted is None: # Whether this artifact has a workspace workspaced = self.__artifact.get_metadata_workspaced() # Whether this artifact's dependencies have workspaces workspaced_dependencies = self.__artifact.get_metadata_workspaced_dependencies() # Other conditions should be or-ed self.__tainted = workspaced or workspaced_dependencies return self.__tainted # __collect_overlaps(): # # A context manager for collecting overlap warnings and errors. # # Any places where code might call Element.stage_artifact() # or Element.stage_dependency_artifacts() should be run in # this context manager. # @contextmanager def __collect_overlaps(self, sandbox): self._overlap_collectors[sandbox] = OverlapCollector(self) try: yield finally: del self._overlap_collectors[sandbox] # __sandbox(): # # A context manager to prepare a Sandbox object at the specified directory, # if the directory is None, then a directory will be chosen automatically # in the configured build directory. # # Args: # directory (str): The local directory where the sandbox will live, or None # stdout (fileobject): The stream for stdout for the sandbox # stderr (fileobject): The stream for stderr for the sandbox # config (SandboxConfig): The SandboxConfig object # allow_remote (bool): Whether the sandbox is allowed to be remote # # Yields: # (Sandbox): A usable sandbox # @contextmanager def __sandbox(self, stdout=None, stderr=None, config=None, allow_remote=True): context = self._get_context() project = self._get_project() platform = context.platform if self._get_workspace(): output_node_properties = ["mtime"] else: output_node_properties = None if allow_remote and context.remote_execution_specs and context.remote_execution_specs.exec_spec: with SandboxRemote( context, project, plugin=self, stdout=stdout, stderr=stderr, config=config, output_node_properties=output_node_properties, ) as sandbox: yield sandbox else: platform = context.platform sandbox = platform.create_sandbox( context, project, plugin=self, stdout=stdout, stderr=stderr, config=config, output_node_properties=output_node_properties, ) with sandbox: yield sandbox # __initialize_from_yaml() # # Normal element initialization procedure. # def __initialize_from_yaml(self, load_element: "LoadElement", plugin_conf: Optional[str]): context = self._get_context() project = self._get_project() # Ensure we have loaded this class's defaults self.__init_defaults(project, plugin_conf, load_element.kind, load_element.first_pass) # Collect the composited variables and resolve them variables = self.__extract_variables(project, load_element) variables["element-name"] = self.name self.__variables = Variables(variables) if not load_element.first_pass: self.__variables.check() # Collect the composited environment now that we have variables unexpanded_env = self.__extract_environment(project, load_element) self.__variables.expand(unexpanded_env) self.__environment = unexpanded_env.strip_node_info() # Collect the environment nocache blacklist list nocache = self.__extract_env_nocache(project, load_element) self.__env_nocache = nocache # Grab public domain data declared for this instance self.__public = self.__extract_public(load_element) self.__variables.expand(self.__public) # Collect the composited element configuration and # ask the element to configure itself. self.__config = self.__extract_config(load_element) self.__variables.expand(self.__config) self._configure(self.__config) # Extract Sandbox config sandbox_config = self.__extract_sandbox_config(project, load_element) self.__variables.expand(sandbox_config) self.__sandbox_config = SandboxConfig.new_from_node(sandbox_config, platform=context.platform) # __initialize_from_artifact_key() # # Initialize the element state from an artifact key # def __initialize_from_artifact_key(self, key: str): # At this point we only know the key which was specified on the command line, # so we will pretend all keys are equal. # # If the artifact is cached, then the real keys will be loaded from the # artifact in `_load_artifact()` and `_load_artifact_done()`. # self.__cache_key = key self.__strict_cache_key = key self.__weak_cache_key = key self._initialize_state() # ArtifactElement requires access to the artifact early on to walk # dependencies. self._load_artifact(pull=False) if not self._cached(): # Remotes are not initialized when artifact elements are loaded. # Always consider pull pending if the artifact is not cached. self.__pull_pending = True else: self._load_artifact_done() @classmethod def __compose_default_splits(cls, project, defaults, first_pass): element_public = defaults.get_mapping(Symbol.PUBLIC, default={}) element_bst = element_public.get_mapping("bst", default={}) element_splits = element_bst.get_mapping("split-rules", default={}) if first_pass: splits = element_splits.clone() else: assert project.splits is not None splits = project.splits.clone() # Extend project wide split rules with any split rules defined by the element element_splits._composite(splits) element_bst["split-rules"] = splits element_public["bst"] = element_bst defaults[Symbol.PUBLIC] = element_public @classmethod def __init_defaults(cls, project, plugin_conf, kind, first_pass): # Defaults are loaded once per class and then reused # if cls.__defaults is None: defaults = Node.from_dict({}) if plugin_conf is not None: # Load the plugin's accompanying .yaml file if one was provided try: defaults = _yaml.load(plugin_conf, os.path.basename(plugin_conf)) except LoadError as e: if e.reason != LoadErrorReason.MISSING_FILE: raise e # Special case; compose any element-wide split-rules declarations cls.__compose_default_splits(project, defaults, first_pass) # Override the element's defaults with element specific # overrides from the project.conf if first_pass: elements = project.first_pass_config.element_overrides else: elements = project.element_overrides overrides = elements.get_mapping(kind, default=None) if overrides: overrides._composite(defaults) # Set the data class wide cls.__defaults = defaults # This will acquire the environment to be used when # creating sandboxes for this element # @classmethod def __extract_environment(cls, project, load_element): default_env = cls.__defaults.get_mapping(Symbol.ENVIRONMENT, default={}) element_env = load_element.node.get_mapping(Symbol.ENVIRONMENT, default={}) or Node.from_dict({}) if load_element.first_pass: environment = Node.from_dict({}) else: environment = project.base_environment.clone() default_env._composite(environment) element_env._composite(environment) environment._assert_fully_composited() return environment @classmethod def __extract_env_nocache(cls, project, load_element): if load_element.first_pass: project_nocache = [] else: project_nocache = project.base_env_nocache default_nocache = cls.__defaults.get_str_list(Symbol.ENV_NOCACHE, default=[]) element_nocache = load_element.node.get_str_list(Symbol.ENV_NOCACHE, default=[]) # Accumulate values from the element default, the project and the element # itself to form a complete list of nocache env vars. env_nocache = set(project_nocache + default_nocache + element_nocache) # Convert back to list now we know they're unique return list(env_nocache) # This will resolve the final variables to be used when # substituting command strings to be run in the sandbox # @classmethod def __extract_variables(cls, project, load_element): default_vars = cls.__defaults.get_mapping(Symbol.VARIABLES, default={}) element_vars = load_element.node.get_mapping(Symbol.VARIABLES, default={}) or Node.from_dict({}) if load_element.first_pass: variables = project.first_pass_config.base_variables.clone() else: variables = project.base_variables.clone() default_vars._composite(variables) element_vars._composite(variables) variables._assert_fully_composited() for var in ("project-name", "element-name", "max-jobs"): node = variables.get_node(var, allow_none=True) if node is None: continue provenance = node.get_provenance() if not provenance._is_synthetic: raise LoadError( "{}: invalid redefinition of protected variable '{}'".format(provenance, var), LoadErrorReason.PROTECTED_VARIABLE_REDEFINED, ) return variables # This will resolve the final configuration to be handed # off to element.configure() # @classmethod def __extract_config(cls, load_element): element_config = load_element.node.get_mapping(Symbol.CONFIG, default={}) or Node.from_dict({}) # The default config is already composited with the project overrides config = cls.__defaults.get_mapping(Symbol.CONFIG, default={}) config = config.clone() element_config._composite(config) config._assert_fully_composited() return config # Sandbox-specific configuration data, to be passed to the sandbox's constructor. # @classmethod def __extract_sandbox_config(cls, project, load_element): element_sandbox = load_element.node.get_mapping(Symbol.SANDBOX, default={}) or Node.from_dict({}) if load_element.first_pass: sandbox_config = Node.from_dict({}) else: sandbox_config = project.sandbox.clone() # The default config is already composited with the project overrides sandbox_defaults = cls.__defaults.get_mapping(Symbol.SANDBOX, default={}) sandbox_defaults = sandbox_defaults.clone() sandbox_defaults._composite(sandbox_config) element_sandbox._composite(sandbox_config) sandbox_config._assert_fully_composited() return sandbox_config # This makes a special exception for the split rules, which # elements may extend but whos defaults are defined in the project. # @classmethod def __extract_public(cls, load_element): element_public = load_element.node.get_mapping(Symbol.PUBLIC, default={}) or Node.from_dict({}) base_public = cls.__defaults.get_mapping(Symbol.PUBLIC, default={}) base_public = base_public.clone() base_bst = base_public.get_mapping("bst", default={}) base_splits = base_bst.get_mapping("split-rules", default={}) element_public = element_public.clone() element_bst = element_public.get_mapping("bst", default={}) element_splits = element_bst.get_mapping("split-rules", default={}) # Allow elements to extend the default splits defined in their project or # element specific defaults element_splits._composite(base_splits) element_bst["split-rules"] = base_splits element_public["bst"] = element_bst element_public._assert_fully_composited() return element_public def __init_splits(self): bstdata = self.get_public_data("bst") splits = bstdata.get_mapping("split-rules") self.__splits = { domain: re.compile( "^(?:" + "|".join([utils._glob2re(r) for r in rules.as_str_list()]) + ")$", re.MULTILINE | re.DOTALL ) for domain, rules in splits.items() } # __split_filter(): # # Returns True if the file with the specified `path` is included in the # specified split domains. This is used by `__split_filter_func()` to create # a filter callback. # # Args: # element_domains (list): All domains for this element # include (list): A list of domains to include files from # exclude (list): A list of domains to exclude files from # orphans (bool): Whether to include files not spoken for by split domains # path (str): The relative path of the file # # Returns: # (bool): Whether to include the specified file # def __split_filter(self, element_domains, include, exclude, orphans, path): # Absolute path is required for matching filename = os.path.join(os.sep, path) include_file = False exclude_file = False claimed_file = False for domain in element_domains: if self.__splits[domain].match(filename): claimed_file = True if domain in include: include_file = True if domain in exclude: exclude_file = True if orphans and not claimed_file: include_file = True return include_file and not exclude_file # __split_filter_func(): # # Returns callable split filter function for use with `copy_files()`, # `link_files()` or `Directory.import_files()`. # # Args: # include (list): An optional list of domains to include files from # exclude (list): An optional list of domains to exclude files from # orphans (bool): Whether to include files not spoken for by split domains # # Returns: # (callable): Filter callback that returns True if the file is included # in the specified split domains. # def __split_filter_func(self, include=None, exclude=None, orphans=True): # No splitting requested, no filter needed if orphans and not (include or exclude): return None if not self.__splits: self.__init_splits() element_domains = list(self.__splits.keys()) if not include: include = element_domains if not exclude: exclude = [] # Ignore domains that dont apply to this element # include = [domain for domain in include if domain in element_domains] exclude = [domain for domain in exclude if domain in element_domains] # The arguments element_domains, include, exclude, and orphans are # the same for all files. Use `partial` to create a function with # the required callback signature: a single `path` parameter. return partial(self.__split_filter, element_domains, include, exclude, orphans) def __compute_splits(self, include=None, exclude=None, orphans=True): filter_func = self.__split_filter_func(include=include, exclude=exclude, orphans=orphans) files_vdir = self.__artifact.get_files() element_files = files_vdir.list_relative_paths() if not filter_func: # No splitting requested, just report complete artifact yield from element_files else: for filename in element_files: if filter_func(filename): yield filename # __load_public_data(): # # Loads the public data from the cached artifact # def __load_public_data(self): self.__assert_cached() assert self.__dynamic_public is None self.__dynamic_public = self.__artifact.load_public_data() def __load_build_result(self): self.__assert_cached() assert self.__build_result is None self.__build_result = self.__artifact.load_build_result() # __update_cache_keys() # # Updates weak and strict cache keys # # Note that it does not update *all* cache keys - In non-strict mode, the # strong cache key is updated in __update_cache_key_non_strict() # # If the element is not resolved, this is a no-op (since inconsistent # elements cannot have cache keys). # # The weak and strict cache keys will be calculated if not already # set. # # The weak cache key is a cache key that doesn't change when the # content of the build dependency graph changes (e.g., the configurations # or source versions of each dependency in Scope.BUILD), but does # take the shape of the build dependency graph into account, such # that adding or removing a dependency will still cause a rebuild to # occur (except for the strict dependency and BST_STRICT_REBUILD element # cases which are treated specially below). # # The strict cache key is a cache key that changes if any dependencies # in Scope.BUILD has changed in any way. # def __update_cache_keys(self): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" if self.__strict_cache_key is not None: # Cache keys already calculated assert self.__weak_cache_key is not None return if not self._has_all_sources_resolved(): # Tracking may still be pending return # Calculate weak cache key first, as it is required for generating the other keys. # # This code can be run multiple times until the strict key can be calculated, # so let's ensure we only ever calculate the weak key once, even though we need # to resolve it before we can resolve the strict key. if self.__weak_cache_key is None: # Weak cache key includes names of direct build dependencies # so as to only trigger rebuilds when the shape of the # dependencies change. # # Some conditions cause dependencies to be strict, such # that this element will be rebuilt anyway if the dependency # changes even in non strict mode, for these cases we just # encode the dependency's weak cache key instead of it's name. # dependencies = [ [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)] if self.BST_STRICT_REBUILD or e in self.__strict_dependencies else [e.project_name, e.name] for e in self._dependencies(_Scope.BUILD) ] self.__weak_cache_key = self._calculate_cache_key(dependencies) context = self._get_context() # Calculate the strict cache key dependencies = [[e.project_name, e.name, e.__strict_cache_key] for e in self._dependencies(_Scope.BUILD)] self.__strict_cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key) if self.__strict_cache_key is None: # Cache keys cannot be calculated yet as a build dependency doesn't # have a cache key yet. return # As the strict cache key has already been calculated, it should always # be possible to calculate the weak cache key as well. assert self.__weak_cache_key is not None if context.get_strict(): # In strict mode, the strong cache key always matches the strict cache key self.__cache_key = self.__strict_cache_key # Update the message kwargs in use for this plugin to dispatch messages with # self._message_kwargs["element_key"] = self._get_display_key() # __update_cache_key_non_strict() # # Calculates the strong cache key if it hasn't already been set. # # When buildstream runs in strict mode, this is identical to the # strict cache key, so no work needs to be done. # # When buildstream is not run in strict mode, this requires the artifact # state (as set in Element._load_artifact()) to be set accordingly, # as the cache key can be loaded from the cache (possibly pulling from # a remote cache). # def __update_cache_key_non_strict(self): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" # The final cache key can be None here only in non-strict mode if self.__cache_key is None: if self._pull_pending(): # Effective strong cache key is unknown until after the pull pass elif self._cached(): # Load the strong cache key from the artifact strong_key, _, _ = self.__artifact.get_metadata_keys() self.__cache_key = strong_key elif self.__assemble_scheduled or self.__assemble_done: # Artifact will or has been built, not downloaded assert self.__weak_cache_key is not None dependencies = [[e.project_name, e.name, e._get_cache_key()] for e in self._dependencies(_Scope.BUILD)] self.__cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key) if self.__cache_key is None: # Strong cache key could not be calculated yet return # The Element may have just become ready for runtime now that the # strong cache key has just been set self._update_ready_for_runtime_and_cached() # Now we have the strong cache key, update the Artifact self.__artifact._cache_key = self.__cache_key # Update the message kwargs in use for this plugin to dispatch messages with self._message_kwargs["element_key"] = self._get_display_key() # _get_normal_name(): # # Get the element name without path separators or # the extension. # # Args: # element_name (str): The element's name # # Returns: # (str): The normalised element name # def _get_normal_name(element_name): return os.path.splitext(element_name.replace(os.sep, "-"))[0] # _compose_artifact_name(): # # Compose the completely resolved 'artifact_name' as a filepath # # Args: # project_name (str): The project's name # normal_name (str): The element's normalised name # cache_key (str): The relevant cache key # # Returns: # (str): The constructed artifact name path # def _compose_artifact_name(project_name, normal_name, cache_key): valid_chars = string.digits + string.ascii_letters + "-._" normal_name = "".join([x if x in valid_chars else "_" for x in normal_name]) # Note that project names are not allowed to contain slashes. Element names containing # a '/' will have this replaced with a '-' upon Element object instantiation. return "{0}/{1}/{2}".format(project_name, normal_name, cache_key) apache-buildstream-27ae392/src/buildstream/exceptions.py000066400000000000000000000105741514607367700234660ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Tiago Gomes """ Exceptions - API for Error Handling =================================== This module contains some Enums used in Error Handling which are useful in testing external plugins. """ from enum import Enum, unique @unique class ErrorDomain(Enum): """ErrorDomain Describes what the error is related to. """ PLUGIN = 1 LOAD = 2 IMPL = 3 PLATFORM = 4 SANDBOX = 5 ARTIFACT = 6 PIPELINE = 7 UTIL = 8 SOURCE = 9 ELEMENT = 10 APP = 11 STREAM = 12 VIRTUAL_FS = 13 CAS = 14 PROG_NOT_FOUND = 15 REMOTE = 16 PROFILE = 17 class LoadErrorReason(Enum): """LoadErrorReason Describes the reason why a :class:`.LoadError` was raised. """ MISSING_FILE = 1 """A file was not found.""" INVALID_YAML = 2 """The parsed data was not valid YAML.""" INVALID_DATA = 3 """Data was malformed, a value was not of the expected type, etc""" ILLEGAL_COMPOSITE = 4 """An error occurred during YAML dictionary composition. This can happen by overriding a value with a new differently typed value, or by overwriting some named value when that was not allowed. """ CIRCULAR_DEPENDENCY = 5 """A circular dependency chain was detected""" UNRESOLVED_VARIABLE = 6 """A variable could not be resolved. This can happen if your project has cyclic dependencies in variable declarations, or, when substituting a string which refers to an undefined variable. """ UNSUPPORTED_PROJECT = 7 """The project requires an incompatible BuildStream version""" UNSUPPORTED_PLUGIN = 8 """Project requires a newer version of a plugin than the one which was loaded """ EXPRESSION_FAILED = 9 """A conditional expression failed to resolve""" USER_ASSERTION = 10 """An assertion was intentionally encoded into project YAML""" TRAILING_LIST_DIRECTIVE = 11 """A list composition directive did not apply to any underlying list""" CONFLICTING_JUNCTION = 12 """Conflicting junctions in subprojects""" INVALID_JUNCTION = 13 """Failure to load a project from a specified junction""" SUBPROJECT_INCONSISTENT = 15 """Subproject has no ref""" INVALID_SYMBOL_NAME = 16 """An invalid symbol name was encountered""" MISSING_PROJECT_CONF = 17 """A project.conf file was missing""" LOADING_DIRECTORY = 18 """Try to load a directory not a yaml file""" PROJ_PATH_INVALID = 19 """A project path leads outside of the project directory""" PROJ_PATH_INVALID_KIND = 20 """A project path points to a file of the not right kind (e.g. a socket) """ RECURSIVE_INCLUDE = 21 """A recursive include has been encountered""" CIRCULAR_REFERENCE_VARIABLE = 22 """A circular variable reference was detected""" PROTECTED_VARIABLE_REDEFINED = 23 """An attempt was made to set the value of a protected variable""" INVALID_DEPENDENCY_CONFIG = 24 """An attempt was made to specify dependency configuration on an element which does not support custom dependency configuration""" LINK_FORBIDDEN_DEPENDENCIES = 25 """A link element declared dependencies""" CIRCULAR_REFERENCE = 26 """A circular element reference was detected""" BAD_ELEMENT_SUFFIX = 27 """ This warning will be produced when an element whose name does not end in .bst is referenced either on the command line or by another element """ BAD_CHARACTERS_IN_NAME = 28 """ This warning will be produced when a filename for a target contains invalid characters in its name. """ UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29 """ Thee source provenance attribute specified was not defined in the project config """ apache-buildstream-27ae392/src/buildstream/node.pxd000066400000000000000000000074061514607367700223750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Benjamin Schubert # Documentation for each class and method here can be found in the adjacent # implementation file (_yaml.pyx) cdef class Node: cdef int file_index cdef int line cdef int column # Public Methods cpdef Node clone(self) cpdef ProvenanceInformation get_provenance(self) cpdef object strip_node_info(self) # Private Methods used in BuildStream cpdef void _assert_fully_composited(self) except * # Protected Methods cdef void _compose_on(self, str key, MappingNode target, list path) except * cdef bint _is_composite_list(self) except * cdef bint _shares_position_with(self, Node target) cdef bint _walk_find(self, Node target, list path) except * cdef class MappingNode(Node): cdef dict value # Public Methods cpdef bint get_bool(self, str key, default=*) except * cpdef object get_enum(self, str key, object constraint, object default=*) cpdef object get_int(self, str key, default=*) cpdef MappingNode get_mapping(self, str key, default=*) cpdef Node get_node(self, str key, list allowed_types=*, bint allow_none=*) cpdef ScalarNode get_scalar(self, str key, default=*) cpdef SequenceNode get_sequence(self, str key, object default=*, list allowed_types=*) cpdef str get_str(self, str key, object default=*) cpdef list get_str_list(self, str key, object default=*) cpdef object items(self) cpdef list keys(self) cpdef void safe_del(self, str key) cpdef void validate_keys(self, list valid_keys) except * cpdef object values(self) # Private Methods used in BuildStream cpdef void _composite(self, MappingNode target) except * cpdef void _composite_under(self, MappingNode target) except * cpdef list _find(self, Node target) # Protected Methods cdef void _compose_on_composite_dict(self, MappingNode target) cdef void _compose_on_list(self, SequenceNode target) # Private Methods cdef void __composite(self, MappingNode target, list path) except * cdef Node _get(self, str key, default, default_constructor) cdef class ScalarNode(Node): cdef str value # Public Methods cpdef bint as_bool(self) except * cpdef object as_enum(self, object constraint) cpdef int as_int(self) except * cpdef str as_str(self) cpdef bint is_none(self) cdef class SequenceNode(Node): cdef list value # Public Methods cpdef void append(self, object value) cpdef list as_str_list(self) cpdef MappingNode mapping_at(self, int index) cpdef Node node_at(self, int index, list allowed_types=*) cpdef ScalarNode scalar_at(self, int index) cpdef SequenceNode sequence_at(self, int index) cdef class ProvenanceInformation: cdef readonly Node _node cdef readonly Node _toplevel cdef readonly _project cdef readonly bint _is_synthetic cdef readonly str _filename cdef readonly str _displayname cdef readonly str _shortname cdef readonly int _col cdef readonly int _line cdef int _SYNTHETIC_FILE_INDEX cdef Py_ssize_t _create_new_file(str filename, str shortname, str displayname, object project) cdef void _set_root_node_for_file(Py_ssize_t file_index, MappingNode contents) except * apache-buildstream-27ae392/src/buildstream/node.pyi000066400000000000000000000114071514607367700223770ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from typing import ( overload, Generic, Union, Iterable, Tuple, List, Mapping, Optional, Sequence, TypeVar, Type, Dict, Any, ) from ._project import Project TNode = TypeVar("TNode", bound="Node") TValidNodeValue = TypeVar("TValidNodeValue", int, str, bool, Mapping, Sequence) class ProvenanceInformation: ... class Node: def clone(self) -> "Node": ... def get_provenance(self) -> ProvenanceInformation: ... def strip_node_info(self) -> Dict[str, Any]: ... # FIXME: We should be able to annotate more specifically what is allowed # in the dictionary here, but this requires recursive type annotations # which appears to not yet be properly supported. # # See: https://github.com/python/mypy/issues/731 # @classmethod def from_dict(cls, value: Dict[str, Any]) -> "MappingNode": ... class ScalarNode(Node): def as_str(self) -> str: ... def clone(self) -> "ScalarNode": ... class SequenceNode(Node, Generic[TNode]): def __iter__(self) -> "SequenceNode": ... def __next__(self) -> Any: ... def as_str_list(self) -> List[str]: ... def clone(self) -> "SequenceNode[TNode]": ... class MappingNode(Node, Generic[TNode]): def __init__(self, file_index: int, line: int, column: int, value: Mapping[str, TValidNodeValue]) -> None: ... def __contains__(self, what: Any) -> bool: ... def clone(self) -> MappingNode[TNode]: ... def keys(self) -> Iterable[str]: ... def items(self) -> Iterable[Tuple[str, Any]]: ... def safe_del(self, key: str) -> None: ... def validate_keys(self, valid_keys: List[str]): ... def values(self) -> List[Node]: ... @overload def get_scalar(self, key: str) -> ScalarNode: ... @overload def get_scalar(self, key: str, default: Union[str, int, bool, None]) -> ScalarNode: ... @overload def get_bool(self, key: str) -> bool: ... @overload def get_bool(self, key: str, default: bool) -> bool: ... @overload def get_str_list(self, key: str) -> List[str]: ... @overload def get_str_list(self, key: str, default: List[str]) -> List[str]: ... @overload def get_str_list(self, key: str, default: Optional[List[str]]) -> Optional[List[str]]: ... @overload def get_str(self, key: str) -> str: ... @overload def get_str(self, key: str, default: str) -> str: ... @overload def get_str(self, key: str, default: Optional[str]) -> Optional[str]: ... @overload def get_int(self, key: str) -> int: ... @overload def get_int(self, key: str, default: int) -> int: ... @overload def get_int(self, key: str, default: Optional[int]) -> Optional[int]: ... @overload def get_enum(self, key: str, constraint: object) -> str: ... @overload def get_enum(self, key: str, constraint: object, default: Optional[object]) -> Optional[str]: ... @overload def get_mapping(self, key: str) -> "MappingNode": ... @overload def get_mapping(self, key: str, default: Union["MappingNode", Dict[str, Any]]) -> "MappingNode": ... @overload def get_mapping( self, key: str, default: Union["MappingNode", Dict[str, Any], None] ) -> Optional["MappingNode"]: ... @overload def get_sequence(self, key: str, *, allowed_types: Optional[List[Type[Node]]]) -> SequenceNode: ... @overload def get_sequence( self, key: str, default: List[Any], *, allowed_types: Optional[List[Type[Node]]] ) -> SequenceNode: ... @overload def get_sequence( self, key: str, default: Optional[List[Any]], *, allowed_types: Optional[List[Type[Node]]] ) -> Optional[SequenceNode]: ... @overload def get_node(self, key: str) -> Node: ... @overload def get_node(self, key: str, allowed_types: Optional[List[Type[Node]]]) -> Node: ... @overload def get_node(self, key: str, allowed_types: Optional[List[Type[Node]]], allow_none: bool) -> Optional[Node]: ... # # Private # def _composite(self, target: "MappingNode") -> None: ... def _assert_symbol_name( symbol_name: str, purpose: str, *, ref_node: Optional[Node], allow_dashes: bool = True ) -> None: ... def _new_synthetic_file(filename: str, project: Optional[Project]) -> MappingNode[TNode]: ... apache-buildstream-27ae392/src/buildstream/node.pyx000066400000000000000000001765311514607367700224300ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Daniel Silverstone # James Ennis # Benjamin Schubert """ Node - Parsed YAML configuration ================================ This module contains the building blocks for handling YAML configuration. Everything that is loaded from YAML is encapsulated in such nodes, which provide helper methods to validate configuration on access. Using node methods when reading configuration will ensure that errors are always coherently notified to the user. .. note:: Plugins are not expected to handle exceptions thrown by node methods for the above reason; They are private. There should always be a way to acquire information without resorting to exception handling. Node types ---------- The most important classes defined here are: * :class:`.MappingNode`: represents a YAML Mapping (dictionary) * :class:`.ScalarNode`: represents a YAML Scalar (string, boolean, integer) * :class:`.SequenceNode`: represents a YAML Sequence (list) Class Reference --------------- """ import string from ._exceptions import LoadError from .exceptions import LoadErrorReason # A sentinel to be used as a default argument for functions that need # to distinguish between a kwarg set to None and an unset kwarg. _sentinel = object() cdef class Node: """This is the base class for YAML document nodes. YAML Nodes contain information to describe the provenance of the YAML which resulted in the Node, allowing mapping back from a Node to the place in the file it came from. .. note:: You should never need to create a :class:`.Node` manually. If you do, you can create :class:`.Node` from dictionaries with :func:`Node.from_dict() `. If something else is needed, please open an issue. """ def __init__(self): raise NotImplementedError("Please do not construct nodes like this. Use Node.from_dict(dict) instead.") def __cinit__(self, int file_index, int line, int column, *args): self.file_index = file_index self.line = line self.column = column # This is in order to ensure we never add a `Node` to a cache key # as ujson will try to convert objects if they have a `__json__` # attribute. def __json__(self): raise ValueError("Nodes should not be allowed when jsonify-ing data", self) def __str__(self): return "{}: {}".format(self.get_provenance(), self.strip_node_info()) ############################################################# # Abstract Public Methods # ############################################################# cpdef Node clone(self): """Clone the node and return the copy. Returns: :class:`.Node`: a clone of the current node """ raise NotImplementedError() cpdef object strip_node_info(self): """ Remove all the node information (provenance) and return the underlying data as plain python objects Returns: (list, dict, str, None): the underlying data that was held in the node structure. """ raise NotImplementedError() ############################################################# # Public Methods # ############################################################# @classmethod def from_dict(cls, dict value): """from_dict(value) Create a new node from the given dictionary. This is a recursive operation, and will transform every value in the dictionary to a :class:`.Node` instance Valid values for keys are `str` Valid values for values are `list`, `dict`, `str`, `int`, `bool` or None. `list` and `dict` can also only contain such types. Args: value (dict): dictionary from which to create a node. Raises: :class:`TypeError`: when the value cannot be converted to a :class:`Node` Returns: :class:`.MappingNode`: a new mapping containing the value """ if value: return __new_node_from_dict(value, MappingNode.__new__( MappingNode, _SYNTHETIC_FILE_INDEX, 0, __next_synthetic_counter(), {})) else: # We got an empty dict, we can shortcut return MappingNode.__new__(MappingNode, _SYNTHETIC_FILE_INDEX, 0, __next_synthetic_counter(), {}) cpdef ProvenanceInformation get_provenance(self): """A convenience accessor to obtain the node's :class:`.ProvenanceInformation` The provenance information allows you to inform the user of where a node came. Transforming the information to a string will show the file, line and column in the file where the node is. An example usage would be: .. code-block:: python # With `config` being your node max_jobs_node = config.get_node('max-jobs') max_jobs = max_jobs_node.as_int() if max_jobs < 1: # We can't get a negative number of jobs raise LoadError("Error at {}: Max jobs needs to be >= 1".format( max_jobs_node.get_provenance() ) # Will print something like: # element.bst [line 4, col 7]: Max jobs needs to be >= 1 Returns: :class:`.ProvenanceInformation`: the provenance information for the node. """ return ProvenanceInformation(self) ############################################################# # Abstract Private Methods used in BuildStream # ############################################################# # _assert_fully_composited() # # This must be called on a fully loaded and composited node, # after all composition has completed. # # This checks that no more composition directives are present # in the data. # # Raises: # (LoadError): If any assertions fail # cpdef void _assert_fully_composited(self) except *: raise NotImplementedError() ############################################################# # Abstract Protected Methods # ############################################################# # _compose_on(key, target, path) # # Compose the current node on the given target. # # Args: # key (str): key on the target on which to compose the current value # target (.Node): target node on which to compose # path (list): path from the root of the target when composing recursively # in order to give accurate error reporting. # # Raises: # (_CompositeError): if an error is encountered during composition # cdef void _compose_on(self, str key, MappingNode target, list path) except *: raise NotImplementedError() # _is_composite_list # # Checks if the node is a Mapping with list composition # directives. # # Returns: # (bool): True if node was a Mapping containing only # list composition directives # # Raises: # (LoadError): If node was a mapping and contained a mix of # list composition directives and other keys # cdef bint _is_composite_list(self) except *: raise NotImplementedError() # _walk_find(target, path) # # Walk the node to search for `target`. # # When this returns `True`, the `path` argument will contain the full path # to the target from the root node. # # Args: # target (.Node): target to find in the node tree # path (list): current path from the root # # Returns: # (bool): whether the target was found in the tree or not # cdef bint _walk_find(self, Node target, list path) except *: raise NotImplementedError() ############################################################# # Protected Methods # ############################################################# # _shares_position_with(target) # # Check whether the current node is at the same position in its tree as the target. # # This is useful when we want to know if two nodes are 'identical', that is they # are at the exact same position in each respective tree, but do not necessarily # have the same content. # # Args: # target (.Node): the target to compare with the current node. # # Returns: # (bool): whether the two nodes share the same position # cdef bint _shares_position_with(self, Node target): return (self.file_index == target.file_index and self.line == target.line and self.column == target.column) cdef class ScalarNode(Node): """This class represents a Scalar (int, str, bool, None) in a YAML document. .. note:: If you need to store another type of scalars, please open an issue on the project. .. note:: You should never have to create a :class:`.ScalarNode` directly """ def __cinit__(self, int file_index, int line, int column, object value): cdef value_type = type(value) if value_type is str: value = value.strip() elif value_type is bool: if value: value = "True" else: value = "False" elif value_type is int: value = str(value) elif value is None: pass else: raise ValueError("ScalarNode can only hold str, int, bool or None objects") self.value = value def __reduce__(self): return ( ScalarNode.__new__, (ScalarNode, self.file_index, self.line, self.column, self.value), ) ############################################################# # Public Methods # ############################################################# cpdef bint as_bool(self) except *: """Get the value of the node as a boolean. .. note:: BuildStream treats the values 'True', 'true' and '1' as True, and the values 'False', 'false' and '0' as False. Any other string values (such as the valid YAML 'TRUE' or 'FALSE' will be considered as an error) Raises: :class:`buildstream._exceptions.LoadError`: if the value cannot be coerced to a bool correctly. Returns: :class:`bool`: the value contained in the node, as a boolean """ if type(self.value) is bool: return self.value # Don't coerce strings to booleans, this makes "False" strings evaluate to True if self.value in ('True', 'true', '1'): return True elif self.value in ('False', 'false', '0'): return False else: provenance = self.get_provenance() path = provenance._toplevel._find(self)[-1] raise LoadError("{}: Value of '{}' is not of the expected type 'boolean'" .format(provenance, path), LoadErrorReason.INVALID_DATA) cpdef object as_enum(self, object constraint): """Get the value of the node as an enum member from `constraint` The constraint must be a :class:`buildstream.types.FastEnum` or a plain python Enum. For example you could do: .. code-block:: python from buildstream.types import FastEnum class SupportedCompressions(FastEnum): NONE = "none" GZIP = "gzip" XZ = "xz" x = config.get_scalar('compress').as_enum(SupportedCompressions) if x == SupportedCompressions.GZIP: print("Using GZIP") Args: constraint (:class:`buildstream.types.FastEnum` or :class:`Enum`): an enum from which to extract the value for the current node. Returns: :class:`FastEnum` or :class:`Enum`: the value contained in the node, as a member of `constraint` """ try: return constraint(self.value) except ValueError: provenance = self.get_provenance() path = provenance._toplevel._find(self)[-1] valid_values = [str(v.value) for v in constraint] raise LoadError("{}: Value of '{}' should be one of '{}'".format( provenance, path, ", ".join(valid_values)), LoadErrorReason.INVALID_DATA) cpdef int as_int(self) except *: """Get the value of the node as an integer. Raises: :class:`buildstream._exceptions.LoadError`: if the value cannot be coerced to an integer correctly. Returns: :class:`int`: the value contained in the node, as a integer """ try: return int(self.value) except ValueError: provenance = self.get_provenance() path = provenance._toplevel._find(self)[-1] raise LoadError("{}: Value of '{}' is not of the expected type '{}'" .format(provenance, path, int.__name__), LoadErrorReason.INVALID_DATA) cpdef str as_str(self): """Get the value of the node as a string. Returns: :class:`str`: the value contained in the node, as a string, or `None` if the content is `None`. """ # We keep 'None' as 'None' to simplify the API's usage and allow chaining for users if self.value is None: return None return str(self.value) cpdef bint is_none(self): """Determine whether the current scalar is `None`. Returns: :class:`bool`: `True` if the value of the scalar is `None`, else `False` """ return self.value is None ############################################################# # Public Methods implementations # ############################################################# cpdef ScalarNode clone(self): return ScalarNode.__new__( ScalarNode, self.file_index, self.line, self.column, self.value ) cpdef object strip_node_info(self): return self.value ############################################################# # Private Methods implementations # ############################################################# cpdef void _assert_fully_composited(self) except *: pass ############################################################# # Protected Methods # ############################################################# cdef void _compose_on(self, str key, MappingNode target, list path) except *: cdef Node target_value = target.value.get(key) if target_value is not None and type(target_value) is not ScalarNode: raise __CompositeError(path, "{}: Cannot compose scalar on non-scalar at {}".format( self.get_provenance(), target_value.get_provenance())) target.value[key] = self.clone() cdef bint _is_composite_list(self) except *: return False cdef bint _walk_find(self, Node target, list path) except *: return self._shares_position_with(target) cdef class MappingNode(Node): """This class represents a Mapping (dict) in a YAML document. It behaves mostly like a :class:`dict`, but doesn't allow untyped value access (Nothing of the form :code:`my_dict[my_value]`. It also doesn't allow anything else than :class:`str` as keys, to align with YAML. You can however use common dict operations in it: .. code-block:: python # Assign a new value to a key my_mapping[key] = my_value # Delete an entry del my_mapping[key] When assigning a key/value pair, the key must be a string, and the value can be any of: * a :class:`Node`, in which case the node is just assigned like normally * a :class:`list`, :class:`dict`, :class:`int`, :class:`str`, :class:`bool` or :class:`None`. In which case, the value will be converted to a :class:`Node` for you. Therefore, all values in a :class:`.MappingNode` will be :class:`Node`. .. note:: You should never create an instance directly. Use :func:`Node.from_dict() ` instead, which will ensure your node is correctly formatted. """ def __cinit__(self, int file_index, int line, int column, dict value): self.value = value def __reduce__(self): return ( MappingNode.__new__, (MappingNode, self.file_index, self.line, self.column, self.value), ) def __contains__(self, what): return what in self.value def __delitem__(self, str key): del self.value[key] def __setitem__(self, str key, object value): cdef Node old_value if type(value) in [MappingNode, ScalarNode, SequenceNode]: self.value[key] = value else: node = __create_node_recursive(value, self) # FIXME: Do we really want to override provenance? # # Related to https://gitlab.com/BuildStream/buildstream/issues/1058 # # There are only two cases were nodes are set in the code (hence without provenance): # - When automatic variables are set by the core (e-g: max-jobs) # - when plugins call Element.set_public_data # # The first case should never throw errors, so it is of limited interests. # # The second is more important. What should probably be done here is to have 'set_public_data' # able of creating a fake provenance with the name of the plugin, the project and probably the # element name. # # We would therefore have much better error messages, and would be able to get rid of most synthetic # nodes. old_value = self.value.get(key) if old_value: node.file_index = old_value.file_index node.line = old_value.line node.column = old_value.column self.value[key] = node ############################################################# # Public Methods # ############################################################# cpdef bint get_bool(self, str key, object default=_sentinel) except *: """get_bool(key, default=sentinel) Get the value of the node for `key` as a boolean. This is equivalent to: :code:`mapping.get_scalar(my_key, my_default).as_bool()`. Args: key (str): key for which to get the value default (bool): default value to return if `key` is not in the mapping Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.ScalarNode` or isn't a valid `boolean` Returns: :class:`bool`: the value at `key` or the default """ cdef ScalarNode scalar = self.get_scalar(key, default) return scalar.as_bool() cpdef object get_enum(self, str key, object constraint, object default=_sentinel): """Get the value of the node as an enum member from `constraint` Args: key (str): key for which to get the value constraint (:class:`buildstream.types.FastEnum` or :class:`Enum`): an enum from which to extract the value for the current node. default (object): default value to return if `key` is not in the mapping Raises: :class:`buildstream._exceptions.LoadError`: if the value is not is not found or not part of the provided enum. Returns: :class:`buildstream.types.Enum` or :class:`Enum`: the value contained in the node, as a member of `constraint` """ cdef object value = self.value.get(key, _sentinel) if value is _sentinel: if default is _sentinel: provenance = self.get_provenance() raise LoadError("{}: Dictionary did not contain expected key '{}'".format(provenance, key), LoadErrorReason.INVALID_DATA) if default is None: return None else: return constraint(default) if type(value) is not ScalarNode: provenance = value.get_provenance() raise LoadError("{}: Value of '{}' is not of the expected type 'scalar'" .format(provenance, key), LoadErrorReason.INVALID_DATA) return ( value).as_enum(constraint) cpdef object get_int(self, str key, object default=_sentinel): """get_int(key, default=sentinel) Get the value of the node for `key` as an integer. This is equivalent to: :code:`mapping.get_scalar(my_key, my_default).as_int()`. Args: key (str): key for which to get the value default (int, None): default value to return if `key` is not in the mapping Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.ScalarNode` or isn't a valid `integer` Returns: :class:`int` or :class:`None`: the value at `key` or the default """ cdef ScalarNode scalar = self.get_scalar(key, default) if default is None and scalar.is_none(): return None return scalar.as_int() cpdef MappingNode get_mapping(self, str key, object default=_sentinel): """get_mapping(key, default=sentinel) Get the value of the node for `key` as a :class:`.MappingNode`. Args: key (str): key for which to get the value default (dict): default value to return if `key` is not in the mapping. It will be converted to a :class:`.MappingNode` before being returned Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.MappingNode` Returns: :class:`.MappingNode`: the value at `key` or the default """ value = self._get(key, default, MappingNode) if type(value) is not MappingNode and value is not None: provenance = value.get_provenance() raise LoadError("{}: Value of '{}' is not of the expected type 'dict'" .format(provenance, key), LoadErrorReason.INVALID_DATA) return value cpdef Node get_node(self, str key, list allowed_types = None, bint allow_none = False): """get_node(key, allowed_types=None, allow_none=False) Get the value of the node for `key` as a :class:`.Node`. This is useful if you have configuration that can be either a :class:`.ScalarNode` or a :class:`.MappingNode` for example. This method will validate that the value is indeed exactly one of those types (not a subclass) and raise an exception accordingly. Args: key (str): key for which to get the value allowed_types (list): list of valid subtypes of :class:`.Node` that are valid return values. If this is `None`, no checks are done on the return value. allow_none (bool): whether to allow the return value to be `None` or not Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not one of the expected types or if it doesn't exist. Returns: :class:`.Node`: the value at `key` or `None` """ cdef value = self.value.get(key, _sentinel) if value is _sentinel: if allow_none: return None provenance = self.get_provenance() raise LoadError("{}: Dictionary did not contain expected key '{}'".format(provenance, key), LoadErrorReason.INVALID_DATA) __validate_node_type(value, allowed_types, key) return value cpdef ScalarNode get_scalar(self, str key, object default=_sentinel): """get_scalar(key, default=sentinel) Get the value of the node for `key` as a :class:`.ScalarNode`. Args: key (str): key for which to get the value default (str, int, bool, None): default value to return if `key` is not in the mapping. It will be converted to a :class:`.ScalarNode` before being returned. Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.MappingNode` Returns: :class:`.ScalarNode`: the value at `key` or the default """ value = self._get(key, default, ScalarNode) if type(value) is not ScalarNode: if value is None: value = ScalarNode.__new__(ScalarNode, self.file_index, 0, __next_synthetic_counter(), None) else: provenance = value.get_provenance() raise LoadError("{}: Value of '{}' is not of the expected type 'scalar'" .format(provenance, key), LoadErrorReason.INVALID_DATA) return value cpdef SequenceNode get_sequence(self, str key, object default=_sentinel, list allowed_types = None): """get_sequence(key, default=sentinel) Get the value of the node for `key` as a :class:`.SequenceNode`. Args: key (str): key for which to get the value default (list): default value to return if `key` is not in the mapping. It will be converted to a :class:`.SequenceNode` before being returned allowed_types (list): list of valid subtypes of :class:`.Node` that are valid for nodes in the sequence. Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.SequenceNode` Returns: :class:`.SequenceNode`: the value at `key` or the default """ cdef Node value = self._get(key, default, SequenceNode) cdef Node node if type(value) is not SequenceNode and value is not None: provenance = value.get_provenance() raise LoadError("{}: Value of '{}' is not of the expected type 'list'" .format(provenance, key), LoadErrorReason.INVALID_DATA) if allowed_types: for node in value: __validate_node_type(node, allowed_types) return value cpdef str get_str(self, str key, object default=_sentinel): """get_str(key, default=sentinel) Get the value of the node for `key` as an string. This is equivalent to: :code:`mapping.get_scalar(my_key, my_default).as_str()`. Args: key (str): key for which to get the value default (str): default value to return if `key` is not in the mapping Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.ScalarNode` or isn't a valid `str` Returns: :class:`str`: the value at `key` or the default """ cdef ScalarNode scalar = self.get_scalar(key, default) return scalar.as_str() cpdef list get_str_list(self, str key, object default=_sentinel): """get_str_list(key, default=sentinel) Get the value of the node for `key` as a list of strings. This is equivalent to: :code:`mapping.get_sequence(my_key, my_default).as_str_list()`. Args: key (str): key for which to get the value default (str): default value to return if `key` is not in the mapping Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.SequenceNode` or if any of its internal values is not a ScalarNode. Returns: :class:`list`: the value at `key` or the default """ cdef SequenceNode sequence = self.get_sequence(key, default) if sequence is not None: return sequence.as_str_list() return None cpdef object items(self): """Get a new view of the mapping items ((key, value) pairs). This is equivalent to running :code:`my_dict.item()` on a `dict`. Returns: :class:`dict_items`: a view on the underlying dictionary """ return self.value.items() cpdef list keys(self): """Get the list of all keys in the mapping. This is equivalent to running :code:`my_dict.keys()` on a `dict`. Returns: :class:`list`: a list of all keys in the mapping """ return list(self.value.keys()) cpdef void safe_del(self, str key): """safe_del(key) Remove the entry at `key` in the dictionary if it exists. This method is a safe equivalent to :code:`del mapping[key]`, that doesn't throw anything if the key doesn't exist. Args: key (str): key to remove from the mapping """ self.value.pop(key, None) cpdef void validate_keys(self, list valid_keys) except *: """validate_keys(valid_keys) Validate that the node doesn't contain extra keys This validates the node so as to ensure the user has not specified any keys which are unrecognized by BuildStream (usually this means a typo which would otherwise not trigger an error). Args: valid_keys (list): A list of valid keys for the specified node Raises: :class:`buildstream._exceptions.LoadError`: In the case that the specified node contained one or more invalid keys """ # Probably the fastest way to do this: https://stackoverflow.com/a/23062482 cdef set valid_keys_set = set(valid_keys) cdef str key for key in self.value: if key not in valid_keys_set: provenance = self.get_node(key).get_provenance() raise LoadError("{}: Unexpected key: {}".format(provenance, key), LoadErrorReason.INVALID_DATA) cpdef object values(self): """Get the values in the mapping. This is equivalent to running :code:`my_dict.values()` on a `dict`. Returns: :class:`dict_values`: a list of all values in the mapping """ return self.value.values() ############################################################# # Public Methods implementations # ############################################################# cpdef MappingNode clone(self): cdef dict copy = {} cdef str key cdef Node value for key, value in self.value.items(): copy[key] = value.clone() return MappingNode.__new__(MappingNode, self.file_index, self.line, self.column, copy) cpdef object strip_node_info(self): cdef str key cdef Node value return {key: value.strip_node_info() for key, value in self.value.items()} ############################################################# # Private Methods used in BuildStream # ############################################################# # _composite() # # Compose one mapping node onto another # # Args: # target (Node): The target to compose into # # Raises: LoadError # cpdef void _composite(self, MappingNode target) except *: try: self.__composite(target, []) except __CompositeError as e: source_provenance = self.get_provenance() error_prefix = "" if source_provenance: error_prefix = "{}: ".format(source_provenance) raise LoadError("{}Failure composing {}: {}" .format(error_prefix, e.path, e.message), LoadErrorReason.ILLEGAL_COMPOSITE) from e # Like self._composite(target), but where values in the target don't get overridden by values in self. # cpdef void _composite_under(self, MappingNode target) except *: target._composite(self) cdef str key cdef Node value cdef list to_delete = [key for key in target.value.keys() if key not in self.value] for key, value in self.value.items(): target.value[key] = value for key in to_delete: del target.value[key] # _find() # # Searches the given node tree for the given target node. # # This is typically used when trying to walk a path to a given node # for the purpose of then modifying a similar tree of objects elsewhere # # Args: # target (Node): The node you are looking for in that tree # # Returns: # (list): A path from `node` to `target` or None if `target` is not in the subtree cpdef list _find(self, Node target): cdef list path = [] if self._walk_find(target, path): return path return None ############################################################# # Private Methods implementations # ############################################################# cpdef void _assert_fully_composited(self) except *: cdef str key cdef Node value for key, value in self.value.items(): # Assert that list composition directives dont remain, this # indicates that the user intended to override a list which # never existed in the underlying data # if key in ('(>)', '(<)', '(=)'): provenance = value.get_provenance() raise LoadError("{}: Attempt to override non-existing list".format(provenance), LoadErrorReason.TRAILING_LIST_DIRECTIVE) value._assert_fully_composited() ############################################################# # Protected Methods # ############################################################# cdef void _compose_on(self, str key, MappingNode target, list path) except *: cdef Node target_value if self._is_composite_list(): if key not in target.value: # Composite list clobbers empty space target.value[key] = self.clone() else: target_value = target.value[key] if type(target_value) is SequenceNode: # Composite list composes into a list self._compose_on_list(target_value) elif target_value._is_composite_list(): # Composite list merges into composite list self._compose_on_composite_dict(target_value) else: # Else composing on top of normal dict or a scalar, so raise... raise __CompositeError(path, "{}: Cannot compose lists onto {}".format( self.get_provenance(), target_value.get_provenance())) else: # We're composing a dict into target now if key not in target.value: # Target lacks a dict at that point, make a fresh one with # the same provenance as the incoming dict target.value[key] = MappingNode.__new__(MappingNode, self.file_index, self.line, self.column, {}) self.__composite(target.value[key], path) # _compose_on_list(target) # # Compose the current node on the given sequence. # # Args: # target (.SequenceNode): sequence on which to compose the current composite dict # cdef void _compose_on_list(self, SequenceNode target): cdef SequenceNode clobber = self.value.get("(=)") cdef SequenceNode prefix = self.value.get("(<)") cdef SequenceNode suffix = self.value.get("(>)") if clobber is not None: target.value.clear() target.value.extend(clobber.value) if prefix is not None: for v in reversed(prefix.value): target.value.insert(0, v) if suffix is not None: target.value.extend(suffix.value) # _compose_on_composite_dict(target) # # Compose the current node on the given composite dict. # # A composite dict is a dict that contains composition directives. # # Args: # target (.MappingNode): sequence on which to compose the current composite dict # cdef void _compose_on_composite_dict(self, MappingNode target): cdef SequenceNode clobber = self.value.get("(=)") cdef SequenceNode prefix = self.value.get("(<)") cdef SequenceNode suffix = self.value.get("(>)") if clobber is not None: # We want to clobber the target list # which basically means replacing the target list # with ourselves target.value["(=)"] = clobber if prefix is not None: target.value["(<)"] = prefix elif "(<)" in target.value: ( target.value["(<)"]).value.clear() if suffix is not None: target.value["(>)"] = suffix elif "(>)" in target.value: ( target.value["(>)"]).value.clear() else: # Not clobbering, so prefix the prefix and suffix the suffix if prefix is not None: if "(<)" in target.value: for v in reversed(prefix.value): ( target.value["(<)"]).value.insert(0, v) else: target.value["(<)"] = prefix if suffix is not None: if "(>)" in target.value: ( target.value["(>)"]).value.extend(suffix.value) else: target.value["(>)"] = suffix cdef bint _is_composite_list(self) except *: cdef bint has_directives = False cdef bint has_keys = False cdef str key for key in self.value.keys(): if key in ["(>)", "(<)", "(=)"]: has_directives = True else: has_keys = True if has_keys and has_directives: provenance = self.get_provenance() raise LoadError("{}: Dictionary contains list composition directives and arbitrary keys" .format(provenance), LoadErrorReason.INVALID_DATA) return has_directives cdef bint _walk_find(self, Node target, list path) except *: cdef str k cdef Node v if self._shares_position_with(target): return True for k, v in self.value.items(): path.append(k) if v._walk_find(target, path): return True del path[-1] return False ############################################################# # Private Methods # ############################################################# # __composite(target, path) # # Helper method to compose the current node on another. # # Args: # target (.MappingNode): target on which to compose the current node # path (list): path from the root of the target when composing recursively # in order to give accurate error reporting. # cdef void __composite(self, MappingNode target, list path) except *: cdef str key cdef Node value for key, value in self.value.items(): path.append(key) value._compose_on(key, target, path) path.pop() # Clobber the provenance of the target mapping node if we're not # synthetic. if self.file_index != _SYNTHETIC_FILE_INDEX: target.file_index = self.file_index target.line = self.line target.column = self.column # _get(key, default, default_constructor) # # Internal helper method to get an entry from the underlying dictionary. # # Args: # key (str): the key for which to retrieve the entry # default (object): default value if the entry is not present # default_constructor (object): method to transform the `default` into a Node # if the entry is not present # # Raises: # (LoadError): if the key is not present and no default has been given. # cdef Node _get(self, str key, object default, object default_constructor): value = self.value.get(key, _sentinel) if value is _sentinel: if default is _sentinel: provenance = self.get_provenance() raise LoadError("{}: Dictionary did not contain expected key '{}'".format(provenance, key), LoadErrorReason.INVALID_DATA) if default is None: value = None else: value = default_constructor.__new__( default_constructor, _SYNTHETIC_FILE_INDEX, 0, __next_synthetic_counter(), default) return value cdef class SequenceNode(Node): """This class represents a Sequence (list) in a YAML document. It behaves mostly like a :class:`list`, but doesn't allow untyped value access (Nothing of the form :code:`my_list[my_value]`). You can however perform common list operations on it: .. code-block:: python # Assign a value my_sequence[key] = value # Get the length len(my_sequence) # Reverse it reversed(my_sequence) # And iter over it for value in my_sequence: print(value) All values in a :class:`SequenceNode` will be :class:`Node`. """ def __cinit__(self, int file_index, int line, int column, list value): self.value = value def __reduce__(self): return ( SequenceNode.__new__, (SequenceNode, self.file_index, self.line, self.column, self.value), ) def __iter__(self): return iter(self.value) def __len__(self): return len(self.value) def __reversed__(self): return reversed(self.value) def __setitem__(self, int key, object value): cdef Node old_value if type(value) in [MappingNode, ScalarNode, SequenceNode]: self.value[key] = value else: node = __create_node_recursive(value, self) # FIXME: Do we really want to override provenance? # See __setitem__ on 'MappingNode' for more context old_value = self.value[key] if old_value: node.file_index = old_value.file_index node.line = old_value.line node.column = old_value.column self.value[key] = node ############################################################# # Public Methods # ############################################################# cpdef void append(self, object value): """append(value) Append the given object to the sequence. Args: value (object): the value to append to the list. This can either be: - a :class:`Node` - a :class:`int`, :class:`bool`, :class:`str`, :class:`None`, :class:`dict` or :class:`list`. In which case, this will be converted into a :class:`Node` beforehand Raises: :class:`TypeError`: when the value cannot be converted to a :class:`Node` """ if type(value) in [MappingNode, ScalarNode, SequenceNode]: self.value.append(value) else: node = __create_node_recursive(value, self) self.value.append(node) cpdef list as_str_list(self): """Get the values of the sequence as a list of strings. Raises: :class:`buildstream._exceptions.LoadError`: if the sequence contains more than :class:`ScalarNode` Returns: :class:`list`: the content of the sequence as a list of strings """ cdef list str_list = [] cdef Node node for node in self.value: if type(node) is not ScalarNode: provenance = node.get_provenance() raise LoadError("{}: List item is not of the expected type 'scalar'" .format(provenance), LoadErrorReason.INVALID_DATA) str_list.append(node.as_str()) return str_list cpdef MappingNode mapping_at(self, int index): """mapping_at(index) Retrieve the entry at `index` as a :class:`.MappingNode`. Args: index (int): index for which to get the value Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.MappingNode` :class:`IndexError`: if no value exists at this index Returns: :class:`.MappingNode`: the value at `index` """ value = self.value[index] if type(value) is not MappingNode: provenance = self.get_provenance() path = ["[{}]".format(p) for p in provenance._toplevel._find(self)] + ["[{}]".format(index)] raise LoadError("{}: Value of '{}' is not of the expected type '{}'" .format(provenance, path, MappingNode.__name__), LoadErrorReason.INVALID_DATA) return value cpdef Node node_at(self, int index, list allowed_types = None): """node_at(index, allowed_types=None) Retrieve the entry at `index` as a :class:`.Node`. This is useful if you have configuration that can be either a :class:`.ScalarNode` or a :class:`.MappingNode` for example. This method will validate that the value is indeed exactly one of those types (not a subclass) and raise an exception accordingly. Args: index (int): index for which to get the value allowed_types (list): list of valid subtypes of :class:`.Node` that are valid return values. If this is `None`, no checks are done on the return value. Raises: :class:`buildstream._exceptions.LoadError`: if the value at `index` is not of one of the expected types :class:`IndexError`: if no value exists at this index Returns: :class:`.Node`: the value at `index` """ cdef value = self.value[index] __validate_node_type(value, allowed_types, str(index)) return value cpdef ScalarNode scalar_at(self, int index): """scalar_at(index) Retrieve the entry at `index` as a :class:`.ScalarNode`. Args: index (int): index for which to get the value Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.ScalarNode` :class:`IndexError`: if no value exists at this index Returns: :class:`.ScalarNode`: the value at `index` """ value = self.value[index] if type(value) is not ScalarNode: provenance = self.get_provenance() path = ["[{}]".format(p) for p in provenance._toplevel._find(self)] + ["[{}]".format(index)] raise LoadError("{}: Value of '{}' is not of the expected type '{}'" .format(provenance, path, ScalarNode.__name__), LoadErrorReason.INVALID_DATA) return value cpdef SequenceNode sequence_at(self, int index): """sequence_at(index) Retrieve the entry at `index` as a :class:`.SequenceNode`. Args: index (int): index for which to get the value Raises: :class:`buildstream._exceptions.LoadError`: if the value at `key` is not a :class:`.SequenceNode` :class:`IndexError`: if no value exists at this index Returns: :class:`.SequenceNode`: the value at `index` """ value = self.value[index] if type(value) is not SequenceNode: provenance = self.get_provenance() path = ["[{}]".format(p) for p in provenance.toplevel._find(self)] + ["[{}]".format(index)] raise LoadError("{}: Value of '{}' is not of the expected type '{}'" .format(provenance, path, SequenceNode.__name__), LoadErrorReason.INVALID_DATA) return value ############################################################# # Public Methods implementations # ############################################################# cpdef SequenceNode clone(self): cdef list copy = [] cdef Node entry for entry in self.value: copy.append(entry.clone()) return SequenceNode.__new__(SequenceNode, self.file_index, self.line, self.column, copy) cpdef object strip_node_info(self): cdef Node value return [value.strip_node_info() for value in self.value] ############################################################# # Private Methods implementations # ############################################################# cpdef void _assert_fully_composited(self) except *: cdef Node value for value in self.value: value._assert_fully_composited() ############################################################# # Protected Methods # ############################################################# cdef void _compose_on(self, str key, MappingNode target, list path) except *: # List clobbers anything list-like cdef Node target_value = target.value.get(key) if not (target_value is None or type(target_value) is SequenceNode or target_value._is_composite_list()): raise __CompositeError(path, "{}: List cannot overwrite {} at: {}" .format(self.get_provenance(), key, target_value.get_provenance())) # If the target is a list of conditional statements, then we are # also conditional statements, and we need to append ourselves # to that list instead of overwriting it in order to preserve the # conditional for later evaluation. if type(target_value) is SequenceNode and key == "(?)": ( target.value[key]).value.extend(self.value) else: # Looks good, clobber it target.value[key] = self.clone() cdef bint _is_composite_list(self) except *: return False cdef bint _walk_find(self, Node target, list path) except *: cdef int i cdef Node v if self._shares_position_with(target): return True for i, v in enumerate(self.value): path.append(i) if v._walk_find(target, path): return True del path[-1] return False # Returned from Node.get_provenance cdef class ProvenanceInformation: """Represents the location of a YAML node in a file. This can effectively be used as a pretty print to display those information in errors consistently. You can retrieve this information for a :class:`Node` with :func:`Node.get_provenance() ` """ def __init__(self, Node nodeish): cdef __FileInfo fileinfo self._node = nodeish if (nodeish is None) or (nodeish.file_index == _SYNTHETIC_FILE_INDEX): self._filename = "" self._shortname = "" self._displayname = "" self._line = 1 self._col = 0 self._toplevel = None self._project = None else: fileinfo = <__FileInfo> __FILE_LIST[nodeish.file_index] self._filename = fileinfo.filename self._shortname = fileinfo.shortname self._displayname = fileinfo.displayname # We add 1 here to convert from computerish to humanish self._line = nodeish.line + 1 self._col = nodeish.column self._toplevel = fileinfo.toplevel self._project = fileinfo.project self._is_synthetic = (self._filename == '') or (self._col < 0) # Convert a Provenance to a string for error reporting def __str__(self): if self._is_synthetic: return "{} [synthetic node]".format(self._displayname) else: return "{} [line {:d} column {:d}]".format(self._displayname, self._line, self._col) ############################################################# # BuildStream Private methods # ############################################################# # Purely synthetic nodes will have _SYNTHETIC_FILE_INDEX for the file number, have line number # zero, and a negative column number which comes from inverting the next value # out of this counter. Synthetic nodes created with a reference node will # have a file number from the reference node, some unknown line number, and # a negative column number from this counter. cdef int _SYNTHETIC_FILE_INDEX = -1 # _assert_symbol_name() # # A helper function to check if a loaded string is a valid symbol # name and to raise a consistent LoadError if not. For strings which # are required to be symbols. # # Args: # symbol_name (str): The loaded symbol name # purpose (str): The purpose of the string, for an error message # ref_node (Node): The node of the loaded symbol, or None # allow_dashes (bool): Whether dashes are allowed for this symbol # # Raises: # LoadError: If the symbol_name is invalid # # Note that dashes are generally preferred for variable names and # usage in YAML, but things such as option names which will be # evaluated with jinja2 cannot use dashes. def _assert_symbol_name(str symbol_name, str purpose, *, Node ref_node=None, bint allow_dashes=True): cdef str valid_chars = string.digits + string.ascii_letters + '_' if allow_dashes: valid_chars += '-' cdef bint valid = True if not symbol_name: valid = False elif any(x not in valid_chars for x in symbol_name): valid = False elif symbol_name[0] in string.digits: valid = False if not valid: detail = "Symbol names must contain only alphanumeric characters, " + \ "may not start with a digit, and may contain underscores" if allow_dashes: detail += " or dashes" message = "Invalid symbol name for {}: '{}'".format(purpose, symbol_name) if ref_node: provenance = ref_node.get_provenance() if provenance is not None: message = "{}: {}".format(provenance, message) raise LoadError(message, LoadErrorReason.INVALID_SYMBOL_NAME, detail=detail) # _create_new_file(filename, shortname, displayname, toplevel, project) # # Create a new synthetic file and return it's index in the `._FILE_LIST`. # # Args: # filename (str): the name to give to the file # shortname (str): a shorter name used when showing information on the screen # displayname (str): the name to give when reporting errors # project (object): project with which to associate the current file (when dealing with junctions) # # Returns: # (int): the index in the `._FILE_LIST` that identifies the new file # cdef Py_ssize_t _create_new_file(str filename, str shortname, str displayname, object project): cdef Py_ssize_t file_number = len(__FILE_LIST) __FILE_LIST.append(__FileInfo(filename, shortname, displayname, None, project)) return file_number # _set_root_node_for_file(file_index, contents) # # Set the root node for the given file # # Args: # file_index (int): the index in the `._FILE_LIST` for the file for which to set the root # contents (.MappingNode): node that should be the root for the file # cdef void _set_root_node_for_file(Py_ssize_t file_index, MappingNode contents) except *: cdef __FileInfo f_info if file_index != _SYNTHETIC_FILE_INDEX: f_info = <__FileInfo> __FILE_LIST[file_index] f_info.toplevel = contents # _new_synthetic_file() # # Create a new synthetic mapping node, with an associated file entry # (in _FILE_LIST) such that later tracking can correctly determine which # file needs writing to in order to persist the changes. # # Args: # filename (str): The name of the synthetic file to create # project (Project): The optional project to associate this synthetic file with # # Returns: # (Node): An empty YAML mapping node, whose provenance is to this new # synthetic file # def _new_synthetic_file(str filename, object project=None): cdef Py_ssize_t file_index = len(__FILE_LIST) cdef Node node = MappingNode.__new__(MappingNode, file_index, 0, 0, {}) __FILE_LIST.append(__FileInfo(filename, filename, "".format(filename), node, project)) return node # _reset_global_state() # # This resets the global variables __FILE_LIST and __counter to their initial # state. This is used by the test suite to improve isolation between tests # running in the same process. # def _reset_global_state(): global __FILE_LIST, __counter __FILE_LIST = [] __counter = 0 ############################################################# # Module local helper Methods # ############################################################# # File name handling cdef list __FILE_LIST = [] # synthetic counter for synthetic nodes cdef int __counter = 0 class __CompositeError(Exception): def __init__(self, path, message): super().__init__(message) self.path = path self.message = message # Metadata container for a yaml toplevel node. # # This class contains metadata around a yaml node in order to be able # to trace back the provenance of a node to the file. # cdef class __FileInfo: cdef str filename, shortname, displayname cdef MappingNode toplevel, cdef object project def __init__(self, str filename, str shortname, str displayname, MappingNode toplevel, object project): self.filename = filename self.shortname = shortname self.displayname = displayname self.toplevel = toplevel self.project = project cdef int __next_synthetic_counter(): global __counter __counter -= 1 return __counter cdef Node __create_node_recursive(object value, Node ref_node): cdef value_type = type(value) if value_type is list: node = __new_node_from_list(value, ref_node) elif value_type in [int, str, bool, type(None)]: node = ScalarNode.__new__(ScalarNode, ref_node.file_index, ref_node.line, __next_synthetic_counter(), value) elif value_type is dict: node = __new_node_from_dict(value, ref_node) else: raise TypeError( "Unable to assign a value of type {} to a Node.".format(value_type)) return node # _new_node_from_dict() # # Args: # indict (dict): The input dictionary # ref_node (Node): The dictionary to take as reference for position # # Returns: # (Node): A new synthetic YAML tree which represents this dictionary # cdef Node __new_node_from_dict(dict indict, Node ref_node): cdef MappingNode ret = MappingNode.__new__( MappingNode, ref_node.file_index, ref_node.line, __next_synthetic_counter(), {}) cdef str k for k, v in indict.items(): ret.value[k] = __create_node_recursive(v, ref_node) return ret # Internal function to help new_node_from_dict() to handle lists cdef Node __new_node_from_list(list inlist, Node ref_node): cdef SequenceNode ret = SequenceNode.__new__( SequenceNode, ref_node.file_index, ref_node.line, __next_synthetic_counter(), []) for v in inlist: ret.value.append(__create_node_recursive(v, ref_node)) return ret # __validate_node_type(node, allowed_types, key) # # Validates that this node is of the expected node type, # and raises a user facing LoadError if not. # # Args: # allowed_types (list): list of valid subtypes of Node, or None # key (str): A key, in case the validated node is a value for a key # # Raises: # (LoadError): If this node is not of the expected type # cdef void __validate_node_type(Node node, list allowed_types = None, str key = None) except *: cdef ProvenanceInformation provenance cdef list human_types cdef str message if allowed_types and type(node) not in allowed_types: provenance = node.get_provenance() human_types = [] if MappingNode in allowed_types: human_types.append("dict") if SequenceNode in allowed_types: human_types.append('list') if ScalarNode in allowed_types: human_types.append('scalar') message = "{}: Value ".format(provenance) if key: message += "of '{}' ".format(key) message += "is not one of the following: {}.".format(", ".join(human_types)) raise LoadError(message, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/src/buildstream/plugin.py000066400000000000000000001113021514607367700225720ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ Plugin - Base plugin class ========================== BuildStream supports third party plugins to define additional kinds of :mod:`Elements ` and :mod:`Sources `. The common API is documented here, along with some information on how external plugin packages are structured. .. _core_plugin_abstract_methods: Abstract Methods ---------------- For both :mod:`Elements ` and :mod:`Sources `, it is mandatory to implement the following abstract methods: * :func:`Plugin.configure() ` Loads the user provided configuration YAML for the given source or element * :func:`Plugin.preflight() ` Early preflight checks allow plugins to bail out early with an error in the case that it can predict that failure is inevitable. * :func:`Plugin.get_unique_key() ` Once all configuration has been loaded and preflight checks have passed, this method is used to inform the core of a plugin's unique configuration. Configurable Warnings --------------------- Warnings raised through calling :func:`Plugin.warn() ` can provide an optional parameter ``warning_token``, this will raise a :class:`PluginError` if the warning is configured as fatal within the project configuration. Configurable warnings will be prefixed with :func:`Plugin.get_kind() ` within buildstream and must be prefixed as such in project configurations. For more detail on project configuration see :ref:`Configurable Warnings `. It is important to document these warnings in your plugin documentation to allow users to make full use of them while configuring their projects. Example ~~~~~~~ If the :class:`git ` plugin uses the warning ``"inconsistent-submodule"`` then it could be referenced in project configuration as ``"git:inconsistent-submodule"``. Plugin Structure ---------------- A plugin should consist of a `setuptools package `_ that advertises contained plugins using `entry points `_. A plugin entry point must be a module that extends a class in the :ref:`core_framework` to be discovered by BuildStream. A YAML file defining plugin default settings with the same name as the module can also be defined in the same directory as the plugin module. .. note:: BuildStream does not support function/class entry points. A sample plugin could be structured as such: .. code-block:: text . ├── elements │   ├── autotools.py │   ├── autotools.yaml │   └── __init__.py ├── MANIFEST.in └── setup.py The setuptools configuration should then contain at least: setup.py: .. literalinclude:: ../source/sample_plugin/setup.py :language: python MANIFEST.in: .. literalinclude:: ../source/sample_plugin/MANIFEST.in :language: text Class Reference --------------- """ import itertools # # In some cases (on Debian with python 3.9 it has been seen) when multiple threads # enter the Plugin.blocking_activity() context manager simultaneously, we get ImportErrors # from the multiprocessing submodules complaining that we are importing symbols from # partially initialized submodules (hinting at possible circular imports). # # Ensuring that these submodules have been initialized up front circumvents these edge # case stack trace bugs from occurring. # import multiprocessing import multiprocessing.queues import multiprocessing.synchronize import multiprocessing.popen_forkserver # type: ignore import os import queue import signal import subprocess import sys import traceback from contextlib import contextmanager, suppress from typing import IO, Any, Callable, Generator, Optional, Sequence, Dict, Tuple, TypeVar, Union, TYPE_CHECKING from weakref import WeakValueDictionary from . import utils, _signals from ._exceptions import PluginError, ImplError from ._message import Message, MessageType from .node import Node, MappingNode from .types import CoreWarnings, SourceRef if TYPE_CHECKING: # pylint: disable=cyclic-import from ._context import Context from ._project import Project # pylint: enable=cyclic-import T1 = TypeVar("T1") # # Some types used for subprocess.Popen() wrappers # # This recipe was gleaned from observing: # https://github.com/python/typeshed/blob/master/stdlib/_typeshed/__init__.pyi # https://github.com/python/typeshed/blob/master/stdlib/subprocess.pyi # _FILE = Union[None, int, IO[Any]] _TXT = Union[bytes, str] _STR_BYTES_PATH = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"] _CMD = Union[_STR_BYTES_PATH, Sequence[_STR_BYTES_PATH]] # _background_job_wrapper() # # Wrapper for running jobs in the background, transparently for users # # This method will put on the queue a response of the form: # (PickleError, OtherError, Result) # # Args: # result_queue: The queue in which to pass back the result # target: function to execute in the background # args: positional arguments to give to the target function # def _background_job_wrapper(result_queue: multiprocessing.Queue, target: Callable[..., T1], args: Any) -> None: result = None try: result = target(*args) result_queue.put((None, result)) except Exception: # pylint: disable=broad-except result_queue.put((traceback.format_exc(), None)) class Plugin: """Plugin() Base Plugin class. Some common features to both Sources and Elements are found in this class. .. note:: Derivation of plugins is not supported. Plugins may only derive from the base :mod:`Source ` and :mod:`Element ` types, and any convenience subclasses (like :mod:`BuildElement `) which are included in the buildstream namespace. """ BST_MIN_VERSION: Optional[str] = None """The minimum required version of BuildStream required by this plugin. The version must be expressed as the string *"."*, where the *major* version number is the API version and the *minor* version number is the revision of the same BuildStream API where new symbols might have been added to the API. **Example:** The following statement means that this plugin works with *BuildStream 2.X*, only if *X >= 2*: .. code:: python class Foo(Source): # Our plugin requires 2.2 BST_MIN_VERSION = "2.2" .. note:: This version works exactly the same was as the :ref:`min-version ` which must be specified in the project.conf file. """ BST_PLUGIN_DEPRECATED = False """True if this element plugin has been deprecated. If this is set to true, BuildStream will emit a deprecation warning in any place where this plugin is used. The deprecation warnings can be suppressed when defining the :ref:`plugin origins in your project configuration ` """ BST_PLUGIN_DEPRECATION_MESSAGE = None """An additional message to report when a plugin is deprecated This can be used to refer the user to a suitable replacement or alternative approach when the plugin is deprecated. """ # Unique id generator for Plugins # # Each plugin gets a unique id at creation. # # Ids are a monotically increasing integer which # starts as 1 (a falsy plugin ID is considered unset # in various parts of the codebase). # __id_generator = itertools.count(1) # Hold on to a lookup table by counter of all instantiated plugins. # We use this to send the id back from child processes so we can lookup # corresponding element/source in the master process. # # Use WeakValueDictionary() so the map we use to lookup objects does not # keep the plugins alive after pipeline destruction. # # Note that Plugins can only be instantiated in the main process before # scheduling tasks. __TABLE = WeakValueDictionary() # type: WeakValueDictionary[int, Plugin] try: __multiprocessing_context: Union[ multiprocessing.context.ForkServerContext, multiprocessing.context.SpawnContext ] = multiprocessing.get_context("forkserver") except ValueError: # We are on a system without `forkserver` support. Let's default to # spawn. This seems to be hanging however in some rare cases. # # Support is not as critical for now, since we do not work on # platforms not supporting forkserver for now (mainly Windows) # # XXX: investigate why we sometimes get deadlocks there __multiprocessing_context = multiprocessing.get_context("spawn") def __init__( self, name: str, context: "Context", project: "Project", provenance_node: Node, type_tag: str, unique_id: Optional[int] = None, ): self.name = name """The plugin name For elements, this is the project relative bst filename, for sources this is the owning element's name with a suffix indicating its index on the owning element. For sources this is for display purposes only. """ # Unique ID # # This id allows to uniquely identify a plugin. # # /!\ the unique id must be an increasing value /!\ # This is because we are depending on it in buildstream.element.Element # to give us a topological sort over all elements. # Modifying how we handle ids here will modify the behavior of the # Element's state handling. if unique_id is None: # Register ourself in the table containing all existing plugins self._unique_id = next(self.__id_generator) self.__TABLE[self._unique_id] = self else: # If the unique ID is passed in the constructor, then it is a cloned # plugin in a subprocess and should use the same ID. self._unique_id = unique_id self.__context = context # The Context object # Note that when pickling jobs over to a child process, we rely on this # reference to the Project, it keeps the plugin factory alive. If the # factory were to be GC'd then we would see undefined behaviour. Make # sure to test plugin pickling if this reference is to be removed. self.__project = project # The Project object self.__provenance_node = provenance_node # The originating YAML node self.__type_tag = type_tag # The type of plugin (element or source) self.__configuring = False # Whether we are currently configuring # Get the full_name as project & type_tag are resolved self.__full_name = self.__get_full_name() # Our message kwargs self._message_kwargs = {"element_name": self._get_full_name()} # Infer the kind identifier modulename = type(self).__module__ self.__kind = modulename.rsplit(".", maxsplit=1)[-1] self.debug("Created: {}".format(self)) def __del__(self): # Dont send anything through the Message() pipeline at destruction time, # any subsequent lookup of plugin by unique id would raise KeyError. if self.__context.log_debug: sys.stderr.write("DEBUG: Destroyed: {}\n".format(self)) def __str__(self): return "{kind} {typetag} at {provenance}".format( kind=self.__kind, typetag=self.__type_tag, provenance=self._get_provenance() ) ############################################################# # Abstract Methods # ############################################################# def configure(self, node: MappingNode) -> None: """Configure the Plugin from loaded configuration data Args: node: The loaded configuration dictionary Raises: :class:`.SourceError`: If it's a :class:`.Source` implementation :class:`.ElementError`: If it's an :class:`.Element` implementation Plugin implementors should implement this method to read configuration data and store it. The :func:`MappingNode.validate_keys() ` method should be used to ensure that the user has not specified keys in `node` which are unsupported by the plugin. """ raise ImplError( "{tag} plugin '{kind}' does not implement configure()".format(tag=self.__type_tag, kind=self.get_kind()) ) def preflight(self) -> None: """Preflight Check Raises: :class:`.SourceError`: If it's a :class:`.Source` implementation :class:`.ElementError`: If it's an :class:`.Element` implementation This method is run after :func:`Plugin.configure() ` and after the pipeline is fully constructed. Implementors should simply raise :class:`.SourceError` or :class:`.ElementError` with an informative message in the case that the host environment is unsuitable for operation. Plugins which require host tools (only sources usually) should obtain them with :func:`utils.get_host_tool() ` which will raise an error automatically informing the user that a host tool is needed. """ raise ImplError( "{tag} plugin '{kind}' does not implement preflight()".format(tag=self.__type_tag, kind=self.get_kind()) ) def get_unique_key(self) -> SourceRef: """Return something which uniquely identifies the plugin input Returns: A string, list or dictionary which uniquely identifies the input This is used to construct unique :ref:`cache keys ` for elements and sources, sources should return something which uniquely identifies the payload, such as an sha256 sum of a tarball content. Elements and Sources should implement this by collecting any configurations which could possibly affect the output and return a dictionary of these settings. For Sources, this is guaranteed to only be called if :func:`Source.is_resolved() ` has returned ``True`` which is to say that the :class:`.Source` is expected to have an exact :ref:`source ref ` indicating exactly what source is going to be staged. .. note:: If your plugin is concerned with API stability, then future extensions of your plugin YAML configuration which affect the unique key returned here should be added to this key with care. A good rule of thumb is to only compute the new value in the returned key if the value of the newly added YAML key is not equal to it's default value. """ raise ImplError( "{tag} plugin '{kind}' does not implement get_unique_key()".format( tag=self.__type_tag, kind=self.get_kind() ) ) ############################################################# # Public Methods # ############################################################# def get_kind(self) -> str: """Fetches the kind of this plugin Returns: The kind of this plugin """ return self.__kind def node_get_project_path(self, node, *, check_is_file=False, check_is_dir=False) -> str: """Fetches a project path from a dictionary node and validates it Paths are asserted to never lead to a directory outside of the project directory. In addition, paths can not point to symbolic links, fifos, sockets and block/character devices. The `check_is_file` and `check_is_dir` parameters can be used to perform additional validations on the path. Note that an exception will always be raised if both parameters are set to ``True``. Args: node (ScalarNode): A Node loaded from YAML containing the path to validate check_is_file (bool): If ``True`` an error will also be raised if path does not point to a regular file. Defaults to ``False`` check_is_dir (bool): If ``True`` an error will also be raised if path does not point to a directory. Defaults to ``False`` Returns: (str): The project path Raises: :class:`.LoadError`: In the case that the project path is not valid or does not exist **Example:** .. code:: python path = self.node_get_project_path(node, 'path') """ return self.__project.get_path_from_node(node, check_is_file=check_is_file, check_is_dir=check_is_dir) def debug(self, brief: str, *, detail: Optional[str] = None) -> None: """Print a debugging message Args: brief: The brief message detail: An optional detailed message, can be multiline output """ if self.__context.log_debug: self.__message(MessageType.DEBUG, brief, detail=detail) def status(self, brief: str, *, detail: Optional[str] = None) -> None: """Print a status message Args: brief: The brief message detail: An optional detailed message, can be multiline output Note: Status messages tell about what a plugin is currently doing """ self.__message(MessageType.STATUS, brief, detail=detail) def info(self, brief: str, *, detail: Optional[str] = None) -> None: """Print an informative message Args: brief: The brief message detail: An optional detailed message, can be multiline output Note: Informative messages tell the user something they might want to know, like if refreshing an element caused it to change. The instance full name of the plugin will be generated with the message, this being the name of the given element, as appose to the class name of the underlying plugin __kind identifier. """ self.__message(MessageType.INFO, brief, detail=detail) def warn(self, brief: str, *, detail: Optional[str] = None, warning_token: Optional[str] = None) -> None: """Print a warning message, checks warning_token against project configuration Args: brief: The brief message detail: An optional detailed message, can be multiline output warning_token: An optional configurable warning assosciated with this warning, this will cause PluginError to be raised if this warning is configured as fatal. Raises: (:class:`.PluginError`): When warning_token is considered fatal by the project configuration """ if warning_token: warning_token = _prefix_warning(self, warning_token) brief = "[{}]: {}".format(warning_token, brief) project = self._get_project() if project._warning_is_fatal(warning_token): detail = detail if detail else "" raise PluginError(message="{}\n{}".format(brief, detail), reason=warning_token) self.__message(MessageType.WARN, brief=brief, detail=detail) def log(self, brief: str, *, detail: Optional[str] = None) -> None: """Log a message into the plugin's log file The message will not be shown in the master log at all (so it will not be displayed to the user on the console). Args: brief: The brief message detail: An optional detailed message, can be multiline output """ self.__message(MessageType.LOG, brief, detail=detail) @contextmanager def timed_activity( self, activity_name: str, *, detail: Optional[str] = None, silent_nested: bool = False ) -> Generator[None, None, None]: """Context manager for performing timed activities in plugins Args: activity_name: The name of the activity detail: An optional detailed message, can be multiline output silent_nested: If specified, nested messages will be silenced This function lets you perform timed tasks in your plugin, the core will take care of timing the duration of your task and printing start / fail / success messages. **Example** .. code:: python # Activity will be logged and timed with self.timed_activity("Mirroring {}".format(self.url)): # This will raise SourceError on its own self.call(... command which takes time ...) """ # Get the plugin kwargs and pass them along plugin_kwargs = self._message_kwargs with self.__context.messenger.timed_activity( activity_name, detail=detail, silent_nested=silent_nested, **plugin_kwargs ): yield def blocking_activity( self, target: Callable[..., T1], args: Sequence[Any], activity_name: str, *, detail: Optional[str] = None, silent_nested: bool = False ) -> T1: """Execute a blocking activity in the background. This is to execute potentially blocking methods in the background, in order to avoid starving the scheduler. The function, its arguments and return value must all be pickleable, as it will be run in another process. The function should not raise an exception. This should be used whenever there is a potential for a blocking syscall to not return in a reasonable (<1s) amount of time. For example, you would use this if you were doing a request to a remote server, without a timeout. Args: target: the function to execute in the background args: positional arguments to pass to the method to call activity_name: The name of the activity detail: An optional detailed message, can be multiline output silent_nested: If specified, nested messages will be silenced Returns: the return value from `target`. """ with self.__context.messenger.timed_activity( activity_name, element_name=self._get_full_name(), detail=detail, silent_nested=silent_nested ): result_queue = self.__multiprocessing_context.Queue() proc = None def kill_proc(): if proc and proc.is_alive(): proc.kill() proc.join() def suspend_proc(): if proc and proc.is_alive(): with suppress(ProcessLookupError): os.kill(proc.pid, signal.SIGSTOP) def resume_proc(): if proc and proc.is_alive(): with suppress(ProcessLookupError): os.kill(proc.pid, signal.SIGCONT) with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc): proc = self.__multiprocessing_context.Process( target=_background_job_wrapper, args=(result_queue, target, args) ) proc.start() should_continue = True last_check = False while should_continue or last_check: last_check = False try: err, result = result_queue.get(timeout=1) break except queue.Empty: if not proc.is_alive() and should_continue: # Let's check one last time, just in case it stopped # between our last check and now last_check = True should_continue = False continue else: raise PluginError("Background process died with error code {}".format(proc.exitcode)) try: proc.join(timeout=15) proc.terminate() except TimeoutError: raise PluginError("Background process didn't exit after 15 seconds and got killed.") if err is not None: raise PluginError("An error happened while running a blocking activity", detail=err) return result def call( self, args: _CMD, fail: Optional[str] = None, fail_temporarily: bool = False, cwd: Optional[_STR_BYTES_PATH] = None, env: Optional[Dict[str, str]] = None, stdin: Optional[_FILE] = None, stdout: Optional[_FILE] = None, stderr: Optional[_FILE] = None, ) -> int: """A wrapper for subprocess.call() Args: args: A sequence of program arguments or else a string or `path-like-object `_ fail: A message to display if the process returns a non zero exit code fail_temporarily: Whether any exceptions should be raised as temporary. cwd: Optionally specify the working directory for the command env: Optionally specify some environment variables for the command stdin: Optionally specify standard input for the command stdout: Optionally specify standard output for the command stderr: Optionally specify standard error for the command Returns: The process exit code. Raises: (:class:`.PluginError`): If a non-zero return code is received and *fail* is specified .. attention:: If *fail* is not specified, then the return value of subprocess.call() is returned even on error, and no exception is automatically raised. .. attention:: If *stderr* and/or *stdout* are specified, then these streams are delivered directly to the plugin instead of being delivered to the associated log file. **Example** .. code:: python # Call some host tool self.tool = utils.get_host_tool('toolname') self.call( [self.tool, '--download-ponies', self.mirror_directory], "Failed to download ponies from {}".format( self.mirror_directory)) """ exit_code, _ = self.__call( args, fail=fail, fail_temporarily=fail_temporarily, cwd=cwd, env=env, stdin=stdin, stdout=stdout, stderr=stderr, ) return exit_code def check_output( self, args: _CMD, fail: Optional[str] = None, fail_temporarily: bool = False, cwd: Optional[_STR_BYTES_PATH] = None, env: Optional[Dict[str, str]] = None, stdin: Optional[_FILE] = None, stderr: Optional[_FILE] = None, ) -> Tuple[int, str]: """A wrapper for subprocess.check_output() Args: args: A sequence of program arguments or else a string or `path-like-object `_ fail: A message to display if the process returns a non zero exit code fail_temporarily: Whether any exceptions should be raised as temporary. cwd: Optionally specify the working directory for the command env: Optionally specify some environment variables for the command stdin: Optionally specify standard input for the command stderr: Optionally specify standard error for the command Returns: A 2-tuple of form (process exit code, process standard output) Raises: (:class:`.PluginError`): If a non-zero return code is received and *fail* is specified .. attention:: If *fail* is not specified, then the return value of subprocess.call() is returned even on error, and no exception is automatically raised. .. attention:: If *stderr* and/or *stdout* are specified, then these streams are delivered directly to the plugin instead of being delivered to the associated log file. **Example** .. code:: python # Get the tool at preflight time self.tool = utils.get_host_tool('toolname') # Call the tool, automatically raise an error _, output = self.check_output( [self.tool, '--print-ponies'], "Failed to print the ponies in {}".format( self.mirror_directory), cwd=self.mirror_directory) # Call the tool, inspect exit code exit_code, output = self.check_output( [self.tool, 'get-ref', tracking], cwd=self.mirror_directory) if exit_code == 128: return elif exit_code != 0: fmt = "{plugin}: Failed to get ref for tracking: {track}" raise SourceError( fmt.format(plugin=self, track=tracking)) from e """ return self.__call( args, collect_stdout=True, fail=fail, fail_temporarily=fail_temporarily, cwd=cwd, env=env, stdin=stdin, stderr=stderr, ) ############################################################# # Private Methods used in BuildStream # ############################################################# # _lookup(): # # Fetch a plugin in the current process by its # unique identifier # # Args: # unique_id: The unique identifier as returned by # plugin._unique_id # # Returns: # (Plugin): The plugin for the given ID, or None # @classmethod def _lookup(cls, unique_id): assert unique_id != 0, "Looking up invalid plugin ID 0, ID counter starts at 1" try: return cls.__TABLE[unique_id] except KeyError: assert False, "Could not find plugin with ID {}".format(unique_id) raise # In case a user is running with "python -O" # _get_context() # # Fetches the invocation context # def _get_context(self): return self.__context # _get_project() # # Fetches the project object associated with this plugin # def _get_project(self): return self.__project # _get_provenance(): # # Fetch bst file, line and column of the entity # def _get_provenance(self): return self.__provenance_node.get_provenance() # Context manager for getting the open file handle to this # plugin's log. Used in the child context to add stuff to # a log. # @contextmanager def _output_file(self): log = self.__context.messenger.get_log_handle() if log is None: with open(os.devnull, "w", encoding="utf-8") as output: yield output else: yield log # _configure(): # # Calls configure() for the plugin, this must be called by # the core instead of configure() directly, so that the # _get_configuring() state is up to date. # # Args: # node (buildstream.node.MappingNode): The loaded configuration dictionary # def _configure(self, node): self.__configuring = True self.configure(node) self.__configuring = False # _get_configuring(): # # Checks whether the plugin is in the middle of having # its Plugin.configure() method called # # Returns: # (bool): Whether we are currently configuring def _get_configuring(self): return self.__configuring # _preflight(): # # Calls preflight() for the plugin, and allows generic preflight # checks to be added # # Raises: # SourceError: If it's a Source implementation # ElementError: If it's an Element implementation # ProgramNotFoundError: If a required host tool is not found # def _preflight(self): self.preflight() # _get_full_name(): # # The instance full name of the plugin prepended with the owning # junction if appropriate. This being the name of the given element, # as appose to the class name of the underlying plugin __kind identifier. # # Returns: # (str): element full name, with prepended owning junction if appropriate # def _get_full_name(self): return self.__full_name ############################################################# # Local Private Methods # ############################################################# # Internal subprocess implementation for the call() and check_output() APIs # def __call( self, args: _CMD, fail: Optional[str] = None, fail_temporarily: bool = False, collect_stdout: bool = False, cwd: Optional[_STR_BYTES_PATH] = None, env: Optional[Dict[str, str]] = None, stdin: Optional[_FILE] = None, stdout: Optional[_FILE] = None, stderr: Optional[_FILE] = None, ): with self._output_file() as output_file: if stdout is None: stdout = output_file if stderr is None: stderr = output_file if collect_stdout: stdout = subprocess.PIPE self.__note_command(output_file, args, cwd) exit_code, output = utils._call(args, cwd=cwd, env=env, stdin=stdin, stdout=stdout, stderr=stderr) if fail and exit_code: raise PluginError("{plugin}: {message}".format(plugin=self, message=fail), temporary=fail_temporarily) return (exit_code, output) # __message(): # # The plugin level focal point for issuing messages. # # Args: # message_type (MessageType): The message type # brief (str): The brief message # kwargs: The remaining Message attributes # def __message(self, message_type, brief, **kwargs): # # Merge the plugin kwargs with the explicitly passed kwargs, give # precedence to the explicitly passed kwargs. # plugin_kwargs = self._message_kwargs.copy() plugin_kwargs.update(kwargs) message = Message(message_type, brief, **plugin_kwargs) self.__context.messenger.message(message) # __note_command(): # # Make a note in the logs that we are running a process on the host. # # Args: # output: The output logging file # args: The command with args which we're about to run # cwd: The host side working directory to run the command, or None # def __note_command(self, output, args, cwd): workdir = cwd or os.getcwd() command = " ".join(args) output.write("Running host command {}: {}\n".format(workdir, command)) output.flush() self.status("Running host command", detail=command) def __get_full_name(self): project = self.__project # Set the name, depending on element or source plugin type name = self._element_name if self.__type_tag == "source" else self.name # pylint: disable=no-member if project.junction: return "{}:{}".format(project.junction._get_full_name(), name) else: return name # A local table for _prefix_warning() # __CORE_WARNINGS = [value for name, value in CoreWarnings.__dict__.items() if not name.startswith("__")] # _prefix_warning(): # # Prefix a warning with the plugin kind. CoreWarnings are not prefixed. # # Args: # plugin (Plugin): The plugin which raised the warning # warning (str): The warning to prefix # # Returns: # (str): A prefixed warning # def _prefix_warning(plugin, warning): if any((warning is core_warning for core_warning in __CORE_WARNINGS)): return warning return "{}:{}".format(plugin.get_kind(), warning) apache-buildstream-27ae392/src/buildstream/plugins/000077500000000000000000000000001514607367700224055ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/plugins/elements/000077500000000000000000000000001514607367700242215ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/plugins/elements/__init__.py000066400000000000000000000000001514607367700263200ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/plugins/elements/compose.py000066400000000000000000000150171514607367700262440ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ compose - Compose the output of multiple elements ================================================= This element creates a selective composition of its dependencies. This is normally used at near the end of a pipeline to prepare something for later deployment. Since this element's output includes its dependencies, it may only depend on elements as `build` type dependencies. The default configuration and possible options are as such: .. literalinclude:: ../../../src/buildstream/plugins/elements/compose.yaml :language: yaml """ import os from buildstream import Element # Element implementation for the 'compose' kind. class ComposeElement(Element): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # The compose element's output is its dependencies, so # we must rebuild if the dependencies change even when # not in strict build plans. # BST_STRICT_REBUILD = True # Compose artifacts must never have indirect dependencies, # so runtime dependencies are forbidden. BST_FORBID_RDEPENDS = True # This element ignores sources, so we should forbid them from being # added, to reduce the potential for confusion BST_FORBID_SOURCES = True def configure(self, node): node.validate_keys(["integrate", "include", "exclude", "include-orphans"]) # We name this variable 'integration' only to avoid # collision with the Element.integrate() method. self.integration = node.get_bool("integrate") self.include = node.get_str_list("include") self.exclude = node.get_str_list("exclude") self.include_orphans = node.get_bool("include-orphans") # Inform the core that we will not need to run any commands in the sandbox if not self.integration: self.BST_RUN_COMMANDS = False def preflight(self): pass def get_unique_key(self): key = {"integrate": self.integration, "include": sorted(self.include), "orphans": self.include_orphans} if self.exclude: key["exclude"] = sorted(self.exclude) return key def configure_sandbox(self, sandbox): pass def stage(self, sandbox): # Stage deps in the sandbox root with self.timed_activity("Staging dependencies", silent_nested=True): self.stage_dependency_artifacts(sandbox) def assemble(self, sandbox): manifest = set() require_split = self.include or self.exclude or not self.include_orphans if require_split: with self.timed_activity("Computing split", silent_nested=True): for dep in self.dependencies(): files = dep.compute_manifest( include=self.include, exclude=self.exclude, orphans=self.include_orphans ) manifest.update(files) # Make a snapshot of all the files. vbasedir = sandbox.get_virtual_directory() removed_files = set() added_files = set() # Run any integration commands provided by the dependencies # once they are all staged and ready if self.integration: with self.timed_activity("Integrating sandbox"): if require_split: # Make a snapshot of all the files before integration-commands are run. snapshot = set(vbasedir.list_relative_paths()) with sandbox.batch(): for dep in self.dependencies(): dep.integrate(sandbox) if require_split: # Calculate added and removed files post_integration_snapshot = vbasedir.list_relative_paths() basedir_contents = set(post_integration_snapshot) for path in manifest: if path in snapshot and path not in basedir_contents: removed_files.add(path) for path in basedir_contents: if path not in snapshot: added_files.add(path) self.info("Integration added {} and removed {} files".format(len(added_files), len(removed_files))) # The remainder of this is expensive, make an early exit if # we're not being selective about what is to be included. if not require_split: return "/" # Update the manifest with files which were added/removed by integration commands # manifest.update(added_files) manifest.difference_update(removed_files) # XXX We should be moving things outside of the build sandbox # instead of into a subdir. The element assemble() method should # support this in some way. # installdir = vbasedir.open_directory("buildstream/install", create=True) # We already saved the manifest for created files in the integration phase, # now collect the rest of the manifest. # lines = [] if self.include: lines.append("Including files from domains: " + ", ".join(self.include)) else: lines.append("Including files from all domains") if self.exclude: lines.append("Excluding files from domains: " + ", ".join(self.exclude)) if self.include_orphans: lines.append("Including orphaned files") else: lines.append("Excluding orphaned files") detail = "\n".join(lines) def import_filter(path): return path in manifest with self.timed_activity("Creating composition", detail=detail, silent_nested=True): self.info("Composing {} files".format(len(manifest))) installdir.import_files(vbasedir, filter_callback=import_filter, collect_result=False) # And we're done return os.path.join(os.sep, "buildstream", "install") # Plugin entry point def setup(): return ComposeElement apache-buildstream-27ae392/src/buildstream/plugins/elements/compose.yaml000066400000000000000000000027411514607367700265560ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Compose element configuration config: # Whether to run the integration commands for the # staged dependencies. # integrate: True # A list of domains to include from each artifact, as # they were defined in the element's 'split-rules'. # # Since domains can be added, it is not an error to # specify domains which may not exist for all of the # elements in this composition. # # The default empty list indicates that all domains # from each dependency should be included. # include: [] # A list of domains to exclude from each artifact, as # they were defined in the element's 'split-rules'. # # In the case that a file is spoken for by a domain # in the 'include' list and another in the 'exclude' # list, then the file will be excluded. exclude: [] # Whether to include orphan files which are not # included by any of the 'split-rules' present on # a given element. # include-orphans: True apache-buildstream-27ae392/src/buildstream/plugins/elements/filter.py000066400000000000000000000240251514607367700260630ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jonathan Maw """ filter - Extract a subset of files from another element ======================================================= Filter another element by producing an output that is a subset of the parent element's output. Subsets are defined by the parent element's :ref:`split rules `. Overview -------- A filter element must have exactly one *build* dependency, where said dependency is the 'parent' element which we would like to filter. Runtime dependencies may also be specified, which can be useful to propagate forward from this filter element onto its reverse dependencies. See :ref:`Dependencies ` to see how we specify dependencies. When workspaces are opened, closed or reset on a filter element, or this element is tracked, the filter element will transparently pass on the command to its parent element (the sole build-dependency). Example ------- Consider a simple import element, ``import.bst`` which imports the local files 'foo', 'bar' and 'baz' (each stored in ``files/``, relative to the project's root): .. code:: yaml kind: import # Specify sources to import sources: - kind: local path: files # Specify public domain data, visible to other elements public: bst: split-rules: foo: - /foo bar: - /bar .. note:: We can make an element's metadata visible to all reverse dependencies by making use of the ``public:`` field. See the :ref:`public data documentation ` for more information. In this example, ``import.bst`` will serve as the 'parent' of the filter element, thus its output will be filtered. It is important to understand that the artifact of the above element will contain the files: 'foo', 'bar' and 'baz'. Now, to produce an element whose artifact contains the file 'foo', and exlusively 'foo', we can define the following filter, ``filter-foo.bst``: .. code:: yaml kind: filter # Declare the sole build-dependency of the filter element build-depends: - import.bst # Declare a list of domains to include in the filter's artifact config: include: - foo It should be noted that an 'empty' ``include:`` list would, by default, include all split-rules specified in the parent element, which, in this example, would be the files 'foo' and 'bar' (the file 'baz' was not covered by any split rules). Equally, we can use the ``exclude:`` statement to create the same artifact (which only contains the file 'foo') by declaring the following element, ``exclude-bar.bst``: .. code:: yaml kind: filter # Declare the sole build-dependency of the filter element build-depends: - import.bst # Declare a list of domains to exclude in the filter's artifact config: exclude: - bar In addition to the ``include:`` and ``exclude:`` fields, there exists an ``include-orphans:`` (Boolean) field, which defaults to ``False``. This will determine whether to include files which are not present in the 'split-rules'. For example, if we wanted to filter out all files which are not included as split rules we can define the following element, ``filter-misc.bst``: .. code:: yaml kind: filter # Declare the sole build-dependency of the filter element build-depends: - import.bst # Filter out all files which are not declared as split rules config: exclude: - foo - bar include-orphans: True The artifact of ``filter-misc.bst`` will only contain the file 'baz'. Below is more information regarding the the default configurations and possible options of the filter element: .. literalinclude:: ../../../src/buildstream/plugins/elements/filter.yaml :language: yaml """ from buildstream import Element, ElementError from buildstream.types import _Scope class FilterElement(Element): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" BST_ARTIFACT_VERSION = 1 # The filter element's output is its dependencies, so # we must rebuild if the dependencies change even when # not in strict build plans. BST_STRICT_REBUILD = True # This element ignores sources, so we should forbid them from being # added, to reduce the potential for confusion BST_FORBID_SOURCES = True # Filter elements do not run any commands BST_RUN_COMMANDS = False def configure(self, node): node.validate_keys(["include", "exclude", "include-orphans", "pass-integration"]) self.include_node = node.get_sequence("include") self.exclude_node = node.get_sequence("exclude") self.include = self.include_node.as_str_list() self.exclude = self.exclude_node.as_str_list() self.include_orphans = node.get_bool("include-orphans") self.pass_integration = node.get_bool("pass-integration", False) def preflight(self): # Exactly one build-depend is permitted build_deps = list(self._dependencies(_Scope.BUILD, recurse=False)) if len(build_deps) != 1: detail = "Full list of build-depends:\n" deps_list = " \n".join([x.name for x in build_deps]) detail += deps_list raise ElementError( "{}: {} element must have exactly 1 build-dependency, actually have {}".format( self, type(self).__name__, len(build_deps) ), detail=detail, reason="filter-bdepend-wrong-count", ) # That build-depend must not also be a runtime-depend runtime_deps = list(self._dependencies(_Scope.RUN, recurse=False)) if build_deps[0] in runtime_deps: detail = "Full list of runtime depends:\n" deps_list = " \n".join([x.name for x in runtime_deps]) detail += deps_list raise ElementError( "{}: {} element's build dependency must not also be a runtime dependency".format( self, type(self).__name__ ), detail=detail, reason="filter-bdepend-also-rdepend", ) # If a parent does not produce an artifact, fail and inform user that the dependency # must produce artifacts if not build_deps[0].BST_ELEMENT_HAS_ARTIFACT: detail = "{} does not produce an artifact, so there is nothing to filter".format(build_deps[0].name) raise ElementError( "{}: {} element's build dependency must produce an artifact".format(self, type(self).__name__), detail=detail, reason="filter-bdepend-no-artifact", ) def get_unique_key(self): key = { "include": sorted(self.include), "exclude": sorted(self.exclude), "orphans": self.include_orphans, } return key def configure_sandbox(self, sandbox): pass def stage(self, sandbox): with self.timed_activity("Staging artifact", silent_nested=True): for dep in self.dependencies(recurse=False): # Check that all the included/excluded domains exist pub_data = dep.get_public_data("bst") split_rules = pub_data.get_mapping("split-rules", {}) unfound_includes = [] for domain in self.include: if domain not in split_rules: unfound_includes.append(domain) unfound_excludes = [] for domain in self.exclude: if domain not in split_rules: unfound_excludes.append(domain) detail = [] if unfound_includes: detail.append("Unknown domains were used in {}".format(self.include_node.get_provenance())) detail.extend([" - {}".format(domain) for domain in unfound_includes]) if unfound_excludes: detail.append("Unknown domains were used in {}".format(self.exclude_node.get_provenance())) detail.extend([" - {}".format(domain) for domain in unfound_excludes]) if detail: detail = "\n".join(detail) raise ElementError("Unknown domains declared.", detail=detail) dep.stage_artifact(sandbox, include=self.include, exclude=self.exclude, orphans=self.include_orphans) def assemble(self, sandbox): if self.pass_integration: build_deps = list(self.dependencies(recurse=False)) assert len(build_deps) == 1 dep = build_deps[0] # Integration commands of the build dependency pub_data = dep.get_public_data("bst") integration_commands = pub_data.get_str_list("integration-commands", []) # Integration commands of the filter element itself filter_pub_data = self.get_public_data("bst") filter_integration_commands = filter_pub_data.get_str_list("integration-commands", []) # Concatenate the command lists filter_pub_data["integration-commands"] = integration_commands + filter_integration_commands self.set_public_data("bst", filter_pub_data) return "" def _get_source_element(self): # Filter elements act as proxies for their sole build-dependency # build_deps = list(self._dependencies(_Scope.BUILD, recurse=False)) assert len(build_deps) == 1 output_elm = build_deps[0]._get_source_element() return output_elm def setup(): return FilterElement apache-buildstream-27ae392/src/buildstream/plugins/elements/filter.yaml000066400000000000000000000027551514607367700264030ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Filter element configuration config: # A list of domains to include in each artifact, as # they were defined as public data in the parent # element's 'split-rules'. # # If a domain is specified that does not exist, the # filter element will fail to build. # # The default empty list indicates that all domains # of the parent's artifact should be included. # include: [] # A list of domains to exclude from each artifact, as # they were defined in the parent element's 'split-rules'. # # In the case that a file is spoken for by a domain # in the 'include' list and another in the 'exclude' # list, then the file will be excluded. exclude: [] # Whether to include orphan files which are not # included by any of the 'split-rules' present in # the parent element. # include-orphans: False # Whether to pass the 'integration-commands' of the # parent element through the filter. # pass-integration: False apache-buildstream-27ae392/src/buildstream/plugins/elements/import.py000066400000000000000000000073021514607367700261070ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ import - Import sources directly ================================ Import elements produce artifacts directly from its sources without any kind of processing. These are typically used to import an SDK to build on top of or to overlay your build with some configuration data. The empty configuration is as such: .. literalinclude:: ../../../src/buildstream/plugins/elements/import.yaml :language: yaml """ import os from buildstream import Element, ElementError # Element implementation for the 'import' kind. class ImportElement(Element): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # Import elements do not run any commands BST_RUN_COMMANDS = False def configure(self, node): node.validate_keys(["source", "target"]) self.source = node.get_str("source") self.target = node.get_str("target") def preflight(self): # Assert that we have at least one source to fetch. sources = list(self.sources()) if not sources: raise ElementError("{}: An import element must have at least one source.".format(self)) def get_unique_key(self): return {"source": self.source, "target": self.target} def configure_sandbox(self, sandbox): pass def stage(self, sandbox): pass def assemble(self, sandbox): # Stage sources into the input directory self.stage_sources(sandbox, "input") rootdir = sandbox.get_virtual_directory() inputdir = rootdir.open_directory("input") outputdir = rootdir.open_directory("output", create=True) # The directory to grab inputdir = inputdir.open_directory(self.source.strip(os.sep)) # The output target directory outputdir = outputdir.open_directory(self.target.strip(os.sep), create=True) if not inputdir: raise ElementError("{}: No files were found inside directory '{}'".format(self, self.source)) # Move it over outputdir.import_files(inputdir, collect_result=False) # And we're done return "/output" def generate_script(self): build_root = self.get_variable("build-root") install_root = self.get_variable("install-root") commands = [] # The directory to grab inputdir = os.path.join(build_root, self.normal_name, self.source.lstrip(os.sep)) inputdir = inputdir.rstrip(os.sep) # The output target directory outputdir = os.path.join(install_root, self.target.lstrip(os.sep)) outputdir = outputdir.rstrip(os.sep) # Ensure target directory parent exists but target directory doesn't commands.append("mkdir -p {}".format(os.path.dirname(outputdir))) commands.append("[ ! -e {outputdir} ] || rmdir {outputdir}".format(outputdir=outputdir)) # Move it over commands.append("mv {} {}".format(inputdir, outputdir)) script = "" for cmd in commands: script += "(set -ex; {}\n) || exit 1\n".format(cmd) return script # Plugin entry point def setup(): return ImportElement apache-buildstream-27ae392/src/buildstream/plugins/elements/import.yaml000066400000000000000000000017431514607367700264240ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # The import element simply stages the given sources # directly to the root of the sandbox and then collects # the output to create an output artifact. # config: # By default we collect everything staged, specify a # directory here to output only a subset of the staged # input sources. source: / # Prefix the output with an optional directory, by default # the input is found at the root of the produced artifact. target: / apache-buildstream-27ae392/src/buildstream/plugins/elements/junction.py000066400000000000000000000326151514607367700264330ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jürg Billeter """ junction - Integrate subprojects ================================ This element acts as a window into another BuildStream project. It allows integration of multiple projects into a single pipeline. Overview -------- .. code:: yaml kind: junction # Specify the BuildStream project source sources: - kind: git url: upstream:projectname.git track: master ref: d0b38561afb8122a3fc6bafc5a733ec502fcaed6 # Specify the junction configuration config: # Override project options options: machine_arch: "%{machine_arch}" debug: True # Optionally look in a subpath of the source repository for the project path: projects/hello # Optionally override elements in subprojects, including junctions. # overrides: subproject-junction.bst: local-junction.bst # Optionally override aliases in subprojects, to allow using mirrors # defined in the parent project. aliases: subproject-alias: local-alias # A default mapping can be set (defaults to none) map-aliases: identity With a junction element in place, local elements can depend on elements in the other BuildStream project using :ref:`element paths `. For example, if you have a ``toolchain.bst`` junction element referring to a project which contains a ``gcc.bst`` element, you can express a build dependency to the compiler like this: .. code:: yaml build-depends: - junction: toolchain.bst:gcc.bst .. important:: **Limitations** Junction elements are only connectors which bring multiple projects together, and as such they are not in the element dependency graph. This means that it is illegal to depend on a junction, and it is also illegal for a junction to have dependencies. While junctions are elements, a limited set of element operations are supported. Junction elements can be tracked and fetched like other elements but they do not produce any artifacts, which means that they cannot be built or staged. Note that when running :ref:`bst source track ` on your project, elements found in subprojects are not tracked by default. You may specify ``--cross-junctions`` to the :ref:`bst source track ` command to explicitly track elements across junction boundaries. Sources ------- The sources of a junction element define how to obtain the BuildStream project that the junction connects to. Most commands, such as :ref:`bst build `, will automatically try to fetch the junction elements required to access any subproject elements which are specified as dependencies of the targets provided. Some commands, such as :ref:`bst show `, do not do this, and in such cases they can be fetched explicitly using :ref:`bst source fetch `: .. code:: bst source fetch junction.bst Options ------- Junction elements can configure the :ref:`project options ` in the subproject, using the ``options`` configuration. .. code:: yaml kind: junction ... config: # Specify the options for this subproject # options: machine_arch: "%{machine_arch}" debug: True Options are never implicitly propagated across junctions, however :ref:`variables ` can be used to explicitly assign configuration in a subproject which matches the toplevel project's configuration. Overriding elements ------------------- It is possible to override elements in subprojects. This can be useful if for example, you need to work with a custom variant or fork of some software in the subproject. This is a better strategy than overlapping and overwriting shared libraries built by the subproject later on, as we can ensure that reverse dependencies in the subproject are built against the overridden element. Overridding elements allows you to build on top of an existing project and benefit from updates and releases for the vast majority of the upstream project, even when there are some parts of the upstream project which need to be customized for your own applications. Even junction elements in subprojects can be overridden, this is sometimes important in order to reconcile conflicts when multiple projects depend on the same subproject, as :ref:`discussed below `. .. code:: yaml kind: junction ... config: # Override elements in a junctioned project # overrides: subproject-element.bst: local-element.bst It is also possible to override elements in deeply nested subprojects, using project relative :ref:`junction paths `: .. code:: yaml kind: junction ... config: # Override deeply nested elements # overrides: subproject.bst:subsubproject-element.bst: local-element.bst .. attention:: Overriding an element causes your project to completely define the element being overridden, which means you will no longer receive updates or security patches to the element in question when updating to newer versions and releases of the upstream project. As such, overriding elements is only recommended in cases where the element is very significantly redefined. Such cases include cases when you need a newer version of the element than the one maintained by the upstream project you are using as a subproject, or when you have significanly modified the code in your own custom ways. If you only need to introduce a security patch, then it is recommended that you create your own downstream branch of the upstream project, not only will this allow you to more easily consume updates with VCS tools like ``git rebase``, but it will also be more convenient for submitting your security patches to the upstream project so that you can drop them in a future update. Similarly, if you only need to enable/disable a specific feature of a module, it is also preferrable to use a downstream branch of the upstream project. In such a case, it is also worth trying to convince the upstream project to support a :ref:`project option ` for your specific element configuration, if it would be of use to other users too. .. _core_junction_nested: Nested Junctions ---------------- Junctions can be nested. That is, subprojects are allowed to have junctions on their own. Nested junctions in different subprojects may point to the same project, however, in most use cases the same project should be loaded only once. As the junctions may differ in source version and options, BuildStream cannot simply use one junction and ignore the others. Due to this, BuildStream requires the user to resolve conflicting nested junctions, and will provide an error message whenever a conflict is detected. .. _core_junction_nested_overrides: Overriding subproject junctions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If your project and a subproject share a subproject in common, then one way to resolve the conflict is to override the subproject's junction with a local in your project. You can override junctions in a subproject in the junction declaration of that subproject, e.g.: .. code:: yaml kind: junction # Here we are junctioning "subproject" which # also junctions "subsubproject", which we also # use directly. # sources: - kind: git url: https://example.com/subproject.git config: # Override `subsubproject.bst` in the subproject using # the locally declared `local-subsubproject.bst` junction. # overrides: subsubproject.bst: local-subsubproject.bst When declaring the ``overrides`` dictionary, the keys (on the left side) refer to :ref:`junction paths ` which are relative to the subproject you are declaring. The values (on the right side) refer to :ref:`junction paths ` which are relative to the project in which your junction is declared. .. warning:: This approach modifies your subproject, causing its output artifacts to differ from that project's expectations. If you rely on validation and guarantees provided by the organization which maintains the subproject, then it is desirable to avoid overriding any details from that upstream project. Linking to other junctions ~~~~~~~~~~~~~~~~~~~~~~~~~~ Another way to resolve the conflict when your project and a subproject both junction a common project, is to simply reuse the same junction from the subproject in your toplevel project. This is preferable to *overrides* because you can avoid modifying the subproject you would otherwise be changing with an override. A convenient way to reuse a nested junction in a higher level project is to create a :mod:`link ` element to that subproject's junction. This will help you avoid redundantly typing out longer :ref:`element paths ` in your project's :ref:`dependency declarations `. This way you can simply create the :mod:`link ` once in your project and use it locally to depend on elements in a nested subproject. **Example:** .. code:: yaml # Declare the `subsubproject-link.bst` link element, which # is a symbolic link to the junction declared in the subproject # kind: link config: target: subproject.bst:subsubproject.bst .. code:: yaml # Depend on elements in the subsubproject using # the subproject's junction directly # kind: autotools depends: - subsubproject-link.bst:glibc.bst .. tip:: When reconciling conflicting junction declarations to the same subproject, it is also possible to use a locally defined :mod:`link ` element from one subproject to override another junction to the same project in an adjacent subproject. Multiple project instances ~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, loading the same project more than once will result in a *conflicting junction error*. There are some use cases which demand that you load the same project more than once in the same build pipeline. In order to allow the loading of multiple instances of the same project in the same build pipeline, please refer to the :ref:`relevant project.conf documentation `. """ from buildstream import Element, ElementError from buildstream._pipeline import PipelineError from buildstream.types import FastEnum class _AliasMappingStrategy(FastEnum): NONE = "none" IDENTITY = "identity" # Element implementation for the 'junction' kind. class JunctionElement(Element): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # Junctions are not allowed any dependencies BST_FORBID_BDEPENDS = True BST_FORBID_RDEPENDS = True def configure(self, node): node.validate_keys(["path", "options", "overrides", "aliases", "map-aliases"]) self.path = node.get_str("path", default="") self.options = node.get_mapping("options", default={}) # The overrides dictionary has the target junction # to override as a key, and the ScalarNode of the # junction name as a value self.overrides = {} overrides_node = node.get_mapping("overrides", {}) for key, junction_name in overrides_node.items(): # Cannot override a subproject with the project itself # if junction_name.as_str() == self.name: raise ElementError( "{}: Attempt to override subproject junction '{}' with the overriding junction '{}' itself".format( junction_name.get_provenance(), key, junction_name.as_str() ), reason="override-junction-with-self", ) self.overrides[key] = junction_name # Map from subproject alias to local alias self.aliases = node.get_mapping("aliases", default={}) self.map_aliases = node.get_enum("map-aliases", _AliasMappingStrategy, default=_AliasMappingStrategy.NONE) def get_parent_alias(self, alias): parent_alias = self.aliases.get_str(alias, default=None) if parent_alias is None and self.map_aliases == _AliasMappingStrategy.IDENTITY: parent_alias = alias return parent_alias def preflight(self): pass def get_unique_key(self): # Junctions do not produce artifacts. get_unique_key() implementation # is still required for `bst source fetch`. return 1 def configure_sandbox(self, sandbox): raise PipelineError("Cannot build junction elements") def stage(self, sandbox): raise PipelineError("Cannot stage junction elements") def generate_script(self): raise PipelineError("Cannot build junction elements") def assemble(self, sandbox): raise PipelineError("Cannot build junction elements") # Plugin entry point def setup(): return JunctionElement apache-buildstream-27ae392/src/buildstream/plugins/elements/link.py000066400000000000000000000045251514607367700255360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan van Berkom """ link - Link elements ================================ This element is a link to another element, allowing one to create a symbolic element which will be resolved to another element. Overview -------- The only configuration allowed in a ``link`` element is the specified target :ref:`element name ` of the link. .. code:: yaml kind: link config: target: element.bst The ``link`` element can be used to refer to elements in subprojects, and can be used to symbolically link :mod:`junction ` elements as well as other elements. """ from buildstream import Element # Element implementation for the 'link' kind. class LinkElement(Element): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # Links are not allowed any dependencies or sources BST_FORBID_BDEPENDS = True BST_FORBID_RDEPENDS = True BST_FORBID_SOURCES = True def configure(self, node): node.validate_keys(["target"]) # Hold onto the node, keep it around for provenance. # self.target_node = node.get_scalar("target") def preflight(self): pass def get_unique_key(self): # This is only used early on but later discarded return 1 def configure_sandbox(self, sandbox): assert False, "link elements should be discarded at load time" def stage(self, sandbox): assert False, "link elements should be discarded at load time" def generate_script(self): assert False, "link elements should be discarded at load time" def assemble(self, sandbox): assert False, "link elements should be discarded at load time" # Plugin entry point def setup(): return LinkElement apache-buildstream-27ae392/src/buildstream/plugins/elements/manual.py000066400000000000000000000026021514607367700260500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ manual - Manual build element ============================= The most basic build element does nothing but allows users to add custom build commands to the array understood by the :mod:`BuildElement ` The empty configuration is as such: .. literalinclude:: ../../../src/buildstream/plugins/elements/manual.yaml :language: yaml See :ref:`built-in functionality documentation ` for details on common configuration options for build elements. """ from buildstream import BuildElement # Element implementation for the 'manual' kind. class ManualElement(BuildElement): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # Plugin entry point def setup(): return ManualElement apache-buildstream-27ae392/src/buildstream/plugins/elements/manual.yaml000066400000000000000000000017041514607367700263640ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Manual build element does not provide any default # build commands config: # Commands for configuring the software # configure-commands: [] # Commands for building the software # build-commands: [] # Commands for installing the software into a # destination folder # install-commands: [] # Commands for stripping installed binaries # strip-commands: - | %{strip-binaries} apache-buildstream-27ae392/src/buildstream/plugins/elements/script.py000066400000000000000000000042321514607367700261000ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jonathan Maw """ script - Run scripts to create output ===================================== This element allows one to run some commands to mutate the input and create some output. .. note:: Script elements may only specify build dependencies. See :ref:`the format documentation ` for more detail on specifying dependencies. The default configuration and possible options are as such: .. literalinclude:: ../../../src/buildstream/plugins/elements/script.yaml :language: yaml """ import buildstream # Element implementation for the 'script' kind. class ScriptElement(buildstream.ScriptElement): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" def configure(self, node): node.validate_keys(["commands", "root-read-only"]) self.add_commands("commands", node.get_str_list("commands")) self.set_work_dir() self.set_install_root() self.set_root_read_only(node.get_bool("root-read-only", default=False)) def configure_dependencies(self, dependencies): for dep in dependencies: # Determine the location to stage each element, default is "/" location = "/" if dep.config: dep.config.validate_keys(["location"]) location = dep.config.get_str("location", location) # Add each element to the layout self.layout_add(dep.element, dep.path, location) # Plugin entry point def setup(): return ScriptElement apache-buildstream-27ae392/src/buildstream/plugins/elements/script.yaml000066400000000000000000000027151514607367700264160ustar00rootroot00000000000000# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # The script element allows staging elements into specific locations # via it's "location" dependency configuration # # For example, if you want to stage "foo-tools.bst" into the "/" of # the sandbox at buildtime, and the "foo-system.bst" element into # the %{build-root}, you can do so as follows: # # build-depends: # - foo-tools.bst # - filename: foo-system.bst # config: # location: "%{build-root}" # # Note: the default of the "location" parameter is "/", so it is not # necessary to specify the location if you want to stage the # element in "/" # # Common script element variables variables: # Defines the directory commands will be run from. cwd: / # Script element configuration config: # Defines whether to run the sandbox with '/' read-only. # It is recommended to set root as read-only wherever possible. root-read-only: False # List of commands to run in the sandbox. commands: [] apache-buildstream-27ae392/src/buildstream/plugins/elements/stack.py000066400000000000000000000133301514607367700257000ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ stack - Symbolic Element for dependency grouping ================================================ Stack elements are simply a symbolic element used for representing a logical group of elements. All dependencies declared in stack elements must always be both :ref:`build and runtime dependencies `. **Example:** .. code:: yaml kind: stack # Declare all of your dependencies in the `depends` list. depends: - libc.bst - coreutils.bst .. note:: Unlike other elements, whose cache keys are a unique identifier of the contents of the artifacts they produce, stack elements do not produce any artifact content. Instead, the cache key of an artifact is a unique identifier for the assembly of its own dependencies. Using intermediate stacks ------------------------- Using a stack element at intermediate levels of your build graph allows you to abstract away some parts of your project into logical subsystems which elements can more conveniently depend on as a whole. In addition to the added convenience, it will allow you to more easily change the implementation of a subsystem later on, without needing to update many reverse dependencies to depend on new elements, or even allow you to conditionally implement a subsystem with various implementations depending on what :ref:`project options ` were specified at build time. Using toplevel stacks --------------------- Stack elements can also be useful as toplevel targets in your build graph to simply indicate all of the components which need to be built for a given system to be complete, or for your integration pipeline to be successful. Checking out and deploying toplevel stacks ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ In case that your software is built remotely, it is possible to checkout the built content of a stack on your own machine for the purposes of inspection or further deployment. To accomplish this, you will need to know the cache key of the stack element which was built remotely, possibly by inspecting the remote build log or by deriving it with an equally configured BuildStream project, and you will need read access to the artifact cache server which the build was uploaded to, this should be configured in your :ref:`user configuration file `. You can then checkout the remotely built stack using the :ref:`bst artifact checkout ` command and providing it with the :ref:`artifact name `: **Example:** .. code:: shell bst artifact checkout --deps build --pull --integrate \\ --directory `pwd`/checkout \\ project/stack/788da21e7c1b5818b7e7b60f7eb75841057ff7e45d362cc223336c606fe47f27 .. note:: It is possible to checkout other elements in the same way, however stack elements are uniquely suited to this purpose, as they cannot have :ref:`runtime only dependencies `, and consequently their cache keys are always a unique representation of their collective dependencies. """ from buildstream import Element, ElementError from buildstream.types import _Scope # Element implementation for the 'stack' kind. class StackElement(Element): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # This plugin does not produce any artifacts when built BST_ELEMENT_HAS_ARTIFACT = False # This element does not allow sources BST_FORBID_SOURCES = True # Stack elements do not run any commands BST_RUN_COMMANDS = False def configure(self, node): pass def preflight(self): # Assert that all dependencies are both build and runtime dependencies. # all_deps = list(self._dependencies(_Scope.ALL, recurse=False)) run_deps = list(self._dependencies(_Scope.RUN, recurse=False)) build_deps = list(self._dependencies(_Scope.BUILD, recurse=False)) if any(dep not in run_deps for dep in all_deps) or any(dep not in build_deps for dep in all_deps): # There is no need to specify the `self` provenance here in preflight() errors, as the base class # will take care of prefixing these for plugin author convenience. raise ElementError( "All dependencies of 'stack' elements must be both build and runtime dependencies", detail="Make sure you declare all dependencies in the `depends` list, without specifying any `type`.", reason="stack-requires-build-and-run", ) def get_unique_key(self): # We do not add anything to the build, only our dependencies # do, so our unique key is just a constant. return 1 def configure_sandbox(self, sandbox): pass def stage(self, sandbox): pass def assemble(self, sandbox): # Just create a dummy empty artifact, its existence is a statement # that all this stack's dependencies are built. vrootdir = sandbox.get_virtual_directory() vrootdir.open_directory("output", create=True) # And we're done return "/output" # Plugin entry point def setup(): return StackElement apache-buildstream-27ae392/src/buildstream/plugins/sourcemirrors/000077500000000000000000000000001514607367700253235ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/plugins/sourcemirrors/default.py000066400000000000000000000026601514607367700273250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Abderrahim Kitouni from typing import Optional, Dict, List, Any, TYPE_CHECKING from buildstream import SourceMirror, MappingNode, SequenceNode, SourceMirrorError class DefaultSourceMirror(SourceMirror): def configure(self, node: MappingNode) -> None: node.validate_keys(["name", "kind", "aliases"]) self._aliases = self._load_aliases(node) def _get_alias_uris(self, alias): if alias in self._aliases: return self._aliases[alias] return [] def _load_aliases(self, node: MappingNode) -> Dict[str, List[str]]: aliases: Dict[str, List[str]] = {} alias_node: MappingNode = node.get_mapping("aliases") for alias, uris in alias_node.items(): aliases[alias] = uris.as_str_list() return aliases def setup(): return DefaultSourceMirror apache-buildstream-27ae392/src/buildstream/plugins/sources/000077500000000000000000000000001514607367700240705ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/plugins/sources/__init__.py000066400000000000000000000000001514607367700261670ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/plugins/sources/local.py000066400000000000000000000127051514607367700255410ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Tiago Gomes """ local - stage local files and directories ========================================= **Usage:** .. code:: yaml # Specify the local source kind kind: local # Specify the project relative path to a file or directory path: files/somefile.txt See :ref:`built-in functionality doumentation ` for details on common configuration options for sources. Reporting :class:`.SourceInfo` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The local source reports the project relative path of the file or directory as the *url*. Further, the local source reports the :attr:`SourceInfoMedium.LOCAL ` *medium* and the :attr:`SourceVersionType.CAS_DIGEST ` *version_type*, for which it reports the CAS digest of the local source as the *version*. The *guess_version* of a local source is meaningless, as it is tied instead to the BuildStream project in which it is contained. """ import os from buildstream import Source, SourceError, SourceInfoMedium, SourceVersionType, Directory class LocalSource(Source): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" BST_STAGE_VIRTUAL_DIRECTORY = True __digest = None def configure(self, node): node.validate_keys(["path", *Source.COMMON_CONFIG_KEYS]) self.path = self.node_get_project_path(node.get_scalar("path")) self.fullpath = os.path.join(self.get_project_directory(), self.path) def preflight(self): pass def is_resolved(self): return True def is_cached(self): return True def get_unique_key(self): # # As a core plugin, we use some private API to optimize file hashing. # # * Use Source._cache_directory() to prepare a Directory # * Do the regular staging activity into the Directory # * Use the hash of the cached digest as the unique key # self.__ensure_digest() return self.__digest.hash # We dont have a ref, we're a local file... def load_ref(self, node): pass def get_ref(self): return None # pragma: nocover def set_ref(self, ref, node): pass # pragma: nocover def fetch(self): # pylint: disable=arguments-differ # Nothing to do here for a local source pass # pragma: nocover def stage_directory(self, directory): # # We've already prepared the CAS while resolving the cache key which # will happen before staging. # # Now just retrieve the previously cached content to stage. # assert isinstance(directory, Directory) assert self.__digest is not None with self._cache_directory(digest=self.__digest) as cached_directory: directory.import_files(cached_directory, collect_result=False) def init_workspace_directory(self, directory): # # FIXME: We should be able to stage the workspace from the content # cached in CAS instead of reimporting from the filesystem # to the new workspace directory with this special case, but # for some reason the writable bits are getting lost on regular # files through the transition. # self.__do_stage(directory) def collect_source_info(self): self.__ensure_digest() version = "{}/{}".format(self.__digest.hash, self.__digest.size_bytes) return [self.create_source_info(self.path, SourceInfoMedium.LOCAL, SourceVersionType.CAS_DIGEST, version)] # As a core element, we speed up some scenarios when this is used for # a junction, by providing the local path to this content directly. # def _get_local_path(self): return self.fullpath # Ensure that the digest is resolved # def __ensure_digest(self): if not self.__digest: with self._cache_directory() as directory: self.__do_stage(directory) self.__digest = directory._get_digest() # Staging is implemented internally, we preemptively put it in the CAS # as a side effect of resolving the cache key, at stage time we just # do an internal CAS stage. # def __do_stage(self, directory): with self.timed_activity("Staging local files into CAS"): if os.path.isdir(self.fullpath) and not os.path.islink(self.fullpath): result = directory.import_files(self.fullpath) else: result = directory.import_single_file(self.fullpath) if result.overwritten or result.ignored: raise SourceError( "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail" ) # Plugin entry point def setup(): return LocalSource apache-buildstream-27ae392/src/buildstream/plugins/sources/remote.py000066400000000000000000000064641514607367700257470ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Ed Baunton """ remote - stage files from remote urls ===================================== **Usage:** .. code:: yaml # Specify the remote source kind kind: remote # Optionally specify a relative staging filename. # If not specified, the basename of the url will be used. # filename: customfilename # Optionally specify whether the downloaded file should be # marked executable. # executable: true # Specify the url. Using an alias defined in your project # configuration is encouraged. 'bst source track' will update the # sha256sum in 'ref' to the downloaded file's sha256sum. url: upstream:foo # Specify the ref. It's a sha256sum of the file you download. ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b See :ref:`built-in base class functionality doumentation ` and :ref:`built-in downloadable file source functionality doumentation ` for details on common configuration options applicable to this source. Reporting :class:`.SourceInfo` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The remote source does not override any of the DownloadableFileSource reporting functionality and as such, behaves as described in the :ref:`default reporting of SourceInfo ` documentation. """ import os from buildstream import DownloadableFileSource, SourceError, utils class RemoteSource(DownloadableFileSource): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" def configure(self, node): super().configure(node) self.filename = node.get_str("filename", os.path.basename(self.url)) self.executable = node.get_bool("executable", default=False) if os.sep in self.filename: raise SourceError( "{}: filename parameter cannot contain directories".format(self), reason="filename-contains-directory" ) node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["filename", "executable"]) def get_unique_key(self): return super().get_unique_key() + [self.filename, self.executable] def stage(self, directory): # Same as in local plugin, don't use hardlinks to stage sources, they # are not write protected in the sandbox. dest = os.path.join(directory, self.filename) with self.timed_activity("Staging remote file to {}".format(dest)): utils.safe_copy(self._get_mirror_file(), dest) # To prevent user's umask introducing variability here, explicitly set # file modes. if self.executable: os.chmod(dest, 0o755) else: os.chmod(dest, 0o644) def setup(): return RemoteSource apache-buildstream-27ae392/src/buildstream/plugins/sources/tar.py000066400000000000000000000244261514607367700252400ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jonathan Maw """ tar - stage files from tar archives =================================== **Host dependencies:** * lzip (for .tar.lz files) **Usage:** .. code:: yaml # Specify the tar source kind kind: tar # Specify the tar url. Using an alias defined in your project # configuration is encouraged. 'bst source track' will update the # sha256sum in 'ref' to the downloaded file's sha256sum. url: upstream:foo.tar # Specify the ref. It's a sha256sum of the file you download. ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b # Specify a glob pattern to indicate the base directory to extract # from the tarball. The first matching directory will be used. # # Note that this is '*' by default since most standard release # tarballs contain a self named subdirectory at the root which # contains the files one normally wants to extract to build. # # To extract the root of the tarball directly, this can be set # to an empty string. base-dir: '*' See :ref:`built-in base class functionality doumentation ` and :ref:`built-in downloadable file source functionality doumentation ` for details on common configuration options applicable to this source. Reporting :class:`.SourceInfo` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The tar source does not override any of the DownloadableFileSource reporting functionality and as such, behaves as described in the :ref:`default reporting of SourceInfo ` documentation. """ import functools import os import sys import tarfile from contextlib import contextmanager from tempfile import TemporaryFile from typing import Optional from buildstream import DownloadableFileSource, SourceError from buildstream import utils class ReadableTarInfo(tarfile.TarInfo): """ The goal is to override `TarFile`'s `extractall` semantics by ensuring that on extraction, the files are readable by the owner of the file. This is done by overriding the accessor for the `mode` attribute in `TarInfo`, the class that encapsulates the internal meta-data of the tarball, so that the owner-read bit is always set. """ # https://github.com/python/mypy/issues/4125 @property # type: ignore def mode(self): # Respect umask instead of the file mode stored in the archive. # The only bit used from the embedded mode is the executable bit for files. umask = utils.get_umask() if self.isdir() or bool(self.__permission & 0o100): return 0o777 & ~umask else: return 0o666 & ~umask @mode.setter def mode(self, permission): self.__permission = permission # pylint: disable=attribute-defined-outside-init class TarSource(DownloadableFileSource): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" def configure(self, node): super().configure(node) self.base_dir = node.get_str("base-dir", "*") node.validate_keys(DownloadableFileSource.COMMON_CONFIG_KEYS + ["base-dir"]) def preflight(self): self.host_lzip = None if self.url.endswith(".lz"): self.host_lzip = utils.get_host_tool("lzip") def get_unique_key(self): return super().get_unique_key() + [self.base_dir] @contextmanager def _run_lzip(self): assert self.host_lzip with TemporaryFile() as lzip_stdout: with open(self._get_mirror_file(), "r") as lzip_file: self.call([self.host_lzip, "-d"], stdin=lzip_file, stdout=lzip_stdout) lzip_stdout.seek(0, 0) yield lzip_stdout @contextmanager def _get_tar(self): if self.url.endswith(".lz"): with self._run_lzip() as lzip_dec: with tarfile.open(fileobj=lzip_dec, mode="r:", tarinfo=ReadableTarInfo) as tar: yield tar else: with tarfile.open(self._get_mirror_file(), tarinfo=ReadableTarInfo) as tar: yield tar def stage(self, directory): try: with self._get_tar() as tar: base_dir = None if self.base_dir: base_dir = self._find_base_dir(tar, self.base_dir) if base_dir and not base_dir.endswith(os.sep): base_dir = base_dir + os.sep filter_function = functools.partial(self._extract_filter, base_dir) filtered_members = [] for member in tar.getmembers(): member = filter_function(member, directory) if member is not None: filtered_members.append(member) if sys.version_info >= (3, 12): tar.extractall(path=directory, members=filtered_members, filter="tar") else: tar.extractall(path=directory, members=filtered_members) except (tarfile.TarError, OSError) as e: raise SourceError("{}: Error staging source: {}".format(self, e)) from e # Assert that a tarfile is safe to extract; specifically, make # sure that we don't do anything outside of the target # directory (this is possible, if, say, someone engineered a # tarfile to contain paths that start with ..). def _assert_safe(self, member: tarfile.TarInfo, target_dir: str): final_path = os.path.abspath(os.path.join(target_dir, member.path)) if not final_path.startswith(target_dir): raise SourceError( "{}: Tarfile attempts to extract outside the staging area: " "{} -> {}".format(self, member.path, final_path) ) if member.islnk(): linked_path = os.path.abspath(os.path.join(target_dir, member.linkname)) if not linked_path.startswith(target_dir): raise SourceError( "{}: Tarfile attempts to hardlink outside the staging area: " "{} -> {}".format(self, member.path, final_path) ) # Don't need to worry about symlinks because they're just # files here and won't be able to do much harm once we are # in a sandbox. def _extract_filter( self, base_dir: Optional[str], member: tarfile.TarInfo, target_dir: str ) -> Optional[tarfile.TarInfo]: if base_dir: # Override and translate which filenames to extract L = len(base_dir) # First, ensure that a member never starts with `./` if member.path.startswith("./"): member.path = member.path[2:] if member.islnk() and member.linkname.startswith("./"): member.linkname = member.linkname[2:] # Now extract only the paths which match the normalized path if not member.path.startswith(base_dir): return None # Hardlinks are smart and collapse into the "original" # when their counterpart doesn't exist. This means we # only need to modify links to files whose location we # change. # # Since we assert that we're not linking to anything # outside the target directory, this should only ever # be able to link to things inside the target # directory, so we should cover all bases doing this. # if member.islnk() and member.linkname.startswith(base_dir): member.linkname = member.linkname[L:] member.path = member.path[L:] self._assert_safe(member, target_dir) # Skip device nodes if member.isdev(): return None return member # We want to iterate over all paths of a tarball, but getmembers() # is not enough because some tarballs simply do not contain the leading # directory paths for the archived files. def _list_tar_paths(self, tar): visited = set() for member in tar.getmembers(): # Remove any possible leading './', offer more consistent behavior # across tarballs encoded with or without a leading '.' member_name = member.name.lstrip("./") if not member.isdir(): # Loop over the components of a path, for a path of a/b/c/d # we will first visit 'a', then 'a/b' and then 'a/b/c', excluding # the final component components = member_name.split("/") for i in range(len(components) - 1): dir_component = "/".join([components[j] for j in range(i + 1)]) if dir_component not in visited: visited.add(dir_component) try: # Dont yield directory members which actually do # exist in the archive _ = tar.getmember(dir_component) except KeyError: if dir_component != ".": yield dir_component continue # Avoid considering the '.' directory, if any is included in the archive # this is to avoid the default 'base-dir: *' value behaving differently # depending on whether the tarball was encoded with a leading '.' or not if member_name == ".": continue yield member_name def _find_base_dir(self, tar, pattern): paths = self._list_tar_paths(tar) matches = sorted(list(utils.glob(paths, pattern))) if not matches: raise SourceError("{}: Could not find base directory matching pattern: {}".format(self, pattern)) return matches[0] def setup(): return TarSource apache-buildstream-27ae392/src/buildstream/plugins/sources/workspace.py000066400000000000000000000130361514607367700264430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ :orphan: workspace - stage an opened workspace directory =============================================== **Usage:** The workspace plugin must not be directly used. This plugin is used as the kind for a synthetic node representing the sources of an element with an open workspace. Reporting :class:`.SourceInfo` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The workspace source reports the project relative or absolute path to the open workspace as the *url*. Further, the workspace source reports the :attr:`SourceInfoMedium.WORKSPACE ` *medium* and the :attr:`SourceVersionType.CAS_DIGEST ` *version_type*, for which it reports the CAS digest of the workspace source as the *version*. The *guess_version* of a workspace source is meaningless and omitted. .. attention:: Observing a SourceInfo with the ``SourceInfoMedium.WORKSPACE`` in the output of :ref:`bst show --format %{source-info} ` is most likely undesirable, given that you are likely interested in observing the source provenance information of the project in a clean state rather than in a state with open workspaces. """ import os from buildstream import Source, SourceError, SourceInfoMedium, SourceVersionType, Directory, MappingNode from buildstream.types import SourceRef class WorkspaceSource(Source): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" BST_STAGE_VIRTUAL_DIRECTORY = True # the digest of the Directory following the import of the workspace __digest = None # the cache key of the last workspace build __last_build = None def configure(self, node: MappingNode) -> None: node.validate_keys(["path", "last_build", "kind"]) self.path = node.get_str("path") self.__last_build = node.get_str("last_build") def preflight(self) -> None: pass # pragma: nocover def is_cached(self): return True def is_resolved(self): return os.path.exists(self._get_local_path()) def get_unique_key(self): # # As a core plugin, we use some private API to optimize file hashing. # # * Use Source._cache_directory() to prepare a Directory # * Do the regular staging activity into the Directory # * Use the hash of the cached digest as the unique key # self.__ensure_digest() return self.__digest.hash def get_ref(self) -> None: return None def load_ref(self, node: MappingNode) -> None: pass # pragma: nocover def set_ref(self, ref: SourceRef, node: MappingNode) -> None: pass # pragma: nocover # init_workspace() # # Raises AssertionError: existing workspaces should not be reinitialized def init_workspace_directory(self, directory: Directory) -> None: raise AssertionError("Attempting to re-open an existing workspace") def fetch(self, *, previous_sources_dir=None) -> None: # pylint: disable=arguments-differ pass # pragma: nocover def stage_directory(self, directory): # # We've already prepared the CAS while resolving the cache key which # will happen before staging. # # Now just retrieve the previously cached content to stage. # assert isinstance(directory, Directory) assert self.__digest is not None with self._cache_directory(digest=self.__digest) as cached_directory: directory._import_files_internal(cached_directory, collect_result=False) def collect_source_info(self): self.__ensure_digest() version = "{}/{}".format(self.__digest.hash, self.__digest.size_bytes) return [self.create_source_info(self.path, SourceInfoMedium.WORKSPACE, SourceVersionType.CAS_DIGEST, version)] # As a core element, we speed up some scenarios when this is used for # a junction, by providing the local path to this content directly. # def _get_local_path(self) -> str: return self.path # Ensure that the digest is resolved # def __ensure_digest(self): if not self.__digest: with self._cache_directory() as directory: self.__do_stage(directory) self.__digest = directory._get_digest() # Staging is implemented internally, we preemptively put it in the CAS # as a side effect of resolving the cache key, at stage time we just # do an internal CAS stage. # def __do_stage(self, directory: Directory) -> None: assert isinstance(directory, Directory) with self.timed_activity("Staging local files"): result = directory._import_files_internal(self.path, properties=["mtime"]) assert result is not None if result.overwritten or result.ignored: raise SourceError( "Failed to stage source: files clash with existing directory", reason="ensure-stage-dir-fail" ) # Plugin entry point def setup(): return WorkspaceSource apache-buildstream-27ae392/src/buildstream/py.typed000066400000000000000000000000001514607367700224110ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/sandbox/000077500000000000000000000000001514607367700223625ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/sandbox/__init__.py000066400000000000000000000014261514607367700244760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat from .sandbox import Sandbox, SandboxCommandError from .sandbox import _SandboxFlags from ._sandboxremote import SandboxRemote from ._sandboxdummy import SandboxDummy apache-buildstream-27ae392/src/buildstream/sandbox/_config.py000066400000000000000000000151371514607367700243470ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jim MacArthur # Tristan Van Berkom # from typing import TYPE_CHECKING, Dict, Optional, Union from .._platform import Platform if TYPE_CHECKING: from ..node import Node, MappingNode # SandboxConfig # # The Sandbox configuration parameters, this object carries configuration # required to instantiate the correct type of sandbox, and assert that # the local or remote worker sandbox has the capabilities required. # # Args: # build_os: The build OS name # build_arch: A canonical machine architecture name, as defined by Platform.canonicalize_arch() # build_uid: The UID for the sandbox process # build_gid: The GID for the sandbox process # remote_apis_socket_path: The path to a UNIX socket providing REAPI access for nested remote execution # # If the build_uid or build_gid is unspecified, then the underlying sandbox implementation # does not guarantee what UID/GID will be used, but generally UID/GID 0 will be used in a # sandbox implementation which supports UID/GID control. # # If the build_uid or build_gid is specified, then the UID/GID is guaranteed to match # the specified UID/GID, if the underlying sandbox implementation does not support UID/GID # control, then an error will be raised when attempting to configure the sandbox. # class SandboxConfig: def __init__( self, *, build_os: str, build_arch: str, build_uid: Optional[int] = None, build_gid: Optional[int] = None, remote_apis_socket_path: Optional[str] = None, remote_apis_socket_action_cache_enable_update: bool = False ): self.build_os = build_os self.build_arch = build_arch self.build_uid = build_uid self.build_gid = build_gid self.remote_apis_socket_path = remote_apis_socket_path self.remote_apis_socket_action_cache_enable_update = remote_apis_socket_action_cache_enable_update # to_dict(): # # Represent the SandboxConfig as a dictionary. # # This dictionary will be stored in the corresponding artifact # whenever an artifact is cached. When loading an element from # an artifact, then this dict will be loaded as a MappingNode # and interpreted by SandboxConfig.new_from_node(). # # This function is also used to contribute to the owning element's cache key. # # Returns: # A dictionary representation of this SandboxConfig # def to_dict(self) -> Dict[str, Union[str, int, Dict]]: # Assign mandatory portions of the sandbox configuration # # /!\ No additional mandatory members can ever be added to # the sandbox configuration, as that would result in # breaking cache key stability. # sandbox_dict: Dict[str, Union[str, int, Dict]] = {"build-os": self.build_os, "build-arch": self.build_arch} # Assign optional portions of the sandbox configuration # # /!\ In order to preserve cache key stability, these attributes # are only ever added to the dictionary if they have been # explicitly set, unset values must not affect the dictionary. # if self.build_uid is not None: sandbox_dict["build-uid"] = self.build_uid if self.build_gid is not None: sandbox_dict["build-gid"] = self.build_gid if self.remote_apis_socket_path is not None: reapi_socket_dict: Dict[str, Union[str, bool]] = {"path": self.remote_apis_socket_path} if self.remote_apis_socket_action_cache_enable_update: reapi_socket_dict["action-cache-enable-update"] = True sandbox_dict["remote-apis-socket"] = reapi_socket_dict return sandbox_dict # new_from_node(): # # Instantiate a new SandboxConfig from YAML configuration. # # If the Platform is specified, then we expect to be loading # from project definitions, and some defaults will be derived # from the Platform. Otherwise, we expect to be loading from # a cached artifact, and values are expected to exist on the # given node. # # Args: # config: The YAML configuration node # platform: The host Platform instance, or None # # Returns: # A new SandboxConfig instance # @classmethod def new_from_node(cls, config: "MappingNode[Node]", *, platform: Optional[Platform] = None) -> "SandboxConfig": config.validate_keys(["build-uid", "build-gid", "build-os", "build-arch", "remote-apis-socket"]) build_os: str build_arch: str if platform: tmp = config.get_str("build-os", None) if tmp: build_os = tmp.lower() else: build_os = platform.get_host_os() tmp = config.get_str("build-arch", None) if tmp: build_arch = Platform.canonicalize_arch(tmp) else: build_arch = platform.get_host_arch() else: build_os = config.get_str("build-os") build_arch = config.get_str("build-arch") build_uid = config.get_int("build-uid", None) build_gid = config.get_int("build-gid", None) remote_apis_socket = config.get_mapping("remote-apis-socket", default=None) if remote_apis_socket: remote_apis_socket.validate_keys(["path", "action-cache-enable-update"]) remote_apis_socket_path = remote_apis_socket.get_str("path") remote_apis_socket_action_cache_enable_update = remote_apis_socket.get_bool( "action-cache-enable-update", default=False ) else: remote_apis_socket_path = None remote_apis_socket_action_cache_enable_update = False return cls( build_os=build_os, build_arch=build_arch, build_uid=build_uid, build_gid=build_gid, remote_apis_socket_path=remote_apis_socket_path, remote_apis_socket_action_cache_enable_update=remote_apis_socket_action_cache_enable_update, ) apache-buildstream-27ae392/src/buildstream/sandbox/_reremote.py000066400000000000000000000064571514607367700247310ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import grpc from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .._protos.build.buildgrid import local_cas_pb2 from .._cas import CASRemote from .._exceptions import RemoteError class RERemote(CASRemote): def __init__(self, cas_spec, remote_execution_specs, casd): super().__init__(cas_spec, casd) self.remote_execution_specs = remote_execution_specs self.exec_service = None self.operations_service = None self.ac_service = None def _configure_protocols(self): local_cas = self.casd.get_local_cas() request = local_cas_pb2.GetInstanceNameForRemotesRequest() if self.remote_execution_specs.storage_spec: self.remote_execution_specs.storage_spec.to_localcas_remote(request.content_addressable_storage) else: self.spec.to_localcas_remote(request.content_addressable_storage) request.content_addressable_storage.read_only = True if self.remote_execution_specs.exec_spec: self.remote_execution_specs.exec_spec.to_localcas_remote(request.execution) request.content_addressable_storage.read_only = False if self.remote_execution_specs.action_spec: self.remote_execution_specs.action_spec.to_localcas_remote(request.action_cache) if self.remote_execution_specs.action_spec.push: request.content_addressable_storage.read_only = False else: request.action_cache.read_only = True response = local_cas.GetInstanceNameForRemotes(request) self.local_cas_instance_name = response.instance_name self.exec_service = self.casd.get_exec_service() self.operations_service = self.casd.get_operations_service() self.ac_service = self.casd.get_ac_service() def _check(self): super()._check() if not self.remote_execution_specs.exec_spec: request = remote_execution_pb2.ExecuteRequest() request.instance_name = self.local_cas_instance_name try: for _ in self.exec_service.Execute(request): pass except grpc.RpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT: # Expected error as the request doesn't specify an action digest. pass elif e.code() == grpc.StatusCode.UNIMPLEMENTED: raise RemoteError( "buildbox-casd >= 1.3.23 is required to support local execution with a remote action cache" ) else: raise RemoteError( "Unexpected error in remote cache initialization {}: {}".format(e.code().name, e.details()) ) apache-buildstream-27ae392/src/buildstream/sandbox/_sandboxbuildboxrun.py000066400000000000000000000277231514607367700270220ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import signal import subprocess import sys from contextlib import ExitStack import grpc import psutil from .. import utils, _signals from . import _SandboxFlags from .._exceptions import SandboxError, SandboxUnavailableError from .._platform import Platform from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from ._reremote import RERemote from ._sandboxreapi import SandboxREAPI # SandboxBuildBoxRun() # # BuildBox-based sandbox implementation. # class SandboxBuildBoxRun(SandboxREAPI): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) context = self._get_context() casd = context.get_casd() re_specs = context.remote_execution_specs if re_specs and re_specs.action_spec: self.re_remote = RERemote(context.remote_cache_spec, re_specs, casd) try: self.re_remote.init() self.re_remote.check() except grpc.RpcError as e: urls = set() if re_specs.storage_spec: urls.add(re_specs.storage_spec.url) urls.add(re_specs.action_spec.url) raise SandboxError("Failed to contact remote cache endpoint at {}: {}".format(sorted(urls), e)) from e else: self.re_remote = None @classmethod def __buildbox_run(cls): return utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox") @classmethod def _setup(cls): try: path = cls.__buildbox_run() except utils.ProgramNotFoundError as e: raise SandboxUnavailableError("buildbox-run not found", reason="unavailable-local-sandbox") from e exit_code, output = utils._call([path, "--capabilities"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if exit_code == 0: # buildbox-run --capabilities prints one capability per line cls._capabilities = set(output.split("\n")) else: # buildbox-run is not functional raise SandboxError( "buildbox-run exited with code {}. Output: {}".format(exit_code, output), reason="buildbox-run-not-functional", ) osfamily_prefix = "platform:OSFamily=" cls._osfamilies = {cap[len(osfamily_prefix) :] for cap in cls._capabilities if cap.startswith(osfamily_prefix)} if not cls._osfamilies: # buildbox-run is too old to list supported OS families, # limit support to native building on the host OS. cls._osfamilies.add(Platform.get_host_os()) isa_prefix = "platform:ISA=" cls._isas = {cap[len(isa_prefix) :] for cap in cls._capabilities if cap.startswith(isa_prefix)} if not cls._isas: # buildbox-run is too old to list supported ISAs, # limit support to native building on the host ISA. cls._isas.add(Platform.get_host_arch()) @classmethod def check_sandbox_config(cls, config): if config.build_os not in cls._osfamilies: raise SandboxUnavailableError("OS '{}' is not supported by buildbox-run.".format(config.build_os)) if config.build_arch not in cls._isas: raise SandboxUnavailableError("ISA '{}' is not supported by buildbox-run.".format(config.build_arch)) if config.build_uid is not None and "platform:unixUID" not in cls._capabilities: raise SandboxUnavailableError("Configuring sandbox UID is not supported by buildbox-run.") if config.build_gid is not None and "platform:unixGID" not in cls._capabilities: raise SandboxUnavailableError("Configuring sandbox GID is not supported by buildbox-run.") if config.remote_apis_socket_path is not None and "platform:remoteApisSocketPath" not in cls._capabilities: raise SandboxUnavailableError("Configuring Remote APIs socket path is not supported by buildbox-run.") def _execute_action(self, action, flags): stdout, stderr = self._get_output() context = self._get_context() casd = context.get_casd() config = self._get_config() if config.remote_apis_socket_path and context.remote_cache_spec and not self.re_remote: raise SandboxError( "Using 'remote-apis-socket' with 'storage-service' requires 'action-cache-service' or 'execution-service' configured in the 'remote-execution' section." ) with utils._tempnamedfile() as action_file, utils._tempnamedfile() as result_file: action_file.write(action.SerializeToString()) action_file.flush() buildbox_command = [ self.__buildbox_run(), "--remote={}".format(casd._connection_string), "--action={}".format(action_file.name), "--action-result={}".format(result_file.name), ] if self.re_remote: buildbox_command.append("--instance={}".format(self.re_remote.local_cas_instance_name)) if config.remote_apis_socket_action_cache_enable_update: buildbox_command.append("--nested-ac-enable-update") # Do not redirect stdout/stderr if "no-logs-capture" in self._capabilities: buildbox_command.append("--no-logs-capture") marked_directories = self._get_marked_directories() mount_sources = self._get_mount_sources() for mount_point in marked_directories: mount_source = mount_sources.get(mount_point) if not mount_source: # Handled by the input tree in the action continue if "bind-mount" not in self._capabilities: context = self._get_context() context.messenger.warn("buildbox-run does not support host-files") break buildbox_command.append("--bind-mount={}:{}".format(mount_source, mount_point)) # If we're interactive, we want to inherit our stdin, # otherwise redirect to /dev/null, ensuring process # disconnected from terminal. if flags & _SandboxFlags.INTERACTIVE: stdin = sys.stdin if "bind-mount" in self._capabilities: # In interactive mode, we want a complete devpts inside # the container, so there is a /dev/console and such. buildbox_command.append("--bind-mount=/dev:/dev") else: stdin = subprocess.DEVNULL self._run_buildbox( buildbox_command, stdin, stdout, stderr, interactive=(flags & _SandboxFlags.INTERACTIVE), ) action_result = remote_execution_pb2.ActionResult().FromString(result_file.read()) if self.re_remote and context.remote_execution_specs.storage_spec and context.remote_cache_spec: # This ensures that the outputs are uploaded to the cache storage-service # in case different CAS remotes have been configured in the `cache` and `remote-execution` sections. self._fetch_action_result_outputs(self.re_remote, action_result) return action_result def _run_buildbox(self, argv, stdin, stdout, stderr, *, interactive): def kill_proc(): if process: # First attempt to gracefully terminate proc = psutil.Process(process.pid) proc.terminate() try: proc.wait(15) except psutil.TimeoutExpired: utils._kill_process_tree(process.pid) def suspend_proc(): group_id = os.getpgid(process.pid) os.killpg(group_id, signal.SIGSTOP) def resume_proc(): group_id = os.getpgid(process.pid) os.killpg(group_id, signal.SIGCONT) with ExitStack() as stack: # We want to launch buildbox-run in a new session in non-interactive # mode so that we handle the SIGTERM and SIGTSTP signals separately # from the nested process, but in interactive mode this causes # launched shells to lack job control as the signals don't reach # the shell process. # if interactive: new_session = False else: new_session = True stack.enter_context(_signals.suspendable(suspend_proc, resume_proc)) stack.enter_context(_signals.terminator(kill_proc)) process = subprocess.Popen( # pylint: disable=consider-using-with argv, close_fds=True, stdin=stdin, stdout=stdout, stderr=stderr, start_new_session=new_session, ) # Wait for the child process to finish, ensuring that # a SIGINT has exactly the effect the user probably # expects (i.e. let the child process handle it). try: while True: try: # Here, we don't use `process.wait()` directly without a timeout # This is because, if we were to do that, and the process would never # output anything, the control would never be given back to the python # process, which might thus not be able to check for request to # shutdown, or kill the process. # We therefore loop with a timeout, to ensure the python process # can act if it needs. returncode = process.wait(timeout=1) # If the process exits due to a signal, we # brutally murder it to avoid zombies if returncode < 0: utils._kill_process_tree(process.pid) except subprocess.TimeoutExpired: continue # Unlike in the bwrap case, here only the main # process seems to receive the SIGINT. We pass # on the signal to the child and then continue # to wait. except _signals.TerminateException: process.send_signal(signal.SIGINT) continue break # If we can't find the process, it has already died of # its own accord, and therefore we don't need to check # or kill anything. except psutil.NoSuchProcess: pass if interactive and stdin.isatty(): # Make this process the foreground process again, otherwise the # next read() on stdin will trigger SIGTTIN and stop the process. # This is required because the sandboxed process does not have # permission to do this on its own (running in separate PID namespace). # # tcsetpgrp() will trigger SIGTTOU when called from a background # process, so ignore it temporarily. handler = signal.signal(signal.SIGTTOU, signal.SIG_IGN) os.tcsetpgrp(0, os.getpid()) signal.signal(signal.SIGTTOU, handler) if returncode != 0: raise SandboxError("buildbox-run failed with returncode {}".format(returncode)) apache-buildstream-27ae392/src/buildstream/sandbox/_sandboxdummy.py000066400000000000000000000023461514607367700256120ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from .._exceptions import SandboxError from .sandbox import Sandbox, SandboxCommandError class SandboxDummy(Sandbox): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._reason = kwargs.get("dummy_reason", "no reason given") def _run(self, command, *, flags, cwd, env): if not self._has_command(command[0], env): raise SandboxCommandError( "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command" ) raise SandboxError( "This platform does not support local builds: {}".format(self._reason), reason="unavailable-local-sandbox" ) apache-buildstream-27ae392/src/buildstream/sandbox/_sandboxreapi.py000066400000000000000000000316451514607367700255630ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import shlex from .sandbox import Sandbox, _SandboxFlags, SandboxCommandError, _SandboxBatch from .. import utils from .._exceptions import ImplError, SandboxError from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 # SandboxREAPI() # # Abstract class providing a skeleton for sandbox implementations based on # the Remote Execution API. # class SandboxREAPI(Sandbox): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._output_node_properties = kwargs.get("output_node_properties") def _run(self, command, *, flags, cwd, env): context = self._get_context() cascache = context.get_cascache() # set up virtual dircetory vdir = self.get_virtual_directory() if not self._has_command(command[0], env): raise SandboxCommandError( "Staged artifacts do not provide command " "'{}'".format(command[0]), reason="missing-command" ) # Ensure working directory exists if len(cwd) > 1: assert cwd.startswith("/") vdir.open_directory(cwd[1:], create=True) # Ensure directories required for sandboxed execution exist for directory in ["dev", "proc", "tmp"]: vsubdir = vdir.open_directory(directory, create=True) if flags & _SandboxFlags.ROOT_READ_ONLY: vsubdir._set_subtree_read_only(False) # Create directories for all marked directories. This emulates # some of the behaviour of other sandboxes, which create these # to use as mount points. read_write_directories = [] mount_sources = self._get_mount_sources() for directory in self._get_marked_directories(): if directory in mount_sources: # Bind mount mount_point = directory.lstrip(os.path.sep) mount_source = mount_sources[directory] # Ensure mount point exists in sandbox if not vdir.exists(mount_point): if os.path.isdir(mount_source): # Mounting a directory, mount point must be a directory vdir.open_directory(mount_point, create=True) else: # Mounting a file or device node, mount point must be a file split_mount_point = mount_point.rsplit(os.path.sep, 1) parent_vdir = vdir.open_directory(split_mount_point[0], create=True) parent_vdir._create_empty_file(split_mount_point[1]) else: # Read-write directory marked_vdir = vdir.open_directory(directory.lstrip(os.path.sep), create=True) read_write_directories.append(directory) if flags & _SandboxFlags.ROOT_READ_ONLY: marked_vdir._set_subtree_read_only(False) if flags & _SandboxFlags.ROOT_READ_ONLY: vdir._set_subtree_read_only(True) else: # The whole sandbox is writable read_write_directories = [os.path.sep] # Generate Action proto input_root_digest = vdir._get_digest() platform = self._create_platform(flags) command_proto = self._create_command(command, cwd, env, read_write_directories, platform) command_digest = cascache.add_object(buffer=command_proto.SerializeToString()) action = remote_execution_pb2.Action( command_digest=command_digest, input_root_digest=input_root_digest, platform=platform ) action_result = self._execute_action(action, flags) # pylint: disable=assignment-from-no-return # Get output of build self._process_job_output( cwd, action_result.output_directories, action_result.output_files, failure=action_result.exit_code != 0 ) # Non-zero exit code means a normal error during the build: # the remote execution system has worked correctly but the command failed. return action_result.exit_code def _create_platform(self, flags): config = self._get_config() platform_dict = {} platform_dict["OSFamily"] = config.build_os platform_dict["ISA"] = config.build_arch if flags & _SandboxFlags.INHERIT_UID: uid = os.geteuid() gid = os.getegid() else: uid = config.build_uid gid = config.build_gid if uid is not None: platform_dict["unixUID"] = str(uid) if gid is not None: platform_dict["unixGID"] = str(gid) if flags & _SandboxFlags.NETWORK_ENABLED: platform_dict["network"] = "on" if config.remote_apis_socket_path: platform_dict["remoteApisSocketPath"] = config.remote_apis_socket_path.lstrip(os.path.sep) # Create Platform message with properties sorted by name in code point order platform = remote_execution_pb2.Platform() for key, value in sorted(platform_dict.items()): platform.properties.add(name=key, value=value) return platform def _create_command(self, command, working_directory, environment, read_write_directories, platform): # Creates a command proto environment_variables = [ remote_execution_pb2.Command.EnvironmentVariable(name=k, value=v) for (k, v) in environment.items() ] # Request read-write directories as output output_directories = [os.path.relpath(dir, start=working_directory) for dir in read_write_directories] return remote_execution_pb2.Command( arguments=command, working_directory=working_directory[1:], environment_variables=environment_variables, output_paths=output_directories, output_node_properties=self._output_node_properties, output_directory_format=remote_execution_pb2.Command.OutputDirectoryFormat.DIRECTORY_ONLY, platform=platform, ) def _fetch_action_result_outputs(self, casremote, action_result): # This also ensures that the outputs are uploaded to the cache # storage-service, if configured context = self._get_context() cascache = context.get_cascache() # Fetch outputs for output_directory in action_result.output_directories: # Now do a pull to ensure we have the full directory structure. # We first try the root_directory_digest we requested, then fall back to tree_digest root_directory_digest = output_directory.root_directory_digest if root_directory_digest and root_directory_digest.hash: cascache.fetch_directory(casremote, root_directory_digest) continue tree_digest = output_directory.tree_digest if tree_digest and tree_digest.hash: cascache.pull_tree(casremote, tree_digest) continue raise SandboxError("Output directory structure had no digest attached.") # Fetch stdout and stderr blobs, if they exist blobs = [] for digest in [action_result.stdout_digest, action_result.stderr_digest]: if digest.hash: blobs.append(digest) if blobs: cascache.fetch_blobs(casremote, blobs) def _process_job_output(self, working_directory, output_directories, output_files, *, failure): # Reads the remote execution server response to an execution request. # # output_directories is an array of OutputDirectory objects. # output_files is an array of OutputFile objects. # if output_files: raise SandboxError("Output files were returned when we didn't request any.") context = self._get_context() cascache = context.get_cascache() vdir = self.get_virtual_directory() for output_directory in output_directories: dir_digest = output_directory.root_directory_digest if dir_digest is None or not dir_digest.hash: tree_digest = output_directory.tree_digest if tree_digest is None or not tree_digest.hash: raise SandboxError("Output directory structure had no digest attached.") # Get digest of output directory from tree digest tree = remote_execution_pb2.Tree() with open(cascache.objpath(tree_digest), "rb") as f: tree.ParseFromString(f.read()) root_directory = tree.root.SerializeToString() dir_digest = utils._message_digest(root_directory) # Create a normalized absolute path (inside the input tree) path = os.path.normpath(os.path.join(working_directory, output_directory.path)).lstrip(os.path.sep) # Get virtual directory at the path of the output directory vsubdir = vdir.open_directory(path, create=True) # Replace contents with returned output vsubdir._reset(digest=dir_digest) def _create_batch(self, main_group, flags, *, collect=None): return _SandboxREAPIBatch(self, main_group, flags, collect=collect) def _execute_action(self, action, flags): raise ImplError("Sandbox of type '{}' does not implement _execute_action()".format(type(self).__name__)) # _SandboxREAPIBatch() # # Command batching by shell script generation. # class _SandboxREAPIBatch(_SandboxBatch): def __init__(self, sandbox, main_group, flags, *, collect=None): super().__init__(sandbox, main_group, flags, collect=collect) self.script = None self.first_command = None self.cwd = None self.env = None def execute(self): self.script = "" self.main_group.execute(self) first = self.first_command if first: context = self.sandbox._get_context() with context.messenger.timed_activity( "Running commands", detail=self.main_group.combined_label(), element_name=self.sandbox._get_element_name(), ): if ( self.sandbox._run_with_flags( ["sh", "-c", "-e", self.script], flags=self.flags, cwd=first.cwd, env=first.env ) != 0 ): raise SandboxCommandError("Command failed", collect=self.collect) def execute_group(self, group): group.execute_children(self) def execute_command(self, command): if self.first_command is None: # First command in batch # Initial working directory and environment of script already matches # the command configuration. self.first_command = command else: # Change working directory for this command if command.cwd != self.cwd: self.script += "mkdir -p {}\n".format(command.cwd) self.script += "cd {}\n".format(command.cwd) # Update environment for this command for key in self.env.keys(): if key not in command.env: self.script += "unset {}\n".format(key) for key, value in command.env.items(): if key not in self.env or self.env[key] != value: self.script += "export {}={}\n".format(key, shlex.quote(value)) # Keep track of current working directory and environment self.cwd = command.cwd self.env = command.env # Actual command execution cmdline = " ".join(shlex.quote(cmd) for cmd in command.command) self.script += "(set -ex; {})".format(cmdline) # Error handling label = command.label or cmdline quoted_label = shlex.quote("'{}'".format(label)) self.script += " || (echo Command {} failed with exitcode $? >&2 ; exit 1)\n".format(quoted_label) def create_empty_file(self, name): self.script += "touch -- {}\n".format(shlex.quote(name)) def clean_directory(self, name): # Do not treat error during cleanup as a fatal build error self.script += "rm -rf -- {} || true\n".format(shlex.quote(name)) if self.first_command: # Working directory may be a subdirectory of the build directory. # Recreate it if necessary as output capture requires the working directory to exist. self.script += "mkdir -p {} || true\n".format(shlex.quote(self.first_command.cwd)) apache-buildstream-27ae392/src/buildstream/sandbox/_sandboxremote.py000066400000000000000000000254411514607367700257530ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jim MacArthur import shutil import grpc from ._reremote import RERemote from ._sandboxreapi import SandboxREAPI from .. import _signals from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .._protos.google.rpc import code_pb2 from .._exceptions import SandboxError from .._protos.google.longrunning import operations_pb2 # SandboxRemote() # # This isn't really a sandbox, it's a stub which sends all the sources and build # commands to a remote server and retrieves the results from it. # class SandboxRemote(SandboxREAPI): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) context = self._get_context() casd = context.get_casd() specs = context.remote_execution_specs if specs is None or specs.exec_spec is None: return self.storage_spec = specs.storage_spec self.exec_spec = specs.exec_spec self.action_spec = specs.action_spec self.operation_name = None self.re_remote = RERemote(context.remote_cache_spec, specs, casd) try: self.re_remote.init() except grpc.RpcError as e: urls = set() if self.storage_spec: urls.add(self.storage_spec.url) urls.add(self.exec_spec.url) if self.action_spec: urls.add(self.action_spec.url) raise SandboxError("Failed to contact remote execution endpoint at {}: {}".format(sorted(urls), e)) from e def run_remote_command(self, action_digest): # Sends an execution request to the remote execution server. # # This function blocks until it gets a response from the server. stub = self.re_remote.exec_service request = remote_execution_pb2.ExecuteRequest( instance_name=self.re_remote.local_cas_instance_name, action_digest=action_digest, skip_cache_lookup=False ) def __run_remote_command(stub, execute_request=None, running_operation=None): try: last_operation = None if execute_request is not None: operation_iterator = stub.Execute(execute_request) else: request = remote_execution_pb2.WaitExecutionRequest(name=running_operation.name) operation_iterator = stub.WaitExecution(request) for operation in operation_iterator: if not self.operation_name: self.operation_name = operation.name if operation.done: return operation else: last_operation = operation except grpc.RpcError as e: status_code = e.code() if status_code in ( grpc.StatusCode.INVALID_ARGUMENT, grpc.StatusCode.FAILED_PRECONDITION, grpc.StatusCode.RESOURCE_EXHAUSTED, grpc.StatusCode.INTERNAL, grpc.StatusCode.DEADLINE_EXCEEDED, ): raise SandboxError( "Failed contacting remote execution server at {}." "{}: {}".format(self.exec_spec.url, status_code.name, e.details()) ) if running_operation and status_code == grpc.StatusCode.UNIMPLEMENTED: raise SandboxError( "Failed trying to recover from connection loss: " "server does not support operation status polling recovery." ) return last_operation # Set up signal handler to trigger cancel_operation on SIGTERM operation = None with self._get_context().messenger.timed_activity( "Waiting for the remote build to complete", element_name=self._get_element_name() ), _signals.terminator(self.cancel_operation): operation = __run_remote_command(stub, execute_request=request) if operation is None: return None elif operation.done: return operation while operation is not None and not operation.done: operation = __run_remote_command(stub, running_operation=operation) return operation def cancel_operation(self): # If we don't have the name can't send request. if self.operation_name is None: return stub = self.re_remote.operations_service request = operations_pb2.CancelOperationRequest(name=str(self.operation_name)) try: stub.CancelOperation(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.UNIMPLEMENTED or e.code() == grpc.StatusCode.INVALID_ARGUMENT: pass else: raise SandboxError( "Failed trying to send CancelOperation request: " "{} ({})".format(e.details(), e.code().name) ) def _fetch_missing_blobs(self, vdir): context = self._get_context() cascache = context.get_cascache() # Fetch the file blobs if self.storage_spec: dir_digest = vdir._get_digest() required_blobs = cascache.required_blobs_for_directory(dir_digest) local_missing_blobs = cascache.missing_blobs(required_blobs) if local_missing_blobs: cascache.fetch_blobs(self.re_remote, local_missing_blobs) def _execute_action(self, action, flags): stdout, stderr = self._get_output() context = self._get_context() cascache = context.get_cascache() action_digest = cascache.add_object(buffer=action.SerializeToString()) casremote = self.re_remote # check action cache download and download if there action_result = self._check_action_cache(action_digest) if not action_result: with self._get_context().messenger.timed_activity( "Uploading input root", element_name=self._get_element_name() ): # Determine blobs missing on remote root_digests = [action.input_root_digest] # Add virtual directories for subsandboxes for subsandbox in self._get_subsandboxes(): vdir = subsandbox.get_virtual_directory() root_digests.append(vdir._get_digest()) missing_blobs = [] try: for root_digest in root_digests: missing_blobs.extend(cascache.missing_blobs_for_directory(root_digest, remote=casremote)) except grpc.RpcError as e: raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e # Add command and action messages to blob list to push missing_blobs.append(action.command_digest) missing_blobs.append(action_digest) # Now, push the missing blobs to the remote. try: cascache.send_blobs(casremote, missing_blobs) except grpc.RpcError as e: raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e # Now request to execute the action operation = self.run_remote_command(action_digest) action_result = self._extract_action_result(operation) self._fetch_action_result_outputs(casremote, action_result) # Forward remote stdout and stderr if stdout: if action_result.stdout_digest.hash: with cascache.open(action_result.stdout_digest, "r") as f: shutil.copyfileobj(f, stdout) elif action_result.stdout_raw: stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore")) if stderr: if action_result.stderr_digest.hash: with cascache.open(action_result.stderr_digest, "r") as f: shutil.copyfileobj(f, stderr) elif action_result.stderr_raw: stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore")) return action_result def _check_action_cache(self, action_digest): # Checks the action cache to see if this artifact has already been built # # Should return either the action response or None if not found, raise # Sandboxerror if other grpc error was raised if not self.action_spec: return None request = remote_execution_pb2.GetActionResultRequest( instance_name=self.re_remote.local_cas_instance_name, action_digest=action_digest ) stub = self.re_remote.ac_service try: result = stub.GetActionResult(request) except grpc.RpcError as e: if e.code() != grpc.StatusCode.NOT_FOUND: raise SandboxError("Failed to query action cache: {} ({})".format(e.code(), e.details())) return None else: context = self._get_context() context.messenger.info("Action result found in action cache", element_name=self._get_element_name()) return result @staticmethod def _extract_action_result(operation): if operation is None: # Failure of remote execution, usually due to an error in BuildStream raise SandboxError("No response returned from server") assert not operation.HasField("error") and operation.HasField("response") execution_response = remote_execution_pb2.ExecuteResponse() # The response is expected to be an ExecutionResponse message assert operation.response.Is(execution_response.DESCRIPTOR) operation.response.Unpack(execution_response) if execution_response.status.code != code_pb2.OK: # An unexpected error during execution: the remote execution # system failed at processing the execution request. if execution_response.status.message: raise SandboxError(execution_response.status.message) # Otherwise, report the failure in a more general manner raise SandboxError("Remote server failed at executing the build request.") return execution_response.result apache-buildstream-27ae392/src/buildstream/sandbox/sandbox.py000066400000000000000000000603231514607367700243760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Andrew Leeming # Tristan Van Berkom """ Sandbox - The build sandbox =========================== :class:`.Element` plugins which want to interface with the sandbox need only understand this interface, while it may be given a different sandbox implementation, any sandbox implementation it is given will conform to this interface. See also: :ref:`sandboxing`. """ import os import shlex import contextlib from contextlib import contextmanager from typing import Dict, Generator, List, Optional, TYPE_CHECKING from .._exceptions import ImplError, SandboxError from ..storage.directory import Directory from ..storage._casbaseddirectory import CasBasedDirectory if TYPE_CHECKING: from typing import Union # pylint: disable=cyclic-import from .._context import Context from .._project import Project # pylint: enable=cyclic-import class SandboxCommandError(SandboxError): """Raised by :class:`.Sandbox` implementations when a command fails. Args: message (str): The error message to report to the user detail (str): The detailed error string collect (str): An optional directory containing partial install contents reason (str): An optional reason string (defaults to 'command-failed') """ def __init__(self, message, *, detail=None, collect=None, reason="command-failed"): super().__init__(message, detail=detail, reason=reason) self.collect = collect # An internal exception which can be used to explicitly trigger a bug / exception # which will be reported with a stack trace instead of reporting a user facing error # class _SandboxBug(Exception): pass class Sandbox: """Sandbox() Sandbox programming interface for :class:`.Element` plugins. """ # Minimal set of devices for the sandbox _dummy_reasons = [] # type: List[str] def __init__(self, context: "Context", project: "Project", **kwargs): self.__context = context self.__project = project self.__directories = [] # type: List[str] self.__cwd = None # type: Optional[str] self.__env = None # type: Optional[Dict[str, str]] self.__mount_sources = {} # type: Dict[str, str] self.__allow_run = True self.__subsandboxes = [] # type: List[Sandbox] # Plugin element full name for logging plugin = kwargs.get("plugin", None) if plugin: self.__element_name = plugin._get_full_name() else: parent = kwargs.get("parent", None) if parent: self.__element_name = parent._get_element_name() else: self.__element_name = None # Configuration from kwargs common to all subclasses self.__config = kwargs["config"] self.__stdout = kwargs["stdout"] self.__stderr = kwargs["stderr"] self._vdir = None # type: Optional[Directory] # Pending command batch self.__batch = None # __enter__() # # Called when entering the with-statement context. # def __enter__(self) -> "Sandbox": return self # __exit__() # # Called when exiting the with-statement context. # def __exit__(self, exc_type, exc_value, traceback) -> None: pass def get_virtual_directory(self) -> Directory: """Fetches the sandbox root directory as a virtual Directory. The root directory is where artifacts for the base runtime environment should be staged. Returns: The sandbox root directory """ if self._vdir is None: cascache = self.__context.get_cascache() self._vdir = CasBasedDirectory(cascache) return self._vdir def set_environment(self, environment: Dict[str, str]) -> None: """Sets the environment variables for the sandbox Args: environment: The environment variables to use in the sandbox """ self.__env = environment def set_work_directory(self, directory: str) -> None: """Sets the work directory for commands run in the sandbox Args: directory: An absolute path within the sandbox """ assert directory.startswith("/"), "The working directory must be an absolute path" self.__cwd = directory def mark_directory(self, directory: str) -> None: """Marks a sandbox directory and ensures it will exist Args: directory: An absolute path within the sandbox to mark .. note:: Any marked directories will be read-write in the sandboxed environment, only the root directory is allowed to be readonly. """ assert directory.startswith("/"), "The directories marked in the sandbox must be absolute paths" self.__directories.append(directory) def run( self, command: List[str], *, root_read_only: bool = False, cwd: Optional[str] = None, env: Optional[Dict[str, str]] = None, label: Optional[str] = None ) -> Optional[int]: """Run a command in the sandbox. If this is called outside a batch context, the command is immediately executed. If this is called in a batch context, the command is added to the batch for later execution. If the command fails, later commands will not be executed. Command flags must match batch flags. Args: command: The command to run in the sandboxed environment, as a list of strings starting with the binary to run. root_read_only: Whether the sandbox root should be readonly. cwd: The sandbox relative working directory in which to run the command. env: A dictionary of string key, value pairs to set as environment variables inside the sandbox environment. label: An optional label for the command, used for logging. Returns: The program exit code, or None if running in batch context. Raises: (:class:`.ProgramNotFoundError`): If a host tool which the given sandbox implementation requires is not found. .. note:: The optional *cwd* argument will default to the value set with :func:`~buildstream.sandbox.Sandbox.set_work_directory` and this function must make sure the directory will be created if it does not exist yet, even if a workspace is being used. """ if root_read_only: flags = _SandboxFlags.ROOT_READ_ONLY else: flags = _SandboxFlags.NONE return self._run_with_flags(command, flags=flags, cwd=cwd, env=env, label=label) @contextmanager def batch( self, *, root_read_only: bool = False, label: Optional[str] = None, collect: Optional[str] = None ) -> Generator[None, None, None]: """Context manager for command batching This provides a batch context that defers execution of commands until the end of the context. If a command fails, the batch will be aborted and subsequent commands will not be executed. Command batches may be nested. Execution will start only when the top level batch context ends. Args: root_read_only: Whether the sandbox root should be readonly. label: An optional label for the batch group, used for logging. collect: An optional directory containing partial install contents on command failure. Raises: (:class:`.SandboxCommandError`): If a command fails. """ if root_read_only: flags = _SandboxFlags.ROOT_READ_ONLY else: flags = _SandboxFlags.NONE group = _SandboxBatchGroup(label=label) if self.__batch: # Nested batch assert flags == self.__batch.flags, "Inconsistent sandbox flags in single command batch" parent_group = self.__batch.current_group parent_group.append(group) self.__batch.current_group = group try: yield finally: self.__batch.current_group = parent_group else: # Top-level batch batch = self._create_batch(group, flags, collect=collect) self.__batch = batch try: yield finally: self.__batch = None batch.execute() ##################################################### # Abstract Methods for Sandbox implementations # ##################################################### # _run() # # Abstract method for running a single command # # Args: # command (list): The command to run in the sandboxed environment, as a list # of strings starting with the binary to run. # flags (:class:`.SandboxFlags`): The flags for running this command. # cwd (str): The sandbox relative working directory in which to run the command. # env (dict): A dictionary of string key, value pairs to set as environment # variables inside the sandbox environment. # # Returns: # (int): The program exit code. # def _run(self, command, *, flags, cwd, env): raise ImplError("Sandbox of type '{}' does not implement _run()".format(type(self).__name__)) # _create_batch() # # Abstract method for creating a batch object. Subclasses can override # this method to instantiate a subclass of _SandboxBatch. # # Args: # main_group (:class:`_SandboxBatchGroup`): The top level batch group. # flags (:class:`.SandboxFlags`): The flags for commands in this batch. # collect (str): An optional directory containing partial install contents # on command failure. # def _create_batch(self, main_group, flags, *, collect=None): return _SandboxBatch(self, main_group, flags, collect=collect) # _fetch_missing_blobs() # # Fetch required file blobs missing from the local cache for sandboxes using # remote execution. This is a no-op for local sandboxes. # # Args: # vdir (Directory): The virtual directory whose blobs to fetch # def _fetch_missing_blobs(self, vdir): pass ################################################ # Private methods # ################################################ # _run_with_flags() # # An internal method for running commands, which exposes the private _SandboxFlags. # # Args: # command: The command to run in the sandboxed environment, as a list # of strings starting with the binary to run. # flags: The SandboxFlags for running this command. # cwd: The sandbox relative working directory in which to run the command. # env: A dictionary of string key, value pairs to set as environment # variables inside the sandbox environment. # label: An optional label for the command, used for logging. # # Returns: # (int): The program exit code, or None if running in batch context. # def _run_with_flags( self, command: List[str], *, flags: int, cwd: Optional[str] = None, env: Optional[Dict[str, str]] = None, label: Optional[str] = None ) -> Optional[int]: if not self.__allow_run: raise _SandboxBug("Element specified BST_RUN_COMMANDS as False but called Sandbox.run()") # Fallback to the sandbox default settings for # the cwd and env. # cwd = self._get_work_directory(cwd=cwd) env = self._get_environment(cwd=cwd, env=env) assert cwd.startswith("/"), "The working directory must be an absolute path" # Convert single-string argument to a list if isinstance(command, str): command = [command] if self.__batch: assert flags == self.__batch.flags, "Inconsistent sandbox flags in single command batch" batch_command = _SandboxBatchCommand(command, cwd=cwd, env=env, label=label) current_group = self.__batch.current_group current_group.append(batch_command) return None else: return self._run(command, flags=flags, cwd=cwd, env=env) # _get_context() # # Fetches the context BuildStream was launched with. # # Returns: # (Context): The context of this BuildStream invocation def _get_context(self): return self.__context # _get_project() # # Fetches the Project this sandbox was created to build for. # # Returns: # (Project): The project this sandbox was created for. def _get_project(self): return self.__project # _get_marked_directories() # # Fetches the marked directories in the sandbox # # Returns: # (List[str]): A list of marked directories. # def _get_marked_directories(self): return self.__directories # _get_mount_source() # # Fetches the list of mount sources # # Returns: # (dict): A dictionary where keys are mount points and values are the mount sources def _get_mount_sources(self): return self.__mount_sources # _set_mount_source() # # Sets the mount source for a given mountpoint # # Args: # mountpoint (str): The absolute mountpoint path inside the sandbox # mount_source (str): the host path to be mounted at the mount point def _set_mount_source(self, mountpoint, mount_source): self.__mount_sources[mountpoint] = mount_source # _get_environment() # # Fetches the environment variables for running commands # in the sandbox. # # Args: # cwd (str): The working directory the command has been requested to run in, if any. # env (str): The environment the command has been requested to run in, if any. # # Returns: # (str): The sandbox work directory def _get_environment(self, *, cwd=None, env=None): cwd = self._get_work_directory(cwd=cwd) if env is None: env = self.__env # Naive getcwd implementations can break when bind-mounts to different # paths on the same filesystem are present. Letting the command know # what directory it is in makes it unnecessary to call the faulty # getcwd. env = dict(env) env["PWD"] = cwd return env # _get_configured_environment() # # Return the environment exactly as configured with `set_environment()`, # or `None` if `set_environment()` has not been called. # # Returns # (Dict[str, str]): The configured environment def _get_configured_environment(self): return self.__env # _get_work_directory() # # Fetches the working directory for running commands # in the sandbox. # # Args: # cwd (str): The working directory the command has been requested to run in, if any. # # Returns: # (str): The sandbox work directory def _get_work_directory(self, *, cwd=None) -> str: return cwd or self.__cwd or "/" # _get_output() # # Fetches the stdout & stderr # # Returns: # (file): The stdout, or None to inherit # (file): The stderr, or None to inherit def _get_output(self): return (self.__stdout, self.__stderr) # _get_config() # # Fetches the sandbox configuration object. # # Returns: # (SandboxConfig): An object containing the configuration # data passed in during construction. def _get_config(self): return self.__config # _has_command() # # Tests whether a command exists inside the sandbox # # Args: # command (list): The command to test. # env (dict): A dictionary of string key, value pairs to set as environment # variables inside the sandbox environment. # Returns: # (bool): Whether a command exists inside the sandbox. def _has_command(self, command, env=None): vroot = self.get_virtual_directory() if os.path.isabs(command): return vroot.exists(command.lstrip(os.sep), follow_symlinks=True) if "/" in command: return False for path in env.get("PATH").split(":"): try_path = os.path.join(path, command).lstrip(os.sep) if vroot.exists(try_path, follow_symlinks=True): return True return False # _create_empty_file() # # Creates an empty file in the current working directory. # # If this is called outside a batch context, the file is created # immediately. # # If this is called in a batch context, creating the file is deferred. # # Args: # path (str): The path of the file to be created # def _create_empty_file(self, name): if self.__batch: batch_file = _SandboxBatchFile(name) current_group = self.__batch.current_group current_group.append(batch_file) else: vdir = self.get_virtual_directory() cwd = self._get_work_directory() cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True) cwd_vdir._create_empty_file(name) # _clean_directory() # # Remove the contents of the specified directory. # # Args: # path (str): The path of the directory to be cleaned # def _clean_directory(self, path): if self.__batch: batch_clean = _SandboxBatchCleanDirectory(path) current_group = self.__batch.current_group current_group.append(batch_clean) else: vdir = self.get_virtual_directory() relative_path = path.lstrip(os.sep) if vdir.exists(relative_path): vdir.remove(relative_path, recursive=True) vdir.open_directory(relative_path, create=True) # _get_element_name() # # Get the plugin's element full name # def _get_element_name(self): return self.__element_name # _disable_run() # # Raise exception if `Sandbox.run()` is called. # # This enforces an invariant by raising an exception if an element # plugin ever set BST_RUN_COMMANDS to False but then proceeded to # attempt to run the sandbox at assemble time. # def _disable_run(self): self.__allow_run = False # _create_subsandbox() # # Create an empty sandbox # # This allows an element to use a secondary sandbox for manipulating artifacts # that does not affect the build sandbox. # def _create_subsandbox(self, **kwargs): sub = Sandbox( self.__context, self.__project, parent=self, stdout=self.__stdout, stderr=self.__stderr, config=self.__config, ) self.__subsandboxes.append(sub) return sub def _get_subsandboxes(self): return self.__subsandboxes # SandboxFlags() # # Flags indicating how the sandbox should be run. # class _SandboxFlags: # Use default sandbox configuration. # NONE = 0 # Whether the root filesystem should be readonly. # # Usually this is true except for when running integration commands ROOT_READ_ONLY = 0x01 # Whether to expose host network. # # This should not be set when running builds, but can # be allowed for running a shell in a sandbox. NETWORK_ENABLED = 0x02 # Whether to run the sandbox interactively. # # This determines if the sandbox should attempt to connect # the terminal through to the calling process, or detach # the terminal entirely. INTERACTIVE = 0x04 # Whether to use the user id and group id from the host environment. # # This determines if processes in the sandbox should run with the # same user id and group id as BuildStream itself. By default, # processes run with user id and group id 0, protected by a user # namespace where available. INHERIT_UID = 0x08 # _SandboxBatch() # # A batch of sandbox commands. # class _SandboxBatch: def __init__(self, sandbox, main_group, flags, *, collect=None): self.sandbox = sandbox self.main_group = main_group self.current_group = main_group self.flags = flags self.collect = collect def execute(self): self.main_group.execute(self) def execute_group(self, group): if group.label: context = self.sandbox._get_context() cm = context.messenger.timed_activity(group.label, element_name=self.sandbox._get_element_name()) else: cm = contextlib.suppress() with cm: group.execute_children(self) def execute_command(self, command): if command.label: context = self.sandbox._get_context() context.messenger.status( "Running command", detail=command.label, element_name=self.sandbox._get_element_name(), ) exitcode = self.sandbox._run(command.command, flags=self.flags, cwd=command.cwd, env=command.env) if exitcode != 0: cmdline = " ".join(shlex.quote(cmd) for cmd in command.command) label = command.label or cmdline raise SandboxCommandError( "Command failed with exitcode {}".format(exitcode), detail=label, collect=self.collect ) def create_empty_file(self, name): vdir = self.sandbox.get_virtual_directory() cwd = self.sandbox._get_work_directory() cwd_vdir = vdir.open_directory(cwd.lstrip(os.sep), create=True) cwd_vdir._create_empty_file(name) def clean_directory(self, name): vdir = self.sandbox.get_virtual_directory() relative_path = name.lstrip(os.sep) if vdir.exists(relative_path): vdir.remove(relative_path, recursive=True) vdir.open_directory(relative_path, create=True) # _SandboxBatchItem() # # An item in a command batch. # class _SandboxBatchItem: def __init__(self, *, label=None): self.label = label def combined_label(self): return self.label # _SandboxBatchCommand() # # A command item in a command batch. # class _SandboxBatchCommand(_SandboxBatchItem): def __init__(self, command, *, cwd, env, label=None): super().__init__(label=label) self.command = command self.cwd = cwd self.env = env def execute(self, batch): batch.execute_command(self) # _SandboxBatchGroup() # # A group in a command batch. # class _SandboxBatchGroup(_SandboxBatchItem): def __init__(self, *, label=None): super().__init__(label=label) self.children = [] def append(self, item): self.children.append(item) def execute(self, batch): batch.execute_group(self) def execute_children(self, batch): for item in self.children: item.execute(batch) def combined_label(self): return "\n".join(filter(None, (item.combined_label() for item in self.children))) # _SandboxBatchFile() # # A file creation item in a command batch. # class _SandboxBatchFile(_SandboxBatchItem): def __init__(self, name): super().__init__() self.name = name def execute(self, batch): batch.create_empty_file(self.name) # _SandboxBatchCleanDirectory() # # A directory cleaning item in a command batch. # class _SandboxBatchCleanDirectory(_SandboxBatchItem): def __init__(self, name): super().__init__() self.name = name def execute(self, batch): batch.clean_directory(self.name) apache-buildstream-27ae392/src/buildstream/scriptelement.py000066400000000000000000000232531514607367700241610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jonathan Maw """ ScriptElement - Abstract class for scripting elements ===================================================== The ScriptElement class is a convenience class one can derive for implementing elements that stage elements and run command-lines on them. Any derived classes must write their own configure() implementation, using the public APIs exposed in this class. Derived classes must also chain up to the parent method in their preflight() implementations. """ import os from collections import OrderedDict from typing import List, Optional, TYPE_CHECKING from .element import Element if TYPE_CHECKING: from typing import Dict, Tuple class ScriptElement(Element): __install_root = "/" __cwd = "/" __root_read_only = False __commands = None # type: OrderedDict[str, List[str]] __layout = {} # type: Dict[str, List[Tuple[Element, str]]] # The compose element's output is its dependencies, so # we must rebuild if the dependencies change even when # not in strict build plans. # BST_STRICT_REBUILD = True # Script artifacts must never have indirect dependencies, # so runtime dependencies are forbidden. BST_FORBID_RDEPENDS = True # This element ignores sources, so we should forbid them from being # added, to reduce the potential for confusion BST_FORBID_SOURCES = True ############################################################# # Public Methods # ############################################################# def set_work_dir(self, work_dir: Optional[str] = None) -> None: """Sets the working dir The working dir (a.k.a. cwd) is the directory which commands will be called from. Args: work_dir: The working directory. If called without this argument set, it'll default to the value of the variable ``cwd``. """ if work_dir is None: self.__cwd = self.get_variable("cwd") or "/" else: self.__cwd = work_dir def set_install_root(self, install_root: Optional[str] = None) -> None: """Sets the install root The install root is the directory which output will be collected from once the commands have been run. Args: install_root: The install root. If called without this argument set, it'll default to the value of the variable ``install-root``. """ if install_root is None: self.__install_root = self.get_variable("install-root") or "/" else: self.__install_root = install_root def set_root_read_only(self, root_read_only: bool) -> None: """Sets root read-only When commands are run, if root_read_only is true, then the root of the filesystem will be protected. This is strongly recommended whenever possible. If this variable is not set, the default permission is read-write. Args: root_read_only: Whether to mark the root filesystem as read-only. """ self.__root_read_only = root_read_only def layout_add(self, element: Element, dependency_path: str, location: str) -> None: """Adds an element to the layout. Layout is a way of defining how dependencies should be added to the staging area for running commands. Args: element (Element): The element to stage. dependency_path (str): The element relative path to the dependency, usually obtained via :attr:`the dependency configuration ` location (str): The path inside the staging area for where to stage this element. If it is not "/", then integration commands will not be run. If this function is never called, then the default behavior is to just stage the build dependencies of the element in question at the sandbox root. Otherwise, the specified elements including their runtime dependencies will be staged in their respective locations. .. note:: The order of directories in the layout is not significant. The paths in the layout will be sorted so that elements are staged in parent directories before subdirectories. The elements for each respective staging directory in the layout will be staged in the predetermined deterministic staging order. """ # # Even if this is an empty dict by default, make sure that it is # instance data instead of appending stuff directly onto class data. # if not self.__layout: self.__layout = {} # Get or create the element list try: element_list = self.__layout[location] except KeyError: element_list = [] self.__layout[location] = element_list element_list.append((element, dependency_path)) def add_commands(self, group_name: str, command_list: List[str]) -> None: """Adds a list of commands under the group-name. .. note:: Command groups will be run in the order they were added. .. note:: This does not perform substitutions automatically. They must be performed beforehand (see :func:`~buildstream.element.Element.node_subst_list`) Args: group_name (str): The name of the group of commands. command_list (list): The list of commands to be run. """ if not self.__commands: self.__commands = OrderedDict() self.__commands[group_name] = command_list ############################################################# # Abstract Method Implementations # ############################################################# def preflight(self): pass def get_unique_key(self): sorted_locations = sorted(self.__layout) layout_key = { location: [dependency_path for _, dependency_path in self.__layout[location]] for location in sorted_locations } return { "commands": self.__commands, "cwd": self.__cwd, "install-root": self.__install_root, "layout": layout_key, "root-read-only": self.__root_read_only, } def configure_sandbox(self, sandbox): # Setup the environment and work directory sandbox.set_work_directory(self.__cwd) # Setup environment sandbox.set_environment(self.get_environment()) # Mark the install root sandbox.mark_directory(self.__install_root) def stage(self, sandbox): # If self.layout_add() was never called, do the default staging of # everything in "/" and run the integration commands if not self.__layout: with self.timed_activity("Staging dependencies", silent_nested=True): self.stage_dependency_artifacts(sandbox) with sandbox.batch(label="Integrating sandbox"): for dep in self.dependencies(): dep.integrate(sandbox) else: # First stage it all # sorted_locations = sorted(self.__layout) for location in sorted_locations: with self.timed_activity("Staging dependencies at: {}".format(location), silent_nested=True): element_list = [element for element, _ in self.__layout[location]] self.stage_dependency_artifacts(sandbox, element_list, path=location) # Now integrate any elements staged in the root # root_list = self.__layout.get("/", None) if root_list: element_list = [element for element, _ in root_list] with self.timed_activity("Integrating sandbox", silent_nested=True), sandbox.batch(): for dep in self.dependencies(element_list): dep.integrate(sandbox) # Ensure the install root exists # sandbox.get_virtual_directory().open_directory(self.__install_root.lstrip(os.sep), create=True) def assemble(self, sandbox): with sandbox.batch(root_read_only=self.__root_read_only, collect=self.__install_root): for groupname, commands in self.__commands.items(): with sandbox.batch(root_read_only=self.__root_read_only, label="Running '{}'".format(groupname)): for cmd in commands: # Note the -e switch to 'sh' means to exit with an error # if any untested command fails. sandbox.run(["sh", "-c", "-e", cmd + "\n"], root_read_only=self.__root_read_only, label=cmd) # Empty the build directory after a successful build to avoid the # overhead of capturing the build directory. self.run_cleanup_commands(sandbox) # Return where the result can be collected from return self.__install_root def setup(): return ScriptElement apache-buildstream-27ae392/src/buildstream/source.py000066400000000000000000002341021514607367700226000ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ Source - Base source class ========================== .. _core_source_builtins: Built-in functionality ---------------------- The Source base class provides built in keys which can be set when instantiating any Source. * Directory The ``directory`` variable can be set for all sources of a type in project.conf or per source within a element. This sets the location within the build root that the content of the source will be loaded in to. If the location does not exist, it will be created. * Provenance The ``provenance`` attribute depicts a dictionary which is used for users to provide additional source provenance related metadata which will later be reported in :class:`.SourceInfo` objects. The ``provenance`` dictionary itself does not have any specific required keys. Any attribute used in the ``provenance`` dictionary of a source must be defined in the project.conf using the ``source-provenance-attributes`` dictionary to define the attribute and its significance. Unless ``source-provenance-attributes`` is purposely defined as an empty dictionary, BuildStream will default it's values to those found in :ref:`builtin defaults `. *Since: 2.5* .. _core_source_abstract_methods: Abstract Methods ---------------- For loading and configuration purposes, Sources must implement the :ref:`Plugin base class abstract methods `. .. attention:: In order to ensure that all configuration data is processed at load time, it is important that all URLs have been processed during :func:`Plugin.configure() `. Source implementations *must* either call :func:`Source.translate_url() ` or :func:`Source.mark_download_url() ` for every URL that has been specified in the configuration during :func:`Plugin.configure() ` Sources expose the following abstract methods. Unless explicitly mentioned, these methods are mandatory to implement. * :func:`Source.load_ref() ` Load the ref from a specific YAML node * :func:`Source.get_ref() ` Fetch the source ref * :func:`Source.set_ref() ` Set a new ref explicitly * :func:`Source.track() ` Automatically derive a new ref from a symbolic tracking branch * :func:`Source.fetch() ` Fetch the actual payload for the currently set ref * :func:`Source.stage() ` / :func:`Source.stage_directory() ` Stage the sources for a given ref at a specified location * :func:`Source.init_workspace() ` / :func:`Source.init_workspace_workspace() ` Stage sources for use as a workspace. **Optional**: If left unimplemented, these will default to calling :func:`Source.stage() ` / :func:`Source.stage_directory() ` * :func:`Source.get_source_fetchers() ` Get the objects that are used for fetching. **Optional**: This only needs to be implemented for sources that need to download from multiple URLs while fetching (e.g. a git repo and its submodules). For details on how to define a SourceFetcher, see :ref:`SourceFetcher `. * :func:`Source.validate_cache() ` Perform any validations which require the sources to be cached. **Optional**: This is completely optional and will do nothing if left unimplemented. * :func:`Source.collect_source_info() ` Collect SourceInfo objects to describe the provenance of sources. **Optional**: BuildStream will function correctly if this is unimplemented, but the ability to generate SBoMs will be impaired, it is highly recommented to implement this. See: :ref:`documentation on generating SourceInfo `. .. _core_source_ref: Working with the source ref --------------------------- The :attr:`~buildstream.types.SourceRef` is used to determine the exact version of data to be addressed by the source. The various responsibilities involving the source reference are described here. Loading and saving ~~~~~~~~~~~~~~~~~~ The source reference is expected to be loaded at :func:`Plugin.configure() ` and and :func:`Source.load_ref() ` time from the provided :class:`.MappingNode`. The :attr:`~buildstream.types.SourceRef` should be loaded from a `single key` in that node, the recommended name for that key is `ref`, but is ultimately up to the implementor to decide. When :func:`Source.set_ref() ` is called, the source reference should be assigned to the `same single key` in the provided :class:`.MappingNode`, this will be used to serialize changed source references to YAML as a result of :ref:`tracking `. Tracking new references ~~~~~~~~~~~~~~~~~~~~~~~ When the user :ref:`tracks ` for new versions of the source, then the new :attr:`~buildstream.types.SourceRef` should be returned from the :func:`Source.track() ` implementation. Managing internal state ~~~~~~~~~~~~~~~~~~~~~~~ Internally the source implementation is expected to keep track of its :attr:`~buildstream.types.SourceRef`. The internal state should be updated when :func:`Plugin.configure() `, :func:`Source.load_ref() ` or :func:`Source.set_ref() ` is called. The internal state should not be updated when :func:`Source.track() ` is called. The internal source ref must be returned on demand whenever :func:`Source.get_ref() ` is called. Generating the unique key ~~~~~~~~~~~~~~~~~~~~~~~~~ When :func:`Plugin.get_unique_key() ` is called, the source's :attr:`~buildstream.types.SourceRef` must be considered as a part of that key. The unique key will be used to generate the cache key of :ref:`cache keys ` of elements using this source, and so the unique key should be comprised of every configuration which may effect how the source is :func:`staged `, as well as any configuration which uniquely identifies the source, which of course includes the :attr:`~buildstream.types.SourceRef`. When plugins :ref:`generate SourceInfo `, it is also important that any configuration attributes which contribute to the generation of SourceInfo also be included in the unique key. Accessing previous sources -------------------------- In the general case, all sources are fetched and tracked independently of one another. In situations where a source needs to access previous source(s) in order to perform its own track and/or fetch, following attributes can be set to request access to previous sources: * :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK` Indicate that access to previous sources is required during track * :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH` Indicate that access to previous sources is required during fetch The intended use of such plugins is to fetch external dependencies of other sources, typically using some kind of package manager, such that all the dependencies of the original source(s) are available at build time. When implementing such a plugin, implementors should adhere to the following guidelines: * Implementations must be able to store the obtained artifacts in a subdirectory. * Implementations must be able to deterministically generate a unique ref, such that two refs are different if and only if they produce different outputs. * Implementations must not introduce host contamination. .. _core_source_info: Generating SourceInfo for provenance information ------------------------------------------------ Source plugins should implement either of the :func:`Source.collect_source_info() ` or :func:`SourceFetcher.get_source_info() ` methods in order to properly report provenance information and contribute to reports such as SBoMs. To implement these methods, you must use :func:`Source.create_source_info() ` to instantiate the :class:`.SourceInfo` object to return from these methods. .. attention:: It is **not** recommented to consider the parameters used for implementing tracking with :func:`Source.track() `. Instead, any versioning information reported should be congruent with the URL and the *current* :ref:`source reference `. Furthermore, if any of the configuration attributes implemented by the plugin contribute to the generation of the SourceInfo objects, these configuration values must be considered in the plugin's :func:`Plugin.get_unique_key() ` implementation. What follows here, are some guidelines and conventions for doing this properly. The URL ~~~~~~~ The URL argument represents the location from which the source is obtained, and should normally be the translated URL, as returned by :func:`Source.translate_url() `. In the case of ``SourceInfoMedium.LOCAL``, the URL can instead be a project relative path to the local data. The medium and version_type arguments ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ These refer to the medium by which the source data was obtained, and the meaning/type of the following "version" argument, respectively. When possible, you should use the :class:`.SourceInfoMedium` and :class:`.SourceVersionType` values which correspond to the the medium and version type which your Source plugin is using. In cases where there is not a suitable value available for your plugin, you can alternatively provide a freeform string which provides these. Documentation ''''''''''''' Your plugin's module level docstring which is used for documenting your plugin, should have a section describing the meaning of these values. This is especially useful to promote interoperability with other tooling, which might want to perform some automations based on the :class:`.SourceInfo` object(s) which your plugin reports. Version ~~~~~~~ This is a string which uniquely identifies the version of the source, and its meaning is described by the "version_type" you specified. Version guess ~~~~~~~~~~~~~ This is a human readable simplified version, more suitable for a cursory reading of a report like an SBoM. Since it is, in most cases not possible to accurately automate the version string intended by upstream maintainers based on the knowledge you have, we refer to this as a *guessed version*. For example, just because you have a tarball named ``pony-1.2.3.tgz`` somewhere, does not guarantee that this is really version ``1.2.3`` of the "pony" project. Configurability ''''''''''''''' When implementing a technique for guessing the version based on the information you have at hand, it is recommended to provide some flexability to users of your plugin, who may have better knowledge about the conventions used by the upstream project and how they choose to express their versioning information. An example of this is the ``version-guess-pattern`` configuration made available in the :ref:`DownloadableFileSource built-in functionality `. Explicit versioning ''''''''''''''''''' In some use cases, it is impossible to derive a guessed version from the information available to the plugin. For instance, consider an upstream which indexes their releases on a web page and then hosts their releases without namespacing their release archives. In such a case you might have a URL that looks something like: ``https://flying-ponies.com/releases/9d0c936c78/pony-flight-release.tgz`` For this reason, the implementing plugin should provide a way for users to manually annotate the source version. An example of this is the ``version`` configuration made available in the :ref:`DownloadableFileSource built-in functionality `. Extra data ~~~~~~~~~~ In the case that the existing fields are insufficient to accurately describe the provenance of this source, extra key/values can be specified when calling :func:`Source.create_source_info() `. .. _core_source_fetcher: SourceFetcher - Object for fetching individual URLs =================================================== Abstract Methods ---------------- SourceFetchers expose the following abstract methods. Unless explicitly mentioned, these methods are mandatory to implement. * :func:`SourceFetcher.fetch() ` Fetches the URL associated with this SourceFetcher, optionally taking an alias override. * :func:`SourceFetcher.get_source_info() ` Get a SourceInfo object to describe the provenance of this source. **Optional**: BuildStream will function correctly if this is unimplemented, but the ability to generate SBoMs will be impaired, it is highly recommented to implement this. Class Reference --------------- """ import os from contextlib import contextmanager from typing import ( Iterable, Iterator, Optional, Tuple, Dict, Any, Set, TYPE_CHECKING, Union, ) from dataclasses import dataclass from . import _yaml, utils from .node import MappingNode, ScalarNode from .plugin import Plugin from .sourcemirror import SourceMirror from .types import SourceRef, CoreWarnings, FastEnum from ._exceptions import BstError, ImplError, PluginError, LoadError from .exceptions import ErrorDomain, LoadErrorReason from ._loader.metasource import MetaSource from ._projectrefs import ProjectRefStorage from ._cachekey import generate_key from .storage import CasBasedDirectory from .storage import FileBasedDirectory from .storage.directory import Directory from ._variables import Variables if TYPE_CHECKING: # pylint: disable=cyclic-import from ._context import Context from ._project import Project # pylint: enable=cyclic-import class SourceError(BstError): """This exception should be raised by :class:`.Source` implementations to report errors to the user. Args: message: The breif error description to report to the user detail: A possibly multiline, more detailed error message reason: An optional machine readable reason string, used for test cases temporary: An indicator to whether the error may occur if the operation was run again. """ def __init__( self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False ): super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary) @dataclass class AliasSubstitution: """AliasSubstitution() An opaque data structure which may be passed through :func:`SourceFetcher.fetch() ` and in such cases must be provided to :func:`Source.translate_url() `. """ _effective_alias: str _mirror: Union[SourceMirror, str] class SourceInfoMedium(FastEnum): """ Indicates the medium in which the source is obtained *Since: 2.5* """ WORKSPACE = "workspace" """ Files in an open workspace """ LOCAL = "local" """ Files stored locally in the project """ REMOTE_FILE = "remote-file" """ A remote file """ GIT = "git" """ A git repository """ BAZAAR = "bzr" """ The Bazaar revision control system """ OCI_IMAGE = "oci-image" """ An OCI image, such as docker or podman images. """ PYTHON_PACKAGE_INDEX = "pypi" """ A python package obtained from a python package index like https://pypi.org """ class SourceVersionType(FastEnum): """ Indicates the type of the version string *Since: 2.5* """ COMMIT = "commit" """ A commit string which accurately represents a version in a source code repository or VCS """ SHA256 = "sha256" """ An sha256 checksum of the content of a file """ CAS_DIGEST = "cas-digest" """ A CAS digest expressed as ``{hash}/{size}``. The ``hash`` and ``size`` components represent the members of a ``Digest`` message as defined in the `remote execution protocol `_ """ OCI_DIGEST = "oci-digest" """ An OCI image digest, as can be used to address images in a docker registry. """ INDEXED_VERSION = "indexed-version" """ This type of version is used in cases where we have repositories which have an interface to index content by version, and that no additional validation is performed to insure the uniqueness of the downloaded content (not recommended). In the case of plugins which use this version type, it is probable that ``SourceInfo.version_guess == SourceInfo.version``. """ class SourceInfo: """SourceInfo() An object representing the provenance of input reported by :func:`Source.collect_source_info() ` and/or :func:`SourceFetcher.get_source_info() ` See: :ref:`documentation on generating SourceInfo `. .. attention:: A given SourceInfo for a given element is **not** guaranteed to be unique for a given :ref:`cache key `. While it is true that plugins which :ref:`generate SourceInfo ` must consider any configuration attributes in their cache keys, so as to produce differing cache keys when source provenance information can be reported differently, this does not account for the special nature of *urls*. When considering the *urls* reported in SourceInfo, the urls are only guaranteed to be the primary urls as defined by the project's :ref:`source aliases `, and arbitrary :ref:`mirror urls ` will not be reported here. Since these aliases are intentionally allowed to change without affecting cache keys, or can be :ref:`redirected with junctions `, it possible to have a *differing* set of SourceInfo objects reported for a project which reports identical *cache keys*, in cases where primary alias mappings are changed. *Since: 2.5* """ # # NOTE: The constructor is not public API, and plugins must # call Source.create_source_info(), the docstring above # starting with `SourceInfo()` ensures that documentation # does not show constructor arguments. # def __init__( self, kind: str, url: str, homepage: Optional[str], issue_tracker: Optional[str], provenance: Optional[MappingNode], medium: Union[SourceInfoMedium, str], version_type: Union[SourceVersionType, str], version: str, *, version_guess: Optional[str] = None, extra_data: Optional[Dict[str, str]] = None, ): self.kind: str = kind """ The Source plugin kind which reported this SourceInfo """ self.url: str = url """ The url of the source input """ self.homepage: Optional[str] = homepage """ The project homepage URL """ self.issue_tracker: Optional[str] = issue_tracker """ The project issue tracking URL """ self.provenance = provenance """ The optional YAML node with source provenance attributes """ self.medium: Union[SourceInfoMedium, str] = medium """ The :class:`.SourceInfoMedium` of the source input, or in the case that an appropriate medium is not defined, a freeform string of the plugin's choice describing the medium. """ self.version_type: Union[SourceVersionType, str] = version_type """ The :class:`.SourceVersionType` of the source input version, or in the case that an appropriate version type is not defined, a freeform string of the plugin's choice depicting the type of version. """ self.version: str = version """ A string which represents a unique version of this source input """ self.version_guess: Optional[str] = version_guess """ A string representing the guessed human readable version of this source input """ self.extra_data: Optional[Dict[str, str]] = extra_data """ Additional plugin defined key/values """ def serialize(self) -> Dict[str, Union[str, Dict[str, str]]]: """Produce a dictionary object suitable for serialization into formats like json or yaml. Returns: A dictionary object with strings as keys and values, except for the extra_data which, if present, is also a dictionary with strings as keys and values. """ # # WARNING: This return value produces output for an API stable interface. # # Dictionary member names cannot be removed, and the meaning of # their values cannot be changed. # version_info: Dict[str, Union[str, Dict[str, str]]] medium_str: str version_type_str: str if isinstance(self.medium, SourceInfoMedium): medium_str = str(self.medium.value) else: medium_str = self.medium if isinstance(self.version_type, SourceVersionType): version_type_str = str(self.version_type.value) else: version_type_str = self.version_type version_info = { "kind": self.kind, "url": self.url, } if self.provenance is not None: # need to keep homepage/issue-tracker [also] at the top-level for backward compat if (homepage := self.provenance.get_str("homepage", None)) is not None: version_info["homepage"] = homepage if (issue_tracker := self.provenance.get_str("issue-tracker", None)) is not None: version_info["issue-tracker"] = issue_tracker version_info["provenance"] = self.provenance.strip_node_info() version_info["medium"] = medium_str version_info["version-type"] = version_type_str version_info["version"] = self.version if self.version_guess is not None: version_info["version-guess"] = self.version_guess if self.extra_data: version_info["extra-data"] = self.extra_data return version_info class SourceFetcher: """SourceFetcher() This interface exists so that a source that downloads from multiple places (e.g. a git source with submodules) has a consistent interface for fetching and substituting aliases. .. attention:: When implementing a SourceFetcher, remember to call :func:`Source.mark_download_url() ` for every URL found in the configuration data at :func:`Plugin.configure() ` time. """ def __init__(self): self.__alias = None ############################################################# # Abstract Methods # ############################################################# def fetch(self, alias_override: Optional[AliasSubstitution] = None, **kwargs) -> None: """Fetch remote sources and mirror them locally, ensuring at least that the specific reference is cached locally. Args: alias_override: The alias to use instead of the default one defined by the :ref:`aliases ` field in the project's config. If provided, it must be used when calling :func:`Source.translate_url() `. Raises: :class:`.SourceError` Implementors should raise :class:`.SourceError` if the there is some network error or if the source reference could not be matched. """ raise ImplError("SourceFetcher '{}' does not implement fetch()".format(type(self))) def get_source_info(self) -> Optional[SourceInfo]: """Get the :class:`.SourceInfo` object describing this source This method should only be called whenever :func:`Source.is_resolved() ` returns ``True``. SourceInfo objects created by implementors should be created with :func:`Source.create_source_info() `. Returns: the :class:`.SourceInfo` object describing this source, or ``None`` if the SourceFetcher does not implement this method. *Since: 2.5* """ return None ############################################################# # Public Methods # ############################################################# def mark_download_url(self, url: str) -> None: """Identifies the URL that this SourceFetcher uses to download This must be called during the fetcher's initialization Args: url: The url used to download. .. note:: While this must be called in a SourceFetcher initializer for the URL which will be used by the fetcher, note that any URLs which are known and specified in the Source configuration YAML must be marked with either :func:`Source.mark_download_url() ` or :func:`Source.translate_url() ` in the :func:`Plugin.configure() ` implementation. """ self.__alias = _extract_alias(url) ############################################################# # Private Methods used in BuildStream # ############################################################# # Returns the alias used by this fetcher def _get_alias(self): return self.__alias class Source(Plugin): """Source() Base Source class. All Sources derive from this class, this interface defines how the core will be interacting with Sources. """ # The defaults from the project __defaults: Optional[Dict[str, Any]] = None BST_CUSTOM_SOURCE_PROVENANCE = False """Whether multiple sources' provenance information are provided Used primarily to override the usage of top-level source provenance of a source where individual sub-source's provenance should instead be provided """ BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False """Whether access to previous sources is required during track When set to True: * all sources listed before this source in the given element will be fetched before this source is tracked * Source.track() will be called with an additional keyword argument `previous_sources_dir` where previous sources will be staged * this source can not be the first source for an element """ BST_REQUIRES_PREVIOUS_SOURCES_FETCH = False """Whether access to previous sources is required during fetch When set to True: * all sources listed before this source in the given element will be fetched before this source is fetched * Source.fetch() will be called with an additional keyword argument `previous_sources_dir` where previous sources will be staged * this source can not be the first source for an element """ BST_REQUIRES_PREVIOUS_SOURCES_STAGE = False """Whether access to previous sources is required during cache When set to True: * All sources listed before current source in the given element will be staged with the source when it's cached. * This source can not be the first source for an element. """ BST_STAGE_VIRTUAL_DIRECTORY = False """Whether we can stage this source directly to a virtual directory When set to True, :func:`Source.stage_directory() ` and :func:`Source.init_workspace_directory() ` will be called in place of :func:`Source.stage() ` and :func:`Source.init_workspace() ` respectively. """ def __init__( self, context: "Context", project: "Project", meta: MetaSource, variables: Variables, *, alias_override: Optional[Tuple[str, AliasSubstitution]] = None, unique_id: Optional[int] = None, ): # Set element_name member before parent init, as needed for debug messaging self.__element_name = meta.element_name # The name of the element owning this source super().__init__( "{}-{}".format(meta.element_name, meta.element_index), context, project, meta.config, "source", unique_id=unique_id, ) self.__element_index = meta.element_index # The index of the source in the owning element's source list self.__element_kind = meta.element_kind # The kind of the element owning this source self._directory = meta.directory # Staging relative directory self.__variables = variables # The variables used to resolve the source's config self.__provenance: Optional[ MappingNode ] = meta.provenance # The source provenance for general user provided SourceInfo if self.__provenance is not None and self.BST_CUSTOM_SOURCE_PROVENANCE: raise SourceError( f"{self._get_provenance()} Custom source provenance plugin: Refusing to use top level source provenance", reason="top-level-provenance-on-custom-implementation", ) self.__key = None # Cache key for source # The alias_override is only set on a re-instantiated Source self.__alias_override = alias_override # Tuple of alias and its override to use instead self.__expected_alias = None # The primary alias # Set of marked download URLs self.__marked_urls: Set[str] = set() # Collect the composited element configuration and # ask the element to configure itself. self.__init_defaults(project, meta) self.__config = self.__extract_config(meta) variables.expand(self.__config) self.__first_pass = meta.first_pass # cached values for commonly access values on the source self.__mirror_directory = None # type: Optional[str] self._configure(self.__config) self.__is_cached = None COMMON_CONFIG_KEYS = ["kind", "directory", "provenance"] """Common source config keys Source config keys that must not be accessed in configure(), and should be checked for using node.validate_keys(). """ ############################################################# # Abstract Methods # ############################################################# def load_ref(self, node: MappingNode) -> None: """Loads the :attr:`~buildstream.types.SourceRef` for this Source from the specified *node*. Args: node: The YAML node to load the ref from Working with the :ref:`source ref is discussed here `. .. note:: The :attr:`~buildstream.types.SourceRef` for the Source is expected to be read at :func:`Plugin.configure() ` time, this will only be used for loading refs from alternative locations than in the `element.bst` file where the given Source object has been declared. """ raise ImplError("Source plugin '{}' does not implement load_ref()".format(self.get_kind())) def get_ref(self) -> SourceRef: """Fetch the :attr:`~buildstream.types.SourceRef` Returns: The internal :attr:`~buildstream.types.SourceRef`, or ``None`` Working with the :ref:`source ref is discussed here `. """ raise ImplError("Source plugin '{}' does not implement get_ref()".format(self.get_kind())) def set_ref(self, ref: SourceRef, node: MappingNode) -> None: """Applies the internal ref, however it is represented Args: ref: The internal :attr:`~buildstream.types.SourceRef` to set, or ``None`` node: The same node which was previously passed to :func:`Plugin.configure() ` and :func:`Source.load_ref() ` The implementor must update the *node* parameter to reflect the new *ref*, and it should store the passed *ref* so that it will be returned in any later calls to :func:`Source.get_ref() `. The passed *ref* parameter is guaranteed to either be a value which has been previously retrieved by the :func:`Source.get_ref() ` method on the same plugin, or ``None``. **Example:** .. code:: python # Implementation of Source.set_ref() # def set_ref(self, ref, node): # Update internal state of the ref self.ref = ref # Update the passed node so that we will read the new ref # next time this source plugin is configured with this node. # node["ref"] = self.ref Working with the :ref:`source ref is discussed here `. """ raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind())) def track(self, *, previous_sources_dir: Optional[str] = None) -> SourceRef: """Resolve a new ref from the plugin's track option Args: previous_sources_dir (str): directory where previous sources are staged. Note that this keyword argument is available only when :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_TRACK` is set to True. Returns: A new :attr:`~buildstream.types.SourceRef`, or None If the backend in question supports resolving references from a symbolic tracking branch or tag, then this should be implemented to perform this task on behalf of :ref:`bst source track ` commands. This usually requires fetching new content from a remote origin to see if a new ref has appeared for your branch or tag. If the backend store allows one to query for a new ref from a symbolic tracking data without downloading then that is desirable. Working with the :ref:`source ref is discussed here `. """ # Allow a non implementation return None def fetch(self, *, previous_sources_dir: Optional[str] = None) -> None: """Fetch remote sources and mirror them locally, ensuring at least that the specific reference is cached locally. Args: previous_sources_dir (str): directory where previous sources are staged. Note that this keyword argument is available only when :attr:`~buildstream.source.Source.BST_REQUIRES_PREVIOUS_SOURCES_FETCH` is set to True. Raises: :class:`.SourceError` Implementors should raise :class:`.SourceError` if the there is some network error or if the source reference could not be matched. """ raise ImplError("Source plugin '{}' does not implement fetch()".format(self.get_kind())) def stage(self, directory: str) -> None: """Stage the sources to a directory Args: directory: Path to stage the source Raises: :class:`.SourceError` Implementors should assume that *directory* already exists and stage already cached sources to the passed directory. Implementors should raise :class:`.SourceError` when encountering some system error. """ raise ImplError("Source plugin '{}' does not implement stage()".format(self.get_kind())) def stage_directory(self, directory: Directory) -> None: """Stage the sources to a directory Args: directory: :class:`.Directory` object to stage the source into Raises: :class:`.SourceError` Implementors should assume that *directory* represents an existing directory root into which the source content can be populated. Implementors should raise :class:`.SourceError` when encountering some system error. .. note:: This will be called *instead* of :func:`Source.stage() ` in the case that :attr:`~buildstream.source.Source.BST_STAGE_VIRTUAL_DIRECTORY` is set for this plugin. """ raise ImplError("Source plugin '{}' does not implement stage_directory()".format(self.get_kind())) def init_workspace(self, directory: str) -> None: """Stage sources for use as a workspace. Args: directory: Path of the workspace to initialize. Raises: :class:`.SourceError` Default implementation is to call :func:`Source.stage() `. Implementors overriding this method should assume that *directory* already exists. Implementors should raise :class:`.SourceError` when encountering some system error. """ self.stage(directory) def init_workspace_directory(self, directory: Directory) -> None: """Stage sources for use as a workspace. Args: directory: :class:`.Directory` object of the workspace to initialize. Raises: :class:`.SourceError` Default implementation is to call :func:`Source.stage_directory() `. Implementors overriding this method should assume that *directory* already exists. Implementors should raise :class:`.SourceError` when encountering some system error. .. note:: This will be called *instead* of :func:`Source.init_workspace() ` in the case that :attr:`~buildstream.source.Source.BST_STAGE_VIRTUAL_DIRECTORY` is set for this plugin. """ self.stage_directory(directory) def get_source_fetchers(self) -> Iterable[SourceFetcher]: """Get the objects that are used for fetching If this source doesn't download from multiple URLs, returning None and falling back on the default behaviour is recommended. Returns: The Source's SourceFetchers, if any. .. note:: Implementors can implement this as a generator. The :func:`SourceFetcher.fetch() ` method will be called on the returned fetchers one by one, before consuming the next fetcher in the list. """ return [] def validate_cache(self) -> None: """Implement any validations once we know the sources are cached This is guaranteed to be called only once for a given session once the sources are known to be cached, before :func:`Source.stage() ` or :func:`Source.init_workspace() ` is called. """ def is_cached(self) -> bool: """Get whether the source has a local copy of its data. This method is guaranteed to only be called whenever :func:`Source.is_resolved() ` returns `True`. Returns: whether the source is cached locally or not. """ raise ImplError("Source plugin '{}' does not implement is_cached()".format(self.get_kind())) def collect_source_info(self) -> Optional[Iterable[SourceInfo]]: """Get the :class:`.SourceInfo` objects describing this source This method should only be called whenever :func:`Source.is_resolved() ` returns ``True``. SourceInfo objects created by implementors should be created with :func:`Source.create_source_info() `. Returns: the :class:`.SourceInfo` objects describing this source, or ``None`` if the Source does not implement this method. .. note:: If your plugin uses :class:`.SourceFetcher` objects, you can implement :func:`Source.get_source_info() ` instead. *Since: 2.5* """ source_info = [] for fetcher in self.get_source_fetchers(): info = fetcher.get_source_info() if info is not None: source_info.append(info) # If there are source fetchers, they can either have returned # SourceInfo objects, or None. # # We need to issue the warning here and return None in the case that no source info # was reported. # if not source_info: self.warn( "{}: Source.collect_source_info() is not implemented in this plugin".format(self), warning_token=CoreWarnings.UNAVAILABLE_SOURCE_INFO, ) return None return source_info ############################################################# # Public Methods # ############################################################# def get_mirror_directory(self) -> str: """Fetches the directory where this source should store things Returns: The directory belonging to this source """ if self.__mirror_directory is None: # Create the directory if it doesnt exist context = self._get_context() directory = os.path.join(context.sourcedir, self.get_kind()) os.makedirs(directory, exist_ok=True) self.__mirror_directory = directory return self.__mirror_directory def translate_url( self, url: str, *, alias_override: Optional[AliasSubstitution] = None, primary: bool = True, suffix: Optional[str] = None, extra_data: Optional[Dict[str, Any]] = None, ) -> str: """Translates the given url which may be specified with an alias into a fully qualified url. Args: url: A URL, which may be using an alias alias_override: Optionally, an URI to override the alias with. primary: Whether this is the primary URL for the source. suffix: an optional suffix to append to the URL (*Since: 2.2*) extra_data: Additional data provided by :class:`SourceMirror ` (*Since: 2.2*) Returns: The fully qualified URL, with aliases resolved .. note:: This must be called for every URL in the configuration during :func:`Plugin.configure() ` if :func:`Source.mark_download_url() ` is not called. The *suffix* argument may be used to translate URLs for which only the base portion of the URL was previously marked with :func:`Source.mark_download_url() ` at :func:`Plugin.configure() ` time. """ project = self._get_project() # Ensure that the download URL is also marked self.mark_download_url(url, primary=primary) if suffix: url = url + suffix # Alias overriding can happen explicitly (by command-line) or # implicitly (the Source being constructed with an __alias_override). # if alias_override or self.__alias_override: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) project_alias_url = project.get_alias_url(url_alias, first_pass=self.__first_pass) if self.__alias_override is not None: override_alias = self.__alias_override[0] override_subst = self.__alias_override[1] # Implicit alias overrides may only be done for one # specific alias, so that sources that fetch from multiple # URLs and use different aliases default to only overriding # one alias, rather than getting confused. # if url_alias != override_alias: return url elif alias_override is not None: override_subst = alias_override else: assert False, "unreachable" # The default source mirror will give prefix URLs if isinstance(override_subst._mirror, str): return override_subst._mirror + url_body # # Delegate the URL translation to the SourceMirror plugin # return override_subst._mirror.translate_url( alias=override_subst._effective_alias, alias_url=project_alias_url, source_url=url_body, extra_data=extra_data, ) else: return project.translate_url(url, source=self, first_pass=self.__first_pass) def mark_download_url(self, url: str, *, primary: bool = True) -> None: """Identifies the URL that this Source uses to download Args: url (str): The URL used to download primary (bool): Whether this is the primary URL for the source .. note:: This must be called for every URL in the configuration during :func:`Plugin.configure() ` if :func:`Source.translate_url() ` is not called. """ # Only mark the Source level aliases on the main instance, not in # a reinstantiated instance in mirroring. if not self.__alias_override: if primary: expected_alias = _extract_alias(url) assert ( self.__expected_alias is None or self.__expected_alias == expected_alias ), "Attempt to mark primary URL with {}, already marked with {}".format( expected_alias, self.__expected_alias ) self.__expected_alias = expected_alias # Enforce proper behaviour of plugins by ensuring that all # aliased URLs have been marked at Plugin.configure() time. # if self._get_configuring(): # Record marked urls while configuring # self.__marked_urls.add(url) else: # If an unknown aliased URL is seen after configuring, # this is an error. # # It is still possible that a URL that was not mentioned # in the element configuration can be marked, this is # the case for git submodules which might be automatically # discovered. # assert url in self.__marked_urls or not _extract_alias( url ), "URL was not seen at configure time: {}".format(url) alias = _extract_alias(url) # Issue a (fatal-able) warning if the source used a URL without specifying an alias if not alias: self.warn( "{}: Use of unaliased source download URL: {}".format(self, url), warning_token=CoreWarnings.UNALIASED_URL, ) # If there is an alias in use, ensure that it exists in the project if alias: project = self._get_project() if not project.alias_exists(alias, first_pass=self.__first_pass, source=self): raise SourceError( "{}: Invalid alias '{}' specified in URL: {}".format(self, alias, url), reason="invalid-source-alias", ) if not project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): raise SourceError( "{}: No fetch URI found for alias '{}'".format(self, alias), detail="Check fetch controls in your user configuration", reason="missing-source-alias-target", ) if not project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True): raise SourceError( "{}: No tracking URI found for alias '{}'".format(self, alias), detail="Check track controls in your user configuration", reason="missing-source-alias-target", ) def get_project_directory(self) -> str: """Fetch the project base directory This is useful for sources which need to load resources stored somewhere inside the project. Returns: The project base directory """ project = self._get_project() return project.directory @contextmanager def tempdir(self) -> Iterator[str]: """Context manager for working in a temporary directory Yields: A path to a temporary directory This should be used by source plugins directly instead of the tempfile module. This one will automatically cleanup in case of termination by catching the signal before os._exit(). It will also use the 'mirror directory' as expected for a source. """ mirrordir = self.get_mirror_directory() with utils._tempdir(dir=mirrordir) as tempdir: yield tempdir def is_resolved(self) -> bool: """Get whether the source is resolved. This has a default implementation that checks whether the source has a ref or not. If it has a ref, it is assumed to be resolved. Sources that never have a ref or have uncommon requirements can override this method to specify when they should be considered resolved Returns: whether the source is fully resolved or not """ return self.get_ref() is not None def create_source_info( self, url: str, medium: Union[SourceInfoMedium, str], version_type: Union[SourceVersionType, str], version: str, *, version_guess: Optional[str] = None, extra_data: Optional[Dict[str, str]] = None, provenance_node: Optional[MappingNode] = None, ) -> SourceInfo: """Create a :class:`.SourceInfo` object This function should be used to generate SourceInfo objects in :func:`Source.is_resolved() ` and :func:`Source.is_resolved() ` implementations. Args: url: The translated URL medium: The :class:`.SourceInfoMedium` of the source input, or in the case that an appropriate medium is not defined, a freeform string of the plugin's choice describing the medium. version_type: The :class:`.SourceVersionType` of the source input version, or in the case that an appropriate version type is not defined, a freeform string of the plugin's choice depicting the type of version. version: A string which represents a unique version of this source input version_guess: An optional string representing the guessed human readable version extra_data: Additional plugin defined key/values provenance_node: An optional :class:`Node ` with source provenance attributes, defaults to the provenance specified at the top level of the source. *Since: 2.5* """ project = self._get_project() source_provenance: MappingNode | None if provenance_node is not None: # Ensure provenance node keys are valid and values are all strings try: provenance_node.validate_keys(project.source_provenance_attributes.keys()) except LoadError as E: raise LoadError( "Specified source provenance attribute not defined in project config\n {}".format(E), LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE, ) # Make sure everything is a string for key, value in provenance_node.items(): if not isinstance(value, ScalarNode): raise LoadError( f"{value}: Expected string for the value of provenance attribute '{key}'", LoadErrorReason.INVALID_DATA, ) source_provenance = provenance_node else: source_provenance = self.__provenance homepage = None issue_tracker = None if source_provenance is not None: homepage = source_provenance.get_str("homepage", None) issue_tracker = source_provenance.get_str("issue-tracker", None) return SourceInfo( self.get_kind(), url, homepage, issue_tracker, source_provenance, medium, version_type, version, version_guess=version_guess, extra_data=extra_data, ) ############################################################# # Private Abstract Methods used in BuildStream # ############################################################# # Returns the local path to the source # # If the source is locally available, this method returns the absolute # path. Otherwise, the return value is None. # # This is an optimization for local sources and optional to implement. # # Returns: # (str): The local absolute path, or None # def _get_local_path(self): return None ############################################################# # Private Methods used in BuildStream # ############################################################# # Wrapper around preflight() method # def _preflight(self): try: self.preflight() except BstError as e: # Prepend provenance to the error raise SourceError("{}: {}".format(self, e), reason=e.reason) from e # Get whether the source is cached by the source plugin # def _is_cached(self): if self.__is_cached is None: # We guarantee we only ever call this when we are resolved. assert self.is_resolved() # Set to 'False' on the first call, this prevents throwing multiple errors if the # plugin throws exception when we display the end result pipeline. # Otherwise, the summary would throw a second exception and we would not # have a nice error reporting. self.__is_cached = False try: self.__is_cached = self.is_cached() # pylint: disable=assignment-from-no-return except SourceError: # SourceErrors should be preserved so that the # plugin can communicate real error cases. raise except Exception as err: # Generic errors point to bugs in the plugin, so # we need to catch them and make sure they do not # cause stacktraces raise PluginError( "Source plugin '{}' failed to check its cached state: {}".format(self.get_kind(), err), reason="source-bug", ) return self.__is_cached # Wrapper function around plugin provided fetch method # # Args: # previous_sources_dir (str): directory where previous sources are staged # def _fetch(self, previous_sources_dir=None): if self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH: self.__do_fetch(previous_sources_dir=previous_sources_dir) else: self.__do_fetch() # _fetch_done() # # Indicates that fetching the source has been done. # # Args: # fetched_original (bool): Whether the original sources had been asked (and fetched) or not # def _fetch_done(self, fetched_original): if fetched_original: # The original was fetched, we know we are cached self.__is_cached = True else: # The original was not requested, we might or might not be cached # Don't recompute, but allow recomputation later if needed self.__is_cached = None # Wrapper for stage() api which gives the source # plugin a fully constructed path considering the # 'directory' option # def _stage(self, directory): self.validate_cache() if isinstance(directory, Directory): self.stage_directory(directory) else: self.stage(directory) # Wrapper for init_workspace() def _init_workspace(self, directory): if self.BST_STAGE_VIRTUAL_DIRECTORY: directory = FileBasedDirectory(external_directory=directory) self.validate_cache() if isinstance(directory, Directory): self.init_workspace_directory(directory) else: self.init_workspace(directory) # _get_unique_key(): # # Wrapper for get_unique_key() api # def _get_unique_key(self): return self.get_unique_key() # _project_refs(): # # Gets the appropriate ProjectRefs object for this source, # which depends on whether the owning element is a junction # # Args: # project (Project): The project to check # def _project_refs(self, project): element_kind = self.__element_kind if element_kind == "junction": return project.junction_refs return project.refs # _load_ref(): # # Loads the ref for the said source. # # Raises: # (SourceError): If the source does not implement load_ref() # # Returns: # (ref): A redundant ref specified inline for a project.refs using project # # This is partly a wrapper around `Source.load_ref()`, it will decide # where to load the ref from depending on which project the source belongs # to and whether that project uses a project.refs file. # # Note the return value is used to construct a summarized warning in the # case that the toplevel project uses project.refs and also lists refs # which will be ignored. # def _load_ref(self): context = self._get_context() project = self._get_project() toplevel = context.get_toplevel_project() redundant_ref = None element_name = self.__element_name element_idx = self.__element_index def do_load_ref(node): try: self.load_ref(node) except ImplError as e: raise SourceError( "{}: Storing refs in project.refs is not supported by '{}' sources".format(self, self.get_kind()), reason="unsupported-load-ref", ) from e # If the main project overrides the ref, use the override if project is not toplevel and toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS: refs = self._project_refs(toplevel) ref_node = refs.lookup_ref(project.name, element_name, element_idx) if ref_node is not None: do_load_ref(ref_node) return redundant_ref # If the project itself uses project.refs, clear the ref which # was already loaded via Source.configure(), as this would # violate the rule of refs being either in project.refs or in # the elements themselves. # if project.ref_storage == ProjectRefStorage.PROJECT_REFS: # First warn if there is a ref already loaded, and reset it redundant_ref = self.get_ref() # pylint: disable=assignment-from-no-return if redundant_ref is not None: self.set_ref(None, MappingNode.from_dict({})) # Try to load the ref refs = self._project_refs(project) ref_node = refs.lookup_ref(project.name, element_name, element_idx) if ref_node is not None: do_load_ref(ref_node) return redundant_ref # _set_ref() # # Persists the ref for this source. This will decide where to save the # ref, or refuse to persist it, depending on active ref-storage project # settings. # # Args: # new_ref (smth): The new reference to save # save (bool): Whether to write the new reference to file or not # # Returns: # (bool): Whether the ref has changed # # Raises: # (SourceError): In the case we encounter errors saving a file to disk # def _set_ref(self, new_ref, *, save): context = self._get_context() project = self._get_project() toplevel = context.get_toplevel_project() toplevel_refs = self._project_refs(toplevel) provenance = self._get_provenance() element_name = self.__element_name element_idx = self.__element_index # # Step 1 - Obtain the node # node = {} if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS: node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True) if project is toplevel and not node: node = provenance._node # # Step 2 - Set the ref in memory, and determine changed state # clean = node.strip_node_info() # Set the ref regardless of whether it changed, the # TrackQueue() will want to update a specific node with # the ref, regardless of whether the original has changed. # # In the following add/del/mod merge algorithm we are working with # dictionaries, but the plugin API calls for a MappingNode. # modify = node.clone() self.set_ref(new_ref, modify) to_modify = modify.strip_node_info() # FIXME: this will save things too often, as a ref might not have # changed. We should optimize this to detect it differently if not save: return False # Ensure the node is not from a junction if not toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS and provenance._project is not toplevel: if provenance._project is project: self.warn("{}: Not persisting new reference in junctioned project".format(self)) elif provenance._project is None: assert provenance._filename == "" assert provenance._shortname == "" raise SourceError("{}: Error saving source reference to synthetic node.".format(self)) else: raise SourceError( "{}: Cannot track source in a fragment from a junction".format(provenance._shortname), reason="tracking-junction-fragment", ) actions = {} for k, v in clean.items(): if k not in to_modify: actions[k] = "del" else: if v != to_modify[k]: actions[k] = "mod" for k in to_modify.keys(): if k not in clean: actions[k] = "add" def walk_container(container, path): # For each step along path, synthesise if we need to. # If we're synthesising missing list entries, we know we're # doing this for project.refs so synthesise empty dicts for the # intervening entries too lpath = path.copy() lpath.append("") # We know the last step will be a string key for step, next_step in zip(lpath, lpath[1:]): if type(step) is str: # pylint: disable=unidiomatic-typecheck # handle dict container if step not in container: if type(next_step) is str: # pylint: disable=unidiomatic-typecheck container[step] = {} else: container[step] = [] container = container[step] else: # handle list container if len(container) <= step: while len(container) <= step: container.append({}) container = container[step] return container def process_value(action, container, path, key, new_value): container = walk_container(container, path) if action == "del": del container[key] elif action == "mod": container[key] = new_value elif action == "add": container[key] = new_value else: assert False, "BUG: Unknown action: {}".format(action) roundtrip_cache = {} for key, action in actions.items(): # Obtain the top level node and its file if action == "add": provenance = node.get_provenance() else: provenance = node.get_node(key).get_provenance() toplevel_node = provenance._toplevel # Get the path to whatever changed if action == "add": path = toplevel_node._find(node) else: full_path = toplevel_node._find(node.get_node(key)) # We want the path to the node containing the key, not to the key path = full_path[:-1] roundtrip_file = roundtrip_cache.get(provenance._filename) if not roundtrip_file: roundtrip_file = roundtrip_cache[provenance._filename] = _yaml.roundtrip_load( provenance._filename, allow_missing=True ) # Get the value of the round trip file that we need to change process_value(action, roundtrip_file, path, key, to_modify.get(key)) # # Step 3 - Apply the change in project data # for filename, data in roundtrip_cache.items(): # This is our roundtrip dump from the track try: _yaml.roundtrip_dump(data, filename) except OSError as e: raise SourceError( "{}: Error saving source reference to '{}': {}".format(self, filename, e), reason="save-ref-error" ) from e return True # Wrapper for track() # # Args: # previous_sources_dir (str): directory where previous sources are staged # def _track(self, previous_sources_dir: Optional[str] = None) -> SourceRef: if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK: new_ref = self.__do_track(previous_sources_dir=previous_sources_dir) else: new_ref = self.__do_track() current_ref = self.get_ref() # pylint: disable=assignment-from-no-return if new_ref is None: # No tracking, keep current ref new_ref = current_ref if current_ref != new_ref: self.info("Found new revision: {}".format(new_ref)) # Save ref in local process for subsequent sources self._set_ref(new_ref, save=False) self._generate_key() return new_ref # _requires_previous_sources() # # If a plugin requires access to previous sources at track or fetch time, # then it cannot be the first source of an elemenet. # # Returns: # (bool): Whether this source requires access to previous sources # def _requires_previous_sources(self): return self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK or self.BST_REQUIRES_PREVIOUS_SOURCES_FETCH # Returns the alias if it's defined in the project def _get_alias(self): alias = self.__expected_alias project = self._get_project() if project.alias_exists(alias, first_pass=self.__first_pass, source=self): # The alias must already be defined in the project's aliases # otherwise http://foo gets treated like it contains an alias return alias else: return None def _generate_key(self): self.__key = generate_key(self._get_unique_key()) @property def _key(self): return self.__key # Gives a ref path that points to where sources are kept in the CAS def _get_source_name(self): # @ is used to prevent conflicts with project names return "{}/{}".format(self.get_kind(), self._key) def _get_brief_display_key(self): context = self._get_context() key = self._key length = min(len(key), context.log_key_length) return key[:length] @property def _element_name(self): return self.__element_name # _cache_directory() # # A context manager to cache and retrieve content. # # If the digest is not specified, then a new directory is prepared, the # content of which can later be addressed by accessing it's digest, # using the private API Directory._get_digest(). # # The hash of the Digest of the cached directory is suitable for use as a # cache key, and the Digest object can be reused later on to do the # staging operation. # # This context manager was added specifically to optimize cases where # we have project or host local data to stage into CAS, such as local # sources and workspaces. # # Args: # digest: A Digest of previously cached content. # # Yields: # (Directory): A handle on the cached content directory # @contextmanager def _cache_directory(self, digest=None): context = self._get_context() cache = context.get_cascache() cas_dir = CasBasedDirectory(cache, digest=digest) yield cas_dir ############################################################# # Local Private Methods # ############################################################# # __clone_for_uri() # # Clone the source with an alternative URI setup for the alias # which this source uses. # # This is used for iteration over source mirrors. # # Args: # uri (str): The alternative URI for this source's alias # # Returns: # (Source): A new clone of this Source, with the specified URI # as the value of the alias this Source has marked as # primary with either mark_download_url() or # translate_url(). # def __clone_for_uri(self, mirror): project = self._get_project() context = self._get_context() alias = self._get_alias() source_kind = type(self) # Rebuild a MetaSource from the current element meta = MetaSource( self.__element_name, self.__element_index, self.__element_kind, self.get_kind(), self._directory, self.__provenance, self.__config, self.__first_pass, ) clone = source_kind( context, project, meta, self.__variables, alias_override=(alias, mirror), unique_id=self._unique_id, ) # Do the necessary post instantiation routines here # clone._preflight() clone._load_ref() return clone # Tries to call fetch for every mirror, stopping once it succeeds def __do_fetch(self, **kwargs): project = self._get_project() context = self._get_context() # Silence the STATUS messages which might happen as a result # of checking the source fetchers. with context.messenger.silence(): source_fetchers = self.get_source_fetchers() # Use the source fetchers if they are provided # if source_fetchers: # Use a contorted loop here, this is to allow us to # silence the messages which can result from consuming # the items of source_fetchers, if it happens to be a generator. # source_fetchers = iter(source_fetchers) while True: with context.messenger.silence(): try: fetcher = next(source_fetchers) except StopIteration: # as per PEP479, we are not allowed to let StopIteration # thrown from a context manager. # Catching it here and breaking instead. break alias = fetcher._get_alias() last_error = None for mirror in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): try: fetcher.fetch(mirror) # FIXME: Need to consider temporary vs. permanent failures, # and how this works with retries. except BstError as e: last_error = e continue # No error, we're done with this fetcher break else: # No break occurred, raise the last detected error raise last_error # Default codepath is to reinstantiate the Source # else: alias = self._get_alias() if self.__first_pass: mirrors = project.first_pass_config.mirrors else: mirrors = project.config.mirrors if not mirrors or not alias: self.fetch(**kwargs) return last_error = None for mirror in project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=False): new_source = self.__clone_for_uri(mirror) try: new_source.fetch(**kwargs) # FIXME: Need to consider temporary vs. permanent failures, # and how this works with retries. except BstError as e: last_error = e continue # No error, we're done here return # Re raise the last detected error raise last_error # Tries to call track for every mirror, stopping once it succeeds def __do_track(self, **kwargs): project = self._get_project() alias = self._get_alias() if self.__first_pass: mirrors = project.first_pass_config.mirrors else: mirrors = project.config.mirrors # If there are no mirrors, or no aliases to replace, there's nothing to do here. if not mirrors or not alias: return self.track(**kwargs) # NOTE: We are assuming here that tracking only requires substituting the # first alias used last_error = None for mirror in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)): new_source = self.__clone_for_uri(mirror) try: ref = new_source.track(**kwargs) # pylint: disable=assignment-from-none # FIXME: Need to consider temporary vs. permanent failures, # and how this works with retries. except BstError as e: last_error = e continue return ref raise last_error @classmethod def __init_defaults(cls, project, meta): if cls.__defaults is None: if meta.first_pass: sources = project.first_pass_config.source_overrides else: sources = project.source_overrides cls.__defaults = sources.get_mapping(meta.kind, default={}) # This will resolve the final configuration to be handed # off to source.configure() # @classmethod def __extract_config(cls, meta): config = cls.__defaults.get_mapping("config", default={}) config = config.clone() meta.config._composite(config) config._assert_fully_composited() return config def _extract_alias(url): parts = url.split(utils._ALIAS_SEPARATOR, 1) if len(parts) > 1 and not parts[0].lower() in utils._URI_SCHEMES: return parts[0] else: return "" apache-buildstream-27ae392/src/buildstream/sourcemirror.py000066400000000000000000000132611514607367700240340ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ SourceMirror - Base source mirror class ======================================= The SourceMirror plugin allows one to customize how :func:`Source.translate_url() ` will behave when looking up mirrors, allowing some additional flexibility in the implementation of source mirrors. .. _core_source_mirror_abstract_methods: Abstract Methods ---------------- For loading and configuration purposes, SourceMirrors may optionally implement the :func:`Plugin base class Plugin.configure() method ` in order to load any custom configuration in the `config` dictionary. The remaining :ref:`Plugin base class abstract methods ` are not relevant to the SourceMirror plugin object and need not be implemented. SourceMirrors expose the following abstract methods. Unless explicitly mentioned, these methods are mandatory to implement. * :func:`SourceMirror.translate_url() ` Produce an appropriate URL for the given URL and alias. Class Reference --------------- """ from typing import Optional, Any, Dict, List, Set, TYPE_CHECKING from .node import MappingNode from .plugin import Plugin from ._exceptions import BstError, ImplError from .exceptions import ErrorDomain if TYPE_CHECKING: # pylint: disable=cyclic-import from ._context import Context from ._project import Project # pylint: enable=cyclic-import class SourceMirrorError(BstError): """This exception should be raised by :class:`.SourceMirror` implementations to report errors to the user. Args: message: The breif error description to report to the user detail: A possibly multiline, more detailed error message reason: An optional machine readable reason string, used for test cases *Since: 2.2* """ def __init__( self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False ): super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason) class SourceMirror(Plugin): """SourceMirror() Base SourceMirror class. All SourceMirror plugins derive from this class, this interface defines how the core will be interacting with SourceMirror plugins. *Since: 2.2* """ # The SourceMirror plugin type is only supported since BuildStream 2.2 BST_MIN_VERSION = "2.2" def __init__( self, context: "Context", project: "Project", node: MappingNode, ): # Note: the MappingNode passed here is already expanded with # the project level base variables, so there is no need # to expand them redundantly here. # # Special case for the default plugin kind = node.get_str("kind", "default") if kind == "default": config_node = node else: node.validate_keys(["name", "kind", "config"]) config_node = node.get_mapping("config", {}) # Do local base class parsing first name: str = node.get_str("name") # Chain up to Plugin super().__init__(name, context, project, node, "source-mirror") self.__aliases: Set[str] = set() self._configure(config_node) ########################################################## # Internal API # ########################################################## # Returns a list of URIs/SourceMirrors for a given alias. def _get_alias_uris(self, alias: str) -> List: assert self.__aliases, "Didn't set aliases during configuring time" if alias in self.__aliases: return [self] return [] ########################################################## # Public API # ########################################################## def set_supported_aliases(self, aliases: List[str]): """Set the aliases for which `self` can translate urls. Args: aliases: The list of aliases supported by this plugin """ assert self._get_configuring(), "Trying to set aliases after configure time" self.__aliases.update(aliases) def translate_url( self, *, alias: str, alias_url: str, source_url: str, extra_data: Optional[Dict[str, Any]], ) -> str: """Produce an alternative url for `url` for the given alias. This method implements the behavior of :func:`Source.translate_url() `. Args: alias: The alias to translate for alias_url: The default URL configured for this alias in the originating project source_url: The URL as specified by original source YAML, excluding the alias extra_data: An optional extra dictionary to return additional data """ raise ImplError( "source mirror plugin '{kind}' does not implement translate_url()".format(kind=self.get_kind()) ) apache-buildstream-27ae392/src/buildstream/storage/000077500000000000000000000000001514607367700223705ustar00rootroot00000000000000apache-buildstream-27ae392/src/buildstream/storage/__init__.py000066400000000000000000000014341514607367700245030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jim MacArthur from ._filebaseddirectory import FileBasedDirectory from ._casbaseddirectory import CasBasedDirectory from .directory import Directory, DirectoryError, FileType, FileStat apache-buildstream-27ae392/src/buildstream/storage/_casbaseddirectory.py000066400000000000000000001075561514607367700266110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jim MacArthur # Tristan van Berkom import os import stat import tarfile as tarfilelib from tarfile import TarFile from contextlib import contextmanager from io import StringIO, BytesIO from typing import Callable, Optional, Union, List, IO, Iterator, Dict from google.protobuf import timestamp_pb2 from .. import utils from .._cas import CASCache from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from .directory import Directory, DirectoryError, FileType, FileStat from ..utils import FileListResult, BST_ARBITRARY_TIMESTAMP # _IndexEntry() # # An object to represent a file, used to track members of a CasBasedDirectory # class _IndexEntry: def __init__( self, cas_cache: CASCache, name: str, entrytype: int, *, digest=None, target: Optional[str] = None, is_executable: bool = False, directory: Optional["CasBasedDirectory"] = None, mtime: Optional[timestamp_pb2.Timestamp] = None # pylint: disable=no-member ) -> None: # The CAS cache self.cas_cache: CASCache = cas_cache # The name of the entry (filename) self.name: str = name # The type of file (FileType) self.type: int = entrytype # The digest of the entry, as calculated by CAS # # This protobuf generated type (remote_execution_pb2.Digest) cannot be annotated self.digest = digest # The target of a symbolic link (for FileType.SYMLINK) self.target: Optional[str] = target # Whether the file is executable (for FileType.REGULAR_FILE) self.is_executable: bool = is_executable # The associated directory object (for FileType.DIRECTORY) self.directory: Optional["CasBasedDirectory"] = directory # The mtime of the file, if provided self.mtime: Optional[timestamp_pb2.Timestamp] = mtime # pylint: disable=no-member def get_directory(self, parent: "CasBasedDirectory") -> "CasBasedDirectory": if self.directory is None: assert self.type == FileType.DIRECTORY self.directory = CasBasedDirectory(self.cas_cache, digest=self.digest, parent=parent, filename=self.name) self.digest = None return self.directory # Get the remote_execution_pb2.Digest for this _IndexEntry # def get_digest(self): if self.directory is not None: # directory with buildstream object return self.directory._get_digest() else: # regular file, symlink or directory without buildstream object return self.digest # clone(): # # Create a deep copy of this object. If this is a directory, a # CasBasedDirectory can also be passed to assign an appropriate # parent directory. # def clone(self) -> "_IndexEntry": return _IndexEntry( self.cas_cache, self.name, self.type, # If this is a directory, the digest will be converted # later if necessary. For other non-file types, digests # are always None. digest=self.get_digest(), target=self.target, is_executable=self.is_executable, mtime=self.mtime, ) def __eq__(self, other: object) -> bool: if not isinstance(other, _IndexEntry): return NotImplemented def get_equivalency_properties(e: _IndexEntry): return (e.name, e.type, e.target, e.is_executable, e.mtime, e.get_digest()) return get_equivalency_properties(self) == get_equivalency_properties(other) # CasBasedDirectory intentionally doesn't call its superclass constuctor, # which is meant to be unimplemented. # pylint: disable=super-init-not-called class CasBasedDirectory(Directory): def __init__( self, cas_cache: CASCache, *, digest=None, parent: Optional["CasBasedDirectory"] = None, filename: Optional[str] = None ) -> None: # The CAS cache self.__cas_cache: CASCache = cas_cache # The name of this directory self.__filename: Optional[str] = filename # The remote_execution_pb2.Digest of this directory self.__digest = digest # The parent directory self.__parent: Optional["CasBasedDirectory"] = parent # An index of directory entries self.__index: Dict[str, _IndexEntry] = {} # Whether this directory and it's subdirectories should be read-only self.__subtree_read_only: Optional[bool] = None if digest: self.__populate_index(digest) def __iter__(self) -> Iterator[str]: yield from self.__index.keys() def __len__(self) -> int: return len(self.__index) def __str__(self) -> str: return "[CAS:{}]".format(self.__get_identifier()) ############################################################# # Implementation of Public API # ############################################################# def open_directory(self, path: str, *, create: bool = False, follow_symlinks: bool = False) -> "CasBasedDirectory": self._validate_path(path) paths = path.split("/") return self.__open_directory(paths, create=create, follow_symlinks=follow_symlinks) def import_single_file(self, external_pathspec: str) -> FileListResult: result = FileListResult() if self.__check_replacement(os.path.basename(external_pathspec), os.path.dirname(external_pathspec), result): self.__add_file( os.path.basename(external_pathspec), external_pathspec, properties=None, ) result.files_written.append(external_pathspec) return result def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int = BST_ARBITRARY_TIMESTAMP) -> None: self._ensure_local() for filename, entry in sorted(self.__index.items()): arcname = os.path.join(destination_dir, filename) if entry.type == FileType.DIRECTORY: tarinfo = tarfilelib.TarInfo(arcname) tarinfo.mtime = mtime tarinfo.type = tarfilelib.DIRTYPE tarinfo.mode = 0o755 tarfile.addfile(tarinfo) self.open_directory(filename).export_to_tar(tarfile, arcname, mtime) elif entry.type == FileType.REGULAR_FILE: source_name = self.__cas_cache.objpath(entry.digest) tarinfo = tarfilelib.TarInfo(arcname) tarinfo.mtime = mtime if entry.is_executable: tarinfo.mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH tarinfo.size = os.path.getsize(source_name) with open(source_name, "rb") as f: tarfile.addfile(tarinfo, f) elif entry.type == FileType.SYMLINK: assert entry.target is not None tarinfo = tarfilelib.TarInfo(arcname) tarinfo.mtime = mtime tarinfo.mode = 0o777 tarinfo.linkname = entry.target tarinfo.type = tarfilelib.SYMTYPE sio = StringIO(entry.target) bio = BytesIO(sio.read().encode("utf8")) tarfile.addfile(tarinfo, bio) else: raise DirectoryError("can not export file type {} to tar".format(entry.type)) def list_relative_paths(self) -> Iterator[str]: yield from self.__list_prefixed_relative_paths() def exists(self, path: str, *, follow_symlinks: bool = False) -> bool: self._validate_path(path) paths = path.split("/") try: self.__entry_from_path(paths, follow_symlinks=follow_symlinks) return True except DirectoryError: return False def stat(self, path: str, *, follow_symlinks: bool = False) -> FileStat: self._validate_path(path) paths = path.split("/") entry = self.__entry_from_path(paths, follow_symlinks=follow_symlinks) if entry.type == FileType.REGULAR_FILE: size = entry.get_digest().size_bytes elif entry.type == FileType.DIRECTORY: size = 0 elif entry.type == FileType.SYMLINK: assert entry.target is not None size = len(entry.target) else: raise DirectoryError("Unsupported file type {}".format(entry.type)) executable = False if entry.type == FileType.DIRECTORY or entry.is_executable: executable = True mtime: float = BST_ARBITRARY_TIMESTAMP if entry.mtime is not None: mtime = utils._parse_protobuf_timestamp(entry.mtime) return FileStat(entry.type, executable=executable, size=size, mtime=mtime) @contextmanager def open_file(self, path: str, *, mode: str = "r") -> Iterator[IO]: self._validate_path(path) paths = path.split("/") subdir = self.__open_directory(paths[:-1]) entry = subdir.__index.get(paths[-1]) if entry and entry.type != FileType.REGULAR_FILE: raise DirectoryError("{} in {} is not a file".format(paths[-1], str(subdir))) if mode not in ["r", "rb", "w", "wb", "w+", "w+b", "x", "xb", "x+", "x+b"]: raise ValueError("Unsupported mode: `{}`".format(mode)) if "b" in mode: encoding = None else: encoding = "utf-8" if "r" in mode: if not entry: raise DirectoryError("{} not found in {}".format(paths[-1], str(subdir))) # Read-only access, allow direct access to CAS object with open(self.__cas_cache.objpath(entry.digest), mode, encoding=encoding) as f: yield f else: if "x" in mode and entry: raise DirectoryError("{} already exists in {}".format(paths[-1], str(subdir))) with utils._tempnamedfile(mode, encoding=encoding, dir=self.__cas_cache.tmpdir) as f: # Make sure the temporary file is readable by buildbox-casd os.chmod(f.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH) yield f # Import written temporary file into CAS f.flush() subdir.__add_file(paths[-1], f.name) def file_digest(self, path: str) -> str: self._validate_path(path) paths = path.split("/") entry = self.__entry_from_path(paths) if entry.type != FileType.REGULAR_FILE: raise DirectoryError("Unsupported file type for digest: {}".format(entry.type)) return entry.digest.hash def readlink(self, path: str) -> str: self._validate_path(path) paths = path.split("/") entry = self.__entry_from_path(paths) if entry.type != FileType.SYMLINK: raise DirectoryError("Unsupported file type for readlink: {}".format(entry.type)) assert entry.target is not None return entry.target def remove(self, path: str, *, recursive: bool = False) -> None: self._validate_path(path) paths = path.split("/") if len(paths) > 1: # Delegate remove to subdirectory subdir = self.__open_directory(paths[:-1]) subdir.remove(paths[-1], recursive=recursive) return name = paths[0] entry = self.__index.get(name) if not entry: raise DirectoryError("{} not found in {}".format(name, str(self))) if entry.type == FileType.DIRECTORY and not recursive: subdir = entry.get_directory(self) if subdir: raise DirectoryError("{} is not empty".format(str(subdir))) del self.__index[name] self.__invalidate_digest() def rename(self, src: str, dest: str) -> None: self._validate_path(src) self._validate_path(dest) src_paths = src.split("/") dest_paths = dest.split("/") srcdir = self.__open_directory(src_paths[:-1]) entry = srcdir.__entry_from_path([src_paths[-1]]) destdir = self.__open_directory(dest_paths[:-1]) srcdir.remove(src_paths[-1], recursive=True) entry.name = dest_paths[-1] destdir.__add_entry(entry) ############################################################# # Implementation of Internal API # ############################################################# def _import_files( self, external_pathspec: Union[Directory, str], *, filter_callback: Optional[Callable[[str], bool]] = None, update_mtime: Optional[float] = None, properties: Optional[List[str]] = None, collect_result: bool = True ) -> Optional[FileListResult]: result = FileListResult() if collect_result else None # See if we can get a source directory to copy from source_directory: Optional[str] = None if isinstance(external_pathspec, str): source_directory = external_pathspec elif isinstance(external_pathspec, Directory): try: source_directory = external_pathspec._get_underlying_directory() except DirectoryError: pass if source_directory: # Import files from local filesystem by first importing complete # directory into CAS (using buildbox-casd) and then importing its # content into this CasBasedDirectory using CAS-to-CAS import # to write the report, handle possible conflicts (if the target # directory is not empty) and apply the optional filter. digest = self.__cas_cache.import_directory(source_directory, properties=properties) external_pathspec = CasBasedDirectory(self.__cas_cache, digest=digest) assert isinstance(external_pathspec, CasBasedDirectory) self.__partial_import_cas_into_cas(external_pathspec, filter_callback, result=result) return result def _export_files(self, to_directory: str, *, can_link: bool = False, can_destroy: bool = False) -> None: # # This is documented to raise DirectoryError, if we are raising a system error # or an error from CAS, it is a bug and we should catch/re-raise from here. # self.__cas_cache.checkout(to_directory, self._get_digest(), can_link=can_link) # We don't store UID/GID in CAS presently, so this can be ignored. def _set_deterministic_user(self) -> None: pass def _ensure_local(self): self.__cas_cache.ensure_tree(self._get_digest()) def _get_underlying_path(self, filename) -> str: try: entry = self.__index[filename] except IndexError as e: raise DirectoryError("Directory does not contain any filename: {}".format(filename)) from e return self.__cas_cache.objpath(entry.digest) def _get_underlying_directory(self) -> str: # There is no underlying directory for a CAS-backed directory, just raise an error. raise DirectoryError( "_get_underlying_directory was called on a CAS-backed directory, which has no underlying directory." ) def _get_size(self) -> int: digest = self._get_digest() total = digest.size_bytes for i in self.__index.values(): if i.type == FileType.DIRECTORY: subdir = i.get_directory(self) total += subdir._get_size() elif i.type == FileType.REGULAR_FILE: total += i.digest.size_bytes # Symlink nodes are encoded as part of the directory serialization. return total ############################################################# # Internal API (specific to CasBasedDirectory) # ############################################################# # _clear(): # # Remove all entries from this directory. # def _clear(self) -> None: self.__invalidate_digest() self.__index = {} # _reset(): # # Replace the contents of this directory with the entries from the specified # directory digest. # # Args: # digest (remote_execution_pb2.Digest): The digest of the replacement directory # def _reset(self, *, digest=None) -> None: self._clear() if digest: self.__digest = digest self.__populate_index(digest) # _set_subtree_read_only() # # Sets this directory as read only # def _set_subtree_read_only(self, read_only: bool) -> None: self.__subtree_read_only = read_only self.__invalidate_digest() # _apply_changes(): # # Apply changes from dir_a to dir_b to this directory. The use # case for this is to merge changes between different workspace # versions into a buildtree. # # If a change was made both to this directory, as well as between # the given directories, it is applied, overwriting any changes to # this directory. This is desirable because we want to keep user # changes, however it may need to be re-considered for other use # cases. # # We perform this computation this way, instead of with a _diff # method and a subsequent _apply_diff, because it prevents leaking # _IndexEntry objects, which contain mutable references and may # therefore cause problems if used outside of this class. # # Args: # dir_a: The directory from which to start computing differences. # dir_b: The directory whose changes to apply # def _apply_changes(self, dir_a: "CasBasedDirectory", dir_b: "CasBasedDirectory") -> None: # If the digests are the same, the directories are the same # (child properties affect the digest). We can skip any work # in such a case. if dir_a._get_digest() == dir_b._get_digest(): return def get_subdir(entry: _IndexEntry, directory: CasBasedDirectory) -> CasBasedDirectory: return directory.__index[entry.name].get_directory(directory) def is_dir_in(entry: _IndexEntry, directory: CasBasedDirectory) -> bool: return directory.__index[entry.name].type == FileType.DIRECTORY # We first check which files were added, and add them to our # directory. for entry in dir_b.__index.values(): if self.__contains_entry(entry): # We can short-circuit checking entries from b that # already exist in our index. continue if not dir_a.__contains_entry(entry): if entry.name in self.__index and is_dir_in(entry, self) and is_dir_in(entry, dir_b): # If the entry changed, and is a directory in both # the current and to-merge-into tree, we need to # merge recursively. # If the entry is not a directory in dir_a, we # want to overwrite the file, but we need an empty # directory for recursion. if entry.name in dir_a.__index and is_dir_in(entry, dir_a): sub_a = get_subdir(entry, dir_a) else: sub_a = CasBasedDirectory(dir_a.__cas_cache) subdir = get_subdir(entry, self) subdir._apply_changes(sub_a, get_subdir(entry, dir_b)) else: # In any other case, we just add/overwrite the file/directory self.__add_entry(entry) # We can't iterate and remove entries at the same time to_remove = [entry for entry in dir_a.__index.values() if entry.name not in dir_b.__index] for entry in to_remove: self.remove(entry.name, recursive=True) self.__invalidate_digest() ############################################################# # Private methods # ############################################################# # _get_digest(): # # Return the Digest for this directory. # # Returns: # (Digest): The Digest protobuf object for the Directory protobuf # # Note that this has a single underscore because it is accessed # by the private _IndexEntry class # def _get_digest(self): if not self.__digest: # Create updated Directory proto pb2_directory = remote_execution_pb2.Directory() if self.__subtree_read_only is not None: node_property = pb2_directory.node_properties.properties.add() node_property.name = "SubtreeReadOnly" node_property.value = "true" if self.__subtree_read_only else "false" for name, entry in sorted(self.__index.items()): if entry.type == FileType.DIRECTORY: dirnode = pb2_directory.directories.add() dirnode.name = name # Update digests for subdirectories in DirectoryNodes. # No need to call entry.get_directory(). # If it hasn't been instantiated, digest must be up-to-date. subdir = entry.directory if subdir is not None: dirnode.digest.CopyFrom(subdir._get_digest()) else: dirnode.digest.CopyFrom(entry.digest) elif entry.type == FileType.REGULAR_FILE: filenode = pb2_directory.files.add() filenode.name = name filenode.digest.CopyFrom(entry.digest) filenode.is_executable = entry.is_executable if entry.mtime is not None: filenode.node_properties.mtime.CopyFrom(entry.mtime) elif entry.type == FileType.SYMLINK: symlinknode = pb2_directory.symlinks.add() symlinknode.name = name symlinknode.target = entry.target self.__digest = self.__cas_cache.add_object(buffer=pb2_directory.SerializeToString()) return self.__digest # __open_directory() # # Open a directory using a list of already separated path components # def __open_directory( self, paths: List[str], *, create: bool = False, follow_symlinks: bool = False ) -> "CasBasedDirectory": # Note: At the moment, creating a directory by opening a directory does # not update this object in the CAS cache. However, performing # an import_files() into a subdirectory of any depth obtained # from this object *will* cause this directory to be updated and stored. current_dir = self for element in paths: # Skip empty path segments if not element: continue entry = current_dir.__index.get(element) if entry: if entry.type == FileType.DIRECTORY: current_dir = entry.get_directory(current_dir) elif follow_symlinks and entry.type == FileType.SYMLINK: assert entry.target is not None linklocation = entry.target newpaths = linklocation.split(os.path.sep) if os.path.isabs(linklocation): current_dir = current_dir.__find_root().__open_directory(newpaths, follow_symlinks=True) else: current_dir = current_dir.__open_directory(newpaths, follow_symlinks=True) else: error = "Cannot open {}, which is a '{}' in the directory {}" raise DirectoryError( error.format(element, current_dir.__index[element].type, current_dir), reason="not-a-directory" ) else: if element == ".": continue if element == "..": if current_dir.__parent is not None: current_dir = current_dir.__parent # In POSIX /.. == / so just stay at the root dir continue if create: current_dir = current_dir.__add_directory(element) else: error = "'{}' not found in {}" raise DirectoryError(error.format(element, str(current_dir)), reason="directory-not-found") return current_dir # __populate_index() # # Populate the _IndexEntry for this digest # # Args: # digest: A remote_execution_pb2.Digest # def __populate_index(self, digest) -> None: try: pb2_directory = remote_execution_pb2.Directory() with open(self.__cas_cache.objpath(digest), "rb") as f: pb2_directory.ParseFromString(f.read()) except FileNotFoundError as e: raise DirectoryError("Directory not found in local cache: {}".format(e)) from e for prop in pb2_directory.node_properties.properties: if prop.name == "SubtreeReadOnly": self.__subtree_read_only = prop.value == "true" for dentry in pb2_directory.directories: self.__index[dentry.name] = _IndexEntry( self.__cas_cache, dentry.name, FileType.DIRECTORY, digest=dentry.digest ) for entry in pb2_directory.files: mtime: Optional[timestamp_pb2.Timestamp] if entry.node_properties.HasField("mtime"): mtime = entry.node_properties.mtime else: mtime = None self.__index[entry.name] = _IndexEntry( self.__cas_cache, entry.name, FileType.REGULAR_FILE, digest=entry.digest, is_executable=entry.is_executable, mtime=mtime, ) for lentry in pb2_directory.symlinks: self.__index[lentry.name] = _IndexEntry( self.__cas_cache, lentry.name, FileType.SYMLINK, target=lentry.target ) def __add_directory(self, name: str) -> "CasBasedDirectory": assert name not in self.__index newdir = CasBasedDirectory(self.__cas_cache, parent=self, filename=name) self.__index[name] = _IndexEntry(self.__cas_cache, name, FileType.DIRECTORY, directory=newdir) self.__invalidate_digest() return newdir def __add_file(self, name: str, path: str, properties: Optional[List[str]] = None) -> None: digest = self.__cas_cache.add_object(path=path) is_executable = os.access(path, os.X_OK) mtime = None if properties and "mtime" in properties: mtime = timestamp_pb2.Timestamp() # pylint: disable=no-member utils._get_file_protobuf_mtimestamp(mtime, path) entry = _IndexEntry( self.__cas_cache, name, FileType.REGULAR_FILE, digest=digest, is_executable=is_executable, mtime=mtime, ) self.__index[name] = entry self.__invalidate_digest() def __add_entry(self, entry: _IndexEntry) -> None: self.__index[entry.name] = entry.clone() self.__invalidate_digest() def __contains_entry(self, entry: _IndexEntry) -> bool: return entry == self.__index.get(entry.name) def __add_new_link_direct(self, name, target) -> None: self.__index[name] = _IndexEntry(self.__cas_cache, name, FileType.SYMLINK, target=target) self.__invalidate_digest() # __check_replacement() # # Checks whether 'name' exists, and if so, whether we can overwrite it. # If we can, add the name to 'overwritten_files' and delete the existing entry. # Returns 'True' if the import should go ahead. # fileListResult.overwritten and fileListResult.ignore are updated depending # on the result. # def __check_replacement(self, name: str, relative_pathname: str, fileListResult: Optional[FileListResult]) -> bool: existing_entry = self.__index.get(name) if existing_entry is None: return True elif existing_entry.type == FileType.DIRECTORY: # If 'name' maps to a DirectoryNode, then there must be an entry in index # pointing to another Directory. subdir = existing_entry.get_directory(self) if not subdir: self.remove(name) if fileListResult is not None: fileListResult.overwritten.append(relative_pathname) return True else: # We can't overwrite a non-empty directory, so we just ignore it. if fileListResult is not None: fileListResult.ignored.append(relative_pathname) return False else: self.remove(name) if fileListResult is not None: fileListResult.overwritten.append(relative_pathname) return True # __partial_import_cas_into_cas() # # Import files from a CAS-based directory. # def __partial_import_cas_into_cas( self, source_directory: "CasBasedDirectory", filter_callback: Optional[Callable[[str], bool]] = None, *, path_prefix: str = "", origin: Optional["CasBasedDirectory"] = None, result: Optional[FileListResult] ) -> None: if origin is None: origin = self for name, entry in source_directory.__index.items(): # The destination filename, relative to the root where the import started relative_pathname = os.path.join(path_prefix, name) is_dir = entry.type == FileType.DIRECTORY if is_dir: create_subdir = name not in self.__index if create_subdir and not filter_callback: # If subdirectory does not exist yet and there is no filter, # we can import the whole source directory by digest instead # of importing each directory entry individually. subdir_digest = entry.get_digest() dest_entry = _IndexEntry(self.__cas_cache, name, FileType.DIRECTORY, digest=subdir_digest) self.__index[name] = dest_entry self.__invalidate_digest() # However, we still need to iterate over the directory entries # to fill in `result.files_written`. # Use source subdirectory object if it already exists, # otherwise create object for destination subdirectory. # This is based on the assumption that the destination # subdirectory is more likely to be modified later on # (e.g., by further import_files() calls). if entry.directory is not None: subdir = entry.directory else: subdir = dest_entry.get_directory(self) if result is not None: subdir.__add_files_to_result(path_prefix=relative_pathname, result=result) else: src_subdir = source_directory.open_directory(name) if src_subdir == origin: continue try: dest_subdir = self.open_directory(name, create=create_subdir) except DirectoryError: filetype = self.__index[name].type raise DirectoryError( "Destination is a {}, not a directory: /{}".format(filetype, relative_pathname) ) dest_subdir.__partial_import_cas_into_cas( src_subdir, filter_callback, path_prefix=relative_pathname, origin=origin, result=result ) if filter_callback and not filter_callback(relative_pathname): if is_dir and create_subdir and not dest_subdir: # Complete subdirectory has been filtered out, remove it self.remove(name) # Entry filtered out, move to next continue if not is_dir: if self.__check_replacement(name, relative_pathname, result): if entry.type == FileType.REGULAR_FILE: self.__add_entry(entry) else: assert entry.type == FileType.SYMLINK self.__add_new_link_direct(name=name, target=entry.target) if result is not None: result.files_written.append(relative_pathname) # __list_prefixed_relative_paths() # # Provide a list of all relative paths. # # Args: # prefix: an optional prefix to the relative paths, this is # also emitted by itself. # # Yields: # ist of all files with relative paths. # def __list_prefixed_relative_paths(self, prefix: str = "") -> Iterator[str]: file_list = list(filter(lambda i: i[1].type != FileType.DIRECTORY, self.__index.items())) directory_list = filter(lambda i: i[1].type == FileType.DIRECTORY, self.__index.items()) if prefix != "": yield prefix for (k, v) in sorted(file_list): yield os.path.join(prefix, k) for (k, v) in sorted(directory_list): subdir = v.get_directory(self) yield from subdir.__list_prefixed_relative_paths(prefix=os.path.join(prefix, k)) def __get_identifier(self) -> str: path = "" if self.__parent: path = self.__parent.__get_identifier() if self.__filename: path += "/" + self.__filename else: path += "/" return path def __find_root(self) -> "CasBasedDirectory": if self.__parent: return self.__parent.__find_root() else: return self def __entry_from_path(self, path: List[str], *, follow_symlinks: bool = False) -> _IndexEntry: subdir = self.__open_directory(path[:-1], follow_symlinks=follow_symlinks) target = subdir.__index.get(path[-1]) if target is None: raise DirectoryError("{} not found in {}".format(path[-1], str(subdir))) if follow_symlinks and target.type == FileType.SYMLINK: assert target.target is not None linklocation = target.target newpath = linklocation.split("/") if os.path.isabs(linklocation): return subdir.__find_root().__entry_from_path(newpath, follow_symlinks=True) return subdir.__entry_from_path(newpath, follow_symlinks=True) else: return target def __invalidate_digest(self) -> None: if self.__digest: self.__digest = None if self.__parent: self.__parent.__invalidate_digest() def __add_files_to_result(self, *, path_prefix: str, result: FileListResult) -> None: for name, entry in self.__index.items(): # The destination filename, relative to the root where the import started relative_pathname = os.path.join(path_prefix, name) if entry.type == FileType.DIRECTORY: subdir = self.open_directory(name) subdir.__add_files_to_result(path_prefix=relative_pathname, result=result) else: result.files_written.append(relative_pathname) apache-buildstream-27ae392/src/buildstream/storage/_filebaseddirectory.py000066400000000000000000000456751514607367700267650ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jim MacArthur # Tristan van Berkom import os import shutil import stat from contextlib import contextmanager from tarfile import TarFile from typing import Callable, Optional, Union, List, IO, Iterator from .directory import Directory, DirectoryError, FileType, FileStat from .. import utils from ..utils import BST_ARBITRARY_TIMESTAMP from ..utils import FileListResult # FileBasedDirectory intentionally doesn't call its superclass constuctor, # which is meant to be unimplemented. # pylint: disable=super-init-not-called class FileBasedDirectory(Directory): def __init__(self, external_directory: str, *, parent: Optional["FileBasedDirectory"] = None) -> None: self.__external_directory: str = external_directory self.__parent: Optional[FileBasedDirectory] = parent def __iter__(self) -> Iterator[str]: yield from os.listdir(self.__external_directory) def __len__(self) -> int: entries = list(os.listdir(self.__external_directory)) return len(entries) def __str__(self) -> str: # This returns the whole path (since we don't know where the directory started) # which exposes the sandbox directory; we will have to assume for the time being # that people will not abuse __str__. return self.__external_directory ############################################################# # Implementation of Public API # ############################################################# def open_directory( self, path: str, *, create: bool = False, follow_symlinks: bool = False ) -> "FileBasedDirectory": self._validate_path(path) paths = path.split("/") return self.__open_directory(paths, create=create, follow_symlinks=follow_symlinks) def import_single_file(self, external_pathspec: str) -> FileListResult: dstpath = os.path.join(self.__external_directory, os.path.basename(external_pathspec)) result = FileListResult() if os.path.exists(dstpath): result.ignored.append(dstpath) else: shutil.copyfile(external_pathspec, dstpath, follow_symlinks=False) return result # Add a directory entry deterministically to a tar file # # This function takes extra steps to ensure the output is deterministic. # First, it sorts the results of os.listdir() to ensure the ordering of # the files in the archive is the same. Second, it sets a fixed # timestamp for each entry. See also https://bugs.python.org/issue24465. def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int = BST_ARBITRARY_TIMESTAMP) -> None: # We need directories here, including non-empty ones, # so list_relative_paths is not used. for filename in sorted(os.listdir(self.__external_directory)): source_name = os.path.join(self.__external_directory, filename) arcname = os.path.join(destination_dir, filename) tarinfo = tarfile.gettarinfo(source_name, arcname) tarinfo.mtime = mtime tarinfo.uid = 0 tarinfo.gid = 0 tarinfo.uname = "" tarinfo.gname = "" if tarinfo.isreg(): with open(source_name, "rb") as f: tarfile.addfile(tarinfo, f) elif tarinfo.isdir(): tarfile.addfile(tarinfo) self.open_directory(filename).export_to_tar(tarfile, arcname, mtime) else: tarfile.addfile(tarinfo) def list_relative_paths(self) -> Iterator[str]: yield from utils.list_relative_paths(self.__external_directory) def exists(self, path: str, *, follow_symlinks: bool = False) -> bool: try: self.stat(path, follow_symlinks=follow_symlinks) return True except DirectoryError: return False def stat(self, path: str, *, follow_symlinks: bool = False) -> FileStat: self._validate_path(path) paths = path.split("/") subdir = self.__open_directory(paths[:-1], follow_symlinks=follow_symlinks) newpath = os.path.join(subdir.__external_directory, paths[-1]) try: st = os.lstat(newpath) except OSError as e: raise DirectoryError("Error accessing path '{}': {}".format(newpath, e)) from e if follow_symlinks and stat.S_ISLNK(st.st_mode): linklocation = os.readlink(newpath) if os.path.isabs(linklocation): # Call stat on the root, remove the leading "/" return subdir.__find_root().stat(linklocation[1:], follow_symlinks=True) return subdir.stat(linklocation, follow_symlinks=True) else: return self.__convert_filestat(st) @contextmanager def open_file(self, path: str, *, mode: str = "r") -> Iterator[IO]: self._validate_path(path) paths = path.split("/") # Use open_directory() to avoid following symlinks (potentially escaping the sandbox) subdir = self.__open_directory(paths[:-1]) newpath = os.path.join(subdir.__external_directory, paths[-1]) if mode not in ["r", "rb", "w", "wb", "w+", "w+b", "x", "xb", "x+", "x+b"]: raise ValueError("Unsupported mode: `{}`".format(mode)) if "b" in mode: encoding = None else: encoding = "utf-8" if "r" in mode: with open(newpath, mode=mode, encoding=encoding) as f: yield f else: if "x" in mode: # This check is not atomic, however, we're operating with a # single thread in a private directory tree. if subdir.exists(path[-1]): raise DirectoryError("{} already exists in {}".format(path[-1], str(subdir))) mode = "w" + mode[1:] with utils.save_file_atomic(newpath, mode=mode, encoding=encoding) as f: yield f def file_digest(self, path: str) -> str: self._validate_path(path) paths = path.split("/") # Use open_directory() to avoid following symlinks (potentially escaping the sandbox) subdir = self.__open_directory(paths[:-1]) if subdir.exists(paths[-1]) and not subdir.isfile(paths[-1]): raise DirectoryError("Unsupported file type for digest") newpath = os.path.join(subdir.__external_directory, paths[-1]) return utils.sha256sum(newpath) def readlink(self, path: str) -> str: self._validate_path(path) paths = path.split("/") # Use open_directory() to avoid following symlinks (potentially escaping the sandbox) subdir = self.__open_directory(paths[:-1]) if subdir.exists(paths[-1]) and not subdir.islink(paths[-1]): raise DirectoryError("Unsupported file type for readlink") newpath = os.path.join(subdir.__external_directory, paths[-1]) return os.readlink(newpath) def remove(self, path: str, *, recursive: bool = False) -> None: self._validate_path(path) paths = path.split("/") # Use open_directory() to avoid following symlinks (potentially escaping the sandbox) subdir = self.__open_directory(paths[:-1]) newpath = os.path.join(subdir.__external_directory, paths[-1]) if subdir.isdir(paths[-1]): if recursive: shutil.rmtree(newpath) else: try: os.rmdir(newpath) except OSError as e: raise DirectoryError("Error removing '{}': {}".format(newpath, e)) else: try: os.unlink(newpath) except OSError as e: raise DirectoryError("Error removing '{}': {}".format(newpath, e)) def rename(self, src: str, dest: str) -> None: self._validate_path(src) self._validate_path(dest) src_paths = src.split("/") dest_paths = dest.split("/") # Use open_directory() to avoid following symlinks (potentially escaping the sandbox) srcdir = self.__open_directory(src_paths[:-1]) destdir = self.__open_directory(dest_paths[:-1]) srcpath = os.path.join(srcdir.__external_directory, src_paths[-1]) destpath = os.path.join(destdir.__external_directory, dest_paths[-1]) if destdir.exists(dest_paths[-1]): destdir.remove(dest_paths[-1]) try: os.rename(srcpath, destpath) except OSError as e: raise DirectoryError("Error renaming '{}' -> '{}': {}".format(srcpath, destpath, e)) ############################################################# # Implementation of Internal API # ############################################################# def _import_files( self, external_pathspec: Union[Directory, str], *, filter_callback: Optional[Callable[[str], bool]] = None, update_mtime: Optional[float] = None, properties: Optional[List[str]] = None, collect_result: bool = True ) -> FileListResult: # See if we can get a source directory to copy from source_directory: Optional[str] = None if isinstance(external_pathspec, str): source_directory = external_pathspec elif isinstance(external_pathspec, Directory): try: source_directory = external_pathspec._get_underlying_directory() except DirectoryError: pass if source_directory: # # We've got a source directory to copy from # import_result = utils.copy_files( source_directory, self.__external_directory, filter_callback=filter_callback, ignore_missing=False, report_written=True, ) if update_mtime: for f in import_result.files_written: os.utime(os.path.join(self.__external_directory, f), times=(update_mtime, update_mtime)) else: # # We're dealing with an abstract Directory object # assert isinstance(external_pathspec, Directory) # Ensure blobs are available locally external_pathspec._ensure_local() def copy_action(src_path, dest_path, mtime, result): utils.safe_copy(src_path, dest_path, result=result) utils._set_file_mtime(dest_path, mtime) import_result = FileListResult() self.__import_files_from_directory( external_pathspec, copy_action, filter_callback, update_mtime=update_mtime, result=import_result, ) return import_result def _export_files(self, to_directory: str, *, can_link: bool = False, can_destroy: bool = False) -> None: if can_destroy: # Try a simple rename of the sandbox root; if that # doesnt cut it, then do the regular link files code path try: os.rename(self.__external_directory, to_directory) return except OSError: # Proceed using normal link/copy pass os.makedirs(to_directory, exist_ok=True) if can_link: utils.link_files(self.__external_directory, to_directory) else: utils.copy_files(self.__external_directory, to_directory) def _set_deterministic_user(self) -> None: utils._set_deterministic_user(self.__external_directory) def _get_underlying_path(self, filename) -> str: return os.path.join(self.__external_directory, filename) def _get_underlying_directory(self) -> str: return self.__external_directory def _get_size(self) -> int: return utils._get_dir_size(self.__external_directory) ############################################################# # Private methods # ############################################################# def __open_directory( self, paths: List[str], *, create: bool = False, follow_symlinks: bool = False ) -> "FileBasedDirectory": current_dir = self for path in paths: # Skip empty path segments if not path: continue if path == ".": continue if path == "..": if current_dir.__parent is not None: current_dir = current_dir.__parent # In POSIX /.. == / so just stay at the root dir continue new_path = os.path.join(current_dir.__external_directory, path) try: st = os.lstat(new_path) if stat.S_ISDIR(st.st_mode): current_dir = FileBasedDirectory(new_path, parent=current_dir) elif follow_symlinks and stat.S_ISLNK(st.st_mode): linklocation = os.readlink(new_path) newpaths = linklocation.split(os.path.sep) if os.path.isabs(linklocation): current_dir = current_dir.__find_root().__open_directory(newpaths, follow_symlinks=True) else: current_dir = current_dir.__open_directory(newpaths, follow_symlinks=True) else: raise DirectoryError( "Cannot open '{}': '{}' is not a directory".format(path, new_path), reason="not-a-directory", ) except FileNotFoundError: if create: os.mkdir(new_path) current_dir = FileBasedDirectory(new_path, parent=current_dir) else: raise DirectoryError("Cannot open '{}': '{}' does not exist".format(path, new_path)) return current_dir # __convert_filestat() # # Convert an os.stat_result into a FileStat # def __convert_filestat(self, st: os.stat_result) -> FileStat: file_type: int = 0 if stat.S_ISREG(st.st_mode): file_type = FileType.REGULAR_FILE elif stat.S_ISDIR(st.st_mode): file_type = FileType.DIRECTORY elif stat.S_ISLNK(st.st_mode): file_type = FileType.SYMLINK # If any of the executable bits are set, lets call it executable executable = bool(st.st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) return FileStat(file_type, executable=executable, size=st.st_size, mtime=st.st_mtime) # __find_root() # # Finds the root of this directory tree by following 'parent' until there is # no parent. # def __find_root(self) -> "FileBasedDirectory": if self.__parent: return self.__parent.__find_root() else: return self # __import_files_from_directory() # # Import files from a CAS-based directory. # def __import_files_from_directory( self, source_directory: Directory, actionfunc: Callable[[str, str, float, FileListResult], None], filter_callback: Optional[Callable[[str], bool]] = None, *, path_prefix: str = "", update_mtime: Optional[float] = None, result: FileListResult ) -> None: # Iterate over entries in the source directory for name in source_directory: # The destination filename, relative to the root where the import started relative_pathname = os.path.join(path_prefix, name) # The full destination path dest_path = os.path.join(self.__external_directory, name) is_dir = source_directory.isdir(name) if is_dir: src_subdir = source_directory.open_directory(name) try: create_subdir = not os.path.lexists(dest_path) dest_subdir = self.open_directory(name, create=create_subdir) except DirectoryError: raise DirectoryError("Destination is not a directory: /{}".format(relative_pathname)) dest_subdir.__import_files_from_directory( src_subdir, actionfunc, filter_callback, path_prefix=relative_pathname, result=result, update_mtime=update_mtime, ) if filter_callback and not filter_callback(relative_pathname): if is_dir and create_subdir and not dest_subdir: # Complete subdirectory has been filtered out, remove it os.rmdir(dest_subdir.__external_directory) # Filename filtered out, move to next continue if not is_dir: if os.path.lexists(dest_path): # Collect overlaps if not os.path.isdir(dest_path): result.overwritten.append(relative_pathname) if not utils.safe_remove(dest_path): result.ignored.append(relative_pathname) continue if source_directory.isfile(name): src_path = source_directory._get_underlying_path(name) filestat = source_directory.stat(name) # hardlink files on request, they wont have their mtimes # updated if hardlinking is performed mtime = update_mtime if mtime is None: mtime = filestat.mtime actionfunc(src_path, dest_path, mtime, result) if filestat.executable: os.chmod( dest_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, ) result.files_written.append(relative_pathname) elif source_directory.islink(name): link_target = source_directory.readlink(name) os.symlink(link_target, dest_path) result.files_written.append(relative_pathname) apache-buildstream-27ae392/src/buildstream/storage/directory.py000066400000000000000000000457111514607367700247560ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Jim MacArthur # Tristan van Berkom """ Directory - Interfacing with files ================================== The Directory class is given to plugins by way of the :class:`.Sandbox` and in some other instances. This API allows plugins to interface with files and directory hierarchies owned by BuildStream. .. _directory_path: Paths ----- The path argument to directory functions depict a relative path. Path elements are separated with the ``/`` character regardless of the platform. Both ``.`` and ``..`` entries are permitted. Absolute paths are not permitted, as such it is illegal to specify a path with a leading ``/`` character. Directory objects represent a directory entry within the context of a *directory tree*, and the directory returned by :func:`Sandbox.get_virtual_directory() ` is the root of the sandbox's *directory tree*. Attempts to escape the root of a *directory tree* using ``..`` entries will not result in an error, instead ``..`` entries which cross the root boundary will be evaluated as the root directory. This behavior matches POSIX behavior of filesystem root directories. """ from contextlib import contextmanager from tarfile import TarFile from typing import Callable, Optional, Union, List, IO, Iterator from .._exceptions import BstError from ..exceptions import ErrorDomain from ..utils import BST_ARBITRARY_TIMESTAMP, FileListResult from ..types import FastEnum class DirectoryError(BstError): """Raised by Directory functions. It is recommended to handle this error and raise a more descriptive user facing :class:`.ElementError` or :class:`.SourceError` from this error. If this is not handled, the BuildStream core will fail the current task where the error occurs and present the user with the error. """ def __init__(self, message: str, reason: Optional[str] = None): super().__init__(message, domain=ErrorDomain.VIRTUAL_FS, reason=reason) class FileType(FastEnum): """Depicts the type of a file""" DIRECTORY: int = 1 """A directory""" REGULAR_FILE: int = 2 """A regular file""" SYMLINK: int = 3 """A symbolic link""" def __str__(self): # https://github.com/PyCQA/pylint/issues/2062 return self.name.lower().replace("_", " ") # pylint: disable=no-member class FileStat: """Depicts stats about a file .. note:: This object can be compared with the equality operator, two :class:`.FileStat` objects will be considered equivalent if they have the same :class:`.FileType` and have equivalent attributes. """ def __init__( self, file_type: int, *, executable: bool = False, size: int = 0, mtime: float = BST_ARBITRARY_TIMESTAMP ) -> None: self.file_type: int = file_type """The :class:`.FileType` of this file""" self.executable: bool = executable """Whether this file is executable""" self.size: int = size """The size of the file in bytes""" self.mtime: float = mtime """The modification time of the file""" def __eq__(self, other: object) -> bool: if not isinstance(other, FileStat): return NotImplemented return ( self.file_type == other.file_type and self.executable == other.file_type and self.size == other.size and self.mtime == other.mtime ) class Directory: """The Directory object represents a directory in a filesystem hierarchy .. tip:: Directory objects behave as a collection of entries in the pythonic sense. Iterating over a directory will yield the entries, and a directory is truthy if it contains any entries and falsy if it is empty. """ def __init__(self, external_directory=None): raise NotImplementedError() def __iter__(self) -> Iterator[str]: raise NotImplementedError() def __len__(self) -> int: raise NotImplementedError() ################################################################### # Public API # ################################################################### def open_directory(self, path: str, *, create: bool = False, follow_symlinks: bool = False) -> "Directory": """Open a Directory object relative to this directory Args: path: A :ref:`path ` relative to this directory. create: If this is true, the directories will be created if they don't already exist. Returns: A Directory object representing the found directory. Raises: DirectoryError: if any of the components in subdirectory_spec cannot be found, or are files, or symlinks to files. """ raise NotImplementedError() # Import and export of files and links def import_files( self, external_pathspec: Union["Directory", str], *, filter_callback: Optional[Callable[[str], bool]] = None, collect_result: bool = True ) -> Optional[FileListResult]: """Imports some or all files from external_path into this directory. Args: external_pathspec: Either a string containing a pathname, or a Directory object, to use as the source. filter_callback: Optional filter callback. Called with the relative path as argument for every file in the source directory. The file is imported only if the callable returns True. If no filter callback is specified, all files will be imported. collect_result: Whether to collect data for the :class:`.FileListResult`, defaults to True. Returns: A :class:`.FileListResult` report of files imported and overwritten, or `None` if `collect_result` is False. Raises: DirectoryError: if any system error occurs. """ return self._import_files_internal( external_pathspec, filter_callback=filter_callback, collect_result=collect_result, ) def import_single_file(self, external_pathspec: str) -> FileListResult: """Imports a single file from an external path Args: external_pathspec: A string containing a pathname. properties: Optional list of strings representing file properties to capture when importing. Returns: A :class:`.FileListResult` report of files imported and overwritten. Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() def export_to_tar(self, tarfile: TarFile, destination_dir: str, mtime: int = BST_ARBITRARY_TIMESTAMP) -> None: """Exports this directory into the given tar file. Args: tarfile: A Python TarFile object to export into. destination_dir: The prefix for all filenames inside the archive. mtime: mtimes of all files in the archive are set to this. Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() def list_relative_paths(self) -> Iterator[str]: """Generate a list of all relative paths in this directory. Yields: All files in this directory with relative paths. """ raise NotImplementedError() def exists(self, path: str, *, follow_symlinks: bool = False) -> bool: """Check whether the specified path exists. Args: path: A :ref:`path ` relative to this directory. follow_symlinks: True to follow symlinks. Returns: True if the path exists, False otherwise. """ raise NotImplementedError() def stat(self, path: str, *, follow_symlinks: bool = False) -> FileStat: """Get the status of a file. Args: path: A :ref:`path ` relative to this directory. follow_symlinks: True to follow symlinks. Returns: A :class:`.FileStat` object. Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() @contextmanager def open_file(self, path: str, *, mode: str = "r") -> Iterator[IO]: """Open file and return a corresponding file object. In text mode, UTF-8 is used as encoding. Args: path: A :ref:`path ` relative to this directory. mode (str): An optional string that specifies the mode in which the file is opened. Yields: The file object for the open file Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() def file_digest(self, path: str) -> str: """Return a unique digest of a file. Args: path: A :ref:`path ` relative to this directory. Raises: DirectoryError: if the specified *path* depicts an entry that is not a :attr:`.FileType.REGULAR_FILE`, or if any system error occurs. """ raise NotImplementedError() def readlink(self, path: str) -> str: """Return a string representing the path to which the symbolic link points. Args: path: A :ref:`path ` relative to this directory. Returns: The path to which the symbolic link points to. Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() def remove(self, path: str, *, recursive: bool = False) -> None: """Remove a file, symlink or directory. Symlinks are not followed. Args: path: A :ref:`path ` relative to this directory. recursive: True to delete non-empty directories. Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() def rename(self, src: str, dest: str) -> None: """Rename a file, symlink or directory. If destination path exists already and is a file or empty directory, it will be replaced. Args: src: A source :ref:`path ` relative to this directory. dest: A destination :ref:`path ` relative to this directory. Raises: DirectoryError: if any system error occurs. """ raise NotImplementedError() def isfile(self, path: str, *, follow_symlinks: bool = False) -> bool: """Check whether the specified path is an existing regular file. Args: path: A :ref:`path ` relative to this directory. follow_symlinks: True to follow symlinks. Returns: True if the path is an existing regular file, False otherwise. """ try: st = self.stat(path, follow_symlinks=follow_symlinks) return st.file_type == FileType.REGULAR_FILE except DirectoryError: return False def isdir(self, path: str, *, follow_symlinks: bool = False) -> bool: """Check whether the specified path is an existing directory. Args: path: A :ref:`path ` relative to this directory. follow_symlinks: True to follow symlinks. Returns: True if the path is an existing directory, False otherwise. """ try: st = self.stat(path, follow_symlinks=follow_symlinks) return st.file_type == FileType.DIRECTORY except DirectoryError: return False def islink(self, path: str, *, follow_symlinks: bool = False) -> bool: """Check whether the specified path is an existing symlink. Args: path: A :ref:`path ` relative to this directory. follow_symlinks: True to follow symlinks. Returns: True if the path is an existing symlink, False otherwise. """ try: st = self.stat(path, follow_symlinks=follow_symlinks) return st.file_type == FileType.SYMLINK except DirectoryError: return False ################################################################### # Internal API # ################################################################### # _import_files_internal() # # Internal API for importing files, which exposes a few more parameters than # the public API exposes. # # Args: # external_pathspec: Either a string containing a pathname, or a # Directory object, to use as the source. # filter_callback: Optional filter callback. Called with the # relative path as argument for every file in the source directory. # The file is imported only if the callable returns True. # If no filter callback is specified, all files will be imported. # update_mtime: Update the access and modification time of each file copied to the time specified in seconds. # properties: Optional list of strings representing file properties to capture when importing. # collect_result: Whether to collect data for the :class:`.FileListResult`, defaults to True. # # Returns: # A :class:`.FileListResult` report of files imported and overwritten, # or `None` if `collect_result` is False. # # Raises: # DirectoryError: if any system error occurs. # def _import_files_internal( self, external_pathspec: Union["Directory", str], *, filter_callback: Optional[Callable[[str], bool]] = None, update_mtime: Optional[float] = None, properties: Optional[List[str]] = None, collect_result: bool = True ) -> Optional[FileListResult]: return self._import_files( external_pathspec, filter_callback=filter_callback, update_mtime=update_mtime, properties=properties, collect_result=collect_result, ) # _import_files() # # Abstract method for backends to import files from an external directory # # Args: # external_pathspec: Either a string containing a pathname, or a # Directory object, to use as the source. # filter_callback: Optional filter callback. Called with the # relative path as argument for every file in the source directory. # The file is imported only if the callable returns True. # If no filter callback is specified, all files will be imported. # update_mtime: Update the access and modification time of each file copied to the time specified in seconds. # properties: Optional list of strings representing file properties to capture when importing. # collect_result: Whether to collect data for the :class:`.FileListResult`, defaults to True. # # Returns: # A :class:`.FileListResult` report of files imported and overwritten, # or `None` if `collect_result` is False. # # Raises: # DirectoryError: if any system error occurs. # def _import_files( self, external_pathspec: Union["Directory", str], *, filter_callback: Optional[Callable[[str], bool]] = None, update_mtime: Optional[float] = None, properties: Optional[List[str]] = None, collect_result: bool = True ) -> Optional[FileListResult]: raise NotImplementedError() # _export_files() # # Exports everything from this directory into to_directory. # # Args: # to_directory: a path outside this directory object where the contents will be copied to. # can_link: Whether we can create hard links in to_directory instead of copying. # Setting this does not guarantee hard links will be used. # can_destroy: Can we destroy the data already in this directory when exporting? If set, # this may allow data to be moved rather than copied which will be quicker. # # Raises: # DirectoryError: if any system error occurs. # def _export_files(self, to_directory: str, *, can_link: bool = False, can_destroy: bool = False) -> None: raise NotImplementedError() # _ensure_local() # # Makes sure the files for the directory are available locally. Should be called before # using _get_underlying_path() def _ensure_local(self): pass # _get_underlying_path() # # Args: # filename: The name of the file in this directory # # Returns the underlying (real) file system path for the file in this # directory # # Raises: # DirectoryError: if the backend doesn't use local files, or if # there is no such file in this directory # def _get_underlying_path(self, filename) -> str: raise NotImplementedError() # _get_underlying_directory() # # Returns the underlying (real) file system directory this # object refers to. # # Raises: # DirectoryError: if the backend doesn't have an underlying directory # def _get_underlying_directory(self) -> str: raise NotImplementedError() # _set_deterministic_user(): # # Abstract method to set all files in this directory to the current user's euid/egid. # def _set_deterministic_user(self): raise NotImplementedError() # _get_size() # # Get an approximation of the storage space in bytes used by this directory # and all files and subdirectories in it. Storage space varies by implementation # and effective space used may be lower than this number due to deduplication. # def _get_size(self) -> int: raise NotImplementedError() # _create_empty_file() # # Utility function to create an empty file # def _create_empty_file(self, path: str) -> None: with self.open_file(path, mode="w"): pass # _validate_path() # # Convenience function for backends to validate path input # def _validate_path(self, path: str) -> None: if path and path[0] == "/": raise ValueError("Invalid path '{}'".format(path)) apache-buildstream-27ae392/src/buildstream/types.py000066400000000000000000000305441514607367700224500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Jim MacArthur # Benjamin Schubert """ Foundation types ================ """ from typing import Any, Dict, List, Union, Optional import os from .node import MappingNode, SequenceNode from ._types import MetaFastEnum class FastEnum(metaclass=MetaFastEnum): """ A reimplementation of a subset of the `Enum` functionality, which is far quicker than `Enum`. :class:`enum.Enum` attributes accesses can be really slow, and slow down the execution noticeably. This reimplementation doesn't suffer the same problems, but also does not reimplement everything. """ name = None """The name of the current Enum entry, same as :func:`enum.Enum.name` """ value = None """The value of the current Enum entry, same as :func:`enum.Enum.value` """ # A dict of all values mapping to the entries in the enum _value_to_entry = {} # type: Dict[str, Any] @classmethod def values(cls): """Get all the possible values for the enum. Returns: list: the list of all possible values for the enum """ return cls._value_to_entry.keys() def __new__(cls, value): try: return cls._value_to_entry[value] except KeyError: if type(value) is cls: # pylint: disable=unidiomatic-typecheck return value raise ValueError("Unknown enum value: {}".format(value)) def __eq__(self, other): if self.__class__ is not other.__class__: return NotImplemented # Enums instances are unique, so creating an instance with the same value as another will just # send back the other one, hence we can use an identity comparison, which is much faster than '==' return self is other def __ne__(self, other): if self.__class__ is not other.__class__: return NotImplemented return self is not other def __hash__(self): return hash(id(self)) def __str__(self): return "{}.{}".format(self.__class__.__name__, self.name) def __reduce__(self): return self.__class__, (self.value,) class CoreWarnings: """CoreWarnings() Some common warnings which are raised by core functionalities within BuildStream are found in this class. """ OVERLAPS = "overlaps" """ This warning will be produced when buildstream detects an overlap on an element which is not whitelisted. See :ref:`Overlap Whitelist ` """ UNSTAGED_FILES = "unstaged-files" """ This warning will be produced when a file cannot be staged. This can happen when a file overlaps with a directory in the sandbox that is not empty. """ REF_NOT_IN_TRACK = "ref-not-in-track" """ This warning will be produced when a source is configured with a reference which is found to be invalid based on the configured track """ UNALIASED_URL = "unaliased-url" """ A URL used for fetching a sources was specified without specifying any :ref:`alias ` """ UNAVAILABLE_SOURCE_INFO = "unavailable-source-info" """ A source was queried for its provenance information but did not implement :func:`Source.collect_source_info() `. """ class OverlapAction(FastEnum): """OverlapAction() Defines what action to take when files staged into the sandbox overlap. .. note:: This only dictates what happens when functions such as :func:`Element.stage_artifact() ` and :func:`Element.stage_dependency_artifacts() ` are called multiple times in an Element's :func:`Element.stage() ` implementation, and the files staged from one function call result in overlapping files staged from previous invocations. If multiple staged elements overlap eachother within a single call to :func:`Element.stage_dependency_artifacts() `, then the :ref:`overlap whitelist ` will be ovserved, and warnings will be issued for overlapping files, which will be fatal warnings if :attr:`CoreWarnings.OVERLAPS ` is specified as a :ref:`fatal warning `. """ ERROR = "error" """ It is an error to overlap previously staged files """ WARNING = "warning" """ A warning will be issued for previously staged files, which will fatal if :attr:`CoreWarnings.OVERLAPS ` is specified as a :ref:`fatal warning ` in the project. """ IGNORE = "ignore" """ Overlapping files are acceptable, and do not cause any warning or error. """ # _Scope(): # # Defines the scope of dependencies to include for a given element # when iterating over the dependency graph in APIs like # Element._dependencies(). # class _Scope(FastEnum): # All elements which the given element depends on, following # all elements required for building. Including the element itself. # ALL = 1 # All elements required for building the element, including their # respective run dependencies. Not including the given element itself. # BUILD = 2 # All elements required for running the element. Including the element # itself. # RUN = 3 # Just the element itself, no dependencies. # NONE = 4 # _KeyStrength(): # # Strength of cache key # class _KeyStrength(FastEnum): # Includes strong cache keys of all build dependencies and their # runtime dependencies. STRONG = 1 # Includes names of direct build dependencies but does not include # cache keys of dependencies. WEAK = 2 # _DisplayKey(): # # The components of a cache key which need to be displayed # # This is a part of Message() so it needs to be a simple serializable object. # # Args: # full: A full hex digest cache key for an Element # brief: An abbreviated hex digest cache key for an Element # strict: Whether the key matches the key which would be used in strict mode # class _DisplayKey: def __init__(self, full: str, brief: str, strict: bool): self.full = full # type: str self.brief = brief # type: str self.strict = strict # type: bool # _SchedulerErrorAction() # # Actions the scheduler can take on error # class _SchedulerErrorAction(FastEnum): # Continue building the rest of the tree CONTINUE = "continue" # finish ongoing work and quit QUIT = "quit" # Abort immediately TERMINATE = "terminate" # _CacheBuildTrees() # # When to cache build trees # class _CacheBuildTrees(FastEnum): # Always store build trees ALWAYS = "always" # Store build trees when they might be useful for BuildStream # (eg: on error, to allow for a shell to debug that) AUTO = "auto" # Never cache build trees NEVER = "never" # _SourceUriPolicy() # # A policy for which URIs to access when fetching and tracking # class _SourceUriPolicy(FastEnum): # Use all URIs from default aliases and mirrors ALL = "all" # Use only the base source aliases defined in project configuration # ALIASES = "aliases" # Use only URIs from source mirrors (whether they are found # in project configuration or user configuration) MIRRORS = "mirrors" # Use only URIs from user configuration, intentionally causing # a failure if we try to access a source for which the user # configuration has not provided a mirror USER = "user" # _PipelineSelection() # # Defines the kind of pipeline selection to make when the pipeline # is provided a list of targets, for whichever purpose. # # These values correspond to the CLI `--deps` arguments for convenience. # class _PipelineSelection(FastEnum): # Select only the target elements in the associated targets NONE = "none" # As NONE, but redirect elements that are capable of it REDIRECT = "redirect" # All dependencies of all targets, including the targets ALL = "all" # All direct build dependencies and their recursive runtime dependencies, # excluding the targets BUILD = "build" # All direct runtime dependencies and their recursive runtime dependencies, # including the targets RUN = "run" def __str__(self): return str(self.value) # _ProjectInformation() # # A descriptive object about a project. # # Args: # project (Project): The project instance # provenance_node (Node): The provenance information, if any # duplicates (list): List of project descriptions which declared this project as a duplicate # internal (list): List of project descriptions which declared this project as internal # class _ProjectInformation: def __init__(self, project, provenance_node, duplicates, internal): self.project = project self.provenance = provenance_node.get_provenance() if provenance_node else None self.duplicates = duplicates self.internal = internal # _HostMount() # # A simple object describing the behavior of a host mount. # class _HostMount: def __init__(self, path: str, host_path: Optional[str] = None, optional: bool = False) -> None: # Support environment variable expansion in host mounts path = os.path.expandvars(path) if host_path is None: host_path = path else: host_path = os.path.expandvars(host_path) self.path: str = path # Path inside the sandbox self.host_path: str = host_path # Path on the host self.optional: bool = optional # Optional mounts do not incur warnings or errors # _SourceMirror() # # A simple object describing a source mirror # # Args: # name: The mirror name # aliases: A dictionary of URI lists, keyed by alias names # class _SourceMirror: def __init__(self, name: str, aliases: Dict[str, List[str]]): self.name: str = name self.aliases: Dict[str, List[str]] = aliases # new_from_node(): # # Creates a _SourceMirror() from a YAML loaded node. # # Args: # node: The configuration node describing the spec. # # Returns: # The described _SourceMirror instance. # # Raises: # LoadError: If the node is malformed. # @classmethod def new_from_node(cls, node: MappingNode) -> "_SourceMirror": node.validate_keys(["name", "aliases"]) name: str = node.get_str("name") aliases: Dict[str, List[str]] = {} alias_node: MappingNode = node.get_mapping("aliases") for alias, uris in alias_node.items(): assert type(uris) is SequenceNode # pylint: disable=unidiomatic-typecheck aliases[alias] = uris.as_str_list() return cls(name, aliases) ######################################## # Type aliases # ######################################## # Internal reference for a given Source SourceRef = Union[None, int, str, List[Any], Dict[str, Any]] """ A simple python object used to describe and exact set of sources This can be ``None`` in order to represent an absense of a source reference, otherwise it can be ``int``, ``str``, or a complex ``list`` or ``dict`` consisting of ``int``, ``str``, ``list`` and ``dict`` types. The order of elements in ``list`` objects is meaningful and should be produced deterministically by :class:`.Source` implementations, as this order will effect :ref:`cache keys `. See the :ref:`source documentation ` for more detils on how :class:`.Source` implementations are expected to handle the source ref. """ apache-buildstream-27ae392/src/buildstream/utils.py000066400000000000000000001466761514607367700224620ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ Utilities ========= """ import calendar import errno import hashlib import math import os import re import shutil import signal import stat from stat import S_ISDIR import subprocess from subprocess import TimeoutExpired import tempfile import threading import itertools from contextlib import contextmanager from pathlib import Path from typing import Callable, IO, Iterable, Iterator, Optional, Tuple, Union, Pattern from google.protobuf import timestamp_pb2 import psutil from . import _signals from ._exceptions import BstError from .exceptions import ErrorDomain from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from . import _site # Contains utils that have been rewritten in Cython for speed benefits # This makes them available when importing from utils from ._utils import url_directory_name # pylint: disable=unused-import # The magic number for timestamps: 2011-11-11 11:11:11 BST_ARBITRARY_TIMESTAMP = calendar.timegm((2011, 11, 11, 11, 11, 11)) # The separator we use for user specified aliases _ALIAS_SEPARATOR = ":" _URI_SCHEMES = ["http", "https", "ftp", "file", "git", "sftp", "ssh"] # The process's file mode creation mask. # Impossible to retrieve without temporarily changing it on POSIX. _UMASK = os.umask(0o777) os.umask(_UMASK) # Only some operating systems have os.copy_file_range and even when present # it might not work _USE_CP_FILE_RANGE = hasattr(os, "copy_file_range") # The default version guessing pattern for utils.guess_version() # _DEFAULT_GUESS_PATTERN = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?") class UtilError(BstError): """Raised by utility functions when system calls fail. This will be handled internally by the BuildStream core, if you need to handle this error, then it should be reraised, or either of the :class:`.ElementError` or :class:`.SourceError` exceptions should be raised from this error. """ def __init__(self, message, reason=None): super().__init__(message, domain=ErrorDomain.UTIL, reason=reason) class ProgramNotFoundError(BstError): """Raised if a required program is not found. It is normally unneeded to handle this exception from plugin code. """ def __init__(self, message, reason=None): super().__init__(message, domain=ErrorDomain.PROG_NOT_FOUND, reason=reason) class DirectoryExistsError(OSError): """Raised when a `os.rename` is attempted but the destination is an existing directory.""" class FileListResult: """An object which stores the result of one of the operations which run on a list of files. """ def __init__(self): self.overwritten = [] """List of files which were overwritten in the target directory""" self.ignored = [] """List of files which were ignored, because they would have replaced a non empty directory""" self.failed_attributes = [] """List of files for which attributes could not be copied over""" self.files_written = [] """List of files that were written.""" def _make_protobuf_timestamp(timestamp: timestamp_pb2.Timestamp, timepoint: float): # pylint: disable=no-member """Obtain the Protobuf Timestamp represented by the time given in seconds. Args: timestamp: the Protobuf Timestamp to set timepoint: the time since the epoch in seconds """ timestamp.seconds = int(timepoint) timestamp.nanos = int(math.modf(timepoint)[0] * 1e9) def _get_file_protobuf_mtimestamp(timestamp: timestamp_pb2.Timestamp, fullpath: str): # pylint: disable=no-member """Obtain the Protobuf Timestamp represented by the mtime of the file at the given path.""" assert isinstance(fullpath, str), "Path to file must be a string: {}".format(str(fullpath)) try: mtime = os.path.getmtime(fullpath) except OSError: raise UtilError("Failed to get mtime of file at {}".format(fullpath)) _make_protobuf_timestamp(timestamp, mtime) def _parse_protobuf_timestamp(timestamp: timestamp_pb2.Timestamp) -> float: # pylint: disable=no-member """Convert Protobuf Timestamp to seconds since epoch. Args: timestamp: the Protobuf Timestamp Returns: The time in seconds since epoch represented by the timestamp. """ return timestamp.seconds + timestamp.nanos / 1e9 def _set_file_mtime(fullpath: str, seconds: Union[int, float]) -> None: """Set the access and modification times of the file at the given path to the given time. The time of the file will be set with nanosecond resolution if supported. Args: fullpath (str): the string representing the path to the file timestamp (int, float): the time in seconds since the UNIX epoch """ assert isinstance(fullpath, str), "Path to file must be a string: {}".format(str(fullpath)) assert isinstance(seconds, (int, float)), "Mtime to set must be a float or integer: {}".format(str(seconds)) set_mtime = seconds * 10**9 try: os.utime(fullpath, times=None, ns=(int(set_mtime), int(set_mtime))) except OSError: errmsg = "Failed to set the times of the file at {} to {}".format(fullpath, str(seconds)) raise UtilError(errmsg) def list_relative_paths(directory: str) -> Iterator[str]: """A generator for walking directory relative paths This generator is useful for checking the full manifest of a directory. Symbolic links will not be followed, but will be included in the manifest. Args: directory: The directory to list files in Yields: Relative filenames in `directory` """ for (dirpath, dirnames, filenames) in os.walk(directory): # os.walk does not decend into symlink directories, which # makes sense because otherwise we might have redundant # directories, or end up descending into directories outside # of the walk() directory. # # But symlinks to directories are still identified as # subdirectories in the walked `dirpath`, so we extract # these symlinks from `dirnames` and add them to `filenames`. # for d in dirnames: fullpath = os.path.join(dirpath, d) if os.path.islink(fullpath): filenames.append(d) # Modifying the dirnames directly ensures that the os.walk() generator # allows us to specify the order in which they will be iterated. dirnames.sort() filenames.sort() relpath = os.path.relpath(dirpath, directory) # We don't want "./" pre-pended to all the entries in the root of # `directory`, prefer to have no prefix in that case. basepath = relpath if relpath != "." and dirpath != directory else "" # First yield the walked directory itself, except for the root if basepath != "": yield basepath # List the filenames in the walked directory for f in filenames: yield os.path.join(basepath, f) # pylint: disable=anomalous-backslash-in-string def glob(paths: Iterable[str], pattern: str) -> Iterator[str]: r"""A generator to yield paths which match the glob pattern Args: paths (iterable): The paths to check pattern (str): A glob pattern This generator will iterate over the passed *paths* and yield only the filenames which matched the provided *pattern*. +--------+------------------------------------------------------------------+ | Meta | Description | +========+==================================================================+ | \* | Zero or more of any character, excepting path separators | +--------+------------------------------------------------------------------+ | \** | Zero or more of any character, including path separators | +--------+------------------------------------------------------------------+ | ? | One of any character, except for path separators | +--------+------------------------------------------------------------------+ | [abc] | One of any of the specified characters | +--------+------------------------------------------------------------------+ | [a-z] | One of the characters in the specified range | +--------+------------------------------------------------------------------+ | [!abc] | Any single character, except the specified characters | +--------+------------------------------------------------------------------+ | [!a-z] | Any single character, except those in the specified range | +--------+------------------------------------------------------------------+ .. note:: Escaping of the metacharacters is not possible """ # Ensure leading slash, just because we want patterns # to match file lists regardless of whether the patterns # or file lists had a leading slash or not. if not pattern.startswith(os.sep): pattern = os.sep + pattern expression = _glob2re(pattern) regexer = re.compile(expression, re.MULTILINE | re.DOTALL) for filename in paths: filename_try = filename if not filename_try.startswith(os.sep): filename_try = os.sep + filename_try if regexer.match(filename_try): yield filename def sha256sum(filename: str) -> str: """Calculate the sha256sum of a file Args: filename: A path to a file on disk Returns: An sha256 checksum string Raises: UtilError: In the case there was an issue opening or reading `filename` """ try: h = hashlib.sha256() with open(filename, "rb") as f: for chunk in iter(lambda: f.read(65536), b""): h.update(chunk) except OSError as e: raise UtilError("Failed to get a checksum of file '{}': {}".format(filename, e)) from e return h.hexdigest() def _copy_file_range(src, dest): global _USE_CP_FILE_RANGE # pylint: disable=global-statement if not _USE_CP_FILE_RANGE: return False with open(src, "rb") as src_file, open(dest, "wb") as dest_file: num_bytes = os.fstat(src_file.fileno()).st_size while num_bytes > 0: try: bytes_read = os.copy_file_range(src_file.fileno(), dest_file.fileno(), num_bytes) if not bytes_read: return True num_bytes -= bytes_read except OSError as error: if error.errno in (errno.ENOSYS, errno.EXDEV): _USE_CP_FILE_RANGE = False return False raise error from None return True def safe_copy(src: str, dest: str, *, copystat: bool = True, result: Optional[FileListResult] = None) -> None: """Copy a file while optionally preserving attributes Args: src: The source filename dest: The destination filename copystat: Whether to preserve attributes result: An optional collective result Raises: UtilError: In the case of unexpected system call failures This is almost the same as shutil.copy2() when copystat is True, except that we unlink *dest* before overwriting it if it exists, just incase *dest* is a hardlink to a different file. """ # First unlink the target if it exists try: os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: raise UtilError("Failed to remove destination file '{}': {}".format(dest, e)) from e try: ret = _copy_file_range(src, dest) if not ret: shutil.copyfile(src, dest) except (OSError, shutil.Error) as e: raise UtilError("Failed to copy '{} -> {}': {}".format(src, dest, e)) from e if copystat: try: shutil.copystat(src, dest) except PermissionError: # If we failed to copy over some file stats, dont treat # it as an unrecoverable error, but provide some feedback # we can use for a warning. # # This has a tendency of happening when attempting to copy # over extended file attributes. if result: result.failed_attributes.append(dest) def safe_link(src: str, dest: str, *, result: Optional[FileListResult] = None, _unlink=False) -> None: """Try to create a hardlink, but resort to copying in the case of cross device links. Args: src: The source filename dest: The destination filename result: An optional collective result Raises: UtilError: In the case of unexpected system call failures """ if _unlink: # First unlink the target if it exists try: os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: raise UtilError("Failed to remove destination file '{}': {}".format(dest, e)) from e # If we can't link it due to cross-device hardlink, copy try: os.link(src, dest) except OSError as e: if e.errno == errno.EEXIST and not _unlink: # Target exists already, unlink and try again safe_link(src, dest, result=result, _unlink=True) elif e.errno in (errno.EXDEV, errno.EPERM): safe_copy(src, dest) else: raise UtilError("Failed to link '{} -> {}': {}".format(src, dest, e)) from e def safe_remove(path: str) -> bool: """Removes a file or directory This will remove a file if it exists, and will remove a directory if the directory is empty. Args: path: The path to remove Returns: True if `path` was removed or did not exist, False if `path` was a non empty directory. Raises: UtilError: In the case of unexpected system call failures """ try: if S_ISDIR(os.lstat(path).st_mode): os.rmdir(path) else: os.unlink(path) # File removed/unlinked successfully return True except OSError as e: if e.errno == errno.ENOTEMPTY: # Path is non-empty directory return False elif e.errno == errno.ENOENT: # Path does not exist return True raise UtilError("Failed to remove '{}': {}".format(path, e)) def copy_files( src: str, dest: str, *, filter_callback: Optional[Callable[[str], bool]] = None, ignore_missing: bool = False, report_written: bool = False ) -> FileListResult: """Copy files from source to destination. Args: src: The source directory dest: The destination directory filter_callback: Optional filter callback. Called with the relative path as argument for every file in the source directory. The file is copied only if the callable returns True. If no filter callback is specified, all files will be copied. ignore_missing: Dont raise any error if a source file is missing report_written: Add to the result object the full list of files written Returns: The result describing what happened during this file operation Raises: UtilError: In the case of unexpected system call failures .. note:: Directories in `dest` are replaced with files from `src`, unless the existing directory in `dest` is not empty in which case the path will be reported in the return value. UNIX domain socket files from `src` are ignored. """ result = FileListResult() try: _process_list( src, dest, safe_copy, result, filter_callback=filter_callback, ignore_missing=ignore_missing, report_written=report_written, ) except OSError as e: raise UtilError("Failed to copy '{} -> {}': {}".format(src, dest, e)) return result def link_files( src: str, dest: str, *, filter_callback: Optional[Callable[[str], bool]] = None, ignore_missing: bool = False, report_written: bool = False ) -> FileListResult: """Hardlink files from source to destination. Args: src: The source directory dest: The destination directory filter_callback: Optional filter callback. Called with the relative path as argument for every file in the source directory. The file is hardlinked only if the callable returns True. If no filter callback is specified, all files will be hardlinked. ignore_missing: Dont raise any error if a source file is missing report_written: Add to the result object the full list of files written Returns: The result describing what happened during this file operation Raises: UtilError: In the case of unexpected system call failures .. note:: Directories in `dest` are replaced with files from `src`, unless the existing directory in `dest` is not empty in which case the path will be reported in the return value. .. note:: If a hardlink cannot be created due to crossing filesystems, then the file will be copied instead. UNIX domain socket files from `src` are ignored. """ result = FileListResult() try: _process_list( src, dest, safe_link, result, filter_callback=filter_callback, ignore_missing=ignore_missing, report_written=report_written, ) except OSError as e: raise UtilError("Failed to link '{} -> {}': {}".format(src, dest, e)) return result def get_host_tool( name: str, ) -> str: """Get the full path of a host tool Args: name (str): The name of the program to search for Returns: The full path to the program, if found Raises: :class:`.ProgramNotFoundError` """ return _get_host_tool_internal(name) def get_bst_version() -> Tuple[int, int]: """Gets the major, minor release portion of the BuildStream version. Returns: A 2-tuple of form (major version, minor version) """ # Import this only conditionally, it's not resolved at bash complete time from . import __version__ # pylint: disable=cyclic-import versions = __version__.split(".")[:2] if versions[0] == "0+untagged": raise UtilError( "Your git repository has no tags - BuildStream can't " "determine its version. Please run `git fetch --tags`." ) try: return _parse_version(__version__) except UtilError as e: raise UtilError("Failed to detect BuildStream version: {}\n".format(e)) from e def move_atomic(source: Union[Path, str], destination: Union[Path, str], *, ensure_parents: bool = True) -> None: """Move the source to the destination using atomic primitives. This uses `os.rename` to move a file or directory to a new destination. It wraps some `OSError` thrown errors to ensure their handling is correct. The main reason for this to exist is that rename can throw different errors for the same symptom (https://www.unix.com/man-page/POSIX/3posix/rename/) when we are moving a directory. We are especially interested here in the case when the destination already exists, is a directory and is not empty. In this case, either EEXIST or ENOTEMPTY can be thrown. In order to ensure consistent handling of these exceptions, this function should be used instead of `os.rename` Args: source: source to rename destination: destination to which to move the source ensure_parents: Whether or not to create the parent's directories of the destination (default: True) Raises: DirectoryExistsError: if the destination directory already exists and is not empty OSError: if another filesystem level error occured """ if ensure_parents: os.makedirs(os.path.dirname(str(destination)), exist_ok=True) try: os.rename(str(source), str(destination)) except OSError as exc: if exc.errno in (errno.EEXIST, errno.ENOTEMPTY): raise DirectoryExistsError(*exc.args) from exc raise @contextmanager def save_file_atomic( filename: str, mode: str = "w", *, buffering: int = -1, encoding: Optional[str] = None, errors: Optional[str] = None, newline: Optional[str] = None, closefd: bool = True, opener: Optional[Callable[[str, int], int]] = None, tempdir: Optional[str] = None ) -> Iterator[IO]: """Save a file with a temporary name and rename it into place when ready. This is a context manager which is meant for saving data to files. The data is written to a temporary file, which gets renamed to the target name when the context is closed. This avoids readers of the file from getting an incomplete file. **Example:** .. code:: python with save_file_atomic('/path/to/foo', 'w') as f: f.write(stuff) The file will be called something like ``tmpCAFEBEEF`` until the context block ends, at which point it gets renamed to ``foo``. The temporary file will be created in the same directory as the output file. The ``filename`` parameter must be an absolute path. If an exception occurs or the process is terminated, the temporary file will be deleted. """ # This feature has been proposed for upstream Python in the past, e.g.: # https://bugs.python.org/issue8604 assert os.path.isabs(filename), "The utils.save_file_atomic() parameter ``filename`` must be an absolute path" if tempdir is None: tempdir = os.path.dirname(filename) fd, tempname = tempfile.mkstemp(dir=tempdir) # Apply mode allowed by umask os.fchmod(fd, 0o666 & ~_UMASK) os.close(fd) f = open( tempname, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd, opener=opener, ) def cleanup_tempfile(): f.close() try: os.remove(tempname) except FileNotFoundError: pass except OSError as e: raise UtilError("Failed to cleanup temporary file {}: {}".format(tempname, e)) from e try: with _signals.terminator(cleanup_tempfile): # Disable type-checking since "IO[Any]" has no attribute "real_filename" f.real_filename = filename # type: ignore yield f f.close() # This operation is atomic, at least on platforms we care about: # https://bugs.python.org/issue8828 os.replace(tempname, filename) except Exception: cleanup_tempfile() raise # get_umask(): # # # # Returns: # (int) # def get_umask() -> int: """ Get the process's file mode creation mask without changing it. Returns: The process's file mode creation mask. """ return _UMASK def guess_version(string: str, *, pattern: Optional[Pattern[str]] = None) -> Optional[str]: """ Attempt to extract a version from an arbitrary string. This function is used by sources who implement :func:`Source.get_source_info() ` in order to provide a guess at what the version is, given some domain specific knowledge such as a git tag or a tarball URL. This function will be traverse the provided string for non-overlapping matches, and in the case of *optional groups* being specified in the pattern; the match with the greatest amount of matched groups will be preferred, allowing for correct handling of cases like: ``https://example.com/releases/1.2/release-1.2.3.tgz`` which may match the *pattern* multiple times. The resulting version will be the captured groups, separated by ``.`` characters. Args: string: The domain specific string to scan for a version pattern: A compiled regex pattern to scan *string*, or None for the default ``(\\d+)\\.(\\d+)(?:\\.(\\d+))?``. Returns: The guessed version, or None if no match was found. .. note:: **Specifying a pattern** When specifying the pattern, any number of capture groups may be specified, and the match containing the most matching groups will be selected. The capture groups must contain only the intended result and not any separating characters. For example, you may parse a string such as ``release-1_2_3-r2`` with the pattern: ``(\\d+)_(\\d+)(?:_(\\d+))?(?:\\-(r\\d+))?``, and this would produce the parsed version ``1.2.3.r2``. **Since: 2.5**. """ version_guess: Optional[str] = None version_guess_groups = 0 if pattern is None: pattern = _DEFAULT_GUESS_PATTERN # Iterate over non-overlapping matches, and prefer a match which is more qualified (i.e. 1.2.3 is better than 1.2) for version_match in pattern.finditer(string): if not version_match: iter_guess = None iter_n_groups = 0 elif pattern.groups == 0: iter_guess = str(version_match.group(0)) iter_n_groups = 1 else: iter_groups = [group for group in version_match.groups() if group is not None] iter_n_groups = len(iter_groups) iter_guess = ".".join(iter_groups) if version_guess is None or iter_n_groups > version_guess_groups: version_guess = iter_guess version_guess_groups = iter_n_groups return version_guess # _get_host_tool_internal(): # # Get the full path of a host tool, including tools bundled inside the Python package. # # Args: # name (str): The name of the program to search for # search_subprojects_dir (str): Optionally search in bundled subprojects directory # # Returns: # The full path to the program, if found # # Raises: # :class:`.ProgramNotFoundError` def _get_host_tool_internal( name: str, search_subprojects_dir: Optional[str] = None, ) -> str: search_path = os.environ.get("PATH", "").split(os.pathsep) if search_subprojects_dir: search_path.append(os.path.join(_site.subprojects, search_subprojects_dir)) program_path = shutil.which(name, path=os.pathsep.join(search_path)) if not program_path: raise ProgramNotFoundError("Did not find '{}' in PATH: {}".format(name, search_path)) return program_path # _get_dir_size(): # # Get the disk usage of a given directory in bytes. # # This function assumes that files do not inadvertantly # disappear while this function is running. # # Arguments: # (str) The path whose size to check. # # Returns: # (int) The size on disk in bytes. # def _get_dir_size(path): path = os.path.abspath(path) def get_size(path): total = 0 for f in os.scandir(path): total += f.stat(follow_symlinks=False).st_size if f.is_dir(follow_symlinks=False): total += get_size(f.path) return total return get_size(path) # _get_volume_size(): # # Gets the overall usage and total size of a mounted filesystem in bytes. # # Args: # path (str): The path to check # # Returns: # (int): The total number of bytes on the volume # (int): The number of available bytes on the volume # def _get_volume_size(path): try: usage = shutil.disk_usage(path) except OSError as e: raise UtilError("Failed to retrieve stats on volume for path '{}': {}".format(path, e)) from e return usage.total, usage.free # _parse_size(): # # Convert a string representing data size to a number of # bytes. E.g. "2K" -> 2048. # # This uses the same format as systemd's # [resource-control](https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html#). # # Arguments: # size (str) The string to parse # volume (str) A path on the volume to consider for percentage # specifications # # Returns: # (int|None) The number of bytes, or None if 'infinity' was specified. # # Raises: # UtilError if the string is not a valid data size. # def _parse_size(size, volume): if size == "infinity": return None matches = re.fullmatch(r"([0-9]+\.?[0-9]*)([KMGT%]?)", size) if matches is None: raise UtilError("{} is not a valid data size.".format(size)) num, unit = matches.groups() if unit == "%": num = float(num) if num > 100: raise UtilError("{}% is not a valid percentage value.".format(num)) disk_size, _ = _get_volume_size(volume) return disk_size * (num / 100) units = ("", "K", "M", "G", "T") return int(num) * 1024 ** units.index(unit) # _parse_percentage(): # # Convert a string representing a percentage between 0% and 100% to a float. # E.g. "80%" -> 0.8. # # Arguments: # percentage (str) The string to parse # # Returns: # (float) The percentage as a float # # Raises: # UtilError if the string is not a valid percentage. # def _parse_percentage(percentage): if not percentage.endswith("%"): raise UtilError("{} is not a valid percentage.".format(percentage)) try: num = float(percentage[:-1]) except ValueError: raise UtilError("{} is not a valid percentage.".format(percentage)) if num < 0 or num > 100: raise UtilError("{} is not between 0% and 100%.".format(percentage)) return num / 100 # _pretty_size() # # Converts a number of bytes into a string representation in KiB, MiB, GiB, TiB # represented as K, M, G, T etc. # # Args: # size (int): The size to convert in bytes. # dec_places (int): The number of decimal places to output to. # # Returns: # (str): The string representation of the number of bytes in the largest def _pretty_size(size, dec_places=0): psize = size unit = "B" units = ("B", "K", "M", "G", "T") for unit in units: if psize < 1024: break if unit != units[-1]: psize /= 1024 return "{size:g}{unit}".format(size=round(psize, dec_places), unit=unit) # _is_in_main_thread() # # Return whether we are running in the main thread or not # def _is_in_main_thread(): return threading.current_thread() is threading.main_thread() # Remove a path and any empty directories leading up to it. # # Args: # basedir - The basedir at which to stop pruning even if # it is empty. # path - A path relative to basedir that should be pruned. # # Raises: # FileNotFoundError - if the path itself doesn't exist. # OSError - if something else goes wrong # def _remove_path_with_parents(basedir: Union[Path, str], path: Union[Path, str]): assert not os.path.isabs(path), "The path ({}) should be relative to basedir ({})".format(path, basedir) path = os.path.join(basedir, path) # Start by removing the path itself os.unlink(path) # Now walk up the directory tree and delete any empty directories path = os.path.dirname(path) while path != basedir: try: os.rmdir(path) except FileNotFoundError: # The parent directory did not exist (race conditions can # cause this), but it's parent directory might still be # ready to prune pass except OSError as e: if e.errno == errno.ENOTEMPTY: # The parent directory was not empty, so we # cannot prune directories beyond this point break raise path = os.path.dirname(path) # Recursively remove directories, ignoring file permissions as much as # possible. def _force_rmtree(rootpath): def fix_permissions(function, path, info): parent = os.path.dirname(path) try: os.chmod(parent, 0o755) except OSError as e: raise UtilError("Failed to ensure write permission on directory '{}': {}".format(parent, e)) # Directories need to be removed with `rmdir`, though # `os.path.isdir` will follow symlinks, so make sure it's # not a symlink first if not os.path.islink(path) and os.path.isdir(path): os.rmdir(path) else: os.remove(path) try: shutil.rmtree(rootpath, onerror=fix_permissions) # pylint: disable=deprecated-argument except OSError as e: raise UtilError("Failed to remove cache directory '{}': {}".format(rootpath, e)) # Recursively make directories in target area def _copy_directories(srcdir, destdir, target): this_dir = os.path.dirname(target) new_dir = os.path.join(destdir, this_dir) old_dir = os.path.join(srcdir, this_dir) if not os.path.lexists(new_dir): if this_dir: yield from _copy_directories(srcdir, destdir, this_dir) if os.path.lexists(old_dir): dir_stat = os.lstat(old_dir) mode = dir_stat.st_mode if stat.S_ISDIR(mode) or stat.S_ISLNK(mode): os.makedirs(new_dir) yield (new_dir, mode) else: raise UtilError("Source directory tree has file where " "directory expected: {}".format(old_dir)) else: if not os.access(new_dir, os.W_OK): # If the destination directory is not writable, change permissions to make it # writable. Callers of this method (like `_process_list`) must # restore the original permissions towards the end of their processing. try: os.chmod(new_dir, 0o755) yield (new_dir, os.lstat(old_dir).st_mode) except PermissionError: raise UtilError("Directory {} is not writable".format(destdir)) # _ensure_real_directory() # # Ensure `path` is a real directory and there are no symlink components. # # Symlink components are allowed in `root`. # def _ensure_real_directory(root, path): destpath = root for name in os.path.split(path): destpath = os.path.join(destpath, name) try: deststat = os.lstat(destpath) if not stat.S_ISDIR(deststat.st_mode): relpath = destpath[len(root) :] if stat.S_ISLNK(deststat.st_mode): filetype = "symlink" elif stat.S_ISREG(deststat.st_mode): filetype = "regular file" else: filetype = "special file" raise UtilError("Destination is a {}, not a directory: {}".format(filetype, relpath)) except FileNotFoundError: os.makedirs(destpath) # _process_list() # # Internal helper for copying/moving/linking file lists # # This will handle directories, symlinks and special files # internally, the `actionfunc` will only be called for regular files. # # Args: # srcdir: The source base directory # destdir: The destination base directory # actionfunc: The function to call for regular files # result: The FileListResult # filter_callback: Optional callback to invoke for every directory entry # ignore_missing: Dont raise any error if a source file is missing # # def _process_list( srcdir, destdir, actionfunc, result, filter_callback=None, ignore_missing=False, report_written=False ): # Keep track of directory permissions, since these need to be set # *after* files have been written. permissions = [] filelist = list_relative_paths(srcdir) if filter_callback: filelist = [path for path in filelist if filter_callback(path)] # Now walk the list for path in filelist: srcpath = os.path.join(srcdir, path) destpath = os.path.join(destdir, path) # Ensure that the parent of the destination path exists without symlink # components. _ensure_real_directory(destdir, os.path.dirname(path)) # Add to the results the list of files written if report_written: result.files_written.append(path) # Collect overlaps if os.path.lexists(destpath) and not os.path.isdir(destpath): result.overwritten.append(path) # The destination directory may not have been created separately permissions.extend(_copy_directories(srcdir, destdir, path)) try: file_stat = os.lstat(srcpath) mode = file_stat.st_mode except FileNotFoundError as e: # Skip this missing file if ignore_missing: continue raise UtilError("Source file is missing: {}".format(srcpath)) from e if stat.S_ISDIR(mode): # Ensure directory exists in destination _ensure_real_directory(destdir, path) permissions.append((destpath, os.stat(srcpath).st_mode)) elif stat.S_ISLNK(mode): if not safe_remove(destpath): result.ignored.append(path) continue target = os.readlink(srcpath) os.symlink(target, destpath) elif stat.S_ISREG(mode): # Process the file. if not safe_remove(destpath): result.ignored.append(path) continue actionfunc(srcpath, destpath, result=result) elif stat.S_ISFIFO(mode): os.mkfifo(destpath, mode) elif stat.S_ISSOCK(mode): # We can't duplicate the process serving the socket anyway pass else: # Unsupported type. raise UtilError("Cannot extract {} into staging-area. Unsupported type.".format(srcpath)) # Write directory permissions now that all files have been written for d, perms in permissions: os.chmod(d, perms) # _set_deterministic_user() # # Set the uid/gid for every file in a directory tree to the process' # euid/guid. # # Args: # directory (str): The directory to recursively set the uid/gid on # def _set_deterministic_user(directory): user = os.geteuid() group = os.getegid() for root, dirs, files in os.walk(directory.encode("utf-8"), topdown=False): for filename in files: os.chown(os.path.join(root, filename), user, group, follow_symlinks=False) for dirname in dirs: os.chown(os.path.join(root, dirname), user, group, follow_symlinks=False) # _tempdir() # # A context manager for doing work in a temporary directory. # # NOTE: Unlike mkdtemp(), this method may not restrict access to other # users. The process umask is the only access restriction, similar # to mkdir(). # This is potentially insecure. Do not create directories in /tmp # with this method. *Only* use this in directories whose parents are # more tightly controlled (i.e., non-public directories). # # Args: # dir (str): A path to a parent directory for the temporary directory # suffix (str): A suffix for the temproary directory name # prefix (str): A prefix for the temporary directory name # # Yields: # (str): The temporary directory # # In addition to the functionality provided by python's # tempfile.TemporaryDirectory() context manager, this one additionally # supports cleaning up the temp directory on SIGTERM. # @contextmanager def _tempdir(*, suffix="", prefix="tmp", dir): # pylint: disable=redefined-builtin # Do not allow fallback to a global temp directory. Due to the chmod # below, this method is not safe to be used in global temp # directories such as /tmp. assert ( dir ), "Creating directories in the public fallback `/tmp` is dangerous. Please use a directory with tight access controls." tempdir = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) def cleanup_tempdir(): if os.path.isdir(tempdir): _force_rmtree(tempdir) try: with _signals.terminator(cleanup_tempdir): # Apply mode allowed by umask os.chmod(tempdir, 0o777 & ~_UMASK) yield tempdir finally: cleanup_tempdir() # _tempnamedfile() # # A context manager for doing work on an open temporary file # which is guaranteed to be named and have an entry in the filesystem. # # Args: # mode (str): The mode in which the file is opened # encoding (str): The name of the encoding used to decode or encode the file # dir (str): A path to a parent directory for the temporary file # suffix (str): A suffix for the temproary file name # prefix (str): A prefix for the temporary file name # # Yields: # (tempfile.NamedTemporaryFile): The temporary file handle # # Do not use tempfile.NamedTemporaryFile() directly, as this will # leak files on the filesystem when BuildStream exits a process # on SIGTERM. # @contextmanager def _tempnamedfile(mode="w+b", encoding=None, suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-builtin temp = None def close_tempfile(): if temp is not None: temp.close() with _signals.terminator(close_tempfile), tempfile.NamedTemporaryFile( mode=mode, encoding=encoding, suffix=suffix, prefix=prefix, dir=dir ) as temp: yield temp # _kill_process_tree() # # Brutally murder a process and all of its children # # Args: # pid (int): Process ID # def _kill_process_tree(pid): proc = psutil.Process(pid) children = proc.children(recursive=True) def kill_proc(p): try: p.kill() except psutil.AccessDenied: # Ignore this error, it can happen with # some setuid bwrap processes. pass except psutil.NoSuchProcess: # It is certain that this has already been sent # SIGTERM, so there is a window where the process # could have exited already. pass # Bloody Murder for child in children: kill_proc(child) kill_proc(proc) # _call() # # A wrapper for subprocess.call() supporting suspend and resume # # Args: # popenargs (list): Popen() arguments # terminate (bool): Whether to attempt graceful termination before killing # rest_of_args (kwargs): Remaining arguments to subprocess.call() # # Returns: # (int): The process exit code. # (str): The program output. # def _call(*popenargs, terminate=False, **kwargs): kwargs["start_new_session"] = True process = None kwargs.setdefault("umask", stat.S_IWGRP | stat.S_IWOTH) # Handle termination, suspend and resume def kill_proc(): if process: # Some callers know that their subprocess can be # gracefully terminated, make an attempt first if terminate: proc = psutil.Process(process.pid) proc.terminate() try: proc.wait(20) except psutil.TimeoutExpired: # Did not terminate within the timeout: murder _kill_process_tree(process.pid) else: # FIXME: This is a brutal but reliable approach # # Other variations I've tried which try SIGTERM first # and then wait for child processes to exit gracefully # have not reliably cleaned up process trees and have # left orphaned git or ssh processes alive. # # This cleans up the subprocesses reliably but may # cause side effects such as possibly leaving stale # locks behind. Hopefully this should not be an issue # as long as any child processes only interact with # the temp directories which we control and cleanup # ourselves. # _kill_process_tree(process.pid) def suspend_proc(): if process: group_id = os.getpgid(process.pid) os.killpg(group_id, signal.SIGSTOP) def resume_proc(): if process: group_id = os.getpgid(process.pid) os.killpg(group_id, signal.SIGCONT) with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc), subprocess.Popen( *popenargs, universal_newlines=True, **kwargs ) as process: # Here, we don't use `process.communicate()` directly without a timeout # This is because, if we were to do that, and the process would never # output anything, the control would never be given back to the python # process, which might thus not be able to check for request to # shutdown, or kill the process. # We therefore loop with a timeout, to ensure the python process # can act if it needs. while True: try: output, _ = process.communicate(timeout=1) break except TimeoutExpired: continue exit_code = process.poll() return (exit_code, output) # _glob2re() # # Function to translate a glob style pattern into a regex # # Args: # pat (str): The glob pattern # # This is a modified version of the python standard library's # fnmatch.translate() function which supports path like globbing # a bit more correctly, and additionally supports recursive glob # patterns with double asterisk. # # Note that this will only support the most basic of standard # glob patterns, and additionally the recursive double asterisk. # # Support includes: # # * Match any pattern except a path separator # ** Match any pattern, including path separators # ? Match any single character # [abc] Match one of the specified characters # [A-Z] Match one of the characters in the specified range # [!abc] Match any single character, except the specified characters # [!A-Z] Match any single character, except those in the specified range # def _glob2re(pat): i, n = 0, len(pat) res = "" while i < n: c = pat[i] i = i + 1 if c == "*": # fnmatch.translate() simply uses the '.*' separator here, # we only want that for double asterisk (bash 'globstar' behavior) # if i < n and pat[i] == "*": res = res + ".*" i = i + 1 else: res = res + "[^/]*" elif c == "?": # fnmatch.translate() simply uses the '.' wildcard here, but # we dont want to match path separators here res = res + "[^/]" elif c == "[": j = i if j < n and pat[j] == "!": j = j + 1 if j < n and pat[j] == "]": j = j + 1 while j < n and pat[j] != "]": j = j + 1 if j >= n: res = res + "\\[" else: stuff = pat[i:j].replace("\\", "\\\\") i = j + 1 if stuff[0] == "!": stuff = "^" + stuff[1:] elif stuff[0] == "^": stuff = "\\" + stuff res = "{}[{}]".format(res, stuff) else: res = res + re.escape(c) return res + r"\Z" # _deduplicate() # # Remove duplicate entries in a list or other iterable. # # Copied verbatim from the unique_everseen() example at # https://docs.python.org/3/library/itertools.html#itertools-recipes # # Args: # iterable (iterable): What to deduplicate # key (callable): Optional function to map from list entry to value # # Returns: # (generator): Generator that produces a deduplicated version of 'iterable' # def _deduplicate(iterable, key=None): seen = set() seen_add = seen.add if key is None: for element in itertools.filterfalse(seen.__contains__, iterable): seen_add(element) yield element else: for element in iterable: k = key(element) if k not in seen: seen_add(k) yield element # _message_digest() # # Args: # message_buffer (str): String to create digest of # # Returns: # (remote_execution_pb2.Digest): Content digest # def _message_digest(message_buffer): sha = hashlib.sha256(message_buffer) digest = remote_execution_pb2.Digest() digest.hash = sha.hexdigest() digest.size_bytes = len(message_buffer) return digest # _search_upward_for_files() # # Searches upwards (from directory, then directory's parent directory...) # for any of the files listed in `filenames`. # # If multiple filenames are specified, and present in the same directory, # the first filename in the list will be returned. # # Args: # directory (str): The directory to begin searching for files from # filenames (list of str): The names of files to search for # # Returns: # (str): The directory a file was found in, or None # (str): The name of the first file that was found in that directory, or None # def _search_upward_for_files(directory, filenames): directory = os.path.abspath(directory) while True: for filename in filenames: file_path = os.path.join(directory, filename) if os.path.isfile(file_path): return directory, filename parent_dir = os.path.dirname(directory) if directory == parent_dir: # i.e. we've reached the root of the filesystem return None, None directory = parent_dir # _get_compression: # # Given a file name infer the compression # # Args: # tar (str): The file name from which to determine compression # # Returns: # (str): One from '', 'gz', 'xz', 'bz2' # # Raises: # UtilError: In the case where an unsupported file extension has been provided, # expecting compression. # # def _get_compression(tar): mapped_extensions = {".tar": "", ".gz": "gz", ".xz": "xz", ".bz2": "bz2"} name, ext = os.path.splitext(tar) try: return mapped_extensions[ext] except KeyError: # If ext not in mapped_extensions, find out if inner ext is .tar # If so, we assume we have been given an unsupported extension, # which expects compression. Raise an error _, suffix = os.path.splitext(name) if suffix == ".tar": raise UtilError( "Expected compression with unknown file extension ('{}'), " "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')".format(ext) ) # Assume just an unconventional name was provided, default to uncompressed return "" # _parse_version(): # # Args: # version (str): The file name from which to determine compression # # Returns: # A 2-tuple of form (major version, minor version) # # Raises: # UtilError: In the case of a malformed version string # def _parse_version(version: str) -> Tuple[int, int]: try: versions = version.split(".") major = int(versions[0]) minor = int(versions[1]) except (IndexError, ValueError, AttributeError) as e: raise UtilError( "Malformed version string: {}".format(version), ) from e return major, minor # _get_bst_api_version(): # # Fetch the current BuildStream API version, this # ensures that we get "2.0" for example when we are # in a development stage leading up to 2.0. # # Returns: # A 2-tuple of form (major version, minor version) # def _get_bst_api_version() -> Tuple[int, int]: bst_major, bst_minor = get_bst_version() if bst_major < 2: bst_major = 2 bst_minor = 0 return (bst_major, bst_minor) apache-buildstream-27ae392/tests/000077500000000000000000000000001514607367700167645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/__init__.py000066400000000000000000000000001514607367700210630ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/000077500000000000000000000000001514607367700215455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/__init__.py000066400000000000000000000000001514607367700236440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/capabilities.py000066400000000000000000000045211514607367700245520ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._project import Project from buildstream import _yaml from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils import dummy_context from tests.testutils.artifactshare import create_dummy_artifact_share DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.datafiles(DATA_DIR) def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. with create_dummy_artifact_share() as share: # Configure artifact share cache_dir = os.path.join(str(tmpdir), "cache") user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) with dummy_context(config=user_config_file) as context: # Load the project project = Project(project_dir, context) project.ensure_fully_loaded() # Create a local artifact cache handle artifactcache = context.artifactcache # Initialize remotes context.initialize_remotes(True, True, None, None) assert ( not artifactcache.has_fetch_remotes() ), "System didn't realize the artifact cache didn't support BuildStream" apache-buildstream-27ae392/tests/artifactcache/config.py000066400000000000000000000250311514607367700233650ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._remotespec import RemoteSpec, RemoteType from buildstream._project import Project from buildstream.utils import _deduplicate from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import runcli from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils import dummy_context DATA_DIR = os.path.dirname(os.path.realpath(__file__)) cache1 = RemoteSpec(RemoteType.ALL, url="https://example.com/cache1", push=True) cache2 = RemoteSpec(RemoteType.ALL, url="https://example.com/cache2", push=False) cache3 = RemoteSpec(RemoteType.ALL, url="https://example.com/cache3", push=False) cache4 = RemoteSpec(RemoteType.ALL, url="https://example.com/cache4", push=False) cache5 = RemoteSpec(RemoteType.ALL, url="https://example.com/cache5", push=False) cache6 = RemoteSpec(RemoteType.ALL, url="https://example.com/cache6", push=True) cache7 = RemoteSpec(RemoteType.INDEX, url="https://index.example.com/cache1", push=True) cache8 = RemoteSpec(RemoteType.STORAGE, url="https://storage.example.com/cache1", push=True) # Generate cache configuration fragments for the user config and project config files. # def configure_remote_caches(override_caches, project_caches=None, user_caches=None): type_strings = {RemoteType.INDEX: "index", RemoteType.STORAGE: "storage", RemoteType.ALL: "all"} if project_caches is None: project_caches = [] if user_caches is None: user_caches = [] user_config = {} if user_caches: user_config["artifacts"] = { "servers": [ {"url": cache.url, "push": cache.push, "type": type_strings[cache.remote_type]} for cache in user_caches ] } if override_caches: user_config["projects"] = { "test": { "artifacts": { "servers": [ {"url": cache.url, "push": cache.push, "type": type_strings[cache.remote_type]} for cache in override_caches ] } } } project_config = {} if project_caches: project_config.update( { "artifacts": [ {"url": cache.url, "push": cache.push, "type": type_strings[cache.remote_type]} for cache in project_caches ] } ) return user_config, project_config # Test that parsing the remote artifact cache locations produces the # expected results. @pytest.mark.parametrize( "override_caches, project_caches, user_caches", [ # The leftmost cache is the highest priority one in all cases here. pytest.param([], [], [], id="empty-config"), pytest.param([], [], [cache1, cache2], id="user-config"), pytest.param([], [cache1, cache2], [cache3], id="project-config"), pytest.param([cache1], [cache2], [cache3], id="project-override-in-user-config"), pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id="list-order"), pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id="duplicates"), pytest.param([cache7, cache8], [], [cache1], id="split-caches"), ], ) def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user_caches): # Produce a fake user and project config with the cache configuration. user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) project_config["name"] = "test" project_config["min-version"] = "2.0" project_dir = tmpdir.mkdir("project") project_config_file = str(project_dir.join("project.conf")) _yaml.roundtrip_dump(project_config, file=project_config_file) with runcli.configured(str(tmpdir), user_config) as user_config_file, dummy_context( config=user_config_file ) as context: project = Project(str(project_dir), context) project.ensure_fully_loaded() # Check the specs which the artifact cache thinks are configured context.initialize_remotes(True, True, None, None) artifactcache = context.artifactcache parsed_cache_specs = artifactcache._project_specs[project.name] # Verify that it was correctly read. expected_cache_specs = list(_deduplicate(override_caches or user_caches)) expected_cache_specs = list(_deduplicate(expected_cache_specs + project_caches)) assert parsed_cache_specs == expected_cache_specs # Assert that if either the client key or client cert is specified # without specifying its counterpart, we get a comprehensive LoadError # instead of an unhandled exception. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) def test_missing_certs(cli, datafiles, config_key, config_value): project = os.path.join(datafiles, "missing-certs") project_conf = { "name": "test", "min-version": "2.0", "artifacts": {"url": "https://cache.example.com:12345", "push": "true", "auth": {config_key: config_value}}, } project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Assert that BuildStream complains when someone attempts to define # only one type of storage. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "override_caches, project_caches, user_caches", [ # The leftmost cache is the highest priority one in all cases here. pytest.param([], [], [cache7], id="index-user"), pytest.param([], [], [cache8], id="storage-user"), pytest.param([], [cache7], [], id="index-project"), pytest.param([], [cache8], [], id="storage-project"), pytest.param([cache7], [], [], id="index-override"), pytest.param([cache8], [], [], id="storage-override"), ], ) def test_only_one(cli, datafiles, override_caches, project_caches, user_caches): project = os.path.join(datafiles, "only-one") # Produce a fake user and project config with the cache configuration. user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) project_config["name"] = "test" project_config["min-version"] = "2.0" cli.configure(user_config) project_config_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_config, file=project_config_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "artifacts_config", ( [ { "url": "http://localhost.test", "auth": { "server-cert": "~/server.crt", "client-cert": "~/client.crt", "client-key": "~/client.key", }, } ], [ { "url": "http://localhost.test", "auth": { "server-cert": "~/server.crt", "client-cert": "~/client.crt", "client-key": "~/client.key", }, }, { "url": "http://localhost2.test", "auth": { "server-cert": "~/server2.crt", "client-cert": "~/client2.crt", "client-key": "~/client2.key", }, }, ], ), ) @pytest.mark.parametrize("in_user_config", [True, False]) def test_paths_for_artifact_config_are_expanded(tmpdir, monkeypatch, artifacts_config, in_user_config): # Produce a fake user and project config with the cache configuration. # user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) # project_config['name'] = 'test' monkeypatch.setenv("HOME", str(tmpdir.join("homedir"))) project_config = {"name": "test", "min-version": "2.0"} user_config = {} if in_user_config: user_config["artifacts"] = {"servers": artifacts_config} else: project_config["artifacts"] = artifacts_config user_config_file = str(tmpdir.join("buildstream.conf")) _yaml.roundtrip_dump(user_config, file=user_config_file) project_dir = tmpdir.mkdir("project") project_config_file = str(project_dir.join("project.conf")) _yaml.roundtrip_dump(project_config, file=project_config_file) with dummy_context(config=user_config_file) as context: project = Project(str(project_dir), context) project.ensure_fully_loaded() # Check the specs which the artifact cache thinks are configured context.initialize_remotes(True, True, None, None) artifactcache = context.artifactcache parsed_cache_specs = artifactcache._project_specs[project.name] if isinstance(artifacts_config, dict): artifacts_config = [artifacts_config] # Build expected artifact config artifacts_config = [ RemoteSpec( RemoteType.ALL, config["url"], push=False, server_cert=os.path.expanduser(config["auth"]["server-cert"]), client_cert=os.path.expanduser(config["auth"]["client-cert"]), client_key=os.path.expanduser(config["auth"]["client-key"]), ) for config in artifacts_config ] assert parsed_cache_specs == artifacts_config apache-buildstream-27ae392/tests/artifactcache/expiry.py000066400000000000000000000340321514607367700234410ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: Tristan Maat # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import time import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing._utils.site import have_subsecond_mtime from tests.testutils import casd_cache, create_element_size, wait_for_cache_granularity DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "expiry") def get_cache_usage(directory): with casd_cache(directory) as cas_cache: wait = 0.1 for _ in range(0, int(5 / wait)): used_size = cas_cache.get_cache_usage().used_size if used_size is not None: return used_size time.sleep(wait) assert False, "Unable to retrieve cache usage" return None # Ensure that the cache successfully removes an old artifact if we do # not have enough space left. @pytest.mark.datafiles(DATA_DIR) def test_artifact_expires(cli, datafiles): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) cli.configure( { "cache": { "quota": 10000000, } } ) # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) create_element_size("target.bst", project, element_path, [], 6000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() assert cli.get_element_state(project, "target.bst") == "cached" # Our cache should now be almost full. Let's create another # artifact and see if we can cause buildstream to delete the old # one. create_element_size("target2.bst", project, element_path, [], 6000000) res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() # Check that the correct element remains in the cache states = cli.get_element_states(project, ["target.bst", "target2.bst"]) assert states["target.bst"] != "cached" assert states["target2.bst"] == "cached" # Ensure that we don't end up deleting the whole cache (or worse) if # we try to store an artifact that is too large to fit in the quota. @pytest.mark.parametrize( "size", [ # Test an artifact that is obviously too large (500000), # Test an artifact that might be too large due to slight overhead # of storing stuff in ostree (399999), ], ) @pytest.mark.datafiles(DATA_DIR) def test_artifact_too_large(cli, datafiles, size): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) cli.configure({"cache": {"quota": 400000}}) # Create an element whose artifact is too large create_element_size("target.bst", project, element_path, [], size) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.CAS, "cache-too-full") @pytest.mark.datafiles(DATA_DIR) def test_expiry_order(cli, datafiles): project = str(datafiles) element_path = "elements" checkout = os.path.join(project, "workspace") cli.configure({"cache": {"quota": 9000000, "low-watermark": "50%"}}) # Create an artifact create_element_size("dep.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "dep.bst"]) res.assert_success() # Create another artifact create_element_size("unrelated.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "unrelated.bst"]) res.assert_success() # And build something else create_element_size("target.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() create_element_size("target2.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() wait_for_cache_granularity() # Now extract dep.bst res = cli.run(project=project, args=["artifact", "checkout", "dep.bst", "--directory", checkout]) res.assert_success() # Finally, build something that will cause the cache to overflow create_element_size("expire.bst", project, element_path, [], 2000000) res = cli.run(project=project, args=["build", "expire.bst"]) res.assert_success() # While dep.bst was the first element to be created, it should not # have been removed. # Note that buildstream will reduce the cache to 50% of the # original size - we therefore remove multiple elements. check_elements = ["unrelated.bst", "target.bst", "target2.bst", "dep.bst", "expire.bst"] states = cli.get_element_states(project, check_elements) assert tuple(states[element] for element in check_elements) == ( "buildable", "buildable", "buildable", "cached", "cached", ) # Ensure that we don't accidentally remove an artifact from something # in the current build pipeline, because that would be embarassing, # wouldn't it? @pytest.mark.datafiles(DATA_DIR) def test_keep_dependencies(cli, datafiles): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) cli.configure({"cache": {"quota": 10000000}}) # Create a pretty big dependency create_element_size("dependency.bst", project, element_path, [], 5000000) res = cli.run(project=project, args=["build", "dependency.bst"]) res.assert_success() # Now create some other unrelated artifact create_element_size("unrelated.bst", project, element_path, [], 4000000) res = cli.run(project=project, args=["build", "unrelated.bst"]) res.assert_success() # Check that the correct element remains in the cache states = cli.get_element_states(project, ["dependency.bst", "unrelated.bst"]) assert states["dependency.bst"] == "cached" assert states["unrelated.bst"] == "cached" # We try to build an element which depends on the LRU artifact, # and could therefore fail if we didn't make sure dependencies # aren't removed. # # Since some artifact caches may implement weak cache keys by # duplicating artifacts (bad!) we need to make this equal in size # or smaller than half the size of its dependencies. # create_element_size("target.bst", project, element_path, ["dependency.bst"], 2000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() states = cli.get_element_states(project, ["target.bst", "unrelated.bst"]) assert states["target.bst"] == "cached" assert states["dependency.bst"] == "cached" assert states["unrelated.bst"] != "cached" # Assert that we never delete a dependency required for a build tree @pytest.mark.datafiles(DATA_DIR) def test_never_delete_required(cli, datafiles): project = str(datafiles) element_path = "elements" # Skip this test if we do not have support for subsecond precision mtimes # # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) cli.configure({"cache": {"quota": 10000000}, "scheduler": {"fetchers": 1, "builders": 1}}) # Create a linear build tree create_element_size("dep1.bst", project, element_path, [], 8000000) create_element_size("dep2.bst", project, element_path, ["dep1.bst"], 8000000) create_element_size("dep3.bst", project, element_path, ["dep2.bst"], 8000000) create_element_size("target.bst", project, element_path, ["dep3.bst"], 8000000) # Build dep1.bst, which should fit into the cache. res = cli.run(project=project, args=["build", "dep1.bst"]) res.assert_success() # We try to build this pipeline, but it's too big for the # cache. Since all elements are required, the build should fail. res = cli.run(project=project, args=["build", "target.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.CAS, "cache-too-full") states = cli.get_element_states(project, ["target.bst"]) assert states["dep1.bst"] == "cached" assert states["dep2.bst"] != "cached" assert states["dep3.bst"] != "cached" assert states["target.bst"] != "cached" # Ensure that only valid cache quotas make it through the loading # process. # # Parameters: # quota (str): A quota size configuration for the config file # err_domain (str): An ErrorDomain, or 'success' or 'warning' # err_reason (str): A reson to compare with an error domain # # If err_domain is 'success', then err_reason is unused. # @pytest.mark.parametrize( "quota,err_domain,err_reason", [ # Valid configurations ("1", "success", None), ("1K", "success", None), ("50%", "success", None), ("infinity", "success", None), ("0", "success", None), # Invalid configurations ("-1", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), ("pony", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), ("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA), ], ) @pytest.mark.datafiles(DATA_DIR) def test_invalid_cache_quota(cli, datafiles, quota, err_domain, err_reason): project = str(datafiles) os.makedirs(os.path.join(project, "elements")) cli.configure( { "cache": { "quota": quota, }, } ) res = cli.run(project=project, args=["workspace", "list"]) if err_domain == "success": res.assert_success() else: res.assert_main_error(err_domain, err_reason) # Ensures that when launching BuildStream with a full artifact cache, # the cache size and cleanup jobs are run before any other jobs. # @pytest.mark.datafiles(DATA_DIR) def test_cleanup_first(cli, datafiles): project = str(datafiles) element_path = "elements" cli.configure( { "cache": { "quota": 10000000, } } ) # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) create_element_size("target.bst", project, element_path, [], 8000000) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() assert cli.get_element_state(project, "target.bst") == "cached" # Now configure with a smaller quota, create a situation # where the cache must be cleaned up before building anything else. # # Fix the fetchers and builders just to ensure a predictable # sequence of events (although it does not effect this test) cli.configure( { "cache": { "quota": 5000000, }, "scheduler": {"fetchers": 1, "builders": 1}, } ) # Our cache is now more than full, BuildStream create_element_size("target2.bst", project, element_path, [], 4000000) res = cli.run(project=project, args=["build", "target2.bst"]) res.assert_success() # Check that the correct element remains in the cache states = cli.get_element_states(project, ["target.bst", "target2.bst"]) assert states["target.bst"] != "cached" assert states["target2.bst"] == "cached" @pytest.mark.datafiles(DATA_DIR) def test_cache_usage_monitor(cli, tmpdir, datafiles): project = str(datafiles) element_path = "elements" assert get_cache_usage(cli.directory) == 0 ELEMENT_SIZE = 1000000 create_element_size("target.bst", project, element_path, [], ELEMENT_SIZE) res = cli.run(project=project, args=["build", "target.bst"]) res.assert_success() assert get_cache_usage(cli.directory) >= ELEMENT_SIZE apache-buildstream-27ae392/tests/artifactcache/expiry/000077500000000000000000000000001514607367700230655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/expiry/project.conf000066400000000000000000000004321514607367700254010ustar00rootroot00000000000000# Project config for cache expiry test name: test min-version: 2.0 element-path: elements aliases: project_dir: file://{project_dir} options: linux: type: bool description: Whether to expect a linux platform default: True split-rules: test: - | /tests/* apache-buildstream-27ae392/tests/artifactcache/junctions.py000066400000000000000000000100171514607367700241320ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream import _yaml from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share, assert_shared, assert_not_shared DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "junctions", ) def project_set_artifacts(project, url): project_conf_file = os.path.join(project, "project.conf") project_config = _yaml.load(project_conf_file, shortname=None) project_config["artifacts"] = [{"url": url, "push": True}] _yaml.roundtrip_dump(project_config.strip_node_info(), file=project_conf_file) @pytest.mark.datafiles(DATA_DIR) def test_push_pull(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "parent") base_project = os.path.join(str(project), "base") with create_artifact_share(os.path.join(str(tmpdir), "artifactshare-parent")) as share, create_artifact_share( os.path.join(str(tmpdir), "artifactshare-base") ) as base_share: # First build it without the artifact cache configured result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # Assert that we are now cached locally state = cli.get_element_state(project, "target.bst") assert state == "cached" state = cli.get_element_state(base_project, "base-element.bst") assert state == "cached" project_set_artifacts(project, share.repo) project_set_artifacts(base_project, base_share.repo) # Now try bst artifact push result = cli.run(project=project, args=["artifact", "push", "--deps", "all", "target.bst"]) assert result.exit_code == 0 # And finally assert that the artifacts are in the right shares # # In the parent project's cache assert_shared(cli, share, project, "target.bst", project_name="parent") assert_shared(cli, share, project, "app.bst", project_name="parent") assert_not_shared(cli, share, base_project, "base-element.bst", project_name="base") # In the junction project's cache assert_not_shared(cli, base_share, project, "target.bst", project_name="parent") assert_not_shared(cli, base_share, project, "app.bst", project_name="parent") assert_shared(cli, base_share, base_project, "base-element.bst", project_name="base") # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # cas = os.path.join(cli.directory, "cas") shutil.rmtree(cas) artifact_dir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifact_dir) # Assert that nothing is cached locally anymore state = cli.get_element_state(project, "target.bst") assert state != "cached" state = cli.get_element_state(base_project, "base-element.bst") assert state != "cached" # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) assert result.exit_code == 0 # And assert that they are again in the local cache, without having built state = cli.get_element_state(project, "target.bst") assert state == "cached" state = cli.get_element_state(base_project, "base-element.bst") assert state == "cached" apache-buildstream-27ae392/tests/artifactcache/junctions/000077500000000000000000000000001514607367700235615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/junctions/parent/000077500000000000000000000000001514607367700250525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/junctions/parent/app.bst000066400000000000000000000001571514607367700263470ustar00rootroot00000000000000kind: import sources: - kind: local path: foo.txt depends: - junction: base.bst filename: base-element.bst apache-buildstream-27ae392/tests/artifactcache/junctions/parent/base.bst000066400000000000000000000000631514607367700264750ustar00rootroot00000000000000kind: junction sources: - kind: local path: base apache-buildstream-27ae392/tests/artifactcache/junctions/parent/base/000077500000000000000000000000001514607367700257645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/junctions/parent/base/base-element.bst000066400000000000000000000000651514607367700310400ustar00rootroot00000000000000kind: import sources: - kind: local path: base.txt apache-buildstream-27ae392/tests/artifactcache/junctions/parent/base/base.txt000066400000000000000000000000241514607367700274330ustar00rootroot00000000000000This is a text file apache-buildstream-27ae392/tests/artifactcache/junctions/parent/base/project.conf000066400000000000000000000000341514607367700302760ustar00rootroot00000000000000name: base min-version: 2.0 apache-buildstream-27ae392/tests/artifactcache/junctions/parent/foo.txt000066400000000000000000000000041514607367700263700ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/artifactcache/junctions/parent/project.conf000066400000000000000000000000361514607367700273660ustar00rootroot00000000000000name: parent min-version: 2.0 apache-buildstream-27ae392/tests/artifactcache/junctions/parent/target.bst000066400000000000000000000001211514607367700270440ustar00rootroot00000000000000kind: stack depends: - junction: base.bst filename: base-element.bst - app.bst apache-buildstream-27ae392/tests/artifactcache/missing-certs/000077500000000000000000000000001514607367700243345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/missing-certs/certificates/000077500000000000000000000000001514607367700270015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/missing-certs/certificates/client.crt000066400000000000000000000000001514607367700307570ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/missing-certs/certificates/client.key000066400000000000000000000000001514607367700307570ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/missing-certs/element.bst000066400000000000000000000000151514607367700264730ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/artifactcache/only-one/000077500000000000000000000000001514607367700233055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/only-one/element.bst000066400000000000000000000000151514607367700254440ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/artifactcache/project/000077500000000000000000000000001514607367700232135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/elements/000077500000000000000000000000001514607367700250275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/elements/compose-all.bst000066400000000000000000000003441514607367700277550ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/artifactcache/project/elements/import-bin.bst000066400000000000000000000000741514607367700276220ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/artifactcache/project/elements/import-dev.bst000066400000000000000000000000741514607367700276300ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/artifactcache/project/elements/target.bst000066400000000000000000000002051514607367700270240ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - import-dev.bst - compose-all.bst apache-buildstream-27ae392/tests/artifactcache/project/files/000077500000000000000000000000001514607367700243155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/bin-files/000077500000000000000000000000001514607367700261655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/bin-files/usr/000077500000000000000000000000001514607367700267765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/bin-files/usr/bin/000077500000000000000000000000001514607367700275465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700305740ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/artifactcache/project/files/dev-files/000077500000000000000000000000001514607367700261735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/dev-files/usr/000077500000000000000000000000001514607367700270045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/dev-files/usr/include/000077500000000000000000000000001514607367700304275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/artifactcache/project/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700315660ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/artifactcache/project/project.conf000066400000000000000000000001341514607367700255260ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/artifactcache/pull.py000066400000000000000000000205611514607367700230770ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._project import Project from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share, dummy_context # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) def tree_maker(cas, tree, directory): if tree.root.ByteSize() == 0: tree.root.CopyFrom(directory) for directory_node in directory.directories: child_directory = tree.children.add() with open(cas.objpath(directory_node.digest), "rb") as f: child_directory.ParseFromString(f.read()) tree_maker(cas, tree, child_directory) @pytest.mark.parametrize( "connection_config", ( None, { "keepalive-time": 60, }, ), ) @pytest.mark.datafiles(DATA_DIR) def test_pull(cli, tmpdir, datafiles, connection_config): project_dir = str(datafiles) # Set up an artifact cache. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share cache_dir = os.path.join(str(tmpdir), "cache") user_config_file = str(tmpdir.join("buildstream.conf")) server = { "url": share.repo, "push": True, } user_config = { "scheduler": {"pushers": 1}, "artifacts": {"servers": [server]}, "cachedir": cache_dir, } if connection_config is not None: server["connection-config"] = connection_config # Write down the user configuration file _yaml.roundtrip_dump(user_config, file=user_config_file) # Ensure CLI calls will use it cli.configure(user_config) # First build the project with the artifact cache configured result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project_dir, "target.bst") == "cached" # Assert that we shared/pushed the cached artifact assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst")) # Delete the artifact locally cli.remove_artifact_from_cache(project_dir, "target.bst") # Assert that we are not cached locally anymore assert cli.get_element_state(project_dir, "target.bst") != "cached" with dummy_context(config=user_config_file) as context: # Load the project project = Project(project_dir, context) project.ensure_fully_loaded() # Assert that the element's artifact is **not** cached element = project.load_elements(["target.bst"])[0] element_key = cli.get_element_key(project_dir, "target.bst") assert not cli.artifact.is_cached(cache_dir, element, element_key) context.cachedir = cache_dir context.casdir = os.path.join(cache_dir, "cas") context.tmpdir = os.path.join(cache_dir, "tmp") # Load the project manually project = Project(project_dir, context) project.ensure_fully_loaded() # Create a local artifact cache handle artifactcache = context.artifactcache # Initialize remotes context.initialize_remotes(True, True, None, None) assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst" assert artifactcache.pull(element, element_key), "Pull operation failed" assert cli.artifact.is_cached(cache_dir, element, element_key) @pytest.mark.datafiles(DATA_DIR) def test_pull_tree(cli, tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share rootcache_dir = os.path.join(str(tmpdir), "cache") user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": rootcache_dir, } # Write down the user configuration file _yaml.roundtrip_dump(user_config, file=user_config_file) # Ensure CLI calls will use it cli.configure(user_config) # First build the project with the artifact cache configured result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project_dir, "target.bst") == "cached" # Assert that we shared/pushed the cached artifact assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst")) with dummy_context(config=user_config_file) as context: # Load the project and CAS cache project = Project(project_dir, context) project.ensure_fully_loaded() cas = context.get_cascache() # Assert that the element's artifact is cached element = project.load_elements(["target.bst"])[0] element_key = cli.get_element_key(project_dir, "target.bst") assert cli.artifact.is_cached(rootcache_dir, element, element_key) # Retrieve the Directory object from the cached artifact artifact_digest = cli.artifact.get_digest(rootcache_dir, element, element_key) # Initialize remotes context.initialize_remotes(True, True, None, None) artifactcache = context.artifactcache assert artifactcache.has_push_remotes() directory = remote_execution_pb2.Directory() with open(cas.objpath(artifact_digest), "rb") as f: directory.ParseFromString(f.read()) # Build the Tree object while we are still cached tree = remote_execution_pb2.Tree() tree_maker(cas, tree, directory) # Push the Tree as a regular message _, remotes = artifactcache.get_remotes(project.name, True) assert len(remotes) == 1 remotes[0].init() tree_digest = cas.add_object( buffer=tree.SerializeToString(), instance_name=remotes[0].local_cas_instance_name ) tree_hash, tree_size = tree_digest.hash, tree_digest.size_bytes assert tree_hash and tree_size # Now delete the artifact locally cli.remove_artifact_from_cache(project_dir, "target.bst") # Assert that we are not cached locally anymore assert cli.get_element_state(project_dir, "target.bst") != "cached" tree_digest = remote_execution_pb2.Digest(hash=tree_hash, size_bytes=tree_size) # Pull the artifact using the Tree object _, remotes = artifactcache.get_remotes(project.name, False) assert len(remotes) == 1 directory_digest = cas.pull_tree(remotes[0], tree_digest) directory_hash, directory_size = directory_digest.hash, directory_digest.size_bytes # Directory size now zero with AaaP and stack element commit #1cbc5e63dc assert directory_hash and not directory_size directory_digest = remote_execution_pb2.Digest(hash=directory_hash, size_bytes=directory_size) # Ensure the entire Tree stucture has been pulled assert os.path.exists(cas.objpath(directory_digest)) apache-buildstream-27ae392/tests/artifactcache/push.py000066400000000000000000000113201514607367700230730ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._project import Project from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share, create_split_share, dummy_context # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) # Push the given element and return its artifact key for assertions. def _push(cli, cache_dir, project_dir, config_file, target): with dummy_context(config=config_file) as context: # Load the project manually project = Project(project_dir, context) project.ensure_fully_loaded() # Assert that the element's artifact is cached element = project.load_elements(["target.bst"])[0] element_key = cli.get_element_key(project_dir, "target.bst") assert cli.artifact.is_cached(cache_dir, element, element_key) # Create a local artifact cache handle artifactcache = context.artifactcache # Initialize remotes context.initialize_remotes(True, True, None, None) # Query local cache element._load_artifact(pull=False) assert artifactcache.has_push_remotes(plugin=element), "No remote configured for element target.bst" assert element._push(), "Push operation failed" return element_key @pytest.mark.datafiles(DATA_DIR) def test_push(cli, tmpdir, datafiles): project_dir = str(datafiles) # First build the project without the artifact cache configured result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project_dir, "target.bst") == "cached" # Set up an artifact cache. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share rootcache_dir = os.path.join(str(tmpdir), "cache") user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": rootcache_dir, } # Write down the user configuration file _yaml.roundtrip_dump(user_config, file=user_config_file) element_key = _push(cli, rootcache_dir, project_dir, user_config_file, "target.bst") assert share.get_artifact(cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key)) @pytest.mark.datafiles(DATA_DIR) def test_push_split(cli, tmpdir, datafiles): project_dir = str(datafiles) # First build the project without the artifact cache configured result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project_dir, "target.bst") == "cached" indexshare = os.path.join(str(tmpdir), "indexshare") storageshare = os.path.join(str(tmpdir), "storageshare") # Set up an artifact cache. with create_split_share(indexshare, storageshare) as (index, storage): rootcache_dir = os.path.join(str(tmpdir), "cache") user_config = { "scheduler": {"pushers": 1}, "artifacts": { "servers": [ {"url": index.repo, "push": True, "type": "index"}, {"url": storage.repo, "push": True, "type": "storage"}, ], }, "cachedir": rootcache_dir, } config_path = str(tmpdir.join("buildstream.conf")) _yaml.roundtrip_dump(user_config, file=config_path) element_key = _push(cli, rootcache_dir, project_dir, config_path, "target.bst") proto = index.get_artifact_proto( cli.get_artifact_name(project_dir, "test", "target.bst", cache_key=element_key) ) assert storage.get_cas_files(proto) is not None apache-buildstream-27ae392/tests/cachekey/000077500000000000000000000000001514607367700205405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/__init__.py000066400000000000000000000000001514607367700226370ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/cachekey.py000066400000000000000000000141041514607367700226660ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Cache Key Test Instructions # # Adding Tests # ~~~~~~~~~~~~ # Cache key tests are bst element files created created in such a way # to exercise a feature which would cause the cache key for an element # or source to be calculated differently. # # Adding tests is a matter to adding files to the project found in the # 'project' subdirectory of this test case. Any files should be depended # on by the main `target.bst` in the toplevel of the project. # # One test is comprised of one `.bst` file and one # '.expected' file in the same directory, containing the # expected cache key. # # Running the cache key test once will reveal what the new element's # cache key should be and will also cause the depending elements to # change cache keys. # # # Updating tests # ~~~~~~~~~~~~~~ # When a test fails it will come with a summary of which cache keys # in the test project have mismatched. # # Also, in the case that the tests have changed or the artifact # versions have changed in some way and the test needs to be # updated; the expected cache keys for the given run are dumped to # '.actual' files beside the corresponding # '.expected' files they mismatched with, all inside # a temporary test directory. # # One can now easily copy over the .actual files from a failed # run over to the corresponding .expected source files and commit # the result. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing._cachekeys import check_cache_key_stability, _parse_output_keys from buildstream._testing.runcli import cli # pylint: disable=unused-import from buildstream._testing._utils.site import IS_LINUX, MACHINE_ARCH from buildstream.plugin import CoreWarnings from buildstream import _yaml # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Cache keys depend on architecture") @pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_cache_key(datafiles, cli): project = str(datafiles) # Workaround bug in recent versions of setuptools: newer # versions of setuptools fail to preserve symbolic links # when creating a source distribution, causing this test # to fail from a dist tarball. goodbye_link = os.path.join(project, "files", "local", "usr", "bin", "goodbye") os.unlink(goodbye_link) os.symlink("hello", goodbye_link) # pytest-datafiles does not copy mode bits # https://github.com/omarkohl/pytest-datafiles/issues/11 os.chmod(goodbye_link, 0o755) check_cache_key_stability(project, cli) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "first_warnings, second_warnings, identical_keys", [ [[], [], True], [[], [CoreWarnings.REF_NOT_IN_TRACK], False], [[CoreWarnings.REF_NOT_IN_TRACK], [], False], [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True], [ [CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS], [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True, ], ], ) def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys): # Builds project, Runs bst show, gathers cache keys def run_get_cache_key(project_name, warnings): config = {"name": "test", "min-version": "2.0", "element-path": "elements", "fatal-warnings": warnings} project_dir = tmpdir.mkdir(project_name) project_config_file = str(project_dir.join("project.conf")) _yaml.roundtrip_dump(config, file=project_config_file) elem_dir = project_dir.mkdir("elements") element_file = str(elem_dir.join("stack.bst")) _yaml.roundtrip_dump({"kind": "stack"}, file=element_file) result = cli.run(project=str(project_dir), args=["show", "--format", "%{name}::%{full-key}", "stack.bst"]) return result.output # Returns true if all keys are identical def compare_cache_keys(first_keys, second_keys): return not any((x != y for x, y in zip(first_keys, second_keys))) first_keys = run_get_cache_key("first", first_warnings) second_keys = run_get_cache_key("second", second_warnings) assert compare_cache_keys(first_keys, second_keys) == identical_keys @pytest.mark.datafiles(DATA_DIR) def test_keys_stable_over_targets(cli, datafiles): root_element = "elements/key-stability/top-level.bst" target1 = "elements/key-stability/t1.bst" target2 = "elements/key-stability/t2.bst" project = str(datafiles) full_graph_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", root_element]) full_graph_result.assert_success() all_cache_keys = _parse_output_keys(full_graph_result.output) ordering1_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target1, target2]) ordering1_result.assert_success() ordering1_cache_keys = _parse_output_keys(ordering1_result.output) ordering2_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target2, target1]) ordering2_result.assert_success() ordering2_cache_keys = _parse_output_keys(ordering2_result.output) elements = ordering1_cache_keys.keys() assert {key: ordering2_cache_keys[key] for key in elements} == ordering1_cache_keys assert {key: all_cache_keys[key] for key in elements} == ordering1_cache_keys apache-buildstream-27ae392/tests/cachekey/project/000077500000000000000000000000001514607367700222065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/elements/000077500000000000000000000000001514607367700240225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/elements/build1.bst000066400000000000000000000005401514607367700257130ustar00rootroot00000000000000# The cache key calculation algorithm is the same # for all build elements, better to only have one test # for it and without too much noise from changes which # would cause is to change the test case. # # Lets stick with manual, the most basic build element. kind: manual sources: - kind: local path: files/local config: build-commands: - make apache-buildstream-27ae392/tests/cachekey/project/elements/build1.expected000066400000000000000000000001001514607367700267140ustar00rootroot0000000000000095cc8ee040690b8abbacb201c8f0d9eeab7d1376db473d62799b5b5c36c71b47apache-buildstream-27ae392/tests/cachekey/project/elements/build2.bst000066400000000000000000000003511514607367700257140ustar00rootroot00000000000000# This tests that sandbox build-uid / build-gid # contributions to the cache key do not regress. # kind: manual sources: - kind: local path: files/local config: build-commands: - make sandbox: build-uid: 20 build-gid: 20 apache-buildstream-27ae392/tests/cachekey/project/elements/build2.expected000066400000000000000000000001001514607367700267150ustar00rootroot00000000000000dccab3eb3e5a9b5ef3a29f409cb0235ee4c393edfda1407114aa7784c35ed588apache-buildstream-27ae392/tests/cachekey/project/elements/build3.bst000066400000000000000000000003441514607367700257170ustar00rootroot00000000000000# This tests that staging into an alternative # prefix affects the cache key # kind: manual sources: - kind: local path: files/local depends: - elements/build1.bst - filename: elements/build1.bst config: location: /opt apache-buildstream-27ae392/tests/cachekey/project/elements/build3.expected000066400000000000000000000001001514607367700267160ustar00rootroot00000000000000492953d9bdef66486c5111caf48f015690848e46d7f8de0f71bde386b68d4e5eapache-buildstream-27ae392/tests/cachekey/project/elements/compose1.bst000066400000000000000000000001061514607367700262570ustar00rootroot00000000000000kind: compose depends: - filename: elements/import1.bst type: build apache-buildstream-27ae392/tests/cachekey/project/elements/compose1.expected000066400000000000000000000001001514607367700272620ustar00rootroot000000000000008406283ca890663b317e055a1c4b8c18248b2c12388ab0900e6ee6ab12af8abdapache-buildstream-27ae392/tests/cachekey/project/elements/compose2.bst000066400000000000000000000001421514607367700262600ustar00rootroot00000000000000kind: compose depends: - filename: elements/import1.bst type: build config: integrate: False apache-buildstream-27ae392/tests/cachekey/project/elements/compose2.expected000066400000000000000000000001001514607367700272630ustar00rootroot00000000000000616a126d43f56ecfa0613093ff3a2b545bfa6568cd03cb2ad8bae8595568aeabapache-buildstream-27ae392/tests/cachekey/project/elements/compose3.bst000066400000000000000000000001501514607367700262600ustar00rootroot00000000000000kind: compose depends: - filename: elements/import1.bst type: build config: include-orphans: False apache-buildstream-27ae392/tests/cachekey/project/elements/compose3.expected000066400000000000000000000001001514607367700272640ustar00rootroot00000000000000f718a75e4fe35bfb2f2000897d0e20e120f3e23c96d6ae9221a29fdb32bb1334apache-buildstream-27ae392/tests/cachekey/project/elements/compose4.bst000066400000000000000000000001601514607367700262620ustar00rootroot00000000000000kind: compose depends: - filename: elements/import1.bst type: build config: include: - runtime - devel apache-buildstream-27ae392/tests/cachekey/project/elements/compose4.expected000066400000000000000000000001001514607367700272650ustar00rootroot0000000000000056a74b2d1bb065e9f83c0415290daca95017478fcbd13a5326e219a9e6a6dd01apache-buildstream-27ae392/tests/cachekey/project/elements/compose5.bst000066400000000000000000000001441514607367700262650ustar00rootroot00000000000000kind: compose depends: - filename: elements/import1.bst type: build config: exclude: - debug apache-buildstream-27ae392/tests/cachekey/project/elements/compose5.expected000066400000000000000000000001001514607367700272660ustar00rootroot000000000000009ab2f1b18d7894ee3cf507c19ace3e46897089532004b60339fd2a63b06f3f59apache-buildstream-27ae392/tests/cachekey/project/elements/import1.bst000066400000000000000000000000701514607367700261240ustar00rootroot00000000000000kind: import sources: - kind: local path: files/local apache-buildstream-27ae392/tests/cachekey/project/elements/import1.expected000066400000000000000000000001001514607367700271270ustar00rootroot000000000000009fa24d24cf3351f522694c0cf39440efabb4ec65d888284b7048285f3eab9429apache-buildstream-27ae392/tests/cachekey/project/elements/import2.bst000066400000000000000000000001241514607367700261250ustar00rootroot00000000000000kind: import sources: - kind: local path: files/local config: source: /usr/bin apache-buildstream-27ae392/tests/cachekey/project/elements/import2.expected000066400000000000000000000001001514607367700271300ustar00rootroot00000000000000bf23c7d9b5a3a68c6fb2ffbc617a596ae65726a17dbd406a4b10acf3247acadcapache-buildstream-27ae392/tests/cachekey/project/elements/import3.bst000066400000000000000000000001201514607367700261220ustar00rootroot00000000000000kind: import sources: - kind: local path: files/local config: target: /opt apache-buildstream-27ae392/tests/cachekey/project/elements/import3.expected000066400000000000000000000001001514607367700271310ustar00rootroot00000000000000982779f72790b19d7fdf59f5f871cb498e5989fec1ebf5f205128f04af016b25apache-buildstream-27ae392/tests/cachekey/project/elements/key-stability/000077500000000000000000000000001514607367700266145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/elements/key-stability/aaa.bst000066400000000000000000000001131514607367700300430ustar00rootroot00000000000000kind: import sources: - kind: local path: elements/key-stability/aaa.bst apache-buildstream-27ae392/tests/cachekey/project/elements/key-stability/t1.bst000066400000000000000000000001641514607367700276530ustar00rootroot00000000000000kind: import sources: - kind: local path: elements/key-stability/t1.bst depends: - elements/key-stability/zzz.bst apache-buildstream-27ae392/tests/cachekey/project/elements/key-stability/t2.bst000066400000000000000000000002251514607367700276520ustar00rootroot00000000000000kind: import sources: - kind: local path: elements/key-stability/t2.bst depends: - elements/key-stability/aaa.bst - elements/key-stability/zzz.bst apache-buildstream-27ae392/tests/cachekey/project/elements/key-stability/top-level.bst000066400000000000000000000002321514607367700312320ustar00rootroot00000000000000kind: import sources: - kind: local path: elements/key-stability/top-level.bst depends: - elements/key-stability/t1.bst - elements/key-stability/t2.bst apache-buildstream-27ae392/tests/cachekey/project/elements/key-stability/zzz.bst000066400000000000000000000001131514607367700301560ustar00rootroot00000000000000kind: import sources: - kind: local path: elements/key-stability/zzz.bst apache-buildstream-27ae392/tests/cachekey/project/elements/script1.bst000066400000000000000000000001631514607367700261210ustar00rootroot00000000000000kind: script depends: - filename: elements/import1.bst type: build config: commands: - echo "Hello World !" apache-buildstream-27ae392/tests/cachekey/project/elements/script1.expected000066400000000000000000000001001514607367700271210ustar00rootroot0000000000000015bed1afee844f2b3ea2a433dcd93a15494ac09c0efc6108a2157013004d2748apache-buildstream-27ae392/tests/cachekey/project/elements/sort0.bst000066400000000000000000000001521514607367700256010ustar00rootroot00000000000000kind: stack depends: - elements/sort8.bst - elements/sort9.bst - elements/sort5.bst - elements/sort3.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort0.expected000066400000000000000000000001011514607367700266040ustar00rootroot0000000000000057263fea3c5595e04ec22e6fa7e531f69192273df2b35d09e47dd8c8660c6215 apache-buildstream-27ae392/tests/cachekey/project/elements/sort1.bst000066400000000000000000000000531514607367700256020ustar00rootroot00000000000000kind: stack depends: - elements/sort9.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort1.expected000066400000000000000000000001011514607367700266050ustar00rootroot000000000000008cc2dc817a224aa1522b70194d7088143884d3889b164d1b164d5ff3bd62d88f apache-buildstream-27ae392/tests/cachekey/project/elements/sort2.bst000066400000000000000000000000531514607367700256030ustar00rootroot00000000000000kind: stack depends: - elements/sort6.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort2.expected000066400000000000000000000001011514607367700266060ustar00rootroot00000000000000c9fb7684e517c668a46ce5d0d36207fb071e6e0094b80c5477d329e6cb4c98a5 apache-buildstream-27ae392/tests/cachekey/project/elements/sort3.bst000066400000000000000000000000531514607367700256040ustar00rootroot00000000000000kind: stack depends: - elements/sort2.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort3.expected000066400000000000000000000001011514607367700266070ustar00rootroot00000000000000d4fb3897bd712221b8348e6beb7a51cce381906bce7dfd05dc310d2ca8b0ed8f apache-buildstream-27ae392/tests/cachekey/project/elements/sort4.bst000066400000000000000000000000531514607367700256050ustar00rootroot00000000000000kind: stack depends: - elements/sort6.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort4.expected000066400000000000000000000001011514607367700266100ustar00rootroot00000000000000210117e90eb052b6d8860e51bfec3dcdac17d6867d0d067d15222b22f62f083b apache-buildstream-27ae392/tests/cachekey/project/elements/sort5.bst000066400000000000000000000000531514607367700256060ustar00rootroot00000000000000kind: stack depends: - elements/sort4.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort5.expected000066400000000000000000000001011514607367700266110ustar00rootroot00000000000000c1e365edf5318dd82d8b3906b0facceee090f794548162af4bbfd6372a2c7ffd apache-buildstream-27ae392/tests/cachekey/project/elements/sort6.bst000066400000000000000000000000531514607367700256070ustar00rootroot00000000000000kind: stack depends: - elements/sort7.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort6.expected000066400000000000000000000001011514607367700266120ustar00rootroot00000000000000292431a52a4f31d9e45ff46e124e74a6306424fb7175dca4166d876d3fd1ae62 apache-buildstream-27ae392/tests/cachekey/project/elements/sort7.bst000066400000000000000000000000531514607367700256100ustar00rootroot00000000000000kind: stack depends: - elements/sort1.bst apache-buildstream-27ae392/tests/cachekey/project/elements/sort7.expected000066400000000000000000000001011514607367700266130ustar00rootroot0000000000000008af1499d29bc396ba61b851a95388f7a394d5412d0b452eeb0546517aed1ae2 apache-buildstream-27ae392/tests/cachekey/project/elements/sort8.bst000066400000000000000000000000151514607367700256070ustar00rootroot00000000000000kind: stack apache-buildstream-27ae392/tests/cachekey/project/elements/sort8.expected000066400000000000000000000001011514607367700266140ustar00rootroot0000000000000094bf67977690c7f438a494779bd418b36dd656a318d49dae525411988ce33e86 apache-buildstream-27ae392/tests/cachekey/project/elements/sort9.bst000066400000000000000000000000151514607367700256100ustar00rootroot00000000000000kind: stack apache-buildstream-27ae392/tests/cachekey/project/elements/sort9.expected000066400000000000000000000001011514607367700266150ustar00rootroot00000000000000a75c0adad402910d65d7c7042433698d55d247fc69d9e43c3ba91c584552e879 apache-buildstream-27ae392/tests/cachekey/project/elements/variables1.bst000066400000000000000000000012671514607367700265730ustar00rootroot00000000000000# # This cache key test attempts to make use of all of the variables # in the default project configuration to ensure their stability. # kind: manual sources: - kind: local path: files/local config: build-commands: - echo "%{prefix}" - echo "%{exec_prefix}" - echo "%{bindir}" - echo "%{sbindir}" - echo "%{libexecdir}" - echo "%{datadir}" - echo "%{sysconfdir}" - echo "%{sharedstatedir}" - echo "%{localstatedir}" - echo "%{lib}" - echo "%{libdir}" - echo "%{debugdir}" - echo "%{includedir}" - echo "%{docdir}" - echo "%{infodir}" - echo "%{mandir}" - echo "%{build-root}" - echo "%{conf-root}" - echo "%{install-root}" - echo "%{strip-binaries}" apache-buildstream-27ae392/tests/cachekey/project/elements/variables1.expected000066400000000000000000000001001514607367700275650ustar00rootroot00000000000000015ec2ae48b66178540db8983a93f536b831b53ff9c4f6fc70c0517b11031170apache-buildstream-27ae392/tests/cachekey/project/files/000077500000000000000000000000001514607367700233105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/local/000077500000000000000000000000001514607367700244025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/local/etc/000077500000000000000000000000001514607367700251555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/local/etc/hello.conf000066400000000000000000000000201514607367700271170ustar00rootroot00000000000000message = Hello apache-buildstream-27ae392/tests/cachekey/project/files/local/etc/ponystyle.conf000066400000000000000000000000051514607367700300650ustar00rootroot00000000000000pink apache-buildstream-27ae392/tests/cachekey/project/files/local/usr/000077500000000000000000000000001514607367700252135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/local/usr/bin/000077500000000000000000000000001514607367700257635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/local/usr/bin/goodbye000077700000000000000000000000001514607367700303572helloustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/local/usr/bin/hello000077500000000000000000000000341514607367700270110ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/cachekey/project/files/patches/000077500000000000000000000000001514607367700247375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/files/patches/patch.diff000066400000000000000000000001451514607367700266700ustar00rootroot00000000000000--- a/usr/bin/hello +++ b/usr/bin/hello @@ -1,3 +1,3 @@ #!/bin/bash -echo "Hello !" +echo "Bye !" apache-buildstream-27ae392/tests/cachekey/project/project.conf000066400000000000000000000001601514607367700245200ustar00rootroot00000000000000# Project config for cache key test name: cachekey min-version: 2.0 aliases: upstream: https://up.stream.org apache-buildstream-27ae392/tests/cachekey/project/sources/000077500000000000000000000000001514607367700236715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/cachekey/project/sources/local1.bst000066400000000000000000000000701514607367700255530ustar00rootroot00000000000000kind: import sources: - kind: local path: files/local apache-buildstream-27ae392/tests/cachekey/project/sources/local1.expected000066400000000000000000000001001514607367700265560ustar00rootroot00000000000000b1b40e934f8c51256dc37990a290c5c66e2e35e98ef497e48b8cf010d8ef712bapache-buildstream-27ae392/tests/cachekey/project/sources/local2.bst000066400000000000000000000001111514607367700255500ustar00rootroot00000000000000kind: import sources: - kind: local path: files/local directory: opt apache-buildstream-27ae392/tests/cachekey/project/sources/local2.expected000066400000000000000000000001001514607367700265570ustar00rootroot000000000000008aff46847f26a99612355026d903b7dea1a18422cc1bf08017697501a01d08dcapache-buildstream-27ae392/tests/cachekey/project/sources/remote1.bst000066400000000000000000000002211514607367700257520ustar00rootroot00000000000000kind: import sources: - kind: remote url: https://example.com/foo.bar ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b apache-buildstream-27ae392/tests/cachekey/project/sources/remote1.expected000066400000000000000000000001001514607367700267570ustar00rootroot0000000000000042b9d1371f692ed06df503967ddda9c11518325f2d856ba33069e41cec4e10e4apache-buildstream-27ae392/tests/cachekey/project/sources/remote2.bst000066400000000000000000000002441514607367700257600ustar00rootroot00000000000000kind: import sources: - kind: remote url: https://example.com/foo.bar executable: true ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b apache-buildstream-27ae392/tests/cachekey/project/sources/remote2.expected000066400000000000000000000001001514607367700267600ustar00rootroot000000000000001f226281aa215e3b2fbb6c301f663cc4bb2257e2c88352e7def71b2d90a2545fapache-buildstream-27ae392/tests/cachekey/project/sources/tar1.bst000066400000000000000000000002431514607367700252510ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://example.com/releases/1.4/foo-1.4.5.tar.gz ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b apache-buildstream-27ae392/tests/cachekey/project/sources/tar1.expected000066400000000000000000000001001514607367700262520ustar00rootroot000000000000007813bf80ae77e66fa7736bd8a5c3eb121a797b2e2069ed5cd598db09e86ff84aapache-buildstream-27ae392/tests/cachekey/project/sources/tar2.bst000066400000000000000000000002631514607367700252540ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://example.com/releases/1.4/foo-1.4.5.tar.gz ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b base-dir: src apache-buildstream-27ae392/tests/cachekey/project/sources/tar2.expected000066400000000000000000000001001514607367700262530ustar00rootroot0000000000000016c0dace226ce67f3b08dad7cd51ed672fb084c7cb74e358771c226490c8cd35apache-buildstream-27ae392/tests/cachekey/project/target.bst000066400000000000000000000013521514607367700242070ustar00rootroot00000000000000kind: stack description: | This is the main entry point including cases in the cache key test. depends: - sources/local1.bst - sources/local2.bst - sources/remote1.bst - sources/remote2.bst - sources/tar1.bst - sources/tar2.bst - elements/build1.bst - elements/build2.bst - elements/build3.bst - elements/compose1.bst - elements/compose2.bst - elements/compose3.bst - elements/compose4.bst - elements/compose5.bst - elements/import1.bst - elements/import2.bst - elements/import3.bst - elements/script1.bst - elements/sort0.bst - elements/sort1.bst - elements/sort2.bst - elements/sort3.bst - elements/sort4.bst - elements/sort5.bst - elements/sort6.bst - elements/sort7.bst - elements/sort8.bst - elements/sort9.bst - elements/variables1.bst apache-buildstream-27ae392/tests/cachekey/project/target.expected000066400000000000000000000001011514607367700252070ustar00rootroot00000000000000ef4f5380ffaa634a6af1177717d874331af1e66cb4d8928611703809b3ee5dab apache-buildstream-27ae392/tests/conftest.py000077500000000000000000000143641514607367700211760ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Maat # from importlib.metadata import entry_points import os import pytest from buildstream._testing import register_repo_kind, sourcetests_collection_hook from buildstream._testing._fixtures import ( # pylint: disable=unused-import default_thread_number, thread_check, ) from buildstream._testing.integration import integration_cache # pylint: disable=unused-import from tests.testutils.repo.tar import Tar # # This file is loaded by pytest, we use it to add a custom # `--integration` option to our test suite, and to install # a session scope fixture. # ################################################# # Implement pytest option # ################################################# def pytest_addoption(parser): parser.addoption("--integration", action="store_true", default=False, help="Run integration tests") parser.addoption("--plugins", action="store_true", default=False, help="Run only plugins tests") parser.addoption("--remote-execution", action="store_true", default=False, help="Run remote-execution tests only") parser.addoption("--remote-cache", action="store_true", default=False, help="Run remote-cache tests only") def pytest_collection_modifyitems(session, config, items): for item in items: # Without --integration: skip tests not marked with 'integration' if not config.getvalue("integration"): if item.get_closest_marker("integration"): item.add_marker(pytest.mark.skip("skipping integration test")) # With --remote-execution: only run tests marked with 'remoteexecution' if config.getvalue("remote_execution"): if not item.get_closest_marker("remoteexecution"): item.add_marker(pytest.mark.skip("skipping non remote-execution test")) # Without --remote-execution: skip tests marked with 'remoteexecution' else: if item.get_closest_marker("remoteexecution"): item.add_marker(pytest.mark.skip("skipping remote-execution test")) # With --remote-cache: only run tests marked with 'remotecache' if config.getvalue("remote_cache"): if not item.get_closest_marker("remotecache"): item.add_marker(pytest.mark.skip("skipping non remote-cache test")) # Without --remote-cache: skip tests marked with 'remotecache' else: if item.get_closest_marker("remotecache"): item.add_marker(pytest.mark.skip("skipping remote-cache test")) # With --plugins only run plugins tests if config.getvalue("plugins"): if not item.get_closest_marker("generic_source_test"): item.add_marker(pytest.mark.skip("Skipping not generic source test")) ################################################# # remote_services fixture # ################################################# # # This is returned by the `remote_services` fixture # class RemoteServices: def __init__(self, **kwargs): self.action_service = kwargs.get("action_service") self.artifact_service = kwargs.get("artifact_service") self.exec_service = kwargs.get("exec_service") self.source_service = kwargs.get("source_service") self.storage_service = kwargs.get("storage_service") self.artifact_index_service = kwargs.get("artifact_index_service") self.artifact_storage_service = kwargs.get("artifact_storage_service") @pytest.fixture(scope="session") def remote_services(request): kwargs = {} # Look for remote services configuration in environment. if "ARTIFACT_CACHE_SERVICE" in os.environ: kwargs["artifact_service"] = os.environ.get("ARTIFACT_CACHE_SERVICE") if "ARTIFACT_INDEX_SERVICE" in os.environ: kwargs["artifact_index_service"] = os.environ.get("ARTIFACT_INDEX_SERVICE") if "ARTIFACT_STORAGE_SERVICE" in os.environ: kwargs["artifact_storage_service"] = os.environ.get("ARTIFACT_STORAGE_SERVICE") if "REMOTE_EXECUTION_SERVICE" in os.environ: kwargs["action_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE") kwargs["exec_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE") kwargs["storage_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE") if "SOURCE_CACHE_SERVICE" in os.environ: kwargs["source_service"] = os.environ.get("SOURCE_CACHE_SERVICE") return RemoteServices(**kwargs) ################################################# # Setup for templated source tests # ################################################# register_repo_kind("tar", Tar, None) # This hook enables pytest to collect the templated source tests from # buildstream._testing def pytest_sessionstart(session): if session.config.getvalue("plugins"): # Enable all plugins that implement the 'buildstream.tests.source_plugins' hook for entrypoint in entry_points(group="buildstream.tests.source_plugins"): module = entrypoint.load() module.register_sources() sourcetests_collection_hook(session) ################################################# # Isolated environment # ################################################# @pytest.fixture(scope="session", autouse=True) def set_xdg_paths(pytestconfig): for env_var, default in [ ("HOME", "tmp"), ("XDG_CACHE_HOME", "tmp/cache"), ("XDG_CONFIG_HOME", "tmp/config"), ("XDG_DATA_HOME", "tmp/share"), ]: value = os.environ.get("BST_TEST_{}".format(env_var)) if value is None: value = os.path.realpath(os.path.join(pytestconfig.getoption("basetemp"), default)) os.environ[env_var] = value apache-buildstream-27ae392/tests/elements/000077500000000000000000000000001514607367700206005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/__init__.py000066400000000000000000000000001514607367700226770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter.py000066400000000000000000000505411514607367700224440ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import create_repo from buildstream._testing import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain from buildstream import _yaml DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "filter", ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_include(datafiles, cli, tmpdir): project = str(datafiles) result = cli.run(project=project, args=["build", "output-include.bst"]) result.assert_success() checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "output-include.bst", "--directory", checkout]) result.assert_success() assert os.path.exists(os.path.join(checkout, "foo")) assert not os.path.exists(os.path.join(checkout, "bar")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_include_dynamic(datafiles, cli, tmpdir): project = str(datafiles) result = cli.run(project=project, args=["build", "output-dynamic-include.bst"]) result.assert_success() checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") result = cli.run( project=project, args=["artifact", "checkout", "output-dynamic-include.bst", "--directory", checkout] ) result.assert_success() assert os.path.exists(os.path.join(checkout, "foo")) assert not os.path.exists(os.path.join(checkout, "bar")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_exclude(datafiles, cli, tmpdir): project = str(datafiles) result = cli.run(project=project, args=["build", "output-exclude.bst"]) result.assert_success() checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "output-exclude.bst", "--directory", checkout]) result.assert_success() assert not os.path.exists(os.path.join(checkout, "foo")) assert os.path.exists(os.path.join(checkout, "bar")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_orphans(datafiles, cli, tmpdir): project = str(datafiles) result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout]) result.assert_success() assert os.path.exists(os.path.join(checkout, "baz")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_deps_ok(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "deps-permitted.bst"]) result.assert_success() result = cli.run(project=project, args=["show", "--deps=run", "--format='%{name}'", "deps-permitted.bst"]) result.assert_success() assert "output-exclude.bst" in result.output assert "output-orphans.bst" in result.output @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_sources(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "forbidden-source.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, "element-forbidden-sources") @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_multi_bdep(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "forbidden-multi-bdep.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-wrong-count") @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_no_bdep(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "forbidden-no-bdep.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-wrong-count") @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_forbid_also_rdep(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "forbidden-also-rdep.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-also-rdepend") @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_open(datafiles, cli, tmpdir): project = str(datafiles) workspace_dir = os.path.join(tmpdir.dirname, tmpdir.basename, "workspace") result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "deps-permitted.bst"]) result.assert_success() assert os.path.exists(os.path.join(workspace_dir, "foo")) assert os.path.exists(os.path.join(workspace_dir, "bar")) assert os.path.exists(os.path.join(workspace_dir, "baz")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_open_multi(datafiles, cli): project = str(datafiles) result = cli.run( cwd=project, project=project, args=["workspace", "open", "deps-permitted.bst", "output-orphans.bst"] ) result.assert_success() assert os.path.exists(os.path.join(project, "input")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_build(datafiles, cli, tmpdir): project = str(datafiles) tempdir = os.path.join(tmpdir.dirname, tmpdir.basename) workspace_dir = os.path.join(tempdir, "workspace") result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"]) result.assert_success() src = os.path.join(workspace_dir, "foo") dst = os.path.join(workspace_dir, "quux") shutil.copyfile(src, dst) result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout_dir = os.path.join(tempdir, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir]) result.assert_success() assert os.path.exists(os.path.join(checkout_dir, "quux")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_close(datafiles, cli, tmpdir): project = str(datafiles) tempdir = os.path.join(tmpdir.dirname, tmpdir.basename) workspace_dir = os.path.join(tempdir, "workspace") result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"]) result.assert_success() src = os.path.join(workspace_dir, "foo") dst = os.path.join(workspace_dir, "quux") shutil.copyfile(src, dst) result = cli.run(project=project, args=["workspace", "close", "deps-permitted.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout_dir = os.path.join(tempdir, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir]) result.assert_success() assert not os.path.exists(os.path.join(checkout_dir, "quux")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_workspace_reset(datafiles, cli, tmpdir): project = str(datafiles) tempdir = os.path.join(tmpdir.dirname, tmpdir.basename) workspace_dir = os.path.join(tempdir, "workspace") result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, "output-orphans.bst"]) result.assert_success() src = os.path.join(workspace_dir, "foo") dst = os.path.join(workspace_dir, "quux") shutil.copyfile(src, dst) result = cli.run(project=project, args=["workspace", "reset", "deps-permitted.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "output-orphans.bst"]) result.assert_success() checkout_dir = os.path.join(tempdir, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "output-orphans.bst", "--directory", checkout_dir]) result.assert_success() assert not os.path.exists(os.path.join(checkout_dir, "quux")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track(datafiles, cli, tmpdir): repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) input_name = "input.bst" project_config = { "name": "filter-track-test", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = {"kind": "filter", "depends": [{"filename": "filter1.bst", "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed assert cli.get_element_state(project, input_name) == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track", "filter2.bst"]) result.assert_success() # Now check that a ref field exists new_input = _yaml.load(input_file, shortname=None) source_node = new_input.get_sequence("sources").mapping_at(0) new_input_ref = source_node.get_str("ref") assert new_input_ref == ref @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_excepted(datafiles, cli, tmpdir): repo = create_repo("tar", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) input_name = "input.bst" project_config = { "name": "filter-track-test", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = {"kind": "filter", "depends": [{"filename": "filter1.bst", "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed assert cli.get_element_state(project, input_name) == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track", "filter2.bst", "--except", "input.bst"]) result.assert_success() # Now check that a ref field exists new_input = _yaml.load(input_file, shortname=None) source_node = new_input.get_sequence("sources").mapping_at(0) assert "ref" not in source_node @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_multi_to_one(datafiles, cli, tmpdir): repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) input_name = "input.bst" project_config = { "name": "filter-track-test", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed assert cli.get_element_state(project, input_name) == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track", "filter1.bst", "filter2.bst"]) result.assert_success() # Now check that a ref field exists new_input = _yaml.load(input_file, shortname=None) source_node = new_input.get_sequence("sources").mapping_at(0) new_ref = source_node.get_str("ref") assert new_ref == ref @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_multi(datafiles, cli, tmpdir): repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) project_config = { "name": "filter-track-test", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } # # Lets do 10 of them at the same time, this may help regression # test against https://github.com/apache/buildstream/issues/1831 # filter_elements = [] input_elements = [] for num in range(0, 10): input_elt = f"input{num}.bst" input_file = os.path.join(elements_dir, input_elt) _yaml.roundtrip_dump(input_config, input_file) filter_config = {"kind": "filter", "depends": [{"filename": input_elt, "type": "build"}]} filter_elt = f"filter{num}.bst" filter_file = os.path.join(elements_dir, filter_elt) _yaml.roundtrip_dump(filter_config, filter_file) input_elements.append(input_elt) filter_elements.append(filter_elt) # Assert that a fetch is needed states = cli.get_element_states(project, input_elements) for key in states: assert states[key] == "no reference" # Now try to track it result = cli.run(project=project, args=["source", "track"] + filter_elements) result.assert_success() # Now check that a ref field exists for elt in input_elements: input_file = os.path.join(elements_dir, elt) new_input = _yaml.load(input_file, shortname=None) source_node = new_input.get_sequence("sources").mapping_at(0) new_ref = source_node.get_str("ref") assert new_ref == ref @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_track_multi_exclude(datafiles, cli, tmpdir): repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(str(datafiles), "files")) elements_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) input_name = "input.bst" input2_name = "input2.bst" project_config = { "name": "filter-track-test", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_file = os.path.join(elements_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) input2_config = dict(input_config) input2_file = os.path.join(elements_dir, input2_name) _yaml.roundtrip_dump(input2_config, input2_file) filter1_config = {"kind": "filter", "depends": [{"filename": input_name, "type": "build"}]} filter1_file = os.path.join(elements_dir, "filter1.bst") _yaml.roundtrip_dump(filter1_config, filter1_file) filter2_config = {"kind": "filter", "depends": [{"filename": input2_name, "type": "build"}]} filter2_file = os.path.join(elements_dir, "filter2.bst") _yaml.roundtrip_dump(filter2_config, filter2_file) # Assert that a fetch is needed states = cli.get_element_states(project, [input_name, input2_name]) assert states == { input_name: "no reference", input2_name: "no reference", } # Now try to track it result = cli.run(project=project, args=["source", "track", "filter1.bst", "filter2.bst", "--except", input_name]) result.assert_success() # Now check that a ref field exists new_input = _yaml.load(input_file, shortname=None) source_node = new_input.get_sequence("sources").mapping_at(0) assert "ref" not in source_node new_input2 = _yaml.load(input2_file, shortname=None) source_node2 = new_input2.get_sequence("sources").mapping_at(0) new_ref2 = source_node2.get_str("ref") assert new_ref2 == ref @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_include_with_indirect_deps(datafiles, cli, tmpdir): project = str(datafiles) result = cli.run(project=project, args=["build", "output-include-with-indirect-deps.bst"]) result.assert_success() checkout = os.path.join(tmpdir.dirname, tmpdir.basename, "checkout") result = cli.run( project=project, args=["artifact", "checkout", "output-include-with-indirect-deps.bst", "--directory", checkout], ) result.assert_success() # direct dependencies should be staged and filtered assert os.path.exists(os.path.join(checkout, "baz")) # indirect dependencies shouldn't be staged and filtered assert not os.path.exists(os.path.join(checkout, "foo")) assert not os.path.exists(os.path.join(checkout, "bar")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_fails_for_nonexisting_domain(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "output-include-nonexistent-domain.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) error = "Unknown domains were used in output-include-nonexistent-domain.bst [line 7 column 2]" assert error in result.stderr assert "- unknown_file" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_pass_integration(datafiles, cli): project = str(datafiles) # Explicitly not passing integration commands should be fine result = cli.run(project=project, args=["build", "no-pass-integration.bst"]) result.assert_success() # Passing integration commands should build nicely result = cli.run(project=project, args=["build", "pass-integration.bst"]) result.assert_success() # Checking out elements which don't pass integration commands should still work checkout_dir = os.path.join(project, "no-pass") result = cli.run( project=project, args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "no-pass-integration.bst"], ) result.assert_success() # Checking out the artifact should fail if we run integration commands, as # the staged artifacts don't have a shell checkout_dir = os.path.join(project, "pass") result = cli.run( project=project, args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "pass-integration.bst"], ) result.assert_main_error(ErrorDomain.STREAM, "missing-command") @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_filter_stack_depend_failure(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["build", "forbidden-stack-dep.bst"]) result.assert_main_error(ErrorDomain.ELEMENT, "filter-bdepend-no-artifact") apache-buildstream-27ae392/tests/elements/filter/000077500000000000000000000000001514607367700220655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/000077500000000000000000000000001514607367700231465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/element_plugins/000077500000000000000000000000001514607367700263405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/element_plugins/dynamic.py000066400000000000000000000016471514607367700303460ustar00rootroot00000000000000from buildstream import Element # Copies files from the dependent element but inserts split-rules using dynamic data class DynamicElement(Element): BST_MIN_VERSION = "2.0" def configure(self, node): node.validate_keys(["split-rules"]) self.split_rules = {key: value.as_str_list() for key, value in node.get_mapping("split-rules").items()} def preflight(self): pass def get_unique_key(self): return {"split-rules": self.split_rules} def configure_sandbox(self, sandbox): pass def stage(self, sandbox): with self.timed_activity("Staging artifact", silent_nested=True): self.stage_dependency_artifacts(sandbox) def assemble(self, sandbox): bstdata = self.get_public_data("bst") bstdata["split-rules"] = self.split_rules self.set_public_data("bst", bstdata) return "" def setup(): return DynamicElement apache-buildstream-27ae392/tests/elements/filter/basic/elements/000077500000000000000000000000001514607367700247625ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/elements/deps-permitted.bst000066400000000000000000000002631514607367700304230ustar00rootroot00000000000000kind: filter depends: - filename: input.bst type: build - filename: output-exclude.bst type: runtime - filename: output-orphans.bst type: runtime config: include: - foo apache-buildstream-27ae392/tests/elements/filter/basic/elements/forbidden-also-rdep.bst000066400000000000000000000002371514607367700313160ustar00rootroot00000000000000kind: filter depends: - filename: output-include.bst type: all - filename: output-exclude.bst type: runtime - filename: output-orphans.bst type: runtime apache-buildstream-27ae392/tests/elements/filter/basic/elements/forbidden-multi-bdep.bst000066400000000000000000000002371514607367700314720ustar00rootroot00000000000000kind: filter depends: - filename: output-include.bst type: build - filename: output-exclude.bst type: build - filename: output-orphans.bst type: runtime apache-buildstream-27ae392/tests/elements/filter/basic/elements/forbidden-no-bdep.bst000066400000000000000000000002431514607367700307510ustar00rootroot00000000000000kind: filter depends: - filename: output-include.bst type: runtime - filename: output-exclude.bst type: runtime - filename: output-orphans.bst type: runtime apache-buildstream-27ae392/tests/elements/filter/basic/elements/forbidden-source.bst000066400000000000000000000002031514607367700307210ustar00rootroot00000000000000kind: filter depends: - filename: output-include.bst type: build config: include: - foo sources: - kind: local path: files apache-buildstream-27ae392/tests/elements/filter/basic/elements/forbidden-stack-dep.bst000066400000000000000000000000721514607367700313000ustar00rootroot00000000000000kind: filter depends: - filename: stack.bst type: build apache-buildstream-27ae392/tests/elements/filter/basic/elements/input-dynamic.bst000066400000000000000000000001721514607367700302550ustar00rootroot00000000000000kind: dynamic depends: - filename: input.bst type: build config: split-rules: foo: - /foo bar: - /bar apache-buildstream-27ae392/tests/elements/filter/basic/elements/input-with-deps.bst000066400000000000000000000002141514607367700305320ustar00rootroot00000000000000kind: import depends: - filename: input.bst sources: - kind: local path: files public: bst: split-rules: baz: - /baz apache-buildstream-27ae392/tests/elements/filter/basic/elements/input.bst000066400000000000000000000002601514607367700266310ustar00rootroot00000000000000kind: import sources: - kind: local path: files public: bst: split-rules: foo: - /foo bar: - /bar integration-commands: - foo - bar apache-buildstream-27ae392/tests/elements/filter/basic/elements/no-pass-integration.bst000066400000000000000000000001341514607367700313730ustar00rootroot00000000000000kind: filter depends: - filename: input.bst type: build config: pass-integration: False apache-buildstream-27ae392/tests/elements/filter/basic/elements/output-dynamic-include.bst000066400000000000000000000001351514607367700320760ustar00rootroot00000000000000kind: filter depends: - filename: input-dynamic.bst type: build config: include: - foo apache-buildstream-27ae392/tests/elements/filter/basic/elements/output-exclude.bst000066400000000000000000000001251514607367700304610ustar00rootroot00000000000000kind: filter depends: - filename: input.bst type: build config: exclude: - foo output-include-nonexistent-domain.bst000066400000000000000000000001371514607367700342200ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/elementskind: filter depends: - filename: input.bst type: build config: include: - unknown_file output-include-with-indirect-deps.bst000066400000000000000000000001051514607367700340730ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/elementskind: filter depends: - filename: input-with-deps.bst type: build apache-buildstream-27ae392/tests/elements/filter/basic/elements/output-include.bst000066400000000000000000000001251514607367700304530ustar00rootroot00000000000000kind: filter depends: - filename: input.bst type: build config: include: - foo apache-buildstream-27ae392/tests/elements/filter/basic/elements/output-orphans.bst000066400000000000000000000001651514607367700305060ustar00rootroot00000000000000kind: filter depends: - filename: input.bst type: build config: exclude: - foo - bar include-orphans: True apache-buildstream-27ae392/tests/elements/filter/basic/elements/pass-integration.bst000066400000000000000000000001331514607367700307600ustar00rootroot00000000000000kind: filter depends: - filename: input.bst type: build config: pass-integration: True apache-buildstream-27ae392/tests/elements/filter/basic/elements/stack.bst000066400000000000000000000000531514607367700265770ustar00rootroot00000000000000kind: stack depends: - filename: input.bst apache-buildstream-27ae392/tests/elements/filter/basic/files/000077500000000000000000000000001514607367700242505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/files/bar000066400000000000000000000000001514607367700247250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/files/baz000066400000000000000000000000001514607367700247350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/files/foo000066400000000000000000000000001514607367700247440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/elements/filter/basic/project.conf000066400000000000000000000001741514607367700254650ustar00rootroot00000000000000name: test min-version: 2.0 element-path: elements plugins: - origin: local path: element_plugins elements: - dynamic apache-buildstream-27ae392/tests/examples/000077500000000000000000000000001514607367700206025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/examples/__init__.py000066400000000000000000000000001514607367700227010ustar00rootroot00000000000000apache-buildstream-27ae392/tests/examples/autotools.py000066400000000000000000000052341514607367700232110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from buildstream._testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "autotools") # Tests a build of the autotools amhello project on a alpine-linux base runtime @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") @pytest.mark.datafiles(DATA_DIR) def test_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Check that the project can be built correctly. result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) result.assert_success() assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # Test running an executable built with autotools. @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") @pytest.mark.datafiles(DATA_DIR) def test_autotools_run(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() result = cli.run(project=project, args=["shell", "hello.bst", "hello"]) result.assert_success() assert result.output == "Hello World!\nThis is amhello 1.0.\n" apache-buildstream-27ae392/tests/examples/developing.py000066400000000000000000000105521514607367700233130ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from buildstream._testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX from tests.testutils import patch pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "developing") # Test that the project builds successfully @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX") @pytest.mark.datafiles(DATA_DIR) def test_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Check that the project can be built correctly. result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) result.assert_success() assert_contains(checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello"]) # Test the unmodified hello command works as expected. @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX") @pytest.mark.datafiles(DATA_DIR) def test_run_unmodified_hello(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() result = cli.run(project=project, args=["shell", "hello.bst", "hello"]) result.assert_success() assert result.output == "Hello World\n" # Test opening a workspace @pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_open_workspace(cli, tmpdir, datafiles): project = str(datafiles) workspace_dir = os.path.join(str(tmpdir), "workspace_hello") result = cli.run( project=project, args=[ "workspace", "open", "-f", "--directory", workspace_dir, "hello.bst", ], ) result.assert_success() result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello.bst"]) result.assert_success() # Test making a change using the workspace @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with SANDBOX") @pytest.mark.datafiles(DATA_DIR) def test_make_change_in_workspace(cli, tmpdir, datafiles): project = str(datafiles) workspace_dir = os.path.join(str(tmpdir), "workspace_hello") result = cli.run(project=project, args=["workspace", "open", "-f", "--directory", workspace_dir, "hello.bst"]) result.assert_success() result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() patch_target = os.path.join(workspace_dir, "hello.c") patch_source = os.path.join(project, "update.patch") patch.apply(patch_target, patch_source) result = cli.run(project=project, args=["build", "hello.bst"]) result.assert_success() result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"]) result.assert_success() assert result.output == "Hello World\nWe can use workspaces!\n" result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello.bst"]) result.assert_success() apache-buildstream-27ae392/tests/examples/first-project.py000066400000000000000000000030741514607367700237530ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from buildstream._testing._utils.site import IS_LINUX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "first-project") @pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_first_project_build_checkout(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", "hello.bst", "--directory", checkout]) assert result.exit_code == 0 assert_contains(checkout, ["/hello.world"]) apache-buildstream-27ae392/tests/examples/integration-commands.py000066400000000000000000000040021514607367700252720ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "integration-commands" ) @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") @pytest.mark.datafiles(DATA_DIR) def test_integration_commands_build(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 # Test running the executable @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") @pytest.mark.datafiles(DATA_DIR) def test_integration_commands_run(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello", "pony"]) assert result.exit_code == 0 assert result.output == "Hello pony\n" apache-buildstream-27ae392/tests/examples/junctions.py000066400000000000000000000052711514607367700231750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "junctions") # Test that the project builds successfully @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap") @pytest.mark.datafiles(DATA_DIR) def test_build(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "callHello.bst"]) result.assert_success() # Test the callHello script works as expected. @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with bubblewrap") @pytest.mark.datafiles(DATA_DIR) def test_shell_call_hello(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "callHello.bst"]) result.assert_success() result = cli.run(project=project, args=["shell", "callHello.bst", "--", "/bin/sh", "callHello.sh"]) result.assert_success() assert result.output == "Calling hello:\nHello World!\nThis is amhello 1.0.\n" # Test opening a cross-junction workspace @pytest.mark.skipif(not IS_LINUX, reason="Only available on linux") @pytest.mark.datafiles(DATA_DIR) def test_open_cross_junction_workspace(cli, tmpdir, datafiles): project = str(datafiles) workspace_dir = os.path.join(str(tmpdir), "workspace_hello_junction") result = cli.run( project=project, args=["workspace", "open", "--directory", workspace_dir, "hello-junction.bst:hello.bst"] ) result.assert_success() result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "hello-junction.bst:hello.bst"]) result.assert_success() apache-buildstream-27ae392/tests/examples/running-commands.py000066400000000000000000000037511514607367700244410ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import IS_LINUX, MACHINE_ARCH, HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..", "doc", "examples", "running-commands") @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") def test_running_commands_build(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 # Test running the executable @pytest.mark.skipif(MACHINE_ARCH != "x86-64", reason="Examples are written for x86-64") @pytest.mark.skipif(not IS_LINUX or not HAVE_SANDBOX, reason="Only available on linux with sandbox") @pytest.mark.datafiles(DATA_DIR) def test_running_commands_run(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "hello.bst"]) assert result.exit_code == 0 result = cli.run(project=project, args=["shell", "hello.bst", "--", "hello"]) assert result.exit_code == 0 assert result.output == "Hello World\n" apache-buildstream-27ae392/tests/format/000077500000000000000000000000001514607367700202545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/__init__.py000066400000000000000000000000001514607367700223530ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/assertion.py000066400000000000000000000041501514607367700226350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "assertion") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,opt_pony,opt_horsy,assertion", [ # Test an unconditional (!) directly in the element ("raw-assertion.bst", "False", "False", "Raw assertion boogey"), # Test an assertion in a conditional ("conditional-assertion.bst", "True", "False", "It's not pony time yet"), # Test that we get the first composited assertion ("ordered-assertion.bst", "True", "True", "It's not horsy time yet"), ], ) def test_assertion_cli(cli, datafiles, target, opt_pony, opt_horsy, assertion): project = str(datafiles) result = cli.run( project=project, silent=True, args=[ "--option", "pony", opt_pony, "--option", "horsy", opt_horsy, "show", "--deps", "none", "--format", "%{vars}", target, ], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.USER_ASSERTION) # Assert that the assertion text provided by the user # is found in the exception text assert assertion in str(result.exception) apache-buildstream-27ae392/tests/format/assertion/000077500000000000000000000000001514607367700222635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/assertion/conditional-assertion.bst000066400000000000000000000002111514607367700272770ustar00rootroot00000000000000kind: manual variables: thepony: "not pony" (?): - pony == True: thepony: "It's a ponay !" (!): It's not pony time yet apache-buildstream-27ae392/tests/format/assertion/ordered-assertion.bst000066400000000000000000000003401514607367700264230ustar00rootroot00000000000000kind: manual variables: thepony: "not pony" (?): - pony == True: thepony: "It's a ponay !" (!): It's not pony time yet - horsy == True: thepony: "It's a horsay !" (!): It's not horsy time yet apache-buildstream-27ae392/tests/format/assertion/project.conf000066400000000000000000000003121514607367700245740ustar00rootroot00000000000000name: test min-version: 2.0 options: pony: type: bool description: Whether a pony or not default: False horsy: type: bool description: Whether a horsy or not default: False apache-buildstream-27ae392/tests/format/assertion/raw-assertion.bst000066400000000000000000000001211514607367700255650ustar00rootroot00000000000000kind: manual variables: thepony: "not pony" (!): | Raw assertion boogey apache-buildstream-27ae392/tests/format/dependencies.py000066400000000000000000000245741514607367700232700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # # Exercising some different ways of loading the dependencies # @pytest.mark.datafiles(DATA_DIR) def test_two_files(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") elements = cli.get_pipeline(project, ["target.bst"]) assert elements == ["firstdep.bst", "target.bst"] @pytest.mark.datafiles(DATA_DIR) def test_shared_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") elements = cli.get_pipeline(project, ["shareddeptarget.bst"]) assert elements == ["firstdep.bst", "shareddep.bst", "shareddeptarget.bst"] @pytest.mark.datafiles(DATA_DIR) def test_dependency_dict(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") elements = cli.get_pipeline(project, ["target-depdict.bst"]) assert elements == ["firstdep.bst", "target-depdict.bst"] @pytest.mark.datafiles(DATA_DIR) def test_invalid_dependency_declaration(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") result = cli.run(project=project, args=["show", "invaliddep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_dependency_type(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") result = cli.run(project=project, args=["show", "invaliddeptype.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_strict_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") result = cli.run(project=project, args=["show", "invalidstrict.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_non_strict_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") result = cli.run(project=project, args=["show", "invalidnonstrict.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_circular_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") result = cli.run(project=project, args=["show", "circulartarget.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_DEPENDENCY) @pytest.mark.datafiles(DATA_DIR) def test_build_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") elements = cli.get_pipeline(project, ["builddep.bst"], scope="run") assert elements == ["builddep.bst"] elements = cli.get_pipeline(project, ["builddep.bst"], scope="build") assert elements == ["firstdep.bst"] @pytest.mark.datafiles(DATA_DIR) def test_runtime_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") elements = cli.get_pipeline(project, ["runtimedep.bst"], scope="build") # FIXME: The empty line should probably never happen here when there are no results. assert elements == [""] elements = cli.get_pipeline(project, ["runtimedep.bst"], scope="run") assert elements == ["firstdep.bst", "runtimedep.bst"] @pytest.mark.datafiles(DATA_DIR) def test_all_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") elements = cli.get_pipeline(project, ["alldep.bst"], scope="build") assert elements == ["firstdep.bst"] elements = cli.get_pipeline(project, ["alldep.bst"], scope="run") assert elements == ["firstdep.bst", "alldep.bst"] @pytest.mark.datafiles(DATA_DIR) def test_list_build_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") # Check that the pipeline includes the build dependency deps = cli.get_pipeline(project, ["builddep-list.bst"], scope="build") assert "firstdep.bst" in deps @pytest.mark.datafiles(DATA_DIR) def test_list_runtime_dependency(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") # Check that the pipeline includes the runtime dependency deps = cli.get_pipeline(project, ["runtimedep-list.bst"], scope="run") assert "firstdep.bst" in deps @pytest.mark.datafiles(DATA_DIR) def test_list_dependencies_combined(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") # Check that runtime deps get combined rundeps = cli.get_pipeline(project, ["list-combine.bst"], scope="run") assert "firstdep.bst" not in rundeps assert "seconddep.bst" in rundeps assert "thirddep.bst" in rundeps # Check that build deps get combined builddeps = cli.get_pipeline(project, ["list-combine.bst"], scope="build") assert "firstdep.bst" in builddeps assert "seconddep.bst" not in builddeps assert "thirddep.bst" in builddeps @pytest.mark.datafiles(DATA_DIR) def test_list_overlap(cli, datafiles): project = os.path.join(str(datafiles), "dependencies1") # Check that dependencies get merged rundeps = cli.get_pipeline(project, ["list-overlap.bst"], scope="run") assert "firstdep.bst" in rundeps builddeps = cli.get_pipeline(project, ["list-overlap.bst"], scope="build") assert "firstdep.bst" in builddeps # # Testing the order of elements reported when iterating through # Element.dependencies() with various scopes. # @pytest.mark.datafiles(DATA_DIR) def test_scope_all(cli, datafiles): project = os.path.join(str(datafiles), "dependencies2") elements = ["target.bst"] element_list = cli.get_pipeline(project, elements, scope="all") assert element_list == [ "build-build.bst", "run-build.bst", "build.bst", "dep-one.bst", "run.bst", "dep-two.bst", "target.bst", ] @pytest.mark.datafiles(DATA_DIR) def test_scope_run(cli, datafiles): project = os.path.join(str(datafiles), "dependencies2") elements = ["target.bst"] element_list = cli.get_pipeline(project, elements, scope="run") assert element_list == [ "dep-one.bst", "run.bst", "dep-two.bst", "target.bst", ] @pytest.mark.datafiles(DATA_DIR) def test_scope_build(cli, datafiles): project = os.path.join(str(datafiles), "dependencies2") elements = ["target.bst"] element_list = cli.get_pipeline(project, elements, scope="build") assert element_list == ["dep-one.bst", "run.bst", "dep-two.bst"] @pytest.mark.datafiles(DATA_DIR) def test_scope_build_of_child(cli, datafiles): project = os.path.join(str(datafiles), "dependencies2") elements = ["target.bst"] element_list = cli.get_pipeline(project, elements, scope="build") # First pass, lets check dep-two element = element_list[2] # Pass two, let's look at these element_list = cli.get_pipeline(project, [element], scope="build") assert element_list == ["run-build.bst", "build.bst"] @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target", [ "merge-separate-lists.bst", "merge-single-list.bst", ], ids=["separate-lists", "single-list"], ) def test_merge(cli, datafiles, target): project = os.path.join(str(datafiles), "dependencies2") # Test both build and run scopes, showing that the two dependencies # have been merged and the run-build.bst is both a runtime and build # time dependency, and is not loaded twice into the build graph. # element_list = cli.get_pipeline(project, [target], scope="build") assert element_list == ["run-build.bst"] element_list = cli.get_pipeline(project, [target], scope="run") assert element_list == ["run-build.bst", target] @pytest.mark.datafiles(DATA_DIR) def test_config_unsupported(cli, datafiles): project = os.path.join(str(datafiles), "dependencies3") result = cli.run(project=project, args=["show", "unsupported.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DEPENDENCY_CONFIG) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,number", [ ("supported1.bst", 1), ("supported2.bst", 2), ], ids=["one", "two"], ) def test_config_supported(cli, datafiles, target, number): project = os.path.join(str(datafiles), "dependencies3") result = cli.run(project=project, args=["show", target]) result.assert_success() assert "TEST PLUGIN FOUND {} ENABLED DEPENDENCIES".format(number) in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_config_runtime_error(cli, datafiles): project = os.path.join(str(datafiles), "dependencies3") # Test that it is considered an error to specify `config` on runtime-only dependencies # result = cli.run(project=project, args=["show", "runtime-error.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,number", [ ("shorthand-config.bst", 2), ("shorthand-junction.bst", 2), ], ids=["config", "junction"], ) def test_shorthand(cli, datafiles, target, number): project = os.path.join(str(datafiles), "dependencies3") result = cli.run(project=project, args=["show", target]) result.assert_success() assert "TEST PLUGIN FOUND {} ENABLED DEPENDENCIES".format(number) in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_invalid_filenames(cli, datafiles): project = os.path.join(str(datafiles), "dependencies3") result = cli.run(project=project, args=["show", "invalid-filenames.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Assert expected provenance assert "invalid-filenames.bst [line 9 column 4]" in result.stderr apache-buildstream-27ae392/tests/format/dependencies1/000077500000000000000000000000001514607367700227635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies1/elements/000077500000000000000000000000001514607367700245775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies1/elements/alldep.bst000066400000000000000000000001721514607367700265520ustar00rootroot00000000000000kind: manual description: This element has a dependency with type 'all' depends: - filename: firstdep.bst type: all apache-buildstream-27ae392/tests/format/dependencies1/elements/builddep-list.bst000066400000000000000000000001751514607367700300550ustar00rootroot00000000000000kind: manual description: This element has a build-only dependency specified via build-depends build-depends: - firstdep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/builddep.bst000066400000000000000000000001671514607367700271050ustar00rootroot00000000000000kind: manual description: This element has a build-only dependency depends: - filename: firstdep.bst type: build apache-buildstream-27ae392/tests/format/dependencies1/elements/circular-firstdep.bst000066400000000000000000000001561514607367700307350ustar00rootroot00000000000000kind: manual description: Depend on another dep which depends on the target depends: - circular-seconddep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/circular-seconddep.bst000066400000000000000000000001551514607367700310600ustar00rootroot00000000000000kind: manual description: Depend on the target, creating a circular dependency depends: - circulartarget.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/circulartarget.bst000066400000000000000000000001671514607367700303300ustar00rootroot00000000000000kind: stack description: This is a main target which introduces a circular dependency depends: - circular-firstdep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/firstdep.bst000066400000000000000000000000671514607367700271340ustar00rootroot00000000000000kind: manual description: This is the first dependency apache-buildstream-27ae392/tests/format/dependencies1/elements/invaliddep.bst000066400000000000000000000001471514607367700274320ustar00rootroot00000000000000kind: pony description: This is an invalid dependency depends: more: it should be a list, not a dict apache-buildstream-27ae392/tests/format/dependencies1/elements/invaliddeptype.bst000066400000000000000000000002061514607367700303300ustar00rootroot00000000000000kind: manual description: This is an invalid dependency type depends: - filename: firstdep.bst type: should be build or runtime apache-buildstream-27ae392/tests/format/dependencies1/elements/invalidnonstrict.bst000066400000000000000000000003621514607367700307040ustar00rootroot00000000000000kind: manual description: | This is an invalid non strict dependency because it is currently illegal to explicitly set a dependency to be non-strict (even though this is the default). depends: - filename: firstdep.bst strict: false apache-buildstream-27ae392/tests/format/dependencies1/elements/invalidstrict.bst000066400000000000000000000002571514607367700301740ustar00rootroot00000000000000kind: manual description: | This is an invalid strict dependency because runtime dependencies cannot be strict. runtime-depends: - filename: firstdep.bst strict: true apache-buildstream-27ae392/tests/format/dependencies1/elements/list-combine.bst000066400000000000000000000002521514607367700276750ustar00rootroot00000000000000kind: manual description: This element depends on three elements in different ways build-depends: - firstdep.bst runtime-depends: - seconddep.bst depends: - thirddep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/list-overlap.bst000066400000000000000000000002401514607367700277260ustar00rootroot00000000000000kind: stack description: This element depends on two elements in different ways build-depends: - firstdep.bst depends: - filename: firstdep.bst type: runtime apache-buildstream-27ae392/tests/format/dependencies1/elements/runtimedep-list.bst000066400000000000000000000001451514607367700304360ustar00rootroot00000000000000kind: manual description: This element has a runtime-only dependency runtime-depends: - firstdep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/runtimedep.bst000066400000000000000000000001731514607367700274660ustar00rootroot00000000000000kind: manual description: This element has a runtime-only dependency depends: - filename: firstdep.bst type: runtime apache-buildstream-27ae392/tests/format/dependencies1/elements/seconddep.bst000066400000000000000000000000701514607367700272520ustar00rootroot00000000000000kind: manual description: This is the second dependency apache-buildstream-27ae392/tests/format/dependencies1/elements/shareddep.bst000066400000000000000000000001161514607367700272460ustar00rootroot00000000000000kind: stack description: This is the first dependency depends: - firstdep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/shareddeptarget.bst000066400000000000000000000001311514607367700304520ustar00rootroot00000000000000kind: stack description: This is the main target depends: - firstdep.bst - shareddep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/target-depdict.bst000066400000000000000000000001241514607367700302060ustar00rootroot00000000000000kind: manual description: This is the main target depends: - filename: firstdep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/target.bst000066400000000000000000000001111514607367700265700ustar00rootroot00000000000000kind: stack description: This is the main target depends: - firstdep.bst apache-buildstream-27ae392/tests/format/dependencies1/elements/thirddep.bst000066400000000000000000000000671514607367700271170ustar00rootroot00000000000000kind: manual description: This is the third dependency apache-buildstream-27ae392/tests/format/dependencies1/project.conf000066400000000000000000000001031514607367700252720ustar00rootroot00000000000000# Basic project name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/dependencies2/000077500000000000000000000000001514607367700227645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies2/build-build.bst000066400000000000000000000001331514607367700256670ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony apache-buildstream-27ae392/tests/format/dependencies2/build.bst000066400000000000000000000002701514607367700245740ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony depends: - filename: build-build.bst type: build - filename: run-build.bst type: runtime apache-buildstream-27ae392/tests/format/dependencies2/dep-one.bst000066400000000000000000000002101514607367700250160ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony depends: - filename: build.bst type: build apache-buildstream-27ae392/tests/format/dependencies2/dep-two.bst000066400000000000000000000002541514607367700250560ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony depends: - filename: build.bst type: build - filename: run.bst type: runtime apache-buildstream-27ae392/tests/format/dependencies2/merge-separate-lists.bst000066400000000000000000000002451514607367700275340ustar00rootroot00000000000000kind: manual description: Depend on the same element twice, as a build and as a runtime dependency build-depends: - run-build.bst runtime-depends: - run-build.bst apache-buildstream-27ae392/tests/format/dependencies2/merge-single-list.bst000066400000000000000000000002771514607367700270330ustar00rootroot00000000000000kind: manual description: Depend on the same element twice, as a build and as a runtime dependency depends: - filename: run-build.bst type: runtime - filename: run-build.bst type: build apache-buildstream-27ae392/tests/format/dependencies2/project.conf000066400000000000000000000001321514607367700252750ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 apache-buildstream-27ae392/tests/format/dependencies2/run-build.bst000066400000000000000000000001331514607367700253740ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony apache-buildstream-27ae392/tests/format/dependencies2/run.bst000066400000000000000000000001331514607367700242770ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony apache-buildstream-27ae392/tests/format/dependencies2/target.bst000066400000000000000000000002001514607367700247540ustar00rootroot00000000000000kind: manual description: Some kinda manual element config: configure-commands: - pony depends: - dep-one.bst - dep-two.bst apache-buildstream-27ae392/tests/format/dependencies3/000077500000000000000000000000001514607367700227655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies3/elements/000077500000000000000000000000001514607367700246015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies3/elements/dep.bst000066400000000000000000000000551514607367700260630ustar00rootroot00000000000000kind: manual description: Some kinda element apache-buildstream-27ae392/tests/format/dependencies3/elements/dep2.bst000066400000000000000000000000551514607367700261450ustar00rootroot00000000000000kind: manual description: Some kinda element apache-buildstream-27ae392/tests/format/dependencies3/elements/invalid-filenames.bst000066400000000000000000000003551514607367700307050ustar00rootroot00000000000000kind: configsupported # Here we test an incorrect type of filename, e.g. a dictionary # depends: - filename: - target-a.bst - target-b.bst - multiple: foo components: bar junction: subproject.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/elements/runtime-error.bst000066400000000000000000000001301514607367700301170ustar00rootroot00000000000000kind: configsupported runtime-depends: - filename: dep.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/elements/shorthand-config.bst000066400000000000000000000002561514607367700305530ustar00rootroot00000000000000kind: configsupported # Here we test specification of multiple files with the same configuration # depends: - filename: - dep.bst - dep2.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/elements/shorthand-junction.bst000066400000000000000000000003201514607367700311270ustar00rootroot00000000000000kind: configsupported # Here we test specification of multiple files through the same junction # depends: - filename: - target-a.bst - target-b.bst junction: subproject.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/elements/subproject.bst000066400000000000000000000000711514607367700274710ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/dependencies3/elements/supported1.bst000066400000000000000000000001201514607367700274120ustar00rootroot00000000000000kind: configsupported depends: - filename: dep.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/elements/supported2.bst000066400000000000000000000006271514607367700274270ustar00rootroot00000000000000kind: configsupported # Here we test that the same dependency will be supplied to the # plugin twice if they are listed twice (even though in this simple # test case, we supply a redundant configuration, in real life it # can be useful to configure the same dependency differently multiple # times) # depends: - filename: dep.bst config: enabled: true - filename: dep.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/elements/unsupported.bst000066400000000000000000000001221514607367700276760ustar00rootroot00000000000000kind: configunsupported depends: - filename: dep.bst config: enabled: true apache-buildstream-27ae392/tests/format/dependencies3/plugins/000077500000000000000000000000001514607367700244465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies3/plugins/configsupported.py000066400000000000000000000011471514607367700302360ustar00rootroot00000000000000from buildstream import Element class ConfigSupported(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def configure_dependencies(self, dependencies): self.configs = [] for dep in dependencies: if dep.config: dep.config.validate_keys(["enabled"]) self.configs.append(dep) self.info("TEST PLUGIN FOUND {} ENABLED DEPENDENCIES".format(len(self.configs))) def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return ConfigSupported apache-buildstream-27ae392/tests/format/dependencies3/plugins/configunsupported.py000066400000000000000000000004461514607367700306020ustar00rootroot00000000000000from buildstream import Element class ConfigUnsupported(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return ConfigUnsupported apache-buildstream-27ae392/tests/format/dependencies3/project.conf000066400000000000000000000002431514607367700253010ustar00rootroot00000000000000# Basic project name: test min-version: 2.0 element-path: elements plugins: - origin: local path: plugins elements: - configsupported - configunsupported apache-buildstream-27ae392/tests/format/dependencies3/subproject/000077500000000000000000000000001514607367700251455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/dependencies3/subproject/project.conf000066400000000000000000000000371514607367700274620ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/dependencies3/subproject/sub.txt000066400000000000000000000000051514607367700264720ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/dependencies3/subproject/target-a.bst000066400000000000000000000000641514607367700273630ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/dependencies3/subproject/target-b.bst000066400000000000000000000000641514607367700273640ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/elementnames.py000066400000000000000000000042451514607367700233100ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.parametrize( "target,domain,reason,provenance", [ # When specifying a bad suffix on the command line we get a different error, we # catch this error earlier on in the load sequence while sorting out element and # artifact names and glob expressions. # ("farm.pony", ErrorDomain.STREAM, "invalid-element-names", None), ('The "quoted" pony.bst', ErrorDomain.LOAD, LoadErrorReason.BAD_CHARACTERS_IN_NAME, None), ( "bad-suffix-dep.bst", ErrorDomain.LOAD, LoadErrorReason.BAD_ELEMENT_SUFFIX, "bad-suffix-dep.bst [line 3 column 2]", ), ( "bad-chars-dep.bst", ErrorDomain.LOAD, LoadErrorReason.BAD_CHARACTERS_IN_NAME, "bad-chars-dep.bst [line 3 column 2]", ), ], ids=["toplevel-bad-suffix", "toplevel-bad-chars", "dependency-bad-suffix", "dependency-bad-chars"], ) @pytest.mark.datafiles(DATA_DIR) def test_invalid_element_names(cli, datafiles, target, domain, reason, provenance): project = os.path.join(str(datafiles), "elementnames") result = cli.run(project=project, silent=True, args=["show", target]) result.assert_main_error(domain, reason) if provenance: assert provenance in result.stderr apache-buildstream-27ae392/tests/format/elementnames/000077500000000000000000000000001514607367700227315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/elementnames/bad-chars-dep.bst000066400000000000000000000000451514607367700260340ustar00rootroot00000000000000kind: stack depends: - .bst apache-buildstream-27ae392/tests/format/elementnames/bad-suffix-dep.bst000066400000000000000000000000441514607367700262370ustar00rootroot00000000000000kind: stack depends: - rainbow.pony apache-buildstream-27ae392/tests/format/elementnames/project.conf000066400000000000000000000001321514607367700252420ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/include.py000066400000000000000000000322521514607367700222550ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import textwrap import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from tests.testutils import generate_junction # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "include") @pytest.mark.datafiles(DATA_DIR) def test_include_project_file(cli, datafiles): project = os.path.join(str(datafiles), "file") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_bool("included") def test_include_missing_file(cli, tmpdir): tmpdir.join("project.conf").write('{"name": "test", "min-version": "2.0"}') element = tmpdir.join("include_missing_file.bst") # Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here # we want to be sure of a stable line and column number. element.write( textwrap.dedent( """ kind: manual "(@)": - nosuch.yaml """ ).strip() ) result = cli.run(project=str(tmpdir), args=["show", str(element.basename)]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Make sure the root cause provenance is in the output. assert "include_missing_file.bst [line 4 column 4]" in result.stderr def test_include_dir(cli, tmpdir): tmpdir.join("project.conf").write('{"name": "test", "min-version": "2.0"}') tmpdir.mkdir("subdir") element = tmpdir.join("include_dir.bst") # Normally we would use dicts and _yaml.roundtrip_dump to write such things, but here # we want to be sure of a stable line and column number. element.write( textwrap.dedent( """ kind: manual "(@)": - subdir/ """ ).strip() ) result = cli.run(project=str(tmpdir), args=["show", str(element.basename)]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.LOADING_DIRECTORY) # Make sure the root cause provenance is in the output. assert "include_dir.bst [line 4 column 4]" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_include_junction_file(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "junction") generate_junction( tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True ) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_bool("included") @pytest.mark.datafiles(DATA_DIR) def test_include_junction_options(cli, datafiles): project = os.path.join(str(datafiles), "options") result = cli.run( project=project, args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("build_arch") == "x86_64" @pytest.mark.datafiles(DATA_DIR) def test_junction_element_partial_project_project(cli, tmpdir, datafiles): """ Junction elements never depend on fully include processed project. """ project = os.path.join(str(datafiles), "junction") subproject_path = os.path.join(project, "subproject") junction_path = os.path.join(project, "junction.bst") repo = create_repo("tar", str(tmpdir)) ref = repo.create(subproject_path) element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, junction_path) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("included", default=None) is None @pytest.mark.datafiles(DATA_DIR) def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles): """ Junction elements never depend on fully include processed project. """ project = os.path.join(str(datafiles), "file_with_subproject") subproject_path = os.path.join(project, "subproject") junction_path = os.path.join(project, "junction.bst") repo = create_repo("tar", str(tmpdir)) ref = repo.create(subproject_path) element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, junction_path) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("included", default=None) is not None @pytest.mark.datafiles(DATA_DIR) def test_include_element_overrides(cli, datafiles): project = os.path.join(str(datafiles), "overrides") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("manual_main_override", default=None) is not None assert loaded.get_str("manual_included_override", default=None) is not None @pytest.mark.datafiles(DATA_DIR) def test_include_element_overrides_composition(cli, datafiles): project = os.path.join(str(datafiles), "overrides") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{config}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str_list("build-commands") == ["first", "second"] @pytest.mark.datafiles(DATA_DIR) def test_list_overide_does_not_fail_upon_first_composition(cli, datafiles): project = os.path.join(str(datafiles), "eventual_overrides") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{public}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) # Assert that the explicitly overwritten public data is present bst = loaded.get_mapping("bst") assert "foo-commands" in bst assert bst.get_str_list("foo-commands") == ["need", "this"] @pytest.mark.datafiles(DATA_DIR) def test_include_element_overrides_sub_include(cli, datafiles): project = os.path.join(str(datafiles), "sub-include") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("included", default=None) is not None @pytest.mark.datafiles(DATA_DIR) def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "overrides-junction") generate_junction( tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True ) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "junction.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("main_override", default=None) is not None assert loaded.get_str("included_override", default=None) is None @pytest.mark.datafiles(DATA_DIR) def test_conditional_in_fragment(cli, datafiles): project = os.path.join(str(datafiles), "conditional") result = cli.run( project=project, args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("size") == "8" @pytest.mark.parametrize( "project_dir", [ "conditional-conflicts-project", "conditional-conflicts-element", "conditional-conflicts-options-included", "conditional-conflicts-complex", "conditional-conflicts-toplevel-precedence", ], ) @pytest.mark.datafiles(DATA_DIR) def test_preserve_conditionals(cli, datafiles, project_dir): project = os.path.join(str(datafiles), project_dir) result = cli.run( project=project, args=["-o", "build_arch", "i586", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("enable-work-around") == "true" assert loaded.get_str("size") == "4" @pytest.mark.datafiles(DATA_DIR) def test_inner(cli, datafiles): project = os.path.join(str(datafiles), "inner") result = cli.run( project=project, args=["-o", "build_arch", "x86_64", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("build_arch") == "x86_64" @pytest.mark.datafiles(DATA_DIR) def test_recursive_include(cli, datafiles): project = os.path.join(str(datafiles), "recursive") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_INCLUDE) assert "line 2 column 2" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_local_to_junction(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "local_to_junction") generate_junction( tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True ) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_bool("included") @pytest.mark.datafiles(DATA_DIR) def test_option_from_junction(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "junction_options") generate_junction( tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True, options={"local_option": "set"}, ) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert not loaded.get_bool("is-default") @pytest.mark.datafiles(DATA_DIR) def test_option_from_junction_element(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "junction_options_element") generate_junction( tmpdir, os.path.join(project, "subproject"), os.path.join(project, "junction.bst"), store_ref=True, options={"local_option": "set"}, ) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert not loaded.get_bool("is-default") @pytest.mark.datafiles(DATA_DIR) def test_option_from_deep_junction(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "junction_options_deep") junction_repo_a = os.path.join(tmpdir, "a") junction_repo_b = os.path.join(tmpdir, "b") generate_junction( junction_repo_a, os.path.join(project, "subproject-2"), os.path.join(project, "subproject-1", "junction-2.bst"), store_ref=True, options={"local_option": "set"}, ) generate_junction( junction_repo_b, os.path.join(project, "subproject-1"), os.path.join(project, "junction-1.bst"), store_ref=True, ) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert not loaded.get_bool("is-default") @pytest.mark.datafiles(DATA_DIR) def test_include_full_path(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "full_path") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("bar") == "red" assert loaded.get_str("foo") == "blue" @pytest.mark.datafiles(DATA_DIR) def test_include_invalid_full_path(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "full_path") result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "invalid.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Make sure the root cause provenance is in the output. assert "invalid.bst [line 4 column 7]" in result.stderr apache-buildstream-27ae392/tests/format/include/000077500000000000000000000000001514607367700216775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/000077500000000000000000000000001514607367700276315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/element.bst000066400000000000000000000000151514607367700317700ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/enable_work_around.yml000066400000000000000000000001531514607367700342130ustar00rootroot00000000000000variables: enable-work-around: "false" (?): - build_arch == "i586": enable-work-around: "true" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/extra_conf.yml000066400000000000000000000001631514607367700325040ustar00rootroot00000000000000(?): - build_arch == "i586": (@): extra_conf_i586.yml - build_arch == "x86_64": (@): extra_conf_x86_64.yml apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/extra_conf_i586.yml000066400000000000000000000000721514607367700332560ustar00rootroot00000000000000variables: (?): - build_arch == "i586": size: 4 apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/extra_conf_x86_64.yml000066400000000000000000000000741514607367700335230ustar00rootroot00000000000000variables: (?): - build_arch == "x86_64": size: 8 apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/options.yml000066400000000000000000000002061514607367700320450ustar00rootroot00000000000000 options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/project.conf000066400000000000000000000001231514607367700321420ustar00rootroot00000000000000name: test min-version: 2.0 (@): - extra_conf.yml - work_around.yml - options.yml apache-buildstream-27ae392/tests/format/include/conditional-conflicts-complex/work_around.yml000066400000000000000000000000751514607367700327100ustar00rootroot00000000000000(?): - build_arch == "i586": (@): enable_work_around.yml apache-buildstream-27ae392/tests/format/include/conditional-conflicts-element/000077500000000000000000000000001514607367700276135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-element/element.bst000066400000000000000000000000721514607367700317550ustar00rootroot00000000000000kind: manual (@): - extra_conf.yml - work_around.yml apache-buildstream-27ae392/tests/format/include/conditional-conflicts-element/extra_conf.yml000066400000000000000000000001601514607367700324630ustar00rootroot00000000000000variables: (?): - build_arch == "i586": size: "4" - build_arch == "x86_64": size: "8" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-element/project.conf000066400000000000000000000002421514607367700321260ustar00rootroot00000000000000name: test min-version: 2.0 options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 apache-buildstream-27ae392/tests/format/include/conditional-conflicts-element/work_around.yml000066400000000000000000000001571514607367700326730ustar00rootroot00000000000000variables: enable-work-around: "false" (?): - build_arch == "i586": enable-work-around: "true" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-options-included/000077500000000000000000000000001514607367700314425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-options-included/element.bst000066400000000000000000000000151514607367700336010ustar00rootroot00000000000000kind: manual extra_conf.yml000066400000000000000000000001601514607367700342330ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-options-includedvariables: (?): - build_arch == "i586": size: "4" - build_arch == "x86_64": size: "8" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-options-included/options.yml000066400000000000000000000002061514607367700336560ustar00rootroot00000000000000 options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 apache-buildstream-27ae392/tests/format/include/conditional-conflicts-options-included/project.conf000066400000000000000000000001231514607367700337530ustar00rootroot00000000000000name: test min-version: 2.0 (@): - options.yml - extra_conf.yml - work_around.yml work_around.yml000066400000000000000000000001571514607367700344430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-options-includedvariables: enable-work-around: "false" (?): - build_arch == "i586": enable-work-around: "true" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-project/000077500000000000000000000000001514607367700276305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-project/element.bst000066400000000000000000000000151514607367700317670ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/conditional-conflicts-project/extra_conf.yml000066400000000000000000000001601514607367700325000ustar00rootroot00000000000000variables: (?): - build_arch == "i586": size: "4" - build_arch == "x86_64": size: "8" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-project/project.conf000066400000000000000000000003171514607367700321460ustar00rootroot00000000000000name: test min-version: 2.0 options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 (@): - extra_conf.yml - work_around.yml apache-buildstream-27ae392/tests/format/include/conditional-conflicts-project/work_around.yml000066400000000000000000000001571514607367700327100ustar00rootroot00000000000000variables: enable-work-around: "false" (?): - build_arch == "i586": enable-work-around: "true" apache-buildstream-27ae392/tests/format/include/conditional-conflicts-toplevel-precedence/000077500000000000000000000000001514607367700321075ustar00rootroot00000000000000element.bst000066400000000000000000000000151514607367700341670ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-toplevel-precedencekind: manual extra_conf.yml000066400000000000000000000001601514607367700347000ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-toplevel-precedencevariables: (?): - build_arch == "i586": size: "4" - build_arch == "x86_64": size: "8" project.conf000066400000000000000000000007221514607367700343460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-toplevel-precedencename: test min-version: 2.0 options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 # The work_around.yml sets this to false in it's conditional # and we set it to true, testing here that the including # fragment still takes precedence over any included fragments. variables: (?): - build_arch == "i586": enable-work-around: "true" (@): - extra_conf.yml - work_around.yml work_around.yml000066400000000000000000000001571514607367700351100ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional-conflicts-toplevel-precedencevariables: enable-work-around: "true" (?): - build_arch == "i586": enable-work-around: "false" apache-buildstream-27ae392/tests/format/include/conditional/000077500000000000000000000000001514607367700242025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/conditional/element.bst000066400000000000000000000000151514607367700263410ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/conditional/extra_conf.yml000066400000000000000000000001601514607367700270520ustar00rootroot00000000000000variables: (?): - build_arch == "i586": size: "4" - build_arch == "x86_64": size: "8" apache-buildstream-27ae392/tests/format/include/conditional/project.conf000066400000000000000000000002731514607367700265210ustar00rootroot00000000000000name: test min-version: 2.0 options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 (@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/eventual_overrides/000077500000000000000000000000001514607367700256045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/eventual_overrides/element.bst000066400000000000000000000002001514607367700277370ustar00rootroot00000000000000kind: manual public: bst: foo-commands: (=): - need - this (@): - extra_conf.yml - extra_conf2.yml apache-buildstream-27ae392/tests/format/include/eventual_overrides/extra_conf.yml000066400000000000000000000001331514607367700304540ustar00rootroot00000000000000public: bst: foo-commands: - wrong - commands variables: included: 'True' apache-buildstream-27ae392/tests/format/include/eventual_overrides/extra_conf2.yml000066400000000000000000000000371514607367700305410ustar00rootroot00000000000000variables: included: 'False' apache-buildstream-27ae392/tests/format/include/eventual_overrides/project.conf000066400000000000000000000000341514607367700301160ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/include/file/000077500000000000000000000000001514607367700226165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/file/element.bst000066400000000000000000000000151514607367700247550ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/file/extra_conf.yml000066400000000000000000000000361514607367700254700ustar00rootroot00000000000000variables: included: 'True' apache-buildstream-27ae392/tests/format/include/file/project.conf000066400000000000000000000000651514607367700251340ustar00rootroot00000000000000name: test min-version: 2.0 (@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/file_with_subproject/000077500000000000000000000000001514607367700261115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/file_with_subproject/element.bst000066400000000000000000000000151514607367700302500ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/file_with_subproject/extra_conf.yml000066400000000000000000000000361514607367700307630ustar00rootroot00000000000000variables: included: 'True' apache-buildstream-27ae392/tests/format/include/file_with_subproject/project.bst000066400000000000000000000000611514607367700302660ustar00rootroot00000000000000name: test (@): - junction.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/file_with_subproject/project.conf000066400000000000000000000000651514607367700304270ustar00rootroot00000000000000name: test min-version: 2.0 (@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/file_with_subproject/subproject/000077500000000000000000000000001514607367700302715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/file_with_subproject/subproject/project.conf000066400000000000000000000000401514607367700326000ustar00rootroot00000000000000name: test-sub min-version: 2.0 apache-buildstream-27ae392/tests/format/include/full_path/000077500000000000000000000000001514607367700236555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/full_path/elements/000077500000000000000000000000001514607367700254715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/full_path/elements/element.bst000066400000000000000000000001661514607367700276370ustar00rootroot00000000000000kind: manual variables: (@): - subproject.bst:sub.yaml - subproject.bst:subsubproject-junction.bst:subsub.yaml apache-buildstream-27ae392/tests/format/include/full_path/elements/invalid.bst000066400000000000000000000001241514607367700276260ustar00rootroot00000000000000kind: manual variables: (@): subproject.bst:subsubproject-junction.bst:pony.yaml apache-buildstream-27ae392/tests/format/include/full_path/elements/subproject.bst000066400000000000000000000000721514607367700303620ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/include/full_path/project.conf000066400000000000000000000000661514607367700261740ustar00rootroot00000000000000name: simple min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/include/full_path/subproject/000077500000000000000000000000001514607367700260355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/full_path/subproject/elements/000077500000000000000000000000001514607367700276515ustar00rootroot00000000000000subsubproject-junction.bst000066400000000000000000000000751514607367700350270ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/full_path/subproject/elementskind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/include/full_path/subproject/files/000077500000000000000000000000001514607367700271375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/full_path/subproject/files/hello.txt000066400000000000000000000000061514607367700307770ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/format/include/full_path/subproject/project.conf000066400000000000000000000000721514607367700303510ustar00rootroot00000000000000name: subproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/include/full_path/subproject/sub.yaml000066400000000000000000000000131514607367700275040ustar00rootroot00000000000000bar: "red" apache-buildstream-27ae392/tests/format/include/full_path/subproject/subsubproject/000077500000000000000000000000001514607367700307275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/full_path/subproject/subsubproject/project.conf000066400000000000000000000000451514607367700332430ustar00rootroot00000000000000name: subsubproject min-version: 2.0 apache-buildstream-27ae392/tests/format/include/full_path/subproject/subsubproject/subsub.yaml000066400000000000000000000000141514607367700331110ustar00rootroot00000000000000foo: "blue" apache-buildstream-27ae392/tests/format/include/inner/000077500000000000000000000000001514607367700230125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/inner/element.bst000066400000000000000000000000151514607367700251510ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/inner/extra_conf.yml000066400000000000000000000001561514607367700256670ustar00rootroot00000000000000build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 apache-buildstream-27ae392/tests/format/include/inner/project.conf000066400000000000000000000001021514607367700253200ustar00rootroot00000000000000name: test min-version: 2.0 options: (@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/junction/000077500000000000000000000000001514607367700235305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction/element.bst000066400000000000000000000000151514607367700256670ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/junction/project.conf000066400000000000000000000001021514607367700260360ustar00rootroot00000000000000name: test min-version: 2.0 (@): - junction.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/junction/subproject/000077500000000000000000000000001514607367700257105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction/subproject/extra_conf.yml000066400000000000000000000000361514607367700305620ustar00rootroot00000000000000variables: included: 'True' apache-buildstream-27ae392/tests/format/include/junction/subproject/project.conf000066400000000000000000000000401514607367700302170ustar00rootroot00000000000000name: test-sub min-version: 2.0 apache-buildstream-27ae392/tests/format/include/junction_options/000077500000000000000000000000001514607367700253035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options/element.bst000066400000000000000000000000151514607367700274420ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/junction_options/project.conf000066400000000000000000000001021514607367700276110ustar00rootroot00000000000000name: test min-version: 2.0 (@): - junction.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/junction_options/subproject/000077500000000000000000000000001514607367700274635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options/subproject/extra_conf.yml000066400000000000000000000002141514607367700323330ustar00rootroot00000000000000(?): - local_option == 'default': variables: is-default: 'True' - local_option == 'set': variables: is-default: 'False' apache-buildstream-27ae392/tests/format/include/junction_options/subproject/project.conf000066400000000000000000000002721514607367700320010ustar00rootroot00000000000000name: test-sub min-version: 2.0 options: local_option: type: enum description: Testing variable: local_option default: default values: - default - set apache-buildstream-27ae392/tests/format/include/junction_options_deep/000077500000000000000000000000001514607367700263005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options_deep/element.bst000066400000000000000000000000151514607367700304370ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/junction_options_deep/project.conf000066400000000000000000000001041514607367700306100ustar00rootroot00000000000000name: test min-version: 2.0 (@): - junction-1.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/junction_options_deep/subproject-1/000077500000000000000000000000001514607367700306165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options_deep/subproject-1/extra_conf.yml000066400000000000000000000000451514607367700334700ustar00rootroot00000000000000(@): junction-2.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/junction_options_deep/subproject-1/project.conf000066400000000000000000000000421514607367700331270ustar00rootroot00000000000000name: test-sub-1 min-version: 2.0 apache-buildstream-27ae392/tests/format/include/junction_options_deep/subproject-2/000077500000000000000000000000001514607367700306175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options_deep/subproject-2/extra_conf.yml000066400000000000000000000002141514607367700334670ustar00rootroot00000000000000(?): - local_option == 'default': variables: is-default: 'True' - local_option == 'set': variables: is-default: 'False' apache-buildstream-27ae392/tests/format/include/junction_options_deep/subproject-2/project.conf000066400000000000000000000002741514607367700331370ustar00rootroot00000000000000name: test-sub-2 min-version: 2.0 options: local_option: type: enum description: Testing variable: local_option default: default values: - default - set apache-buildstream-27ae392/tests/format/include/junction_options_element/000077500000000000000000000000001514607367700270145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options_element/element.bst000066400000000000000000000000631514607367700311560ustar00rootroot00000000000000kind: manual (@): - junction.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/junction_options_element/project.conf000066400000000000000000000000341514607367700313260ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/include/junction_options_element/subproject/000077500000000000000000000000001514607367700311745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/junction_options_element/subproject/extra_conf.yml000066400000000000000000000002141514607367700340440ustar00rootroot00000000000000(?): - local_option == 'default': variables: is-default: 'True' - local_option == 'set': variables: is-default: 'False' apache-buildstream-27ae392/tests/format/include/junction_options_element/subproject/project.conf000066400000000000000000000002721514607367700335120ustar00rootroot00000000000000name: test-sub min-version: 2.0 options: local_option: type: enum description: Testing variable: local_option default: default values: - default - set apache-buildstream-27ae392/tests/format/include/local_to_junction/000077500000000000000000000000001514607367700254045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/local_to_junction/element.bst000066400000000000000000000000151514607367700275430ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/local_to_junction/project.conf000066400000000000000000000001021514607367700277120ustar00rootroot00000000000000name: test min-version: 2.0 (@): - junction.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/local_to_junction/subproject/000077500000000000000000000000001514607367700275645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/local_to_junction/subproject/extra_conf.yml000066400000000000000000000000261514607367700324350ustar00rootroot00000000000000(@): - internal.yml apache-buildstream-27ae392/tests/format/include/local_to_junction/subproject/internal.yml000066400000000000000000000000361514607367700321220ustar00rootroot00000000000000variables: included: 'True' apache-buildstream-27ae392/tests/format/include/local_to_junction/subproject/project.conf000066400000000000000000000000401514607367700320730ustar00rootroot00000000000000name: test-sub min-version: 2.0 apache-buildstream-27ae392/tests/format/include/options/000077500000000000000000000000001514607367700233725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/options/element.bst000066400000000000000000000000151514607367700255310ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/options/extra_conf.yml000066400000000000000000000002051514607367700262420ustar00rootroot00000000000000options: build_arch: type: arch description: Architecture variable: build_arch values: - i586 - x86_64 apache-buildstream-27ae392/tests/format/include/options/project.conf000066400000000000000000000000651514607367700257100ustar00rootroot00000000000000name: test min-version: 2.0 (@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/overrides-junction/000077500000000000000000000000001514607367700255305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/overrides-junction/element.bst000066400000000000000000000000151514607367700276670ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/overrides-junction/project.conf000066400000000000000000000004461514607367700300510ustar00rootroot00000000000000name: test min-version: 2.0 elements: junction: variables: main_override: True manual: variables: manual_main_override: True config: build-commands: - "first" sources: git: variables: from_main: True (@): - junction.bst:extra_conf.yml apache-buildstream-27ae392/tests/format/include/overrides-junction/subproject/000077500000000000000000000000001514607367700277105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/overrides-junction/subproject/extra_conf.yml000066400000000000000000000003771514607367700325720ustar00rootroot00000000000000elements: junction: variables: included_override: True manual: variables: manual_included_override: True config: build-commands: (>): - "second" sources: git: variables: from_included: True apache-buildstream-27ae392/tests/format/include/overrides-junction/subproject/project.conf000066400000000000000000000000401514607367700322170ustar00rootroot00000000000000name: test-sub min-version: 2.0 apache-buildstream-27ae392/tests/format/include/overrides/000077500000000000000000000000001514607367700237015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/overrides/element.bst000066400000000000000000000000151514607367700260400ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/overrides/extra_conf.yml000066400000000000000000000003611514607367700265540ustar00rootroot00000000000000elements: junction: variables: included_override: True manual: variables: manual_included_override: True config: build-commands: - "ignored" sources: git: variables: from_included: True apache-buildstream-27ae392/tests/format/include/overrides/extra_conf2.yml000066400000000000000000000001101514607367700266260ustar00rootroot00000000000000elements: manual: config: build-commands: - "first" apache-buildstream-27ae392/tests/format/include/overrides/project.conf000066400000000000000000000004751514607367700262240ustar00rootroot00000000000000name: test min-version: 2.0 elements: junction: variables: main_override: True manual: variables: manual_main_override: True config: build-commands: (>): - "second" sources: git: variables: from_main: True (@): - extra_conf.yml - extra_conf2.yml apache-buildstream-27ae392/tests/format/include/overrides/subproject/000077500000000000000000000000001514607367700260615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/overrides/subproject/project.conf000066400000000000000000000000401514607367700303700ustar00rootroot00000000000000name: test-sub min-version: 2.0 apache-buildstream-27ae392/tests/format/include/recursive/000077500000000000000000000000001514607367700237065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/recursive/element.bst000066400000000000000000000000151514607367700260450ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/recursive/extra_conf.yml000066400000000000000000000000311514607367700265530ustar00rootroot00000000000000(@): - extra_conf2.yml apache-buildstream-27ae392/tests/format/include/recursive/extra_conf2.yml000066400000000000000000000000301514607367700266340ustar00rootroot00000000000000(@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/recursive/project.conf000066400000000000000000000000651514607367700262240ustar00rootroot00000000000000name: test min-version: 2.0 (@): - extra_conf.yml apache-buildstream-27ae392/tests/format/include/string/000077500000000000000000000000001514607367700232055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/string/element.bst000066400000000000000000000000151514607367700253440ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/string/extra_conf.yml000066400000000000000000000000361514607367700260570ustar00rootroot00000000000000variables: included: 'True' apache-buildstream-27ae392/tests/format/include/string/project.conf000066400000000000000000000000611514607367700255170ustar00rootroot00000000000000name: test min-version: 2.0 (@): extra_conf.yml apache-buildstream-27ae392/tests/format/include/sub-include/000077500000000000000000000000001514607367700241115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/include/sub-include/element.bst000066400000000000000000000000151514607367700262500ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/include/sub-include/manual_conf.yml000066400000000000000000000000341514607367700271130ustar00rootroot00000000000000variables: included: True apache-buildstream-27ae392/tests/format/include/sub-include/project.conf000066400000000000000000000001221514607367700264210ustar00rootroot00000000000000name: test min-version: 2.0 elements: manual: (@): - manual_conf.yml apache-buildstream-27ae392/tests/format/include_composition.py000066400000000000000000000117741514607367700247060ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from contextlib import contextmanager from buildstream._project import Project from buildstream._includes import Includes from buildstream import _yaml from tests.testutils import dummy_context @contextmanager def make_includes(basedir): _yaml.roundtrip_dump({"name": "test", "min-version": "2.0"}, os.path.join(basedir, "project.conf")) with dummy_context() as context: project = Project(basedir, context) loader = project.loader yield Includes(loader) def test_main_has_priority(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": ["main"]}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) includes.process(main) assert main.get_str_list("test") == ["main"] def test_include_cannot_append(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": ["main"]}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": {"(>)": ["a"]}}, str(tmpdir.join("a.yml"))) includes.process(main) assert main.get_str_list("test") == ["main"] def test_main_can_append(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml"], "test": {"(>)": ["main"]}}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) includes.process(main) assert main.get_str_list("test") == ["a", "main"] def test_sibling_cannot_append_backward(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": {"(>)": ["a"]}}, str(tmpdir.join("a.yml"))) _yaml.roundtrip_dump({"test": ["b"]}, str(tmpdir.join("b.yml"))) includes.process(main) assert main.get_str_list("test") == ["b"] def test_sibling_can_append_forward(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) _yaml.roundtrip_dump({"test": {"(>)": ["b"]}}, str(tmpdir.join("b.yml"))) includes.process(main) assert main.get_str_list("test") == ["a", "b"] def test_lastest_sibling_has_priority(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml", "b.yml"]}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) _yaml.roundtrip_dump({"test": ["b"]}, str(tmpdir.join("b.yml"))) includes.process(main) assert main.get_str_list("test") == ["b"] def test_main_keeps_keys(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump({"(@)": ["a.yml"], "something": "else"}, str(tmpdir.join("main.yml"))) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) _yaml.roundtrip_dump({"test": ["a"]}, str(tmpdir.join("a.yml"))) includes.process(main) assert main.get_str_list("test") == ["a"] assert main.get_str("something") == "else" def test_overwrite_directive_on_later_composite(tmpdir): with make_includes(str(tmpdir)) as includes: _yaml.roundtrip_dump( {"(@)": ["a.yml", "b.yml"], "test": {"(=)": ["Overwritten"]}}, str(tmpdir.join("main.yml")) ) main = _yaml.load(str(tmpdir.join("main.yml")), shortname=None) # a.yml _yaml.roundtrip_dump( {"test": ["some useless", "list", "to be overwritten"], "foo": "should not be present"}, str(tmpdir.join("a.yml")), ) # b.yaml isn't going to have a 'test' node to overwrite _yaml.roundtrip_dump({"foo": "should be present"}, str(tmpdir.join("b.yml"))) includes.process(main) assert main.get_str_list("test") == ["Overwritten"] assert main.get_str("foo") == "should be present" apache-buildstream-27ae392/tests/format/invalid-keys/000077500000000000000000000000001514607367700226535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/invalid-keys/included-source.bst000066400000000000000000000000601514607367700264460ustar00rootroot00000000000000kind: import sources: - (@): somesource.yaml apache-buildstream-27ae392/tests/format/invalid-keys/no-path-specified.bst000066400000000000000000000000471514607367700266650ustar00rootroot00000000000000kind: import sources: - kind: local apache-buildstream-27ae392/tests/format/invalid-keys/optional-source.bst000066400000000000000000000001121514607367700265020ustar00rootroot00000000000000kind: import sources: - (?): - not thingy: kind: local apache-buildstream-27ae392/tests/format/invalid-keys/project.conf000066400000000000000000000001541514607367700251700ustar00rootroot00000000000000name: test min-version: 2.0 options: thingy: type: bool description: A thingy default: false apache-buildstream-27ae392/tests/format/invalid-keys/somesource.yaml000066400000000000000000000000141514607367700257160ustar00rootroot00000000000000kind: local apache-buildstream-27ae392/tests/format/invalid_keys.py000066400000000000000000000031221514607367700233050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "invalid-keys") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( ("element", "location"), [ ("no-path-specified.bst", "line 4 column 4"), ("optional-source.bst", "line 6 column 10"), ("included-source.bst", "line 4 column 4"), ], ) def test_compositied_node_fails_usefully(cli, datafiles, element, location): project = str(datafiles) result = cli.run(project=project, args=["show", element]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) assert "synthetic node" not in result.stderr assert "{} [{}]: Dictionary did not contain expected key 'path'".format(element, location) in result.stderr apache-buildstream-27ae392/tests/format/junctions.py000066400000000000000000001066511514607367700226530ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from tests.testutils import generate_junction DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "junctions", ) def update_project(project_path, updated_configuration): project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf.update(updated_configuration) _yaml.roundtrip_dump(project_conf, project_conf_path) # # Test behavior of `bst show` on a junction element # @pytest.mark.datafiles(DATA_DIR) def test_simple_show(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "simple") assert cli.get_element_state(project, "subproject.bst") == "junction" # # Test that we can build build a pipeline with a junction # @pytest.mark.datafiles(DATA_DIR) def test_simple_build(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "simple") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file from the subproject assert os.path.exists(os.path.join(checkoutdir, "base.txt")) # # Test failure when there is a missing project.conf # @pytest.mark.datafiles(DATA_DIR) def test_junction_missing_project_conf(cli, datafiles): project = os.path.join(str(datafiles), "simple") # Just remove the project.conf from the simple test and assert the error os.remove(os.path.join(project, "subproject", "project.conf")) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) assert "target.bst [line 4 column 2]" in result.stderr # # Test failure when there is a missing project.conf in a workspaced junction # @pytest.mark.datafiles(DATA_DIR) def test_workspaced_junction_missing_project_conf(cli, datafiles): project = os.path.join(str(datafiles), "simple") workspace_dir = os.path.join(project, "workspace") result = cli.run(project=project, args=["workspace", "open", "subproject.bst", "--directory", workspace_dir]) result.assert_success() # Remove the project.conf from the workspace directory os.remove(os.path.join(workspace_dir, "project.conf")) # Assert the same missing project.conf error result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # Assert that we have the expected provenance encoded into the error assert "target.bst [line 4 column 2]" in result.stderr # # Test successful builds of deeply nested targets # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expected", [ ("target.bst", ["sub.txt", "subsub.txt"]), ("deeptarget.bst", ["sub.txt", "subsub.txt", "subsubsub.txt"]), ], ids=["simple", "deep"], ) def test_nested(cli, tmpdir, datafiles, target, expected): project = os.path.join(str(datafiles), "nested") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from all subprojects for filename in expected: assert os.path.exists(os.path.join(checkoutdir, filename)) # # Test successful builds of nested junctions with deep includes # @pytest.mark.datafiles(DATA_DIR) def test_nested_include(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "nested-include") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from all subprojects for filename in ["sub.txt", "subsub.txt"]: assert os.path.exists(os.path.join(checkoutdir, filename)) # # Test successful builds of deeply nested targets with both the top-level project # and a subproject configured to use `project.refs` # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expected", [ ("target.bst", ["sub.txt", "subsub.txt"]), ("deeptarget.bst", ["sub.txt", "subsub.txt", "subsubsub.txt"]), ], ids=["simple", "deep"], ) def test_nested_ref_storage(cli, tmpdir, datafiles, target, expected): project = os.path.join(str(datafiles), "nested") subproject = os.path.join(project, "subproject") subsubproject = os.path.join(subproject, "subsubproject") checkoutdir = os.path.join(str(tmpdir), "checkout") # Configure both the top-level project and the subproject to use project.refs / junction.refs update_project(project, {"ref-storage": "project.refs"}) update_project(subproject, {"ref-storage": "project.refs"}) # Move the subsubproject from a local directory to a repo to test `junction.refs` in the subproject subsubref = generate_junction( tmpdir, subsubproject, os.path.join(subproject, "subsubproject.bst"), store_ref=False ) shutil.rmtree(subsubproject) # Store ref to the subsubproject repo in the subproject's `junction.refs` project_refs = {"projects": {"subtest": {"subsubproject.bst": [{"ref": subsubref}]}}} _yaml.roundtrip_dump(project_refs, os.path.join(subproject, "junction.refs")) # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from all subprojects for filename in expected: assert os.path.exists(os.path.join(checkoutdir, filename)) # # Test missing elements/junctions in subprojects # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,provenance", [ ("target.bst", "target.bst [line 4 column 2]"), ("sub-target.bst", "junction-A.bst:target.bst [line 4 column 2]"), ("bad-junction.bst", "bad-junction.bst [line 3 column 2]"), ("sub-target-bad-junction.bst", "junction-A.bst:bad-junction-target.bst [line 4 column 2]"), ], ids=["subproject-target", "subsubproject-target", "local-junction", "subproject-junction"], ) def test_missing_files(cli, datafiles, target, provenance): project = os.path.join(str(datafiles), "missing-element") result = cli.run(project=project, args=["show", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error assert provenance in result.stderr # # Test various invalid junction configuraions # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,domain,reason,provenance", [ # Test a junction which itself has dependencies ( "junction-with-deps.bst", ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION, "base-with-deps.bst [line 6 column 2]", ), # Test having a dependency directly on a junction ("junction-dep.bst", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "junction-dep.bst [line 3 column 2]"), # Test that we error correctly when we junction-depend on a non-junction ( "junctiondep-not-a-junction.bst", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "junctiondep-not-a-junction.bst [line 3 column 2]", ), # Test that overriding a subproject junction with the junction # declaring the override itself will result in an error ( "target-self-override.bst", ErrorDomain.ELEMENT, "override-junction-with-self", "subproject-self-override.bst [line 16 column 20]", ), ], ids=["junction-with-deps", "deps-on-junction", "use-element-as-junction", "override-with-self"], ) def test_invalid(cli, datafiles, target, domain, reason, provenance): project = os.path.join(str(datafiles), "invalid") result = cli.run(project=project, args=["build", target]) result.assert_main_error(domain, reason) assert provenance in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expect_exists,expect_not_exists", [ ("target-default.bst", "pony.txt", "horsy.txt"), ("target-explicit.bst", "horsy.txt", "pony.txt"), ], ids=["check-values", "set-explicit-values"], ) def test_options(cli, tmpdir, datafiles, target, expect_exists, expect_not_exists): project = os.path.join(str(datafiles), "options") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() assert os.path.exists(os.path.join(checkoutdir, expect_exists)) assert not os.path.exists(os.path.join(checkoutdir, expect_not_exists)) # # Test propagation of options through a junction # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "animal,expect_exists,expect_not_exists", [ ("pony", "pony.txt", "horsy.txt"), ("horsy", "horsy.txt", "pony.txt"), ], ids=["pony", "horsy"], ) def test_options_propagate(cli, tmpdir, datafiles, animal, expect_exists, expect_not_exists): project = os.path.join(str(datafiles), "options") checkoutdir = os.path.join(str(tmpdir), "checkout") update_project( project, { "options": { "animal": { "type": "enum", "description": "The kind of animal", "values": ["pony", "horsy"], "default": "pony", "variable": "animal", } } }, ) # Build, checkout result = cli.run(project=project, args=["--option", "animal", animal, "build", "target-propagate.bst"]) result.assert_success() result = cli.run( project=project, args=[ "--option", "animal", animal, "artifact", "checkout", "target-propagate.bst", "--directory", checkoutdir, ], ) result.assert_success() assert os.path.exists(os.path.join(checkoutdir, expect_exists)) assert not os.path.exists(os.path.join(checkoutdir, expect_not_exists)) # # A lot of testing is using local sources for the junctions for # speed and convenience, however there are some internal optimizations # for local sources, so we need to test some things using a real # source which involves triggering fetches. # # We use the tar source for this since it is a core plugin. # @pytest.mark.datafiles(DATA_DIR) def test_tar_show(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "use-repo") # Create the repo from 'baserepo' subdir repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project, "baserepo")) # Write out junction element with tar source element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(project, "base.bst")) # Check that bst show succeeds with implicit subproject fetching and the # pipeline includes the subproject element element_list = cli.get_pipeline(project, ["target.bst"]) assert "base.bst:target.bst" in element_list @pytest.mark.datafiles(DATA_DIR) def test_tar_build(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "use-repo") checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the repo from 'baserepo' subdir repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project, "baserepo")) # Write out junction element with tar source element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(project, "base.bst")) # Build (with implicit fetch of subproject), checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file from the subproject assert os.path.exists(os.path.join(checkoutdir, "base.txt")) @pytest.mark.datafiles(DATA_DIR) def test_tar_missing_project_conf(cli, tmpdir, datafiles): project = datafiles / "use-repo" # Remove the project.conf from this repo os.remove(datafiles / "use-repo" / "baserepo" / "project.conf") # Create the repo from 'base' subdir repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project, "baserepo")) # Write out junction element with tar source element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, str(project / "base.bst")) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # Assert that we have the expected provenance encoded into the error assert "target.bst [line 3 column 2]" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_build_tar_cross_junction_names(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "use-repo") checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the repo from 'base' subdir repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project, "baserepo")) # Write out junction element with tar source element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(project, "base.bst")) # Build (with implicit fetch of subproject), checkout result = cli.run(project=project, args=["build", "base.bst:target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "base.bst:target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from both projects assert os.path.exists(os.path.join(checkoutdir, "base.txt")) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target", [ "junction-full-path.bst", "element-full-path.bst", "subproject.bst:subsubproject.bst:subsubsubproject.bst:target.bst", ], ids=["junction", "element", "command-line"], ) def test_full_path(cli, tmpdir, datafiles, target): project = os.path.join(str(datafiles), "full-path") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file from base assert os.path.exists(os.path.join(checkoutdir, "subsubsub.txt")) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,provenance", [ ("junction-full-path-notfound.bst", "junction-full-path-notfound.bst [line 3 column 2]"), ("element-full-path-notfound.bst", "element-full-path-notfound.bst [line 3 column 2]"), ("subproject.bst:subsubproject.bst:pony.bst", None), ], ids=["junction", "element", "command-line"], ) def test_full_path_not_found(cli, tmpdir, datafiles, target, provenance): project = os.path.join(str(datafiles), "full-path") # Build result = cli.run(project=project, args=["build", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Check that provenance was provided if expected if provenance: assert provenance in result.stderr # # Test overridding elements # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expected", [ # Override an element in a subproject, this dependency will depend on # the same element in the subproject as the overridden element did. ("override-subproject-element.bst", ["element.txt", "subelement-override.txt", "subdep.txt"]), # Override an element in a subproject while depending on an element which depends # on the overridden element, in this case we ensure that the reverse dependencies # of the replaced element are built against the replacement. ("override-subproject-dep.bst", ["element.txt", "sub.txt", "subdep-override.txt"]), # Override an element in a subproject with a local link element which points to another # element in the same subproject. ("override-subproject-element-with-link.bst", ["element.txt", "sub-alternative.txt", "subdep.txt"]), # Override a link to an element in a subproject with an alternative element # in the same subproject. ("override-subproject-element-using-link.bst", ["element.txt", "sub-alternative.txt", "subdep.txt"]), # Override an element in a nested subsubproject, where the intermediate project also overrides # the same element ("override-subsubproject.bst", ["element.txt", "subsub.txt", "subdep-override.txt"]), ], ids=[ "element-with-deps", "dependency-of-element", "with-link", "using-link", "priority", ], ) def test_override_element(cli, tmpdir, datafiles, target, expected): project = os.path.join(str(datafiles), "override-element") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file(s) for expect in expected: assert os.path.exists(os.path.join(checkoutdir, expect)) # # Test overridding junctions # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expected", [ # Test that we can override a subproject junction of a subproject ("target-overridden-subsubproject.bst", "subsubsub.txt"), # Test that we can override a subproject junction of a subproject, when that junction is a link ("target-overridden-subsubproject-link.bst", "subsubsub.txt"), # Test that we can override a subproject junction of a subproject's subproject ("target-overridden-subsubsubproject.bst", "surprise.txt"), # Test that we can override a subproject junction of a subproject's subproject, which using links to address them ("target-overridden-subsubsubproject-link.bst", "surprise.txt"), # Test that we can override a subproject junction of a subproject's subproject, using various levels of links indirection ("target-overridden-subsubsubproject-indirect-link.bst", "surprise.txt"), # Test that we can override a subproject junction with a deep subproject path ("target-overridden-with-deepsubproject.bst", "deepsurprise.txt"), ], ids=[ "override-subproject", "override-subproject-link", "override-subsubproject", "override-subsubproject-link", "override-subsubproject-indirect-link", "override-subproject-with-subsubproject", ], ) def test_override_junction(cli, tmpdir, datafiles, target, expected): project = os.path.join(str(datafiles), "overrides") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file assert os.path.exists(os.path.join(checkoutdir, expected)) # Tests a situation where the same deep subproject is overridden # more than once. # @pytest.mark.datafiles(DATA_DIR) def test_override_twice(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "override-twice") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file assert os.path.exists(os.path.join(checkoutdir, "overridden-again.txt")) # Tests the case where we override an element in a self junction # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expected_result", [ ("target.bst", "pony"), ("self-junction.bst:target.bst", "horsy"), ], ids=["direct-target", "override-target"], ) def test_override_self(cli, datafiles, target, expected_result): project = os.path.join(str(datafiles), "override-self") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target], ) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str("animal") == expected_result # # Test conflicting junction scenarios # # Note here we assert 2 provenances, we want to ensure that both # provenances leading up to the use of a project are accounted for # in a conflicting junction error. # # The second provenance can be None, because there will be no # provenance for the originally loaded project if it was the toplevel # project, or in some cases when a full path to a deep element was # specified directly on the command line. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir,target,provenances", [ # Test a stack element which depends directly on the same project twice ( "conflicts", "simple-conflict.bst", ["simple-conflict.bst [line 5 column 2]", "simple-conflict.bst [line 4 column 2]"], ), # Test a dependency chain leading deep into a project which conflicts with the toplevel ( "conflicts", "nested-conflict-toplevel.bst", ["subproject.bst:subsubproject-conflict-target.bst [line 4 column 2]"], ), # Test an attempt to override a subproject with a subproject of that same subproject through a different junction ( "conflicts", "override-conflict.bst", [ "subproject-override-conflicting-path.bst [line 13 column 23]", "override-conflict.bst [line 8 column 2]", ], ), # Same test as above, but specifying the target as a full path instead of a stack element ( "conflicts", "subproject-override-conflicting-path.bst:subsubproject.bst:target.bst", ["subproject-override-conflicting-path.bst [line 13 column 23]"], ), # Test a dependency on a subproject conflicting with an include of a file from a different # version of the same project ( "conflicts", "include-conflict-target.bst", ["include-conflict-target.bst [line 5 column 2]", "include-conflict.bst [line 4 column 7]"], ), # Test an element kind which needs to load it's plugin from a subproject, but # the element has a dependency on an element from a different version of the same project ( "conflicts", "plugin-conflict.bst", ["project.conf [line 4 column 2]", "plugin-conflict.bst [line 4 column 2]"], ), # Test a project which subproject's the same project twice, but only lists it # as a duplicate via one of it's junctions. ( "duplicates-simple-incomplete", "target.bst", ["target.bst [line 4 column 2]", "target.bst [line 5 column 2]"], ), # Test a project which subproject's the same project twice, but only lists it # as a duplicate via one of it's junctions. ( "duplicates-nested-incomplete", "target.bst", ["target.bst [line 6 column 2]", "target.bst [line 4 column 2]", "target.bst [line 5 column 2]"], ), # Test a project which uses an internal subsubproject, but also uses that same subsubproject twice # at the toplevel, this test ensures we also get the provenance of the internal project in the error. ( "internal-and-conflict", "target.bst", [ "subproject.bst:subtarget.bst [line 10 column 2]", "target.bst [line 5 column 2]", "target.bst [line 6 column 2]", ], ), ], ids=[ "simple", "nested", "override", "override-full-path", "include", "plugin", "incomplete-duplicates", "incomplete-nested-duplicates", "internal", ], ) def test_conflict(cli, tmpdir, datafiles, project_dir, target, provenances): project = os.path.join(str(datafiles), project_dir) # Special case setup the conflicting project.conf if target == "plugin-conflict.bst": update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject2.bst", "elements": ["found"], } ] }, ) result = cli.run(project=project, args=["build", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CONFLICTING_JUNCTION) # Assert expected provenances for provenance in provenances: assert provenance in result.stderr # # Test circular references in junction override cycles # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,provenance1,provenance2", [ # Override a subprojects subsubproject, with a subproject of the # subsubproject being overridden. ( "target-overridden-subsubproject-circular.bst", "subproject-overriden-with-circular-reference.bst [line 8 column 23]", None, ), ( "target-overridden-subsubproject-circular-link.bst", "link-subsubsubproject.bst [line 4 column 10]", "target-overridden-subsubproject-circular-link.bst [line 4 column 2]", ), ], ids=["override-self", "override-self-using-link"], ) def test_circular_reference(cli, tmpdir, datafiles, target, provenance1, provenance2): project = os.path.join(str(datafiles), "circular-references") result = cli.run(project=project, args=["build", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_REFERENCE) assert provenance1 in result.stderr if provenance2: assert provenance2 in result.stderr # # Test explicitly marked duplicates # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir", [ # Test a project with two direct dependencies on the same project ("duplicates-simple"), # Test a project with a dependency on a project with two duplicate subprojects, # while additionally adding a dependency on that duplicated subproject at the toplevel ("duplicates-nested"), # Same as previous test, but duplicate the subprojects only from the toplevel, # ensuring that the pathing and addressing of elements works. ("duplicates-nested-full-path"), # Test a project with two direct dependencies on the same project, one of them # referred to via a link to the junction. ("duplicates-simple-link"), # Test a project where the toplevel duplicates a link in a subproject ("duplicates-nested-link1"), # Test a project where the toplevel duplicates a link to a nested subproject ("duplicates-nested-link2"), # Test a project which overrides the a subsubproject which is marked as a duplicate by the subproject, # ensure that the duplicate relationship for the subproject/subsubproject is preserved. ("duplicates-override-dup"), # Test a project which overrides a deep subproject multiple times in the hierarchy, the intermediate # junction to the deep subproject (which is overridden by the toplevel) marks that deep subproject as # a duplicate using a link element in the project.conf to mark the duplicate, this link is otherwise unused. ("duplicates-override-twice-link"), ], ids=[ "simple", "nested", "nested-full-path", "simple-link", "link-in-subproject", "link-to-subproject", "overridden", "overridden-twice-link", ], ) def test_duplicates(cli, tmpdir, datafiles, project_dir): project = os.path.join(str(datafiles), project_dir) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # # Test errors which occur when duplicate lists refer to elements which # don't exist. # # While subprojects are not loaded by virtue of searching the duplicate # lists, we do attempt to load elements in loaded projects in order to # ensure that we properly traverse `link` elements. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir,provenance", [ # Test a not found duplicate at the toplevel ("duplicates-simple-not-found", "project.conf [line 8 column 6]"), # Test a listed duplicate of a broken `link` target in a subproject ("duplicates-nested-not-found", "subproject.bst:subproject1-link.bst [line 4 column 10]"), ], ids=["simple", "broken-nested-link"], ) def test_duplicates_not_found(cli, tmpdir, datafiles, project_dir, provenance): project = os.path.join(str(datafiles), project_dir) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Check that provenance was provided if expected assert provenance in result.stderr # # Test internal projects # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir,expected_files", [ # Test a project which repeats a subproject which is also # internal to another subproject. ("internal-simple", ["subsub.txt", "subsub-again.txt"]), # Test a project which repeats a subproject which is also # internal to two other subprojects. ("internal-double", ["subsub1.txt", "subsub2.txt", "subsub-again.txt"]), # Test a project which repeats a subproject which is also # internal to another subproject, which marks it internal using a link. ("internal-link", ["subsub.txt", "subsub-again.txt"]), # Test a project which repeats a subproject which is also internal to another # subproject, and also overrides that same internal subproject. ("internal-override", ["subsub-override.txt", "subsub-again.txt"]), ], ids=["simple", "double", "link", "override"], ) def test_internal(cli, tmpdir, datafiles, project_dir, expected_files): project = os.path.join(str(datafiles), project_dir) checkoutdir = os.path.join(str(tmpdir), "checkout") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file for expected in expected_files: assert os.path.exists(os.path.join(checkoutdir, expected)) # This test verifies that variables declared in subproject include files # are resolved in their respective subproject, rather than being imported # literally and resolved in the including project. # @pytest.mark.datafiles(DATA_DIR) def test_include_vars(cli, datafiles): project = os.path.join(str(datafiles), "include-vars") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "target.bst"] ) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str("resolved") == "The animal is a horsy" # This test verifies that project option conditional statements made # in an include file are resolved in the context of the project where # the include file originates. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "use_species,expected_result", [ ("True", "The species is a horsy"), ("False", "The animal is a horsy"), ], ids=["branch1", "branch2"], ) def test_include_vars_optional(cli, datafiles, use_species, expected_result): project = os.path.join(str(datafiles), "include-vars-optional") result = cli.run( project=project, silent=True, args=["--option", "use_species", use_species, "show", "--deps", "none", "--format", "%{vars}", "target.bst"], ) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str("resolved") == expected_result # This test verifies that project option conditional statements made # in an include file are resolved in the context of the project where # the include file originates. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target", ["target.bst", "subproject.bst:target.bst"], ids=["toplevel-target", "subproject-target"], ) @pytest.mark.parametrize( "animal,expected_result", [ ("pony", "target pony"), ("horsy", "target horsy"), ], ids=["branch1", "branch2"], ) def test_include_vars_cross_junction_element(cli, datafiles, target, animal, expected_result): project = os.path.join(str(datafiles), "include-complex") result = cli.run( project=project, silent=True, args=[ "--option", "animal", animal, "show", "--deps", "none", "--format", "%{vars}", target, ], ) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str("target_animal_variable") == expected_result apache-buildstream-27ae392/tests/format/junctions/000077500000000000000000000000001514607367700222705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/000077500000000000000000000000001514607367700262135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/link-subsubsubproject.bst000066400000000000000000000001731514607367700332650ustar00rootroot00000000000000kind: link config: target: subproject-overriden-with-circular-reference-link.bst:subsubproject.bst:subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/circular-references/project.conf000066400000000000000000000000341514607367700305250ustar00rootroot00000000000000name: test min-version: 2.0 subproject-overriden-with-circular-reference-link.bst000066400000000000000000000002001514607367700404530ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-referenceskind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst: link-subsubsubproject.bst subproject-overriden-with-circular-reference.bst000066400000000000000000000002761514607367700375350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-referenceskind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst: subproject-overriden-with-circular-reference.bst:subsubproject.bst:subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject.bst000066400000000000000000000000711514607367700311030ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/000077500000000000000000000000001514607367700303735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/project.conf000066400000000000000000000000371514607367700327100ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/sub.txt000066400000000000000000000000051514607367700317200ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubproject.bst000066400000000000000000000000741514607367700340000ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubproject/000077500000000000000000000000001514607367700332655ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700355170ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubprojectname: subsubtest min-version: 2.0 subsub.txt000066400000000000000000000000051514607367700352450ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubprojectfile subsubsubproject.bst000066400000000000000000000000771514607367700373300ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject subsubsubproject/000077500000000000000000000000001514607367700366125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubprojectproject.conf000066400000000000000000000000451514607367700411260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 subsubsub.txt000066400000000000000000000000051514607367700413630ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubproject/subsubsubprojectfile target.bst000066400000000000000000000000721514607367700406110ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: subsubsub.txt target.bst000066400000000000000000000000671514607367700352110ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/subsubprojectkind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/circular-references/subproject/target.bst000066400000000000000000000000641514607367700323730ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt target-overridden-subsubproject-circular-link.bst000066400000000000000000000001531514607367700377170ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-referenceskind: stack depends: - subproject-overriden-with-circular-reference-link.bst:subsubproject.bst:target.bst target-overridden-subsubproject-circular.bst000066400000000000000000000001461514607367700367660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/circular-referenceskind: stack depends: - subproject-overriden-with-circular-reference.bst:subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/000077500000000000000000000000001514607367700242545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/include-conflict-target.bst000066400000000000000000000001111514607367700314650ustar00rootroot00000000000000kind: stack depends: - include-conflict.bst - subproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/include-conflict.bst000066400000000000000000000001401514607367700302030ustar00rootroot00000000000000kind: manual variables: (@): subproject2.bst:inc.yaml depends: - subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/nested-conflict-toplevel.bst000066400000000000000000000001111514607367700316700ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subsubproject-conflict-target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/override-conflict.bst000066400000000000000000000003141514607367700304020ustar00rootroot00000000000000kind: stack # # To trigger the conflict, we need to traverse the path of # the overridden `subsubproject.bst` junction. # depends: - subproject-override-conflicting-path.bst:subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/plugin-conflict.bst000066400000000000000000000000621514607367700300610ustar00rootroot00000000000000kind: found depends: - subproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/project.conf000066400000000000000000000000341514607367700265660ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/conflicts/simple-conflict.bst000066400000000000000000000001171514607367700300550ustar00rootroot00000000000000kind: stack depends: - subproject.bst:target.bst - subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject-override-conflicting-path.bst000066400000000000000000000005301514607367700342100ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject # # Here we are declaring a junction to subproject, and trying to override # it's subproject with a deep subproject, using a different junction to # the same subproject `subproject.bst` # config: overrides: subsubproject.bst: subproject.bst:subsubproject.bst:subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject.bst000066400000000000000000000000711514607367700271440ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/000077500000000000000000000000001514607367700264345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/project.conf000066400000000000000000000000371514607367700307510ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/sub.txt000066400000000000000000000000051514607367700277610ustar00rootroot00000000000000file subsubproject-conflict-target.bst000066400000000000000000000000761514607367700350470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subprojectkind: stack depends: - subsubproject-conflict.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject-conflict.bst000066400000000000000000000001051514607367700336330ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-conflict apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject-conflict/000077500000000000000000000000001514607367700331255ustar00rootroot00000000000000deepsurprise.txt000066400000000000000000000000051514607367700363140ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject-conflictfile project.conf000066400000000000000000000000341514607367700353600ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject-conflictname: test min-version: 2.0 target.bst000066400000000000000000000000751514607367700350500ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject-conflictkind: import sources: - kind: local path: deepsurprise.txt apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject.bst000066400000000000000000000000741514607367700320410ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/000077500000000000000000000000001514607367700313265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/project.conf000066400000000000000000000000421514607367700336370ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/subsub.txt000066400000000000000000000000051514607367700333650ustar00rootroot00000000000000file subsubsubproject.bst000066400000000000000000000000771514607367700353710ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject subsubsubproject/000077500000000000000000000000001514607367700346535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubprojectproject.conf000066400000000000000000000000451514607367700371670ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 subsubsub.txt000066400000000000000000000000051514607367700374240ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/subsubsubprojectfile target.bst000066400000000000000000000000721514607367700366520ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: subsubsub.txt apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/subsubproject/target.bst000066400000000000000000000000671514607367700333310ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject/target.bst000066400000000000000000000000641514607367700304340ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2.bst000066400000000000000000000000721514607367700272270ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/000077500000000000000000000000001514607367700265165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/inc.yaml000066400000000000000000000000131514607367700301450ustar00rootroot00000000000000test: Pony apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/plugins/000077500000000000000000000000001514607367700301775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/plugins/found.py000066400000000000000000000004161514607367700316650ustar00rootroot00000000000000from buildstream import Element class Found(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return Found apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/project.conf000066400000000000000000000001371514607367700310340ustar00rootroot00000000000000name: subtest min-version: 2.0 plugins: - origin: local path: plugins elements: - found apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/sub2.txt000066400000000000000000000000051514607367700301250ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/conflicts/subproject2/target.bst000066400000000000000000000000651514607367700305170ustar00rootroot00000000000000kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/000077500000000000000000000000001514607367700275775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/project.conf000066400000000000000000000002461514607367700321160ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubtest: - subproject.bst:subproject1.bst - subproject.bst:subproject2.bst - subproject3.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject.bst000066400000000000000000000000711514607367700324670ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/000077500000000000000000000000001514607367700317575ustar00rootroot00000000000000project.conf000066400000000000000000000000371514607367700342150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subprojectname: subtest min-version: 2.0 subproject1.bst000066400000000000000000000000721514607367700346520ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subprojectkind: junction sources: - kind: local path: subproject1 subproject1/000077500000000000000000000000001514607367700341415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subprojectproject.conf000066400000000000000000000000421514607367700364520ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/subproject1name: subsubtest min-version: 2.0 sub.txt000066400000000000000000000000051514607367700354660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/subproject1file target.bst000066400000000000000000000000641514607367700361410ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/subproject1kind: import sources: - kind: local path: sub.txt subproject2.bst000066400000000000000000000000721514607367700346530ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subprojectkind: junction sources: - kind: local path: subproject2 subproject2/000077500000000000000000000000001514607367700341425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subprojectproject.conf000066400000000000000000000000421514607367700364530ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/subproject2name: subsubtest min-version: 2.0 sub2.txt000066400000000000000000000000051514607367700355510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/subproject2file target.bst000066400000000000000000000000651514607367700361430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject3.bst000066400000000000000000000000721514607367700325530ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject3/000077500000000000000000000000001514607367700320425ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700342740ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject3name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject3/sub3.txt000066400000000000000000000000051514607367700334520ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/subproject3/target.bst000066400000000000000000000000651514607367700340430ustar00rootroot00000000000000kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-full-path/target.bst000066400000000000000000000002131514607367700315730ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subproject1.bst:target.bst - subproject.bst:subproject2.bst:target.bst - subproject3.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/000077500000000000000000000000001514607367700300425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/project.conf000066400000000000000000000001341514607367700323550ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubtest: - subproject3.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject.bst000066400000000000000000000000711514607367700327320ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/000077500000000000000000000000001514607367700322225ustar00rootroot00000000000000project.conf000066400000000000000000000001371514607367700344610ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subprojectname: subtest min-version: 2.0 junctions: duplicates: subsubtest: - subproject1.bst subproject1.bst000066400000000000000000000000721514607367700351150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subprojectkind: junction sources: - kind: local path: subproject1 subproject1/000077500000000000000000000000001514607367700344045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subprojectproject.conf000066400000000000000000000000421514607367700367150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/subproject1name: subsubtest min-version: 2.0 sub.txt000066400000000000000000000000051514607367700357310ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/subproject1file target.bst000066400000000000000000000000641514607367700364040ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/subproject1kind: import sources: - kind: local path: sub.txt subproject2.bst000066400000000000000000000000721514607367700351160ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subprojectkind: junction sources: - kind: local path: subproject2 subproject2/000077500000000000000000000000001514607367700344055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subprojectproject.conf000066400000000000000000000000421514607367700367160ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/subproject2name: subsubtest min-version: 2.0 sub2.txt000066400000000000000000000000051514607367700360140ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/subproject2file target.bst000066400000000000000000000000651514607367700364060ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject3.bst000066400000000000000000000000721514607367700330160ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject3/000077500000000000000000000000001514607367700323055ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700345370ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject3name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject3/sub3.txt000066400000000000000000000000051514607367700337150ustar00rootroot00000000000000file target.bst000066400000000000000000000000651514607367700342270ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/subproject3kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-incomplete/target.bst000066400000000000000000000002131514607367700320360ustar00rootroot00000000000000kind: stack depends: - subproject3.bst:target.bst - subproject.bst:subproject1.bst:target.bst - subproject.bst:subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/000077500000000000000000000000001514607367700267215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/project.conf000066400000000000000000000002061514607367700312340ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubtest: - subproject3.bst - subproject.bst:subproject1-link.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject.bst000066400000000000000000000000711514607367700316110ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/000077500000000000000000000000001514607367700311015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/project.conf000066400000000000000000000001371514607367700334170ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: duplicates: subsubtest: - subproject2.bst subproject1-link.bst000066400000000000000000000000561514607367700347310ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subprojectkind: link config: target: subproject1.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject1.bst000066400000000000000000000000721514607367700340530ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject1/000077500000000000000000000000001514607367700333425ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700355740ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject1name: subsubtest min-version: 2.0 sub.txt000066400000000000000000000000051514607367700346100ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject1file target.bst000066400000000000000000000000641514607367700352630ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject1kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject2.bst000066400000000000000000000000721514607367700340540ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject2/000077500000000000000000000000001514607367700333435ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700355750ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject2name: subsubtest min-version: 2.0 sub2.txt000066400000000000000000000000051514607367700346730ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject2file target.bst000066400000000000000000000000651514607367700352650ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject3.bst000066400000000000000000000000721514607367700316750ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject3/000077500000000000000000000000001514607367700311645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject3/project.conf000066400000000000000000000000421514607367700334750ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject3/sub3.txt000066400000000000000000000000051514607367700325740ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/subproject3/target.bst000066400000000000000000000000651514607367700331650ustar00rootroot00000000000000kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link1/target.bst000066400000000000000000000002131514607367700307150ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subproject1.bst:target.bst - subproject.bst:subproject2.bst:target.bst - subproject3.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/000077500000000000000000000000001514607367700267225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/project.conf000066400000000000000000000001341514607367700312350ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubtest: - subproject3.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject.bst000066400000000000000000000000711514607367700316120ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/000077500000000000000000000000001514607367700311025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/project.conf000066400000000000000000000001651514607367700334210ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: duplicates: subsubtest: - subproject1.bst - subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject1.bst000066400000000000000000000000721514607367700340540ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject1/000077500000000000000000000000001514607367700333435ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700355750ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject1name: subsubtest min-version: 2.0 sub.txt000066400000000000000000000000051514607367700346110ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject1file target.bst000066400000000000000000000000641514607367700352640ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject1kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject2.bst000066400000000000000000000000721514607367700340550ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject2/000077500000000000000000000000001514607367700333445ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700355760ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject2name: subsubtest min-version: 2.0 sub2.txt000066400000000000000000000000051514607367700346740ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject2file target.bst000066400000000000000000000000651514607367700352660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject2-link.bst000066400000000000000000000000751514607367700326330ustar00rootroot00000000000000kind: link config: target: subproject.bst:subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject3.bst000066400000000000000000000000721514607367700316760ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject3/000077500000000000000000000000001514607367700311655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject3/project.conf000066400000000000000000000000421514607367700334760ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject3/sub3.txt000066400000000000000000000000051514607367700325750ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/subproject3/target.bst000066400000000000000000000000651514607367700331660ustar00rootroot00000000000000kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-link2/target.bst000066400000000000000000000002011514607367700307130ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subproject1.bst:target.bst - subproject2-link.bst:target.bst - subproject3.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/000077500000000000000000000000001514607367700276145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/project.conf000066400000000000000000000002061514607367700321270ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubtest: - subproject3.bst - subproject.bst:subproject1-link.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject.bst000066400000000000000000000000711514607367700325040ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/000077500000000000000000000000001514607367700317745ustar00rootroot00000000000000project.conf000066400000000000000000000001371514607367700342330ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subprojectname: subtest min-version: 2.0 junctions: duplicates: subsubtest: - subproject2.bst subproject1-link.bst000066400000000000000000000000561514607367700356240ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subprojectkind: link config: target: subproject5.bst subproject1.bst000066400000000000000000000000721514607367700346670ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subprojectkind: junction sources: - kind: local path: subproject1 subproject1/000077500000000000000000000000001514607367700341565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subprojectproject.conf000066400000000000000000000000421514607367700364670ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/subproject1name: subsubtest min-version: 2.0 sub.txt000066400000000000000000000000051514607367700355030ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/subproject1file target.bst000066400000000000000000000000641514607367700361560ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/subproject1kind: import sources: - kind: local path: sub.txt subproject2.bst000066400000000000000000000000721514607367700346700ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subprojectkind: junction sources: - kind: local path: subproject2 subproject2/000077500000000000000000000000001514607367700341575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subprojectproject.conf000066400000000000000000000000421514607367700364700ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/subproject2name: subsubtest min-version: 2.0 sub2.txt000066400000000000000000000000051514607367700355660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/subproject2file target.bst000066400000000000000000000000651514607367700361600ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject3.bst000066400000000000000000000000721514607367700325700ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject3/000077500000000000000000000000001514607367700320575ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700343110ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject3name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject3/sub3.txt000066400000000000000000000000051514607367700334670ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/subproject3/target.bst000066400000000000000000000000651514607367700340600ustar00rootroot00000000000000kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested-not-found/target.bst000066400000000000000000000002131514607367700316100ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subproject1.bst:target.bst - subproject.bst:subproject2.bst:target.bst - subproject3.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/000077500000000000000000000000001514607367700257055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/project.conf000066400000000000000000000001341514607367700302200ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubtest: - subproject3.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject.bst000066400000000000000000000000711514607367700305750ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/000077500000000000000000000000001514607367700300655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/project.conf000066400000000000000000000001651514607367700324040ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: duplicates: subsubtest: - subproject1.bst - subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject1.bst000066400000000000000000000000721514607367700330370ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject1/000077500000000000000000000000001514607367700323265ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700345600ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject1name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject1/sub.txt000066400000000000000000000000051514607367700336530ustar00rootroot00000000000000file target.bst000066400000000000000000000000641514607367700342470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject1kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject2.bst000066400000000000000000000000721514607367700330400ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject2/000077500000000000000000000000001514607367700323275ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700345610ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject2name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject2/sub2.txt000066400000000000000000000000051514607367700337360ustar00rootroot00000000000000file target.bst000066400000000000000000000000651514607367700342510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject3.bst000066400000000000000000000000721514607367700306610ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject3/000077500000000000000000000000001514607367700301505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject3/project.conf000066400000000000000000000000421514607367700324610ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject3/sub3.txt000066400000000000000000000000051514607367700315600ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/subproject3/target.bst000066400000000000000000000000651514607367700321510ustar00rootroot00000000000000kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-nested/target.bst000066400000000000000000000002131514607367700277010ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subproject1.bst:target.bst - subproject.bst:subproject2.bst:target.bst - subproject3.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/000077500000000000000000000000001514607367700270305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/project.conf000066400000000000000000000000341514607367700313420ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject.bst000066400000000000000000000001641514607367700317230ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subproject1.bst: subproject3.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/000077500000000000000000000000001514607367700312105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/project.conf000066400000000000000000000001651514607367700335270ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: duplicates: subsubtest: - subproject1.bst - subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject1.bst000066400000000000000000000000721514607367700341620ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject1/000077500000000000000000000000001514607367700334515ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700357030ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject1name: subsubtest min-version: 2.0 sub.txt000066400000000000000000000000051514607367700347170ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject1file target.bst000066400000000000000000000000641514607367700353720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject1kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject2.bst000066400000000000000000000000721514607367700341630ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject2/000077500000000000000000000000001514607367700334525ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700357040ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject2name: subsubtest min-version: 2.0 sub2.txt000066400000000000000000000000051514607367700350020ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject2file target.bst000066400000000000000000000000651514607367700353740ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject3.bst000066400000000000000000000000721514607367700320040ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject3 apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject3/000077500000000000000000000000001514607367700312735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject3/project.conf000066400000000000000000000000421514607367700336040ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject3/sub3.txt000066400000000000000000000000051514607367700327030ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/subproject3/target.bst000066400000000000000000000000651514607367700332740ustar00rootroot00000000000000kind: import sources: - kind: local path: sub3.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-override-dup/target.bst000066400000000000000000000001561514607367700310320ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subproject1.bst:target.bst - subproject.bst:subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/000077500000000000000000000000001514607367700303065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/duplicate.bst000066400000000000000000000000701514607367700327670ustar00rootroot00000000000000kind: junction sources: - kind: local path: duplicate apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/duplicate/000077500000000000000000000000001514607367700322605ustar00rootroot00000000000000duplicate.txt000066400000000000000000000000051514607367700347070ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/duplicatefile project.conf000066400000000000000000000000451514607367700345150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/duplicatename: subsubsubtest min-version: 2.0 target.bst000066400000000000000000000000721514607367700342000ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/duplicatekind: import sources: - kind: local path: duplicate.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/override.bst000066400000000000000000000000671514607367700326420ustar00rootroot00000000000000kind: junction sources: - kind: local path: override apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/override/000077500000000000000000000000001514607367700321255ustar00rootroot00000000000000overridden-again.txt000066400000000000000000000000051514607367700360200ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/overridefile project.conf000066400000000000000000000000451514607367700343620ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/overridename: subsubsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/override/target.bst000066400000000000000000000001011514607367700341150ustar00rootroot00000000000000kind: import sources: - kind: local path: overridden-again.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/project.conf000066400000000000000000000001351514607367700326220ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subsubsubtest: - duplicate.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject.bst000066400000000000000000000002101514607367700331710ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst:subsubsubproject.bst: override.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/000077500000000000000000000000001514607367700324665ustar00rootroot00000000000000override.bst000066400000000000000000000000671514607367700347430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subprojectkind: junction sources: - kind: local path: override override/000077500000000000000000000000001514607367700342265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subprojectoverridden.txt000066400000000000000000000000051514607367700371230ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/overridefile project.conf000066400000000000000000000000451514607367700365420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/overridename: subsubsubtest min-version: 2.0 target.bst000066400000000000000000000000731514607367700362260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/overridekind: import sources: - kind: local path: overridden.txt project.conf000066400000000000000000000001541514607367700347240ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subprojectname: subtest min-version: 2.0 junctions: duplicates: subsubsubtest: - subsubsubproject-link.bst subsubproject.bst000066400000000000000000000001711514607367700360120ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subprojectkind: junction sources: - kind: local path: subsubproject config: overrides: subsubsubproject.bst: override.bst subsubproject/000077500000000000000000000000001514607367700353015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subprojectproject.conf000066400000000000000000000000421514607367700376120ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/subsubprojectname: subsubtest min-version: 2.0 subsubsubproject.bst000066400000000000000000000000771514607367700414230ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject subsubsubproject/000077500000000000000000000000001514607367700407055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/subsubprojectoriginal.txt000066400000000000000000000000051514607367700432450ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/subsubproject/subsubsubprojectfile project.conf000066400000000000000000000000451514607367700432210ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 target.bst000066400000000000000000000000711514607367700427030ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: original.txt subsubsubproject-link.bst000066400000000000000000000001051514607367700374540ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/subprojectkind: link config: target: subsubproject.bst:subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-override-twice-link/target.bst000066400000000000000000000001641514607367700323070ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subsubproject.bst:subsubsubproject.bst:target.bst - duplicate.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/000077500000000000000000000000001514607367700300515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/project.conf000066400000000000000000000001311514607367700323610ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subtest: - subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject1.bst000066400000000000000000000000721514607367700330230ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject1/000077500000000000000000000000001514607367700323125ustar00rootroot00000000000000project.conf000066400000000000000000000000371514607367700345500ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject1name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject1/sub.txt000066400000000000000000000000051514607367700336370ustar00rootroot00000000000000file target.bst000066400000000000000000000000641514607367700342330ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject1kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject2.bst000066400000000000000000000000721514607367700330240ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject2/000077500000000000000000000000001514607367700323135ustar00rootroot00000000000000project.conf000066400000000000000000000000371514607367700345510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject2name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject2/sub2.txt000066400000000000000000000000051514607367700337220ustar00rootroot00000000000000file target.bst000066400000000000000000000000651514607367700342350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/subproject2kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-incomplete/target.bst000066400000000000000000000001201514607367700320420ustar00rootroot00000000000000kind: stack depends: - subproject1.bst:target.bst - subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/000077500000000000000000000000001514607367700266475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/project.conf000066400000000000000000000001641514607367700311650ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subtest: - subproject1.bst - subproject2-link.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject1.bst000066400000000000000000000000721514607367700316210ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject1/000077500000000000000000000000001514607367700311105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject1/project.conf000066400000000000000000000000371514607367700334250ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject1/sub.txt000066400000000000000000000000051514607367700324350ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject1/target.bst000066400000000000000000000000641514607367700331100ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject2-link.bst000066400000000000000000000000561514607367700325570ustar00rootroot00000000000000kind: link config: target: subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject2.bst000066400000000000000000000000721514607367700316220ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject2/000077500000000000000000000000001514607367700311115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject2/project.conf000066400000000000000000000000371514607367700334260ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject2/sub2.txt000066400000000000000000000000051514607367700325200ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/subproject2/target.bst000066400000000000000000000000651514607367700331120ustar00rootroot00000000000000kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-link/target.bst000066400000000000000000000001251514607367700306450ustar00rootroot00000000000000kind: stack depends: - subproject1.bst:target.bst - subproject2-link.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/000077500000000000000000000000001514607367700276235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/project.conf000066400000000000000000000001571514607367700321430ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subtest: - subproject1.bst - subproject5.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject1.bst000066400000000000000000000000721514607367700325750ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject1/000077500000000000000000000000001514607367700320645ustar00rootroot00000000000000project.conf000066400000000000000000000000371514607367700343220ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject1name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject1/sub.txt000066400000000000000000000000051514607367700334110ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject1/target.bst000066400000000000000000000000641514607367700340640ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject2.bst000066400000000000000000000000721514607367700325760ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject2/000077500000000000000000000000001514607367700320655ustar00rootroot00000000000000project.conf000066400000000000000000000000371514607367700343230ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject2name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject2/sub2.txt000066400000000000000000000000051514607367700334740ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/subproject2/target.bst000066400000000000000000000000651514607367700340660ustar00rootroot00000000000000kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple-not-found/target.bst000066400000000000000000000001201514607367700316140ustar00rootroot00000000000000kind: stack depends: - subproject1.bst:target.bst - subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/000077500000000000000000000000001514607367700257145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/project.conf000066400000000000000000000001571514607367700302340ustar00rootroot00000000000000name: test min-version: 2.0 junctions: duplicates: subtest: - subproject1.bst - subproject2.bst apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject1.bst000066400000000000000000000000721514607367700306660ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject1/000077500000000000000000000000001514607367700301555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject1/project.conf000066400000000000000000000000371514607367700324720ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject1/sub.txt000066400000000000000000000000051514607367700315020ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject1/target.bst000066400000000000000000000000641514607367700321550ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject2.bst000066400000000000000000000000721514607367700306670ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject2/000077500000000000000000000000001514607367700301565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject2/project.conf000066400000000000000000000000371514607367700324730ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject2/sub2.txt000066400000000000000000000000051514607367700315650ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/subproject2/target.bst000066400000000000000000000000651514607367700321570ustar00rootroot00000000000000kind: import sources: - kind: local path: sub2.txt apache-buildstream-27ae392/tests/format/junctions/duplicates-simple/target.bst000066400000000000000000000001201514607367700277050ustar00rootroot00000000000000kind: stack depends: - subproject1.bst:target.bst - subproject2.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/full-path/000077500000000000000000000000001514607367700241645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/element-full-path-notfound.bst000066400000000000000000000001011514607367700320430ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subsubproject.bst:pony.bst apache-buildstream-27ae392/tests/format/junctions/full-path/element-full-path.bst000066400000000000000000000001301514607367700302130ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subsubproject.bst:subsubsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/full-path/junction-full-path-notfound.bst000066400000000000000000000001271514607367700322530ustar00rootroot00000000000000kind: stack depends: - junction: subproject.bst:subsubproject.bst filename: pony.bst apache-buildstream-27ae392/tests/format/junctions/full-path/junction-full-path.bst000066400000000000000000000001561514607367700304230ustar00rootroot00000000000000kind: stack depends: - junction: subproject.bst:subsubproject.bst:subsubsubproject.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/full-path/project.conf000066400000000000000000000000341514607367700264760ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/full-path/subproject.bst000066400000000000000000000000711514607367700270540ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/000077500000000000000000000000001514607367700263445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/project.conf000066400000000000000000000000371514607367700306610ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/sub.txt000066400000000000000000000000051514607367700276710ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject.bst000066400000000000000000000000741514607367700317510ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/000077500000000000000000000000001514607367700312365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/project.conf000066400000000000000000000000421514607367700335470ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/subsub.txt000066400000000000000000000000051514607367700332750ustar00rootroot00000000000000file subsubsubproject.bst000066400000000000000000000000771514607367700353010ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject subsubsubproject/000077500000000000000000000000001514607367700345635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubprojectproject.conf000066400000000000000000000000451514607367700370770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 subsubsub.txt000066400000000000000000000000051514607367700373340ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/subsubsubprojectfile target.bst000066400000000000000000000000721514607367700365620ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: subsubsub.txt apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/subsubproject/target.bst000066400000000000000000000000671514607367700332410ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/full-path/subproject/target.bst000066400000000000000000000000641514607367700303440ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/include-complex/000077500000000000000000000000001514607367700253605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/include-complex/project.conf000066400000000000000000000003101514607367700276670ustar00rootroot00000000000000name: test min-version: 2.0 (@): - subproject.bst:include.yml options: animal: type: enum description: the animal values: - pony - horsy default: pony variable: animal apache-buildstream-27ae392/tests/format/junctions/include-complex/subproject.bst000066400000000000000000000001541514607367700302520ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: options: target_animal: '%{animal}' apache-buildstream-27ae392/tests/format/junctions/include-complex/subproject/000077500000000000000000000000001514607367700275405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/include-complex/subproject/include.yml000066400000000000000000000003241514607367700317050ustar00rootroot00000000000000 variables: target_animal_variable: "no target animal" (?): - target_animal == "pony": target_animal_variable: "target pony" - target_animal == "horsy": target_animal_variable: "target horsy" apache-buildstream-27ae392/tests/format/junctions/include-complex/subproject/project.conf000066400000000000000000000002411514607367700320520ustar00rootroot00000000000000name: subtest min-version: 2.0 options: target_animal: type: enum description: the target animal values: - pony - horsy default: pony apache-buildstream-27ae392/tests/format/junctions/include-complex/subproject/target.bst000066400000000000000000000000401514607367700315320ustar00rootroot00000000000000kind: stack (@): - include.yml apache-buildstream-27ae392/tests/format/junctions/include-complex/target.bst000066400000000000000000000000631514607367700273570ustar00rootroot00000000000000kind: manual depends: - subproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/000077500000000000000000000000001514607367700265075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/project.conf000066400000000000000000000002721514607367700310250ustar00rootroot00000000000000name: test min-version: 2.0 variables: animal: pony options: use_species: type: bool description: Whether to use the species classifier in the subproject default: True apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/subproject.bst000066400000000000000000000002701514607367700314000ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: options: (?): - use_species: classifier: species - not use_species: classifier: animal apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/subproject/000077500000000000000000000000001514607367700306675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/subproject/include.yml000066400000000000000000000004311514607367700330330ustar00rootroot00000000000000 # # Test that conditional statements are resolved in the context # of the project from where the include originated. # variables: (?): - classifier == "species": resolved: The species is a %{animal} - classifier == "animal": resolved: The animal is a %{animal} apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/subproject/project.conf000066400000000000000000000003201514607367700331770ustar00rootroot00000000000000name: subtest min-version: 2.0 variables: animal: horsy options: classifier: type: enum description: how to classify the horsy or pony values: - animal - species default: animal apache-buildstream-27ae392/tests/format/junctions/include-vars-optional/target.bst000066400000000000000000000000551514607367700305070ustar00rootroot00000000000000kind: stack (@): subproject.bst:include.yml apache-buildstream-27ae392/tests/format/junctions/include-vars/000077500000000000000000000000001514607367700246645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/include-vars/project.conf000066400000000000000000000000671514607367700272040ustar00rootroot00000000000000name: test min-version: 2.0 variables: animal: pony apache-buildstream-27ae392/tests/format/junctions/include-vars/subproject.bst000066400000000000000000000000711514607367700275540ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/include-vars/subproject/000077500000000000000000000000001514607367700270445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/include-vars/subproject/include.yml000066400000000000000000000000621514607367700312100ustar00rootroot00000000000000 variables: resolved: The animal is a %{animal} apache-buildstream-27ae392/tests/format/junctions/include-vars/subproject/project.conf000066400000000000000000000000731514607367700313610ustar00rootroot00000000000000name: subtest min-version: 2.0 variables: animal: horsy apache-buildstream-27ae392/tests/format/junctions/include-vars/target.bst000066400000000000000000000000551514607367700266640ustar00rootroot00000000000000kind: stack (@): subproject.bst:include.yml apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/000077500000000000000000000000001514607367700264435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/project.conf000066400000000000000000000000341514607367700307550ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject.bst000066400000000000000000000000711514607367700313330ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/000077500000000000000000000000001514607367700306235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/project.conf000066400000000000000000000001151514607367700331350ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: internal: - subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/subsubproject.bst000066400000000000000000000000741514607367700342300ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/subsubproject/000077500000000000000000000000001514607367700335155ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700357470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/subsubprojectname: subsubtest min-version: 2.0 subsub.txt000066400000000000000000000000051514607367700354750ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/subsubprojectfile target.bst000066400000000000000000000000671514607367700354410ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/subsubprojectkind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subproject/subtarget.bst000066400000000000000000000004161514607367700333360ustar00rootroot00000000000000kind: compose # Use a build dependency on our internal subsubproject target. # # There is currently no validation for this but it may be # introduced in the future. For correctness of this test, # use a build dependency. # build-depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-again.bst000066400000000000000000000001021514607367700331150ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-again apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-again/000077500000000000000000000000001514607367700324125ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700346440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-againname: subsubtest min-version: 2.0 subsub-again.txt000066400000000000000000000000051514607367700354470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-againfile target.bst000066400000000000000000000000751514607367700343350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-againkind: import sources: - kind: local path: subsub-again.txt apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-conflict.bst000066400000000000000000000001051514607367700336420ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-conflict apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-conflict/000077500000000000000000000000001514607367700331345ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700353660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-conflictname: subsubtest min-version: 2.0 subsub-again.txt000066400000000000000000000000051514607367700361710ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-conflictfile target.bst000066400000000000000000000000751514607367700350570ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/subsubproject-conflictkind: import sources: - kind: local path: subsub-again.txt apache-buildstream-27ae392/tests/format/junctions/internal-and-conflict/target.bst000066400000000000000000000002021514607367700304350ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subtarget.bst - subsubproject-again.bst:target.bst - subsubproject-conflict.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-double/000077500000000000000000000000001514607367700253545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/project.conf000066400000000000000000000000341514607367700276660ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1.bst000066400000000000000000000000721514607367700303260ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject1 apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/000077500000000000000000000000001514607367700276155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/project.conf000066400000000000000000000001161514607367700321300ustar00rootroot00000000000000name: subtest1 min-version: 2.0 junctions: internal: - subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/subsubproject.bst000066400000000000000000000000741514607367700332220ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/subsubproject/000077500000000000000000000000001514607367700325075ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700347410ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/subsubprojectname: subsubtest min-version: 2.0 subsub1.txt000066400000000000000000000000051514607367700345500ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/subsubprojectfile target.bst000066400000000000000000000000701514607367700344250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/subsubprojectkind: import sources: - kind: local path: subsub1.txt apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject1/subtarget.bst000066400000000000000000000004161514607367700323300ustar00rootroot00000000000000kind: compose # Use a build dependency on our internal subsubproject target. # # There is currently no validation for this but it may be # introduced in the future. For correctness of this test, # use a build dependency. # build-depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2.bst000066400000000000000000000000721514607367700303270ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject2 apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/000077500000000000000000000000001514607367700276165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/project.conf000066400000000000000000000001161514607367700321310ustar00rootroot00000000000000name: subtest2 min-version: 2.0 junctions: internal: - subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/subsubproject.bst000066400000000000000000000000741514607367700332230ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/subsubproject/000077500000000000000000000000001514607367700325105ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700347420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/subsubprojectname: subsubtest min-version: 2.0 subsub2.txt000066400000000000000000000000051514607367700345520ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/subsubprojectfile target.bst000066400000000000000000000000701514607367700344260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/subsubprojectkind: import sources: - kind: local path: subsub2.txt apache-buildstream-27ae392/tests/format/junctions/internal-double/subproject2/subtarget.bst000066400000000000000000000004161514607367700323310ustar00rootroot00000000000000kind: compose # Use a build dependency on our internal subsubproject target. # # There is currently no validation for this but it may be # introduced in the future. For correctness of this test, # use a build dependency. # build-depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-double/subsubproject-again.bst000066400000000000000000000001021514607367700320260ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-again apache-buildstream-27ae392/tests/format/junctions/internal-double/subsubproject-again/000077500000000000000000000000001514607367700313235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subsubproject-again/project.conf000066400000000000000000000000421514607367700336340ustar00rootroot00000000000000name: subsubtest min-version: 2.0 subsub-again.txt000066400000000000000000000000051514607367700343600ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-double/subsubproject-againfile apache-buildstream-27ae392/tests/format/junctions/internal-double/subsubproject-again/target.bst000066400000000000000000000000751514607367700333250ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub-again.txt apache-buildstream-27ae392/tests/format/junctions/internal-double/target.bst000066400000000000000000000001731514607367700273550ustar00rootroot00000000000000kind: stack depends: - subproject1.bst:subtarget.bst - subproject2.bst:subtarget.bst - subsubproject-again.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-link/000077500000000000000000000000001514607367700250375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-link/project.conf000066400000000000000000000000341514607367700273510ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject.bst000066400000000000000000000000711514607367700277270ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/000077500000000000000000000000001514607367700272175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/project.conf000066400000000000000000000001221514607367700315270ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: internal: - subsubproject-link.bst apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subsubproject-link.bst000066400000000000000000000000601514607367700335520ustar00rootroot00000000000000kind: link config: target: subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subsubproject.bst000066400000000000000000000000741514607367700326240ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subsubproject/000077500000000000000000000000001514607367700321115ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700343430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subsubprojectname: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subsubproject/subsub.txt000066400000000000000000000000051514607367700341500ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subsubproject/target.bst000066400000000000000000000000671514607367700341140ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/internal-link/subproject/subtarget.bst000066400000000000000000000004161514607367700317320ustar00rootroot00000000000000kind: compose # Use a build dependency on our internal subsubproject target. # # There is currently no validation for this but it may be # introduced in the future. For correctness of this test, # use a build dependency. # build-depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-link/subsubproject-again.bst000066400000000000000000000001021514607367700315110ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-again apache-buildstream-27ae392/tests/format/junctions/internal-link/subsubproject-again/000077500000000000000000000000001514607367700310065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-link/subsubproject-again/project.conf000066400000000000000000000000421514607367700333170ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-link/subsubproject-again/subsub-again.txt000066400000000000000000000000051514607367700341220ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/internal-link/subsubproject-again/target.bst000066400000000000000000000000751514607367700330100ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub-again.txt apache-buildstream-27ae392/tests/format/junctions/internal-link/target.bst000066400000000000000000000001321514607367700270330ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subtarget.bst - subsubproject-again.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-override/000077500000000000000000000000001514607367700257215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/project.conf000066400000000000000000000000341514607367700302330ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject.bst000066400000000000000000000002011514607367700306040ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst: subsubproject-override.bst apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/000077500000000000000000000000001514607367700301015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/project.conf000066400000000000000000000001151514607367700324130ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: internal: - subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/subsubproject.bst000066400000000000000000000000741514607367700335060ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/subsubproject/000077500000000000000000000000001514607367700327735ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700352250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/subsubprojectname: subsubtest min-version: 2.0 subsub.txt000066400000000000000000000000051514607367700347530ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/subsubprojectfile target.bst000066400000000000000000000000671514607367700347170ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/subsubprojectkind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/internal-override/subproject/subtarget.bst000066400000000000000000000004161514607367700326140ustar00rootroot00000000000000kind: compose # Use a build dependency on our internal subsubproject target. # # There is currently no validation for this but it may be # introduced in the future. For correctness of this test, # use a build dependency. # build-depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-again.bst000066400000000000000000000001021514607367700323730ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-again apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-again/000077500000000000000000000000001514607367700316705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-again/project.conf000066400000000000000000000000421514607367700342010ustar00rootroot00000000000000name: subsubtest min-version: 2.0 subsub-again.txt000066400000000000000000000000051514607367700347250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-againfile apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-again/target.bst000066400000000000000000000000751514607367700336720ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub-again.txt apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-override.bst000066400000000000000000000001051514607367700331360ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-override apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-override/000077500000000000000000000000001514607367700324305ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700346620ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-overridename: subsubtest min-version: 2.0 subsub-override.txt000066400000000000000000000000051514607367700362250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-overridefile target.bst000066400000000000000000000001001514607367700343400ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-override/subsubproject-overridekind: import sources: - kind: local path: subsub-override.txt apache-buildstream-27ae392/tests/format/junctions/internal-override/target.bst000066400000000000000000000001321514607367700277150ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subtarget.bst - subsubproject-again.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-simple/000077500000000000000000000000001514607367700253735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/project.conf000066400000000000000000000000341514607367700277050ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject.bst000066400000000000000000000000711514607367700302630ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/000077500000000000000000000000001514607367700275535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/project.conf000066400000000000000000000001151514607367700320650ustar00rootroot00000000000000name: subtest min-version: 2.0 junctions: internal: - subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/subsubproject.bst000066400000000000000000000000741514607367700331600ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/subsubproject/000077500000000000000000000000001514607367700324455ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700346770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/subsubprojectname: subsubtest min-version: 2.0 subsub.txt000066400000000000000000000000051514607367700344250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/subsubprojectfile target.bst000066400000000000000000000000671514607367700343710ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/subsubprojectkind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/internal-simple/subproject/subtarget.bst000066400000000000000000000004161514607367700322660ustar00rootroot00000000000000kind: compose # Use a build dependency on our internal subsubproject target. # # There is currently no validation for this but it may be # introduced in the future. For correctness of this test, # use a build dependency. # build-depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/internal-simple/subsubproject-again.bst000066400000000000000000000001021514607367700320450ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject-again apache-buildstream-27ae392/tests/format/junctions/internal-simple/subsubproject-again/000077500000000000000000000000001514607367700313425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/subsubproject-again/project.conf000066400000000000000000000000421514607367700336530ustar00rootroot00000000000000name: subsubtest min-version: 2.0 subsub-again.txt000066400000000000000000000000051514607367700343770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/internal-simple/subsubproject-againfile apache-buildstream-27ae392/tests/format/junctions/internal-simple/subsubproject-again/target.bst000066400000000000000000000000751514607367700333440ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub-again.txt apache-buildstream-27ae392/tests/format/junctions/internal-simple/target.bst000066400000000000000000000001321514607367700273670ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subtarget.bst - subsubproject-again.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/invalid/000077500000000000000000000000001514607367700237165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/invalid/app.bst000066400000000000000000000000641514607367700252100ustar00rootroot00000000000000kind: import sources: - kind: local path: foo.txt apache-buildstream-27ae392/tests/format/junctions/invalid/base-with-deps.bst000066400000000000000000000001061514607367700272410ustar00rootroot00000000000000kind: junction sources: - kind: local path: base depends: - app.bst apache-buildstream-27ae392/tests/format/junctions/invalid/base.bst000066400000000000000000000000631514607367700253410ustar00rootroot00000000000000kind: junction sources: - kind: local path: base apache-buildstream-27ae392/tests/format/junctions/invalid/base/000077500000000000000000000000001514607367700246305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/invalid/base/base.txt000066400000000000000000000000241514607367700262770ustar00rootroot00000000000000This is a text file apache-buildstream-27ae392/tests/format/junctions/invalid/base/project.conf000066400000000000000000000000341514607367700271420ustar00rootroot00000000000000name: base min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/invalid/base/target.bst000066400000000000000000000000651514607367700266310ustar00rootroot00000000000000kind: import sources: - kind: local path: base.txt apache-buildstream-27ae392/tests/format/junctions/invalid/foo.txt000066400000000000000000000000041514607367700252340ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/format/junctions/invalid/junction-dep.bst000066400000000000000000000000401514607367700270210ustar00rootroot00000000000000kind: stack depends: - base.bst apache-buildstream-27ae392/tests/format/junctions/invalid/junction-with-deps.bst000066400000000000000000000001131514607367700301560ustar00rootroot00000000000000kind: stack depends: - junction: base-with-deps.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/invalid/junctiondep-not-a-junction.bst000066400000000000000000000001001514607367700316040ustar00rootroot00000000000000kind: stack depends: - junction: app.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/invalid/project.conf000066400000000000000000000000371514607367700262330ustar00rootroot00000000000000name: invalid min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/invalid/subproject-self-override.bst000066400000000000000000000006411514607367700313550ustar00rootroot00000000000000kind: junction sources: - kind: local path: base # # In this case, the "base" subproject does not really # have a subproject to override, but we're using this # setup to test the error of overriding a subproject # with the junction declaring the override, which will # happen sooner than noticing there is not a subproject.bst # to override. # config: overrides: subproject.bst: subproject-self-override.bst apache-buildstream-27ae392/tests/format/junctions/invalid/target-self-override.bst000066400000000000000000000001001514607367700304510ustar00rootroot00000000000000kind: stack depends: - subproject-self-override.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/missing-element/000077500000000000000000000000001514607367700253705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/missing-element/bad-junction.bst000066400000000000000000000001101514607367700304470ustar00rootroot00000000000000kind: stack depends: - junction: missingfile.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/missing-element/junction-A.bst000066400000000000000000000000701514607367700301060ustar00rootroot00000000000000kind: junction sources: - kind: local path: junctionA apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/000077500000000000000000000000001514607367700273225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/bad-junction-target.bst000066400000000000000000000001221514607367700336700ustar00rootroot00000000000000kind: manual depends: - filename: noelement.bst junction: missing-junction.bst apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/junction-B.bst000066400000000000000000000000701514607367700320410ustar00rootroot00000000000000kind: junction sources: - kind: local path: junctionB apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/junctionB/000077500000000000000000000000001514607367700312555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/junctionB/project.conf000066400000000000000000000000401514607367700335640ustar00rootroot00000000000000name: projectB min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/project.conf000066400000000000000000000000401514607367700316310ustar00rootroot00000000000000name: projectA min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/missing-element/junctionA/target.bst000066400000000000000000000001111514607367700313130ustar00rootroot00000000000000kind: stack depends: - filename: missing.bst junction: junction-B.bst apache-buildstream-27ae392/tests/format/junctions/missing-element/project.conf000066400000000000000000000000341514607367700277020ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/missing-element/sub-target-bad-junction.bst000066400000000000000000000001251514607367700325300ustar00rootroot00000000000000kind: stack depends: - filename: bad-junction-target.bst junction: junction-A.bst apache-buildstream-27ae392/tests/format/junctions/missing-element/sub-target.bst000066400000000000000000000001101514607367700301470ustar00rootroot00000000000000kind: stack depends: - filename: target.bst junction: junction-A.bst apache-buildstream-27ae392/tests/format/junctions/missing-element/target.bst000066400000000000000000000001111514607367700273610ustar00rootroot00000000000000kind: stack depends: - filename: missing.bst junction: junction-A.bst apache-buildstream-27ae392/tests/format/junctions/nested-include/000077500000000000000000000000001514607367700251735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested-include/project.conf000066400000000000000000000001171514607367700275070ustar00rootroot00000000000000name: test min-version: 2.0 (@): subproject.bst:subsubproject.bst:include.yml apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject.bst000066400000000000000000000000711514607367700300630ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/000077500000000000000000000000001514607367700273535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/project.conf000066400000000000000000000001031514607367700316620ustar00rootroot00000000000000name: subtest min-version: 2.0 (@): subsubproject.bst:include.yml apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/sub.txt000066400000000000000000000000051514607367700307000ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subsubproject.bst000066400000000000000000000000741514607367700327600ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subsubproject/000077500000000000000000000000001514607367700322455ustar00rootroot00000000000000include.yml000066400000000000000000000000001514607367700343220ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subsubprojectproject.conf000066400000000000000000000000421514607367700344770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subsubprojectname: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subsubproject/subsub.txt000066400000000000000000000000051514607367700343040ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subsubproject/target.bst000066400000000000000000000000671514607367700342500ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/subtarget.bst000066400000000000000000000000651514607367700320660ustar00rootroot00000000000000kind: stack depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/nested-include/subproject/target.bst000066400000000000000000000000641514607367700313530ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/nested-include/target.bst000066400000000000000000000001211514607367700271650ustar00rootroot00000000000000kind: stack depends: - subproject.bst:target.bst - subproject.bst:subtarget.bst apache-buildstream-27ae392/tests/format/junctions/nested/000077500000000000000000000000001514607367700235525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/deeptarget.bst000066400000000000000000000001631514607367700264100ustar00rootroot00000000000000kind: stack depends: - subproject.bst:target.bst - subproject.bst:subtarget.bst - subproject.bst:subsubtarget.bst apache-buildstream-27ae392/tests/format/junctions/nested/project.conf000066400000000000000000000000341514607367700260640ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/nested/subproject.bst000066400000000000000000000000711514607367700264420ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/nested/subproject/000077500000000000000000000000001514607367700257325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/subproject/project.conf000066400000000000000000000000371514607367700302470ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/nested/subproject/sub.txt000066400000000000000000000000051514607367700272570ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject.bst000066400000000000000000000000741514607367700313370ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/000077500000000000000000000000001514607367700306245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/project.conf000066400000000000000000000000421514607367700331350ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/subsub.txt000066400000000000000000000000051514607367700326630ustar00rootroot00000000000000file subsubsubproject.bst000066400000000000000000000000771514607367700346670ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/subsubsubproject/000077500000000000000000000000001514607367700342305ustar00rootroot00000000000000project.conf000066400000000000000000000000451514607367700364650ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 subsubsub.txt000066400000000000000000000000051514607367700367220ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/subsubsubprojectfile target.bst000066400000000000000000000000721514607367700361500ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: subsubsub.txt apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/subtarget.bst000066400000000000000000000000701514607367700333330ustar00rootroot00000000000000kind: stack depends: - subsubsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubproject/target.bst000066400000000000000000000000671514607367700326270ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subsubtarget.bst000066400000000000000000000000701514607367700311530ustar00rootroot00000000000000kind: stack depends: - subsubproject.bst:subtarget.bst apache-buildstream-27ae392/tests/format/junctions/nested/subproject/subtarget.bst000066400000000000000000000000651514607367700304450ustar00rootroot00000000000000kind: stack depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/nested/subproject/target.bst000066400000000000000000000000641514607367700277320ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/nested/target.bst000066400000000000000000000001211514607367700255440ustar00rootroot00000000000000kind: stack depends: - subproject.bst:target.bst - subproject.bst:subtarget.bst apache-buildstream-27ae392/tests/format/junctions/options/000077500000000000000000000000001514607367700237635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/options/base-default.bst000066400000000000000000000000731514607367700270310ustar00rootroot00000000000000kind: junction sources: - kind: local path: options-base apache-buildstream-27ae392/tests/format/junctions/options/base-explicit.bst000066400000000000000000000001401514607367700272210ustar00rootroot00000000000000kind: junction sources: - kind: local path: options-base config: options: animal: horsy apache-buildstream-27ae392/tests/format/junctions/options/base-propagate.bst000066400000000000000000000001461514607367700273700ustar00rootroot00000000000000kind: junction sources: - kind: local path: options-base config: options: animal: '%{animal}' apache-buildstream-27ae392/tests/format/junctions/options/options-base/000077500000000000000000000000001514607367700263665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/options/options-base/horsy.txt000066400000000000000000000000061514607367700302670ustar00rootroot00000000000000horsy apache-buildstream-27ae392/tests/format/junctions/options/options-base/pony.txt000066400000000000000000000000051514607367700301070ustar00rootroot00000000000000pony apache-buildstream-27ae392/tests/format/junctions/options/options-base/project.conf000066400000000000000000000002371514607367700307050ustar00rootroot00000000000000name: options-base min-version: 2.0 options: animal: type: enum description: The kind of animal values: - pony - horsy default: pony apache-buildstream-27ae392/tests/format/junctions/options/options-base/target.bst000066400000000000000000000002031514607367700303610ustar00rootroot00000000000000kind: import sources: - kind: local (?): - animal == "pony": path: pony.txt - animal == "horsy": path: horsy.txt apache-buildstream-27ae392/tests/format/junctions/options/project.conf000066400000000000000000000000371514607367700263000ustar00rootroot00000000000000name: options min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/options/target-default.bst000066400000000000000000000001111514607367700273760ustar00rootroot00000000000000kind: stack depends: - junction: base-default.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/options/target-explicit.bst000066400000000000000000000001121514607367700275740ustar00rootroot00000000000000kind: stack depends: - junction: base-explicit.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/options/target-propagate.bst000066400000000000000000000001131514607367700277360ustar00rootroot00000000000000kind: stack depends: - junction: base-propagate.bst filename: target.bst apache-buildstream-27ae392/tests/format/junctions/override-element/000077500000000000000000000000001514607367700255365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/element.txt000066400000000000000000000000001514607367700277160ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/override-subproject-dep.bst000066400000000000000000000003431514607367700330130ustar00rootroot00000000000000kind: import sources: - kind: local path: element.txt # # Depend on the subproject using a junction which overrides the dependency # of the element we're depending on. # depends: - subproject-override-dep.bst:subelement.bst override-subproject-element-using-link.bst000066400000000000000000000004321514607367700356720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-elementkind: import sources: - kind: local path: element.txt # # Depend on the subproject using a junction which overrides the # element we're depending on, using a link to address the element # we are overridding. # depends: - subproject-override-element-using-link.bst:subelement.bst override-subproject-element-with-link.bst000066400000000000000000000004461514607367700355250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-elementkind: import sources: - kind: local path: element.txt # # Depend on the subproject using a junction which overrides the # element we're depending on, with a local link to an alternative # element in that same subproject. # depends: - subproject-override-element-with-link.bst:subelement.bst apache-buildstream-27ae392/tests/format/junctions/override-element/override-subproject-element.bst000066400000000000000000000003251514607367700336740ustar00rootroot00000000000000kind: import sources: - kind: local path: element.txt # # Depend on the subproject using a junction which overrides the # element we're depending on. # depends: - subproject-override-element.bst:subelement.bst apache-buildstream-27ae392/tests/format/junctions/override-element/override-subsubproject.bst000066400000000000000000000005751514607367700327660ustar00rootroot00000000000000kind: import sources: - kind: local path: element.txt # # Depend on the subsubproject element, and override it's dependency, the intermediate # project also overrides the same subsubproject element, and this test ensures # that the toplevel override takes precedence. # depends: - subproject-override-subsubproject-element.bst:subsubproject-override-dep.bst:subsubelement.bst apache-buildstream-27ae392/tests/format/junctions/override-element/project.conf000066400000000000000000000000341514607367700300500ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/override-element/subdep-override.bst000066400000000000000000000001001514607367700313360ustar00rootroot00000000000000kind: import sources: - kind: local path: subdep-override.txt apache-buildstream-27ae392/tests/format/junctions/override-element/subdep-override.txt000066400000000000000000000000001514607367700313640ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subelement-alternative-link.bst000066400000000000000000000003451514607367700336640ustar00rootroot00000000000000kind: link # # This link replaces subelement.bst in the same project, use the # same junction to access the alternative in the subproject. # config: target: subproject-override-element-with-link.bst:subelement-alternative.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subelement-override.bst000066400000000000000000000005741514607367700322360ustar00rootroot00000000000000kind: import sources: - kind: local path: subelement-override.txt # We override the subproject direct dependency with this element, # and this element explicitly depends on it's dependency in that # subproject. # # The dependency we depend on happens to be the same dependency # as the element we are overridding. # depends: - subproject-override-element.bst:subdependency.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subelement-override.txt000066400000000000000000000000001514607367700322450ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject-override-dep.bst000066400000000000000000000001721514607367700330130ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subdependency.bst: subdep-override.bst subproject-override-element-using-link.bst000066400000000000000000000002561514607367700356760ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-elementkind: junction sources: - kind: local path: subproject config: overrides: subelement-link.bst: subproject-override-element-using-link.bst:subelement-alternative.bst subproject-override-element-with-link.bst000066400000000000000000000002031514607367700355140ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-elementkind: junction sources: - kind: local path: subproject config: overrides: subelement.bst: subelement-alternative-link.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subproject-override-element.bst000066400000000000000000000001731514607367700336750ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subelement.bst: subelement-override.bst subproject-override-subsubproject-element.bst000066400000000000000000000002341514607367700365040ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-elementkind: junction sources: - kind: local path: subproject config: overrides: subsubproject-override-dep.bst:subsubdependency.bst: subdep-override.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/000077500000000000000000000000001514607367700277165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/project.conf000066400000000000000000000000371514607367700322330ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/sub-alternative.txt000066400000000000000000000000001514607367700335520ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/sub.txt000066400000000000000000000000001514607367700312360ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subdep.txt000066400000000000000000000000001514607367700317270ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subdependency.bst000066400000000000000000000000671514607367700332630ustar00rootroot00000000000000kind: import sources: - kind: local path: subdep.txt subelement-alternative.bst000066400000000000000000000001361514607367700350300ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subprojectkind: import sources: - kind: local path: sub-alternative.txt depends: - subdependency.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subelement-link.bst000066400000000000000000000002371514607367700335300ustar00rootroot00000000000000kind: link # # A link to the subelement, used by the override in # the subproject-override-element-using-link.bst junction # config: target: subelement.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subelement.bst000066400000000000000000000001221514607367700325660ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt depends: - subdependency.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubdep-override.bst000066400000000000000000000001031514607367700342330ustar00rootroot00000000000000kind: import sources: - kind: local path: subsubdep-override.txt apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubdep-override.txt000066400000000000000000000000001514607367700342560ustar00rootroot00000000000000subsubproject-override-dep.bst000066400000000000000000000002031514607367700356210ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subprojectkind: junction sources: - kind: local path: subsubproject config: overrides: subsubdependency.bst: subsubdep-override.bst apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubproject/000077500000000000000000000000001514607367700326105ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700350420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubprojectname: subsubtest min-version: 2.0 subsub.txt000066400000000000000000000000001514607367700345630ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubprojectsubsubdep.txt000066400000000000000000000000001514607367700352540ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubprojectsubsubdependency.bst000066400000000000000000000000721514607367700366040ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubprojectkind: import sources: - kind: local path: subsubdep.txt subsubelement.bst000066400000000000000000000001301514607367700361120ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-element/subproject/subsubprojectkind: import sources: - kind: local path: subsub.txt depends: - subsubdependency.bst apache-buildstream-27ae392/tests/format/junctions/override-self/000077500000000000000000000000001514607367700250365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-self/alternative.bst000066400000000000000000000000501514607367700300610ustar00rootroot00000000000000kind: stack variables: animal: horsy apache-buildstream-27ae392/tests/format/junctions/override-self/project.conf000066400000000000000000000001121514607367700273450ustar00rootroot00000000000000name: test min-version: 2.0 junctions: internal: - self-junction.bst apache-buildstream-27ae392/tests/format/junctions/override-self/self-junction.bst000066400000000000000000000001461514607367700303310ustar00rootroot00000000000000kind: junction sources: - kind: local path: . config: overrides: target.bst: alternative.bst apache-buildstream-27ae392/tests/format/junctions/override-self/target.bst000066400000000000000000000000471514607367700270370ustar00rootroot00000000000000kind: stack variables: animal: pony apache-buildstream-27ae392/tests/format/junctions/override-twice/000077500000000000000000000000001514607367700252205ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/override.bst000066400000000000000000000000671514607367700275540ustar00rootroot00000000000000kind: junction sources: - kind: local path: override apache-buildstream-27ae392/tests/format/junctions/override-twice/override/000077500000000000000000000000001514607367700270375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/override/overridden-again.txt000066400000000000000000000000051514607367700330110ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/override-twice/override/project.conf000066400000000000000000000000451514607367700313530ustar00rootroot00000000000000name: subsubsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/override-twice/override/target.bst000066400000000000000000000001011514607367700310270ustar00rootroot00000000000000kind: import sources: - kind: local path: overridden-again.txt apache-buildstream-27ae392/tests/format/junctions/override-twice/project.conf000066400000000000000000000000341514607367700275320ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject.bst000066400000000000000000000002101514607367700301030ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst:subsubsubproject.bst: override.bst apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/000077500000000000000000000000001514607367700274005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/override.bst000066400000000000000000000000671514607367700317340ustar00rootroot00000000000000kind: junction sources: - kind: local path: override apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/override/000077500000000000000000000000001514607367700312175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/override/overridden.txt000066400000000000000000000000051514607367700341140ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/override/project.conf000066400000000000000000000000451514607367700335330ustar00rootroot00000000000000name: subsubsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/override/target.bst000066400000000000000000000000731514607367700332170ustar00rootroot00000000000000kind: import sources: - kind: local path: overridden.txt apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/project.conf000066400000000000000000000000371514607367700317150ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubproject.bst000066400000000000000000000001711514607367700330030ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject config: overrides: subsubsubproject.bst: override.bst apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubproject/000077500000000000000000000000001514607367700322725ustar00rootroot00000000000000project.conf000066400000000000000000000000421514607367700345240ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubprojectname: subsubtest min-version: 2.0 subsubsubproject.bst000066400000000000000000000000771514607367700363350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject subsubsubproject/000077500000000000000000000000001514607367700356175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubprojectoriginal.txt000066400000000000000000000000051514607367700401570ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubproject/subsubsubprojectfile project.conf000066400000000000000000000000451514607367700401330ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 target.bst000066400000000000000000000000711514607367700376150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/override-twice/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: original.txt apache-buildstream-27ae392/tests/format/junctions/override-twice/target.bst000066400000000000000000000001311514607367700272130ustar00rootroot00000000000000kind: stack depends: - subproject.bst:subsubproject.bst:subsubsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/overrides/000077500000000000000000000000001514607367700242725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/overridden-subsubproject.bst000066400000000000000000000004111514607367700320310ustar00rootroot00000000000000# This junction resides at the toplevel project # # It is used to override the subrpoject's junction to # the subsubproject, and instead point that to the # subsubsubproject. # kind: junction sources: - kind: local path: subproject/subsubproject/subsubsubproject apache-buildstream-27ae392/tests/format/junctions/overrides/overridden-subsubsubproject.bst000066400000000000000000000004011514607367700325420ustar00rootroot00000000000000# This junction resides at the toplevel project # # It is used to override the subrpoject's subsubproject's # junction to the subsubsubproject, and instead point that to # the surpriseproject. # kind: junction sources: - kind: local path: surpriseproject apache-buildstream-27ae392/tests/format/junctions/overrides/project.conf000066400000000000000000000000341514607367700266040ustar00rootroot00000000000000name: test min-version: 2.0 subproject-overriden-with-deep-subproject.bst000066400000000000000000000002221514607367700351360ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrideskind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst: surpriseproject.bst:deepsurpriseproject.bst subproject-with-deep-override-indirect-link.bst000066400000000000000000000002271514607367700353410ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrideskind: junction sources: - kind: local path: subproject config: overrides: subsubsubproject-indirect-link.bst: overridden-subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject-with-deep-override-link.bst000066400000000000000000000002401514607367700336140ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject-link.bst:subsubsubproject.bst: overridden-subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject-with-deep-override.bst000066400000000000000000000002331514607367700326630ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst:subsubsubproject.bst: overridden-subsubsubproject.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject-with-overridden-link.bst000066400000000000000000000002101514607367700332200ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject-link.bst: overridden-subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject-with-override.bst000066400000000000000000000002031514607367700317450ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject config: overrides: subsubproject.bst: overridden-subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject.bst000066400000000000000000000000711514607367700271620ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/000077500000000000000000000000001514607367700264525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/project.conf000066400000000000000000000000371514607367700307670ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/sub.txt000066400000000000000000000000051514607367700277770ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject-link.bst000066400000000000000000000000601514607367700330050ustar00rootroot00000000000000kind: link config: target: subsubproject.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject.bst000066400000000000000000000000741514607367700320570ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/000077500000000000000000000000001514607367700313445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/project.conf000066400000000000000000000000421514607367700336550ustar00rootroot00000000000000name: subsubtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/subsub.txt000066400000000000000000000000051514607367700334030ustar00rootroot00000000000000file subsubsubproject-link.bst000066400000000000000000000000631514607367700363350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubprojectkind: link config: target: subsubsubproject.bst subsubsubproject.bst000066400000000000000000000000771514607367700354070ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubprojectkind: junction sources: - kind: local path: subsubsubproject subsubsubproject/000077500000000000000000000000001514607367700346715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubprojectproject.conf000066400000000000000000000000451514607367700372050ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/subsubsubprojectname: subsubsubtest min-version: 2.0 subsubsub.txt000066400000000000000000000000051514607367700374420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/subsubsubprojectfile target.bst000066400000000000000000000000721514607367700366700ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/subsubsubprojectkind: import sources: - kind: local path: subsubsub.txt apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/subsubproject/target.bst000066400000000000000000000000671514607367700333470ustar00rootroot00000000000000kind: import sources: - kind: local path: subsub.txt subsubsubproject-indirect-link.bst000066400000000000000000000001171514607367700352420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/subprojectkind: link config: target: subsubproject-link.bst:subsubsubproject-link.bst apache-buildstream-27ae392/tests/format/junctions/overrides/subproject/target.bst000066400000000000000000000000641514607367700304520ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject.bst000066400000000000000000000000761514607367700302520ustar00rootroot00000000000000kind: junction sources: - kind: local path: surpriseproject apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/000077500000000000000000000000001514607367700275355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/deepsurpriseproject.bst000066400000000000000000000001021514607367700343410ustar00rootroot00000000000000kind: junction sources: - kind: local path: deepsurpriseproject apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/deepsurpriseproject/000077500000000000000000000000001514607367700336365ustar00rootroot00000000000000deepsurprise.txt000066400000000000000000000000051514607367700370250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/deepsurpriseprojectfile project.conf000066400000000000000000000000441514607367700360720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/deepsurpriseprojectname: deepsurprise min-version: 2.0 target.bst000066400000000000000000000000751514607367700355610ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/deepsurpriseprojectkind: import sources: - kind: local path: deepsurprise.txt apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/project.conf000066400000000000000000000000401514607367700320440ustar00rootroot00000000000000name: surprise min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/surprise.txt000066400000000000000000000000051514607367700321450ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/overrides/surpriseproject/target.bst000066400000000000000000000000711514607367700315330ustar00rootroot00000000000000kind: import sources: - kind: local path: surprise.txt apache-buildstream-27ae392/tests/format/junctions/overrides/target-overridden-subsubproject-link.bst000066400000000000000000000003771514607367700342630ustar00rootroot00000000000000kind: stack # Similar test as target-overridden-subsubproject.bst, but we # test that this works equally if we are referring to a subsubproject # which is in fact a link. # depends: - subproject-with-overridden-link.bst:subsubproject-link.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/overrides/target-overridden-subsubproject.bst000066400000000000000000000006031514607367700333200ustar00rootroot00000000000000kind: stack # Here we depend on the target in subproject's subsubproject, # however we've overridden the subproject's subsubproject with # our own. # # We should still be able to address that overridden subproject # and access the project we've overridden it with, which will # turn out to be the subsubsubproject. # depends: - subproject-with-override.bst:subsubproject.bst:target.bst target-overridden-subsubsubproject-indirect-link.bst000066400000000000000000000003611514607367700365060ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrideskind: stack # Similar test as target-overridden-subsubsubproject.bst, except that # we use links in the addressing of overrides. # depends: - subproject-with-deep-override-indirect-link.bst:subsubproject.bst:subsubsubproject.bst:target.bst target-overridden-subsubsubproject-link.bst000066400000000000000000000003501514607367700347050ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrideskind: stack # Similar test as target-overridden-subsubsubproject.bst, except that # we use links in the addressing of overrides. # depends: - subproject-with-deep-override-link.bst:subsubproject.bst:subsubsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/overrides/target-overridden-subsubsubproject.bst000066400000000000000000000004541514607367700340360ustar00rootroot00000000000000kind: stack # Here we depend on the target in subproject's subsubproject's # subsubsubproject, however we've overridden the subproject's # subsubproject's subsubsubproject with our own surprise project # depends: - subproject-with-deep-override.bst:subsubproject.bst:subsubsubproject.bst:target.bst target-overridden-with-deepsubproject.bst000066400000000000000000000001431514607367700343350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/overrideskind: stack depends: - subproject-overriden-with-deep-subproject.bst:subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/simple/000077500000000000000000000000001514607367700235615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/simple/project.conf000066400000000000000000000000341514607367700260730ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/simple/subproject.bst000066400000000000000000000000711514607367700264510ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/junctions/simple/subproject/000077500000000000000000000000001514607367700257415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/simple/subproject/base.txt000066400000000000000000000000051514607367700274070ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/format/junctions/simple/subproject/project.conf000066400000000000000000000000371514607367700302560ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/simple/subproject/target.bst000066400000000000000000000000651514607367700277420ustar00rootroot00000000000000kind: import sources: - kind: local path: base.txt apache-buildstream-27ae392/tests/format/junctions/simple/target.bst000066400000000000000000000000621514607367700255570ustar00rootroot00000000000000kind: stack depends: - subproject.bst:target.bst apache-buildstream-27ae392/tests/format/junctions/use-repo/000077500000000000000000000000001514607367700240275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/use-repo/baserepo/000077500000000000000000000000001514607367700256275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/junctions/use-repo/baserepo/base.txt000066400000000000000000000000241514607367700272760ustar00rootroot00000000000000This is a text file apache-buildstream-27ae392/tests/format/junctions/use-repo/baserepo/project.conf000066400000000000000000000000341514607367700301410ustar00rootroot00000000000000name: base min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/use-repo/baserepo/target.bst000066400000000000000000000000651514607367700276300ustar00rootroot00000000000000kind: import sources: - kind: local path: base.txt apache-buildstream-27ae392/tests/format/junctions/use-repo/project.conf000066400000000000000000000000331514607367700263400ustar00rootroot00000000000000name: foo min-version: 2.0 apache-buildstream-27ae392/tests/format/junctions/use-repo/target.bst000066400000000000000000000001011514607367700260170ustar00rootroot00000000000000kind: stack depends: - junction: base.bst filename: target.bst apache-buildstream-27ae392/tests/format/link.py000066400000000000000000000207651514607367700215750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain, LoadErrorReason DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "link", ) # # Test links to elements, this tests both specifying the link as # the main target, and also as a dependency of the main target. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("target", ["target.bst", "hello-link.bst"]) def test_simple_link(cli, tmpdir, datafiles, target): project = os.path.join(str(datafiles), "simple") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from sub-sub-project assert os.path.exists(os.path.join(checkoutdir, "hello.txt")) # # Test links to elements, this tests both specifying the link as # the main target, and also as a dependency of the main target, while # also using a conditional statement in the link # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("target", ["target.bst", "target-link.bst"]) @pytest.mark.parametrize("greeting,expected_file", [("hello", "hello.txt"), ("goodbye", "goodbye.txt")]) def test_conditional_link(cli, tmpdir, datafiles, target, greeting, expected_file): project = os.path.join(str(datafiles), "conditional") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["-o", "greeting", greeting, "build", target]) result.assert_success() result = cli.run( project=project, args=["-o", "greeting", greeting, "artifact", "checkout", target, "--directory", checkoutdir] ) result.assert_success() # Check that the checkout contains the expected files from sub-sub-project assert os.path.exists(os.path.join(checkoutdir, expected_file)) # # Test links to junctions from local projects and subprojects # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target", ["target-local.bst", "target-nested.bst", "full-path-link.bst", "target-full-path.bst"] ) def test_simple_junctions(cli, tmpdir, datafiles, target): project = os.path.join(str(datafiles), "simple-junctions") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from sub-sub-project assert os.path.exists(os.path.join(checkoutdir, "hello.txt")) # # Test links which resolve junction targets conditionally # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("greeting,expected_file", [("hello", "hello.txt"), ("goodbye", "goodbye.txt")]) def test_conditional_junctions(cli, tmpdir, datafiles, greeting, expected_file): project = os.path.join(str(datafiles), "conditional-junctions") checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["-o", "greeting", greeting, "build", "target.bst"]) result.assert_success() result = cli.run( project=project, args=["-o", "greeting", greeting, "artifact", "checkout", "target.bst", "--directory", checkoutdir], ) result.assert_success() # Check that the checkout contains the expected files from sub-sub-project assert os.path.exists(os.path.join(checkoutdir, expected_file)) # # Tests links which refer to non-existing elements or junctions # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,provenance", [ # Target is a link to a non-existing local element ("link-target.bst", "link-target.bst [line 4 column 10]"), # Target is a stack depending on a link to a non-existing local element ( "depends-on-link-target.bst", "link-target.bst [line 4 column 10]", ), # Depends on non-existing subproject element, via a local link ( "linked-local-junction-target.bst", "linked-local-junction-target.bst [line 4 column 2]", ), # Depends on non-existing subsubproject element, via a local link ( "linked-nested-junction-target.bst", "linked-nested-junction-target.bst [line 4 column 2]", ), # Depends on an element via a link to a non-existing local junction ( "linked-local-junction.bst", "subproject-link-notfound.bst [line 4 column 10]", ), # Depends on an element via a link to a non-existing subproject junction ( "linked-nested-junction.bst", "subsubproject-link-notfound.bst [line 4 column 10]", ), # Target is a link to a non-existing nested element referred to with a full path ("link-full-path.bst", "link-full-path.bst [line 4 column 10]"), # Target depends on a link to a non-existing nested element referred to with a full path ("target-full-path.bst", "link-full-path.bst [line 4 column 10]"), ], ) def test_link_not_found(cli, tmpdir, datafiles, target, provenance): project = os.path.join(str(datafiles), "notfound") result = cli.run(project=project, args=["build", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) assert provenance in result.stderr # # Tests links with invalid configurations # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,expected_error,expected_reason", [ # Test link which declares sources, either directly of via a dependency ("link-with-sources.bst", ErrorDomain.ELEMENT, "element-forbidden-sources"), ("target-link-with-sources.bst", ErrorDomain.ELEMENT, "element-forbidden-sources"), # Test link which declares dependencies, either directly of via a dependency ("link-with-dependencies.bst", ErrorDomain.LOAD, LoadErrorReason.LINK_FORBIDDEN_DEPENDENCIES), ], ) def test_link_invalid_config(cli, tmpdir, datafiles, target, expected_error, expected_reason): project = os.path.join(str(datafiles), "invalid") result = cli.run(project=project, args=["show", target]) result.assert_main_error(expected_error, expected_reason) # # Test including files across the boundry a link to a subproject's junction # @pytest.mark.datafiles(DATA_DIR) def test_cross_link_junction_include(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "cross-link-junction-include") # Show the variables and parse our test variable from the subsubproject result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{vars}", "target.bst"]) result.assert_success() # Read back some of our project defaults from the env variables = _yaml.load_data(result.output) assert variables.get_str("test") == "the test" # # Test two links to the same element, both links are dependencies of the build target. # @pytest.mark.datafiles(DATA_DIR) def test_multiple_links_same_target(cli, tmpdir, datafiles): project = os.path.join(str(datafiles), "multiple-links-same-target") checkoutdir = os.path.join(str(tmpdir), "checkout") target = "target.bst" # Build, checkout result = cli.run(project=project, args=["build", target]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected files from sub-sub-project assert os.path.exists(os.path.join(checkoutdir, "hello.txt")) apache-buildstream-27ae392/tests/format/link/000077500000000000000000000000001514607367700212115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/000077500000000000000000000000001514607367700255265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/elements/000077500000000000000000000000001514607367700273425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/elements/subproject.bst000066400000000000000000000000721514607367700322330ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/link/conditional-junctions/elements/subsubproject-link.bst000066400000000000000000000003151514607367700337000ustar00rootroot00000000000000kind: link config: (?): - greeting == "hello": target: subproject.bst:subsubproject-hello-junction.bst - greeting == "goodbye": target: subproject.bst:subsubproject-goodbye-junction.bst apache-buildstream-27ae392/tests/format/link/conditional-junctions/elements/target.bst000066400000000000000000000000721514607367700313410ustar00rootroot00000000000000kind: stack depends: - subsubproject-link.bst:target.bst apache-buildstream-27ae392/tests/format/link/conditional-junctions/project.conf000066400000000000000000000002671514607367700300500ustar00rootroot00000000000000name: conditional min-version: 2.0 element-path: elements options: greeting: type: enum description: The greeting values: - hello - goodbye default: hello apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/000077500000000000000000000000001514607367700277065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/elements/000077500000000000000000000000001514607367700315225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/elements/hello.bst000066400000000000000000000000751514607367700333410ustar00rootroot00000000000000kind: import sources: - kind: local path: files/hello.txt subsubproject-goodbye-junction.bst000066400000000000000000000001051514607367700403200ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/elementskind: junction sources: - kind: local path: subsubproject-goodbye subsubproject-hello-junction.bst000066400000000000000000000001031514607367700377710ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/elementskind: junction sources: - kind: local path: subsubproject-hello apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/project.conf000066400000000000000000000000721514607367700322220ustar00rootroot00000000000000name: subproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-goodbye/000077500000000000000000000000001514607367700342265ustar00rootroot00000000000000elements/000077500000000000000000000000001514607367700357635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-goodbyetarget.bst000066400000000000000000000000771514607367700377670ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-goodbye/elementskind: import sources: - kind: local path: files/goodbye.txt files/000077500000000000000000000000001514607367700352515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-goodbyegoodbye.txt000066400000000000000000000000061514607367700374360ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-goodbye/fileshello project.conf000066400000000000000000000000751514607367700364660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-goodbyename: subsubproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-hello/000077500000000000000000000000001514607367700337015ustar00rootroot00000000000000elements/000077500000000000000000000000001514607367700354365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-hellotarget.bst000066400000000000000000000000751514607367700374400ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-hello/elementskind: import sources: - kind: local path: files/hello.txt files/000077500000000000000000000000001514607367700347245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-hellohello.txt000066400000000000000000000000061514607367700365640ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-hello/fileshello project.conf000066400000000000000000000000751514607367700361410ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional-junctions/subproject/subsubproject-helloname: subsubproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/conditional/000077500000000000000000000000001514607367700235145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional/elements/000077500000000000000000000000001514607367700253305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional/elements/goodbye.bst000066400000000000000000000000771514607367700274760ustar00rootroot00000000000000kind: import sources: - kind: local path: files/goodbye.txt apache-buildstream-27ae392/tests/format/link/conditional/elements/hello.bst000066400000000000000000000000751514607367700271470ustar00rootroot00000000000000kind: import sources: - kind: local path: files/hello.txt apache-buildstream-27ae392/tests/format/link/conditional/elements/target-link.bst000066400000000000000000000002011514607367700302540ustar00rootroot00000000000000kind: link config: (?): - greeting == "hello": target: hello.bst - greeting == "goodbye": target: goodbye.bst apache-buildstream-27ae392/tests/format/link/conditional/elements/target.bst000066400000000000000000000000501514607367700273230ustar00rootroot00000000000000kind: stack depends: - target-link.bst apache-buildstream-27ae392/tests/format/link/conditional/files/000077500000000000000000000000001514607367700246165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/conditional/files/goodbye.txt000066400000000000000000000000061514607367700270030ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/format/link/conditional/files/hello.txt000066400000000000000000000000061514607367700264560ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/format/link/conditional/project.conf000066400000000000000000000002671514607367700260360ustar00rootroot00000000000000name: conditional min-version: 2.0 element-path: elements options: greeting: type: enum description: The greeting values: - hello - goodbye default: hello apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/000077500000000000000000000000001514607367700265455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/project.conf000066400000000000000000000000751514607367700310640ustar00rootroot00000000000000name: test min-version: 2.0 (@): subsubproject.bst:file.yml apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject.bst000066400000000000000000000000711514607367700314350ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/000077500000000000000000000000001514607367700307255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/project.conf000066400000000000000000000000371514607367700332420ustar00rootroot00000000000000name: subtest min-version: 2.0 apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/sub.txt000066400000000000000000000000051514607367700322520ustar00rootroot00000000000000file subsubproject.bst000066400000000000000000000000741514607367700342530ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subprojectkind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subsubproject/000077500000000000000000000000001514607367700336175ustar00rootroot00000000000000file.yml000066400000000000000000000000341514607367700351770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subsubprojectvariables: test: the test project.conf000066400000000000000000000000421514607367700360510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subsubprojectname: subsubtest min-version: 2.0 subsub.txt000066400000000000000000000000051514607367700355770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subsubprojectfile subtarget.bst000066400000000000000000000000701514607367700362470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subsubprojectkind: stack depends: - subsubsubproject.bst:target.bst target.bst000066400000000000000000000000671514607367700355430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subsubprojectkind: import sources: - kind: local path: subsub.txt apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/subtarget.bst000066400000000000000000000000651514607367700334400ustar00rootroot00000000000000kind: stack depends: - subsubproject.bst:target.bst apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subproject/target.bst000066400000000000000000000000641514607367700327250ustar00rootroot00000000000000kind: import sources: - kind: local path: sub.txt apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/subsubproject.bst000066400000000000000000000000771514607367700321550ustar00rootroot00000000000000kind: link config: target: subproject.bst:subsubproject.bst apache-buildstream-27ae392/tests/format/link/cross-link-junction-include/target.bst000066400000000000000000000001211514607367700305370ustar00rootroot00000000000000kind: stack depends: - subproject.bst:target.bst - subproject.bst:subtarget.bst apache-buildstream-27ae392/tests/format/link/invalid/000077500000000000000000000000001514607367700226375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/invalid/elements/000077500000000000000000000000001514607367700244535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/invalid/elements/base-file.bst000066400000000000000000000000571514607367700270160ustar00rootroot00000000000000kind: import sources: - kind: local path: . apache-buildstream-27ae392/tests/format/link/invalid/elements/link-with-dependencies.bst000066400000000000000000000001061514607367700315140ustar00rootroot00000000000000kind: link depends: - base-file.bst config: target: base-file.bst apache-buildstream-27ae392/tests/format/link/invalid/elements/link-with-sources.bst000066400000000000000000000001161514607367700305520ustar00rootroot00000000000000kind: link sources: - kind: local path: . config: target: base-file.bst apache-buildstream-27ae392/tests/format/link/invalid/elements/target-link-with-sources.bst000066400000000000000000000000561514607367700320410ustar00rootroot00000000000000kind: stack depends: - link-with-sources.bst apache-buildstream-27ae392/tests/format/link/invalid/project.conf000066400000000000000000000000671514607367700251570ustar00rootroot00000000000000name: invalid min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/000077500000000000000000000000001514607367700263715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/elements/000077500000000000000000000000001514607367700302055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/elements/hello-link-1.bst000066400000000000000000000000501514607367700331060ustar00rootroot00000000000000kind: link config: target: hello.bst apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/elements/hello-link-2.bst000066400000000000000000000000501514607367700331070ustar00rootroot00000000000000kind: link config: target: hello.bst apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/elements/hello.bst000066400000000000000000000000751514607367700320240ustar00rootroot00000000000000kind: import sources: - kind: local path: files/hello.txt apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/elements/target.bst000066400000000000000000000000741514607367700322060ustar00rootroot00000000000000kind: stack depends: - hello-link-1.bst - hello-link-2.bst apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/files/000077500000000000000000000000001514607367700274735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/files/hello.txt000066400000000000000000000000061514607367700313330ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/format/link/multiple-links-same-target/project.conf000066400000000000000000000001141514607367700307020ustar00rootroot00000000000000name: multiple-links-single-target min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/notfound/000077500000000000000000000000001514607367700230455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/notfound/elements/000077500000000000000000000000001514607367700246615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/notfound/elements/depends-on-link-target.bst000066400000000000000000000000501514607367700316410ustar00rootroot00000000000000kind: stack depends: - link-target.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/link-full-path.bst000066400000000000000000000001211514607367700302140ustar00rootroot00000000000000kind: link config: target: subproject.bst:subsubproject-junction.bst:pony.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/link-target.bst000066400000000000000000000000631514607367700276130ustar00rootroot00000000000000kind: link config: target: no-element-found.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/linked-local-junction-target.bst000066400000000000000000000000661514607367700330460ustar00rootroot00000000000000kind: stack depends: - subproject-link.bst:hello.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/linked-local-junction.bst000066400000000000000000000001011514607367700315500ustar00rootroot00000000000000kind: stack depends: - subproject-link-notfound.bst:element.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/linked-nested-junction-target.bst000066400000000000000000000000711514607367700332320ustar00rootroot00000000000000kind: stack depends: - subsubproject-link.bst:hello.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/linked-nested-junction.bst000066400000000000000000000001041514607367700317430ustar00rootroot00000000000000kind: stack depends: - subsubproject-link-notfound.bst:element.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/subproject-link-notfound.bst000066400000000000000000000000661514607367700323420ustar00rootroot00000000000000kind: link config: target: subproject-notfound.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/subproject-link.bst000066400000000000000000000000551514607367700305060ustar00rootroot00000000000000kind: link config: target: subproject.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/subproject.bst000066400000000000000000000000721514607367700275520ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/link/notfound/elements/subsubproject-link-notfound.bst000066400000000000000000000001211514607367700330440ustar00rootroot00000000000000kind: link config: target: subproject.bst:subsubproject-junction-notfound.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/subsubproject-link.bst000066400000000000000000000001101514607367700312100ustar00rootroot00000000000000kind: link config: target: subproject.bst:subsubproject-junction.bst apache-buildstream-27ae392/tests/format/link/notfound/elements/target-full-path.bst000066400000000000000000000000531514607367700305510ustar00rootroot00000000000000kind: stack depends: - link-full-path.bst apache-buildstream-27ae392/tests/format/link/notfound/project.conf000066400000000000000000000000701514607367700253570ustar00rootroot00000000000000name: notfound min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/notfound/subproject/000077500000000000000000000000001514607367700252255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/notfound/subproject/elements/000077500000000000000000000000001514607367700270415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/notfound/subproject/elements/subsubproject-junction.bst000066400000000000000000000000751514607367700342760ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/link/notfound/subproject/project.conf000066400000000000000000000000721514607367700275410ustar00rootroot00000000000000name: subproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/notfound/subproject/subsubproject/000077500000000000000000000000001514607367700301175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/notfound/subproject/subsubproject/project.conf000066400000000000000000000000451514607367700324330ustar00rootroot00000000000000name: subsubproject min-version: 2.0 apache-buildstream-27ae392/tests/format/link/simple-junctions/000077500000000000000000000000001514607367700245145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/000077500000000000000000000000001514607367700263305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/full-path-link.bst000066400000000000000000000001221514607367700316640ustar00rootroot00000000000000kind: link config: target: subproject.bst:subsubproject-junction.bst:hello.bst apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/subproject-link.bst000066400000000000000000000000551514607367700321550ustar00rootroot00000000000000kind: link config: target: subproject.bst apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/subproject.bst000066400000000000000000000000721514607367700312210ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/subsubproject-link.bst000066400000000000000000000001101514607367700326570ustar00rootroot00000000000000kind: link config: target: subproject.bst:subsubproject-junction.bst apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/target-full-path.bst000066400000000000000000000000531514607367700322200ustar00rootroot00000000000000kind: stack depends: - full-path-link.bst apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/target-local.bst000066400000000000000000000000661514607367700314220ustar00rootroot00000000000000kind: stack depends: - subproject-link.bst:hello.bst apache-buildstream-27ae392/tests/format/link/simple-junctions/elements/target-nested.bst000066400000000000000000000000711514607367700316060ustar00rootroot00000000000000kind: stack depends: - subsubproject-link.bst:hello.bst apache-buildstream-27ae392/tests/format/link/simple-junctions/project.conf000066400000000000000000000000661514607367700270330ustar00rootroot00000000000000name: simple min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/000077500000000000000000000000001514607367700266745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/elements/000077500000000000000000000000001514607367700305105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/elements/hello.bst000066400000000000000000000000751514607367700323270ustar00rootroot00000000000000kind: import sources: - kind: local path: files/hello.txt subsubproject-junction.bst000066400000000000000000000000751514607367700356660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/elementskind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/files/000077500000000000000000000000001514607367700277765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/files/hello.txt000066400000000000000000000000061514607367700316360ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/project.conf000066400000000000000000000000721514607367700312100ustar00rootroot00000000000000name: subproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/subsubproject/000077500000000000000000000000001514607367700315665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/subsubproject/elements/000077500000000000000000000000001514607367700334025ustar00rootroot00000000000000hello.bst000066400000000000000000000000751514607367700351420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/subsubproject/elementskind: import sources: - kind: local path: files/hello.txt apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/subsubproject/files/000077500000000000000000000000001514607367700326705ustar00rootroot00000000000000hello.txt000066400000000000000000000000061514607367700344510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/subsubproject/fileshello apache-buildstream-27ae392/tests/format/link/simple-junctions/subproject/subsubproject/project.conf000066400000000000000000000000751514607367700341050ustar00rootroot00000000000000name: subsubproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/link/simple/000077500000000000000000000000001514607367700225025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple/elements/000077500000000000000000000000001514607367700243165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple/elements/hello-link.bst000066400000000000000000000000501514607367700270610ustar00rootroot00000000000000kind: link config: target: hello.bst apache-buildstream-27ae392/tests/format/link/simple/elements/hello.bst000066400000000000000000000000751514607367700261350ustar00rootroot00000000000000kind: import sources: - kind: local path: files/hello.txt apache-buildstream-27ae392/tests/format/link/simple/elements/target.bst000066400000000000000000000000471514607367700263170ustar00rootroot00000000000000kind: stack depends: - hello-link.bst apache-buildstream-27ae392/tests/format/link/simple/files/000077500000000000000000000000001514607367700236045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/link/simple/files/hello.txt000066400000000000000000000000061514607367700254440ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/format/link/simple/project.conf000066400000000000000000000000661514607367700250210ustar00rootroot00000000000000name: simple min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/list-directive-error-element/000077500000000000000000000000001514607367700257615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/list-directive-error-element/config.bst000066400000000000000000000002111514607367700277320ustar00rootroot00000000000000kind: manual config: outigration-frob-mans: (>): - name: Appending to something - value: that is not integration-commands apache-buildstream-27ae392/tests/format/list-directive-error-element/environment.bst000066400000000000000000000001531514607367700310360ustar00rootroot00000000000000kind: manual environment: foo: (=): - name: Overriding - value: Something that doesnt exist apache-buildstream-27ae392/tests/format/list-directive-error-element/project.conf000066400000000000000000000000341514607367700302730ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/list-directive-error-element/public.bst000066400000000000000000000002021514607367700277430ustar00rootroot00000000000000kind: manual public: foo: the-foo-list: (>): - name: Appending to something - value: that does not exist apache-buildstream-27ae392/tests/format/list-directive-error-element/variables.bst000066400000000000000000000001541514607367700304430ustar00rootroot00000000000000kind: manual variables: foo: (<): - name: Prepending - value: To something that doesnt exist apache-buildstream-27ae392/tests/format/list-directive-error-project/000077500000000000000000000000001514607367700257765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/list-directive-error-project/element.bst000066400000000000000000000000151514607367700301350ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/list-directive-error-project/project.conf000066400000000000000000000001551514607367700303140ustar00rootroot00000000000000name: test min-version: 2.0 artifacts: (>): - name: Appending - value: To something that doesnt exist apache-buildstream-27ae392/tests/format/list-directive-type-error/000077500000000000000000000000001514607367700253115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/list-directive-type-error/element.bst000066400000000000000000000001361514607367700274540ustar00rootroot00000000000000kind: manual sources: (?): - arch == "x86_64": - url: https://example.com/x86_64 apache-buildstream-27ae392/tests/format/list-directive-type-error/project.conf000066400000000000000000000002211514607367700276210ustar00rootroot00000000000000name: test min-version: 2.0 options: arch: type: arch description: Example architecture option values: [ x86-32, x86-64, aarch64 ]apache-buildstream-27ae392/tests/format/listdirectiveerrors.py000066400000000000000000000041141514607367700247350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) def test_project_error(cli, datafiles): project = os.path.join(datafiles, "list-directive-error-project") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.TRAILING_LIST_DIRECTIVE) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("target", [("variables.bst"), ("environment.bst"), ("config.bst"), ("public.bst")]) def test_element_error(cli, datafiles, target): project = os.path.join(datafiles, "list-directive-error-element") result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.TRAILING_LIST_DIRECTIVE) @pytest.mark.datafiles(DATA_DIR) def test_project_composite_error(cli, datafiles): project = os.path.join(datafiles, "list-directive-type-error") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.ILLEGAL_COMPOSITE) apache-buildstream-27ae392/tests/format/option-arch-alias/000077500000000000000000000000001514607367700235665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-arch-alias/element.bst000066400000000000000000000002311514607367700257250ustar00rootroot00000000000000kind: manual variables: result: "Nothing" (?): - machine_arch == "arm": result: "Army" - machine_arch == "x86_64": result: "X86-64y" apache-buildstream-27ae392/tests/format/option-arch-alias/project.conf000066400000000000000000000002221514607367700260770ustar00rootroot00000000000000name: test min-version: 2.0 options: machine_arch: type: arch description: The machine architecture values: - arm - x86_64 apache-buildstream-27ae392/tests/format/option-arch-unknown/000077500000000000000000000000001514607367700241745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-arch-unknown/element.bst000066400000000000000000000003251514607367700263370ustar00rootroot00000000000000kind: manual variables: result: "Nothing" (?): - machine_arch == "aarch32": result: "Army" - machine_arch == "aarch64": result: "Aarchy" - machine_arch == "x86-128": result: "X86-128y" apache-buildstream-27ae392/tests/format/option-arch-unknown/project.conf000066400000000000000000000002451514607367700265120ustar00rootroot00000000000000name: test min-version: 2.0 options: machine_arch: type: arch description: The machine architecture values: - aarch32 - aarch64 - x86-128 apache-buildstream-27ae392/tests/format/option-arch/000077500000000000000000000000001514607367700224775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-arch/element.bst000066400000000000000000000002351514607367700246420ustar00rootroot00000000000000kind: manual variables: result: "Nothing" (?): - machine_arch == "aarch32": result: "Army" - machine_arch == "aarch64": result: "Aarchy" apache-buildstream-27ae392/tests/format/option-arch/project.conf000066400000000000000000000002271514607367700250150ustar00rootroot00000000000000name: test min-version: 2.0 options: machine_arch: type: arch description: The machine architecture values: - aarch32 - aarch64 apache-buildstream-27ae392/tests/format/option-bool/000077500000000000000000000000001514607367700225155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-bool/element-equals.bst000066400000000000000000000001371514607367700261510ustar00rootroot00000000000000kind: manual variables: thepony: "not pony" (?): - pony == True: thepony: "a pony" apache-buildstream-27ae392/tests/format/option-bool/element-not-equals.bst000066400000000000000000000001371514607367700267470ustar00rootroot00000000000000kind: manual variables: thepony: "a pony" (?): - pony != True: thepony: "not pony" apache-buildstream-27ae392/tests/format/option-bool/element-not.bst000066400000000000000000000001331514607367700254530ustar00rootroot00000000000000kind: manual variables: thepony: "a pony" (?): - not pony: thepony: "not pony" apache-buildstream-27ae392/tests/format/option-bool/element.bst000066400000000000000000000001271514607367700246600ustar00rootroot00000000000000kind: manual variables: thepony: "not pony" (?): - pony: thepony: "a pony" apache-buildstream-27ae392/tests/format/option-bool/project.conf000066400000000000000000000001671514607367700250360ustar00rootroot00000000000000name: test min-version: 2.0 options: pony: type: bool description: Whether a pony or not default: False apache-buildstream-27ae392/tests/format/option-element-mask-invalid/000077500000000000000000000000001514607367700255705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-element-mask-invalid/pony.bst000066400000000000000000000001441514607367700272660ustar00rootroot00000000000000kind: manual variables: debug: False (?): - ("pony.bst" in debug_elements): debug: True apache-buildstream-27ae392/tests/format/option-element-mask-invalid/project.conf000066400000000000000000000003701514607367700301050ustar00rootroot00000000000000name: test min-version: 2.0 options: debug_elements: type: element-mask description: The elements to build in debug mode # Values are not allowed to be declared on element mask options values: - pony - horsy - zebry apache-buildstream-27ae392/tests/format/option-element-mask/000077500000000000000000000000001514607367700241445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-element-mask/giraffy.bst000066400000000000000000000001471514607367700263070ustar00rootroot00000000000000kind: manual variables: debug: False (?): - ("giraffy.bst" in debug_elements): debug: True apache-buildstream-27ae392/tests/format/option-element-mask/horsy.bst000066400000000000000000000001451514607367700260220ustar00rootroot00000000000000kind: manual variables: debug: False (?): - ("horsy.bst" in debug_elements): debug: True apache-buildstream-27ae392/tests/format/option-element-mask/pony.bst000066400000000000000000000001441514607367700256420ustar00rootroot00000000000000kind: manual variables: debug: False (?): - ("pony.bst" in debug_elements): debug: True apache-buildstream-27ae392/tests/format/option-element-mask/project.conf000066400000000000000000000002041514607367700264550ustar00rootroot00000000000000name: test min-version: 2.0 options: debug_elements: type: element-mask description: The elements to build in debug mode apache-buildstream-27ae392/tests/format/option-element-mask/zebry.bst000066400000000000000000000001451514607367700260110ustar00rootroot00000000000000kind: manual variables: debug: False (?): - ("zebry.bst" in debug_elements): debug: True apache-buildstream-27ae392/tests/format/option-element-override/000077500000000000000000000000001514607367700250305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-element-override/element.bst000066400000000000000000000000151514607367700271670ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-element-override/project.conf000066400000000000000000000005511514607367700273460ustar00rootroot00000000000000name: test min-version: 2.0 options: animal: type: enum description: The kind of animal values: - pony - horsy default: pony variables: result: "a sloppy joe" elements: manual: (?): - animal == "pony": variables: result: "a pony" - animal == "horsy": variables: result: "a horsy" apache-buildstream-27ae392/tests/format/option-enum-missing/000077500000000000000000000000001514607367700241755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-enum-missing/element.bst000066400000000000000000000000151514607367700263340ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-enum-missing/project.conf000066400000000000000000000002041514607367700265060ustar00rootroot00000000000000name: test min-version: 2.0 options: empty: type: enum description: Invalid because no values are defined values: [] apache-buildstream-27ae392/tests/format/option-enum/000077500000000000000000000000001514607367700225265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-enum/element-compare.bst000066400000000000000000000001411514607367700263110ustar00rootroot00000000000000kind: manual variables: result: "different" (?): - brother == sister: result: "same" apache-buildstream-27ae392/tests/format/option-enum/element.bst000066400000000000000000000002221514607367700246650ustar00rootroot00000000000000kind: manual variables: result: "a pony" (?): - brother == "zebry": result: "a zebry" - brother == "horsy": result: "a horsy" apache-buildstream-27ae392/tests/format/option-enum/project.conf000066400000000000000000000005011514607367700250370ustar00rootroot00000000000000name: test min-version: 2.0 options: brother: type: enum description: The kind of animal of the brother values: - pony - horsy - zebry default: pony sister: type: enum description: The kind of animal of the sister values: - pony - horsy - zebry default: zebry apache-buildstream-27ae392/tests/format/option-exports/000077500000000000000000000000001514607367700232665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-exports/element.bst000066400000000000000000000001201514607367700254220ustar00rootroot00000000000000kind: config config: animal: "%{exported-enum}" sleepy: "%{exported-bool}" apache-buildstream-27ae392/tests/format/option-exports/plugins/000077500000000000000000000000001514607367700247475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-exports/plugins/config.py000066400000000000000000000007151514607367700265710ustar00rootroot00000000000000from buildstream import Element, FastEnum class AnimalEnum(FastEnum): PONY = "pony" HORSY = "horsy" ZEBRY = "zebry" class Config(Element): BST_MIN_VERSION = "2.0" def configure(self, node): self.animal = node.get_enum("animal", AnimalEnum) self.sleepy = node.get_bool("sleepy") def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return Config apache-buildstream-27ae392/tests/format/option-exports/project.conf000066400000000000000000000011001514607367700255730ustar00rootroot00000000000000name: test min-version: 2.0 plugins: - origin: local path: plugins elements: - config options: bool_export: type: bool description: Exported boolean option default: False variable: exported-bool enum_export: type: enum description: Exported enum option values: - pony - horsy - zebry default: pony variable: exported-enum flags_export: type: flags description: Exported flags option values: - pony - horsy - zebry default: - pony - horsy variable: exported-flags apache-buildstream-27ae392/tests/format/option-flags-missing/000077500000000000000000000000001514607367700243255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-flags-missing/element.bst000066400000000000000000000000151514607367700264640ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-flags-missing/project.conf000066400000000000000000000002051514607367700266370ustar00rootroot00000000000000name: test min-version: 2.0 options: empty: type: flags description: Invalid because no values are defined values: [] apache-buildstream-27ae392/tests/format/option-flags/000077500000000000000000000000001514607367700226565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-flags/element-in.bst000066400000000000000000000003061514607367700254240ustar00rootroot00000000000000kind: manual variables: result: "a pony" (?): - ("zebry" in farm): result: "a zebry" - ("pony" not in farm): result: "no pony" - (animal not in farm): result: "no horsy" apache-buildstream-27ae392/tests/format/option-flags/element.bst000066400000000000000000000003611514607367700250210ustar00rootroot00000000000000kind: manual variables: result: "a pony" (?): - farm == [ "zebry" ]: result: "a zebry" - farm == [ "horsy", "pony" ]: result: "a pony and a horsy" - farm == [ "horsy", "pony", "zebry" ]: result: "all the animals" apache-buildstream-27ae392/tests/format/option-flags/project.conf000066400000000000000000000006061514607367700251750ustar00rootroot00000000000000name: test min-version: 2.0 options: # Include an enum option here so we can compare it animal: type: enum description: The kind of animal values: - pony - horsy - zebry default: horsy # A flags value to test farm: type: flags description: The kinds of animals on this farm values: - pony - horsy - zebry default: - pony apache-buildstream-27ae392/tests/format/option-list-directive.py000066400000000000000000000012461514607367700250660ustar00rootroot00000000000000# Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("mount_devices", [("true"), ("false")]) def test_override(cli, datafiles, mount_devices): project = os.path.join(datafiles, "option-list-directive") bst_args = ["--option", "shell_mount_devices", mount_devices, "build"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() apache-buildstream-27ae392/tests/format/option-list-directive/000077500000000000000000000000001514607367700245115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-list-directive/project.conf000066400000000000000000000004441514607367700270300ustar00rootroot00000000000000name: test min-version: 2.0 options: shell_mount_devices: type: bool description: whether to mount devices in the shell default: false shell: host-files: - '/etc/passwd' - '/etc/group' (?): - shell_mount_devices: host-files: (>): - '/dev/dri' apache-buildstream-27ae392/tests/format/option-os/000077500000000000000000000000001514607367700222035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-os/element.bst000066400000000000000000000004001514607367700243400ustar00rootroot00000000000000kind: manual variables: result: "Nothing" (?): - machine_os == "Linux": result: "Linuxy" - machine_os == "Darwin": result: "Darwiny" - machine_os == "SunOS": result: "SunOSy" - machine_os == "FreeBSD": result: "FreeBSDy"apache-buildstream-27ae392/tests/format/option-os/project.conf000066400000000000000000000002461514607367700245220ustar00rootroot00000000000000name: test min-version: 2.0 options: machine_os: type: os description: The operating system values: - Linux - Darwin - FreeBSD - SunOS apache-buildstream-27ae392/tests/format/option-overrides/000077500000000000000000000000001514607367700235645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-overrides/element.bst000066400000000000000000000000151514607367700257230ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-overrides/project.conf000066400000000000000000000006161514607367700261040ustar00rootroot00000000000000# Test case ensuring that we can use options # in the element overrides. # name: test min-version: 2.0 options: arch: type: arch description: architecture values: [i686, x86_64] elements: manual: variables: (?): - arch == 'i686': conf-global: --host=i686-unknown-linux-gnu - arch == 'x86_64': conf-global: --host=x86_64-unknown-linux-gnu apache-buildstream-27ae392/tests/format/option-project-root/000077500000000000000000000000001514607367700242115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-project-root/element.bst000066400000000000000000000000151514607367700263500ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-project-root/project.conf000066400000000000000000000004241514607367700265260ustar00rootroot00000000000000name: test min-version: 2.0 options: animal: type: enum description: The kind of animal values: - pony - horsy default: pony (?): - animal == "pony": variables: result: "a pony" - animal == "horsy": variables: result: "a horsy" apache-buildstream-27ae392/tests/format/option-restricted-name/000077500000000000000000000000001514607367700246505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-restricted-name/element.bst000066400000000000000000000000151514607367700270070ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-restricted-name/project.conf000066400000000000000000000003641514607367700271700ustar00rootroot00000000000000name: test min-version: 2.0 options: animal: type: enum description: The kind of animal values: - pony - horsy default: pony (?): - animal == "pony": name: ponyproject - animal == "horsy": name: horsyproject apache-buildstream-27ae392/tests/format/option-restricted-options/000077500000000000000000000000001514607367700254235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/option-restricted-options/element.bst000066400000000000000000000000151514607367700275620ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/option-restricted-options/project.conf000066400000000000000000000006451514607367700277450ustar00rootroot00000000000000name: test min-version: 2.0 options: animal: type: enum description: The kind of animal values: - pony - horsy default: pony (?): - animal == "pony": options: pony: type: bool description: Whether a pony or not default: False - animal == "horsy": options: horsy: type: bool description: Whether a horsy or not default: False apache-buildstream-27ae392/tests/format/optionarch.py000066400000000000000000000070751514607367700230050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils import override_platform_uname # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "machine,value,expected", [ # Test explicitly provided arches ("arm", "aarch32", "Army"), ("arm", "aarch64", "Aarchy"), # Test automatically derived arches ("arm", None, "Army"), ("aarch64", None, "Aarchy"), # Test that explicitly provided arches dont error out # when the `uname` reported arch is not supported ("i386", "aarch32", "Army"), ("x86_64", "aarch64", "Aarchy"), ], ) def test_conditional(cli, datafiles, machine, value, expected): with override_platform_uname(machine=machine): project = os.path.join(datafiles, "option-arch") bst_args = [] if value is not None: bst_args += ["--option", "machine_arch", value] bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) def test_unsupported_arch(cli, datafiles): with override_platform_uname(machine="x86_64"): project = os.path.join(datafiles, "option-arch") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_alias(cli, datafiles): with override_platform_uname(machine="arm"): project = os.path.join(datafiles, "option-arch-alias") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_unknown_host_arch(cli, datafiles): with override_platform_uname(machine="x86_128"): project = os.path.join(datafiles, "option-arch") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.PLATFORM, None) @pytest.mark.datafiles(DATA_DIR) def test_unknown_project_arch(cli, datafiles): project = os.path.join(datafiles, "option-arch-unknown") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/optionbool.py000066400000000000000000000073741514607367700230250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,option,expected", [ # Test 'foo' syntax, and valid values of 'True' / 'False' ("element.bst", "True", "a pony"), ("element.bst", "true", "a pony"), ("element.bst", "False", "not pony"), ("element.bst", "false", "not pony"), # Test 'not foo' syntax ("element-not.bst", "False", "not pony"), ("element-not.bst", "True", "a pony"), # Test 'foo == True' syntax ("element-equals.bst", "False", "not pony"), ("element-equals.bst", "True", "a pony"), # Test 'foo != True' syntax ("element-not-equals.bst", "False", "not pony"), ("element-not-equals.bst", "True", "a pony"), ], ) def test_conditional_cli(cli, datafiles, target, option, expected): project = os.path.join(datafiles, "option-bool") result = cli.run( project=project, silent=True, args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{vars}", target], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("thepony") == expected # Test configuration of boolean option in the config file # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,option,expected", [ ("element.bst", True, "a pony"), ("element.bst", False, "not pony"), ], ) def test_conditional_config(cli, datafiles, target, option, expected): project = os.path.join(datafiles, "option-bool") cli.configure({"projects": {"test": {"options": {"pony": option}}}}) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("thepony") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("cli_option", [("falsey"), ("pony"), ("trUE")]) def test_invalid_value_cli(cli, datafiles, cli_option): project = os.path.join(datafiles, "option-bool") result = cli.run( project=project, silent=True, args=["--option", "pony", cli_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("config_option", [("pony"), (["its", "a", "list"]), ({"dic": "tionary"})]) def test_invalid_value_config(cli, datafiles, config_option): project = os.path.join(datafiles, "option-bool") cli.configure({"projects": {"test": {"options": {"pony": config_option}}}}) result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/optioneltmask.py000066400000000000000000000060371514607367700235250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,value,expected", [ ("pony.bst", "pony.bst", "True"), ("horsy.bst", "pony.bst, horsy.bst", "True"), ("zebry.bst", "pony.bst, horsy.bst", "False"), ], ) def test_conditional_cli(cli, datafiles, target, value, expected): project = os.path.join(datafiles, "option-element-mask") result = cli.run( project=project, silent=True, args=["--option", "debug_elements", value, "show", "--deps", "none", "--format", "%{vars}", target], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,value,expected", [ ("pony.bst", ["pony.bst"], "True"), ("horsy.bst", ["pony.bst", "horsy.bst"], "True"), ("zebry.bst", ["pony.bst", "horsy.bst"], "False"), ], ) def test_conditional_config(cli, datafiles, target, value, expected): project = os.path.join(datafiles, "option-element-mask") cli.configure({"projects": {"test": {"options": {"debug_elements": value}}}}) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) def test_invalid_declaration(cli, datafiles): project = os.path.join(datafiles, "option-element-mask-invalid") result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "pony.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_value(cli, datafiles): project = os.path.join(datafiles, "option-element-mask") result = cli.run( project=project, silent=True, args=["--option", "debug_elements", "kitten.bst", "show", "--deps", "none", "--format", "%{vars}", "pony.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/optionenum.py000066400000000000000000000101231514607367700230200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,option,value,expected", [ # Test 'var == "foo"' syntax ("element.bst", "brother", "pony", "a pony"), ("element.bst", "brother", "zebry", "a zebry"), ("element.bst", "brother", "horsy", "a horsy"), # Test 'var1 == var2' syntax ("element-compare.bst", "brother", "horsy", "different"), ("element-compare.bst", "brother", "zebry", "same"), ("element-compare.bst", "sister", "pony", "same"), ], ) def test_conditional_cli(cli, datafiles, target, option, value, expected): project = os.path.join(datafiles, "option-enum") result = cli.run( project=project, silent=True, args=["--option", option, value, "show", "--deps", "none", "--format", "%{vars}", target], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,option,value,expected", [ # Test 'var == "foo"' syntax ("element.bst", "brother", "pony", "a pony"), ("element.bst", "brother", "zebry", "a zebry"), ("element.bst", "brother", "horsy", "a horsy"), # Test 'var1 == var2' syntax ("element-compare.bst", "brother", "horsy", "different"), ("element-compare.bst", "brother", "zebry", "same"), ("element-compare.bst", "sister", "pony", "same"), ], ) def test_conditional_config(cli, datafiles, target, option, value, expected): project = os.path.join(datafiles, "option-enum") cli.configure({"projects": {"test": {"options": {option: value}}}}) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) def test_invalid_value_cli(cli, datafiles): project = os.path.join(datafiles, "option-enum") result = cli.run( project=project, silent=True, args=["--option", "brother", "giraffy", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("config_option", [("giraffy"), (["its", "a", "list"]), ({"dic": "tionary"})]) def test_invalid_value_config(cli, datafiles, config_option): project = os.path.join(datafiles, "option-enum") cli.configure({"projects": {"test": {"options": {"brother": config_option}}}}) result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_missing_values(cli, datafiles): project = os.path.join(datafiles, "option-enum-missing") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/optionexports.py000066400000000000000000000035251514607367700235700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "option_name,option_value,var_name,var_value", [ # Test boolean ("bool_export", "False", "exported-bool", "0"), ("bool_export", "True", "exported-bool", "1"), # Enum ("enum_export", "pony", "exported-enum", "pony"), ("enum_export", "horsy", "exported-enum", "horsy"), # Flags ("flags_export", "pony", "exported-flags", "pony"), ("flags_export", "pony, horsy", "exported-flags", "horsy,pony"), ], ) def test_export(cli, datafiles, option_name, option_value, var_name, var_value): project = os.path.join(datafiles, "option-exports") result = cli.run( project=project, silent=True, args=["--option", option_name, option_value, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str(var_name) == var_value apache-buildstream-27ae392/tests/format/optionflags.py000066400000000000000000000110161514607367700231520ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,option,value,expected", [ # Test (var == [ "foo" ]) syntax ("element.bst", "farm", "pony", "a pony"), ("element.bst", "farm", "zebry", "a zebry"), ("element.bst", "farm", "pony, horsy", "a pony and a horsy"), ("element.bst", "farm", "zebry,horsy , pony", "all the animals"), # Test ("literal" in var) syntax ("element-in.bst", "farm", "zebry, horsy, pony", "a zebry"), # Test ("literal" not in var) syntax ("element-in.bst", "farm", "zebry, horsy", "no pony"), # Test (var1 not in var2) syntax (where var1 is enum and var2 is flags) ("element-in.bst", "farm", "zebry, pony", "no horsy"), ], ) def test_conditional_cli(cli, datafiles, target, option, value, expected): project = os.path.join(datafiles, "option-flags") result = cli.run( project=project, silent=True, args=["--option", option, value, "show", "--deps", "none", "--format", "%{vars}", target], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,option,value,expected", [ # Test 'var == [ "foo" ]' syntax ("element.bst", "farm", ["pony"], "a pony"), ("element.bst", "farm", ["zebry"], "a zebry"), ("element.bst", "farm", ["pony", "horsy"], "a pony and a horsy"), ("element.bst", "farm", ["zebry", "horsy", "pony"], "all the animals"), ], ) def test_conditional_config(cli, datafiles, target, option, value, expected): project = os.path.join(datafiles, "option-flags") cli.configure({"projects": {"test": {"options": {option: value}}}}) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "cli_option", [("giraffy"), ("horsy pony")], # Not a valid animal for the farm option # Does not include comma separators ) def test_invalid_value_cli(cli, datafiles, cli_option): project = os.path.join(datafiles, "option-flags") result = cli.run( project=project, silent=True, args=["--option", "farm", cli_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "config_option", [ ("pony"), # Not specified as a list (["horsy", "pony", "giraffy"]), # Invalid giraffy animal for farm option ({"dic": "tionary"}), # Dicts also dont make sense in the config for flags ], ) def test_invalid_value_config(cli, datafiles, config_option): project = os.path.join(datafiles, "option-flags") cli.configure({"projects": {"test": {"options": {"farm": config_option}}}}) result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_missing_values(cli, datafiles): project = os.path.join(datafiles, "option-flags-missing") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/optionos.py000066400000000000000000000047141514607367700225060ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils import override_platform_uname DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "system,value,expected", [ # Test explicitly provided arches ("Darwin", "Linux", "Linuxy"), ("SunOS", "FreeBSD", "FreeBSDy"), # Test automatically derived arches ("Linux", None, "Linuxy"), ("Darwin", None, "Darwiny"), # Test that explicitly provided arches dont error out # when the `uname` reported arch is not supported ("ULTRIX", "Linux", "Linuxy"), ("HaikuOS", "SunOS", "SunOSy"), ], ) def test_conditionals(cli, datafiles, system, value, expected): with override_platform_uname(system=system): project = os.path.join(datafiles, "option-os") bst_args = [] if value is not None: bst_args += ["--option", "machine_os", value] bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected @pytest.mark.datafiles(DATA_DIR) def test_unsupported_arch(cli, datafiles): with override_platform_uname(system="ULTRIX"): project = os.path.join(datafiles, "option-os") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/optionoverrides.py000066400000000000000000000027631514607367700240710ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("arch", [("i686"), ("x86_64")]) def test_override(cli, datafiles, arch): project = os.path.join(datafiles, "option-overrides") bst_args = ["--option", "arch", arch] bst_args += ["show", "--deps", "none", "--format", "%{vars}", "element.bst"] result = cli.run(project=project, silent=True, args=bst_args) result.assert_success() # See the associated project.conf for the expected values expected_value = "--host={}-unknown-linux-gnu".format(arch) loaded = _yaml.load_data(result.output) assert loaded.get_str("conf-global") == expected_value apache-buildstream-27ae392/tests/format/optionprojectroot.py000066400000000000000000000056731514607367700244440ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # # Test that project option conditionals can be resolved in the project root # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("value,expected", [("pony", "a pony"), ("horsy", "a horsy")], ids=["pony", "horsy"]) def test_resolve_project_root_conditional(cli, datafiles, value, expected): project = os.path.join(datafiles, "option-project-root") result = cli.run( project=project, silent=True, args=["--option", "animal", value, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected # # Test that project option conditionals can be resolved in element overrides # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("value,expected", [("pony", "a pony"), ("horsy", "a horsy")], ids=["pony", "horsy"]) def test_resolve_element_override_conditional(cli, datafiles, value, expected): project = os.path.join(datafiles, "option-element-override") result = cli.run( project=project, silent=True, args=["--option", "animal", value, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("result") == expected # # Test that restricted keys error out correctly if specified conditionally # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir,provenance", [ ("option-restricted-name", "project.conf [line 15 column 10]"), ("option-restricted-options", "project.conf [line 16 column 6]"), ], ids=["name", "options"], ) def test_restricted_conditionals(cli, datafiles, project_dir, provenance): project = os.path.join(datafiles, project_dir) result = cli.run( project=project, silent=True, args=["show", "element.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.ILLEGAL_COMPOSITE) assert provenance in result.stderr apache-buildstream-27ae392/tests/format/options.py000066400000000000000000000215361514607367700223300ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "options") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir", [ ("invalid-name-spaces"), ("invalid-name-dashes"), ("invalid-name-plus"), ("invalid-name-leading-number"), ], ) def test_invalid_option_name(cli, datafiles, project_dir): project = os.path.join(datafiles, project_dir) result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project_dir", [ ("invalid-variable-name-spaces"), ("invalid-variable-name-plus"), ], ) def test_invalid_variable_name(cli, datafiles, project_dir): project = os.path.join(datafiles, project_dir) result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(DATA_DIR) def test_invalid_option_type(cli, datafiles): project = os.path.join(datafiles, "invalid-type") # Test with the opt option set result = cli.run( project=project, silent=True, args=["--option", "opt", "funny", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_option_cli(cli, datafiles): project = os.path.join(datafiles, "simple-condition") # Test with the opt option set result = cli.run( project=project, silent=True, args=["--option", "fart", "funny", "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_option_config(cli, datafiles): project = os.path.join(datafiles, "simple-condition") cli.configure({"projects": {"test": {"options": {"fart": "Hello"}}}}) result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_invalid_expression(cli, datafiles): project = os.path.join(datafiles, "invalid-expression") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.EXPRESSION_FAILED) @pytest.mark.datafiles(DATA_DIR) def test_undefined(cli, datafiles): project = os.path.join(datafiles, "undefined-variable") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.EXPRESSION_FAILED) @pytest.mark.datafiles(DATA_DIR) def test_invalid_condition(cli, datafiles): project = os.path.join(datafiles, "invalid-condition") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"] ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "opt_option,expected_prefix", [ ("False", "/usr"), ("True", "/opt"), ], ) def test_simple_conditional(cli, datafiles, opt_option, expected_prefix): project = os.path.join(datafiles, "simple-condition") # Test with the opt option set result = cli.run( project=project, silent=True, args=["--option", "opt", opt_option, "show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("prefix") == expected_prefix @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "debug,logging,expected", [ ("False", "False", "False"), ("True", "False", "False"), ("False", "True", "False"), ("True", "True", "True"), ], ) def test_nested_conditional(cli, datafiles, debug, logging, expected): project = os.path.join(datafiles, "nested-condition") # Test with the opt option set result = cli.run( project=project, silent=True, args=[ "--option", "debug", debug, "--option", "logging", logging, "show", "--deps", "none", "--format", "%{vars}", "element.bst", ], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "debug,logging,expected", [ ("False", "False", "False"), ("True", "False", "False"), ("False", "True", "False"), ("True", "True", "True"), ], ) def test_compound_and_conditional(cli, datafiles, debug, logging, expected): project = os.path.join(datafiles, "compound-and-condition") # Test with the opt option set result = cli.run( project=project, silent=True, args=[ "--option", "debug", debug, "--option", "logging", logging, "show", "--deps", "none", "--format", "%{vars}", "element.bst", ], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("debug") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "debug,logging,expected", [ ("False", "False", "False"), ("True", "False", "True"), ("False", "True", "True"), ("True", "True", "True"), ], ) def test_compound_or_conditional(cli, datafiles, debug, logging, expected): project = os.path.join(datafiles, "compound-or-condition") # Test with the opt option set result = cli.run( project=project, silent=True, args=[ "--option", "debug", debug, "--option", "logging", logging, "show", "--deps", "none", "--format", "%{vars}", "element.bst", ], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("logging") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "option,expected", [ ("False", "horsy"), ("True", "pony"), ], ) def test_deep_nesting_level1(cli, datafiles, option, expected): project = os.path.join(datafiles, "deep-nesting") result = cli.run( project=project, silent=True, args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{public}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) shallow_list = loaded.get_sequence("shallow-nest") first_dict = shallow_list.mapping_at(0) assert first_dict.get_str("animal") == expected @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "option,expected", [ ("False", "horsy"), ("True", "pony"), ], ) def test_deep_nesting_level2(cli, datafiles, option, expected): project = os.path.join(datafiles, "deep-nesting") result = cli.run( project=project, silent=True, args=["--option", "pony", option, "show", "--deps", "none", "--format", "%{public}", "element-deeper.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) shallow_list = loaded.get_sequence("deep-nest") deeper_list = shallow_list.sequence_at(0) first_dict = deeper_list.mapping_at(0) assert first_dict.get_str("animal") == expected apache-buildstream-27ae392/tests/format/options/000077500000000000000000000000001514607367700217475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/compound-and-condition/000077500000000000000000000000001514607367700263175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/compound-and-condition/element.bst000066400000000000000000000000151514607367700304560ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/compound-and-condition/project.conf000066400000000000000000000005431514607367700306360ustar00rootroot00000000000000name: test min-version: 2.0 options: debug: type: bool description: Whether debugging is enabled default: False logging: type: bool description: Whether logging is enabled default: False variables: debug: 'False' (?): # Debugging is not enabled unless logging is also enabled - logging and debug: debug: 'True' apache-buildstream-27ae392/tests/format/options/compound-or-condition/000077500000000000000000000000001514607367700261755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/compound-or-condition/element.bst000066400000000000000000000000151514607367700303340ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/compound-or-condition/project.conf000066400000000000000000000005531514607367700305150ustar00rootroot00000000000000name: test min-version: 2.0 options: debug: type: bool description: Whether debugging is enabled default: False logging: type: bool description: Whether logging is enabled default: False variables: logging: 'False' (?): # Logging is enabled if specified or if debugging is requested - logging or debug: logging: 'True' apache-buildstream-27ae392/tests/format/options/deep-nesting/000077500000000000000000000000001514607367700243315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/deep-nesting/element-deeper.bst000066400000000000000000000002351514607367700277360ustar00rootroot00000000000000kind: manual # Yep, this is a list of lists of dictionaries. # public: deep-nest: - - animal: horsy (?): - pony: animal: pony apache-buildstream-27ae392/tests/format/options/deep-nesting/element.bst000066400000000000000000000001421514607367700264710ustar00rootroot00000000000000kind: manual public: shallow-nest: - animal: horsy (?): - pony: animal: pony apache-buildstream-27ae392/tests/format/options/deep-nesting/project.conf000066400000000000000000000001661514607367700266510ustar00rootroot00000000000000name: test min-version: 2.0 options: pony: type: bool description: Whether a pony or not default: False apache-buildstream-27ae392/tests/format/options/invalid-condition/000077500000000000000000000000001514607367700253615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-condition/element.bst000066400000000000000000000000151514607367700275200ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-condition/project.conf000066400000000000000000000003551514607367700277010ustar00rootroot00000000000000name: test min-version: 2.0 options: opt: type: bool description: Whether to build in an opt prefix default: False variables: (?): - not: Allowed any adjacent keys beside the expression. opt: prefix: "/opt" apache-buildstream-27ae392/tests/format/options/invalid-expression/000077500000000000000000000000001514607367700255725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-expression/element.bst000066400000000000000000000000151514607367700277310ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-expression/project.conf000066400000000000000000000002771514607367700301150ustar00rootroot00000000000000name: test min-version: 2.0 options: opt: type: bool description: Whether to build in an opt prefix default: False variables: (?): - is defined pony: prefix: "/opt" apache-buildstream-27ae392/tests/format/options/invalid-name-dashes/000077500000000000000000000000001514607367700255605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-name-dashes/element.bst000066400000000000000000000000151514607367700277170ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-name-dashes/project.conf000066400000000000000000000002041514607367700300710ustar00rootroot00000000000000name: test min-version: 2.0 options: name-with-dashes: type: bool description: An invalid option name default: False apache-buildstream-27ae392/tests/format/options/invalid-name-leading-number/000077500000000000000000000000001514607367700272025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-name-leading-number/element.bst000066400000000000000000000000151514607367700313410ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-name-leading-number/project.conf000066400000000000000000000002061514607367700315150ustar00rootroot00000000000000name: test min-version: 2.0 options: 123number_is_first: type: bool description: An invalid option name default: False apache-buildstream-27ae392/tests/format/options/invalid-name-plus/000077500000000000000000000000001514607367700252745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-name-plus/element.bst000066400000000000000000000000151514607367700274330ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-name-plus/project.conf000066400000000000000000000001771514607367700276160ustar00rootroot00000000000000name: test min-version: 2.0 options: name_with_+: type: bool description: An invalid option name default: False apache-buildstream-27ae392/tests/format/options/invalid-name-spaces/000077500000000000000000000000001514607367700255675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-name-spaces/element.bst000066400000000000000000000000151514607367700277260ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-name-spaces/project.conf000066400000000000000000000002041514607367700301000ustar00rootroot00000000000000name: test min-version: 2.0 options: name with spaces: type: bool description: An invalid option name default: False apache-buildstream-27ae392/tests/format/options/invalid-type/000077500000000000000000000000001514607367700243545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-type/element.bst000066400000000000000000000000151514607367700265130ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-type/project.conf000066400000000000000000000002731514607367700266730ustar00rootroot00000000000000name: test min-version: 2.0 options: opt: type: funny description: This aint really an option type default: False variables: (?): - opt is funny: prefix: "/opt" apache-buildstream-27ae392/tests/format/options/invalid-variable-name-plus/000077500000000000000000000000001514607367700270575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-variable-name-plus/element.bst000066400000000000000000000000161514607367700312170ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-variable-name-plus/project.conf000066400000000000000000000002041514607367700313700ustar00rootroot00000000000000name: test min-version: 2.0 options: pony: type: bool description: Whether a pony default: False variable: pony+ apache-buildstream-27ae392/tests/format/options/invalid-variable-name-spaces/000077500000000000000000000000001514607367700273525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/invalid-variable-name-spaces/element.bst000066400000000000000000000000161514607367700315120ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/invalid-variable-name-spaces/project.conf000066400000000000000000000002131514607367700316630ustar00rootroot00000000000000name: test min-version: 2.0 options: pony: type: bool description: Whether a pony default: False variable: the variable apache-buildstream-27ae392/tests/format/options/nested-condition/000077500000000000000000000000001514607367700252155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/nested-condition/element.bst000066400000000000000000000000151514607367700273540ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/nested-condition/project.conf000066400000000000000000000005731514607367700275370ustar00rootroot00000000000000name: test min-version: 2.0 options: debug: type: bool description: Whether debugging is enabled default: False logging: type: bool description: Whether logging is enabled default: False variables: debug: 'False' (?): - logging: # Debugging is not enabled unless logging is also enabled (?): - debug: debug: 'True' apache-buildstream-27ae392/tests/format/options/simple-condition/000077500000000000000000000000001514607367700252245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/simple-condition/element.bst000066400000000000000000000000151514607367700273630ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/simple-condition/project.conf000066400000000000000000000002631514607367700275420ustar00rootroot00000000000000name: test min-version: 2.0 options: opt: type: bool description: Whether to build in an opt prefix default: False variables: (?): - opt: prefix: "/opt" apache-buildstream-27ae392/tests/format/options/undefined-variable/000077500000000000000000000000001514607367700254735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/options/undefined-variable/element.bst000066400000000000000000000000161514607367700276330ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/options/undefined-variable/project.conf000066400000000000000000000002511514607367700300060ustar00rootroot00000000000000name: test min-version: 2.0 options: pony: type: bool description: Whether a pony default: False variables: (?): - pony == foo: prefix: "/opt" apache-buildstream-27ae392/tests/format/project-overrides/000077500000000000000000000000001514607367700237225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project-overrides/prepend-configure-commands/000077500000000000000000000000001514607367700311355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project-overrides/prepend-configure-commands/element.bst000066400000000000000000000000201514607367700332700ustar00rootroot00000000000000kind: autotools apache-buildstream-27ae392/tests/format/project-overrides/prepend-configure-commands/project.conf000066400000000000000000000004111514607367700334460ustar00rootroot00000000000000name: test min-version: 2.0 plugins: - origin: pip package-name: sample-plugins elements: - autotools # Test that prepending to configure-commands works elements: autotools: config: configure-commands: (<): - echo "Hello World!" apache-buildstream-27ae392/tests/format/project.py000066400000000000000000000247761514607367700223140ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import generate_project from tests.testutils import filetypegenerator # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize("args", [["workspace", "list"], ["show", "pony.bst"]], ids=["list-workspace", "show-element"]) def test_missing_project_conf(cli, datafiles, args): project = str(datafiles) result = cli.run(project=project, args=args) result.assert_main_error(ErrorDomain.STREAM, "project-not-loaded") @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_project_name(cli, datafiles): project = os.path.join(datafiles, "missingname") result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_element(cli, datafiles): project = os.path.join(datafiles, "missing-element") result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error assert "manual.bst [line 4 column 2]" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_missing_junction(cli, datafiles): project = os.path.join(datafiles, "missing-junction") result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) # Assert that we have the expected provenance encoded into the error assert "manual.bst [line 4 column 2]" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_empty_project_name(cli, datafiles): project = os.path.join(datafiles, "emptyname") result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_invalid_project_name(cli, datafiles): project = os.path.join(datafiles, "invalidname") result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_invalid_yaml(cli, datafiles): project = os.path.join(datafiles, "invalid-yaml") result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_YAML) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_load_default_project(cli, datafiles): project = os.path.join(datafiles, "default") result = cli.run(project=project, args=["show", "--format", "%{env}", "manual.bst"]) result.assert_success() # Read back some of our project defaults from the env env = _yaml.load_data(result.output) assert env.get_str("USER") == "tomjon" assert env.get_str("TERM") == "dumb" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_load_project_from_subdir(cli, datafiles): project = os.path.join(datafiles, "project-from-subdir") result = cli.run( project=project, cwd=os.path.join(project, "subdirectory"), args=["show", "--format", "%{env}", "manual.bst"] ) result.assert_success() # Read back some of our project defaults from the env env = _yaml.load_data(result.output) assert env.get_str("USER") == "tomjon" assert env.get_str("TERM") == "dumb" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_override_project_path(cli, datafiles): project = os.path.join(datafiles, "overridepath") result = cli.run(project=project, args=["show", "--format", "%{env}", "manual.bst"]) result.assert_success() # Read back the overridden path env = _yaml.load_data(result.output) assert env.get_str("PATH") == "/bin:/sbin" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_project_unsupported(cli, datafiles): project = os.path.join(datafiles, "unsupported") result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNSUPPORTED_PROJECT) @pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path")) def test_missing_element_path_directory(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) @pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path")) def test_element_path_not_a_directory(cli, datafiles): project = str(datafiles) path = os.path.join(project, "elements") for _file_type in filetypegenerator.generate_file_types(path): result = cli.run(project=project, args=["workspace", "list"]) if not os.path.isdir(path): result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) else: result.assert_success() @pytest.mark.datafiles(os.path.join(DATA_DIR, "local-plugin")) def test_missing_local_plugin_directory(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) @pytest.mark.datafiles(os.path.join(DATA_DIR, "local-plugin")) def test_local_plugin_not_directory(cli, datafiles): project = str(datafiles) path = os.path.join(project, "plugins") for _file_type in filetypegenerator.generate_file_types(path): result = cli.run(project=project, args=["workspace", "list"]) if not os.path.isdir(path): result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) else: result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_plugin_no_load_ref(cli, datafiles, ref_storage): project = os.path.join(datafiles, "plugin-no-load-ref") # Generate project with access to the noloadref plugin and project.refs enabled # config = { "name": "test", "min-version": "2.0", "ref-storage": ref_storage, "plugins": [{"origin": "local", "path": "plugins", "sources": ["noloadref"]}], } _yaml.roundtrip_dump(config, os.path.join(project, "project.conf")) result = cli.run(project=project, silent=True, args=["show", "noloadref.bst"]) # There is no error if project.refs is not in use, otherwise we # assert our graceful failure if ref_storage == "inline": result.assert_success() else: result.assert_main_error(ErrorDomain.SOURCE, "unsupported-load-ref") @pytest.mark.datafiles(DATA_DIR) def test_plugin_preflight_error(cli, datafiles): project = os.path.join(datafiles, "plugin-preflight-error") result = cli.run(project=project, args=["source", "fetch", "error.bst"]) result.assert_main_error(ErrorDomain.SOURCE, "the-preflight-error") @pytest.mark.datafiles(DATA_DIR) def test_duplicate_plugins(cli, datafiles): project = os.path.join(datafiles, "duplicate-plugins") result = cli.run(project=project, silent=True, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "duplicate-plugin") # Assert that we get a different cache key for target.bst, depending # on a conditional statement we have placed in the project.refs file. # @pytest.mark.datafiles(DATA_DIR) def test_project_refs_options(cli, datafiles): project = os.path.join(datafiles, "refs-options") result1 = cli.run( project=project, silent=True, args=["--option", "test", "True", "show", "--deps", "none", "--format", "%{key}", "target.bst"], ) result1.assert_success() result2 = cli.run( project=project, silent=True, args=["--option", "test", "False", "show", "--deps", "none", "--format", "%{key}", "target.bst"], ) result2.assert_success() # Assert that the cache keys are different assert result1.output != result2.output # Assert that we can correctly track an element that has multiple sources such # that a remote source (i.e. `kind: tar`) comes after a `kind: local` source # when using project.refs # # `kind: local` sources are supposed to leave a gap in the project.refs file. However, # older versions of buildstream would incorrectly account for this gap. See issue for # more details: https://github.com/apache/buildstream/issues/1851 @pytest.mark.datafiles(os.path.join(DATA_DIR, "project-refs-gap")) def test_project_refs_gap(cli, tmpdir, datafiles): project = str(datafiles) generate_project( project, config={ "aliases": {"tmpdir": "file:///" + str(tmpdir)}, "ref-storage": "project.refs", }, ) result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() @pytest.mark.datafiles(os.path.join(DATA_DIR, "element-path")) def test_element_path_project_path_contains_symlinks(cli, datafiles, tmpdir): real_project = str(datafiles) linked_project = os.path.join(str(tmpdir), "linked") os.symlink(real_project, linked_project) os.makedirs(os.path.join(real_project, "elements"), exist_ok=True) with open(os.path.join(real_project, "elements", "element.bst"), "w", encoding="utf-8") as f: f.write("kind: manual\n") result = cli.run(project=linked_project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_empty_depends(cli, datafiles): project = os.path.join(datafiles, "empty-depends") result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/project/000077500000000000000000000000001514607367700217225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/default/000077500000000000000000000000001514607367700233465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/default/manual.bst000066400000000000000000000000151514607367700253310ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/project/default/project.conf000066400000000000000000000001331514607367700256600ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 apache-buildstream-27ae392/tests/format/project/duplicate-plugins/000077500000000000000000000000001514607367700253535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/duplicate-plugins/bar/000077500000000000000000000000001514607367700261175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/duplicate-plugins/bar/__init__.py000066400000000000000000000000001514607367700302160ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/duplicate-plugins/bar/foo.py000066400000000000000000000001531514607367700272530ustar00rootroot00000000000000from buildstream import Element class FooElement(Element): pass def setup(): return FooElement apache-buildstream-27ae392/tests/format/project/duplicate-plugins/bar/frob.py000066400000000000000000000001551514607367700274220ustar00rootroot00000000000000from buildstream import Element class FrobElement(Element): pass def setup(): return FrobElement apache-buildstream-27ae392/tests/format/project/duplicate-plugins/baz/000077500000000000000000000000001514607367700261275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/duplicate-plugins/baz/__init__.py000066400000000000000000000000001514607367700302260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/duplicate-plugins/baz/foo.py000066400000000000000000000001531514607367700272630ustar00rootroot00000000000000from buildstream import Element class FooElement(Element): pass def setup(): return FooElement apache-buildstream-27ae392/tests/format/project/duplicate-plugins/baz/frob.py000066400000000000000000000001551514607367700274320ustar00rootroot00000000000000from buildstream import Element class FrobElement(Element): pass def setup(): return FrobElement apache-buildstream-27ae392/tests/format/project/duplicate-plugins/element.bst000066400000000000000000000000151514607367700275120ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/project/duplicate-plugins/project.conf000066400000000000000000000002561514607367700276730ustar00rootroot00000000000000name: test min-version: 2.0 plugins: - origin: local path: bar elements: - foo sources: - frob - origin: local path: baz elements: - foo sources: - frob apache-buildstream-27ae392/tests/format/project/element-path/000077500000000000000000000000001514607367700243055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/element-path/project.conf000066400000000000000000000000621514607367700266200ustar00rootroot00000000000000name: foo min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/project/empty-depends/000077500000000000000000000000001514607367700245005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/empty-depends/manual.bst000066400000000000000000000000271514607367700264660ustar00rootroot00000000000000kind: manual depends: apache-buildstream-27ae392/tests/format/project/empty-depends/project.conf000066400000000000000000000000341514607367700270120ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/project/emptyname/000077500000000000000000000000001514607367700237215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/emptyname/project.conf000066400000000000000000000001771514607367700262430ustar00rootroot00000000000000# A project configuration with an invalid symbol for a project name, # this one is an empty string # name: '' min-version: 2.0 apache-buildstream-27ae392/tests/format/project/invalid-yaml/000077500000000000000000000000001514607367700243105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/invalid-yaml/manual.bst000066400000000000000000000000151514607367700262730ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/project/invalid-yaml/project.conf000066400000000000000000000001761514607367700266310ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 variables: sbindir: "%{bindir} apache-buildstream-27ae392/tests/format/project/invalidname/000077500000000000000000000000001514607367700242115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/invalidname/project.conf000066400000000000000000000002071514607367700265250ustar00rootroot00000000000000# A project configuration with an invalid symbol for a project name, # this one contains a space # name: Project Name min-version: 2.0 apache-buildstream-27ae392/tests/format/project/local-plugin/000077500000000000000000000000001514607367700243105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/local-plugin/project.conf000066400000000000000000000001341514607367700266230ustar00rootroot00000000000000name: foo min-version: 2.0 plugins: - origin: local path: plugins sources: - mysource apache-buildstream-27ae392/tests/format/project/missing-element/000077500000000000000000000000001514607367700250225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/missing-element/manual.bst000066400000000000000000000000451514607367700270100ustar00rootroot00000000000000kind: manual depends: - missing.bst apache-buildstream-27ae392/tests/format/project/missing-element/project.conf000066400000000000000000000000341514607367700273340ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/project/missing-junction/000077500000000000000000000000001514607367700252225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/missing-junction/manual.bst000066400000000000000000000001071514607367700272070ustar00rootroot00000000000000kind: manual depends: - filename: element.bst junction: missing.bst apache-buildstream-27ae392/tests/format/project/missing-junction/project.conf000066400000000000000000000000341514607367700275340ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/project/missingname/000077500000000000000000000000001514607367700242345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/missingname/project.conf000066400000000000000000000000001514607367700265370ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/overridepath/000077500000000000000000000000001514607367700244165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/overridepath/manual.bst000066400000000000000000000000151514607367700264010ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/project/overridepath/project.conf000066400000000000000000000002151514607367700267310ustar00rootroot00000000000000# A project configuration which overrides the sandbox PATH environment variable name: foo min-version: 2.0 environment: PATH: /bin:/sbin apache-buildstream-27ae392/tests/format/project/plugin-no-load-ref/000077500000000000000000000000001514607367700253215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/plugin-no-load-ref/noloadref.bst000066400000000000000000000001701514607367700300020ustar00rootroot00000000000000kind: import description: | Import a source which does not support the load_ref() method sources: - kind: noloadref apache-buildstream-27ae392/tests/format/project/plugin-no-load-ref/plugins/000077500000000000000000000000001514607367700270025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/plugin-no-load-ref/plugins/noloadref.py000066400000000000000000000011151514607367700313230ustar00rootroot00000000000000from buildstream import Source # Just a dummy plugin which does not support the new load_ref() method. # # Use this to test that the core behaves as expected with such plugins. # class NoLoadRefSource(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def get_ref(self): return None def set_ref(self, ref, node): pass def fetch(self): pass def stage(self, directory): pass def setup(): return NoLoadRefSource apache-buildstream-27ae392/tests/format/project/plugin-no-load-ref/project.refs000066400000000000000000000001751514607367700276530ustar00rootroot00000000000000# A project.refs file with an existing ref for the noloadref element # projects: test: noloadref.bst: - ref: dummy apache-buildstream-27ae392/tests/format/project/plugin-preflight-error/000077500000000000000000000000001514607367700263315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/plugin-preflight-error/error.bst000066400000000000000000000001551514607367700301750ustar00rootroot00000000000000kind: import description: An element with a failing source at preflight time sources: - kind: preflighterror apache-buildstream-27ae392/tests/format/project/plugin-preflight-error/errorplugin/000077500000000000000000000000001514607367700307015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/plugin-preflight-error/errorplugin/__init__.py000066400000000000000000000000001514607367700330000ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py000066400000000000000000000011531514607367700343110ustar00rootroot00000000000000from buildstream import Source, SourceError class PreflightErrorSource(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): # Raise a preflight error unconditionally raise SourceError("Unsatisfied requirements in preflight, raising this error", reason="the-preflight-error") def get_unique_key(self): return {} def get_ref(self): return None def set_ref(self, ref, node): pass def fetch(self): pass def stage(self, directory): pass def setup(): return PreflightErrorSource apache-buildstream-27ae392/tests/format/project/plugin-preflight-error/project.conf000066400000000000000000000003451514607367700306500ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 # Whitelist the local test Source "errorplugin" to be loaded # plugins: - origin: local path: errorplugin sources: - preflighterror apache-buildstream-27ae392/tests/format/project/project-from-subdir/000077500000000000000000000000001514607367700256175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/project-from-subdir/manual.bst000066400000000000000000000000151514607367700276020ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/format/project/project-from-subdir/project.conf000066400000000000000000000001331514607367700301310ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 apache-buildstream-27ae392/tests/format/project/project-from-subdir/subdirectory/000077500000000000000000000000001514607367700303355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/project-from-subdir/subdirectory/README000066400000000000000000000001151514607367700312120ustar00rootroot00000000000000This directory is used to test running commands from a project subdirectory. apache-buildstream-27ae392/tests/format/project/project-refs-gap/000077500000000000000000000000001514607367700250725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/project-refs-gap/file000066400000000000000000000000171514607367700257320ustar00rootroot00000000000000This is a file apache-buildstream-27ae392/tests/format/project/project-refs-gap/target.bst000066400000000000000000000001261514607367700270710ustar00rootroot00000000000000kind: import sources: - kind: local path: file - kind: remote url: tmpdir:/file apache-buildstream-27ae392/tests/format/project/refs-options/000077500000000000000000000000001514607367700243525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/refs-options/project.conf000066400000000000000000000002101514607367700266600ustar00rootroot00000000000000name: test min-version: 2.0 ref-storage: project.refs options: test: type: bool description: Test boolean default: False apache-buildstream-27ae392/tests/format/project/refs-options/project.refs000066400000000000000000000003101514607367700266730ustar00rootroot00000000000000# A project.refs file with a conditional statement # projects: test: target.bst: - ref: pony # Optionally override the ref (?): - test: target.bst: - ref: horsy apache-buildstream-27ae392/tests/format/project/refs-options/target.bst000066400000000000000000000002001514607367700263420ustar00rootroot00000000000000kind: import description: | Import some tar repo with optional refs sources: - kind: tar url: http://pony.com/tarball.tgz apache-buildstream-27ae392/tests/format/project/unsupported/000077500000000000000000000000001514607367700243125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/project/unsupported/project.conf000066400000000000000000000001301514607367700266210ustar00rootroot00000000000000# A project which requires a too new version of the format name: foo min-version: 2.900 apache-buildstream-27ae392/tests/format/projectoverrides.py000066400000000000000000000032121514607367700242150ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project-overrides") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_prepend_configure_commands(cli, datafiles): project = os.path.join(datafiles, "prepend-configure-commands") result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", "element.bst"] ) result.assert_success() loaded = _yaml.load_data(result.output) config_commands = loaded.get_str_list("configure-commands") assert len(config_commands) == 3 assert config_commands[0] == 'echo "Hello World!"' apache-buildstream-27ae392/tests/format/stack.py000066400000000000000000000025411514607367700217350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "stack") # # Assert that we have errors when trying to have runtime-only or # build-only dependencies. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target", [ "build-only-stack.bst", "runtime-only-stack.bst", ], ) def test_require_build_and_run(cli, datafiles, target): project = str(datafiles) result = cli.run(project=project, args=["show", target]) result.assert_main_error(ErrorDomain.ELEMENT, "stack-requires-build-and-run") apache-buildstream-27ae392/tests/format/stack/000077500000000000000000000000001514607367700213615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/stack/elements/000077500000000000000000000000001514607367700231755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/stack/elements/build-only-stack.bst000066400000000000000000000000551514607367700270700ustar00rootroot00000000000000kind: stack build-depends: - dependency.bst apache-buildstream-27ae392/tests/format/stack/elements/dependency.bst000066400000000000000000000000571514607367700260270ustar00rootroot00000000000000kind: manual description: This is a dependency apache-buildstream-27ae392/tests/format/stack/elements/runtime-only-stack.bst000066400000000000000000000000571514607367700274560ustar00rootroot00000000000000kind: stack runtime-depends: - dependency.bst apache-buildstream-27ae392/tests/format/stack/project.conf000066400000000000000000000001031514607367700236700ustar00rootroot00000000000000# Basic project name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/format/substitutions.py000066400000000000000000000024231514607367700235660ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project", "default") # Test that output is formatted correctly, when there are multiple matches of a # variable that is known to BuildStream. # @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_match_multiple(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["show", "--format", "%{name} {name} %{name}", "manual.bst"]) result.assert_success() assert result.output == "manual.bst {name} manual.bst\n" apache-buildstream-27ae392/tests/format/userconfig.py000066400000000000000000000023641514607367700227770ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_ensure_misformed_project_overrides_give_sensible_errors(cli, datafiles): userconfig = {"projects": {"test": []}} cli.configure(userconfig) result = cli.run(project=datafiles, args=["show"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/format/variables.py000066400000000000000000000316451514607367700226070ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import sys import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "variables") # List of BuildStream protected variables PROTECTED_VARIABLES = [("project-name"), ("element-name"), ("max-jobs")] def print_warning(msg): RED, END = "\033[91m", "\033[0m" print(("\n{}{}{}").format(RED, msg, END), file=sys.stderr) ############################################################### # Test proper loading of some default commands from plugins # ############################################################### @pytest.mark.parametrize( "target,varname,expected", [("autotools.bst", "make-install", 'make -j1 DESTDIR="/buildstream-install" install')], ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults")) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_defaults(cli, datafiles, target, varname, expected): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str(varname) == expected ################################################################ # Test overriding of variables to produce different commands # ################################################################ @pytest.mark.parametrize( "target,varname,expected", [("autotools.bst", "make-install", 'make -j1 DESTDIR="/custom/install/root" install')], ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "overrides")) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_overrides(cli, datafiles, target, varname, expected): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", target]) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str(varname) == expected @pytest.mark.parametrize( "element,provenance", [ # This test makes a reference to an undefined variable in a build command ("manual.bst", "manual.bst [line 5 column 6]"), # This test makes a reference to an undefined variable by another variable, # ensuring that we validate variables even when they are unused ("manual2.bst", "manual2.bst [line 4 column 8]"), # This test uses a build command to refer to some variables which ultimately # refer to an undefined variable, testing a more complex case. ("manual3.bst", "manual3.bst [line 6 column 8]"), ], ids=["build-command", "variables", "complex"], ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "missing_variables")) def test_undefined(cli, datafiles, element, provenance): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE) assert provenance in result.stderr @pytest.mark.parametrize( "element,provenances", [ # Test a simple a -> b and b -> a reference ("simple-cyclic.bst", ["simple-cyclic.bst [line 4 column 5]", "simple-cyclic.bst [line 5 column 5]"]), # Test a simple a -> b and b -> a reference with some text involved ("cyclic.bst", ["cyclic.bst [line 5 column 10]", "cyclic.bst [line 4 column 5]"]), # Test an indirect circular dependency ( "indirect-cyclic.bst", [ "indirect-cyclic.bst [line 5 column 5]", "indirect-cyclic.bst [line 6 column 5]", "indirect-cyclic.bst [line 7 column 5]", "indirect-cyclic.bst [line 8 column 5]", ], ), # Test an indirect circular dependency ("self-reference.bst", ["self-reference.bst [line 4 column 5]"]), ], ids=["simple", "simple-text", "indirect", "self-reference"], ) @pytest.mark.timeout(30, method="signal") @pytest.mark.datafiles(os.path.join(DATA_DIR, "cyclic_variables")) def test_circular_reference(cli, datafiles, element, provenances): print_warning("Performing cyclic test, if this test times out it will exit the test sequence") project = str(datafiles) result = cli.run(project=project, silent=True, args=["build", element]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_REFERENCE_VARIABLE) for provenance in provenances: assert provenance in result.stderr # Test that variables which refer to eachother very deeply are # still resolved correctly, this ensures that we are not relying # on a recursive algorithm limited by stack depth. # @pytest.mark.parametrize( "maxvars", [50, 500, 5000], ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults")) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_deep_references(cli, datafiles, maxvars): project = str(datafiles) # Generate an element with very, very many variables to resolve, # each which expand to the value of the previous variable. # # The bottom variable defines a test value which we check for # in the top variable in `bst show` output. # topvar = "var{}".format(maxvars) bottomvar = "var0" testvalue = "testvalue {}".format(maxvars) # Generate variables = {"var{}".format(idx + 1): "%{var" + str(idx) + "}" for idx in range(maxvars)} variables[bottomvar] = testvalue element = {"kind": "manual", "variables": variables} _yaml.roundtrip_dump(element, os.path.join(project, "test.bst")) # Run `bst show` result = cli.run(project=project, args=["show", "--format", "%{vars}", "test.bst"]) result.assert_success() # Test results result_vars = _yaml.load_data(result.output) assert result_vars.get_str(topvar) == testvalue @pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES) @pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars")) def test_use_of_protected_var_project_conf(cli, datafiles, protected_var): project = str(datafiles) conf = {"name": "test", "min-version": "2.0", "variables": {protected_var: "some-value"}} _yaml.roundtrip_dump(conf, os.path.join(project, "project.conf")) element = { "kind": "import", "sources": [{"kind": "local", "path": "foo.txt"}], } _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) @pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES) @pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars")) def test_use_of_protected_var_element_overrides(cli, datafiles, protected_var): project = str(datafiles) conf = {"name": "test", "min-version": "2.0", "elements": {"manual": {"variables": {protected_var: "some-value"}}}} _yaml.roundtrip_dump(conf, os.path.join(project, "project.conf")) element = { "kind": "manual", "sources": [{"kind": "local", "path": "foo.txt"}], } _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) @pytest.mark.parametrize("protected_var", PROTECTED_VARIABLES) @pytest.mark.datafiles(os.path.join(DATA_DIR, "protected-vars")) def test_use_of_protected_var_in_element(cli, datafiles, protected_var): project = str(datafiles) element = { "kind": "import", "sources": [{"kind": "local", "path": "foo.txt"}], "variables": {protected_var: "some-value"}, } _yaml.roundtrip_dump(element, os.path.join(project, "target.bst")) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROTECTED_VARIABLE_REDEFINED) @pytest.mark.datafiles(os.path.join(DATA_DIR, "shared_variables")) def test_variables_are_resolved_in_elements_context(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build"]) result.assert_success() checkout_dir = os.path.join(project, "checkout") for elem in ["one", "two"]: result = cli.run( project=project, args=["artifact", "checkout", "--directory", os.path.join(checkout_dir, elem), "{}.bst".format(elem)], ) result.assert_success() assert (os.listdir(os.path.join(checkout_dir, "one")), os.listdir(os.path.join(checkout_dir, "two"))) == ( ["one.bst"], ["two.bst"], ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "public_data_variables")) def test_variables_are_resolved_in_public_section(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["show", "--format", "%{public}", "public.bst"]) result.assert_success() output = _yaml.load_data(result.output).strip_node_info() expected = {"integration-commands": ["echo expanded"], "test": "expanded"} assert {k: v for k, v in output.items() if k in expected} == expected @pytest.mark.datafiles(os.path.join(DATA_DIR, "public_data_variables")) def test_variables_resolving_errors_in_public_section(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["show", "--format", "%{public}", "public_unresolved.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE) @pytest.mark.datafiles(os.path.join(DATA_DIR, "partial_context")) def test_partial_context_junctions(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["show", "--format", "%{vars}", "test.bst"]) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str("eltvar") == "/bar/foo/baz" # The notparallel tests use a custom plugin which recreates a situation where # a plugin substitutes an environment variable with the protected %{max-jobs} # variable, which is set depending on whether the plugin declared notparallel. # # These are a regression test against issue #1360, where we found variable # substitution at the plugin default YAML was buggy when multiple instances # were not getting the correct results. # @pytest.mark.datafiles(os.path.join(DATA_DIR, "notparallel")) def test_notparallel(cli, datafiles): project = str(datafiles) # Test the vars result = cli.run(project=project, args=["show", "--format", "%{vars}%{env}", "notparallel.bst"]) result.assert_success() result_vars = _yaml.load_data(result.output) assert result_vars.get_str("element-name") == "notparallel.bst" assert result_vars.get_str("max-jobs") == "1" assert result_vars.get_str("MAKEFLAGS") == "-j1" @pytest.mark.datafiles(os.path.join(DATA_DIR, "notparallel")) def test_notparallel_twice(cli, datafiles): project = str(datafiles) # # Explicitly configure default max-jobs using user configuration # cli.configure({"build": {"max-jobs": 2}}) # Fetch the variables and environment of both elements, where parallel.bst depends on notparallel.bst result = cli.run(project=project, args=["show", "--format", "%{vars}%{env}", "parallel.bst"]) result.assert_success() # Split on the empty line, which separates elements in bst show output groups = result.output.split("\n\n") assert len(groups) >= 2 notparallel_vars = _yaml.load_data(groups[0]) parallel_vars = _yaml.load_data(groups[1]) # Test the first group for the expected notparallel state assert notparallel_vars.get_str("element-name") == "notparallel.bst" assert notparallel_vars.get_str("max-jobs") == "1" assert notparallel_vars.get_str("MAKEFLAGS") == "-j1" # Test the second group for the expected !notparallel state assert parallel_vars.get_str("element-name") == "parallel.bst" assert parallel_vars.get_str("max-jobs") == "2" assert parallel_vars.get_str("MAKEFLAGS") == "-j2" apache-buildstream-27ae392/tests/format/variables/000077500000000000000000000000001514607367700222245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/cyclic_variables/000077500000000000000000000000001514607367700255225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/cyclic_variables/cyclic.bst000066400000000000000000000001121514607367700274740ustar00rootroot00000000000000kind: manual variables: a: "%{prefix}/a" prefix: "%{a}/some_prefix/" apache-buildstream-27ae392/tests/format/variables/cyclic_variables/indirect-cyclic.bst000066400000000000000000000001271514607367700313010ustar00rootroot00000000000000kind: manual variables: foo: "%{a}" a: "%{b}" b: "%{c}" c: "%{d}" d: "%{a}" apache-buildstream-27ae392/tests/format/variables/cyclic_variables/project.conf000066400000000000000000000000341514607367700300340ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/format/variables/cyclic_variables/self-reference.bst000066400000000000000000000000751514607367700311230ustar00rootroot00000000000000kind: manual variables: a: "Referencing itself with %{a}" apache-buildstream-27ae392/tests/format/variables/cyclic_variables/simple-cyclic.bst000066400000000000000000000000611514607367700307660ustar00rootroot00000000000000kind: manual variables: a: "%{b}" b: "%{a}" apache-buildstream-27ae392/tests/format/variables/defaults/000077500000000000000000000000001514607367700240335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/defaults/autotools.bst000066400000000000000000000000721514607367700265750ustar00rootroot00000000000000kind: autotools description: Some kinda autotools element apache-buildstream-27ae392/tests/format/variables/defaults/project.conf000066400000000000000000000002531514607367700263500ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 plugins: - origin: pip package-name: sample-plugins elements: - autotools apache-buildstream-27ae392/tests/format/variables/missing_variables/000077500000000000000000000000001514607367700257255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/missing_variables/manual.bst000066400000000000000000000001171514607367700277130ustar00rootroot00000000000000kind: manual config: build-commands: - some undefined variable %{foo} apache-buildstream-27ae392/tests/format/variables/missing_variables/manual2.bst000066400000000000000000000000621514607367700277740ustar00rootroot00000000000000kind: manual variables: test: hello %{missing} apache-buildstream-27ae392/tests/format/variables/missing_variables/manual3.bst000066400000000000000000000003421514607367700277760ustar00rootroot00000000000000kind: manual variables: hello: "Hello mister %{pony}" greeting: "The %{hello} string twice: %{hello} again" pony: "The pony is %{undefined}" config: build-commands: - Some indirectly undefined variable %{greeting} apache-buildstream-27ae392/tests/format/variables/missing_variables/project.conf000066400000000000000000000001321514607367700302360ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 apache-buildstream-27ae392/tests/format/variables/notparallel/000077500000000000000000000000001514607367700245415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/notparallel/notparallel.bst000066400000000000000000000000551514607367700275700ustar00rootroot00000000000000kind: custom variables: notparallel: true apache-buildstream-27ae392/tests/format/variables/notparallel/parallel.bst000066400000000000000000000000511514607367700270430ustar00rootroot00000000000000kind: custom depends: - notparallel.bst apache-buildstream-27ae392/tests/format/variables/notparallel/plugins/000077500000000000000000000000001514607367700262225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/notparallel/plugins/custom.py000066400000000000000000000002751514607367700301120ustar00rootroot00000000000000from buildstream import BuildElement # A custom build element class CustomElement(BuildElement): BST_MIN_VERSION = "2.0" # Plugin entry point def setup(): return CustomElement apache-buildstream-27ae392/tests/format/variables/notparallel/plugins/custom.yaml000066400000000000000000000003431514607367700304200ustar00rootroot00000000000000# # This element tests an odd corner case we've discovered with # elements which use the `%{max-jobs}` to substitute a nocache # environment variable. # environment: MAKEFLAGS: -j%{max-jobs} environment-nocache: - MAKEFLAGS apache-buildstream-27ae392/tests/format/variables/notparallel/project.conf000066400000000000000000000002751514607367700270620ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: zebra min-version: 2.0 variables: notparallel: false plugins: - origin: local path: plugins elements: - custom apache-buildstream-27ae392/tests/format/variables/overrides/000077500000000000000000000000001514607367700242265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/overrides/autotools.bst000066400000000000000000000001721514607367700267710ustar00rootroot00000000000000kind: autotools description: Some kinda autotools element variables: install-root: /custom/install/root prefix: /opt apache-buildstream-27ae392/tests/format/variables/overrides/project.conf000066400000000000000000000002531514607367700265430ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 plugins: - origin: pip package-name: sample-plugins elements: - autotools apache-buildstream-27ae392/tests/format/variables/partial_context/000077500000000000000000000000001514607367700254245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/partial_context/base.bst000066400000000000000000000000631514607367700270470ustar00rootroot00000000000000kind: junction sources: - kind: local path: base apache-buildstream-27ae392/tests/format/variables/partial_context/base/000077500000000000000000000000001514607367700263365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/partial_context/base/project.conf000066400000000000000000000000351514607367700306510ustar00rootroot00000000000000name: base min-version: 2.0 apache-buildstream-27ae392/tests/format/variables/partial_context/base/vars.yml000066400000000000000000000000341514607367700300310ustar00rootroot00000000000000variables: subvar: "/bar" apache-buildstream-27ae392/tests/format/variables/partial_context/project.conf000066400000000000000000000001301514607367700277330ustar00rootroot00000000000000name: test min-version: 2.0 (@): base.bst:vars.yml variables: var: "%{subvar}/foo" apache-buildstream-27ae392/tests/format/variables/partial_context/test.bst000066400000000000000000000000571514607367700271170ustar00rootroot00000000000000kind: manual variables: eltvar: "%{var}/baz" apache-buildstream-27ae392/tests/format/variables/protected-vars/000077500000000000000000000000001514607367700251665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/protected-vars/foo.txt000066400000000000000000000000041514607367700265040ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/format/variables/protected-vars/project.conf000066400000000000000000000000331514607367700274770ustar00rootroot00000000000000name: foo min-version: 2.0 apache-buildstream-27ae392/tests/format/variables/public_data_variables/000077500000000000000000000000001514607367700265235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/public_data_variables/project.conf000066400000000000000000000001011514607367700310300ustar00rootroot00000000000000name: foo min-version: 2.0 variables: expand-me: "expanded" apache-buildstream-27ae392/tests/format/variables/public_data_variables/public.bst000066400000000000000000000002151514607367700305110ustar00rootroot00000000000000kind: import sources: - kind: local path: public.bst public: integration-commands: - echo %{expand-me} test: "%{expand-me}" apache-buildstream-27ae392/tests/format/variables/public_data_variables/public_unresolved.bst000066400000000000000000000002031514607367700327540ustar00rootroot00000000000000kind: import sources: - kind: local path: public_unresolved.bst public: integration-commands: - echo %{non-existent} apache-buildstream-27ae392/tests/format/variables/shared_variables/000077500000000000000000000000001514607367700255225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/format/variables/shared_variables/one.bst000066400000000000000000000001011514607367700270050ustar00rootroot00000000000000kind: import (@): shared.yml variables: import-path: one.bst apache-buildstream-27ae392/tests/format/variables/shared_variables/project.conf000066400000000000000000000000431514607367700300340ustar00rootroot00000000000000name: shared-vars min-version: 2.0 apache-buildstream-27ae392/tests/format/variables/shared_variables/shared.yml000066400000000000000000000000641514607367700275130ustar00rootroot00000000000000sources: - kind: local path: "%{import-path}" apache-buildstream-27ae392/tests/format/variables/shared_variables/two.bst000066400000000000000000000001011514607367700270350ustar00rootroot00000000000000kind: import (@): shared.yml variables: import-path: two.bst apache-buildstream-27ae392/tests/frontend/000077500000000000000000000000001514607367700206035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/__init__.py000066400000000000000000000020011514607367700227050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from buildstream import _yaml # Shared function to configure the project.conf inline # def configure_project(path, config): config["name"] = "test" config["min-version"] = "2.0" config["element-path"] = "elements" config["plugins"] = [ { "origin": "pip", "package-name": "sample-plugins", "sources": ["git"], } ] _yaml.roundtrip_dump(config, os.path.join(path, "project.conf")) apache-buildstream-27ae392/tests/frontend/artifact-show/000077500000000000000000000000001514607367700233565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/elements/000077500000000000000000000000001514607367700251725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/elements/compose-all.bst000066400000000000000000000003441514607367700301200ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/frontend/artifact-show/elements/import-bin.bst000066400000000000000000000000741514607367700277650ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/frontend/artifact-show/elements/import-dev.bst000066400000000000000000000000741514607367700277730ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/frontend/artifact-show/elements/manual.bst000066400000000000000000000001631514607367700271610ustar00rootroot00000000000000kind: manual config: build-commands: - echo "hello" sources: - kind: local path: elements/manual.bst apache-buildstream-27ae392/tests/frontend/artifact-show/elements/target.bst000066400000000000000000000001641514607367700271730ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - compose-all.bst apache-buildstream-27ae392/tests/frontend/artifact-show/files/000077500000000000000000000000001514607367700244605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/bin-files/000077500000000000000000000000001514607367700263305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/bin-files/usr/000077500000000000000000000000001514607367700271415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/bin-files/usr/bin/000077500000000000000000000000001514607367700277115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700307370ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/frontend/artifact-show/files/dev-files/000077500000000000000000000000001514607367700263365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/dev-files/usr/000077500000000000000000000000001514607367700271475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/dev-files/usr/include/000077500000000000000000000000001514607367700305725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact-show/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700317310ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/artifact-show/project.conf000066400000000000000000000002461514607367700256750ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/artifact_checkout.py000066400000000000000000000074201514607367700246420ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain from tests.testutils import create_artifact_share DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # # Test modes of `bst artifact checkout --pull` when given an artifact name # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "deps,expect_exist,expect_noexist", [ # Deps none: We only expect the file from target-import.bst ("none", ["foo"], ["usr/bin/hello", "usr/include/pony.h"]), # Deps build: We only expect the files from the build dependencies ("build", ["usr/bin/hello", "usr/include/pony.h"], ["foo"]), # Deps run: not supported without a local project ("run", [], []), # Deps all: not supported without a local project ("all", [], []), ], ids=["none", "build", "run", "all"], ) @pytest.mark.parametrize("with_project", [True, False], ids=["with-project", "without-project"]) def test_checkout(cli, tmpdir, datafiles, deps, expect_exist, expect_noexist, with_project): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Build the element to push it to cache cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Build it result = cli.run(project=project, args=["build", "target-import.bst"]) result.assert_success() # Assert it is cached locally and remotely assert cli.get_element_state(project, "target-import.bst") == "cached" assert share.get_artifact(cli.get_artifact_name(project, "test", "target-import.bst")) # Obtain the artifact name for pulling purposes artifact_name = cli.get_artifact_name(project, "test", "target-import.bst") # Discard the local cache shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas"))) shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts"))) assert cli.get_element_state(project, "target-import.bst") != "cached" # Delete the project.conf if we're going to try this without a project if not with_project: os.remove(os.path.join(project, "project.conf")) # Now checkout the artifact result = cli.run( project=project, args=["artifact", "checkout", "--directory", checkout, "--deps", deps, artifact_name], ) if deps in ["all", "run"]: result.assert_main_error(ErrorDomain.STREAM, "deps-not-supported") else: result.assert_success() # After checkout, assert that we have the expected files and assert that # we don't have any of the unexpected files. # for expect in expect_exist: filename = os.path.join(checkout, expect) assert os.path.exists(filename) for expect in expect_noexist: filename = os.path.join(checkout, expect) assert not os.path.exists(filename) apache-buildstream-27ae392/tests/frontend/artifact_delete.py000066400000000000000000000243631514607367700243040ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.element import _get_normal_name from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) # Test that we can delete the artifact of the element which corresponds # to the current project state @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_element(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Build the element and ensure it's cached result = cli.run(project=project, args=["build", element]) result.assert_success() assert cli.get_element_state(project, element) == "cached" result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() assert cli.get_element_state(project, element) != "cached" # Test that we can delete an artifact by specifying its ref. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("with_project", [True, False], ids=["with-project", "without-project"]) def test_artifact_delete_artifact(cli, tmpdir, datafiles, with_project): project = str(datafiles) element = "target.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # Delete the project.conf if we're going to try this without a project if not with_project: os.remove(os.path.join(project, "project.conf")) # Delete the artifact result = cli.run(project=project, args=["artifact", "delete", artifact]) result.assert_success() # Check that the ARTIFACT is no longer in the cache assert not os.path.exists(os.path.join(local_cache, "cas", "refs", "heads", artifact)) # Test the `bst artifact delete` command with multiple, different arguments. @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" dep = "compose-all.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact result = cli.run(project=project, args=["build", element]) result.assert_success() assert cli.get_element_states(project, [element, dep], deps="none") == { element: "cached", dep: "cached", } # Obtain the artifact ref cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # Delete the artifact result = cli.run(project=project, args=["artifact", "delete", artifact, dep]) result.assert_success() # Check that the ARTIFACT is no longer in the cache assert not os.path.exists(os.path.join(local_cache, "artifacts", artifact)) # Check that the dependency ELEMENT is no longer cached assert cli.get_element_state(project, dep) != "cached" # Test that we receive the appropriate stderr when we try to delete an artifact # that is not present in the cache. @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # delete it, just in case it's there _ = cli.run(project=project, args=["artifact", "delete", element]) # Ensure the element is not cached assert cli.get_element_state(project, element) != "cached" # Now try and remove it again (now we know its not there) result = cli.run(project=project, args=["artifact", "delete", element]) cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) expected_err = "WARNING Could not find ref '{}'".format(artifact) assert expected_err in result.stderr # Test that an artifact pulled from it's remote cache (without it's buildtree) will not # throw an Exception when trying to prune the cache. @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Set up remote and local shares local_cache = os.path.join(str(tmpdir), "artifacts") with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: cli.configure({"artifacts": {"servers": [{"url": remote.repo, "push": True}]}, "cachedir": local_cache}) # Build the element result = cli.run(project=project, args=["build", element]) result.assert_success() # Make sure it's in the share assert remote.get_artifact(cli.get_artifact_name(project, "test", element)) # Delete and then pull the artifact (without its buildtree) result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() assert cli.get_element_state(project, element) != "cached" result = cli.run(project=project, args=["artifact", "pull", element]) result.assert_success() assert cli.get_element_state(project, element) == "cached" # Now delete it again (it should have been pulled without the buildtree, but # a digest of the buildtree is pointed to in the artifact's metadata result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() assert cli.get_element_state(project, element) != "cached" # Test that we can delete the build deps of an element @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_elements_build_deps(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Build the element and ensure it's cached result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert element and build deps are cached assert cli.get_element_state(project, element) == "cached" bdep_states = cli.get_element_states(project, [element], deps="build") for state in bdep_states.values(): assert state == "cached" result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", element]) result.assert_success() # Assert that the build deps have been deleted and that the artifact remains cached assert cli.get_element_state(project, element) == "cached" bdep_states = cli.get_element_states(project, [element], deps="build") for state in bdep_states.values(): assert state != "cached" # Test that we can delete the build deps of an artifact by providing an artifact ref @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_artifacts_build_deps(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) # First build an element so that we can find its artifact result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Explicitly check that the ARTIFACT exists in the cache assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # get the artifact refs of the build dependencies bdep_refs = [] bdep_states = cli.get_element_states(project, [element], deps="build") for bdep in bdep_states.keys(): bdep_refs.append(os.path.join("test", _get_normal_name(bdep), cli.get_element_key(project, bdep))) # Assert build dependencies are cached for ref in bdep_refs: assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", ref)) # Delete the artifact result = cli.run(project=project, args=["artifact", "delete", "--deps", "build", artifact]) result.assert_success() # Check that the artifact's build deps are no longer in the cache # Assert build dependencies have been deleted and that the artifact remains for ref in bdep_refs: assert not os.path.exists(os.path.join(local_cache, "artifacts", "refs", ref)) assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact)) # Test that `--deps all` option fails if an artifact ref is specified @pytest.mark.datafiles(DATA_DIR) def test_artifact_delete_artifact_with_deps_all_fails(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # First build an element so that we can find its artifact result = cli.run(project=project, args=["build", element]) result.assert_success() # Obtain the artifact ref cache_key = cli.get_element_key(project, element) artifact = os.path.join("test", os.path.splitext(element)[0], cache_key) # Try to delete the artifact with all of its dependencies result = cli.run(project=project, args=["artifact", "delete", "--deps", "all", artifact]) result.assert_main_error(ErrorDomain.STREAM, "deps-not-supported") apache-buildstream-27ae392/tests/frontend/artifact_list_contents.py000066400000000000000000000142601514607367700257250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "artifact_list_contents", ) def prepare_symlink(project): # Create the link before running the tests. # This is needed for users working on Windows, git checks out symlinks as files which content is the name # of the symlink and the test therefore doesn't have the correct content os.symlink( os.path.join("..", "basicfile"), os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"), ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("target", ["element-name", "artifact-name"]) @pytest.mark.parametrize("with_project", [True, False], ids=["with-project", "without-project"]) def test_artifact_list_exact_contents(cli, datafiles, target, with_project): project = str(datafiles) prepare_symlink(project) # Ensure we have an artifact to read result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() if target == "element-name": arg_bin = "import-bin.bst" arg_links = "import-links.bst" elif target == "artifact-name": key_bin = cli.get_element_key(project, "import-bin.bst") key_links = cli.get_element_key(project, "import-links.bst") arg_bin = "test/import-bin/" + key_bin arg_links = "test/import-links/" + key_links else: assert False, "unreachable" # Delete the project.conf if we're going to try this without a project if not with_project: os.remove(os.path.join(project, "project.conf")) expected_output_bin = ("{target}:\n" "\tusr\n" "\tusr/bin\n" "\tusr/bin/hello\n\n").format(target=arg_bin) expected_output_links = ( "{target}:\n" "\tbasicfile\n" "\tbasicfolder\n" "\tbasicfolder/basicsymlink\n" "\tbasicfolder/subdir-file\n\n" ).format(target=arg_links) for arg, expected_output in [(arg_bin, expected_output_bin), (arg_links, expected_output_links)]: # List the contents via the key result = cli.run(project=project, args=["artifact", "list-contents", arg]) # Expect to fail if we try to list by element name and there is no project if target == "element-name" and not with_project: result.assert_main_error(ErrorDomain.STREAM, "project-not-loaded") else: result.assert_success() assert expected_output in result.output # NOTE: The pytest-datafiles package has an issue where it fails to transfer any # mode bits when copying files into the temporary directory: # # https://github.com/omarkohl/pytest-datafiles/issues/11 # # This is why the /usr/bin/hello file appears to not be executable # in the test below, in real life the /usr/bin/hello file will # appear executable. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("target", ["element-name", "artifact-name"]) def test_artifact_list_exact_contents_long(cli, datafiles, target): project = str(datafiles) prepare_symlink(project) # Ensure we have an artifact to read result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 if target == "element-name": arg_bin = "import-bin.bst" arg_links = "import-links.bst" elif target == "artifact-name": key_bin = cli.get_element_key(project, "import-bin.bst") key_links = cli.get_element_key(project, "import-links.bst") arg_bin = "test/import-bin/" + key_bin arg_links = "test/import-links/" + key_links else: assert False, "unreachable" expected_output_bin = ( "{target}:\n" "\tdrwxr-xr-x dir 0 usr\n" "\tdrwxr-xr-x dir 0 usr/bin\n" "\t-rwxr-xr-x exe 28 usr/bin/hello\n\n" ).format(target=arg_bin) expected_output_links = ( "{target}:\n" "\t-rw-r--r-- reg 14 basicfile\n" "\tdrwxr-xr-x dir 0 basicfolder\n" "\tlrwxrwxrwx link 12 basicfolder/basicsymlink -> ../basicfile\n" "\t-rw-r--r-- reg 0 basicfolder/subdir-file\n\n" ).format(target=arg_links) # List the contents via the element name for arg, expected_output in [(arg_bin, expected_output_bin), (arg_links, expected_output_links)]: result = cli.run(project=project, args=["artifact", "list-contents", "--long", arg]) assert result.exit_code == 0 assert expected_output in result.output @pytest.mark.datafiles(DATA_DIR) def test_artifact_list_exact_contents_glob(cli, datafiles): project = str(datafiles) prepare_symlink(project) # Ensure we have an artifact to read result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 # List the contents via glob result = cli.run(project=project, args=["artifact", "list-contents", "test/**"]) assert result.exit_code == 0 # get the cahe keys for each element in the glob import_bin_key = cli.get_element_key(project, "import-bin.bst") import_links_key = cli.get_element_key(project, "import-links.bst") target_key = cli.get_element_key(project, "target.bst") expected_artifacts = [ "test/import-bin/" + import_bin_key, "test/import-links/" + import_links_key, "test/target/" + target_key, ] for artifact in expected_artifacts: assert artifact in result.output apache-buildstream-27ae392/tests/frontend/artifact_list_contents/000077500000000000000000000000001514607367700253505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/elements/000077500000000000000000000000001514607367700271645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/elements/import-bin.bst000066400000000000000000000000741514607367700317570ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/frontend/artifact_list_contents/elements/import-links.bst000066400000000000000000000001021514607367700323170ustar00rootroot00000000000000kind: import sources: - kind: local path: files/files-and-links apache-buildstream-27ae392/tests/frontend/artifact_list_contents/elements/target.bst000066400000000000000000000001651514607367700311660ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - import-links.bst apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/000077500000000000000000000000001514607367700264525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/bin-files/000077500000000000000000000000001514607367700303225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/bin-files/usr/000077500000000000000000000000001514607367700311335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/bin-files/usr/bin/000077500000000000000000000000001514607367700317035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700327310ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/files-and-links/000077500000000000000000000000001514607367700314325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/files-and-links/basicfile000066400000000000000000000000161514607367700332730ustar00rootroot00000000000000file contents apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/files-and-links/basicfolder/000077500000000000000000000000001514607367700337075ustar00rootroot00000000000000subdir-file000066400000000000000000000000001514607367700357460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/artifact_list_contents/files/files-and-links/basicfolderapache-buildstream-27ae392/tests/frontend/artifact_list_contents/project.conf000066400000000000000000000002461514607367700276670ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/artifact_log.py000066400000000000000000000071251514607367700236200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import re import pytest from buildstream._testing import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("target", ["artifact", "artifact-glob"]) @pytest.mark.parametrize("with_project", [True, False], ids=["with-project", "without-project"]) def test_artifact_log(cli, datafiles, target, with_project): project = str(datafiles) # Get the cache key of our test element result = cli.run( project=project, silent=True, args=["--no-colors", "show", "--deps", "none", "--format", "%{full-key}", "target.bst"], ) key = result.output.strip() # Ensure we have an artifact to read result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Collect the log by running `bst artifact log` on the element name first result = cli.run(project=project, args=["artifact", "log", "target.bst"]) result.assert_success() log = result.output assert log != "" # Delete the project.conf if we're going to try this without a project if not with_project: os.remove(os.path.join(project, "project.conf")) args = ["artifact", "log"] if target == "artifact": args.append("test/target/{}".format(key)) elif target == "artifact-glob": args.append("test/target/*") # Run bst artifact log result = cli.run(project=project, args=args) result.assert_success() assert result.output == log @pytest.mark.datafiles(DATA_DIR) def test_artifact_log_files(cli, datafiles): project = str(datafiles) # Ensure we have an artifact to read result = cli.run(project=project, args=["build", "target.bst"]) assert result.exit_code == 0 logfiles = os.path.join(project, "logfiles") target = os.path.join(project, logfiles, "target.log") import_bin = os.path.join(project, logfiles, "import-bin.log") # Ensure the logfile doesn't exist before the command is run assert not os.path.exists(logfiles) assert not os.path.exists(target) assert not os.path.exists(import_bin) # Run the command and ensure the file now exists result = cli.run(project=project, args=["artifact", "log", "--out", logfiles, "target.bst", "import-bin.bst"]) assert result.exit_code == 0 assert os.path.exists(logfiles) assert os.path.exists(target) assert os.path.exists(import_bin) # Ensure the file contains the logs by checking for the LOG line pattern = r"\[..:..:..\] LOG \[.*\] target.bst" with open(target, "r", encoding="utf-8") as f: data = f.read() assert len(re.findall(pattern, data, re.MULTILINE)) > 0 pattern = r"\[..:..:..\] LOG \[.*\] import-bin.bst" with open(import_bin, "r", encoding="utf-8") as f: data = f.read() assert len(re.findall(pattern, data, re.MULTILINE)) > 0 apache-buildstream-27ae392/tests/frontend/artifact_pull.py000066400000000000000000000076751514607367700240250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain from tests.testutils import create_artifact_share DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # # Test modes of `bst artifact pull` when given an artifact # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "deps,expect_cached", [ # When pulling an artifact with --deps none, we expect that artifact to be pulled ("none", ["target.bst"]), # When pulling an artifact with --deps build, we expect the build deps to be pulled ("build", ["import-bin.bst", "compose-all.bst"]), # Pulling an artifact with --deps run is not supported without a local project ("run", []), # Pulling an artifact with --deps all is not supported without a local project ("all", []), ], ids=["none", "build", "run", "all"], ) @pytest.mark.parametrize("with_project", [True, False], ids=["with-project", "without-project"]) def test_pull(cli, tmpdir, datafiles, deps, expect_cached, with_project): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Build the element to push it to cache, and explicitly configure local cache so we can check it local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache, "artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Build it result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert it is cached locally and remotely assert cli.get_element_state(project, "target.bst") == "cached" assert share.get_artifact(cli.get_artifact_name(project, "test", "target.bst")) # Obtain the artifact name for pulling purposes artifact_name = cli.get_artifact_name(project, "test", "target.bst") # Translate the expected element names into artifact names expect_cached_artifacts = [ cli.get_artifact_name(project, "test", element_name) for element_name in expect_cached ] # Discard the local cache shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "cas"))) shutil.rmtree(str(os.path.join(str(tmpdir), "cache", "artifacts"))) assert cli.get_element_state(project, "target.bst") != "cached" # Delete the project.conf if we're going to try this without a project if not with_project: os.remove(os.path.join(project, "project.conf")) # Now run our pull test result = cli.run(project=project, args=["artifact", "pull", "--deps", deps, artifact_name]) if deps in ["all", "run"]: result.assert_main_error(ErrorDomain.STREAM, "deps-not-supported") else: result.assert_success() # After pulling, assert that we have the expected elements cached again. # # Note that we do not use cli.get_element_states() here because the project.conf # might not be present, so we poke at the cache directly for this assertion. for expect in expect_cached_artifacts: assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", expect)) apache-buildstream-27ae392/tests/frontend/artifact_show.py000066400000000000000000000225401514607367700240150ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share from . import configure_project # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "artifact-show", ) SIMPLE_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "simple", ) # Test artifact show @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_element_name(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() assert "not cached {}".format(element) in result.output result = cli.run(project=project, args=["build", element]) result.assert_success() result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() assert "cached {}".format(element) in result.output # Test artifact show on a failed element @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_failed_element(cli, tmpdir, datafiles): project = str(datafiles) element = "manual.bst" result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() assert "not cached {}".format(element) in result.output result = cli.run(project=project, args=["build", element]) result.assert_task_error(ErrorDomain.SANDBOX, "missing-command") result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() assert "failed {}".format(element) in result.output # Test artifact show with a deleted dependency @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_element_missing_deps(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" dependency = "import-bin.bst" result = cli.run(project=project, args=["build", element]) result.assert_success() result = cli.run(project=project, args=["artifact", "delete", dependency]) result.assert_success() result = cli.run(project=project, args=["artifact", "show", "--deps", "all", element]) result.assert_success() assert "not cached {}".format(dependency) in result.output assert "cached {}".format(element) in result.output # Test artifact show with artifact ref @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("with_project", [True, False], ids=["with-project", "without-project"]) def test_artifact_show_artifact_name(cli, tmpdir, datafiles, with_project): project = str(datafiles) element = "target.bst" result = cli.run(project=project, args=["build", element]) result.assert_success() cache_key = cli.get_element_key(project, element) artifact_ref = "test/target/" + cache_key # Delete the project.conf if we're going to try this without a project if not with_project: os.remove(os.path.join(project, "project.conf")) result = cli.run(project=project, args=["artifact", "show", artifact_ref]) result.assert_success() assert "cached {}".format(artifact_ref) in result.output # Test artifact show glob behaviors @pytest.mark.datafiles(SIMPLE_DIR) @pytest.mark.parametrize( "pattern,expected_prefixes", [ # List only artifact results in the test/project # ("test/**", ["test/target/", "test/compose-all/", "test/import-bin", "test/import-dev"]), # List only artifact results by their .bst element names # ("**.bst", ["import-bin.bst", "import-dev.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), # List only the import artifact results # ("import*.bst", ["import-bin.bst", "import-dev.bst"]), ], ids=["test/**", "**.bst", "import*.bst"], ) def test_artifact_show_glob(cli, tmpdir, datafiles, pattern, expected_prefixes): project = str(datafiles) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "show", pattern]) result.assert_success() output = result.output.strip().splitlines() # Assert that the number of results match the number of expected results assert len(output) == len(expected_prefixes) # Assert that each expected result was found. for expected_prefix in expected_prefixes: found = False for result_line in output: result_split = result_line.split() if result_split[-1].startswith(expected_prefix): found = True break assert found, "Expected result {} not found".format(expected_prefix) # Test artifact show artifact in remote @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("config", ["config-project", "config-user", "config-cli"]) @pytest.mark.parametrize("by_element_name", [True, False], ids=["by-element-name", "by-artifact-name"]) def test_artifact_show_available_remotely(cli, tmpdir, datafiles, config, by_element_name): project = str(datafiles) element = "target.bst" # # Skip this configuration, BuildStream intentionally ignores the local project.conf # if an artifact name is specified. # if config == "config-project" and not by_element_name: pytest.skip("No project.conf in context") # Set up remote and local shares local_cache = os.path.join(str(tmpdir), "artifacts") cli.configure( { "cachedir": local_cache, } ) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: extra_cli_args = [] if config == "config-project": configure_project( project, { "artifacts": [ { "url": remote.repo, "push": True, } ] }, ) elif config == "config-user": cli.configure( { "artifacts": {"servers": [{"url": remote.repo, "push": True}]}, } ) else: extra_cli_args = ["--artifact-remote", remote.repo] # Build the element result = cli.run(project=project, args=["build"] + extra_cli_args + [element]) result.assert_success() artifact_name = cli.get_artifact_name(project, "test", element) # Make sure it's in the share assert remote.get_artifact(artifact_name) # Delete the artifact from the local cache result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() assert cli.get_element_state(project, element) != "cached" # Do the artifact show and assert element_or_artifact = element if by_element_name else artifact_name result = cli.run(project=project, args=["artifact", "show"] + extra_cli_args + [element_or_artifact]) result.assert_success() assert "available {}".format(element_or_artifact) in result.output # Test out --ignore-project-artifact-remotes @pytest.mark.datafiles(DATA_DIR) def test_artifact_show_ignore_project_remotes(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Set up remote and local shares local_cache = os.path.join(str(tmpdir), "artifacts") cli.configure( { "cachedir": local_cache, } ) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as remote: configure_project( project, { "artifacts": [ { "url": remote.repo, "push": True, } ] }, ) # Build the element result = cli.run(project=project, args=["build", element]) result.assert_success() # Make sure it's in the share assert remote.get_artifact(cli.get_artifact_name(project, "test", element)) # Delete the artifact from the local cache result = cli.run(project=project, args=["artifact", "delete", element]) result.assert_success() assert cli.get_element_state(project, element) != "cached" # It is available remotely with the project configured remote result = cli.run(project=project, args=["artifact", "show", element]) result.assert_success() assert "available {}".format(element) in result.output # Ignoring project remotes, it is not found in any remote result = cli.run(project=project, args=["artifact", "show", "--ignore-project-artifact-remotes", element]) result.assert_success() assert "not cached {}".format(element) in result.output apache-buildstream-27ae392/tests/frontend/buildcheckout.py000066400000000000000000001337061514607367700240140ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import tarfile import hashlib import re import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from buildstream._testing._utils.site import CASD_SEPARATE_USER from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream import utils from tests.testutils import generate_junction, create_artifact_share from . import configure_project # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) def strict_args(args, strict): if strict != "strict": return ["--no-strict", *args] return args @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "strict,hardlinks", [ ("strict", "copies"), ("strict", "hardlinks"), ("non-strict", "copies"), ("non-strict", "hardlinks"), ], ) def test_build_checkout(datafiles, cli, strict, hardlinks): if CASD_SEPARATE_USER and hardlinks == "hardlinks": pytest.xfail("Cannot hardlink with buildbox-casd running as a separate user") project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # First build it result = cli.run(project=project, args=strict_args(["build", "target.bst"], strict)) result.assert_success() # Assert that after a successful build, the builddir is empty builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) # Prepare checkout args checkout_args = strict_args(["artifact", "checkout"], strict) if hardlinks == "hardlinks": checkout_args += ["--hardlinks"] checkout_args += ["target.bst", "--directory", checkout] # Now check it out result = cli.run(project=project, args=checkout_args) result.assert_success() # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_include_dot_bst_in_checkout_dirname(datafiles, cli): # put '.bst' on the end of the checkout directory # this is a regression test for a bug. The bug would # remove '.bst' if it appeared at the end of of the # "--directory" argument. project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.bst") # build the artifact result = cli.run(project=project, args=["--no-strict", "build", "target.bst"]) result.assert_success() # checkout result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_success() # check that the executable hello file is found in the checkout (ie # confirm files were checked out to directory called "checkout.bst" # (as expected) and not to a directory calld "checkout") filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_non_strict_build_strict_checkout(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # First build it in non-strict mode. # As this is a clean build from scratch, the result and also the cache keys # should be identical to a build in strict mode. result = cli.run(project=project, args=["--no-strict", "build", "target.bst"]) result.assert_success() # Now check it out in strict mode. # This verifies that the clean build in non-strict mode produced an artifact # matching the strict cache key. result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_success() # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_non_strict_pull_build_strict_checkout(datafiles, cli, tmpdir): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Target with at least one (indirect) build-only dependency element_name = "target.bst" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo}]}}) # First build it in non-strict mode with an artifact server configured. # With this configuration BuildStream will attempt to pull the build-only # dependencies after attempting to pull the target element. This means # that the cache key calculation of the target element has to be deferred # until the pull attempt of the build-only dependencies, exercising a # different code path. # As this is a clean build from scratch, the result and also the cache keys # should be identical to a build in strict mode. result = cli.run(project=project, args=["--no-strict", "build", element_name]) result.assert_success() # Now check it out in strict mode. # This verifies that the clean build in non-strict mode produced an artifact # matching the strict cache key. result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) # Regression test for https://github.com/apache/buildstream/issues/1469 # Test that artifact checkout without pull doesn't trigger a BUG in non-strict mode. @pytest.mark.datafiles(DATA_DIR) def test_non_strict_checkout_uncached(datafiles, cli, tmpdir): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "target.bst" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo}]}}) # Attempt to checkout an uncached artifact with remote artifact server # configured but pull disabled. result = cli.run( project=project, args=["--no-strict", "artifact", "checkout", element_name, "--directory", checkout] ) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("deps", [("run"), ("none"), ("build"), ("all")]) def test_build_checkout_deps(datafiles, cli, deps): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "checkout-deps.bst" # First build it result = cli.run(project=project, args=["build", element_name]) result.assert_success() # Assert that after a successful build, the builddir is empty builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) # Now check it out result = cli.run( project=project, args=["artifact", "checkout", element_name, "--deps", deps, "--directory", checkout] ) result.assert_success() # Verify output of this element filename = os.path.join(checkout, "etc", "buildstream", "config") if deps == "build": assert not os.path.exists(filename) else: assert os.path.exists(filename) # Verify output of this element's build dependencies filename = os.path.join(checkout, "usr", "include", "pony.h") if deps in ["build", "all"]: assert os.path.exists(filename) else: assert not os.path.exists(filename) # Verify output of this element's runtime dependencies filename = os.path.join(checkout, "usr", "bin", "hello") if deps in ["run", "all"]: assert os.path.exists(filename) else: assert not os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_deps_build(datafiles, cli): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" result = cli.run(project=project, args=["build", "--deps", "build", target]) result.assert_success() states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert states[build_dep] == "cached" assert states[target] == "buildable" assert states[runtime_dep] == "buildable" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_unbuilt(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Check that checking out an unbuilt element fails nicely result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_compression_no_tar(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() checkout_args = ["artifact", "checkout", "--directory", checkout, "--compression", "gz", "target.bst"] result = cli.run(project=project, args=checkout_args) assert "ERROR: --compression can only be provided if --tar is provided" in result.stderr assert result.exit_code != 0 # If we don't support the extension, we default to an uncompressed tarball @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tar_with_unconventional_name(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.foo") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.open(name=checkout, mode="r") as tar: assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tar_with_unsupported_ext(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar.foo") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) assert ( "Invalid file extension given with '--tar': Expected compression with unknown file extension ('.foo'), " "supported extensions are ('.tar'), ('.gz'), ('.xz'), ('.bz2')" in result.stderr ) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tar_no_compression(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar.gz") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.open(name=checkout, mode="r:gz") as tar: assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar") # Work-around datafiles not preserving mode os.chmod(os.path.join(project, "files/bin-files/usr/bin/hello"), 0o0755) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--tar", checkout, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.TarFile(checkout) as tar: tarinfo = tar.getmember(os.path.join(".", "usr", "bin", "hello")) assert tarinfo.mode == 0o755 assert tarinfo.uid == 0 and tarinfo.gid == 0 assert tarinfo.uname == "" and tarinfo.gname == "" tarinfo = tar.getmember(os.path.join(".", "usr", "include", "pony.h")) assert tarinfo.mode == 0o644 assert tarinfo.uid == 0 and tarinfo.gid == 0 assert tarinfo.uname == "" and tarinfo.gname == "" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_using_ref(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() key = cli.get_element_key(project, "checkout-deps.bst") checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "none", "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_success() filename = os.path.join(checkout, "etc", "buildstream", "config") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_using_ref(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar") result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) key = cli.get_element_key(project, "checkout-deps.bst") checkout_args = ["artifact", "checkout", "--deps", "none", "--tar", checkout, "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.TarFile(checkout) as tar: assert os.path.join(".", "etc", "buildstream", "config") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_build_deps_using_ref(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() key = cli.get_element_key(project, "checkout-deps.bst") checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "build", "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_success() build_dep_files = os.path.join(checkout, "usr", "include", "pony.h") runtime_dep_files = os.path.join(checkout, "usr", "bin", "hello") target_files = os.path.join(checkout, "etc", "buildstream", "config") assert os.path.exists(build_dep_files) assert not os.path.exists(runtime_dep_files) assert not os.path.exists(target_files) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_runtime_deps_using_ref_fails(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() key = cli.get_element_key(project, "checkout-deps.bst") checkout_args = ["artifact", "checkout", "--directory", checkout, "--deps", "run", "test/checkout-deps/" + key] result = cli.run(project=project, args=checkout_args) result.assert_main_error(ErrorDomain.STREAM, "deps-not-supported") @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_invalid_ref(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar") result = cli.run(project=project, args=["build", "checkout-deps.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) non_existent_artifact = "test/checkout-deps/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" checkout_args = ["artifact", "checkout", "--deps", "none", "--tar", checkout, non_existent_artifact] result = cli.run(project=project, args=checkout_args) result.assert_main_error(ErrorDomain.STREAM, "missing-sandbox-config") @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_no_tar_no_directory(datafiles, cli, tmpdir): project = str(datafiles) runtestdir = str(tmpdir) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() checkout_args = ["artifact", "checkout", "target.bst"] result = cli.run(cwd=runtestdir, project=project, args=checkout_args) result.assert_success() filename = os.path.join(runtestdir, "target", "usr", "bin", "hello") assert os.path.exists(filename) filename = os.path.join(runtestdir, "target", "usr", "include", "pony.h") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("compression", [("gz"), ("xz"), ("bz2")]) def test_build_checkout_tarball_compression(datafiles, cli, compression): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--tar", checkout, "--compression", compression, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.open(name=checkout, mode="r:" + compression) as tar: assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_stdout(datafiles, cli): project = str(datafiles) tarball = os.path.join(cli.directory, "tarball.tar") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--tar", "-", "target.bst"] result = cli.run(project=project, args=checkout_args, binary_capture=True) result.assert_success() with open(tarball, "wb") as f: f.write(result.output) with tarfile.TarFile(tarball) as tar: assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_mtime_nonzero(datafiles, cli): project = str(datafiles) tarpath = os.path.join(cli.directory, "mtime_tar.tar") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() checkout_args = ["artifact", "checkout", "--tar", tarpath, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.TarFile(tarpath) as tar: for tarinfo in tar.getmembers(): # An mtime of zero can be confusing to other software, # e.g. ninja build and template toolkit have both taken zero mtime to # mean 'file does not exist'. assert tarinfo.mtime > 0 @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_is_deterministic(datafiles, cli): project = str(datafiles) tarball1 = os.path.join(cli.directory, "tarball1.tar") tarball2 = os.path.join(cli.directory, "tarball2.tar") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--force", "target.bst"] checkout_args1 = checkout_args + ["--tar", tarball1] result = cli.run(project=project, args=checkout_args1) result.assert_success() checkout_args2 = checkout_args + ["--tar", tarball2] result = cli.run(project=project, args=checkout_args2) result.assert_success() with open(tarball1, "rb") as f: contents = f.read() hash1 = hashlib.sha1(contents).hexdigest() with open(tarball2, "rb") as f: contents = f.read() hash2 = hashlib.sha1(contents).hexdigest() assert hash1 == hash2 @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_tarball_links(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout.tar") extract = os.path.join(cli.directory, "extract") # Create the link before running the tests. # This is needed for users working on Windows, git checks out symlinks as files which content is the name # of the symlink and the test therefore doesn't have the correct content os.symlink( os.path.join("..", "basicfile"), os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"), ) result = cli.run(project=project, args=["build", "import-links.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--tar", checkout, "import-links.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.open(name=checkout, mode="r:") as tar: # This tarball is generated by the above `bst artifact checkout`, so there is no security concern. # Use the `tar` extraction filter if available (upstream Python 3.12+) for consistent behavior with # future Python versions and as safeguard against bugs in `bst artifact checkout`. tar.extraction_filter = getattr(tarfile, "tar_filter", (lambda member, path: member)) tar.extractall(extract) with open(os.path.join(extract, "basicfolder", "basicsymlink"), encoding="utf-8") as fp: data = fp.read() assert data == "file contents\n" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_links(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Create the link before running the tests. # This is needed for users working on Windows, git checks out symlinks as files which content is the name # of the symlink and the test therefore doesn't have the correct content os.symlink( os.path.join("..", "basicfile"), os.path.join(project, "files", "files-and-links", "basicfolder", "basicsymlink"), ) result = cli.run(project=project, args=["build", "import-links.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) checkout_args = ["artifact", "checkout", "--directory", checkout, "import-links.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with open(os.path.join(checkout, "basicfolder", "basicsymlink"), encoding="utf-8") as fp: data = fp.read() assert data == "file contents\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("hardlinks", [("copies"), ("hardlinks")]) def test_build_checkout_nonempty(datafiles, cli, hardlinks): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") filename = os.path.join(checkout, "file.txt") # First build it result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that after a successful build, the builddir is empty builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) # Create the checkout dir and add a file to it, should cause checkout to fail os.makedirs(checkout, exist_ok=True) with open(filename, "w", encoding="utf-8") as f: f.write("Hello") # Prepare checkout args checkout_args = ["artifact", "checkout"] if hardlinks == "hardlinks": checkout_args += ["--hardlinks"] checkout_args += ["target.bst", "--directory", checkout] # Now check it out result = cli.run(project=project, args=checkout_args) result.assert_main_error(ErrorDomain.STREAM, None) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("hardlinks", [("copies"), ("hardlinks")]) def test_build_checkout_force(datafiles, cli, hardlinks): if CASD_SEPARATE_USER and hardlinks == "hardlinks": pytest.xfail("Cannot hardlink with buildbox-casd running as a separate user") project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") filename = os.path.join(checkout, "file.txt") # First build it result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that after a successful build, the builddir is empty builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) # Create the checkout dir and add a file to it, should cause checkout to fail os.makedirs(checkout, exist_ok=True) with open(filename, "w", encoding="utf-8") as f: f.write("Hello") # Prepare checkout args checkout_args = ["artifact", "checkout", "--force"] if hardlinks == "hardlinks": checkout_args += ["--hardlinks"] checkout_args += ["target.bst", "--directory", checkout] # Now check it out result = cli.run(project=project, args=checkout_args) result.assert_success() # Check that the file we added is still there filename = os.path.join(checkout, "file.txt") assert os.path.exists(filename) # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_force_tarball(datafiles, cli): project = str(datafiles) tarball = os.path.join(cli.directory, "tarball.tar") result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() builddir = os.path.join(cli.directory, "build") assert os.path.isdir(builddir) assert not os.listdir(builddir) with open(tarball, "w", encoding="utf-8") as f: f.write("Hello") checkout_args = ["artifact", "checkout", "--force", "--tar", tarball, "target.bst"] result = cli.run(project=project, args=checkout_args) result.assert_success() with tarfile.TarFile(tarball) as tar: assert os.path.join(".", "usr", "bin", "hello") in tar.getnames() assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # if ref_storage == "project.refs": project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: contents = f.read() assert contents == "animal=Pony\n" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") workspace = os.path.join(cli.directory, "workspace") checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now open a workspace on the junction # result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, "junction.bst"]) result.assert_success() filename = os.path.join(workspace, "files", "etc-files", "etc", "animal.conf") # Assert the content of /etc/animal.conf in the workspace assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: contents = f.read() assert contents == "animal=Pony\n" # Modify the content of the animal.conf in the workspace with open(filename, "w", encoding="utf-8") as f: f.write("animal=Horsy\n") # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the workspace modified content of /etc/animal.conf filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: contents = f.read() assert contents == "animal=Horsy\n" @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_cross_junction(datafiles, cli, tmpdir): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") checkout = os.path.join(cli.directory, "checkout") generate_junction(tmpdir, subproject_path, junction_path) result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() result = cli.run( project=project, args=["artifact", "checkout", "junction.bst:import-etc.bst", "--directory", checkout] ) result.assert_success() filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_build_junction_short_notation(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst"]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: contents = f.read() assert contents == "animal=Pony\n" @pytest.mark.datafiles(DATA_DIR) def test_build_junction_short_notation_filename(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") checkout = os.path.join(cli.directory, "checkout") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = {"kind": "stack", "depends": [{"filename": "junction.bst:import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should automatically result in fetching # the junction itself at load time. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_success() # Assert that it's cached now assert cli.get_element_state(project, "junction-dep.bst") == "cached" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout]) result.assert_success() # Assert the content of /etc/animal.conf filename = os.path.join(checkout, "etc", "animal.conf") assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: contents = f.read() assert contents == "animal=Pony\n" @pytest.mark.datafiles(DATA_DIR) def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = { "kind": "stack", "depends": [ { "filename": "junction.bst:import-etc.bst", "junction": "junction.bst", } ], } _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as filenames should not contain # `:` when junction is explicity specified result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) @pytest.mark.datafiles(DATA_DIR) def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path) # Create a stack element to depend on a cross junction element, using # colon (:) as the separator element = {"kind": "stack", "depends": ["junction.bst:import-etc.bst:foo.bst"]} _yaml.roundtrip_dump(element, element_path) # Now try to build it, this should fail as recursive lookups for # cross-junction elements is not allowed. result = cli.run(project=project, args=["build", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Should check that after a build we have partial artifacts locally, but should # then attempt to fetch them when doing a artifact checkout @pytest.mark.datafiles(DATA_DIR) def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): project = str(datafiles) checkout_dir = os.path.join(str(tmpdir), "checkout") repo = create_repo("tar", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) element_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) project_config = { "name": "partial-artifact-checkout-fetch", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_name = "input.bst" input_file = os.path.join(element_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["source", "track", input_name]) result.assert_success() result = cli.run(project=project, args=["build", input_name]) result.assert_success() # A push artifact cache means we have to pull to push to them, so # delete some blobs from that CAS such that we have to fetch digest = utils.sha256sum(os.path.join(project, "files", "bin-files", "usr", "bin", "hello")) objpath = os.path.join(cli.directory, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # Verify that the build-only dependency is not (complete) in the local cache cli.configure({"artifacts": {}}) result = cli.run(project=project, args=["artifact", "checkout", input_name, "--directory", checkout_dir]) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") # Verify that the pull method fetches relevant artifacts in order to stage cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["artifact", "checkout", input_name, "--directory", checkout_dir]) result.assert_success() # should have pulled whatever was deleted previous assert input_name in result.get_pulled_elements() @pytest.mark.datafiles(DATA_DIR) def test_partial_checkout_fail(tmpdir, datafiles, cli): project = str(datafiles) build_elt = "import-bin.bst" checkout_dir = os.path.join(str(tmpdir), "checkout") with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) res = cli.run(project=project, args=["artifact", "checkout", build_elt, "--directory", checkout_dir]) res.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") assert re.findall(r"Remote \((\S+)\) does not have artifact (\S+) cached", res.stderr) # Regression test for https://gitlab.com/BuildStream/buildstream/-/issues/1367. # Make sure that `artifact checkout` fails gracefully when no arguments are # provided. @pytest.mark.datafiles(DATA_DIR) def test_fail_no_args(datafiles, cli): project = str(datafiles) result = cli.run(project=project, args=["artifact", "checkout"]) result.assert_main_error(ErrorDomain.APP, None) assert "Missing argument" in result.stderr # This test reproduces a scenario where BuildStream can get confused # if the strictness of a dependency is not taken into account in the # strong artifact cache key, as reported in issue #1270. # # While we were unable to reproduce the exact experience, we can test # the expected behavior. # STRICT_DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "strict-scenario", ) @pytest.mark.datafiles(STRICT_DATA_DIR) def test_changing_strict_dependency_scenario(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") target_path = os.path.join(project, "elements", "target.bst") # Function to (re)write the target element so that it depends # on the base.bst element strictly or non-strictly # def configure_target(strict): dependency = {"filename": "base.bst"} if strict: dependency["strict"] = True config = { "kind": "import", "depends": [dependency], "sources": [{"kind": "local", "path": "files/target.txt"}], } _yaml.roundtrip_dump(config, target_path) # First build where the target normally depends on the base element configure_target(False) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Now configure the target to *strictly* depend on the base, try to check it out # # This will fail in both strict more or non-strict mode, as the strictness of the # dependency will affect both keys. configure_target(True) result = cli.run(project=project, args=["artifact", "checkout", "--directory", checkout, "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") shutil.rmtree(checkout) result = cli.run( project=project, args=["--no-strict", "artifact", "checkout", "--directory", checkout, "target.bst"] ) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") shutil.rmtree(checkout) # Now perform a build on the newly strict dependency, which should cause it to be # available under both strict and non-strict checkout scenarios result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "--directory", checkout, "target.bst"]) result.assert_success() shutil.rmtree(checkout) result = cli.run( project=project, args=["--no-strict", "artifact", "checkout", "--directory", checkout, "target.bst"] ) result.assert_success() shutil.rmtree(checkout) apache-buildstream-27ae392/tests/frontend/completions.py000066400000000000000000000327251514607367700235220ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "completions") MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "shell ", "show ", "source ", "workspace "] MAIN_OPTIONS = [ "--builders ", "-c ", "-C ", "--cache-buildtrees ", "--colors ", "--config ", "--debug ", "--default-mirror ", "--directory ", "--error-lines ", "--fetchers ", "--log-file ", "--max-jobs ", "--message-lines ", "--network-retries ", "--no-colors ", "--no-debug ", "--no-interactive ", "--no-strict ", "--no-verbose ", "-o ", "--option ", "--on-error ", "--pull-buildtrees ", "--pushers ", "--strict ", "--verbose ", "--version ", ] SOURCE_COMMANDS = [ "checkout ", "fetch ", "push ", "track ", ] ARTIFACT_COMMANDS = [ "checkout ", "delete ", "push ", "pull ", "log ", "list-contents ", "show ", ] WORKSPACE_COMMANDS = ["close ", "list ", "open ", "reset "] PROJECT_ELEMENTS = [ "compose-all.bst", "compose-exclude-dev.bst", "compose-include-bin.bst", "import-bin.bst", "import-dev.bst", "target.bst", ] INVALID_ELEMENTS = [ "target.foo", "target.bst.bar", ] MIXED_ELEMENTS = PROJECT_ELEMENTS + INVALID_ELEMENTS def assert_completion(cli, cmd, word_idx, expected, cwd=None): result = cli.run( project=".", cwd=cwd, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)} ) words = [] if result.output: words = result.output.splitlines() # The order is meaningless, bash will # take the results and order it by its # own little heuristics words = sorted(words) expected = sorted(expected) assert words == expected def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None): result = cli.run(cwd=cwd, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)}) words = [] if result.output: words = result.output.splitlines() # The order is meaningless, bash will # take the results and order it by its # own little heuristics words = sorted(words) expected = sorted(expected) assert words != expected @pytest.mark.parametrize( "cmd,word_idx,expected", [ ("bst", 0, []), ("bst ", 1, MAIN_COMMANDS), ("bst artifact ", 2, ARTIFACT_COMMANDS), ("bst source ", 2, SOURCE_COMMANDS), ("bst w ", 1, ["workspace "]), ("bst workspace ", 2, WORKSPACE_COMMANDS), ], ) def test_commands(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) @pytest.mark.parametrize( "cmd,word_idx,expected", [ ("bst -", 1, MAIN_OPTIONS), ("bst --l", 1, ["--log-file "]), # Test that options of subcommands also complete ( "bst --no-colors build -", 3, [ "--deps ", "-d ", "--artifact-remote ", "--source-remote ", "--ignore-project-artifact-remotes ", "--ignore-project-source-remotes ", "--retry-failed ", "-r ", ], ), # Test the behavior of completing after an option that has a # parameter that cannot be completed, vs an option that has # no parameter ("bst --fetchers ", 2, []), ("bst --no-colors ", 2, MAIN_COMMANDS), ], ) def test_options(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) @pytest.mark.parametrize( "cmd,word_idx,expected", [ ("bst --on-error ", 2, ["continue ", "quit ", "terminate "]), ("bst --cache-buildtrees ", 2, ["always ", "auto ", "never "]), ("bst show --deps ", 3, ["all ", "build ", "none ", "run "]), ("bst show --deps=", 2, ["all ", "build ", "none ", "run "]), ("bst show --deps b", 3, ["build "]), ("bst show --deps=b", 2, ["build "]), ("bst show --deps r", 3, ["run "]), ("bst source track --deps ", 4, ["all ", "build ", "none ", "run "]), ], ) def test_option_choice(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize( "cmd,word_idx,expected,subdir", [ # Note that elements/ and files/ are partial completions and # as such do not come with trailing whitespace ("bst --config ", 2, ["cache/", "elements/", "files/", "project.conf "], None), ("bst --log-file ", 2, ["cache/", "elements/", "files/", "project.conf "], None), ("bst --config f", 2, ["files/"], None), ("bst --log-file f", 2, ["files/"], None), ("bst --config files", 2, ["files/bin-files/", "files/dev-files/"], None), ("bst --log-file files", 2, ["files/bin-files/", "files/dev-files/"], None), ("bst --config files/", 2, ["files/bin-files/", "files/dev-files/"], None), ("bst --log-file elements/", 2, [os.path.join("elements", e) + " " for e in PROJECT_ELEMENTS], None), ("bst --config ../", 2, ["../cache/", "../elements/", "../files/", "../project.conf "], "files"), ("bst --config ../elements/", 2, [os.path.join("..", "elements", e) + " " for e in PROJECT_ELEMENTS], "files"), ("bst --config ../nofile", 2, [], "files"), ("bst --config /pony/rainbow/nobodyhas/this/file", 2, [], "files"), ], ) def test_option_file(datafiles, cli, cmd, word_idx, expected, subdir): cwd = str(datafiles) if subdir: cwd = os.path.join(cwd, subdir) assert_completion(cli, cmd, word_idx, expected, cwd=cwd) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize( "cmd,word_idx,expected,subdir", [ # Note that regular files like project.conf are not returned when # completing for a directory ("bst --directory ", 2, ["cache/", "elements/", "files/"], None), ("bst --directory elements/", 2, [], None), ("bst --directory ", 2, ["dev-files/", "bin-files/"], "files"), ("bst --directory ../", 2, ["../cache/", "../elements/", "../files/"], "files"), ], ) def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir): cwd = str(datafiles) if subdir: cwd = os.path.join(cwd, subdir) assert_completion(cli, cmd, word_idx, expected, cwd=cwd) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project,cmd,word_idx,expected,subdir", [ # When running in the project directory ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], None), ( "project", "bst build com", 2, ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], None, ), # When running from the files subdir ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], "files"), ( "project", "bst build com", 2, ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], "files", ), # When passing the project directory ("project", "bst --directory ../ show ", 4, [e + " " for e in PROJECT_ELEMENTS], "files"), ( "project", "bst --directory ../ build com", 4, ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], "files", ), # Also try multi arguments together ("project", "bst --directory ../ artifact checkout t ", 5, ["target.bst "], "files"), ("project", "bst --directory ../ artifact checkout --directory ", 6, ["bin-files/", "dev-files/"], "files"), # When running in the project directory ("no-element-path", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], None), ( "no-element-path", "bst build com", 2, ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], None, ), # When running from the files subdir ("no-element-path", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], "files"), ( "no-element-path", "bst build com", 2, ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], "files", ), # When passing the project directory ("no-element-path", "bst --directory ../ show ", 4, [e + " " for e in PROJECT_ELEMENTS] + ["files/"], "files"), ("no-element-path", "bst --directory ../ show f", 4, ["files/"], "files"), ("no-element-path", "bst --directory ../ show files/", 4, ["files/bin-files/", "files/dev-files/"], "files"), ( "no-element-path", "bst --directory ../ build com", 4, ["compose-all.bst ", "compose-include-bin.bst ", "compose-exclude-dev.bst "], "files", ), # Also try multi arguments together ("no-element-path", "bst --directory ../ artifact checkout t ", 5, ["target.bst "], "files"), ( "no-element-path", "bst --directory ../ artifact checkout --directory ", 6, ["bin-files/", "dev-files/"], "files", ), # When element-path have sub-folders ("sub-folders", "bst show base", 2, ["base/wanted.bst "], None), ("sub-folders", "bst show base/", 2, ["base/wanted.bst "], None), ], ) def test_argument_element(datafiles, cli, project, cmd, word_idx, expected, subdir): cwd = os.path.join(str(datafiles), project) if subdir: cwd = os.path.join(cwd, subdir) assert_completion(cli, cmd, word_idx, expected, cwd=cwd) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "project,cmd,word_idx,expected,subdir", [ # When element has invalid suffix ("project", "bst --directory ../ show ", 4, [e + " " for e in MIXED_ELEMENTS], "files") ], ) def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expected, subdir): cwd = os.path.join(str(datafiles), project) if subdir: cwd = os.path.join(cwd, subdir) assert_completion_failed(cli, cmd, word_idx, expected, cwd=cwd) @pytest.mark.parametrize( "cmd,word_idx,expected", [ ("bst he", 1, ["help "]), ("bst help ", 2, MAIN_COMMANDS), ("bst help artifact ", 3, ARTIFACT_COMMANDS), ("bst help in", 2, ["init "]), ("bst help source ", 3, SOURCE_COMMANDS), ("bst help artifact ", 3, ARTIFACT_COMMANDS), ("bst help w", 2, ["workspace "]), ("bst help workspace ", 3, WORKSPACE_COMMANDS), ], ) def test_help_commands(cli, cmd, word_idx, expected): assert_completion(cli, cmd, word_idx, expected) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_argument_artifact(cli, datafiles): project = str(datafiles) # Build an import element with no dependencies (this will generate one artifact with 2 keys) result = cli.run(project=project, args=["build", "import-bin.bst"]) # Has no dependencies result.assert_success() # Use hard coded artifact names, cache keys should be stable now artifacts = [ "test/import-bin/4066f11d4e3f681bc7b9f738af8093473478907946801be2f28033e0f4c904a1", "test/import-bin/b9f5aa9c50349a109431e2fbc23d3384a3a0bf490158b0352e896b82033c5137", ] # Test autocompletion of the artifact cmds = ["bst artifact log ", "bst artifact log t", "bst artifact log test/"] for i, cmd in enumerate(cmds): word_idx = 3 result = cli.run( project=project, cwd=project, env={"_BST_COMPLETION": "complete", "COMP_WORDS": cmd, "COMP_CWORD": str(word_idx)}, ) if result.output: words = result.output.splitlines() # This leaves an extra space on each e.g. ['foo.bst '] words = [word.strip() for word in words] # We should now be able to see the artifacts, but the order in which artifacts # are displayed in the completion list is not guaranteed to be ordered, so we # test for both orders. if i == 0: expected1 = PROJECT_ELEMENTS + artifacts expected2 = PROJECT_ELEMENTS + list(reversed(artifacts)) elif i == 1: expected1 = ["target.bst"] + artifacts expected2 = ["target.bst"] + list(reversed(artifacts)) elif i == 2: expected1 = artifacts expected2 = list(reversed(artifacts)) else: assert False, "unreachable" assert words in (expected1, expected2) apache-buildstream-27ae392/tests/frontend/completions/000077500000000000000000000000001514607367700231375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/000077500000000000000000000000001514607367700261345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/compose-all.bst000066400000000000000000000003441514607367700310620ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/frontend/completions/no-element-path/compose-exclude-dev.bst000066400000000000000000000004251514607367700325170ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False # Exclude the dev domain exclude: - devel apache-buildstream-27ae392/tests/frontend/completions/no-element-path/compose-include-bin.bst000066400000000000000000000004301514607367700324770ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False # Only include the runtim include: - runtime apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/000077500000000000000000000000001514607367700272365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/bin-files/000077500000000000000000000000001514607367700311065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/bin-files/usr/000077500000000000000000000000001514607367700317175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/bin-files/usr/bin/000077500000000000000000000000001514607367700324675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700335150ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/dev-files/000077500000000000000000000000001514607367700311145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/dev-files/usr/000077500000000000000000000000001514607367700317255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/dev-files/usr/include/000077500000000000000000000000001514607367700333505ustar00rootroot00000000000000pony.h000066400000000000000000000003711514607367700344300ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/no-element-path/files/dev-files/usr/include#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/completions/no-element-path/import-bin.bst000066400000000000000000000000741514607367700307270ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/frontend/completions/no-element-path/import-dev.bst000066400000000000000000000000741514607367700307350ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/frontend/completions/no-element-path/project.conf000066400000000000000000000001051514607367700304450ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 apache-buildstream-27ae392/tests/frontend/completions/no-element-path/target.bst000066400000000000000000000001641514607367700301350ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - compose-all.bst apache-buildstream-27ae392/tests/frontend/completions/project/000077500000000000000000000000001514607367700246055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/elements/000077500000000000000000000000001514607367700264215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/elements/compose-all.bst000066400000000000000000000003441514607367700313470ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/frontend/completions/project/elements/compose-exclude-dev.bst000066400000000000000000000004251514607367700330040ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False # Exclude the dev domain exclude: - devel apache-buildstream-27ae392/tests/frontend/completions/project/elements/compose-include-bin.bst000066400000000000000000000004301514607367700327640ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False # Only include the runtim include: - runtime apache-buildstream-27ae392/tests/frontend/completions/project/elements/import-bin.bst000066400000000000000000000000741514607367700312140ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/frontend/completions/project/elements/import-dev.bst000066400000000000000000000000741514607367700312220ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/frontend/completions/project/elements/target.bst000066400000000000000000000001641514607367700304220ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - compose-all.bst apache-buildstream-27ae392/tests/frontend/completions/project/files/000077500000000000000000000000001514607367700257075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/bin-files/000077500000000000000000000000001514607367700275575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/bin-files/usr/000077500000000000000000000000001514607367700303705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/bin-files/usr/bin/000077500000000000000000000000001514607367700311405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700321660ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/frontend/completions/project/files/dev-files/000077500000000000000000000000001514607367700275655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/dev-files/usr/000077500000000000000000000000001514607367700303765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/dev-files/usr/include/000077500000000000000000000000001514607367700320215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/project/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700331600ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/completions/project/project.conf000066400000000000000000000001341514607367700271200ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/completions/sub-folders/000077500000000000000000000000001514607367700253645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/sub-folders/base/000077500000000000000000000000001514607367700262765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/sub-folders/base/unwanted.bst000066400000000000000000000000741514607367700306360ustar00rootroot00000000000000kind: manual description: | Not auto-completed element apache-buildstream-27ae392/tests/frontend/completions/sub-folders/elements/000077500000000000000000000000001514607367700272005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/sub-folders/elements/base.bst000066400000000000000000000001001514607367700306130ustar00rootroot00000000000000kind: stack description: Base stack depends: - base/wanted.bst apache-buildstream-27ae392/tests/frontend/completions/sub-folders/elements/base/000077500000000000000000000000001514607367700301125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/completions/sub-folders/elements/base/wanted.bst000066400000000000000000000000701514607367700321030ustar00rootroot00000000000000kind: manual description: | Auto-completed element apache-buildstream-27ae392/tests/frontend/completions/sub-folders/elements/hello.bst000066400000000000000000000000551514607367700310150ustar00rootroot00000000000000kind: manual description: | Hello world apache-buildstream-27ae392/tests/frontend/completions/sub-folders/project.conf000066400000000000000000000001341514607367700276770ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/compose_splits.py000066400000000000000000000033371514607367700242260ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing.runcli import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.parametrize("target", [("compose-include-bin.bst"), ("compose-exclude-dev.bst")]) @pytest.mark.datafiles(DATA_DIR) def test_compose_splits(datafiles, cli, target): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # First build it result = cli.run(project=project, args=["build", target]) result.assert_success() # Now check it out result = cli.run(project=project, args=["artifact", "checkout", target, "--directory", checkout]) result.assert_success() # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "include", "pony.h") assert not os.path.exists(filename) apache-buildstream-27ae392/tests/frontend/configurable_warnings.py000066400000000000000000000047621514607367700255360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.plugin import CoreWarnings from buildstream.exceptions import ErrorDomain from buildstream import _yaml from buildstream._testing.runcli import cli # pylint: disable=unused-import TOP_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configuredwarning") def get_project(fatal_warnings): return { "name": "test", "min-version": "2.0", "element-path": "elements", "plugins": [{"origin": "local", "path": "plugins", "elements": ["warninga", "warningb", "corewarn"]}], "fatal-warnings": fatal_warnings, } def build_project(datafiles, fatal_warnings): project_path = str(datafiles) project = get_project(fatal_warnings) _yaml.roundtrip_dump(project, os.path.join(project_path, "project.conf")) return project_path @pytest.mark.datafiles(TOP_DIR) @pytest.mark.parametrize( "element_name, fatal_warnings, expect_fatal, error_domain", [ ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM), ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM), ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM), ("corewarn.bst", [], False, None), ("warninga.bst", [], False, None), ("warningb.bst", [], False, None), ("warninga.bst", [CoreWarnings.OVERLAPS], False, None), ("warningb.bst", [CoreWarnings.OVERLAPS], False, None), ], ) def test_fatal_warnings(cli, datafiles, element_name, fatal_warnings, expect_fatal, error_domain): project_path = build_project(datafiles, fatal_warnings) result = cli.run(project=project_path, args=["build", element_name]) if expect_fatal: result.assert_main_error(error_domain, None, "Expected fatal execution") else: result.assert_success("Unexpected fatal execution") apache-buildstream-27ae392/tests/frontend/configuredwarning/000077500000000000000000000000001514607367700243165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/configuredwarning/elements/000077500000000000000000000000001514607367700261325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/configuredwarning/elements/corewarn.bst000066400000000000000000000000161514607367700304610ustar00rootroot00000000000000kind: corewarnapache-buildstream-27ae392/tests/frontend/configuredwarning/elements/warninga.bst000066400000000000000000000000171514607367700304500ustar00rootroot00000000000000kind: warninga apache-buildstream-27ae392/tests/frontend/configuredwarning/elements/warningb.bst000066400000000000000000000000171514607367700304510ustar00rootroot00000000000000kind: warningb apache-buildstream-27ae392/tests/frontend/configuredwarning/plugins/000077500000000000000000000000001514607367700257775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/configuredwarning/plugins/corewarn.py000066400000000000000000000010251514607367700301670ustar00rootroot00000000000000from buildstream import Element from buildstream.plugin import CoreWarnings class CoreWarn(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): pass def configure_sandbox(self, sandbox): pass def stage(self, sandbox): pass def assemble(self, sandbox): self.warn("Testing: CoreWarning produced during assemble", warning_token=CoreWarnings.OVERLAPS) def setup(): return CoreWarn apache-buildstream-27ae392/tests/frontend/configuredwarning/plugins/warninga.py000066400000000000000000000007641514607367700301660ustar00rootroot00000000000000from buildstream import Element WARNING_A = "warning-a" class WarningA(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): pass def configure_sandbox(self, sandbox): pass def stage(self, sandbox): pass def assemble(self, sandbox): self.warn("Testing: warning-a produced during assemble", warning_token=WARNING_A) def setup(): return WarningA apache-buildstream-27ae392/tests/frontend/configuredwarning/plugins/warningb.py000066400000000000000000000007641514607367700301670ustar00rootroot00000000000000from buildstream import Element WARNING_B = "warning-b" class WarningB(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): pass def configure_sandbox(self, sandbox): pass def stage(self, sandbox): pass def assemble(self, sandbox): self.warn("Testing: warning-b produced during assemble", warning_token=WARNING_B) def setup(): return WarningB apache-buildstream-27ae392/tests/frontend/configuredwarning/project.conf000066400000000000000000000002121514607367700266260ustar00rootroot00000000000000name: test min-version: 2.0 element-path: elements plugins: - origin: local path: element_plugins elements: - warninga - warningb apache-buildstream-27ae392/tests/frontend/consistencyerror/000077500000000000000000000000001514607367700242165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/consistencyerror/__init__.py000066400000000000000000000000001514607367700263150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/consistencyerror/bug.bst000066400000000000000000000002111514607367700254770ustar00rootroot00000000000000kind: import description: An element with an unhandled exception in checking whether it is cached or not sources: - kind: consistencybug apache-buildstream-27ae392/tests/frontend/consistencyerror/error.bst000066400000000000000000000002071514607367700260600ustar00rootroot00000000000000kind: import description: An element with a failing source when checking whether it is cached or not sources: - kind: consistencyerror apache-buildstream-27ae392/tests/frontend/consistencyerror/plugins/000077500000000000000000000000001514607367700256775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/consistencyerror/plugins/__init__.py000066400000000000000000000000001514607367700277760ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/consistencyerror/plugins/consistencybug.py000066400000000000000000000012641514607367700313130ustar00rootroot00000000000000from buildstream import Source class CustomError(Exception): pass class ConsistencyBugSource(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def is_resolved(self): return True def is_cached(self): # Raise an unhandled exception (not a BstError) raise CustomError("Something went terribly wrong") def get_ref(self): return None def set_ref(self, ref, node): pass def fetch(self, **kwargs): pass def stage(self, directory): pass def setup(): return ConsistencyBugSource apache-buildstream-27ae392/tests/frontend/consistencyerror/plugins/consistencyerror.py000066400000000000000000000012551514607367700316670ustar00rootroot00000000000000from buildstream import Source, SourceError class ConsistencyErrorSource(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def is_resolved(self): return True def is_cached(self): # Raise an error unconditionally raise SourceError("Something went terribly wrong", reason="the-consistency-error") def get_ref(self): return None def set_ref(self, ref, node): pass def fetch(self, **kwargs): pass def stage(self, directory): pass def setup(): return ConsistencyErrorSource apache-buildstream-27ae392/tests/frontend/consistencyerror/project.conf000066400000000000000000000003341514607367700265330ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: test min-version: 2.0 # Whitelist the local test Sources # plugins: - origin: local path: plugins sources: - consistencyerror - consistencybug apache-buildstream-27ae392/tests/frontend/cross_junction_workspace.py000066400000000000000000000123051514607367700262760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from buildstream import _yaml def prepare_junction_project(cli, tmpdir): main_project = tmpdir.join("main") sub_project = tmpdir.join("sub") os.makedirs(str(main_project)) os.makedirs(str(sub_project)) _yaml.roundtrip_dump({"name": "main", "min-version": "2.0"}, str(main_project.join("project.conf"))) _yaml.roundtrip_dump({"name": "sub", "min-version": "2.0"}, str(sub_project.join("project.conf"))) import_dir = tmpdir.join("import") os.makedirs(str(import_dir)) with open(str(import_dir.join("hello.txt")), "w", encoding="utf-8") as f: f.write("hello!") import_repo_dir = tmpdir.join("import_repo") os.makedirs(str(import_repo_dir)) import_repo = create_repo("tar", str(import_repo_dir)) import_ref = import_repo.create(str(import_dir)) _yaml.roundtrip_dump( {"kind": "import", "sources": [import_repo.source_config(ref=import_ref)]}, str(sub_project.join("data.bst")) ) sub_repo_dir = tmpdir.join("sub_repo") os.makedirs(str(sub_repo_dir)) sub_repo = create_repo("tar", str(sub_repo_dir)) sub_ref = sub_repo.create(str(sub_project)) _yaml.roundtrip_dump( {"kind": "junction", "sources": [sub_repo.source_config(ref=sub_ref)]}, str(main_project.join("sub.bst")) ) args = ["source", "fetch", "sub.bst"] result = cli.run(project=str(main_project), args=args) result.assert_success() return str(main_project) def open_cross_junction(cli, tmpdir): project = prepare_junction_project(cli, tmpdir) element = "sub.bst:data.bst" oldkey = cli.get_element_key(project, element) workspace = tmpdir.join("workspace") args = ["workspace", "open", "--directory", str(workspace), element] result = cli.run(project=project, args=args) result.assert_success() assert cli.get_element_state(project, element) == "buildable" assert os.path.exists(str(workspace.join("hello.txt"))) assert cli.get_element_key(project, element) != oldkey return project, workspace def test_open_cross_junction(cli, tmpdir): open_cross_junction(cli, tmpdir) def test_list_cross_junction(cli, tmpdir): project, _ = open_cross_junction(cli, tmpdir) element = "sub.bst:data.bst" args = ["workspace", "list"] result = cli.run(project=project, args=args) result.assert_success() loaded = _yaml.load_data(result.output) workspaces = loaded.get_sequence("workspaces") assert len(workspaces) == 1 first_workspace = workspaces.mapping_at(0) assert "element" in first_workspace assert first_workspace.get_str("element") == element def test_close_cross_junction(cli, tmpdir): project, workspace = open_cross_junction(cli, tmpdir) element = "sub.bst:data.bst" args = ["workspace", "close", "--remove-dir", element] result = cli.run(project=project, args=args) result.assert_success() assert not os.path.exists(str(workspace)) args = ["workspace", "list"] result = cli.run(project=project, args=args) result.assert_success() loaded = _yaml.load_data(result.output) workspaces = loaded.get_sequence("workspaces") assert not workspaces def test_close_all_cross_junction(cli, tmpdir): project, workspace = open_cross_junction(cli, tmpdir) args = ["workspace", "close", "--remove-dir", "--all"] result = cli.run(project=project, args=args) result.assert_success() assert not os.path.exists(str(workspace)) args = ["workspace", "list"] result = cli.run(project=project, args=args) result.assert_success() loaded = _yaml.load_data(result.output) workspaces = loaded.get_sequence("workspaces") assert not workspaces def test_subdir_command_cross_junction(cli, tmpdir): # i.e. commands can be run successfully from a subdirectory of the # junction's workspace, in case project loading logic has gone wrong project = prepare_junction_project(cli, tmpdir) workspace = os.path.join(str(tmpdir), "workspace") junction_element = "sub.bst" # Open the junction as a workspace args = ["workspace", "open", "--directory", workspace, junction_element] result = cli.run(project=project, args=args) result.assert_success() # Run commands from a subdirectory of the workspace newdir = os.path.join(str(workspace), "newdir") element_name = "data.bst" os.makedirs(newdir) result = cli.run(project=str(workspace), args=["-C", newdir, "show", element_name]) result.assert_success() apache-buildstream-27ae392/tests/frontend/default-target/000077500000000000000000000000001514607367700235135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/default-target/elements/000077500000000000000000000000001514607367700253275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/default-target/elements/dummy_1.bst000066400000000000000000000000141514607367700274070ustar00rootroot00000000000000kind: stack apache-buildstream-27ae392/tests/frontend/default-target/elements/dummy_2.bst000066400000000000000000000000141514607367700274100ustar00rootroot00000000000000kind: stack apache-buildstream-27ae392/tests/frontend/default-target/elements/dummy_3.bst000066400000000000000000000000141514607367700274110ustar00rootroot00000000000000kind: stack apache-buildstream-27ae392/tests/frontend/default-target/elements/dummy_stack.bst000066400000000000000000000000621514607367700303570ustar00rootroot00000000000000kind: stack depends: - dummy_1.bst - dummy_2.bst apache-buildstream-27ae392/tests/frontend/default-target/files/000077500000000000000000000000001514607367700246155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/default-target/files/sub-project/000077500000000000000000000000001514607367700270525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/default-target/files/sub-project/elements/000077500000000000000000000000001514607367700306665ustar00rootroot00000000000000dummy_subproject.bst000066400000000000000000000000141514607367700347070ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/default-target/files/sub-project/elementskind: stack apache-buildstream-27ae392/tests/frontend/default-target/files/sub-project/project.conf000066400000000000000000000001151514607367700313640ustar00rootroot00000000000000name: test-default-target-subproject min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/default-target/project.conf000066400000000000000000000001021514607367700260210ustar00rootroot00000000000000name: test-default-target min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/default_target.py000066400000000000000000000166471514607367700241650ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli, create_repo # pylint: disable=unused-import from tests.testutils import create_artifact_share # project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "default-target", ) # # When no target is default, then expect all targets to be built # @pytest.mark.datafiles(DATA_DIR) def test_no_default(cli, datafiles): project = str(datafiles) all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_3.bst", "dummy_stack.bst"] result = cli.run(project=project, args=["build"]) result.assert_success() states = cli.get_element_states(project, all_targets) assert all(states[e] == "cached" for e in all_targets) # # When the stack is specified as the default target, then # expect only it and it's dependencies to be built # @pytest.mark.datafiles(DATA_DIR) def test_default_target(cli, datafiles): project = str(datafiles) project_path = os.path.join(project, "project.conf") # First, modify project configuration to set a default target project_conf = { "name": "test-default-target", "min-version": "2.0", "element-path": "elements", "defaults": {"targets": ["dummy_stack.bst"]}, } _yaml.roundtrip_dump(project_conf, project_path) # dummy_stack only depends on dummy_1 and dummy_2, but not dummy_3 all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_stack.bst"] result = cli.run(project=project, args=["build"]) result.assert_success() states = cli.get_element_states(project, all_targets) assert all(states[e] == "cached" for e in all_targets) # assert that dummy_3 isn't included in the output assert "dummy_3.bst" not in states # # Even when there is a junction, expect that the elements in the # subproject referred to by the toplevel project are built when # calling `bst build` and no default is specified. # @pytest.mark.datafiles(DATA_DIR) def test_no_default_with_junction(cli, datafiles): project = str(datafiles) junction_path = os.path.join(project, "elements", "junction.bst") target_path = os.path.join(project, "elements", "junction-target.bst") # First, create a junction element to refer to the subproject junction_config = { "kind": "junction", "sources": [ { "kind": "local", "path": "files/sub-project", } ], } _yaml.roundtrip_dump(junction_config, junction_path) # Then, create a stack element with dependency on cross junction element target_config = {"kind": "stack", "depends": ["junction.bst:dummy_subproject.bst"]} _yaml.roundtrip_dump(target_config, target_path) # Now try to perform a build # This should automatically fetch the junction at load time. result = cli.run(project=project, args=["build"]) result.assert_success() assert cli.get_element_state(project, "junction.bst:dummy_subproject.bst") == "cached" assert cli.get_element_state(project, "junction-target.bst") == "cached" ################################### # track/fetch operations # ################################### @pytest.mark.datafiles(DATA_DIR) def test_default_target_track(cli, tmpdir, datafiles): project = str(datafiles) project_path = os.path.join(project, "project.conf") target = "track-fetch-test.bst" # First, create an element with trackable sources repo = create_repo("tar", str(tmpdir)) repo.create(project) element_conf = {"kind": "import", "sources": [repo.source_config()]} _yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target)) # Then, make it the default target project_conf = { "name": "test-default-target", "min-version": "2.0", "element-path": "elements", "defaults": {"targets": [target]}, } _yaml.roundtrip_dump(project_conf, project_path) # Setup finished. Track it now assert cli.get_element_state(project, target) == "no reference" result = cli.run(project=project, args=["source", "track"]) result.assert_success() # Tracking will result in fetching it automatically, so we expect the state # to be buildable. assert cli.get_element_state(project, target) == "buildable" @pytest.mark.datafiles(DATA_DIR) def test_default_target_fetch(cli, tmpdir, datafiles): project = str(datafiles) project_path = os.path.join(project, "project.conf") target = "track-fetch-test.bst" # First, create an element with trackable sources repo = create_repo("tar", str(tmpdir)) ref = repo.create(project) element_conf = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target)) # Then, make it the default target project_conf = { "name": "test-default-target", "min-version": "2.0", "element-path": "elements", "defaults": {"targets": [target]}, } _yaml.roundtrip_dump(project_conf, project_path) # Setup finished. Track it now assert cli.get_element_state(project, target) == "fetch needed" result = cli.run(project=project, args=["source", "fetch"]) result.assert_success() assert cli.get_element_state(project, target) == "buildable" ################################### # pull/push operations # ################################### @pytest.mark.datafiles(DATA_DIR) def test_default_target_push_pull(cli, tmpdir, datafiles): project = str(datafiles) project_path = os.path.join(project, "project.conf") target = "dummy_1.bst" # Set a default target project_conf = { "name": "test-default-target", "min-version": "2.0", "element-path": "elements", "defaults": {"targets": [target]}, } _yaml.roundtrip_dump(project_conf, project_path) # Build the target result = cli.run(project=project, args=["build"]) result.assert_success() assert cli.get_element_state(project, target) == "cached" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Push the artifacts cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["artifact", "push"]) result.assert_success() # Delete local artifacts # Note that `artifact delete` does not support default targets result = cli.run(project=project, args=["artifact", "delete", target]) result.assert_success() # Target should be buildable now, and we should be able to pull it assert cli.get_element_state(project, target) == "buildable" result = cli.run(project=project, args=["artifact", "pull"]) assert cli.get_element_state(project, target) == "cached" apache-buildstream-27ae392/tests/frontend/exceptions/000077500000000000000000000000001514607367700227645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/exceptions/build.bst000066400000000000000000000001511514607367700245720ustar00rootroot00000000000000kind: manual description: Some kinda manual element depends: - first-level-1.bst - first-level-2.bst apache-buildstream-27ae392/tests/frontend/exceptions/first-level-1.bst000066400000000000000000000001321514607367700260640ustar00rootroot00000000000000kind: manual description: Depends on a removed dependency depends: - second-level-1.bst apache-buildstream-27ae392/tests/frontend/exceptions/first-level-2.bst000066400000000000000000000001471514607367700260730ustar00rootroot00000000000000kind: manual description: Shares a dependency with a removed dependency depends: - third-level-2.bst apache-buildstream-27ae392/tests/frontend/exceptions/fourth-level-1.bst000066400000000000000000000000541514607367700262470ustar00rootroot00000000000000kind: manual description: Should be removed apache-buildstream-27ae392/tests/frontend/exceptions/fourth-level-2.bst000066400000000000000000000000671514607367700262540ustar00rootroot00000000000000kind: manual description: Should also ~not~ be removed apache-buildstream-27ae392/tests/frontend/exceptions/fourth-level-3.bst000066400000000000000000000000601514607367700262460ustar00rootroot00000000000000kind: manual description: Should not be removed apache-buildstream-27ae392/tests/frontend/exceptions/project.conf000066400000000000000000000001321514607367700252750ustar00rootroot00000000000000# Basic project configuration that doesnt override anything # name: pony min-version: 2.0 apache-buildstream-27ae392/tests/frontend/exceptions/second-level-1.bst000066400000000000000000000002761514607367700262210ustar00rootroot00000000000000kind: manual description: Depends uniquely on one dependency, shares another, has another unique nested dependency depends: - third-level-1.bst - third-level-2.bst - third-level-3.bst apache-buildstream-27ae392/tests/frontend/exceptions/third-level-1.bst000066400000000000000000000001501514607367700260470ustar00rootroot00000000000000kind: manual description: Should be removed, and not "revive" its child depends: - fourth-level-1.bst apache-buildstream-27ae392/tests/frontend/exceptions/third-level-2.bst000066400000000000000000000001221514607367700260470ustar00rootroot00000000000000kind: manual description: Should ~not~ be removed depends: - fourth-level-2.bst apache-buildstream-27ae392/tests/frontend/exceptions/third-level-3.bst000066400000000000000000000001721514607367700260550ustar00rootroot00000000000000kind: manual description: Should be an explicit dependency, and *not* remove its children depends: - fourth-level-3.bst apache-buildstream-27ae392/tests/frontend/exceptions/unrelated-1.bst000066400000000000000000000002051514607367700256140ustar00rootroot00000000000000kind: manual description: Unrelated to the rest of the pipeline, not loaded when targeting build.bst depends: - second-level-1.bst apache-buildstream-27ae392/tests/frontend/exceptions/unrelated-2.bst000066400000000000000000000002041514607367700256140ustar00rootroot00000000000000kind: manual description: Unrelated to the rest of the pipeline, not loaded when targeting build.bst depends: - first-level-2.bst apache-buildstream-27ae392/tests/frontend/fetch.py000066400000000000000000000141221514607367700222460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from tests.testutils import generate_junction from . import configure_project # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") # Test all possible choices of the `--deps` option. # # NOTE: Elements used in this test must have sources that are not already # cached. The kind of the sources do not matter so long as they need to # be fetched from somewhere. # Currently we use remote sources for this purpose. # @pytest.mark.datafiles(os.path.join(TOP_DIR, "source-fetch")) @pytest.mark.parametrize( "deps, expected_states", [ ("build", ("fetch needed", "buildable", "fetch needed")), ("none", ("waiting", "fetch needed", "fetch needed")), ("run", ("waiting", "fetch needed", "buildable")), ("all", ("waiting", "buildable", "buildable")), ], ) def test_fetch_deps(cli, datafiles, deps, expected_states): project = str(datafiles) target = "bananas.bst" build_dep = "apples.bst" runtime_dep = "oranges.bst" # Assert that none of the sources are cached states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert all(state == "fetch needed" for state in states.values()) # Now fetch the specified sources result = cli.run(project=project, args=["source", "fetch", "--deps", deps, target]) result.assert_success() # Finally assert that we have fetched _only_ the desired sources states = cli.get_element_states(project, [target, build_dep, runtime_dep]) states_flattened = (states[target], states[build_dep], states[runtime_dep]) assert states_flattened == expected_states @pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_fetch_consistency_error(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["source", "fetch", "error.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, "the-consistency-error") @pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_fetch_consistency_bug(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["source", "fetch", "bug.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, "source-bug") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"]) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_unfetched_junction(cli, tmpdir, datafiles, strict, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) cli.configure({"projects": {"test": {"strict": strict}}}) # Create a repo to hold the subproject and generate a junction element for it ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # if ref_storage == "project.refs": project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Now try to fetch it, this should automatically result in fetching # the junction itself. result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to fetch it, this will bail with the appropriate error # informing the user to track the junction first result = cli.run(project=project, args=["source", "fetch", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr apache-buildstream-27ae392/tests/frontend/help.py000066400000000000000000000026001514607367700221030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import pytest from buildstream._testing.runcli import cli # pylint: disable=unused-import def assert_help(cli_output): expected_start = "Usage: " if not cli_output.startswith(expected_start): raise AssertionError( "Help output expected to begin with '{}',".format(expected_start) + " output was: {}".format(cli_output) ) def test_help_main(cli): result = cli.run(args=["--help"]) result.assert_success() assert_help(result.output) @pytest.mark.parametrize("command", [("artifact"), ("build"), ("shell"), ("show"), ("source"), ("workspace")]) def test_help(cli, command): result = cli.run(args=[command, "--help"]) result.assert_success() assert_help(result.output) apache-buildstream-27ae392/tests/frontend/init.py000066400000000000000000000136611514607367700221270ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml from buildstream import utils from buildstream._frontend.app import App from buildstream.exceptions import ErrorDomain, LoadErrorReason def get_default_min_version(): bst_major, bst_minor = utils.get_bst_version() return "{}.{}".format(bst_major, bst_minor) def test_defaults(cli, tmpdir): project = str(tmpdir) project_path = os.path.join(project, "project.conf") result = cli.run(args=["init", "--project-name", "foo", project]) result.assert_success() project_conf = _yaml.load(project_path, shortname=None) assert project_conf.get_str("name") == "foo" assert project_conf.get_str("min-version") == get_default_min_version() assert project_conf.get_str("element-path") == "elements" def test_all_options(cli, tmpdir): project = str(tmpdir) project_path = os.path.join(project, "project.conf") result = cli.run( args=["init", "--project-name", "foo", "--min-version", "2.0", "--element-path", "ponies", project] ) result.assert_success() project_conf = _yaml.load(project_path, shortname=None) assert project_conf.get_str("name") == "foo" assert project_conf.get_str("min-version") == "2.0" assert project_conf.get_str("element-path") == "ponies" elements_dir = os.path.join(project, "ponies") assert os.path.isdir(elements_dir) def test_no_project_name(cli, tmpdir): result = cli.run(args=["init", str(tmpdir)]) result.assert_main_error(ErrorDomain.APP, "unspecified-project-name") def test_project_exists(cli, tmpdir): project = str(tmpdir) project_path = os.path.join(project, "project.conf") with open(project_path, "w", encoding="utf-8") as f: f.write("name: pony\n") result = cli.run(args=["init", "--project-name", "foo", project]) result.assert_main_error(ErrorDomain.APP, "project-exists") def test_force_overwrite_project(cli, tmpdir): project = str(tmpdir) project_path = os.path.join(project, "project.conf") with open(project_path, "w", encoding="utf-8") as f: f.write("name: pony\n") result = cli.run(args=["init", "--project-name", "foo", "--force", project]) result.assert_success() project_conf = _yaml.load(project_path, shortname=None) assert project_conf.get_str("name") == "foo" assert project_conf.get_str("min-version") == get_default_min_version() def test_relative_path_directory_as_argument(cli, tmpdir): project = os.path.join(str(tmpdir), "child-directory") os.makedirs(project, exist_ok=True) project_path = os.path.join(project, "project.conf") rel_path = os.path.relpath(project) result = cli.run(args=["init", "--project-name", "foo", rel_path]) result.assert_success() project_conf = _yaml.load(project_path, shortname=None) assert project_conf.get_str("name") == "foo" assert project_conf.get_str("min-version") == get_default_min_version() assert project_conf.get_str("element-path") == "elements" def test_set_directory_and_directory_as_argument(cli, tmpdir): result = cli.run(args=["-C", "/foo/bar", "init", "--project-name", "foo", "/boo/far"]) result.assert_main_error(ErrorDomain.APP, "init-with-set-directory") @pytest.mark.parametrize("project_name", [("Micheal Jackson"), ("one+one")]) def test_bad_project_name(cli, tmpdir, project_name): result = cli.run(args=["init", "--project-name", str(tmpdir)]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_SYMBOL_NAME) @pytest.mark.parametrize("min_version", [("-1"), ("1.4"), ("2.900"), ("abc")]) def test_bad_min_version(cli, tmpdir, min_version): result = cli.run(args=["init", "--project-name", "foo", "--min-version", min_version, str(tmpdir)]) result.assert_main_error(ErrorDomain.APP, "invalid-min-version") @pytest.mark.parametrize("element_path", [("/absolute/path"), ("../outside/of/project")]) def test_bad_element_path(cli, tmpdir, element_path): result = cli.run(args=["init", "--project-name", "foo", "--element-path", element_path, str(tmpdir)]) result.assert_main_error(ErrorDomain.APP, "invalid-element-path") @pytest.mark.parametrize("element_path", [("foo"), ("foo/bar")]) def test_element_path_interactive(cli, tmp_path, monkeypatch, element_path): project = tmp_path project_conf_path = project.joinpath("project.conf") class DummyInteractiveApp(App): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.interactive = True @classmethod def create(cls, *args, **kwargs): return DummyInteractiveApp(*args, **kwargs) def _init_project_interactive(self, *args, **kwargs): # pylint: disable=signature-differs return ("project_name", "2.0", element_path) monkeypatch.setattr(App, "create", DummyInteractiveApp.create) result = cli.run(args=["init", str(project)]) result.assert_success() full_element_path = project.joinpath(element_path) assert full_element_path.exists() project_conf = _yaml.load(str(project_conf_path), shortname=None) assert project_conf.get_str("name") == "project_name" assert project_conf.get_str("min-version") == "2.0" assert project_conf.get_str("element-path") == element_path apache-buildstream-27ae392/tests/frontend/interactive_init.py000066400000000000000000000035221514607367700245170ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import pexpect from buildstream import _yaml from buildstream import utils from tests.testutils.constants import PEXPECT_TIMEOUT_SHORT def test_init(tmpdir): session = pexpect.spawn("bst", ["--no-colors", "init", str(tmpdir)], timeout=PEXPECT_TIMEOUT_SHORT) name = "test-project" min_version = "2.0" element_path = "my-elements" bst_major, bst_minor = utils.get_bst_version() # For the version check, artificially set the version to at least # version 2.0 # # TODO: Remove this code block after releasing 2.0 # if bst_major < 2: bst_major = 2 bst_minor = 0 session.expect_exact("Project name:") session.sendline(name) session.expect_exact("Minimum version [{}.{}]:".format(bst_major, bst_minor)) session.sendline(str(min_version)) session.expect_exact("Element path [elements]:") session.sendline(element_path) session.expect_exact("Created project.conf") session.close() # Now assert that a project.conf got created with expected values project_conf = _yaml.load(os.path.join(str(tmpdir), "project.conf"), shortname=None) assert project_conf.get_str("name") == name assert project_conf.get_str("min-version") == min_version assert project_conf.get_str("element-path") == element_path apache-buildstream-27ae392/tests/frontend/invalid_element_path/000077500000000000000000000000001514607367700247565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/invalid_element_path/project.conf000066400000000000000000000001351514607367700272720ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 elephant-path: elements apache-buildstream-27ae392/tests/frontend/large_directory.py000066400000000000000000000056021514607367700243360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name from contextlib import contextmanager import os import pytest import grpc from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import create_artifact_share, assert_shared # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @contextmanager def limit_grpc_message_length(limit): orig_insecure_channel = grpc.insecure_channel def new_insecure_channel(target): return orig_insecure_channel(target, options=(("grpc.max_send_message_length", limit),)) grpc.insecure_channel = new_insecure_channel try: yield finally: grpc.insecure_channel = orig_insecure_channel @pytest.mark.datafiles(DATA_DIR) def test_large_directory(cli, tmpdir, datafiles): project = str(datafiles) # Number of files chosen to ensure the complete list of digests exceeds # our 1 MB gRPC message limit. I.e., test message splitting. MAX_MESSAGE_LENGTH = 1024 * 1024 NUM_FILES = MAX_MESSAGE_LENGTH // 64 + 1 large_directory_dir = os.path.join(project, "files", "large-directory") os.mkdir(large_directory_dir) for i in range(NUM_FILES): with open(os.path.join(large_directory_dir, str(i)), "w", encoding="utf-8") as f: # The files need to have different content as we want different digests. f.write(str(i)) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure bst to push to the artifact share cli.configure( { "artifacts": { "servers": [ {"url": share.repo, "push": True}, ] } } ) # Enforce 1 MB gRPC message limit with limit_grpc_message_length(MAX_MESSAGE_LENGTH): # Build and push result = cli.run(project=project, args=["build", "import-large-directory.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project, "import-large-directory.bst") == "cached" # Assert that the push was successful assert_shared(cli, share, project, "import-large-directory.bst") apache-buildstream-27ae392/tests/frontend/logging.py000066400000000000000000000220311514607367700226010ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import re import pytest from buildstream._testing import create_repo from buildstream import _yaml from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_default_logging(cli, tmpdir, datafiles): project = str(datafiles) bin_files_path = os.path.join(project, "files", "bin-files") element_path = os.path.join(project, "elements") element_name = "fetch-test-git.bst" # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # repo = create_repo("tar", str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Now try to fetch it result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() m = re.search(r"\[\d\d:\d\d:\d\d\]\[\s*\]\[.*\] SUCCESS Query cache", result.stderr) assert m is not None @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_custom_logging(cli, tmpdir, datafiles): project = str(datafiles) bin_files_path = os.path.join(project, "files", "bin-files") element_path = os.path.join(project, "elements") element_name = "fetch-test-git.bst" custom_log_format = "%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us},%{key},%{element},%{action},%{message}" user_config = {"logging": {"message-format": custom_log_format}} cli.configure(user_config) # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # repo = create_repo("tar", str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Now try to fetch it result = cli.run(project=project, args=["source", "fetch", element_name]) result.assert_success() m = re.search( r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*,SUCCESS,Query cache", result.stderr, ) assert m is not None @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_failed_build_listing(cli, datafiles): project = str(datafiles) element_names = [] for i in range(3): element_name = "testfail-{}.bst".format(i) element_path = os.path.join("elements", element_name) element = {"kind": "script", "config": {"commands": ["false"]}} _yaml.roundtrip_dump(element, os.path.join(project, element_path)) element_names.append(element_name) result = cli.run(project=project, args=["--on-error=continue", "build", *element_names]) result.assert_main_error(ErrorDomain.STREAM, None) # Check that we re-print the failure summaries only in the "Failure Summary" # section. # e.g. # # Failure Summary # testfail-0.bst: # [00:00:00][44f1b8c3][ build:testfail-0.bst ] FAILURE Running 'commands' # failure_heading_pos = re.search(r"^Failure Summary$", result.stderr, re.MULTILINE).start() pipeline_heading_pos = re.search(r"^Pipeline Summary$", result.stderr, re.MULTILINE).start() failure_summary_range = range(failure_heading_pos, pipeline_heading_pos) matches = tuple(re.finditer(r"^\s+testfail-.\.bst:$", result.stderr, re.MULTILINE)) for m in matches: assert m.start() in failure_summary_range assert m.end() in failure_summary_range assert len(matches) == 3 # each element should be matched once. # Note that if we mess up the 'element_name' of Messages, they won't be printed # with the name of the relevant element, e.g. 'testfail-1.bst'. Check that # they have the name as expected. pattern = r"\[..:..:..\] FAILURE \[.*\] testfail-.\.bst: Staged artifacts do not provide command 'sh'" assert len(re.findall(pattern, result.stderr, re.MULTILINE)) == 6 # each element should be matched twice. # This test ensures that we get the expected element name and cache key in log lines. # # * The master build log should show the element name and cache key # of the task element, i.e. the element currently being built, not # the element issuing the message. # # * In the individual task log, we expect to see the name and cache # key of the element issuing messages, since the entire log file # is contextual to the task, it makes more sense to provide the # full context of the element issuing the log in this case. # # The order and format of log lines are UI and as such might change # in which case this test needs to be adapted, the important part of # this test is only that we see task elements reported in the aggregated # master log file, and that we see message originating elements in # a task specific log file. # @pytest.mark.datafiles(os.path.join(DATA_DIR, "logging")) def test_log_line_element_names(cli, datafiles): project = str(datafiles) # First discover the cache keys, this will give us a dictionary # where we can look up the brief cache key (as displayed in the logs) # by the element name. # keys = {} result = cli.run(project=project, args=["show", "--deps", "all", "--format", "%{name}||%{key}", "logtest.bst"]) result.assert_success() lines = result.output.splitlines() for line in lines: split = line.split(sep="||") keys[split[0]] = split[1] # Run a build of the import elements, so that we can observe only the build of the logtest.bst element # at the end. # result = cli.run(project=project, args=["build", "foo.bst", "bar.bst"]) # Now run the build # result = cli.run(project=project, args=["build", "logtest.bst"]) result.assert_success() master_log = result.stderr # Now run `bst artifact log` to conveniently collect the build log so we can compare it. logfiles = os.path.join(project, "logfiles") logfile = os.path.join(project, "logfiles", "logtest.log") result = cli.run(project=project, args=["artifact", "log", "--out", logfiles, "logtest.bst"]) result.assert_success() with open(logfile, "r", encoding="utf-8") as f: task_log = f.read() ######################################################### # Parse and assert master log # ######################################################### # In the master log, we're looking for lines like this: # # [--:--:--][10dc28c5][ build:logtest.bst ] STATUS Staging bar.bst/40ff1c5a # [--:--:--][10dc28c5][ build:logtest.bst ] STATUS Staging foo.bst/e5ab75a1 # Capture (log key, element name, staged element name, staged element key) pattern = r"\[--:--:--\]\[(\S*)\]\[\s*build:(\S*)\s*] STATUS Staging\s*(\S*)/(\S*)" lines = re.findall(pattern, master_log, re.MULTILINE) # We staged 2 elements assert len(lines) == 2 # Assert that the logtest.bst element name and it's cache key is used in the master log for line in lines: log_key, log_name, staged_name, staged_key = line assert log_name == "logtest.bst" assert log_key == keys["logtest.bst"] ######################################################### # Parse and assert artifact log # ######################################################### # In the task specific build log, we're looking for lines like this: # # [--:--:--] STATUS [40ff1c5a] bar.bst: Staging bar.bst/40ff1c5a # [--:--:--] STATUS [e5ab75a1] foo.bst: Staging foo.bst/e5ab75a1 # Capture (log key, element name, staged element name, staged element key) pattern = r"\[--:--:--\] STATUS \[(\S*)\] (\S*): Staging\s*(\S*)/(\S*)" lines = re.findall(pattern, task_log, re.MULTILINE) # We staged 2 elements assert len(lines) == 2 # Assert that the originating element names and cache keys are used in # log lines when recorded to the task specific log file for line in lines: log_key, log_name, staged_name, staged_key = line assert log_name == staged_name assert log_key == staged_key assert log_key == keys[log_name] apache-buildstream-27ae392/tests/frontend/logging/000077500000000000000000000000001514607367700222315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/logging/elements/000077500000000000000000000000001514607367700240455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/logging/elements/bar.bst000066400000000000000000000000661514607367700253250ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bar apache-buildstream-27ae392/tests/frontend/logging/elements/foo.bst000066400000000000000000000000661514607367700253440ustar00rootroot00000000000000kind: import sources: - kind: local path: files/foo apache-buildstream-27ae392/tests/frontend/logging/elements/logtest.bst000066400000000000000000000000541514607367700262370ustar00rootroot00000000000000kind: logtest depends: - foo.bst - bar.bst apache-buildstream-27ae392/tests/frontend/logging/files/000077500000000000000000000000001514607367700233335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/logging/files/bar000066400000000000000000000000001514607367700240100ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/logging/files/foo000066400000000000000000000000001514607367700240270ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/logging/plugins/000077500000000000000000000000001514607367700237125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/logging/plugins/logtest.py000066400000000000000000000024561514607367700257540ustar00rootroot00000000000000from buildstream import Element class LogTest(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def configure_sandbox(self, sandbox): pass def stage(self, sandbox): # Here we stage the artifacts of dependencies individually without # using a timed activity or suppressing the logging. # # This allows us to test the logging behavior when log lines are # triggered by an element which is not the element being processed. # # * The master build log should show the element name and cache key # of the task element, i.e. the element currently being built, not # the element issuing the message. # # * In the individual task log, we expect to see the name and cache # key of the element issuing messages, since the entire log file # is contextual to the task, it makes more sense to provide the # full context of the element issuing the log in this case. # for dep in self.dependencies(): dep.stage_artifact(sandbox) def assemble(self, sandbox): return "/" # Plugin entry point def setup(): return LogTest apache-buildstream-27ae392/tests/frontend/logging/project.conf000066400000000000000000000002361514607367700245470ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements plugins: - origin: local path: plugins elements: - logtest apache-buildstream-27ae392/tests/frontend/main.py000066400000000000000000000026731514607367700221110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import click import pytest from buildstream._frontend.app import _prefix_choice_value_proc def test_prefix_choice_value_proc_full_match(): value_proc = _prefix_choice_value_proc(["foo", "bar", "baz"]) assert value_proc("foo") == "foo" assert value_proc("bar") == "bar" assert value_proc("baz") == "baz" def test_prefix_choice_value_proc_prefix_match(): value_proc = _prefix_choice_value_proc(["foo"]) assert value_proc("f") == "foo" def test_prefix_choice_value_proc_ambigous_match(): value_proc = _prefix_choice_value_proc(["bar", "baz"]) assert value_proc("bar") == "bar" assert value_proc("baz") == "baz" with pytest.raises(click.UsageError): value_proc("ba") def test_prefix_choice_value_proc_value_not_in_choices(): value_proc = _prefix_choice_value_proc(["bar", "baz"]) with pytest.raises(click.UsageError): value_proc("foo") apache-buildstream-27ae392/tests/frontend/mirror.py000066400000000000000000001314761514607367700225030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream import CoreWarnings, _yaml from buildstream.exceptions import ErrorDomain from buildstream._testing import create_repo from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils.repo.git import Git from tests.testutils.repo.tar import Tar from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(output_file): element = { "kind": "import", "sources": [ { "kind": "fetch_source", "output-text": output_file, "urls": ["foo:repo1", "bar:repo2"], "fetch-succeeds": { "FOO/repo1": True, "BAR/repo2": False, "OOF/repo1": False, "RAB/repo2": True, "OFO/repo1": False, "RBA/repo2": False, "ooF/repo1": False, "raB/repo2": False, }, } ], } return element DEFAULT_MIRROR_LIST = [ { "name": "middle-earth", "aliases": { "foo": ["OOF/"], "bar": ["RAB/"], }, }, { "name": "arrakis", "aliases": { "foo": ["OFO/"], "bar": ["RBA/"], }, }, { "name": "oz", "aliases": { "foo": ["ooF/"], "bar": ["raB/"], }, }, ] SUCCESS_MIRROR_LIST = [ { "name": "middle-earth", "aliases": { "foo": ["OOF/"], "bar": ["RAB/"], }, }, { "name": "arrakis", "aliases": { "foo": ["FOO/"], "bar": ["RBA/"], }, }, { "name": "oz", "aliases": { "foo": ["ooF/"], "bar": ["raB/"], }, }, ] FAIL_MIRROR_LIST = [ { "name": "middle-earth", "aliases": { "foo": ["pony/"], "bar": ["horzy/"], }, }, { "name": "arrakis", "aliases": { "foo": ["donkey/"], "bar": ["rabbit/"], }, }, { "name": "oz", "aliases": { "foo": ["bear/"], "bar": ["buffalo/"], }, }, ] class MirrorConfig: NO_MIRRORS = 0 SUCCESS_MIRRORS = 1 FAIL_MIRRORS = 2 DEFAULT_MIRRORS = 3 def generate_project(mirror_config=MirrorConfig.DEFAULT_MIRRORS, base_alias_succeed=False): aliases = { "foo": "FOO/", "bar": "BAR/", } if base_alias_succeed: aliases["bar"] = "RAB/" project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": aliases, "plugins": [{"origin": "local", "path": "sources", "sources": ["fetch_source"]}], } if mirror_config == MirrorConfig.SUCCESS_MIRRORS: project["mirrors"] = SUCCESS_MIRROR_LIST elif mirror_config == MirrorConfig.FAIL_MIRRORS: project["mirrors"] = FAIL_MIRROR_LIST elif mirror_config == MirrorConfig.DEFAULT_MIRRORS: project["mirrors"] = DEFAULT_MIRROR_LIST return project @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) @pytest.mark.parametrize("mirror", [("no-mirror"), ("mirror"), ("unrelated-mirror")]) def test_mirror_fetch_ref_storage(cli, tmpdir, datafiles, ref_storage, mirror): bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") upstream_repodir = os.path.join(str(tmpdir), "upstream") mirror_repodir = os.path.join(str(tmpdir), "mirror") project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) element_dir = os.path.join(project_dir, "elements") # Create repo objects of the upstream and mirror upstream_repo = create_repo("tar", upstream_repodir) upstream_repo.create(bin_files_path) mirror_repo = upstream_repo.copy(mirror_repodir) upstream_ref = upstream_repo.create(dev_files_path) element = { "kind": "import", "sources": [upstream_repo.source_config(ref=upstream_ref if ref_storage == "inline" else None)], } element_name = "test.bst" element_path = os.path.join(element_dir, element_name) full_repo = element["sources"][0]["url"] upstream_map, repo_name = os.path.split(full_repo) alias = "foo" aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo full_mirror = mirror_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) os.makedirs(element_dir) _yaml.roundtrip_dump(element, element_path) if ref_storage == "project.refs": # Manually set project.refs to avoid caching the repo prematurely project_refs = {"projects": {"test": {element_name: [{"ref": upstream_ref}]}}} project_refs_path = os.path.join(project_dir, "project.refs") _yaml.roundtrip_dump(project_refs, project_refs_path) project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": {alias: upstream_map + "/"}, "ref-storage": ref_storage, } if mirror != "no-mirror": mirror_data = [{"name": "middle-earth", "aliases": {alias: [mirror_map + "/"]}}] if mirror == "unrelated-mirror": mirror_data.insert(0, {"name": "narnia", "aliases": {"frob": ["http://www.example.com/repo"]}}) project["mirrors"] = mirror_data project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") @pytest.mark.parametrize( "project_config,user_config,expect_success", [ # User defined mirror configuration (MirrorConfig.NO_MIRRORS, MirrorConfig.SUCCESS_MIRRORS, True), # Project defined mirror configuration (MirrorConfig.SUCCESS_MIRRORS, MirrorConfig.NO_MIRRORS, True), # Both configurations active with success (MirrorConfig.FAIL_MIRRORS, MirrorConfig.SUCCESS_MIRRORS, True), # Both configurations active with failure, this ensures that # the user configuration does not regress to extending project defined # mirrors but properly overrides project defined mirrors. (MirrorConfig.SUCCESS_MIRRORS, MirrorConfig.FAIL_MIRRORS, False), ], ids=["user-config", "project-config", "override-success", "override-fail"], ) def test_mirror_fetch_multi(cli, tmpdir, project_config, user_config, expect_success): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, "project.conf") project = generate_project(project_config) _yaml.roundtrip_dump(project, project_file) if user_config == MirrorConfig.SUCCESS_MIRRORS: cli.configure({"projects": {"test": {"mirrors": SUCCESS_MIRROR_LIST}}}) elif user_config == MirrorConfig.FAIL_MIRRORS: cli.configure({"projects": {"test": {"mirrors": FAIL_MIRROR_LIST}}}) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) if expect_success: result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents else: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") @pytest.mark.parametrize( "project_config,user_config,alias_success,expect_success,source,expect_missing_aliases", [ # # Test "alias" fetch source policy (aliases only) # # Test that we fail to fetch from the primary alias even if the user config defines a mirror (MirrorConfig.NO_MIRRORS, MirrorConfig.SUCCESS_MIRRORS, False, False, "aliases", False), # Test that we fail to fetch from the primary alias even if the project config defines a mirror (MirrorConfig.SUCCESS_MIRRORS, MirrorConfig.NO_MIRRORS, False, False, "aliases", False), # Test that we succeed to fetch from the primary alias even if the user config defines a failing mirror (MirrorConfig.FAIL_MIRRORS, MirrorConfig.FAIL_MIRRORS, True, True, "aliases", False), # # Test "mirrors" fetch source policy (mirrors only, no base aliases) # # Test that we fail to fetch from primary alias even if it is the only one configured to succeed (MirrorConfig.FAIL_MIRRORS, MirrorConfig.FAIL_MIRRORS, True, False, "mirrors", False), # Test that we succeed to fetch from mirrors when primary alias is set to succeed # (doesn't prove that primary alias is not consulted, but tests that we indeed consult # mirrors when configued in mirror mode) (MirrorConfig.SUCCESS_MIRRORS, MirrorConfig.NO_MIRRORS, True, True, "mirrors", False), (MirrorConfig.FAIL_MIRRORS, MirrorConfig.SUCCESS_MIRRORS, True, True, "mirrors", False), # # Test preflight errors when there are missing mirror alias targets # (MirrorConfig.NO_MIRRORS, MirrorConfig.NO_MIRRORS, True, False, "mirrors", True), # # Test "user" fetch source policy (only mirrors defined in user configuration) # # Test that we fail to fetch even if the alias is good and the project defined mirrors are good (MirrorConfig.SUCCESS_MIRRORS, MirrorConfig.FAIL_MIRRORS, True, False, "user", False), # Test that we succeed to fetch when alias is bad and project mirrors are bad # (this doesn't prove that project aliases and mirrors are not consulted, but here for completeness) (MirrorConfig.FAIL_MIRRORS, MirrorConfig.SUCCESS_MIRRORS, False, True, "user", False), ], ids=[ "aliases-fail-user-config", "aliases-fail-project-config", "aliases-success-bad-mirrors", "mirrors-fail-bad-mirrors", "mirrors-success-project-config", "mirrors-success-user-config", "mirrors-fail-inactive", "user-fail", "user-succees", ], ) def test_mirror_fetch_source( cli, tmpdir, project_config, user_config, alias_success, expect_success, source, expect_missing_aliases ): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, "project.conf") project = generate_project(project_config, alias_success) _yaml.roundtrip_dump(project, project_file) # Configure the fetch source cli.configure({"fetch": {"source": source}}) if user_config == MirrorConfig.SUCCESS_MIRRORS: cli.configure({"projects": {"test": {"mirrors": SUCCESS_MIRROR_LIST}}}) elif user_config == MirrorConfig.FAIL_MIRRORS: cli.configure({"projects": {"test": {"mirrors": FAIL_MIRROR_LIST}}}) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) if expect_success: result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents else: # # Special case check this failure mode # if expect_missing_aliases: result.assert_main_error(ErrorDomain.SOURCE, "missing-source-alias-target") else: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") def test_mirror_fetch_default_cmdline(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name]) result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() print(contents) # Success if fetching from arrakis' mirror happened before middle-earth's arrakis_str = "OFO/repo1" arrakis_pos = contents.find(arrakis_str) assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str) me_str = "OOF/repo1" me_pos = contents.find(me_str) assert me_pos != -1, "'{}' wasn't found".format(me_str) assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") def test_mirror_fetch_default_userconfig(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) userconfig = {"projects": {"test": {"default-mirror": "oz"}}} cli.configure(userconfig) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() print(contents) # Success if fetching from Oz' mirror happened before middle-earth's oz_str = "ooF/repo1" oz_pos = contents.find(oz_str) assert oz_pos != -1, "'{}' wasn't found".format(oz_str) me_str = "OOF/repo1" me_pos = contents.find(me_str) assert me_pos != -1, "'{}' wasn't found".format(me_str) assert oz_pos < me_pos, "'{}' wasn't found before '{}'".format(oz_str, me_str) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, "project.conf") project = generate_project() _yaml.roundtrip_dump(project, project_file) userconfig = {"projects": {"test": {"default-mirror": "oz"}}} cli.configure(userconfig) result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name]) result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() print(contents) # Success if fetching from arrakis' mirror happened before middle-earth's arrakis_str = "OFO/repo1" arrakis_pos = contents.find(arrakis_str) assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str) me_str = "OOF/repo1" me_pos = contents.find(me_str) assert me_pos != -1, "'{}' wasn't found".format(me_str) assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles): # Test that it behaves as expected with submodules, both defined in config # and discovered when fetching. foo_file = os.path.join(str(datafiles), "files", "foo") bar_file = os.path.join(str(datafiles), "files", "bar") bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") mirror_dir = os.path.join(str(datafiles), "mirror") defined_subrepo = Git(str(tmpdir), "defined_subrepo") defined_subrepo.create(bin_files_path) defined_subrepo.copy(mirror_dir) defined_subrepo.add_file(foo_file) found_subrepo = Git(str(tmpdir), "found_subrepo") found_subrepo.create(dev_files_path) main_repo = Git(str(tmpdir)) main_mirror_ref = main_repo.create(bin_files_path) main_repo.add_submodule("defined", "file://" + defined_subrepo.repo) main_repo.add_submodule("found", "file://" + found_subrepo.repo) main_mirror = main_repo.copy(mirror_dir) main_repo.add_file(bar_file) project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir) element = {"kind": "import", "sources": [main_repo.source_config(ref=main_mirror_ref)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) # Alias the main repo full_repo = element["sources"][0]["url"] _, repo_name = os.path.split(full_repo) alias = "foo" aliased_repo = alias + ":" + repo_name element["sources"][0]["url"] = aliased_repo # Hide the found subrepo del element["sources"][0]["submodules"]["found"] # Alias the defined subrepo subrepo = element["sources"][0]["submodules"]["defined"]["url"] _, repo_name = os.path.split(subrepo) aliased_repo = alias + ":" + repo_name element["sources"][0]["submodules"]["defined"]["url"] = aliased_repo _yaml.roundtrip_dump(element, element_path) full_mirror = main_mirror.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": {alias: "http://www.example.com/"}, "plugins": [ { "origin": "pip", "package-name": "sample-plugins", "sources": ["git"], } ], "mirrors": [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, }, ], } project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles): # Main repo has no mirror or alias. # One submodule is overridden to use a mirror. # There is another submodules not overriden. # Upstream for overriden submodule is down. # # We expect: # - overriden submodule is fetched from mirror. # - other submodule is fetched. bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") upstream_bin_repodir = os.path.join(str(tmpdir), "bin-upstream") mirror_bin_repodir = os.path.join(str(tmpdir), "bin-mirror") upstream_bin_repo = Git(upstream_bin_repodir) upstream_bin_repo.create(bin_files_path) mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir) dev_repodir = os.path.join(str(tmpdir), "dev-upstream") dev_repo = Git(dev_repodir) dev_repo.create(dev_files_path) main_files = os.path.join(str(tmpdir), "main-files") os.makedirs(main_files) with open(os.path.join(main_files, "README"), "w", encoding="utf-8") as f: f.write("TEST\n") main_repodir = os.path.join(str(tmpdir), "main-upstream") main_repo = Git(main_repodir) main_repo.create(main_files) upstream_url = "file://{}".format(upstream_bin_repo.repo) main_repo.add_submodule("bin", url=upstream_url) main_repo.add_submodule("dev", url="file://{}".format(dev_repo.repo)) # Unlist 'dev'. del main_repo.submodules["dev"] main_ref = main_repo.latest_commit() upstream_map, repo_name = os.path.split(upstream_url) alias = "foo" aliased_repo = "{}:{}".format(alias, repo_name) main_repo.submodules["bin"]["url"] = aliased_repo full_mirror = mirror_bin_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) element_dir = os.path.join(project_dir, "elements") element = {"kind": "import", "sources": [main_repo.source_config_extra(ref=main_ref, checkout_submodules=True)]} element_name = "test.bst" element_path = os.path.join(element_dir, element_name) os.makedirs(element_dir) _yaml.roundtrip_dump(element, element_path) project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": {alias: upstream_map + "/"}, "plugins": [ { "origin": "pip", "package-name": "sample-plugins", "sources": ["git"], } ], "mirrors": [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, } ], } project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. os.rename(upstream_bin_repo.repo, "{}.bak".format(upstream_bin_repo.repo)) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() result = cli.run(project=project_dir, args=["build", element_name]) result.assert_success() checkout = os.path.join(str(tmpdir), "checkout") result = cli.run(project=project_dir, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert os.path.exists(os.path.join(checkout, "bin", "bin", "hello")) assert os.path.exists(os.path.join(checkout, "dev", "include", "pony.h")) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles): # Main repo has mirror. But does not list submodules. # # We expect: # - we will fetch submodules anyway bin_files_path = os.path.join(str(datafiles), "files", "bin-files", "usr") dev_files_path = os.path.join(str(datafiles), "files", "dev-files", "usr") bin_repodir = os.path.join(str(tmpdir), "bin-repo") bin_repo = Git(bin_repodir) bin_repo.create(bin_files_path) dev_repodir = os.path.join(str(tmpdir), "dev-repo") dev_repo = Git(dev_repodir) dev_repo.create(dev_files_path) main_files = os.path.join(str(tmpdir), "main-files") os.makedirs(main_files) with open(os.path.join(main_files, "README"), "w", encoding="utf-8") as f: f.write("TEST\n") upstream_main_repodir = os.path.join(str(tmpdir), "main-upstream") upstream_main_repo = Git(upstream_main_repodir) upstream_main_repo.create(main_files) upstream_main_repo.add_submodule("bin", url="file://{}".format(bin_repo.repo)) upstream_main_repo.add_submodule("dev", url="file://{}".format(dev_repo.repo)) # Unlist submodules. del upstream_main_repo.submodules["bin"] del upstream_main_repo.submodules["dev"] upstream_main_ref = upstream_main_repo.latest_commit() mirror_main_repodir = os.path.join(str(tmpdir), "main-mirror") mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir) upstream_url = mirror_main_repo.source_config()["url"] upstream_map, repo_name = os.path.split(upstream_url) alias = "foo" aliased_repo = "{}:{}".format(alias, repo_name) full_mirror = mirror_main_repo.source_config()["url"] mirror_map, _ = os.path.split(full_mirror) project_dir = os.path.join(str(tmpdir), "project") os.makedirs(project_dir) element_dir = os.path.join(project_dir, "elements") element = { "kind": "import", "sources": [upstream_main_repo.source_config_extra(ref=upstream_main_ref, checkout_submodules=True)], } element["sources"][0]["url"] = aliased_repo element_name = "test.bst" element_path = os.path.join(element_dir, element_name) os.makedirs(element_dir) _yaml.roundtrip_dump(element, element_path) project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": {alias: upstream_map + "/"}, "plugins": [ { "origin": "pip", "package-name": "sample-plugins", "sources": ["git"], } ], "mirrors": [ { "name": "middle-earth", "aliases": { alias: [mirror_map + "/"], }, } ], } project_file = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(project, project_file) # Now make the upstream unavailable. os.rename(upstream_main_repo.repo, "{}.bak".format(upstream_main_repo.repo)) result = cli.run(project=project_dir, args=["source", "fetch", element_name]) result.assert_success() result = cli.run(project=project_dir, args=["build", element_name]) result.assert_success() checkout = os.path.join(str(tmpdir), "checkout") result = cli.run(project=project_dir, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert os.path.exists(os.path.join(checkout, "bin", "bin", "hello")) assert os.path.exists(os.path.join(checkout, "dev", "include", "pony.h")) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") def test_mirror_expand_project_and_toplevel_root(cli, tmpdir): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) project_file = os.path.join(project_dir, "project.conf") project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": { "foo": "FOO/", "bar": "BAR/", }, "mirrors": [ { "name": "middle-earth", "aliases": { "foo": ["OOF/"], "bar": ["RAB/"], }, }, { "name": "arrakis", "aliases": { "foo": ["%{project-root}/OFO/"], "bar": ["%{project-root}/RBA/"], }, }, { "name": "oz", "aliases": { "foo": ["ooF/"], "bar": ["raB/"], }, }, ], "plugins": [{"origin": "local", "path": "sources", "sources": ["fetch_source"]}], } _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name]) result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() print(contents) foo_str = os.path.join(project_dir, "OFO/repo1") bar_str = os.path.join(project_dir, "RBA/repo2") # Success if the expanded %{project-root} is found assert foo_str in contents assert bar_str in contents # Test a simple SourceMirror implementation which reads # plugin configuration and behaves in the same way as default # mirrors but using data in the plugin configuration instead. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") @pytest.mark.parametrize("origin", [("local"), ("junction"), ("pip")]) def test_source_mirror_plugin(cli, tmpdir, origin): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) element_name = "test.bst" element_path = os.path.join(element_dir, element_name) element = generate_element(output_file) _yaml.roundtrip_dump(element, element_path) def source_mirror_plugin_origin(): if origin == "local": return {"origin": "local", "path": "sourcemirrors", "source-mirrors": ["mirror"]} elif origin == "pip": return { "origin": "pip", "package-name": "sample-plugins>=1.2", "source-mirrors": ["mirror"], } elif origin == "junction": # For junction loading, just copy in the sample-plugins into a subdir and # create a local junction sample_plugins_dir = os.path.join(TOP_DIR, "..", "plugins", "sample-plugins") sample_plugins_copy_dir = os.path.join(project_dir, "sample-plugins-copy") junction_file = os.path.join(element_dir, "sample-plugins.bst") shutil.copytree(sample_plugins_dir, sample_plugins_copy_dir) _yaml.roundtrip_dump( {"kind": "junction", "sources": [{"kind": "local", "path": "sample-plugins-copy"}]}, junction_file ) return { "origin": "junction", "junction": "sample-plugins.bst", "source-mirrors": ["mirror"], } else: assert False project_file = os.path.join(project_dir, "project.conf") project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": { "foo": "FOO/", "bar": "BAR/", }, "mirrors": [ { "name": "middle-earth", "kind": "mirror", "config": { "aliases": { "foo": ["OOF/"], "bar": ["RAB/"], }, }, }, { "name": "arrakis", "kind": "mirror", "config": { "aliases": { "foo": ["%{project-root}/OFO/"], "bar": ["%{project-root}/RBA/"], }, }, }, { "name": "oz", "kind": "mirror", "config": { "aliases": { "foo": ["ooF/"], "bar": ["raB/"], }, }, }, ], "plugins": [ {"origin": "local", "path": "sources", "sources": ["fetch_source"]}, source_mirror_plugin_origin(), ], } _yaml.roundtrip_dump(project, project_file) result = cli.run(project=project_dir, args=["--default-mirror", "arrakis", "source", "fetch", element_name]) result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() print(contents) foo_str = os.path.join(project_dir, "OFO/repo1") bar_str = os.path.join(project_dir, "RBA/repo2") # Success if the expanded %{project-root} is found assert foo_str in contents assert bar_str in contents # Test subproject alias mapping. As there are quite a few corner cases and interactions with other # features that need to be tested, this test relies heavily on parametrization. Here is what each # parameter means: # * subproject_mirrors: whether the subproject defines a (failing) mirror # * unaliased_sources: whether the subproject has unaliased sources # * disallow_subproject_uris: define disallow-subproject-uris in the parent project # * fetch_source: whether to fetch from aliases or mirrors # * alias_override: which aliases to override ("global" means use "map-aliases") # * alias_mapping: how to map aliases to the overrides # * source_mirror: whether to use a custom source mirror plugin @pytest.mark.datafiles(DATA_DIR) @pytest.mark.usefixtures("datafiles") @pytest.mark.parametrize("subproject_mirrors", [True, False]) @pytest.mark.parametrize("unaliased_sources", [True, False]) @pytest.mark.parametrize("disallow_subproject_uris", [True, False]) @pytest.mark.parametrize("fetch_source", ["aliases", "mirrors"]) @pytest.mark.parametrize("alias_override", [["foo"], ["foo", "bar"], "global"]) @pytest.mark.parametrize("alias_mapping", ["identity", "project-prefix", "invalid"]) @pytest.mark.parametrize("source_mirror", [True, False]) def test_mirror_subproject_aliases( cli, tmpdir, subproject_mirrors, unaliased_sources, disallow_subproject_uris, fetch_source, alias_override, alias_mapping, source_mirror, ): if alias_override == "global": if alias_mapping == "invalid": # we can't have an invalid mapping using a predefined option pytest.skip() elif alias_mapping == "project-prefix": # project-prefix alias mapping not yet implemented pytest.xfail() output_file = os.path.join(str(tmpdir), "output.txt") project_dir = tmpdir element_dir = project_dir / "elements" os.makedirs(element_dir, exist_ok=True) subproject_dir = project_dir / "subproject" subproject_element_dir = subproject_dir / "elements" os.makedirs(subproject_element_dir, exist_ok=True) subproject_bar_alias_succeed = ( fetch_source == "aliases" and not disallow_subproject_uris and alias_override == ["foo"] ) subproject = { "name": "test-subproject", "min-version": "2.0", "element-path": "elements", "aliases": { "foo": "OOF/", "bar": "RAB/" if subproject_bar_alias_succeed else "BAR/", }, "plugins": [ {"origin": "local", "path": "sources", "sources": ["fetch_source"]}, ], } if subproject_mirrors: subproject["mirrors"] = FAIL_MIRROR_LIST _yaml.roundtrip_dump(subproject, str(subproject_dir / "project.conf")) element_name = "test.bst" element = generate_element(output_file) if unaliased_sources: element["sources"][0]["urls"] = ["foo:repo1", "RAB/repo2"] _yaml.roundtrip_dump(element, str(subproject_element_dir / element_name)) # copy the source plugin to the subproject shutil.copytree(project_dir / "sources", subproject_dir / "sources") if alias_mapping == "identity": def map_alias(x): return x elif alias_mapping == "project-prefix": def map_alias(x): return subproject["name"] + "/" + x else: def map_alias(x): return "invalid-" + x if alias_mapping != "invalid": map_alias_valid = map_alias else: def map_alias_valid(x): return x project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": { map_alias_valid("foo"): "FOO/", map_alias_valid("bar"): "RAB/" if fetch_source == "aliases" else "BAR/", }, "plugins": [{"origin": "local", "path": "sources", "sources": ["fetch_source"]}], # Copy of SUCCESS_MIRROR_LIST from above "mirrors": [ { "name": "middle-earth", "aliases": { map_alias_valid("foo"): ["OOF/"], map_alias_valid("bar"): ["RAB/"], }, }, { "name": "arrakis", "aliases": { map_alias_valid("foo"): ["FOO/"], map_alias_valid("bar"): ["RBA/"], }, }, { "name": "oz", "aliases": { map_alias_valid("foo"): ["ooF/"], map_alias_valid("bar"): ["raB/"], }, }, ], } if source_mirror: project["plugins"].append({"origin": "local", "path": "sourcemirrors", "source-mirrors": ["mirror"]}) mirrors = [] for mirror in project["mirrors"]: mirrors.append({"name": mirror["name"], "kind": "mirror", "config": {"aliases": mirror["aliases"]}}) project["mirrors"] = mirrors if disallow_subproject_uris: project["junctions"] = {"disallow-subproject-uris": "true"} _yaml.roundtrip_dump(project, str(project_dir / "project.conf")) junction_name = "subproject.bst" junction = { "kind": "junction", "sources": [ { "kind": "local", "path": "subproject", } ], } if alias_override == "global": junction["config"] = {"map-aliases": alias_mapping} else: junction["config"] = {"aliases": {alias: map_alias(alias) for alias in alias_override}} _yaml.roundtrip_dump(junction, str(element_dir / junction_name)) userconfig = {"fetch": {"source": fetch_source}} cli.configure(userconfig) result = cli.run(project=project_dir, args=["source", "fetch", "{}:{}".format(junction_name, element_name)]) if alias_mapping == "invalid": # Mapped alias does not exist in the parent project result.assert_main_error(ErrorDomain.SOURCE, "invalid-source-alias") elif disallow_subproject_uris and unaliased_sources: # Subproject defines unaliased source and the parent project disallows subproject URIs result.assert_main_error(ErrorDomain.PLUGIN, CoreWarnings.UNALIASED_URL) elif disallow_subproject_uris and alias_override == ["foo"]: # No alias mapping defined for `bar` and the parent project disallows subproject URIs result.assert_main_error(ErrorDomain.SOURCE, "missing-alias-mapping") elif fetch_source == "mirrors" and not unaliased_sources and alias_override == ["foo"]: # Mirror required and no alias mapping defined for `bar` if not subproject_mirrors: # and the subproject has no mirror configured result.assert_main_error(ErrorDomain.SOURCE, "missing-source-alias-target") else: # and the subproject has a failing mirror configured result.assert_task_error(ErrorDomain.SOURCE, None) with open(output_file, encoding="utf-8") as f: contents = f.read() assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents assert "Fetch bar:repo2 failed from rabbit/repo2" in contents assert "Fetch bar:repo2 failed from buffalo/repo2" in contents else: result.assert_success() with open(output_file, encoding="utf-8") as f: contents = f.read() assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents if unaliased_sources: assert "Fetch RAB/repo2 succeeded from RAB/repo2" in contents else: assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents # Test the behavior of loading a SourceMirror plugin across a junction, # when the cross junction SourceMirror object has a mirror. # # Check what happens when the mirror does not need to be exercized (success) # # Check what happens when the mirror needs to be exercised in order to obtain # the mirror plugin itself (failure) and check the failure mode. # # @pytest.mark.parametrize("fetch_source", [("all"), ("mirrors")], ids=["normal", "circular"]) def test_source_mirror_circular_junction(cli, tmpdir, fetch_source): project_dir = str(tmpdir) element_dir = os.path.join(project_dir, "elements") os.makedirs(element_dir, exist_ok=True) cli.configure({"fetch": {"source": fetch_source}}) # Generate a 2 tar repos with the sample plugins # sample_plugins_dir = os.path.join(TOP_DIR, "..", "plugins", "sample-plugins") base_sample_plugins_repodir = os.path.join(str(tmpdir), "base_sample_plugins") base_sample_plugins_repo = Tar(base_sample_plugins_repodir) base_sample_plugins_ref = base_sample_plugins_repo.create(sample_plugins_dir) mirror_sample_plugins_repodir = os.path.join(str(tmpdir), "mirror_sample_plugins") mirror_sample_plugins_repo = Tar(mirror_sample_plugins_repodir) # Don't expect determinism from python tar, just copy over the Tar repo file # and we need to use the same ref for both. shutil.copyfile( os.path.join(base_sample_plugins_repo.repo, "file.tar.gz"), os.path.join(mirror_sample_plugins_repo.repo, "file.tar.gz"), ) # Generate junction for sample plugins # sample_plugins_junction = { "kind": "junction", "sources": [ { "kind": "tar", "url": "samplemirror:file.tar.gz", "ref": base_sample_plugins_ref, } ], } element_path = os.path.join(element_dir, "sample-plugins.bst") _yaml.roundtrip_dump(sample_plugins_junction, element_path) # Generate project.conf # project_file = os.path.join(project_dir, "project.conf") project = { "name": "test", "min-version": "2.0", "element-path": "elements", "aliases": { "samplemirror": "file://" + base_sample_plugins_repo.repo + "/", }, "mirrors": [ { "name": "alternative", "kind": "mirror", "config": { "aliases": { "samplemirror": ["file://" + mirror_sample_plugins_repo.repo + "/"], }, }, }, ], "plugins": [ {"origin": "junction", "junction": "sample-plugins.bst", "source-mirrors": ["mirror"]}, ], } _yaml.roundtrip_dump(project, project_file) # Make a silly element element = {"kind": "import", "sources": [{"kind": "local", "path": "project.conf"}]} element_path = os.path.join(element_dir, "test.bst") _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project_dir, args=["show", "test.bst"]) if fetch_source == "all": result.assert_success() elif fetch_source == "mirrors": # # This error looks like this: # # Error loading project: tar source at sample-plugins.bst [line 3 column 2]: No fetch URI found for alias 'samplemirror' # # Check fetch controls in your user configuration # # This is not 100% ideal, as we could theoretically have Source.mark_download_url() detect # the case that we are currently instantiating the specific SourceMirror plugin required # to resolve the URL needed to obtain the same said SourceMirror plugin, and report # something about this being a circular dependency error. # # However, this would be fairly complex to reason about in the code, especially considering # the source alias redirects, and the possibility that a subproject's source mirror is being # redirected to a parent project's aliases and corresponding mirrors. # result.assert_main_error(ErrorDomain.SOURCE, "missing-source-alias-target") apache-buildstream-27ae392/tests/frontend/order.py000066400000000000000000000117061514607367700222750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import create_repo from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) # create_element() # # Args: # project (str): The project directory where testing is happening # name (str): The element name to create # dependencies (list): The list of dependencies to dump into YAML format # # Returns: # (Repo): The corresponding git repository created for the element def create_element(project, name, dependencies): dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") repo = create_repo("tar", project, "{}-repo".format(name)) ref = repo.create(dev_files_path) element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies} _yaml.roundtrip_dump(element, os.path.join(element_path, name)) return repo # This tests a variety of scenarios and checks that the order in # which things are processed remains stable. # # This is especially important in order to ensure that our # depth sorting and optimization of which elements should be # processed first is doing it's job right, and that we are # promoting elements to the build queue as soon as possible # # Parameters: # targets (target elements): The targets to invoke bst with # template (dict): The project template dictionary, for create_element() # expected (list): A list of element names in the expected order # @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize( "target,template,expected_stage_order,expected_build_order", [ # First simple test ( "3.bst", {"0.bst": ["1.bst"], "1.bst": [], "2.bst": ["0.bst"], "3.bst": ["0.bst", "1.bst", "2.bst"]}, ["1.bst", "0.bst", "2.bst", "3.bst"], ["1.bst", "0.bst", "2.bst", "3.bst"], ), # A more complicated test with build of build dependencies ( "target.bst", { "a.bst": [], "base.bst": [], "timezones.bst": [], "middleware.bst": [{"filename": "base.bst", "type": "build"}], "app.bst": [{"filename": "middleware.bst", "type": "build"}], "target.bst": ["a.bst", "base.bst", "middleware.bst", "app.bst", "timezones.bst"], }, ["a.bst", "base.bst", "middleware.bst", "app.bst", "timezones.bst", "target.bst"], ["base.bst", "middleware.bst", "a.bst", "app.bst", "timezones.bst", "target.bst"], ), ], ids=["simple", "complex"], ) @pytest.mark.parametrize("operation", [("show"), ("fetch"), ("build")]) def test_order(cli, datafiles, operation, target, template, expected_stage_order, expected_build_order): project = str(datafiles) # Configure to only allow one fetcher at a time, make it easy to # determine what is being planned in what order. cli.configure({"scheduler": {"fetchers": 1, "builders": 1}}) # Build the project from the template, make import elements # all with the same repo # for element, dependencies in template.items(): create_element(project, element, dependencies) # Run test and collect results if operation == "show": result = cli.run(args=["show", "--deps", "all", "--format", "%{name}", target], project=project, silent=True) result.assert_success() results = result.output.splitlines() else: if operation == "fetch": result = cli.run(args=["source", "fetch", "--deps", "all", target], project=project, silent=True) else: result = cli.run(args=["build", target], project=project, silent=True) result.assert_success() results = result.get_start_order(operation) # When running `bst build`, the elements are depth sorted for optimal processing if operation == "build": expected = expected_build_order else: # Otherwise, we get the usual deterministic staging order expected = expected_stage_order # Assert the order print("Expected order: {}".format(expected)) print("Observed result order: {}".format(results)) assert results == expected apache-buildstream-27ae392/tests/frontend/overlaps.py000066400000000000000000000213371514607367700230160ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing.runcli import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream import _yaml from buildstream import CoreWarnings, OverlapAction from tests.testutils import generate_junction # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "overlaps") def gen_project(project_dir, fatal_warnings, *, project_name="test", use_plugin=False): template = {"name": project_name, "min-version": "2.0"} template["fatal-warnings"] = [CoreWarnings.OVERLAPS, CoreWarnings.UNSTAGED_FILES] if fatal_warnings else [] if use_plugin: template["plugins"] = [{"origin": "local", "path": "plugins", "elements": ["overlap"]}] projectfile = os.path.join(project_dir, "project.conf") _yaml.roundtrip_dump(template, projectfile) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("error", [False, True], ids=["warning", "error"]) def test_unstaged_files(cli, datafiles, error): project_dir = str(datafiles) gen_project(project_dir, error) result = cli.run(project=project_dir, silent=True, args=["build", "unstaged.bst"]) if error: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.UNSTAGED_FILES) else: result.assert_success() assert "WARNING [unstaged-files]" in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("error", [False, True], ids=["warning", "error"]) def test_overlaps(cli, datafiles, error): project_dir = str(datafiles) gen_project(project_dir, error) result = cli.run(project=project_dir, silent=True, args=["build", "collect.bst"]) if error: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() assert "WARNING [overlaps]" in result.stderr # # When the overlap is whitelisted, there is no warning or error. # # Still test this in fatal/nonfatal warning modes # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("error", [False, True], ids=["warning", "error"]) def test_overlaps_whitelisted(cli, datafiles, error): project_dir = str(datafiles) gen_project(project_dir, error) result = cli.run(project=project_dir, silent=True, args=["build", "collect-whitelisted.bst"]) result.assert_success() assert "WARNING [overlaps]" not in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("error", [False, True], ids=["warning", "error"]) def test_overlaps_whitelist_on_overlapper(cli, datafiles, error): # Tests that the overlapping element is responsible for whitelisting, # i.e. that if A overlaps B overlaps C, and the B->C overlap is permitted, # it'll still fail because A doesn't permit overlaps. project_dir = str(datafiles) gen_project(project_dir, error) result = cli.run(project=project_dir, silent=True, args=["build", "collect-partially-whitelisted.bst"]) if error: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() assert "WARNING [overlaps]" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_overlaps_whitelist_undefined_variable(cli, datafiles): project_dir = str(datafiles) gen_project(project_dir, False) result = cli.run(project=project_dir, silent=True, args=["show", "whitelist-undefined.bst"]) # Assert that we get the expected undefined variable error, # and that it has the provenance we expect from whitelist-undefined.bst # result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE) assert "whitelist-undefined.bst [line 13 column 6]" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_overlaps_script(cli, datafiles): # Test overlaps with script element to test # Element.stage_dependency_artifacts() with Scope.RUN project_dir = str(datafiles) gen_project(project_dir, False) result = cli.run(project=project_dir, silent=True, args=["build", "script.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("project_policy", [("fail"), ("warn")]) @pytest.mark.parametrize("subproject_policy", [("fail"), ("warn")]) def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_policy): project_dir = str(datafiles) subproject_dir = os.path.join(project_dir, "sub-project") junction_path = os.path.join(project_dir, "sub-project.bst") gen_project(project_dir, bool(project_policy == "fail"), project_name="test") gen_project(subproject_dir, bool(subproject_policy == "fail"), project_name="subtest") generate_junction(tmpdir, subproject_dir, junction_path) # Here we have a dependency chain where the project element # always overlaps with the subproject element. # # Test that overlap error vs warning policy for this overlap # is always controlled by the project and not the subproject. # result = cli.run(project=project_dir, silent=True, args=["build", "sub-collect.bst"]) if project_policy == "fail": result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() assert "WARNING [overlaps]" in result.stderr # Test unstaged-files warnings when staging to an alternative location than "/" # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("error", [False, True], ids=["warning", "error"]) def test_unstaged_files_relocated(cli, datafiles, error): project_dir = str(datafiles) gen_project(project_dir, error, use_plugin=True) result = cli.run(project=project_dir, silent=True, args=["build", "relocated-unstaged.bst"]) if error: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.UNSTAGED_FILES) else: result.assert_success() assert "WARNING [unstaged-files]" in result.stderr # Test overlap warnings when staging to an alternative location than "/" # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("error", [False, True], ids=["warning", "error"]) def test_overlaps_relocated(cli, datafiles, error): project_dir = str(datafiles) gen_project(project_dir, error, use_plugin=True) result = cli.run(project=project_dir, silent=True, args=["build", "relocated.bst"]) if error: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() assert "WARNING [overlaps]" in result.stderr # Test overlap warnings as a result of multiple calls to Element.stage_dependency_artifacts() # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target,action,error", [ ("multistage-overlap-ignore.bst", OverlapAction.IGNORE, False), ("multistage-overlap.bst", OverlapAction.WARNING, False), ("multistage-overlap.bst", OverlapAction.WARNING, True), ("multistage-overlap-error.bst", OverlapAction.ERROR, True), ], ids=["ignore", "warn-warning", "warn-error", "error"], ) def test_overlaps_multistage(cli, datafiles, target, action, error): project_dir = str(datafiles) gen_project(project_dir, error, use_plugin=True) result = cli.run(project=project_dir, silent=True, args=["build", target]) if action == OverlapAction.WARNING: if error: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() assert "WARNING [overlaps]" in result.stderr elif action == OverlapAction.IGNORE: result.assert_success() assert "WARNING [overlaps]" not in result.stderr elif action == OverlapAction.ERROR: result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.ELEMENT, "overlaps") apache-buildstream-27ae392/tests/frontend/overlaps/000077500000000000000000000000001514607367700224365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/a-whitelisted.bst000066400000000000000000000002421514607367700257110ustar00rootroot00000000000000kind: import config: source: / target: / depends: - b-whitelisted.bst sources: - kind: local path: "a" public: bst: overlap-whitelist: - "/file*" apache-buildstream-27ae392/tests/frontend/overlaps/a.bst000066400000000000000000000001411514607367700233640ustar00rootroot00000000000000kind: import config: source: / target: / depends: - b.bst sources: - kind: local path: "a" apache-buildstream-27ae392/tests/frontend/overlaps/a/000077500000000000000000000000001514607367700226565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/a/file1000066400000000000000000000000041514607367700235730ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/frontend/overlaps/a/file2000066400000000000000000000000041514607367700235740ustar00rootroot00000000000000bar apache-buildstream-27ae392/tests/frontend/overlaps/b-whitelisted.bst000066400000000000000000000002761514607367700257210ustar00rootroot00000000000000kind: import config: source: / target: / depends: - c.bst sources: - kind: local path: "b" variables: FILE: /file public: bst: overlap-whitelist: - /file2 - "%{FILE}3" apache-buildstream-27ae392/tests/frontend/overlaps/b.bst000066400000000000000000000001411514607367700233650ustar00rootroot00000000000000kind: import config: source: / target: / depends: - c.bst sources: - kind: local path: "b" apache-buildstream-27ae392/tests/frontend/overlaps/b/000077500000000000000000000000001514607367700226575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/b/file2000066400000000000000000000000041514607367700235750ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/frontend/overlaps/b/file3000066400000000000000000000000041514607367700235760ustar00rootroot00000000000000bar apache-buildstream-27ae392/tests/frontend/overlaps/c-whitelisted.bst000066400000000000000000000002051514607367700257120ustar00rootroot00000000000000kind: import config: source: / target: / sources: - kind: local path: "c" public: bst: overlap-whitelist: - "/file*" apache-buildstream-27ae392/tests/frontend/overlaps/c.bst000066400000000000000000000001201514607367700233630ustar00rootroot00000000000000kind: import config: source: / target: / sources: - kind: local path: "c" apache-buildstream-27ae392/tests/frontend/overlaps/c/000077500000000000000000000000001514607367700226605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/c/file1000066400000000000000000000000041514607367700235750ustar00rootroot00000000000000baz apache-buildstream-27ae392/tests/frontend/overlaps/c/file2000066400000000000000000000000041514607367700235760ustar00rootroot00000000000000bar apache-buildstream-27ae392/tests/frontend/overlaps/c/file3000066400000000000000000000000041514607367700235770ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/frontend/overlaps/collect-partially-whitelisted.bst000066400000000000000000000002041514607367700311130ustar00rootroot00000000000000kind: compose depends: - filename: a.bst type: build - filename: b-whitelisted.bst type: build - filename: c.bst type: build apache-buildstream-27ae392/tests/frontend/overlaps/collect-whitelisted.bst000066400000000000000000000002201514607367700271120ustar00rootroot00000000000000kind: compose depends: - filename: a-whitelisted.bst type: build - filename: b-whitelisted.bst type: build - filename: c.bst type: build apache-buildstream-27ae392/tests/frontend/overlaps/collect.bst000066400000000000000000000001701514607367700245730ustar00rootroot00000000000000kind: compose depends: - filename: a.bst type: build - filename: b.bst type: build - filename: c.bst type: build apache-buildstream-27ae392/tests/frontend/overlaps/directory-file.bst000066400000000000000000000001731514607367700260720ustar00rootroot00000000000000kind: import config: source: / target: / depends: - with-directory.bst sources: - kind: local path: "directory-file" apache-buildstream-27ae392/tests/frontend/overlaps/directory-file/000077500000000000000000000000001514607367700253575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/directory-file/directory-file000066400000000000000000000000051514607367700302160ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/frontend/overlaps/multistage-overlap-error.bst000066400000000000000000000002311514607367700301170ustar00rootroot00000000000000kind: overlap build-depends: - filename: subdir-a.bst config: location: / - filename: c.bst config: location: /opt config: action: error apache-buildstream-27ae392/tests/frontend/overlaps/multistage-overlap-ignore.bst000066400000000000000000000002321514607367700302520ustar00rootroot00000000000000kind: overlap build-depends: - filename: subdir-a.bst config: location: / - filename: c.bst config: location: /opt config: action: ignore apache-buildstream-27ae392/tests/frontend/overlaps/multistage-overlap.bst000066400000000000000000000002331514607367700267720ustar00rootroot00000000000000kind: overlap build-depends: - filename: subdir-a.bst config: location: / - filename: c.bst config: location: /opt config: action: warning apache-buildstream-27ae392/tests/frontend/overlaps/plugins/000077500000000000000000000000001514607367700241175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/plugins/overlap.py000066400000000000000000000032771514607367700261520ustar00rootroot00000000000000from buildstream import Element, OverlapAction # A testing element to test the behavior of staging overlapping files # class OverlapElement(Element): BST_MIN_VERSION = "2.0" def configure(self, node): node.validate_keys(["action"]) self.overlap_action = node.get_enum("action", OverlapAction) def configure_dependencies(self, dependencies): self.layout = {} for dep in dependencies: location = "/" if dep.config: dep.config.validate_keys(["location"]) location = dep.config.get_str("location") try: element_list = self.layout[location] except KeyError: element_list = [] self.layout[location] = element_list element_list.append((dep.element, dep.path)) def preflight(self): pass def get_unique_key(self): sorted_locations = sorted(self.layout) layout_key = { location: [dependency_path for _, dependency_path in self.layout[location]] for location in sorted_locations } return {"action": str(self.overlap_action), "layout": layout_key} def configure_sandbox(self, sandbox): for location in self.layout: sandbox.mark_directory(location) def stage(self, sandbox): sorted_locations = sorted(self.layout) for location in sorted_locations: element_list = [element for element, _ in self.layout[location]] self.stage_dependency_artifacts(sandbox, element_list, path=location, action=self.overlap_action) def assemble(self, sandbox): return "/" def setup(): return OverlapElement apache-buildstream-27ae392/tests/frontend/overlaps/relocated-unstaged.bst000066400000000000000000000001651514607367700267240ustar00rootroot00000000000000kind: overlap build-depends: - filename: directory-file.bst config: location: /opt config: action: warning apache-buildstream-27ae392/tests/frontend/overlaps/relocated.bst000066400000000000000000000003061514607367700251110ustar00rootroot00000000000000kind: overlap build-depends: - filename: a.bst config: location: /opt - filename: b.bst config: location: /opt - filename: c.bst config: location: /opt config: action: warning apache-buildstream-27ae392/tests/frontend/overlaps/script.bst000066400000000000000000000001661514607367700244570ustar00rootroot00000000000000kind: script depends: - filename: a.bst type: build - filename: b.bst type: build - filename: c.bst type: build apache-buildstream-27ae392/tests/frontend/overlaps/sub-collect.bst000066400000000000000000000002701514607367700253630ustar00rootroot00000000000000kind: compose depends: - filename: c.bst type: build - filename: a-sub.bst junction: sub-project.bst type: build - filename: z-sub.bst junction: sub-project.bst type: build apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/000077500000000000000000000000001514607367700246735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/a-sub.bst000066400000000000000000000001261514607367700264130ustar00rootroot00000000000000kind: import config: source: / target: / sources: - kind: local path: "files/a" apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/files/000077500000000000000000000000001514607367700257755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/files/a/000077500000000000000000000000001514607367700262155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/files/a/file3000066400000000000000000000000061514607367700271360ustar00rootroot00000000000000barny apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/files/z/000077500000000000000000000000001514607367700262465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/files/z/file1000066400000000000000000000000041514607367700271630ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/files/z/file2000066400000000000000000000000041514607367700271640ustar00rootroot00000000000000bar apache-buildstream-27ae392/tests/frontend/overlaps/sub-project/z-sub.bst000066400000000000000000000001261514607367700264440ustar00rootroot00000000000000kind: import config: source: / target: / sources: - kind: local path: "files/z" apache-buildstream-27ae392/tests/frontend/overlaps/subdir-a.bst000066400000000000000000000001231514607367700246520ustar00rootroot00000000000000kind: import config: source: / target: /opt sources: - kind: local path: "a" apache-buildstream-27ae392/tests/frontend/overlaps/unstaged.bst000066400000000000000000000000631514607367700247610ustar00rootroot00000000000000kind: compose build-depends: - directory-file.bst apache-buildstream-27ae392/tests/frontend/overlaps/whitelist-undefined.bst000066400000000000000000000002631514607367700271240ustar00rootroot00000000000000kind: import config: source: / target: / depends: - b-whitelisted.bst sources: - kind: local path: "a" public: bst: overlap-whitelist: - "%{undefined-variable}/*" apache-buildstream-27ae392/tests/frontend/overlaps/with-directory.bst000066400000000000000000000001351514607367700261240ustar00rootroot00000000000000kind: import config: source: / target: / sources: - kind: local path: "with-directory" apache-buildstream-27ae392/tests/frontend/overlaps/with-directory/000077500000000000000000000000001514607367700254135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/with-directory/directory-file/000077500000000000000000000000001514607367700303345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/overlaps/with-directory/directory-file/file000066400000000000000000000000051514607367700311710ustar00rootroot00000000000000file apache-buildstream-27ae392/tests/frontend/progress.py000066400000000000000000000126051514607367700230250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from tests.testutils import generate_junction # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_show_progress_tally(cli, datafiles): # Check that the progress reporting messages give correct tallies project = str(datafiles) result = cli.run(project=project, args=["show", "compose-all.bst"]) result.assert_success() assert " 3 subtasks processed" in result.stderr assert "3 of 3 subtasks processed" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_junction_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in junctions project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Assert the correct progress tallies are in the logging result = cli.run(project=project, args=["show", "junction-dep.bst"]) assert " 2 subtasks processed" in result.stderr assert "2 of 2 subtasks processed" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_nested_junction_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in # junctions of junctions project = str(datafiles) sub1_path = os.path.join(project, "files", "sub-project") sub2_path = os.path.join(project, "files", "sub2-project") # A junction element which pulls sub1 into sub2 sub1_element = os.path.join(project, "files", "sub2-project", "elements", "sub-junction.bst") # A junction element which pulls sub2 into the main project sub2_element = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") generate_junction(tmpdir / "sub-project", sub1_path, sub1_element, store_ref=True) generate_junction(tmpdir / "sub2-project", sub2_path, sub2_element, store_ref=True) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-sub.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Assert the correct progress tallies are in the logging result = cli.run(project=project, args=["show", "junction-dep.bst"]) assert " 3 subtasks processed" in result.stderr assert "3 of 3 subtasks processed" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_junction_dep_tally(cli, tmpdir, datafiles): # Check that the progress reporting messages count elements in junctions project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Add dependencies to the junction (not allowed, but let's do it # anyway) with open(junction_path, "a", encoding="utf-8") as f: deps = {"depends": ["manual.bst"]} _yaml.roundtrip_dump(deps, f) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction-dep.bst"]) # Since we aren't allowed to specify any dependencies on a # junction, we should fail result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_JUNCTION) # We don't get a final tally in this case assert "subtasks processed" not in result.stderr apache-buildstream-27ae392/tests/frontend/project/000077500000000000000000000000001514607367700222515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/000077500000000000000000000000001514607367700240655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/checkout-deps.bst000066400000000000000000000003631514607367700273370ustar00rootroot00000000000000kind: import description: It is important for this element to have both build and runtime dependencies sources: - kind: local path: files/etc-files depends: - filename: import-dev.bst type: build - filename: import-bin.bst type: runtime apache-buildstream-27ae392/tests/frontend/project/elements/compose-all.bst000066400000000000000000000003441514607367700270130ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/frontend/project/elements/compose-exclude-dev.bst000066400000000000000000000004251514607367700304500ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False # Exclude the dev domain exclude: - devel apache-buildstream-27ae392/tests/frontend/project/elements/compose-include-bin.bst000066400000000000000000000004301514607367700304300ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False # Only include the runtim include: - runtime apache-buildstream-27ae392/tests/frontend/project/elements/format-deps.bst000066400000000000000000000005361514607367700270240ustar00rootroot00000000000000kind: import description: > It is important that this element has both and build and runtime dependencies. It is also important that it has a dependency that is needed at both build time and runtime. sources: - kind: local path: files/etc-files depends: - import-links.bst build-depends: - import-dev.bst runtime-depends: - import-bin.bst apache-buildstream-27ae392/tests/frontend/project/elements/import-bin.bst000066400000000000000000000000741514607367700266600ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/frontend/project/elements/import-dev.bst000066400000000000000000000000741514607367700266660ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/frontend/project/elements/import-large-directory.bst000066400000000000000000000001021514607367700311740ustar00rootroot00000000000000kind: import sources: - kind: local path: files/large-directory apache-buildstream-27ae392/tests/frontend/project/elements/import-links.bst000066400000000000000000000001021514607367700272200ustar00rootroot00000000000000kind: import sources: - kind: local path: files/files-and-links apache-buildstream-27ae392/tests/frontend/project/elements/invalid-alias.bst000066400000000000000000000000741514607367700273150ustar00rootroot00000000000000kind: import sources: - kind: tar url: zebry:tarball.tar apache-buildstream-27ae392/tests/frontend/project/elements/manual.bst000066400000000000000000000001631514607367700260540ustar00rootroot00000000000000kind: manual config: build-commands: - echo "hello" sources: - kind: local path: elements/manual.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/000077500000000000000000000000001514607367700274515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/dependency/000077500000000000000000000000001514607367700315675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/dependency/horsey.bst000066400000000000000000000000771514607367700336160ustar00rootroot00000000000000kind: manual depends: - multiple_targets/dependency/pony.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/dependency/pony.bst000066400000000000000000000000151514607367700332620ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/dependency/zebry.bst000066400000000000000000000001011514607367700334240ustar00rootroot00000000000000kind: manual depends: - multiple_targets/dependency/horsey.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/000077500000000000000000000000001514607367700305645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/0.bst000066400000000000000000000002561514607367700314400ustar00rootroot00000000000000kind: manual description: Root node depends: - multiple_targets/order/2.bst - multiple_targets/order/3.bst - filename: multiple_targets/order/run.bst type: runtime apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/1.bst000066400000000000000000000001161514607367700314340ustar00rootroot00000000000000kind: manual description: Root node depends: - multiple_targets/order/9.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/2.bst000066400000000000000000000001331514607367700314340ustar00rootroot00000000000000kind: manual description: First dependency level depends: - multiple_targets/order/3.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/3.bst000066400000000000000000000002361514607367700314410ustar00rootroot00000000000000kind: manual description: Second dependency level depends: - multiple_targets/order/4.bst - multiple_targets/order/5.bst - multiple_targets/order/6.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/4.bst000066400000000000000000000000611514607367700314360ustar00rootroot00000000000000kind: manual description: Third level dependency apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/5.bst000066400000000000000000000000611514607367700314370ustar00rootroot00000000000000kind: manual description: Fifth level dependency apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/6.bst000066400000000000000000000001341514607367700314410ustar00rootroot00000000000000kind: manual description: Fourth level dependency depends: - multiple_targets/order/5.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/7.bst000066400000000000000000000001331514607367700314410ustar00rootroot00000000000000kind: manual description: Third level dependency depends: - multiple_targets/order/6.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/8.bst000066400000000000000000000001341514607367700314430ustar00rootroot00000000000000kind: manual description: Second level dependency depends: - multiple_targets/order/7.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/9.bst000066400000000000000000000001331514607367700314430ustar00rootroot00000000000000kind: manual description: First level dependency depends: - multiple_targets/order/8.bst apache-buildstream-27ae392/tests/frontend/project/elements/multiple_targets/order/run.bst000066400000000000000000000001241514607367700320770ustar00rootroot00000000000000kind: manual description: Not a root node, yet built at the same time as root nodes apache-buildstream-27ae392/tests/frontend/project/elements/random.bst000066400000000000000000000000241514607367700260530ustar00rootroot00000000000000kind: randomelement apache-buildstream-27ae392/tests/frontend/project/elements/rebuild-target.bst000066400000000000000000000000531514607367700275070ustar00rootroot00000000000000kind: compose build-depends: - target.bst apache-buildstream-27ae392/tests/frontend/project/elements/source-bundle/000077500000000000000000000000001514607367700266345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/source-bundle/source-bundle-hello.bst000066400000000000000000000002171514607367700332160ustar00rootroot00000000000000kind: import description: the kind of this element must implement generate_script() method sources: - kind: local path: files/source-bundle apache-buildstream-27ae392/tests/frontend/project/elements/target-import.bst000066400000000000000000000001431514607367700273730ustar00rootroot00000000000000kind: import sources: - kind: local path: files/foo depends: - import-bin.bst - compose-all.bst apache-buildstream-27ae392/tests/frontend/project/elements/target.bst000066400000000000000000000001641514607367700260660ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - compose-all.bst apache-buildstream-27ae392/tests/frontend/project/elements/target.foo000066400000000000000000000001071514607367700260560ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test apache-buildstream-27ae392/tests/frontend/project/elements/test.bst000066400000000000000000000000001514607367700255440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/elements/unaliased-tar.bst000066400000000000000000000001201514607367700273210ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://unaliased-url.org/tarball.tar apache-buildstream-27ae392/tests/frontend/project/files/000077500000000000000000000000001514607367700233535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/bar000066400000000000000000000000001514607367700240300ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/bin-files/000077500000000000000000000000001514607367700252235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/bin-files/usr/000077500000000000000000000000001514607367700260345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/bin-files/usr/bin/000077500000000000000000000000001514607367700266045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700276320ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/frontend/project/files/dev-files/000077500000000000000000000000001514607367700252315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/dev-files/usr/000077500000000000000000000000001514607367700260425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/dev-files/usr/include/000077500000000000000000000000001514607367700274655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700306240ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/project/files/etc-files/000077500000000000000000000000001514607367700252265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/etc-files/etc/000077500000000000000000000000001514607367700260015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/etc-files/etc/buildstream/000077500000000000000000000000001514607367700303145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/etc-files/etc/buildstream/config000066400000000000000000000000071514607367700315010ustar00rootroot00000000000000config apache-buildstream-27ae392/tests/frontend/project/files/files-and-links/000077500000000000000000000000001514607367700263335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/files-and-links/basicfile000066400000000000000000000000161514607367700301740ustar00rootroot00000000000000file contents apache-buildstream-27ae392/tests/frontend/project/files/files-and-links/basicfolder/000077500000000000000000000000001514607367700306105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/files-and-links/basicfolder/subdir-file000066400000000000000000000000001514607367700327260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/foo000066400000000000000000000000001514607367700240470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/source-bundle/000077500000000000000000000000001514607367700261225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/source-bundle/llamas.txt000066400000000000000000000000071514607367700301310ustar00rootroot00000000000000llamas apache-buildstream-27ae392/tests/frontend/project/files/sub-project/000077500000000000000000000000001514607367700256105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub-project/elements/000077500000000000000000000000001514607367700274245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub-project/elements/import-etc.bst000066400000000000000000000000741514607367700322220ustar00rootroot00000000000000kind: import sources: - kind: local path: files/etc-files apache-buildstream-27ae392/tests/frontend/project/files/sub-project/files/000077500000000000000000000000001514607367700267125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub-project/files/deps.bst000066400000000000000000000000211514607367700303500ustar00rootroot00000000000000- import-etc.bst apache-buildstream-27ae392/tests/frontend/project/files/sub-project/files/etc-files/000077500000000000000000000000001514607367700305655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub-project/files/etc-files/etc/000077500000000000000000000000001514607367700313405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub-project/files/etc-files/etc/animal.conf000066400000000000000000000000141514607367700334430ustar00rootroot00000000000000animal=Pony apache-buildstream-27ae392/tests/frontend/project/files/sub-project/project.conf000066400000000000000000000001371514607367700301260ustar00rootroot00000000000000# Project config for frontend build test name: subtest min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/000077500000000000000000000000001514607367700256725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/elements/000077500000000000000000000000001514607367700275065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/elements/import-sub.bst000066400000000000000000000001251514607367700323170ustar00rootroot00000000000000kind: stack depends: - junction: 'sub-junction.bst' filename: 'import-etc.bst' apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/files/000077500000000000000000000000001514607367700267745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/files/etc-files/000077500000000000000000000000001514607367700306475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/files/etc-files/etc/000077500000000000000000000000001514607367700314225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/files/etc-files/etc/animal.conf000066400000000000000000000000141514607367700335250ustar00rootroot00000000000000animal=Pony apache-buildstream-27ae392/tests/frontend/project/files/sub2-project/project.conf000066400000000000000000000001401514607367700302020ustar00rootroot00000000000000# Project config for frontend build test name: sub2test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/project/plugins/000077500000000000000000000000001514607367700237325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/plugins/randomelement.py000066400000000000000000000012601514607367700271350ustar00rootroot00000000000000import os from buildstream import Element class RandomElement(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): pass def configure_sandbox(self, sandbox): pass def stage(self, sandbox): pass def assemble(self, sandbox): rootdir = sandbox.get_virtual_directory() outputdir = rootdir.open_directory("output", create=True) # Generate non-reproducible output with outputdir.open_file("random", mode="wb") as f: f.write(os.urandom(64)) return "/output" def setup(): return RandomElement apache-buildstream-27ae392/tests/frontend/project/project.conf000066400000000000000000000002461514607367700245700ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/project/sourcemirrors/000077500000000000000000000000001514607367700251675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/sourcemirrors/mirror.py000066400000000000000000000016141514607367700270550ustar00rootroot00000000000000from typing import Optional, Dict, Any from buildstream import SourceMirror, MappingNode # This mirror plugin basically implements the default behavior # by loading the alias definitions as custom "config" configuration # instead, and implementing the translate_url method. # class Sample(SourceMirror): def configure(self, node): node.validate_keys(["aliases"]) self.aliases = {} aliases = node.get_mapping("aliases") for alias_name, url_list in aliases.items(): self.aliases[alias_name] = url_list.as_str_list() self.set_supported_aliases(self.aliases.keys()) def translate_url( self, *, alias: str, alias_url: str, source_url: str, extra_data: Optional[Dict[str, Any]], ) -> str: return self.aliases[alias][0] + source_url # Plugin entry point def setup(): return Sample apache-buildstream-27ae392/tests/frontend/project/sources/000077500000000000000000000000001514607367700237345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project/sources/fetch_source.py000066400000000000000000000054551514607367700267700ustar00rootroot00000000000000import os from buildstream import Source, SourceError, SourceFetcher # Expected config # sources: # - output-text: $FILE # urls: # - foo:bar # - baz:quux # fetch-succeeds: # Foo/bar: true # ooF/bar: false class FetchFetcher(SourceFetcher): def __init__(self, source, url, primary=False): super().__init__() self.source = source self.original_url = url self.primary = primary self.mark_download_url(url) def fetch(self, alias_override=None): url = self.source.translate_url(self.original_url, alias_override=alias_override, primary=self.primary) with open(self.source.output_file, "a") as f: success = url in self.source.fetch_succeeds and self.source.fetch_succeeds[url] message = "Fetch {} {} from {}\n".format(self.original_url, "succeeded" if success else "failed", url) f.write(message) if not success: raise SourceError("Failed to fetch {}".format(url)) class FetchSource(Source): BST_MIN_VERSION = "2.0" # Read config to know which URLs to fetch def configure(self, node): self.original_urls = node.get_str_list("urls") self.output_file = node.get_str("output-text") self.fetch_succeeds = {key: value.as_bool() for key, value in node.get_mapping("fetch-succeeds", {}).items()} # First URL is the primary one for this test # primary = True self.fetchers = [] for url in self.original_urls: self.mark_download_url(url, primary=primary) fetcher = FetchFetcher(self, url, primary=primary) self.fetchers.append(fetcher) primary = False def get_source_fetchers(self): return self.fetchers def preflight(self): output_dir = os.path.dirname(self.output_file) if not os.path.exists(output_dir): raise SourceError("Directory '{}' does not exist".format(output_dir)) def stage(self, directory): pass def fetch(self): for fetcher in self.fetchers: fetcher.fetch() def get_unique_key(self): return {"urls": self.original_urls, "output_file": self.output_file} def is_resolved(self): return True def is_cached(self) -> bool: if not os.path.exists(self.output_file): return False with open(self.output_file, "r") as f: contents = f.read() for url in self.original_urls: if url not in contents: return False return True # We dont have a ref, we're a local file... def load_ref(self, node): pass def get_ref(self): return None # pragma: nocover def set_ref(self, ref, node): pass # pragma: nocover def setup(): return FetchSource apache-buildstream-27ae392/tests/frontend/project_fail/000077500000000000000000000000001514607367700232445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project_fail/elements/000077500000000000000000000000001514607367700250605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project_fail/elements/compose-all.bst000066400000000000000000000002731514607367700300070ustar00rootroot00000000000000kind: compose depends: - fileNAME: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/frontend/project_fail/elements/import-dev.bst000066400000000000000000000000741514607367700276610ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/frontend/project_fail/elements/target.bst000066400000000000000000000001431514607367700270560ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - compose-all.bst apache-buildstream-27ae392/tests/frontend/project_fail/files/000077500000000000000000000000001514607367700243465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project_fail/files/dev-files/000077500000000000000000000000001514607367700262245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project_fail/files/dev-files/usr/000077500000000000000000000000001514607367700270355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project_fail/files/dev-files/usr/include/000077500000000000000000000000001514607367700304605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/project_fail/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700316170ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/project_fail/project.conf000066400000000000000000000001341514607367700255570ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/pull.py000066400000000000000000000616431514607367700221430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import stat import pytest from buildstream import utils, _yaml from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from tests.testutils import ( create_artifact_share, create_split_share, generate_junction, assert_shared, assert_not_shared, ) # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) # Tests that: # # * `bst build` pushes all build elements to configured 'push' cache # * `bst artifact pull --deps DEPS` downloads necessary artifacts from the cache # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "deps, expected_states", [ ("build", ("buildable", "cached", "buildable")), ("none", ("cached", "buildable", "buildable")), ("run", ("cached", "buildable", "cached")), ("all", ("cached", "cached", "cached")), ], ) def test_push_pull_deps(cli, tmpdir, datafiles, deps, expected_states): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" all_elements = [target, build_dep, runtime_dep] with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", target]) result.assert_success() # Assert that everything is now cached in the remote. for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore states = cli.get_element_states(project, all_elements) assert not any(states[e] == "cached" for e in all_elements) # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", "--deps", deps, target]) result.assert_success() # And assert that the pulled elements are again in the local cache states = cli.get_element_states(project, all_elements) states_flattended = (states[target], states[build_dep], states[runtime_dep]) assert states_flattended == expected_states # Tests that: # # * `bst build` pushes all build elements ONLY to configured 'push' cache # * `bst artifact pull` finds artifacts that are available only in the secondary cache # @pytest.mark.datafiles(DATA_DIR) def test_pull_secondary_cache(cli, tmpdir, datafiles): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "artifactshare2") ) as share2: # Build the target and push it to share2 only. cli.configure( { "artifacts": { "servers": [ {"url": share1.repo, "push": False}, {"url": share2.repo, "push": True}, ] } } ) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert_not_shared(cli, share1, project, "target.bst") assert_shared(cli, share2, project, "target.bst") # Delete the user's local artifact cache. casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that the element is not cached anymore. assert cli.get_element_state(project, "target.bst") != "cached" # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", "target.bst"]) result.assert_success() # And assert that it's again in the local cache, without having built, # i.e. we found it in share2. assert cli.get_element_state(project, "target.bst") == "cached" # Tests that: # # * `bst artifact push --artifact-remote` pushes to the given remote, not one from the config # * `bst artifact pull --artifact-remote` pulls from the given remote # @pytest.mark.datafiles(DATA_DIR) def test_push_pull_specific_remote(cli, tmpdir, datafiles): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "goodartifactshare")) as good_share, create_artifact_share( os.path.join(str(tmpdir), "badartifactshare") ) as bad_share: # Build the target so we have it cached locally only. result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() state = cli.get_element_state(project, "target.bst") assert state == "cached" # Configure the default push location to be bad_share; we will assert that # nothing actually gets pushed there. cli.configure( { "artifacts": { "servers": [ {"url": bad_share.repo, "push": True}, ] } } ) # Now try `bst artifact push` to the good_share. result = cli.run( project=project, args=["artifact", "push", "target.bst", "--artifact-remote", good_share.repo] ) result.assert_success() # Assert that all the artifacts are in the share we pushed # to, and not the other. assert_shared(cli, good_share, project, "target.bst") assert_not_shared(cli, bad_share, project, "target.bst") # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the good_share. # casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) result = cli.run( project=project, args=["artifact", "pull", "target.bst", "--artifact-remote", good_share.repo] ) result.assert_success() # And assert that it's again in the local cache, without having built assert cli.get_element_state(project, "target.bst") == "cached" # Tests that: # # * In non-strict mode, dependency changes don't block artifact reuse # @pytest.mark.datafiles(DATA_DIR) def test_push_pull_non_strict(cli, tmpdir, datafiles): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. cli.configure( {"artifacts": {"servers": [{"url": share.repo, "push": True}]}, "projects": {"test": {"strict": False}}} ) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert cli.get_element_state(project, "target.bst") == "cached" # Assert that everything is now cached in the remote. all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"] for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore for element_name in all_elements: assert cli.get_element_state(project, element_name) != "cached" # Add a file to force change in strict cache key of import-bin.bst with open(os.path.join(str(project), "files", "bin-files", "usr", "bin", "world"), "w", encoding="utf-8") as f: f.write("world") # Assert that the workspaced element requires a rebuild assert cli.get_element_state(project, "import-bin.bst") == "buildable" # Assert that the target is still waiting due to --no-strict assert cli.get_element_state(project, "target.bst") == "waiting" # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", "target.bst"]) result.assert_success() # And assert that the target is again in the local cache, without having built assert cli.get_element_state(project, "target.bst") == "cached" @pytest.mark.datafiles(DATA_DIR) def test_push_pull_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # First build the target element and push to the remote. cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" cache_dir = os.path.join(project, "cache", "cas") shutil.rmtree(cache_dir) artifact_dir = os.path.join(project, "cache", "artifacts") shutil.rmtree(artifact_dir) assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "buildable" # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", "junction.bst:import-etc.bst"]) result.assert_success() # And assert that it's again in the local cache, without having built assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" def _test_pull_missing_blob(cli, project, index, storage): # First build the target element and push to the remote. result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert cli.get_element_state(project, "target.bst") == "cached" # Assert that everything is now cached in the remote. all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"] for element_name in all_elements: project_name = "test" artifact_name = cli.get_artifact_name(project, project_name, element_name) artifact_proto = index.get_artifact_proto(artifact_name) assert artifact_proto assert storage.get_cas_files(artifact_proto) # Now we've pushed, delete the user's local artifact cache # directory and try to redownload it from the share # casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore for element_name in all_elements: assert cli.get_element_state(project, element_name) != "cached" # Now delete blobs in the remote without deleting the artifact ref. # This simulates scenarios with concurrent artifact expiry. remote_objdir = os.path.join(storage.repodir, "cas", "objects") shutil.rmtree(remote_objdir) # Now try bst build result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that no artifacts were pulled assert not result.get_pulled_elements() @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_blob(cli, tmpdir, datafiles): project = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) _test_pull_missing_blob(cli, project, share, share) @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_blob_split_share(cli, tmpdir, datafiles): project = str(datafiles) indexshare = os.path.join(str(tmpdir), "indexshare") storageshare = os.path.join(str(tmpdir), "storageshare") with create_split_share(indexshare, storageshare) as (index, storage): cli.configure( { "artifacts": { "servers": [ {"url": index.repo, "push": True, "type": "index"}, {"url": storage.repo, "push": True, "type": "storage"}, ] } } ) _test_pull_missing_blob(cli, project, index, storage) @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_local_blob(cli, tmpdir, datafiles): project = str(datafiles) repo = create_repo("tar", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) element_dir = os.path.join(str(tmpdir), "elements") project = str(tmpdir) project_config = { "name": "pull-missing-local-blob", "min-version": "2.0", "element-path": "elements", } project_file = os.path.join(str(tmpdir), "project.conf") _yaml.roundtrip_dump(project_config, project_file) input_config = { "kind": "import", "sources": [repo.source_config()], } input_name = "input.bst" input_file = os.path.join(element_dir, input_name) _yaml.roundtrip_dump(input_config, input_file) depends_name = "depends.bst" depends_config = {"kind": "stack", "depends": [input_name]} depends_file = os.path.join(element_dir, depends_name) _yaml.roundtrip_dump(depends_config, depends_file) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the import-bin element and push to the remote. cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["source", "track", input_name]) result.assert_success() result = cli.run(project=project, args=["build", input_name]) result.assert_success() assert cli.get_element_state(project, input_name) == "cached" # Delete a file blob from the local cache. # This is a placeholder to test partial CAS handling until we support # partial artifact pulling (or blob-based CAS expiry). # digest = utils.sha256sum(os.path.join(project, "files", "bin-files", "usr", "bin", "hello")) objpath = os.path.join(cli.directory, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # Now try bst build result = cli.run(project=project, args=["build", depends_name]) result.assert_success() # Assert that the import-bin artifact was pulled (completing the partial artifact) assert result.get_pulled_elements() == [input_name] @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo}]}}) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert not result.get_pulled_elements(), "No elements should have been pulled since the cache was empty" assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr assert "SKIPPED Pull" in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_build_remote_option(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as shareuser, create_artifact_share( os.path.join(str(tmpdir), "artifactshare2") ) as sharecli: # Configure shareuser remote in user conf cli.configure({"artifacts": {"servers": [{"url": shareuser.repo, "push": True}]}}) # Push the artifacts to the shareuser remote. # Assert that shareuser has the artfifacts cached, but sharecli doesn't, # then delete locally cached elements result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"] for element_name in all_elements: assert element_name in result.get_pushed_elements() assert_not_shared(cli, sharecli, project, element_name) assert_shared(cli, shareuser, project, element_name) cli.remove_artifact_from_cache(project, element_name) # Now check that a build with cli set as sharecli results in nothing being pulled, # as it doesn't have them cached and shareuser should be ignored. This # will however result in the artifacts being built and pushed to it result = cli.run(project=project, args=["build", "--artifact-remote", sharecli.repo, "target.bst"]) result.assert_success() for element_name in all_elements: assert element_name not in result.get_pulled_elements() assert_shared(cli, sharecli, project, element_name) cli.remove_artifact_from_cache(project, element_name) # Now check that a clean build with cli set as sharecli should result in artifacts only # being pulled from it, as that was provided via the cli and is populated result = cli.run(project=project, args=["build", "--artifact-remote", sharecli.repo, "target.bst"]) result.assert_success() for element_name in all_elements: assert cli.get_element_state(project, element_name) == "cached" assert element_name in result.get_pulled_elements() assert shareuser.repo not in result.stderr assert sharecli.repo in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_pull_access_rights(cli, tmpdir, datafiles): project = str(datafiles) checkout = os.path.join(str(tmpdir), "checkout") umask = utils.get_umask() # Work-around datafiles not preserving mode os.chmod(os.path.join(project, "files/bin-files/usr/bin/hello"), 0o0755) # We need a big file that does not go into a batch to test a different # code path os.makedirs(os.path.join(project, "files/dev-files/usr/share"), exist_ok=True) with open(os.path.join(project, "files/dev-files/usr/share/big-file"), "w", encoding="utf-8") as f: buf = " " * 4096 for _ in range(1024): f.write(buf) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", "compose-all.bst"]) result.assert_success() result = cli.run( project=project, args=["artifact", "checkout", "--no-integrate", "compose-all.bst", "--directory", checkout], ) result.assert_success() st = os.lstat(os.path.join(checkout, "usr/include/pony.h")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0666 & ~umask st = os.lstat(os.path.join(checkout, "usr/bin/hello")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0777 & ~umask st = os.lstat(os.path.join(checkout, "usr/share/big-file")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0666 & ~umask shutil.rmtree(checkout) casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) result = cli.run(project=project, args=["artifact", "pull", "compose-all.bst"]) result.assert_success() result = cli.run( project=project, args=["artifact", "checkout", "--no-integrate", "compose-all.bst", "--directory", checkout], ) result.assert_success() st = os.lstat(os.path.join(checkout, "usr/include/pony.h")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0666 & ~umask st = os.lstat(os.path.join(checkout, "usr/bin/hello")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0777 & ~umask st = os.lstat(os.path.join(checkout, "usr/share/big-file")) assert stat.S_ISREG(st.st_mode) assert stat.S_IMODE(st.st_mode) == 0o0666 & ~umask # Tests `bst artifact pull $artifact_ref` @pytest.mark.datafiles(DATA_DIR) def test_pull_artifact(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Assert that the target is shared (note that assert shared will use the artifact name) assert_shared(cli, share, project, element) # Now we've pushed, remove the local cache shutil.rmtree(os.path.join(local_cache, "artifacts")) # Assert that nothing is cached locally anymore assert not os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Now try bst artifact pull result = cli.run(project=project, args=["artifact", "pull", artifact_ref]) result.assert_success() # And assert that it's again in the local cache, without having built assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "deps, expected_pulled_elements, expected_states", [ ("none", ["checkout-deps.bst"], ("cached", "buildable", "buildable")), ("run", ["checkout-deps.bst", "import-bin.bst"], ("cached", "buildable", "cached")), ("all", ["checkout-deps.bst", "import-bin.bst", "import-dev.bst"], ("cached", "cached", "cached")), ], ) def test_dynamic_build_plan(cli, tmpdir, datafiles, deps, expected_pulled_elements, expected_states): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" all_elements = [target, build_dep, runtime_dep] with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target element and push to the remote. cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", target]) result.assert_success() # Assert that everything is now cached in the remote. for element_name in all_elements: assert_shared(cli, share, project, element_name) # Now we've pushed, delete the user's local artifact cache directory casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) artifactdir = os.path.join(cli.directory, "artifacts") shutil.rmtree(artifactdir) # Assert that nothing is cached locally anymore states = cli.get_element_states(project, all_elements) assert not any(states[e] == "cached" for e in all_elements) # Now try to rebuild target result = cli.run(project=project, args=["build", "--deps", deps, target]) result.assert_success() # Assert that we only pulled the needed dependencies # (dynamic build plan). assert sorted(result.get_pulled_elements()) == expected_pulled_elements # And assert that the pulled elements are again in the local cache states = cli.get_element_states(project, all_elements) states_flattended = (states[target], states[build_dep], states[runtime_dep]) assert states_flattended == expected_states apache-buildstream-27ae392/tests/frontend/push.py000066400000000000000000001021471514607367700221410ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: Tristan Van Berkom # Sam Thursfield # Jürg Billeter # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream.exceptions import ErrorDomain from buildstream._testing import cli, generate_project, Cli # pylint: disable=unused-import from buildstream._testing._utils.site import have_subsecond_mtime from tests.testutils import ( create_artifact_share, create_element_size, generate_junction, wait_for_cache_granularity, assert_shared, assert_not_shared, ) # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) # Tests that: # # * `bst artifact push` fails if there are no remotes configured for pushing # * `bst artifact push` successfully pushes to any remote that is configured for pushing # @pytest.mark.datafiles(DATA_DIR) def test_push(cli, tmpdir, datafiles): project = str(datafiles) # First build the project without the artifact cache configured result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project, "target.bst") == "cached" # Set up two artifact shares. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1: with create_artifact_share(os.path.join(str(tmpdir), "artifactshare2")) as share2: # Try pushing with no remotes configured. This should fail. result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Configure bst to pull but not push from a cache and run `bst artifact push`. # This should also fail. cli.configure({"artifacts": {"servers": [{"url": share1.repo, "push": False}]}}) result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Configure bst to push to one of the caches and run `bst artifact push`. This works. cli.configure( { "artifacts": { "servers": [ {"url": share1.repo, "push": False}, {"url": share2.repo, "push": True}, ] } } ) cli.run(project=project, args=["artifact", "push", "target.bst"]) assert_not_shared(cli, share1, project, "target.bst") assert_shared(cli, share2, project, "target.bst") # Now try pushing to both with create_artifact_share(os.path.join(str(tmpdir), "artifactshare2")) as share2: cli.configure( { "artifacts": { "servers": [ {"url": share1.repo, "push": True}, {"url": share2.repo, "push": True}, ] } } ) cli.run(project=project, args=["artifact", "push", "target.bst"]) assert_shared(cli, share1, project, "target.bst") assert_shared(cli, share2, project, "target.bst") # Tests `bst artifact push $artifact_ref` @pytest.mark.datafiles(DATA_DIR) def test_push_artifact(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Configure artifact share cli.configure( { # # FIXME: This test hangs "sometimes" if we allow # concurrent push. # # It's not too bad to ignore since we're # using the local artifact cache functionality # only, but it should probably be fixed. # "scheduler": {"pushers": 1}, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, } ) # Now try bst artifact push all the deps result = cli.run(project=project, args=["artifact", "push", artifact_ref]) result.assert_success() # And finally assert that all the artifacts are in the share # # Note that assert shared tests that an element is shared by obtaining # the artifact ref and asserting that the path exists in the share assert_shared(cli, share, project, element) @pytest.mark.datafiles(DATA_DIR) def test_push_artifact_glob(cli, tmpdir, datafiles): project = str(datafiles) element = "target.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Configure artifact share cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Run bst artifact push with a wildcard, there is only one artifact # matching "test/target/*", even though it can be accessed both by it's # strong and weak key. # result = cli.run(project=project, args=["artifact", "push", "test/target/*"]) result.assert_success() assert len(result.get_pushed_elements()) == 1 # Tests that: # # * `bst artifact push` fails if the element is not cached locally # * `bst artifact push` fails if multiple elements are not cached locally # @pytest.mark.datafiles(DATA_DIR) def test_push_fails(cli, tmpdir, datafiles): project = str(datafiles) # Set up the share with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure bst to be able to push to the share cli.configure( { "artifacts": { "servers": [ {"url": share.repo, "push": True}, ] } } ) # First ensure that the target is *NOT* cache assert cli.get_element_state(project, "target.bst") != "cached" # Now try and push the target result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) assert "Push failed: target.bst is not cached" in result.stderr # Now ensure that deps are also not cached assert cli.get_element_state(project, "import-bin.bst") != "cached" assert cli.get_element_state(project, "import-dev.bst") != "cached" assert cli.get_element_state(project, "compose-all.bst") != "cached" # Tests that: # # * `bst artifact push` fails when one of the targets is not cached, but still pushes the others # @pytest.mark.datafiles(DATA_DIR) def test_push_fails_with_on_error_continue(cli, tmpdir, datafiles): project = str(datafiles) # Set up the share with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build the target (and its deps) result = cli.run(project=project, args=["build", "target.bst"]) assert cli.get_element_state(project, "target.bst") == "cached" assert cli.get_element_state(project, "import-dev.bst") == "cached" # Now delete the artifact of a dependency and ensure it is not in the cache result = cli.run(project=project, args=["artifact", "delete", "import-dev.bst"]) assert cli.get_element_state(project, "import-dev.bst") != "cached" # Configure bst to be able to push to the share cli.configure( { "artifacts": { "servers": [ {"url": share.repo, "push": True}, ] } } ) # Now try and push the target with its deps using --on-error continue # and assert that push failed, but what could be pushed was pushed result = cli.run( project=project, args=["--on-error=continue", "artifact", "push", "--deps", "all", "target.bst"] ) # The overall process should return as failed result.assert_main_error(ErrorDomain.STREAM, None) # We should still have pushed what we could assert_shared(cli, share, project, "import-bin.bst") assert_shared(cli, share, project, "compose-all.bst") assert_shared(cli, share, project, "target.bst") assert_not_shared(cli, share, project, "import-dev.bst") assert "Push failed: import-dev.bst is not cached" in result.stderr # Tests that `bst artifact push --deps DEPS` pushes selected dependencies of # the given element. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "deps, expected_states", [ ("build", [False, True, False]), ("none", [True, False, False]), ("run", [True, False, True]), ("all", [True, True, True]), ], ) def test_push_deps(cli, tmpdir, datafiles, deps, expected_states): project = str(datafiles) target = "checkout-deps.bst" build_dep = "import-dev.bst" runtime_dep = "import-bin.bst" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured result = cli.run(project=project, args=["build", target]) result.assert_success() # Configure artifact share cli.configure( { # # FIXME: This test hangs "sometimes" if we allow # concurrent push. # # It's not too bad to ignore since we're # using the local artifact cache functionality # only, but it should probably be fixed. # "scheduler": {"pushers": 1}, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, } ) # Now try bst artifact push all the deps result = cli.run(project=project, args=["artifact", "push", target, "--deps", deps]) result.assert_success() # And finally assert that the selected artifacts are in the share states = [] for element in (target, build_dep, runtime_dep): is_cached = share.get_artifact(cli.get_artifact_name(project, "test", element)) is not None states.append(is_cached) assert states == expected_states # Tests that `bst artifact push --deps run $artifact_ref` fails @pytest.mark.datafiles(DATA_DIR) def test_push_artifacts_all_deps_fails(cli, tmpdir, datafiles): project = str(datafiles) element = "checkout-deps.bst" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # First build it without the artifact cache configured result = cli.run(project=project, args=["build", element]) result.assert_success() # Assert that the *artifact* is cached locally cache_key = cli.get_element_key(project, element) artifact_ref = os.path.join("test", os.path.splitext(element)[0], cache_key) assert os.path.exists(os.path.join(local_cache, "artifacts", "refs", artifact_ref)) # Configure artifact share cli.configure( { # # FIXME: This test hangs "sometimes" if we allow # concurrent push. # # It's not too bad to ignore since we're # using the local artifact cache functionality # only, but it should probably be fixed. # "scheduler": {"pushers": 1}, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, } ) # Now try bst artifact push all the deps result = cli.run(project=project, args=["artifact", "push", "--deps", "all", artifact_ref]) result.assert_main_error(ErrorDomain.STREAM, "deps-not-supported") # Tests that `bst build` won't push artifacts to the cache it just pulled from. # # Regression test for https://gitlab.com/BuildStream/buildstream/issues/233. @pytest.mark.datafiles(DATA_DIR) def test_push_after_pull(cli, tmpdir, datafiles): project = str(datafiles) # Set up two artifact shares. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "artifactshare2") ) as share2: # Set the scene: share1 has the artifact, share2 does not. # cli.configure( { "artifacts": { "servers": [ {"url": share1.repo, "push": True}, ] } } ) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() cli.remove_artifact_from_cache(project, "target.bst") assert_shared(cli, share1, project, "target.bst") assert_not_shared(cli, share2, project, "target.bst") assert cli.get_element_state(project, "target.bst") != "cached" # Now run the build again. Correct `bst build` behaviour is to download the # artifact from share1 but not push it back again. # result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert "target.bst" in result.get_pulled_elements() assert "target.bst" not in result.get_pushed_elements() # Delete the artifact locally again. cli.remove_artifact_from_cache(project, "target.bst") # Now we add share2 into the mix as a second push remote. This time, # `bst build` should push to share2 after pulling from share1. cli.configure( { "artifacts": { "servers": [ {"url": share1.repo, "push": True}, {"url": share2.repo, "push": True}, ] } } ) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert "target.bst" in result.get_pulled_elements() assert "target.bst" in result.get_pushed_elements() # Ensure that when an artifact's size exceeds available disk space # the least recently pushed artifact is deleted in order to make room for # the incoming artifact. @pytest.mark.datafiles(DATA_DIR) def test_artifact_expires(cli, datafiles, tmpdir): project = str(datafiles) element_path = "elements" # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare # Set a 22 MB quota with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share: # Configure bst to push to the cache cli.configure( { "artifacts": { "servers": [ {"url": share.repo, "push": True}, ] } } ) # Create and build an element of 15 MB create_element_size("element1.bst", project, element_path, [], int(15e6)) result = cli.run(project=project, args=["build", "element1.bst"]) result.assert_success() # Create and build an element of 5 MB create_element_size("element2.bst", project, element_path, [], int(5e6)) result = cli.run(project=project, args=["build", "element2.bst"]) result.assert_success() # check that element's 1 and 2 are cached both locally and remotely states = cli.get_element_states(project, ["element1.bst", "element2.bst"]) assert states == { "element1.bst": "cached", "element2.bst": "cached", } assert_shared(cli, share, project, "element1.bst") assert_shared(cli, share, project, "element2.bst") # Create and build another element of 5 MB (This will exceed the free disk space available) create_element_size("element3.bst", project, element_path, [], int(5e6)) result = cli.run(project=project, args=["build", "element3.bst"]) result.assert_success() # Ensure it is cached both locally and remotely assert cli.get_element_state(project, "element3.bst") == "cached" assert_shared(cli, share, project, "element3.bst") # Ensure element1 has been removed from the share assert_not_shared(cli, share, project, "element1.bst") # Ensure that elemen2 remains assert_shared(cli, share, project, "element2.bst") # Test that a large artifact, whose size exceeds the quota, is not pushed # to the remote share @pytest.mark.datafiles(DATA_DIR) def test_artifact_too_large(cli, datafiles, tmpdir): project = str(datafiles) element_path = "elements" # Create an artifact share (remote cache) in tmpdir/artifactshare # Mock a file system with 5 MB total space with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(5e6)) as share: # Configure bst to push to the remote cache cli.configure( { "artifacts": { "servers": [{"url": share.repo, "push": True}], } } ) # Create and push a 3MB element create_element_size("small_element.bst", project, element_path, [], int(3e6)) result = cli.run(project=project, args=["build", "small_element.bst"]) result.assert_success() # Create and try to push a 6MB element. create_element_size("large_element.bst", project, element_path, [], int(6e6)) result = cli.run(project=project, args=["build", "large_element.bst"]) # This should fail; the server will refuse to store the CAS # blobs for the artifact, and then fail to find the files for # the uploaded artifact proto. # # FIXME: This should be extremely uncommon in practice, since # the artifact needs to be at least half the cache size for # this to happen. Nonetheless, a nicer error message would be # nice (perhaps we should just disallow uploading artifacts # that large). result.assert_main_error(ErrorDomain.STREAM, None) # Ensure that the small artifact is still in the share states = cli.get_element_states(project, ["small_element.bst", "large_element.bst"]) assert states["small_element.bst"] == "cached" assert_shared(cli, share, project, "small_element.bst") # Ensure that the artifact is cached locally but NOT remotely assert states["large_element.bst"] == "cached" assert_not_shared(cli, share, project, "large_element.bst") # Test that when an element is pulled recently, it is not considered the LRU element. @pytest.mark.datafiles(DATA_DIR) def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): project = str(datafiles) element_path = "elements" # The artifact expiry logic relies on mtime changes, in real life second precision # should be enough for this to work almost all the time, but test cases happen very # quickly, resulting in all artifacts having the same mtime. # # This test requires subsecond mtime to be reliable. # if not have_subsecond_mtime(project): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) # Create an artifact share (remote cache) in tmpdir/artifactshare # Set a 22 MB quota with create_artifact_share(os.path.join(str(tmpdir), "artifactshare"), quota=int(22e6)) as share: # Configure bst to push to the cache cli.configure( { "artifacts": { "servers": [{"url": share.repo, "push": True}], } } ) # Create and build 2 elements, one 5 MB and one 15 MB. create_element_size("element1.bst", project, element_path, [], int(5e6)) result = cli.run(project=project, args=["build", "element1.bst"]) result.assert_success() create_element_size("element2.bst", project, element_path, [], int(15e6)) result = cli.run(project=project, args=["build", "element2.bst"]) result.assert_success() # Ensure they are cached locally states = cli.get_element_states(project, ["element1.bst", "element2.bst"]) assert states == { "element1.bst": "cached", "element2.bst": "cached", } # Ensure that they have been pushed to the cache assert_shared(cli, share, project, "element1.bst") assert_shared(cli, share, project, "element2.bst") # Pull the element1 from the remote cache (this should update its mtime). # Use a separate local cache for this to ensure the complete element is pulled. cli2_path = os.path.join(str(tmpdir), "cli2") cli2 = Cli(cli2_path) result = cli2.run(project=project, args=["artifact", "pull", "element1.bst", "--artifact-remote", share.repo]) result.assert_success() # Ensure element1 is cached locally assert cli2.get_element_state(project, "element1.bst") == "cached" wait_for_cache_granularity() # Create and build the element3 (of 5 MB) create_element_size("element3.bst", project, element_path, [], int(5e6)) result = cli.run(project=project, args=["build", "element3.bst"]) result.assert_success() # Make sure it's cached locally and remotely assert cli.get_element_state(project, "element3.bst") == "cached" assert_shared(cli, share, project, "element3.bst") # Ensure that element2 was deleted from the share and element1 remains assert_not_shared(cli, share, project, "element2.bst") assert_shared(cli, share, project, "element1.bst") @pytest.mark.datafiles(DATA_DIR) def test_push_cross_junction(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) result = cli.run(project=project, args=["build", "junction.bst:import-etc.bst"]) result.assert_success() assert cli.get_element_state(project, "junction.bst:import-etc.bst") == "cached" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure( { "artifacts": { "servers": [{"url": share.repo, "push": True}], } } ) cli.run(project=project, args=["artifact", "push", "junction.bst:import-etc.bst"]) cache_key = cli.get_element_key(project, "junction.bst:import-etc.bst") assert share.get_artifact(cli.get_artifact_name(project, "subtest", "import-etc.bst", cache_key=cache_key)) @pytest.mark.datafiles(DATA_DIR) def test_push_already_cached(caplog, cli, tmpdir, datafiles): project = str(datafiles) caplog.set_level(1) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert "SKIPPED Push" not in result.stderr result = cli.run(project=project, args=["artifact", "push", "target.bst"]) result.assert_success() assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated" assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr assert "SKIPPED Push" in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("use_remote", [True, False], ids=["with_cli_remote", "without_cli_remote"]) @pytest.mark.parametrize("ignore_project", [True, False], ids=["ignore_project_caches", "include_project_caches"]) def test_build_remote_option(caplog, cli, tmpdir, datafiles, use_remote, ignore_project): project = str(datafiles) caplog.set_level(1) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare1")) as shareuser, create_artifact_share( os.path.join(str(tmpdir), "artifactshare2") ) as shareproject, create_artifact_share(os.path.join(str(tmpdir), "artifactshare3")) as sharecli: # Add shareproject repo url to project.conf with open(os.path.join(project, "project.conf"), "a", encoding="utf-8") as projconf: projconf.write("artifacts:\n- url: {}\n push: True".format(shareproject.repo)) # Configure shareuser remote in user conf cli.configure({"artifacts": {"servers": [{"url": shareuser.repo, "push": True}]}}) args = ["build", "target.bst"] if use_remote: args += ["--artifact-remote", sharecli.repo] if ignore_project: args += ["--ignore-project-artifact-remotes"] result = cli.run(project=project, args=args) # Artifacts should have only been pushed to sharecli, as that was provided via the cli result.assert_success() all_elements = ["target.bst", "import-bin.bst", "compose-all.bst"] for element_name in all_elements: assert element_name in result.get_pushed_elements() # Test shared state of project recommended cache depending # on whether we decided to ignore project suggestions. # if ignore_project: assert_not_shared(cli, shareproject, project, element_name) else: assert_shared(cli, shareproject, project, element_name) # If we specified a remote on the command line, this replaces any remotes # specified in user configuration. # if use_remote: assert_not_shared(cli, shareuser, project, element_name) assert_shared(cli, sharecli, project, element_name) else: assert_shared(cli, shareuser, project, element_name) assert_not_shared(cli, sharecli, project, element_name) # This test ensures that we are able to run `bst artifact push` in non strict mode # and that we do not crash when trying to push elements even though they # have not yet been pulled. # # This is a regression test for issue #990 # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("buildtrees", [("buildtrees"), ("normal")]) def test_push_no_strict(caplog, cli, tmpdir, datafiles, buildtrees): project = str(datafiles) caplog.set_level(1) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure( {"artifacts": {"servers": [{"url": share.repo, "push": True}]}, "projects": {"test": {"strict": False}}} ) # First get us a build result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Now cause one of the dependenies to change their cache key # # Here we just add a file, causing the strong cache key of the # import-bin.bst element to change due to the local files it # imports changing. path = os.path.join(project, "files", "bin-files", "newfile") with open(path, "w", encoding="utf-8") as f: f.write("PONY !") # Now build again after having changed the dependencies result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Now run `bst artifact push`. # # Optionally try it with --pull-buildtrees, since this causes # a pull queue to be added to the `push` command, the behavior # around this is different. args = [] if buildtrees == "buildtrees": args += ["--pull-buildtrees"] args += ["artifact", "push", "--deps", "all", "target.bst"] result = cli.run(project=project, args=args) result.assert_success() # Test that push works after rebuilding an incomplete artifact # of a non-reproducible element. @pytest.mark.datafiles(DATA_DIR) def test_push_after_rebuild(cli, tmpdir, datafiles): project = str(datafiles) generate_project( project, config={ "element-path": "elements", "min-version": "2.0", "plugins": [{"origin": "local", "path": "plugins", "elements": ["randomelement"]}], }, ) # First build the element result = cli.run(project=project, args=["build", "random.bst"]) result.assert_success() assert cli.get_element_state(project, "random.bst") == "cached" # Delete the artifact blobs but keep the artifact proto, # i.e., now we have an incomplete artifact casdir = os.path.join(cli.directory, "cas") shutil.rmtree(casdir) assert cli.get_element_state(project, "random.bst") != "cached" with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Now rebuild the element and push it result = cli.run(project=project, args=["build", "random.bst"]) result.assert_success() assert result.get_pushed_elements() == ["random.bst"] assert cli.get_element_state(project, "random.bst") == "cached" # Test that push after rebuilding a non-reproducible element updates the # artifact on the server. @pytest.mark.datafiles(DATA_DIR) def test_push_update_after_rebuild(cli, tmpdir, datafiles): project = str(datafiles) generate_project( project, config={ "element-path": "elements", "min-version": "2.0", "plugins": [{"origin": "local", "path": "plugins", "elements": ["randomelement"]}], }, ) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Build the element and push the artifact result = cli.run(project=project, args=["build", "random.bst"]) result.assert_success() assert result.get_pushed_elements() == ["random.bst"] assert cli.get_element_state(project, "random.bst") == "cached" # Now delete the artifact and ensure it is not in the cache result = cli.run(project=project, args=["artifact", "delete", "random.bst"]) assert cli.get_element_state(project, "random.bst") != "cached" # Now rebuild the element. Reset config to disable pulling. cli.config = None result = cli.run(project=project, args=["build", "random.bst"]) result.assert_success() assert cli.get_element_state(project, "random.bst") == "cached" # Push the new build cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["artifact", "push", "random.bst"]) assert result.get_pushed_elements() == ["random.bst"] apache-buildstream-27ae392/tests/frontend/rebuild.py000066400000000000000000000071741514607367700226140ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) def strict_args(args, strict): if strict != "strict": return ["--no-strict", *args] return args @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", ["strict", "non-strict"]) def test_rebuild(datafiles, cli, strict): project = str(datafiles) # First build intermediate target.bst result = cli.run(project=project, args=strict_args(["build", "target.bst"], strict)) result.assert_success() # Modify base import with open(os.path.join(project, "files", "dev-files", "usr", "include", "new.h"), "w", encoding="utf-8") as f: f.write("#define NEW") # Rebuild base import and build top-level rebuild-target.bst # In non-strict mode, this does not rebuild intermediate target.bst, # which means that a weakly cached target.bst will be staged as dependency. result = cli.run(project=project, args=strict_args(["build", "rebuild-target.bst"], strict)) result.assert_success() built_elements = result.get_built_elements() assert "rebuild-target.bst" in built_elements if strict == "strict": assert "target.bst" in built_elements else: assert "target.bst" not in built_elements # Test that a cached artifact matching the strict cache key is preferred # to a more recent artifact matching only the weak cache key. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", ["strict", "non-strict"]) def test_modify_and_revert(datafiles, cli, strict): project = str(datafiles) # First build target and dependencies result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Remember cache key of first build target_cache_key = cli.get_element_key(project, "target.bst") # Modify dependency new_header_path = os.path.join(project, "files", "dev-files", "usr", "include", "new.h") with open(new_header_path, "w", encoding="utf-8") as f: f.write("#define NEW") # Trigger rebuild. This will also rebuild the unmodified target as this # follows a strict build plan. result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert "target.bst" in result.get_built_elements() assert cli.get_element_key(project, "target.bst") != target_cache_key # Revert previous modification in dependency os.unlink(new_header_path) # Rebuild again, everything should be cached. result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() assert len(result.get_built_elements()) == 0 # Verify that cache key now again matches the first build in both # strict and non-strict mode. cli.configure({"projects": {"test": {"strict": strict == "strict"}}}) assert cli.get_element_key(project, "target.bst") == target_cache_key apache-buildstream-27ae392/tests/frontend/remote-caches.py000066400000000000000000000132671514607367700237050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml from tests.testutils import create_artifact_share, create_element_size DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def message_handler(message, context): pass @pytest.mark.datafiles(DATA_DIR) def test_build_checkout(cli, tmpdir, datafiles): cachedir = os.path.join(str(tmpdir), "cache") project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") with create_artifact_share(os.path.join(str(tmpdir), "remote-cache")) as remote_cache: # Enable remote cache cli.configure({"cache": {"storage-service": {"url": remote_cache.repo}}}) # First build it result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() # Discard the local CAS cache shutil.rmtree(str(os.path.join(cachedir, "cas"))) # Now check it out, this should automatically fetch the necessary blobs # from the remote cache result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkout]) result.assert_success() # Check that the executable hello file is found in the checkout filename = os.path.join(checkout, "usr", "bin", "hello") assert os.path.exists(filename) filename = os.path.join(checkout, "usr", "include", "pony.h") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_source_artifact_caches(cli, tmpdir, datafiles): cachedir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) element_path = os.path.join(project_dir, "elements") with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cachedir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) create_element_size("repo.bst", project_dir, element_path, [], 10000) res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() assert "Pushed source " in res.stderr assert "Pushed artifact " in res.stderr # delete local sources and artifacts and check it pulls them shutil.rmtree(os.path.join(cachedir, "cas")) shutil.rmtree(os.path.join(cachedir, "sources")) # this should just fetch the artifacts res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() assert "Pulled artifact " in res.stderr assert "Pulled source " not in res.stderr @pytest.mark.datafiles(DATA_DIR) def test_source_cache_empty_artifact_cache(cli, tmpdir, datafiles): cachedir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) element_path = os.path.join(project_dir, "elements") with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "artifacts": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cachedir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) create_element_size("repo.bst", project_dir, element_path, [], 10000) res = cli.run(project=project_dir, args=["source", "push", "repo.bst"]) res.assert_success() assert "Pushed source " in res.stderr # delete local sources and check it pulls sources, builds # and then pushes the artifacts shutil.rmtree(os.path.join(cachedir, "cas")) shutil.rmtree(os.path.join(cachedir, "sources")) res = cli.run(project=project_dir, args=["build", "repo.bst"]) res.assert_success() assert "Remote ({}) does not have artifact ".format(share.repo) in res.stderr assert "Pulled source" in res.stderr assert "Caching artifact" in res.stderr assert "Pushed artifact" in res.stderr apache-buildstream-27ae392/tests/frontend/show.py000066400000000000000000000763051514607367700221500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import sys import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream.types import CoreWarnings from tests.testutils import generate_junction from . import configure_project # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize( "target,fmt,expected", [ ("import-bin.bst", "%{name}", "import-bin.bst"), ("import-bin.bst", "%{state}", "buildable"), ("compose-all.bst", "%{state}", "waiting"), ("import-bin.bst", "%{kind}", "import"), ("compose-all.bst", "%{kind}", "compose"), ], ) def test_show(cli, datafiles, target, fmt, expected): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", fmt, target]) result.assert_success() if result.output.strip() != expected: raise AssertionError("Expected output:\n{}\nInstead received output:\n{}".format(expected, result.output)) @pytest.mark.datafiles( os.path.join( os.path.dirname(os.path.realpath(__file__)), "invalid_element_path", ) ) def test_show_invalid_element_path(cli, datafiles): project = str(datafiles) cli.run(project=project, silent=True, args=["show", "foo.bst"]) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project_fail")) def test_show_fail(cli, datafiles): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Test behaviors of user supplied glob patterns @pytest.mark.datafiles(os.path.join(DATA_DIR, "simple")) @pytest.mark.parametrize( "pattern,expected_elements", [ # Only bst files, same as "**" for `bst show` # ("**.bst", ["import-bin.bst", "import-dev.bst", "compose-all.bst", "target.bst", "subdir/target.bst"]), # Use regular globbing without matching path separators, this should exclude # the target in the subdirectory. # ("*.bst", ["import-bin.bst", "import-dev.bst", "compose-all.bst", "target.bst"]), # Report only targets in the subdirectory # ("subdir/*.bst", ["subdir/target.bst"]), # Report both targets which end in "target.bst" # ("**target.bst", ["target.bst", "subdir/target.bst"]), # All elements starting with the prefix "import" # ("import*.bst", ["import-bin.bst", "import-dev.bst"]), ], ids=["**.bst", "*.bst", "subdir/*", "**target.bst", "import*"], ) def test_show_glob(cli, tmpdir, datafiles, pattern, expected_elements): project = str(datafiles) result = cli.run(project=project, args=["show", "--deps", "none", "--format", "%{name}", pattern]) result.assert_success() output = result.output.strip().splitlines() # Assert that the number of results match the number of expected results assert len(output) == len(expected_elements) # Assert that each expected result was found. for expected in expected_elements: assert expected in output, "Expected result {} not found".format(expected) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize( "target,except_,expected", [ ("target.bst", "import-bin.bst", ["import-dev.bst", "compose-all.bst", "target.bst"]), ("target.bst", "import-dev.bst", ["import-bin.bst", "compose-all.bst", "target.bst"]), ("target.bst", "compose-all.bst", ["import-bin.bst", "target.bst"]), ("compose-all.bst", "import-bin.bst", ["import-dev.bst", "compose-all.bst"]), ], ) def test_show_except_simple(cli, datafiles, target, except_, expected): project = str(datafiles) result = cli.run( project=project, silent=True, args=["show", "--deps", "all", "--format", "%{name}", "--except", except_, target], ) result.assert_success() results = result.output.strip().splitlines() if results != expected: raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}".format(expected, results)) # This test checks various constructions of a pipeline # with one or more targets and 0 or more exception elements, # each data set provides the targets, exceptions and expected # result list. # @pytest.mark.datafiles(os.path.join(DATA_DIR, "exceptions")) @pytest.mark.parametrize( "targets,exceptions,expected", [ # Test without exceptions, lets just see the whole list here ( ["build.bst"], None, [ "fourth-level-1.bst", "third-level-1.bst", "fourth-level-2.bst", "third-level-2.bst", "fourth-level-3.bst", "third-level-3.bst", "second-level-1.bst", "first-level-1.bst", "first-level-2.bst", "build.bst", ], ), # Test one target and excepting a part of the pipeline, this # removes forth-level-1 and third-level-1 ( ["build.bst"], ["third-level-1.bst"], [ "fourth-level-2.bst", "third-level-2.bst", "fourth-level-3.bst", "third-level-3.bst", "second-level-1.bst", "first-level-1.bst", "first-level-2.bst", "build.bst", ], ), # Test one target and excepting a part of the pipeline, check that # excepted dependencies remain in the pipeline if depended on from # outside of the except element ( ["build.bst"], ["second-level-1.bst"], [ "fourth-level-2.bst", "third-level-2.bst", # first-level-2 depends on this, so not excepted "first-level-1.bst", "first-level-2.bst", "build.bst", ], ), # The same as the above test, but excluding the toplevel build.bst, # instead only select the two toplevel dependencies as targets ( ["first-level-1.bst", "first-level-2.bst"], ["second-level-1.bst"], [ "fourth-level-2.bst", "third-level-2.bst", # first-level-2 depends on this, so not excepted "first-level-1.bst", "first-level-2.bst", ], ), # Test one target and excepting an element outisde the pipeline ( ["build.bst"], ["unrelated-1.bst"], [ "fourth-level-2.bst", "third-level-2.bst", # first-level-2 depends on this, so not excepted "first-level-1.bst", "first-level-2.bst", "build.bst", ], ), # Test one target and excepting two elements ( ["build.bst"], ["unrelated-1.bst", "unrelated-2.bst"], [ "first-level-1.bst", "build.bst", ], ), ], ) def test_show_except(cli, datafiles, targets, exceptions, expected): basedir = str(datafiles) results = cli.get_pipeline(basedir, targets, except_=exceptions, scope="all") if results != expected: raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}".format(expected, results)) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_target_is_dependency(cli, datafiles): project = str(datafiles) elements = ["multiple_targets/dependency/zebry.bst", "multiple_targets/dependency/horsey.bst"] args = ["show", "-d", "all", "-f", "%{name}", *elements] result = cli.run(project=project, args=args) result.assert_success() # Get the order names = result.output.splitlines() names = [name[len("multiple_targets/dependency/") :] for name in names] assert names == ["pony.bst", "horsey.bst", "zebry.bst"] @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) @pytest.mark.parametrize("element_name", ["junction-dep.bst", "junction.bst:import-etc.bst"]) @pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"]) def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name, workspaced): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it ref = generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == "inline")) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Dump a project.refs if we're using project.refs storage # if ref_storage == "project.refs": project_refs = {"projects": {"test": {"junction.bst": [{"ref": ref}]}}} _yaml.roundtrip_dump(project_refs, os.path.join(project, "junction.refs")) # Open a workspace if we're testing workspaced behavior if workspaced: result = cli.run( project=project, silent=True, args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], ) result.assert_success() # Assert successful bst show (requires implicit subproject fetching) result = cli.run(project=project, silent=True, args=["show", element_name]) result.assert_success() @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) @pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, workspaced): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Open a workspace if we're testing workspaced behavior if workspaced: result = cli.run( project=project, silent=True, args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], ) result.assert_success() # Assert the correct error when trying to show the pipeline dep_result = cli.run(project=project, silent=True, args=["show", "junction-dep.bst"]) # Assert the correct error when trying to show the pipeline etc_result = cli.run(project=project, silent=True, args=["show", "junction.bst:import-etc.bst"]) # If a workspace is open, no ref is needed if workspaced: dep_result.assert_success() etc_result.assert_success() else: # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in dep_result.stderr dep_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) etc_result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize("element_name", ["junction-dep.bst", "junction.bst:import-etc.bst"]) @pytest.mark.parametrize("workspaced", [True, False], ids=["workspace", "no-workspace"]) def test_fetched_junction(cli, tmpdir, datafiles, element_name, workspaced): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) result = cli.run(project=project, silent=True, args=["source", "fetch", "junction.bst"]) result.assert_success() # Open a workspace if we're testing workspaced behavior if workspaced: result = cli.run( project=project, silent=True, args=["workspace", "open", "--no-checkout", "--directory", subproject_path, "junction.bst"], ) result.assert_success() # Assert the correct error when trying to show the pipeline result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}-%{state}", element_name]) results = result.output.strip().splitlines() assert "junction.bst:import-etc.bst-buildable" in results ############################################################### # Testing recursion depth # ############################################################### @pytest.mark.parametrize("dependency_depth", [100, 150, 1200]) def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth): project_name = "recursion-test" path = str(tmpdir) project_path = os.path.join(path, project_name) def setup_test(): """ Creates a bst project with dependencydepth + 1 elements, each of which depends of the previous element to be created. Each element created is of type import and has an empty source file. """ os.mkdir(project_path) result = cli.run(silent=True, args=["init", "--project-name", project_name, project_path]) result.assert_success() sourcefiles_path = os.path.join(project_path, "files") os.mkdir(sourcefiles_path) element_path = os.path.join(project_path, "elements") for i in range(0, dependency_depth + 1): element = { "kind": "import", "sources": [{"kind": "local", "path": "files/source{}".format(str(i))}], "depends": ["element{}.bst".format(str(i - 1))], } if i == 0: del element["depends"] _yaml.roundtrip_dump(element, os.path.join(element_path, "element{}.bst".format(str(i)))) source = os.path.join(sourcefiles_path, "source{}".format(str(i))) open(source, "x", encoding="utf-8").close() # pylint: disable=consider-using-with assert os.path.exists(source) setup_test() result = cli.run(project=project_path, silent=True, args=["show", "element{}.bst".format(str(dependency_depth))]) recursion_limit = sys.getrecursionlimit() if dependency_depth <= recursion_limit: result.assert_success() else: # Assert exception is thown and handled assert not result.unhandled_exception assert result.exit_code == -1 shutil.rmtree(project_path) ############################################################### # Testing format symbols # ############################################################### @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize( "dep_kind, expected_deps", [ ("%{deps}", "- import-dev.bst\n- import-links.bst\n- import-bin.bst"), ("%{build-deps}", "- import-dev.bst\n- import-links.bst"), ("%{runtime-deps}", "- import-links.bst\n- import-bin.bst"), ], ids=["deps", "build-deps", "runtime-deps"], ) def test_format_deps(cli, datafiles, dep_kind, expected_deps): project = str(datafiles) target = "format-deps.bst" result = cli.run(project=project, silent=True, args=["show", "--deps", "none", "--format", dep_kind, target]) result.assert_success() expected = "{deps}".format(deps=expected_deps) if result.output.strip() != expected: raise AssertionError("Expected output:\n{}\nInstead received output:\n{}".format(expected, result.output)) # This tests the resolved value of the 'max-jobs' variable, # ensuring at least that the variables are resolved according # to how the user has configured max-jobs # @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize( "cli_value, config_value", [ (None, None), (None, "16"), ("16", None), ("5", "16"), ("0", "16"), ("16", "0"), ], ) def test_max_jobs(cli, datafiles, cli_value, config_value): project = str(datafiles) target = "target.bst" # Specify `--max-jobs` if this test sets it args = [] if cli_value is not None: args += ["--max-jobs", cli_value] args += ["show", "--deps", "none", "--format", "%{vars}", target] # Specify `max-jobs` in user configuration if this test sets it if config_value is not None: cli.configure({"build": {"max-jobs": config_value}}) result = cli.run(project=project, silent=True, args=args) result.assert_success() loaded = _yaml.load_data(result.output) loaded_value = loaded.get_int("max-jobs") # We expect the value provided on the command line to take # precedence over the configuration file value, if specified. # # If neither are specified then we expect the default expected_value = cli_value or config_value or "0" if expected_value == "0": # If we are expecting the automatic behavior of using the maximum # number of cores available, just check that it is a value > 0 assert loaded_value > 0, "Automatic setting of max-jobs didnt work" else: # Check that we got the explicitly set value assert loaded_value == int(expected_value) # This tests that cache keys behave as expected when # dependencies have been specified as `strict` and # when building in strict mode. # # This test will: # # * Build the target once (and assert that it is cached) # * Modify some local files which are imported # by an import element which the target depends on # * Assert that the cached state of the target element # is as expected # # We run the test twice, once with an element which strict # depends on the changing import element, and one which # depends on it regularly. # @pytest.mark.datafiles(os.path.join(DATA_DIR, "strict-depends")) @pytest.mark.parametrize( "target, expected_state", [ ("non-strict-depends.bst", "cached"), ("strict-depends.bst", "waiting"), ], ) def test_strict_dependencies(cli, datafiles, target, expected_state): project = str(datafiles) # Configure non strict mode, this will have # an effect on the build and the `bst show` # commands run via cli.get_element_states() cli.configure({"projects": {"test": {"strict": False}}}) result = cli.run(project=project, silent=True, args=["build", target]) result.assert_success() states = cli.get_element_states(project, ["base.bst", target]) assert states["base.bst"] == "cached" assert states[target] == "cached" # Now modify the file, effectively causing the common base.bst # dependency to change it's cache key hello_path = os.path.join(project, "files", "hello.txt") with open(hello_path, "w", encoding="utf-8") as f: f.write("Goodbye") # Now assert that we have the states we expect as a result states = cli.get_element_states(project, ["base.bst", target]) assert states["base.bst"] == "buildable" assert states[target] == expected_state @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) @pytest.mark.parametrize("fatal", [True, False], ids=["fatal", "non-fatal"]) def test_unaliased_url(cli, tmpdir, datafiles, fatal): project = str(datafiles) if fatal: configure_project(project, {"fatal-warnings": [CoreWarnings.UNALIASED_URL]}) result = cli.run(project=project, silent=True, args=["show", "unaliased-tar.bst"]) if fatal: result.assert_main_error(ErrorDomain.PLUGIN, CoreWarnings.UNALIASED_URL) else: result.assert_success() assert "WARNING [unaliased-url]" in result.stderr @pytest.mark.datafiles(os.path.join(DATA_DIR, "project")) def test_invalid_alias(cli, tmpdir, datafiles): project = str(datafiles) configure_project(project, {"aliases": {"pony": "https://pony.org/tarballs", "horsy": "http://horsy.tv/shows"}}) result = cli.run(project=project, silent=True, args=["show", "invalid-alias.bst"]) result.assert_main_error(ErrorDomain.SOURCE, "invalid-source-alias") @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) @pytest.mark.parametrize( "target, expected_kind, expected_url, expected_medium, expected_version_type, expected_version, expected_guess_version, expected_homepage, expected_issue_tracker", [ ( "local.bst", "local", "files/testfile", "local", "cas-digest", "9391a5943daf287b46520c4289d41cab5f6b33e643f7661bcf620de7f02c1c9b/82", None, None, None, ), ( "tar.bst", "tar", "https://flying-ponies.com/releases/1.2/pony-flight-1.2.3.tgz", "remote-file", "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "1.2.3", None, None, ), ( "tar-no-micro.bst", "tar", "https://flying-ponies.com/releases/pony-flight-1.2.tgz", "remote-file", "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "1.2", None, None, ), ( "tar-custom-version.bst", "tar", "https://flying-ponies.com/releases/pony_v2_4_93.tgz", "remote-file", "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "2.4.93", None, None, ), ( "tar-explicit.bst", "tar", "https://flying-ponies.com/releases/9d0c936c78/pony-flight-release.tgz", "remote-file", "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "3.2.1", None, None, ), ( "testsource.bst", "testsource", "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", "12", None, None, ), ( "user-provenance.bst", "tar", "https://flying-ponies.com/releases/1.2/pony-flight-1.2.3.tgz", "remote-file", "sha256", "9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501", "1.2.3", "https://flying-ponies.com/index.html", "https://bugs.flying-ponies.com/issues", ), ], ids=[ "local", "tar-full-version", "tar-no-micro", "tar-custom-version", "tar-explicit", "testsource", "user-provenance", ], ) def test_source_info( cli, datafiles, target, expected_url, expected_kind, expected_medium, expected_version_type, expected_version, expected_guess_version, expected_homepage, expected_issue_tracker, ): project = str(datafiles) result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", target]) result.assert_success() loaded = _yaml.load_data(result.output) sources = loaded.get_sequence(target) source_info = sources.mapping_at(0) assert source_info.get_str("kind") == expected_kind assert source_info.get_str("url") == expected_url assert source_info.get_str("medium") == expected_medium assert source_info.get_str("version-type") == expected_version_type assert source_info.get_str("version") == expected_version # Optional fields assert source_info.get_str("version-guess", None) == expected_guess_version assert source_info.get_str("homepage", None) == expected_homepage assert source_info.get_str("issue-tracker", None) == expected_issue_tracker @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) def test_source_info_extra_data(cli, datafiles): project = str(datafiles) result = cli.run( project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "extradata.bst"] ) result.assert_success() loaded = _yaml.load_data(result.output) sources = loaded.get_sequence("extradata.bst") source_info = sources.mapping_at(0) assert source_info.get_str("kind") == "extradata" assert source_info.get_str("url") == "http://ponyfarm.com/ponies" assert source_info.get_str("medium") == "pony-ride" assert source_info.get_str("version-type") == "pony-age" assert source_info.get_str("version") == "1234567" assert source_info.get_str("version-guess", None) == "12" extra_data = source_info.get_mapping("extra-data", None) assert extra_data is not None assert extra_data.get_str("pony", "green") # Test what happens when encountering a source that doesn't implement collect_source_info() # @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info-unimplemented")) @pytest.mark.parametrize( "subdir, expect_fatal", [ ("non-fatal", False), ("fatal", True), ], ids=["non-fatal", "fatal"], ) def test_source_info_unimplemented(cli, datafiles, subdir, expect_fatal): project = os.path.join(str(datafiles), subdir) result = cli.run(project=project, silent=True, args=["show", "--format", "%{source-info}", "unimplemented.bst"]) if expect_fatal: result.assert_main_error(ErrorDomain.PLUGIN, CoreWarnings.UNAVAILABLE_SOURCE_INFO) else: # Assert empty list but no errors for a source not implementing collect_source_info() # # Note that buildstream internal _yaml doesn't support loading a list as a toplevel element # in the stream, so we just assert the string instead. result.assert_success() assert result.output == "[]\n\n" assert "WARNING [unavailable-source-info]" in result.stderr # This checks how Source.collect_source_info() works on a workspace, # in order to do this we use a workspace *on a tarball* which we generate, # so that we avoid the weird case of opening a workspace on a local source. # @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) def test_source_info_workspace(cli, datafiles, tmpdir): project = str(datafiles) element_dir = os.path.join(project, "elements") workspace_path = os.path.join(project, "workspace") # Create a tar repo repo = create_repo("tar", str(tmpdir)) repo.create(os.path.join(str(datafiles), "files")) # Generate an import element for this tarball input_config = { "kind": "import", "sources": [repo.source_config()], } input_file = os.path.join(element_dir, "test-workspace.bst") _yaml.roundtrip_dump(input_config, input_file) # Track the element so that there is a ref (required for workspace checkout) result = cli.run( project=project, silent=True, args=["source", "track", "test-workspace.bst"], ) result.assert_success() # Open a workspace on the tarball result = cli.run( project=project, silent=True, args=["workspace", "open", "--directory", workspace_path, "test-workspace.bst"], ) result.assert_success() # Lets see what happens result = cli.run( project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "test-workspace.bst"] ) result.assert_success() # Lets check the results loaded = _yaml.load_data(result.output) sources = loaded.get_sequence("test-workspace.bst") source_info = sources.mapping_at(0) # The URL is a local path for a workspace, so we just check it exists assert source_info.get_str("url") is not None assert source_info.get_str("medium") == "workspace" assert source_info.get_str("version-type") == "cas-digest" # Tarball repo generation is not deterministic, so we just assert that there is a digest assert source_info.get_str("version", None) is not None # There is no version guessing for a workspace assert source_info.get_str("version-guess", None) is None @pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) def test_multi_source_info(cli, datafiles): project = str(datafiles) result = cli.run( project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "multisource.bst"] ) result.assert_success() loaded = _yaml.load_data(result.output) sources = loaded.get_sequence("multisource.bst") source_info = sources.mapping_at(0) assert source_info.get_str("kind") == "multisource" assert source_info.get_str("url") == "http://ponyfarm.com/ponies" assert source_info.get_str("medium") == "pony-ride" assert source_info.get_str("version-type") == "pony-age" assert source_info.get_str("version") == "1234567" assert source_info.get_str("version-guess", None) == "12" assert "homepage" not in source_info source_info = sources.mapping_at(1) assert source_info.get_str("kind") == "multisource" assert source_info.get_str("url") == "http://ponyfarm.com/happy" assert source_info.get_str("medium") == "pony-ride" assert source_info.get_str("version-type") == "pony-age" assert source_info.get_str("version") == "1234567" assert source_info.get_str("version-guess", None) == "12" assert source_info.get_str("homepage") == "http://happy.ponyfarm.com" apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest.py000066400000000000000000000202141514607367700261760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils import ( create_artifact_share, assert_shared, assert_not_shared, ) DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "show_artifact_cas_digest_project") # This tests that a target that hasn't been built locally # and that isn't cached remotely has not artifact CAS # digest. # # The test is performed without a remote cache. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target", [ "import-basic-files.bst", "import-executable-files.bst", "import-symlinks.bst", ], ids=["basic-files", "executable-files", "symlinks"], ) def test_show_artifact_cas_digest_uncached(cli, tmpdir, datafiles, target): project = str(datafiles) expected_no_digest = "" # Check the target has not been built locally and is not existing in the remote cache assert ( # May be "buildable" or "waiting" but shouldn't be "cached" cli.get_element_state(project, target) != "cached" ) # Check the target has no artifact digest result = cli.run(project=project, silent=True, args=["show", "--format", "%{name},%{artifact-cas-digest}", target]) result.assert_success() received_digest = result.output.splitlines()[0] assert received_digest == "{target},{digest}".format(target=target, digest=expected_no_digest) # This tests that a target that has been built locally and # with no remote has an artifact CAS digest. # # The test is performed without a remote cache. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target, expected_digest", [ ("import-basic-files.bst", "7093d3c89029932ce1518bd2192e1d3cf60fd88e356b39195d10b87b598c78f0/168"), ("import-executable-files.bst", "133a9ae2eda30945a363272ac14bb2c8a941770b5a37c2847c99934f2972ce4f/170"), ("import-symlinks.bst", "95947ea55021e26cec4fd4f9de90d2b7f4f7d803ccc91656b3e1f2c9923ddf19/131"), ], ids=["basic-files", "executable-files", "symlinks"], ) def test_show_artifact_cas_digest_cached(cli, tmpdir, datafiles, target, expected_digest): project = str(datafiles) # Build the target locally result = cli.run(project=project, silent=True, args=["build", target]) result.assert_success() # Check the target has been built locally and is existing in the remote cache assert cli.get_element_state(project, target) == "cached" # Check the target has an artifact digest result = cli.run(project=project, silent=True, args=["show", "--format", "%{name},%{artifact-cas-digest}", target]) result.assert_success() received_digest = result.output.splitlines()[0] assert received_digest == "{target},{digest}".format(target=target, digest=expected_digest) # This tests that an import element which produces the same content # as it's dependency, has the same CAS digest as the dependency. # # This is tested to ensure that we are indeed producing the content hash # of the files portion of the artifact, and that they are indeed a match. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "target, expected_digests", [ ( "dependencies.bst", { # Note that these expect exactly the same cas digest ! "dependencies.bst": "7093d3c89029932ce1518bd2192e1d3cf60fd88e356b39195d10b87b598c78f0/168", "import-basic-files.bst": "7093d3c89029932ce1518bd2192e1d3cf60fd88e356b39195d10b87b598c78f0/168", }, ), ], ids=["dependencies"], ) def test_show_artifact_cas_digest_dependencies(cli, tmpdir, datafiles, target, expected_digests): project = str(datafiles) # Build the target and its dependencies locally result = cli.run(project=project, silent=True, args=["build", target]) result.assert_success() # Check the target and its dependencies have the same CAS digest result = cli.run(project=project, silent=True, args=["show", "--format", "%{name},%{artifact-cas-digest}", target]) result.assert_success() digests = dict(line.split(",", 2) for line in result.output.splitlines()) assert len(digests) == len(expected_digests) for component, received in sorted(digests.items()): assert received == expected_digests[component] # This tests: # - that a target that hasn't been built locally and that # isn't cached remotely has no artifact CAS digest, # - that a target that has been built locally and that is # cached remotely has an artifact CAS digest, # - that a target that hasn't been built locally and that # is cached remotely has no artifact CAS digest. # # The test is performed with a remote cache, multiple tests # are performed at once and on a single element because # setting up a share is expensive. # @pytest.mark.datafiles(DATA_DIR) def test_show_artifact_cas_digest_remote(cli, tmpdir, datafiles): project = str(datafiles) target = "import-basic-files.bst" expected_no_digest = "" # Configure a local cache local_cache = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": local_cache}) with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Test a target cached neither locally or remotely has no digest # Check the target has not been built locally and is not existing in the remote cache assert cli.get_element_state(project, target) == "buildable" assert_not_shared(cli, share, project, target) # Check the target has no artifact digest result = cli.run( project=project, silent=True, args=["show", "--format", "%{name},%{artifact-cas-digest}", target] ) result.assert_success() received_digest = result.output.splitlines()[0] assert received_digest == "{target},{digest}".format(target=target, digest=expected_no_digest) # Test a target cached locally has a digest # Build the target locally and cache it remotely result = cli.run(project=project, silent=True, args=["build", target]) result.assert_success() # Check the target has been built and shared assert cli.get_element_state(project, target) == "cached" assert_shared(cli, share, project, target) # Check the target has an artifact digest result = cli.run( project=project, silent=True, args=["show", "--format", "%{name},%{artifact-cas-digest}", target] ) result.assert_success() received_digest = result.output.splitlines()[0] assert received_digest != "{target},{digest}".format(target=target, digest=expected_no_digest) # Test a target cached remotely but not locally has no digest # Delete the locally cached target result = cli.run(project=project, silent=True, args=["artifact", "delete", target]) result.assert_success() # Check the target has been deleted locally but not remotely assert cli.get_element_state(project, target) == "buildable" assert_shared(cli, share, project, target) # Check the target has an artifact digest result = cli.run( project=project, silent=True, args=["show", "--format", "%{name},%{artifact-cas-digest}", target] ) result.assert_success() received_digest = result.output.splitlines()[0] assert received_digest == "{target},{digest}".format(target=target, digest=expected_no_digest) apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/000077500000000000000000000000001514607367700273535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/elements/000077500000000000000000000000001514607367700311675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/elements/dependencies.bst000066400000000000000000000001401514607367700343220ustar00rootroot00000000000000kind: import sources: - kind: local path: files/basic-files depends: - import-basic-files.bst import-basic-files.bst000066400000000000000000000000761514607367700353160ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/elementskind: import sources: - kind: local path: files/basic-files import-executable-files.bst000066400000000000000000000001031514607367700363450ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/elementskind: import sources: - kind: local path: files/executable-files import-symlinks.bst000066400000000000000000000000731514607367700350030ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/elementskind: import sources: - kind: local path: files/symlinks apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/000077500000000000000000000000001514607367700304555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/basic-files/000077500000000000000000000000001514607367700326365ustar00rootroot00000000000000basicfile000066400000000000000000000000161514607367700344200ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/basic-filesfile contents basicfolder/000077500000000000000000000000001514607367700350345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/basic-filessubdir-file000066400000000000000000000000001514607367700371520ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/basic-files/basicfolderapache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/executable-files/000077500000000000000000000000001514607367700336765ustar00rootroot00000000000000basicfile000077500000000000000000000000161514607367700354630ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/executable-filesfile contents basicfolder/000077500000000000000000000000001514607367700360745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/executable-filessubdir-file000077500000000000000000000000001514607367700402150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/executable-files/basicfolderapache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/symlinks/000077500000000000000000000000001514607367700323265ustar00rootroot00000000000000broken-symlink000077700000000000000000000000001514607367700374422missing-fileustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/symlinksapache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/symlinks/symlink000077700000000000000000000000001514607367700351432targetustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/files/symlinks/target000066400000000000000000000000071514607367700335340ustar00rootroot00000000000000target apache-buildstream-27ae392/tests/frontend/show_artifact_cas_digest_project/project.conf000066400000000000000000000002461514607367700316720ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/simple/000077500000000000000000000000001514607367700220745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/elements/000077500000000000000000000000001514607367700237105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/elements/compose-all.bst000066400000000000000000000003441514607367700266360ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/frontend/simple/elements/import-bin.bst000066400000000000000000000000741514607367700265030ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/frontend/simple/elements/import-dev.bst000066400000000000000000000000741514607367700265110ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/frontend/simple/elements/subdir/000077500000000000000000000000001514607367700252005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/elements/subdir/target.bst000066400000000000000000000001321514607367700271740ustar00rootroot00000000000000kind: stack description: | Another target in a subdirectory depends: - import-dev.bst apache-buildstream-27ae392/tests/frontend/simple/elements/target.bst000066400000000000000000000001641514607367700257110ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - compose-all.bst apache-buildstream-27ae392/tests/frontend/simple/files/000077500000000000000000000000001514607367700231765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/bin-files/000077500000000000000000000000001514607367700250465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/bin-files/usr/000077500000000000000000000000001514607367700256575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/bin-files/usr/bin/000077500000000000000000000000001514607367700264275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700274550ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/frontend/simple/files/dev-files/000077500000000000000000000000001514607367700250545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/dev-files/usr/000077500000000000000000000000001514607367700256655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/dev-files/usr/include/000077500000000000000000000000001514607367700273105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/simple/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700304470ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/simple/project.conf000066400000000000000000000001341514607367700244070ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/frontend/source-fetch/000077500000000000000000000000001514607367700231725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-fetch/apples.bst000066400000000000000000000002201514607367700251620ustar00rootroot00000000000000kind: import sources: - kind: remote url: project-root:/files/apples ref: 39ac5969800f779603237d01dc0be28dd795a33c591af6c7736264482265930a apache-buildstream-27ae392/tests/frontend/source-fetch/bananas.bst000066400000000000000000000003161514607367700253070ustar00rootroot00000000000000kind: import build-depends: - apples.bst runtime-depends: - oranges.bst sources: - kind: remote url: project-root:/files/bananas ref: e49295702f7da8670778e9b95a281b72b41b31cb16afa376034b45f59a18ea3f apache-buildstream-27ae392/tests/frontend/source-fetch/files/000077500000000000000000000000001514607367700242745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-fetch/files/apples000066400000000000000000000000071514607367700255000ustar00rootroot00000000000000apples apache-buildstream-27ae392/tests/frontend/source-fetch/files/bananas000066400000000000000000000000101514607367700256110ustar00rootroot00000000000000bananas apache-buildstream-27ae392/tests/frontend/source-fetch/files/oranges000066400000000000000000000000101514607367700256440ustar00rootroot00000000000000oranges apache-buildstream-27ae392/tests/frontend/source-fetch/oranges.bst000066400000000000000000000002211514607367700253350ustar00rootroot00000000000000kind: import sources: - kind: remote url: project-root:/files/oranges ref: b19fe3ce519db5b79021aed3e83d9533631d69b15813f9e198a4ba4a3107deaa apache-buildstream-27ae392/tests/frontend/source-fetch/project.conf000066400000000000000000000001141514607367700255030ustar00rootroot00000000000000name: test min-version: 2.0 aliases: project-root: "%{project-root-uri}" apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/000077500000000000000000000000001514607367700257005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/fatal/000077500000000000000000000000001514607367700267675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/fatal/elements/000077500000000000000000000000001514607367700306035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/fatal/elements/unimplemented.bst000066400000000000000000000000631514607367700341620ustar00rootroot00000000000000kind: import sources: - kind: fatal_unimplemented apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/fatal/plugins/000077500000000000000000000000001514607367700304505ustar00rootroot00000000000000fatal_unimplemented.py000066400000000000000000000007211514607367700347600ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/fatal/pluginsfrom buildstream import Source class Unimplemented(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False # Plugin entry point def setup(): return Unimplemented apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/fatal/project.conf000066400000000000000000000003321514607367700313020ustar00rootroot00000000000000# Project config for bst show source-info test name: test min-version: 2.0 element-path: elements plugins: - origin: local path: plugins sources: - fatal_unimplemented fatal-warnings: - unavailable-source-info apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/non-fatal/000077500000000000000000000000001514607367700275575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/non-fatal/elements/000077500000000000000000000000001514607367700313735ustar00rootroot00000000000000unimplemented.bst000066400000000000000000000000671514607367700346770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/non-fatal/elementskind: import sources: - kind: non_fatal_unimplemented apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/non-fatal/plugins/000077500000000000000000000000001514607367700312405ustar00rootroot00000000000000non_fatal_unimplemented.py000066400000000000000000000007211514607367700364220ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/non-fatal/pluginsfrom buildstream import Source class Unimplemented(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False # Plugin entry point def setup(): return Unimplemented apache-buildstream-27ae392/tests/frontend/source-info-unimplemented/non-fatal/project.conf000066400000000000000000000002631514607367700320750ustar00rootroot00000000000000# Project config for bst show source-info test name: test min-version: 2.0 element-path: elements plugins: - origin: local path: plugins sources: - non_fatal_unimplemented apache-buildstream-27ae392/tests/frontend/source-info/000077500000000000000000000000001514607367700230345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info/elements/000077500000000000000000000000001514607367700246505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info/elements/extradata.bst000066400000000000000000000000511514607367700273330ustar00rootroot00000000000000kind: import sources: - kind: extradata apache-buildstream-27ae392/tests/frontend/source-info/elements/local.bst000066400000000000000000000000741514607367700264550ustar00rootroot00000000000000kind: import sources: - kind: local path: files/testfile apache-buildstream-27ae392/tests/frontend/source-info/elements/multisource.bst000066400000000000000000000000531514607367700277330ustar00rootroot00000000000000kind: import sources: - kind: multisource apache-buildstream-27ae392/tests/frontend/source-info/elements/tar-custom-version.bst000066400000000000000000000003301514607367700311370ustar00rootroot00000000000000kind: import sources: - kind: tar version-guess-pattern: >- (\d+)_(\d+)_(\d+) url: https://flying-ponies.com/releases/pony_v2_4_93.tgz ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 apache-buildstream-27ae392/tests/frontend/source-info/elements/tar-explicit.bst000066400000000000000000000003111514607367700277620ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://flying-ponies.com/releases/9d0c936c78/pony-flight-release.tgz ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 version: 3.2.1 apache-buildstream-27ae392/tests/frontend/source-info/elements/tar-no-micro.bst000066400000000000000000000002511514607367700276670ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://flying-ponies.com/releases/pony-flight-1.2.tgz ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 apache-buildstream-27ae392/tests/frontend/source-info/elements/tar.bst000066400000000000000000000002571514607367700261540ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://flying-ponies.com/releases/1.2/pony-flight-1.2.3.tgz ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 apache-buildstream-27ae392/tests/frontend/source-info/elements/testsource.bst000066400000000000000000000000521514607367700275570ustar00rootroot00000000000000kind: import sources: - kind: testsource apache-buildstream-27ae392/tests/frontend/source-info/elements/user-provenance.bst000066400000000000000000000004511514607367700304760ustar00rootroot00000000000000kind: import sources: - kind: tar url: https://flying-ponies.com/releases/1.2/pony-flight-1.2.3.tgz ref: 9d0c936c78d0dfe3a67cae372c9a2330476ea87a2eec16b2daada64a664ca501 provenance: homepage: https://flying-ponies.com/index.html issue-tracker: https://bugs.flying-ponies.com/issues apache-buildstream-27ae392/tests/frontend/source-info/files/000077500000000000000000000000001514607367700241365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info/files/testfile000066400000000000000000000000711514607367700256760ustar00rootroot00000000000000The perverted pink pony mounts the unwilling hippopotamusapache-buildstream-27ae392/tests/frontend/source-info/plugins/000077500000000000000000000000001514607367700245155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-info/plugins/extradata.py000066400000000000000000000014171514607367700270470ustar00rootroot00000000000000from buildstream import Source class ExtraData(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False def collect_source_info(self): return [ self.create_source_info( "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12", extra_data={"pony": "green"}, ) ] # Plugin entry point def setup(): return ExtraData apache-buildstream-27ae392/tests/frontend/source-info/plugins/multisource.py000066400000000000000000000020101514607367700274330ustar00rootroot00000000000000from buildstream import Node, Source class MultiSource(Source): BST_MIN_VERSION = "2.0" BST_CUSTOM_SOURCE_PROVENANCE = True def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False def collect_source_info(self): return [ self.create_source_info( "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12" ), self.create_source_info( "http://ponyfarm.com/happy", "pony-ride", "pony-age", "1234567", version_guess="12", provenance_node=Node.from_dict({"homepage": "http://happy.ponyfarm.com"}), ), ] # Plugin entry point def setup(): return MultiSource apache-buildstream-27ae392/tests/frontend/source-info/plugins/testsource.py000066400000000000000000000012321514607367700272650ustar00rootroot00000000000000from buildstream import Source class Sample(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False def collect_source_info(self): return [ self.create_source_info( "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12" ) ] # Plugin entry point def setup(): return Sample apache-buildstream-27ae392/tests/frontend/source-info/project.conf000066400000000000000000000004261514607367700253530ustar00rootroot00000000000000# Project config for bst show source-info test name: test min-version: 2.0 element-path: elements plugins: - origin: pip package-name: sample-plugins sources: - git - origin: local path: plugins sources: - extradata - multisource - testsource - unimplemented apache-buildstream-27ae392/tests/frontend/source-track/000077500000000000000000000000001514607367700232055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-track/apples.bst000066400000000000000000000001101514607367700251730ustar00rootroot00000000000000kind: import sources: - kind: remote url: project-root:/files/apples apache-buildstream-27ae392/tests/frontend/source-track/bananas.bst000066400000000000000000000002061514607367700253200ustar00rootroot00000000000000kind: import build-depends: - apples.bst runtime-depends: - oranges.bst sources: - kind: remote url: project-root:/files/bananas apache-buildstream-27ae392/tests/frontend/source-track/comments.bst000066400000000000000000000003401514607367700255410ustar00rootroot00000000000000kind: manual config: build-commands: (<): # This is a multi-line comment. # It should have no effect and should be ignored safely. - | true sources: - kind: remote url: project-root:/files/apples apache-buildstream-27ae392/tests/frontend/source-track/files/000077500000000000000000000000001514607367700243075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/source-track/files/apples000066400000000000000000000000071514607367700255130ustar00rootroot00000000000000apples apache-buildstream-27ae392/tests/frontend/source-track/files/bananas000066400000000000000000000000101514607367700256240ustar00rootroot00000000000000bananas apache-buildstream-27ae392/tests/frontend/source-track/files/oranges000066400000000000000000000000101514607367700256570ustar00rootroot00000000000000oranges apache-buildstream-27ae392/tests/frontend/source-track/oranges.bst000066400000000000000000000001111514607367700253460ustar00rootroot00000000000000kind: import sources: - kind: remote url: project-root:/files/oranges apache-buildstream-27ae392/tests/frontend/source_checkout.py000066400000000000000000000232151514607367700243450ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import tarfile import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream import utils, _yaml # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) def generate_remote_import_element(input_path, output_path): return { "kind": "import", "sources": [ { "kind": "remote", "url": "file://{}".format(input_path), "filename": output_path, "ref": utils.sha256sum(input_path), } ], } @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "with_workspace,guess_element", [(True, True), (True, False), (False, False)], ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"], ) def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace, guess_element): tmpdir = tmpdir_factory.mktemp(os.path.basename(__file__)) project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") target = "checkout-deps.bst" workspace = os.path.join(str(tmpdir), "workspace") elm_cmd = [target] if not guess_element else [] if with_workspace: ws_cmd = ["-C", workspace] result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target]) result.assert_success() else: ws_cmd = [] args = ws_cmd + ["source", "checkout", "--deps", "none", "--directory", checkout, *elm_cmd] result = cli.run(project=project, args=args) result.assert_success() assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("force_flag", ["--force", "-f"]) def test_source_checkout_force(datafiles, cli, force_flag): project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") target = "checkout-deps.bst" # Make the checkout directory with 'some-thing' inside it os.makedirs(os.path.join(checkout, "some-thing")) result = cli.run( project=project, args=["source", "checkout", force_flag, "--deps", "none", "--directory", checkout, target] ) result.assert_success() assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_tar(datafiles, cli): project = str(datafiles) tar = os.path.join(cli.directory, "source-checkout.tar") target = "checkout-deps.bst" result = cli.run(project=project, args=["source", "checkout", "--tar", tar, "--deps", "none", target]) result.assert_success() assert os.path.exists(tar) with tarfile.open(tar) as tf: expected_content = os.path.join(tar, "checkout-deps", "etc", "buildstream", "config") tar_members = [f.name for f in tf] for member in tar_members: assert member in expected_content @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("compression", [("gz"), ("xz"), ("bz2")]) def test_source_checkout_compressed_tar(datafiles, cli, compression): project = str(datafiles) tarfile_name = "source-checkout.tar" + compression tar = os.path.join(cli.directory, tarfile_name) target = "checkout-deps.bst" result = cli.run( project=project, args=["source", "checkout", "--tar", tar, "--compression", compression, "--deps", "none", target], ) result.assert_success() with tarfile.open(name=tar, mode="r:" + compression) as tar: assert os.path.join("checkout-deps", "etc", "buildstream", "config") in tar.getnames() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("deps", [("build"), ("none"), ("run"), ("all")]) def test_source_checkout_deps(datafiles, cli, deps): project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") target = "checkout-deps.bst" result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, "--deps", deps, target]) result.assert_success() # Sources of the target if deps == "build": assert not os.path.exists(os.path.join(checkout, "checkout-deps")) else: assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) # Sources of the target's build dependencies if deps in ("build", "all"): assert os.path.exists(os.path.join(checkout, "import-dev", "usr", "include", "pony.h")) else: assert not os.path.exists(os.path.join(checkout, "import-dev")) # Sources of the target's runtime dependencies if deps in ("run", "all"): assert os.path.exists(os.path.join(checkout, "import-bin", "usr", "bin", "hello")) else: assert not os.path.exists(os.path.join(checkout, "import-bin")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_except(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") target = "checkout-deps.bst" result = cli.run( project=project, args=["source", "checkout", "--directory", checkout, "--deps", "all", "--except", "import-bin.bst", target], ) result.assert_success() # Sources for the target should be present assert os.path.exists(os.path.join(checkout, "checkout-deps", "etc", "buildstream", "config")) # Sources for import-bin.bst should not be present assert not os.path.exists(os.path.join(checkout, "import-bin")) # Sources for other dependencies should be present assert os.path.exists(os.path.join(checkout, "import-dev", "usr", "include", "pony.h")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_fetch(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") target = "remote-import-dev.bst" target_path = os.path.join(project, "elements", target) # Create an element with remote source element = generate_remote_import_element( os.path.join(project, "files", "dev-files", "usr", "include", "pony.h"), "pony.h" ) _yaml.roundtrip_dump(element, target_path) # Testing implicit fetching requires that we do not have the sources # cached already assert cli.get_element_state(project, target) == "fetch needed" args = ["source", "checkout"] args += [target, checkout] result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, target]) result.assert_success() assert os.path.exists(os.path.join(checkout, "remote-import-dev", "pony.h")) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_build_scripts(cli, tmpdir, datafiles): project_path = str(datafiles) element_name = "source-bundle/source-bundle-hello.bst" normal_name = "source-bundle-source-bundle-hello" checkout = os.path.join(str(tmpdir), "source-checkout") args = ["source", "checkout", "--include-build-scripts", "--directory", checkout, element_name] result = cli.run(project=project_path, args=args) result.assert_success() # There sould be a script for each element (just one in this case) and a top level build script expected_scripts = ["build.sh", "build-" + normal_name] for script in expected_scripts: assert script in os.listdir(checkout) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles): project_path = str(datafiles) element_name = "source-bundle/source-bundle-hello.bst" normal_name = "source-bundle-source-bundle-hello" tar_file = os.path.join(str(tmpdir), "source-checkout.tar") args = ["source", "checkout", "--include-build-scripts", "--tar", tar_file, element_name] result = cli.run(project=project_path, args=args) result.assert_success() expected_scripts = ["build.sh", "build-" + normal_name] with tarfile.open(tar_file, "r") as tf: for script in expected_scripts: assert script in tf.getnames() # Test that the --directory and --tar options conflict @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_options_tar_and_dir_conflict(cli, tmpdir, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") tar_file = os.path.join(str(tmpdir), "source-checkout.tar") target = "checkout-deps.bst" result = cli.run(project=project, args=["source", "checkout", "--directory", checkout, "--tar", tar_file, target]) assert result.exit_code != 0 assert "ERROR: options --directory and --tar conflict" in result.stderr # Test that the --compression option without --tar fails @pytest.mark.datafiles(DATA_DIR) def test_source_checkout_compression_without_tar(cli, tmpdir, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "source-checkout") target = "checkout-deps.bst" result = cli.run( project=project, args=["source", "checkout", "--directory", checkout, "--compression", "xz", target] ) assert result.exit_code != 0 assert "ERROR: --compression specified without --tar" in result.stderr apache-buildstream-27ae392/tests/frontend/strict-depends/000077500000000000000000000000001514607367700235335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/strict-depends/elements/000077500000000000000000000000001514607367700253475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/strict-depends/elements/base.bst000066400000000000000000000000631514607367700267720ustar00rootroot00000000000000kind: import sources: - kind: local path: files apache-buildstream-27ae392/tests/frontend/strict-depends/elements/non-strict-depends.bst000066400000000000000000000000411514607367700315740ustar00rootroot00000000000000kind: stack depends: - base.bst apache-buildstream-27ae392/tests/frontend/strict-depends/elements/strict-depends.bst000066400000000000000000000000721514607367700310100ustar00rootroot00000000000000kind: stack depends: - filename: base.bst strict: true apache-buildstream-27ae392/tests/frontend/strict-depends/files/000077500000000000000000000000001514607367700246355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/strict-depends/files/hello.txt000066400000000000000000000000051514607367700264740ustar00rootroot00000000000000pony apache-buildstream-27ae392/tests/frontend/strict-depends/project.conf000066400000000000000000000000631514607367700260470ustar00rootroot00000000000000name: test element-path: elements min-version: 2.0 apache-buildstream-27ae392/tests/frontend/strict-scenario/000077500000000000000000000000001514607367700237145ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/strict-scenario/elements/000077500000000000000000000000001514607367700255305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/strict-scenario/elements/base.bst000066400000000000000000000000741514607367700271550ustar00rootroot00000000000000kind: import sources: - kind: local path: files/base.txt apache-buildstream-27ae392/tests/frontend/strict-scenario/elements/target.bst000066400000000000000000000001541514607367700275300ustar00rootroot00000000000000kind: import depends: - filename: base.bst strict: true sources: - kind: local path: files/target.txt apache-buildstream-27ae392/tests/frontend/strict-scenario/files/000077500000000000000000000000001514607367700250165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/strict-scenario/files/base.txt000066400000000000000000000000051514607367700264640ustar00rootroot00000000000000pony apache-buildstream-27ae392/tests/frontend/strict-scenario/files/target.txt000066400000000000000000000000061514607367700270410ustar00rootroot00000000000000horsy apache-buildstream-27ae392/tests/frontend/strict-scenario/project.conf000066400000000000000000000000631514607367700262300ustar00rootroot00000000000000name: test element-path: elements min-version: 2.0 apache-buildstream-27ae392/tests/frontend/track-cross-junction/000077500000000000000000000000001514607367700246655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-cross-junction/files/000077500000000000000000000000001514607367700257675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-cross-junction/files/usr/000077500000000000000000000000001514607367700266005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-cross-junction/files/usr/include/000077500000000000000000000000001514607367700302235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-cross-junction/files/usr/include/pony.h000066400000000000000000000003711514607367700313620ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/track-cross-junction/subproject-junction.bst000066400000000000000000000000711514607367700314040ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/frontend/track-cross-junction/subproject.bst000066400000000000000000000001241514607367700275540ustar00rootroot00000000000000kind: stack depends: - filename: subtarget.bst junction: subproject-junction.bst apache-buildstream-27ae392/tests/frontend/track-cross-junction/subproject/000077500000000000000000000000001514607367700270455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-cross-junction/subproject/project.conf000066400000000000000000000001511514607367700313570ustar00rootroot00000000000000name: subtest min-version: 2.0 plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/track-optional-inline/000077500000000000000000000000001514607367700250065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-inline/files/000077500000000000000000000000001514607367700261105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-inline/files/usr/000077500000000000000000000000001514607367700267215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-inline/files/usr/include/000077500000000000000000000000001514607367700303445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-inline/files/usr/include/pony.h000066400000000000000000000003711514607367700315030ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/track-optional-inline/project.conf000066400000000000000000000002671514607367700273300ustar00rootroot00000000000000name: test min-version: 2.0 options: test: type: bool description: Test boolean default: False plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/track-optional-inline/target.bst000066400000000000000000000003361514607367700270100ustar00rootroot00000000000000# Optionally track the test branch # kind: import (?): - test: sources: - kind: git url: file://{repo} track: test - not test: sources: - kind: git url: file://{repo} track: master apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/000077500000000000000000000000001514607367700261335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/files/000077500000000000000000000000001514607367700272355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/files/usr/000077500000000000000000000000001514607367700300465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/files/usr/include/000077500000000000000000000000001514607367700314715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/files/usr/include/pony.h000066400000000000000000000003711514607367700326300ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/project.conf000066400000000000000000000003231514607367700304460ustar00rootroot00000000000000name: test min-version: 2.0 ref-storage: project.refs options: test: type: bool description: Test boolean default: False plugins: - origin: pip package-name: sample-plugins sources: - git apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/project.refs000066400000000000000000000002031514607367700304550ustar00rootroot00000000000000projects: test: (?): - test: target.bst: - ref: '' - not test: target.bst: - ref: '' apache-buildstream-27ae392/tests/frontend/track-optional-project-refs/target.bst000066400000000000000000000002421514607367700301310ustar00rootroot00000000000000# Optionally track the test branch # kind: import sources: - kind: git url: file://{repo} (?): - test: track: test - not test: track: master apache-buildstream-27ae392/tests/frontend/track.py000066400000000000000000000512701514607367700222660ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import stat import os import re import pytest from buildstream._testing import create_repo, generate_project from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing._utils.site import have_subsecond_mtime from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream import _yaml from tests.testutils import generate_junction from tests.testutils.repo.git import Git from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON from . import configure_project # Project directory TOP_DIR = os.path.dirname(os.path.realpath(__file__)) DATA_DIR = os.path.join(TOP_DIR, "project") def generate_element(repo, element_path, dep_name=None): element = {"kind": "import", "sources": [repo.source_config()]} if dep_name: element["depends"] = [dep_name] _yaml.roundtrip_dump(element, element_path) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_track_single(cli, tmpdir, datafiles): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_dep_name = "track-test-dep.bst" element_target_name = "track-test-target.bst" # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = Git(str(tmpdir)) repo.create(dev_files_path) # Write out our test targets generate_element(repo, os.path.join(element_path, element_dep_name)) generate_element(repo, os.path.join(element_path, element_target_name), dep_name=element_dep_name) # Assert that tracking is needed for both elements states = cli.get_element_states(project, [element_target_name]) assert states == { element_dep_name: "no reference", element_target_name: "no reference", } # Now first try to track only one element result = cli.run(project=project, args=["source", "track", "--deps", "none", element_target_name]) result.assert_success() # And now fetch it result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_target_name]) result.assert_success() # Assert that the dependency is waiting and the target has still never been tracked states = cli.get_element_states(project, [element_target_name]) assert states == { element_dep_name: "no reference", element_target_name: "waiting", } @pytest.mark.datafiles(os.path.join(TOP_DIR)) @pytest.mark.parametrize("ref_storage", [("inline"), ("project-refs")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_track_optional(cli, tmpdir, datafiles, ref_storage): project = os.path.join(datafiles, "track-optional-" + ref_storage) dev_files_path = os.path.join(project, "files") element_path = os.path.join(project, "target.bst") # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = Git(str(tmpdir)) repo.create(dev_files_path) # Now create an optional test branch and add a commit to that, # so two branches with different heads now exist. # repo.branch("test") repo.add_commit() # Substitute the {repo} for the git repo we created with open(element_path, encoding="utf-8") as f: target_bst = f.read() target_bst = target_bst.format(repo=repo.repo) with open(element_path, "w", encoding="utf-8") as f: f.write(target_bst) # First track for both options # # We want to track and persist the ref separately in this test # result = cli.run(project=project, args=["--option", "test", "False", "source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["--option", "test", "True", "source", "track", "target.bst"]) result.assert_success() # Now fetch the key for both options # result = cli.run( project=project, args=["--option", "test", "False", "show", "--deps", "none", "--format", "%{key}", "target.bst"], ) result.assert_success() master_key = result.output result = cli.run( project=project, args=["--option", "test", "True", "show", "--deps", "none", "--format", "%{key}", "target.bst"], ) result.assert_success() test_key = result.output # Assert that the keys are different when having # tracked separate branches assert test_key != master_key # Test all possible choices of the `--deps` option. # # NOTE: Elements used in this test must have sources that are trackable and do # not have a reference already. The kinds of the sources do not matter so # long as they can be tracked from somewhere. # Currently we use remote sources for this purpose. # @pytest.mark.datafiles(os.path.join(TOP_DIR, "source-track")) @pytest.mark.parametrize( "deps, expected_states", [ ("build", ("no reference", "buildable", "no reference")), ("none", ("waiting", "no reference", "no reference")), ("run", ("waiting", "no reference", "buildable")), ("all", ("waiting", "buildable", "buildable")), ], ) def test_track_deps(cli, datafiles, deps, expected_states): project = str(datafiles) generate_project(project, {"aliases": {"project-root": "file:///" + project}}) target = "bananas.bst" build_dep = "apples.bst" runtime_dep = "oranges.bst" # Assert that none of the sources have a reference states = cli.get_element_states(project, [target, build_dep, runtime_dep]) assert all(state == "no reference" for state in states.values()) # Now track the specified sources result = cli.run(project=project, args=["source", "track", "--deps", deps, target]) result.assert_success() # Finally assert that we have tracked _only_ the desired sources states = cli.get_element_states(project, [target, build_dep, runtime_dep]) states_flattened = (states[target], states[build_dep], states[runtime_dep]) assert states_flattened == expected_states @pytest.mark.datafiles(os.path.join(TOP_DIR, "track-cross-junction")) @pytest.mark.parametrize("cross_junction", [("cross"), ("nocross")]) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_track_cross_junction(cli, tmpdir, datafiles, cross_junction, ref_storage): project = str(datafiles) dev_files_path = os.path.join(project, "files") target_path = os.path.join(project, "target.bst") subtarget_path = os.path.join(project, "subproject", "subtarget.bst") # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = Git(str(tmpdir)) repo.create(dev_files_path) # Generate two elements using the git source, one in # the main project and one in the subproject. generate_element(repo, target_path, dep_name="subproject.bst") generate_element(repo, subtarget_path) # Generate project.conf # project_conf = { "name": "test", "min-version": "2.0", "ref-storage": ref_storage, "plugins": [ { "origin": "pip", "package-name": "sample-plugins", "sources": ["git"], } ], } _yaml.roundtrip_dump(project_conf, os.path.join(project, "project.conf")) # # FIXME: This can be simplified when we have support # for addressing of junctioned elements. # def get_subproject_element_state(): result = cli.run(project=project, args=["show", "--deps", "all", "--format", "%{name}|%{state}", "target.bst"]) result.assert_success() # Create two dimentional list of the result, # first line should be the junctioned element lines = [line.split("|") for line in result.output.splitlines()] assert lines[0][0] == "subproject-junction.bst:subtarget.bst" return lines[0][1] # # Assert that we have no reference yet for the cross junction element # assert get_subproject_element_state() == "no reference" # Track recursively across the junction args = ["source", "track", "--deps", "all"] if cross_junction == "cross": args += ["--cross-junctions"] args += ["target.bst"] result = cli.run(project=project, args=args) if ref_storage == "inline": if cross_junction == "cross": # # Cross junction tracking is not allowed when the toplevel project # is using inline ref storage. # result.assert_main_error(ErrorDomain.STREAM, "untrackable-sources") else: # # No cross juction tracking was requested # result.assert_success() assert get_subproject_element_state() == "no reference" else: # # Tracking is allowed with project.refs ref storage # result.assert_success() # # If cross junction tracking was enabled, we should now be buildable # if cross_junction == "cross": assert get_subproject_element_state() == "buildable" else: assert get_subproject_element_state() == "no reference" @pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_track_consistency_error(cli, datafiles): project = str(datafiles) # Track the element causing a consistency error in `is_cached()` result = cli.run(project=project, args=["source", "track", "error.bst"]) # We expect tracking to succeed as `is_cached()` is not required for tracking. result.assert_success() @pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror")) def test_track_consistency_bug(cli, datafiles): project = str(datafiles) # Track the element causing an unhandled exception in `is_cached()` result = cli.run(project=project, args=["source", "track", "bug.bst"]) # We expect tracking to succeed as `is_cached()` is not required for tracking. result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # Now try to track it, this will bail with the appropriate error # informing the user to track the junction first result = cli.run(project=project, args=["source", "track", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [("inline"), ("project.refs")]) def test_junction_element(cli, tmpdir, datafiles, ref_storage): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") junction_path = os.path.join(project, "elements", "junction.bst") element_path = os.path.join(project, "elements", "junction-dep.bst") configure_project(project, {"ref-storage": ref_storage}) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Create a stack element to depend on a cross junction element # element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]} _yaml.roundtrip_dump(element, element_path) # First demonstrate that showing the pipeline yields an error result = cli.run(project=project, args=["show", "junction-dep.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) # Assert that we have the expected provenance encoded into the error element_node = _yaml.load(element_path, shortname="junction-dep.bst") ref_node = element_node.get_sequence("depends").mapping_at(0) provenance = ref_node.get_provenance() assert str(provenance) in result.stderr # Now track the junction itself result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # Now assert element state (via bst show under the hood) of the dep again assert cli.get_element_state(project, "junction-dep.bst") == "waiting" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_track_error_cannot_write_file(cli, tmpdir, datafiles): if os.geteuid() == 0: pytest.skip("This is not testable with root permissions") project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_name = "track-test.bst" configure_project(project, {"ref-storage": "inline"}) repo = Git(str(tmpdir)) repo.create(dev_files_path) element_full_path = os.path.join(element_path, element_name) generate_element(repo, element_full_path) st = os.stat(element_path) try: read_mask = stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH os.chmod(element_path, stat.S_IMODE(st.st_mode) & ~read_mask) result = cli.run(project=project, args=["source", "track", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, "save-ref-error") finally: os.chmod(element_path, stat.S_IMODE(st.st_mode)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_no_needless_overwrite(cli, tmpdir, datafiles): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") target = "track-test-target.bst" # Skip this test if we do not have support for subsecond precision mtimes # if not have_subsecond_mtime(project): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(project)) # Create our repo object of the given source type with # the dev files, and then collect the initial ref. # repo = Git(str(tmpdir)) repo.create(dev_files_path) # Write out our test target and assert it exists generate_element(repo, os.path.join(element_path, target)) path_to_target = os.path.join(element_path, target) assert os.path.exists(path_to_target) creation_mtime = os.path.getmtime(path_to_target) # Assert tracking is needed states = cli.get_element_states(project, [target]) assert states[target] == "no reference" # Perform the track result = cli.run(project=project, args=["source", "track", target]) result.assert_success() track1_mtime = os.path.getmtime(path_to_target) assert creation_mtime != track1_mtime # Now (needlessly) track again result = cli.run(project=project, args=["source", "track", target]) result.assert_success() track2_mtime = os.path.getmtime(path_to_target) assert track1_mtime == track2_mtime # Regression test for https://gitlab.com/BuildStream/buildstream/-/issues/1265. # Ensure that we can successfully track a `.bst` file that has comments inside # one of our YAML directives (like list append, prepend etc). @pytest.mark.datafiles(os.path.join(TOP_DIR, "source-track")) def test_track_with_comments(cli, datafiles): project = str(datafiles) generate_project(project, {"aliases": {"project-root": "file:///" + project}}) target = "comments.bst" # Assert that it needs to be tracked assert cli.get_element_state(project, target) == "no reference" # Track and fetch the sources result = cli.run(project=project, args=["source", "track", target]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", target]) result.assert_success() # Assert that the sources are cached assert cli.get_element_state(project, target) == "buildable" # Test that elements which contain only sources which do not implement Source.track() # produce a SKIP message in the logs instead of a SUCCESS message when tracking the # element. # # Also test the same for an open workspace, which would be trackable if the # workspace was not open. # # Also test that elements which do not have any sources do not produce any messages at all, # as these jobs are discarded before ever processing. # @pytest.mark.datafiles(DATA_DIR) def test_track_skip(cli, tmpdir, datafiles): project = str(datafiles) dev_files_path = os.path.join(project, "files", "dev-files") element_path = os.path.join(project, "elements") element_dep_name = "track-test-dep.bst" element_workspace_name = "track-test-workspace.bst" element_target_name = "track-test-target.bst" workspace_dir = os.path.join(str(tmpdir), "workspace") # Generate an import element with some local source plugins, these # do not implement track() and thus can be skipped. # element = { "kind": "import", "sources": [ {"kind": "local", "path": "files/dev-files", "directory": "/foo"}, {"kind": "local", "path": "files/dev-files", "directory": "/bar"}, ], } _yaml.roundtrip_dump(element, os.path.join(element_path, element_dep_name)) # Generate a regular import element which will have a workspace open # repo = create_repo("tar", str(tmpdir)) repo.create(dev_files_path) generate_element(repo, os.path.join(element_path, element_workspace_name)) # Generate a stack element which depends on the import of local files # # Stack elements do not have any sources, as such they are also skipped. # element = { "kind": "stack", "depends": [element_dep_name, element_workspace_name], } _yaml.roundtrip_dump(element, os.path.join(element_path, element_target_name)) # First track and fetch the workspace element result = cli.run(project=project, args=["source", "track", "--deps", "none", element_workspace_name]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "--deps", "none", element_workspace_name]) result.assert_success() # Open the workspace so it really is a workspace result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, element_workspace_name]) result.assert_success() # Now run track on the stack and all the deps result = cli.run(project=project, args=["source", "track", "--deps", "all", element_target_name]) result.assert_success() # Assert we got the expected skip messages pattern = r"\[.*track:track-test-dep\.bst.*\] SKIPPED" assert len(re.findall(pattern, result.stderr, re.MULTILINE)) == 1 pattern = r"\[.*track:track-test-workspace\.bst.*\] SKIPPED" assert len(re.findall(pattern, result.stderr, re.MULTILINE)) == 1 # For now, we expect to not see the job for stack elements # # This may be revisited, need to consider if we should emit # START/SKIPPED message pairs for jobs which were assessed to # be unneeded before ever processing. # pattern = r"\[.*track:track-test-target\.bst.*\]" assert len(re.findall(pattern, result.stderr, re.MULTILINE)) == 0 apache-buildstream-27ae392/tests/frontend/version.py000066400000000000000000000024441514607367700226460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name from buildstream._testing.runcli import cli # pylint: disable=unused-import # For utils.get_bst_version() from buildstream import utils def assert_version(cli_version_output): major, minor = utils.get_bst_version() expected_start = "{}.{}".format(major, minor) if not cli_version_output.startswith(expected_start): raise AssertionError( "Version output expected to begin with '{}',".format(expected_start) + " output was: {}".format(cli_version_output) ) def test_version(cli): result = cli.run(args=["--version"]) result.assert_success() assert_version(result.output) apache-buildstream-27ae392/tests/frontend/workspace.py000066400000000000000000001416141514607367700231620ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: Tristan Van Berkom # Tristan Maat # Chandan Singh # Phillip Smyth # Jonathan Maw # Richard Maw # William Salmon # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import stat import shutil import tempfile import pytest from buildstream._testing import create_repo, ALL_REPO_KINDS from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._workspaces import BST_WORKSPACE_FORMAT_VERSION from tests.testutils import create_artifact_share, create_element_size, wait_for_cache_granularity repo_kinds = ALL_REPO_KINDS # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) BASE_FILENAME = os.path.basename(__file__) class WorkspaceCreator: def __init__(self, cli, tmpdir, datafiles, project_path=None): self.cli = cli self.tmpdir = tmpdir self.datafiles = datafiles if not project_path: project_path = str(datafiles) else: shutil.copytree(str(datafiles), project_path) self.project_path = project_path self.bin_files_path = os.path.join(project_path, "files", "bin-files") self.workspace_cmd = os.path.join(self.project_path, "workspace_cmd") def create_workspace_element(self, kind, suffix="", workspace_dir=None, element_attrs=None): element_name = "workspace-test-{}{}.bst".format(kind, suffix) element_path = os.path.join(self.project_path, "elements") if not workspace_dir: workspace_dir = os.path.join(self.workspace_cmd, element_name) if workspace_dir[-4:] == ".bst": workspace_dir = workspace_dir[:-4] # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # And ensure we store it in a suffix-specific directory, to avoid clashes # if using multiple times the same kind element here. repo = create_repo(kind, str(self.tmpdir), "repo-for-{}".format(element_name)) with tempfile.TemporaryDirectory() as tempdir: dst_repo = os.path.join(tempdir, "repo") shutil.copytree(self.bin_files_path, dst_repo) # Touch a file with the element name in, to allow validating that this # is the correct repo # pylint: disable=consider-using-with open(os.path.join(dst_repo, element_name), "a", encoding="utf-8").close() ref = repo.create(os.path.join(tempdir, "repo")) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} if element_attrs: element = {**element, **element_attrs} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) return element_name, element_path, workspace_dir def create_workspace_elements(self, kinds, suffixs=None, workspace_dir_usr=None, element_attrs=None): element_tuples = [] if suffixs is None: suffixs = [ "", ] * len(kinds) else: assert len(suffixs) == len(kinds) for suffix, kind in zip(suffixs, kinds): element_name, _, workspace_dir = self.create_workspace_element( kind, suffix, workspace_dir_usr, element_attrs ) element_tuples.append((element_name, workspace_dir)) # Assert that there is a fetch is needed states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples]) assert not any(states[e] != "fetch needed" for e, _ in element_tuples) return element_tuples def open_workspaces(self, kinds, suffixs=None, workspace_dir=None, element_attrs=None, no_checkout=False): element_tuples = self.create_workspace_elements(kinds, suffixs, workspace_dir, element_attrs) os.makedirs(self.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. args = ["workspace", "open"] if no_checkout: args.append("--no-checkout") if workspace_dir is not None: assert len(element_tuples) == 1, "test logic error" _, workspace_dir = element_tuples[0] args.extend(["--directory", workspace_dir]) args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args) result.assert_success() if not no_checkout: # Assert that we are now buildable because the source is now cached. states = self.cli.get_element_states(self.project_path, [e for e, _ in element_tuples]) assert not any(states[e] != "buildable" for e, _ in element_tuples) # Check that the executable hello file is found in each workspace for _, workspace in element_tuples: filename = os.path.join(workspace, "usr", "bin", "hello") assert os.path.exists(filename) return element_tuples def open_workspace( cli, tmpdir, datafiles, kind, suffix="", workspace_dir=None, project_path=None, element_attrs=None, no_checkout=False, ): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path) workspaces = workspace_object.open_workspaces((kind,), (suffix,), workspace_dir, element_attrs, no_checkout) assert len(workspaces) == 1 element_name, workspace = workspaces[0] return element_name, workspace_object.project_path, workspace @pytest.mark.datafiles(DATA_DIR) def test_open_multi(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) workspaces = workspace_object.open_workspaces(repo_kinds) for (elname, workspace), kind in zip(workspaces, repo_kinds): assert kind in elname workspace_lsdir = os.listdir(workspace) assert elname in workspace_lsdir @pytest.mark.skipif(os.geteuid() == 0, reason="root may have CAP_DAC_OVERRIDE and ignore permissions") @pytest.mark.datafiles(DATA_DIR) def test_open_multi_unwritable(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) element_tuples = workspace_object.create_workspace_elements(repo_kinds, repo_kinds) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. args = ["workspace", "open"] args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) cli.configure({"workspacedir": workspace_object.workspace_cmd}) cwdstat = os.stat(workspace_object.workspace_cmd) try: os.chmod(workspace_object.workspace_cmd, cwdstat.st_mode - stat.S_IWRITE) result = workspace_object.cli.run(project=workspace_object.project_path, args=args) finally: # Using this finally to make sure we always put thing back how they should be. os.chmod(workspace_object.workspace_cmd, cwdstat.st_mode) result.assert_main_error(ErrorDomain.STREAM, "workspace-directory-failure") # Normally we avoid checking stderr in favour of using the mechine readable result.assert_main_error # But Tristan was very keen that the names of the elements left needing workspaces were present in the out put assert " ".join([element_name for element_name, workspace_dir_suffix in element_tuples[1:]]) in result.stderr @pytest.mark.datafiles(DATA_DIR) def test_open_defaultlocation(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) # pylint: disable=unbalanced-tuple-unpacking ((element_name, workspace_dir),) = workspace_object.create_workspace_elements(["tar"], ["tar"]) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. args = ["workspace", "open"] args.append(element_name) # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional # argument, cwd for the workspace_object.cli.run function. But hear we set the default # workspace location to workspace_object.workspace_cmd and run the cli.run function with # no cwd option so that it runs in the project directory. cli.configure({"workspacedir": workspace_object.workspace_cmd}) result = workspace_object.cli.run(project=workspace_object.project_path, args=args) result.assert_success() assert cli.get_element_state(workspace_object.project_path, element_name) == "buildable" # Check that the executable hello file is found in the workspace # even though the cli.run function was not run with cwd = workspace_object.workspace_cmd # the workspace should be created in there as we used the 'workspacedir' configuration # option. filename = os.path.join(workspace_dir, "usr", "bin", "hello") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) def test_open_defaultlocation_exists(cli, tmpdir, datafiles): workspace_object = WorkspaceCreator(cli, tmpdir, datafiles) # pylint: disable=unbalanced-tuple-unpacking ((element_name, workspace_dir),) = workspace_object.create_workspace_elements(["tar"], ["tar"]) os.makedirs(workspace_object.workspace_cmd, exist_ok=True) with open(workspace_dir, "w", encoding="utf-8") as fl: fl.write("foo") # Now open the workspace, this should have the effect of automatically # tracking & fetching the source from the repo. args = ["workspace", "open"] args.append(element_name) # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional # argument, cwd for the workspace_object.cli.run function. But hear we set the default # workspace location to workspace_object.workspace_cmd and run the cli.run function with # no cwd option so that it runs in the project directory. cli.configure({"workspacedir": workspace_object.workspace_cmd}) result = workspace_object.cli.run(project=workspace_object.project_path, args=args) result.assert_main_error(ErrorDomain.STREAM, "bad-directory") @pytest.mark.datafiles(DATA_DIR) def test_open_track(cli, tmpdir, datafiles): open_workspace(cli, tmpdir, datafiles, "tar") @pytest.mark.datafiles(DATA_DIR) def test_open_noclose_open(cli, tmpdir, datafiles): # opening the same workspace twice without closing it should fail element_name, project, _ = open_workspace(cli, tmpdir, datafiles, "tar") result = cli.run(project=project, args=["workspace", "open", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) @pytest.mark.datafiles(DATA_DIR) def test_open_force(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") # Close the workspace result = cli.run(project=project, args=["workspace", "close", element_name]) result.assert_success() # Assert the workspace dir still exists assert os.path.exists(workspace) # Now open the workspace again with --force, this should happily succeed result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_open_force_open(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") result = cli.run(project=project, args=["workspace", "close", element_name]) result.assert_success() # Assert the workspace dir exists assert os.path.exists(workspace) # Now open the workspace again with --force, this should happily succeed result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name]) result.assert_success() # Regression test for #1086. @pytest.mark.datafiles(DATA_DIR) def test_open_force_open_no_checkout(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") hello_path = os.path.join(workspace, "hello.txt") # Assert the workspace dir exists assert os.path.exists(workspace) # Create a new file in the workspace with open(hello_path, "w", encoding="utf-8") as f: f.write("hello") # Now open the workspace again with --force and --no-checkout result = cli.run( project=project, args=["workspace", "open", "--force", "--no-checkout", "--directory", workspace, element_name] ) result.assert_success() # Ensure that our files were not overwritten assert os.path.exists(hello_path) with open(hello_path, encoding="utf-8") as f: assert f.read() == "hello" @pytest.mark.datafiles(DATA_DIR) def test_open_force_different_workspace(cli, tmpdir, datafiles): _, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar", "-alpha") # Assert the workspace dir exists assert os.path.exists(workspace) hello_path = os.path.join(workspace, "usr", "bin", "hello") hello1_path = os.path.join(workspace, "usr", "bin", "hello1") tmpdir = os.path.join(str(tmpdir), "-beta") shutil.move(hello_path, hello1_path) element_name2, _, workspace2 = open_workspace(cli, tmpdir, datafiles, "tar", "-beta") # Assert the workspace dir exists assert os.path.exists(workspace2) # Assert that workspace 1 contains the modified file assert os.path.exists(hello1_path) # Assert that workspace 2 contains the unmodified file assert os.path.exists(os.path.join(workspace2, "usr", "bin", "hello")) result = cli.run(project=project, args=["workspace", "close", element_name2]) result.assert_success() # Now open the workspace again with --force, this should happily succeed result = cli.run(project=project, args=["workspace", "open", "--force", "--directory", workspace, element_name2]) result.assert_success() # Assert that the file in workspace 1 has been replaced # With the file from workspace 2 assert os.path.exists(hello_path) assert not os.path.exists(hello1_path) @pytest.mark.datafiles(DATA_DIR) def test_close(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") # Close the workspace result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted assert not os.path.exists(workspace) @pytest.mark.datafiles(DATA_DIR) def test_close_external_after_move_project(cli, tmpdir, datafiles): workspace_dir = os.path.join(str(tmpdir), "workspace") project_path = os.path.join(str(tmpdir), "initial_project") element_name, _, _ = open_workspace(cli, tmpdir, datafiles, "tar", "", workspace_dir, project_path) assert os.path.exists(workspace_dir) moved_dir = os.path.join(str(tmpdir), "external_project") shutil.move(project_path, moved_dir) assert os.path.exists(moved_dir) # Close the workspace result = cli.run(project=moved_dir, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted assert not os.path.exists(workspace_dir) @pytest.mark.datafiles(DATA_DIR) def test_close_internal_after_move_project(cli, tmpdir, datafiles): initial_dir = os.path.join(str(tmpdir), "initial_project") initial_workspace = os.path.join(initial_dir, "workspace") element_name, _, _ = open_workspace( cli, tmpdir, datafiles, "tar", workspace_dir=initial_workspace, project_path=initial_dir ) moved_dir = os.path.join(str(tmpdir), "internal_project") shutil.move(initial_dir, moved_dir) assert os.path.exists(moved_dir) # Close the workspace result = cli.run(project=moved_dir, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted workspace = os.path.join(moved_dir, "workspace") assert not os.path.exists(workspace) @pytest.mark.datafiles(DATA_DIR) def test_close_removed(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") # Remove it first, closing the workspace should work shutil.rmtree(workspace) # Close the workspace result = cli.run(project=project, args=["workspace", "close", element_name]) result.assert_success() # Assert the workspace dir has been deleted assert not os.path.exists(workspace) @pytest.mark.datafiles(DATA_DIR) def test_close_nonexistant_element(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") element_path = os.path.join(datafiles, "elements", element_name) # First brutally remove the element.bst file, ensuring that # the element does not exist anymore in the project where # we want to close the workspace. os.remove(element_path) # Close the workspace result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Assert the workspace dir has been deleted assert not os.path.exists(workspace) @pytest.mark.datafiles(DATA_DIR) def test_close_multiple(cli, tmpdir, datafiles): tmpdir_alpha = os.path.join(str(tmpdir), "alpha") tmpdir_beta = os.path.join(str(tmpdir), "beta") alpha, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "tar", suffix="-alpha") beta, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "tar", suffix="-beta") # Close the workspaces result = cli.run(project=project, args=["workspace", "close", "--remove-dir", alpha, beta]) result.assert_success() # Assert the workspace dirs have been deleted assert not os.path.exists(workspace_alpha) assert not os.path.exists(workspace_beta) @pytest.mark.datafiles(DATA_DIR) def test_close_all(cli, tmpdir, datafiles): tmpdir_alpha = os.path.join(str(tmpdir), "alpha") tmpdir_beta = os.path.join(str(tmpdir), "beta") _, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "tar", suffix="-alpha") _, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "tar", suffix="-beta") # Close the workspaces result = cli.run(project=project, args=["workspace", "close", "--remove-dir", "--all"]) result.assert_success() # Assert the workspace dirs have been deleted assert not os.path.exists(workspace_alpha) assert not os.path.exists(workspace_beta) @pytest.mark.datafiles(DATA_DIR) def test_reset(cli, tmpdir, datafiles): # Open the workspace element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") # Modify workspace shutil.rmtree(os.path.join(workspace, "usr", "bin")) os.makedirs(os.path.join(workspace, "etc")) with open(os.path.join(workspace, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") # Now reset the open workspace, this should have the # effect of reverting our changes. result = cli.run(project=project, args=["workspace", "reset", element_name]) result.assert_success() assert os.path.exists(os.path.join(workspace, "usr", "bin", "hello")) assert not os.path.exists(os.path.join(workspace, "etc", "pony.conf")) @pytest.mark.datafiles(DATA_DIR) def test_reset_soft(cli, tmpdir, datafiles): # Open the workspace element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") assert cli.get_element_state(project, element_name) == "buildable" hello_path = os.path.join(workspace, "usr", "bin", "hello") pony_path = os.path.join(workspace, "etc", "pony.conf") assert os.path.exists(os.path.join(workspace, "usr", "bin")) assert os.path.exists(hello_path) assert not os.path.exists(pony_path) key_1 = cli.get_element_key(project, element_name) assert key_1 != "{:?<64}".format("") result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" key_2 = cli.get_element_key(project, element_name) assert key_2 != "{:?<64}".format("") # workspace keys are not recalculated assert key_1 == key_2 wait_for_cache_granularity() # Modify workspace shutil.rmtree(os.path.join(workspace, "usr", "bin")) os.makedirs(os.path.join(workspace, "etc")) with open(os.path.join(workspace, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") assert not os.path.exists(os.path.join(workspace, "usr", "bin")) assert os.path.exists(pony_path) # Now soft-reset the open workspace, this should not revert the changes result = cli.run(project=project, args=["workspace", "reset", "--soft", element_name]) result.assert_success() # we removed this dir assert not os.path.exists(os.path.join(workspace, "usr", "bin")) # and added this one assert os.path.exists(os.path.join(workspace, "etc", "pony.conf")) assert cli.get_element_state(project, element_name) == "buildable" key_3 = cli.get_element_key(project, element_name) assert key_3 != "{:?<64}".format("") assert key_1 != key_3 @pytest.mark.datafiles(DATA_DIR) def test_reset_multiple(cli, tmpdir, datafiles): # Open the workspaces tmpdir_alpha = os.path.join(str(tmpdir), "alpha") tmpdir_beta = os.path.join(str(tmpdir), "beta") alpha, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "tar", suffix="-alpha") beta, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "tar", suffix="-beta") # Modify workspaces shutil.rmtree(os.path.join(workspace_alpha, "usr", "bin")) os.makedirs(os.path.join(workspace_beta, "etc")) with open(os.path.join(workspace_beta, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") # Now reset the open workspaces, this should have the # effect of reverting our changes. result = cli.run( project=project, args=[ "workspace", "reset", alpha, beta, ], ) result.assert_success() assert os.path.exists(os.path.join(workspace_alpha, "usr", "bin", "hello")) assert not os.path.exists(os.path.join(workspace_beta, "etc", "pony.conf")) @pytest.mark.datafiles(DATA_DIR) def test_reset_all(cli, tmpdir, datafiles): # Open the workspaces tmpdir_alpha = os.path.join(str(tmpdir), "alpha") tmpdir_beta = os.path.join(str(tmpdir), "beta") _, project, workspace_alpha = open_workspace(cli, tmpdir_alpha, datafiles, "tar", suffix="-alpha") _, project, workspace_beta = open_workspace(cli, tmpdir_beta, datafiles, "tar", suffix="-beta") # Modify workspaces shutil.rmtree(os.path.join(workspace_alpha, "usr", "bin")) os.makedirs(os.path.join(workspace_beta, "etc")) with open(os.path.join(workspace_beta, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") # Now reset the open workspace, this should have the # effect of reverting our changes. result = cli.run(project=project, args=["workspace", "reset", "--all"]) result.assert_success() assert os.path.exists(os.path.join(workspace_alpha, "usr", "bin", "hello")) assert not os.path.exists(os.path.join(workspace_beta, "etc", "pony.conf")) @pytest.mark.datafiles(DATA_DIR) def test_list(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") # Now list the workspaces result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() loaded = _yaml.load_data(result.output) workspaces = loaded.get_sequence("workspaces") assert len(workspaces) == 1 space = workspaces.mapping_at(0) assert space.get_str("element") == element_name assert space.get_str("directory") == workspace @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("kind", repo_kinds) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) @pytest.mark.parametrize( "from_workspace,guess_element", [(False, False), (True, True), (True, False)], ids=["project-no-guess", "workspace-guess", "workspace-no-guess"], ) def test_build(cli, tmpdir_factory, datafiles, kind, strict, from_workspace, guess_element): tmpdir = tmpdir_factory.mktemp(BASE_FILENAME) element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False) checkout = os.path.join(str(tmpdir), "checkout") args_dir = ["-C", workspace] if from_workspace else [] args_elm = [element_name] if not guess_element else [] # Modify workspace shutil.rmtree(os.path.join(workspace, "usr", "bin")) os.makedirs(os.path.join(workspace, "etc")) with open(os.path.join(workspace, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") # Configure strict mode strict_mode = True if strict != "strict": strict_mode = False cli.configure({"projects": {"test": {"strict": strict_mode}}}) # Build modified workspace assert cli.get_element_state(project, element_name) == "buildable" key_1 = cli.get_element_key(project, element_name) assert key_1 != "{:?<64}".format("") result = cli.run(project=project, args=args_dir + ["build", *args_elm]) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" key_2 = cli.get_element_key(project, element_name) assert key_2 != "{:?<64}".format("") # workspace keys are not recalculated assert key_1 == key_2 # Checkout the result result = cli.run(project=project, args=args_dir + ["artifact", "checkout", "--directory", checkout, *args_elm]) result.assert_success() # Check that the pony.conf from the modified workspace exists filename = os.path.join(checkout, "etc", "pony.conf") assert os.path.exists(filename) # Check that the original /usr/bin/hello is not in the checkout assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello")) @pytest.mark.datafiles(DATA_DIR) def test_buildable_no_ref(cli, tmpdir, datafiles): project = str(datafiles) element_name = "workspace-test-no-ref.bst" element_path = os.path.join(project, "elements") # Write out our test target without any source ref repo = create_repo("tar", str(tmpdir)) element = {"kind": "import", "sources": [repo.source_config()]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Assert that this target is not buildable when no workspace is associated. assert cli.get_element_state(project, element_name) == "no reference" # Now open the workspace. We don't need to checkout the source though. workspace = os.path.join(str(tmpdir), "workspace-no-ref") os.makedirs(workspace) args = ["workspace", "open", "--no-checkout", "--directory", workspace, element_name] result = cli.run(project=project, args=args) result.assert_success() # Assert that the target is now buildable. assert cli.get_element_state(project, element_name) == "buildable" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("modification", [("addfile"), ("removefile"), ("modifyfile")]) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_detect_modifications(cli, tmpdir, datafiles, modification, strict): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") checkout = os.path.join(str(tmpdir), "checkout") # Configure strict mode strict_mode = True if strict != "strict": strict_mode = False cli.configure({"projects": {"test": {"strict": strict_mode}}}) # Build clean workspace assert cli.get_element_state(project, element_name) == "buildable" key_1 = cli.get_element_key(project, element_name) assert key_1 != "{:?<64}".format("") result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" key_2 = cli.get_element_key(project, element_name) assert key_2 != "{:?<64}".format("") # workspace keys are not recalculated assert key_1 == key_2 wait_for_cache_granularity() # Modify the workspace in various different ways, ensuring we # properly detect the changes. # if modification == "addfile": os.makedirs(os.path.join(workspace, "etc")) with open(os.path.join(workspace, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") elif modification == "removefile": os.remove(os.path.join(workspace, "usr", "bin", "hello")) elif modification == "modifyfile": with open(os.path.join(workspace, "usr", "bin", "hello"), "w", encoding="utf-8") as f: f.write("cookie") else: # This cannot be reached assert 0 # First assert that the state is properly detected assert cli.get_element_state(project, element_name) == "buildable" key_3 = cli.get_element_key(project, element_name) assert key_3 != "{:?<64}".format("") # Since there are different things going on at `bst build` time # than `bst show` time, we also want to build / checkout again, # and ensure that the result contains what we expect. result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" key_4 = cli.get_element_key(project, element_name) assert key_4 != "{:?<64}".format("") # workspace keys are not recalculated assert key_3 == key_4 # workspace keys are determined by the files assert key_1 != key_3 # Checkout the result result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() # Check the result for the changes we made # if modification == "addfile": filename = os.path.join(checkout, "etc", "pony.conf") assert os.path.exists(filename) elif modification == "removefile": assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello")) elif modification == "modifyfile": with open(os.path.join(workspace, "usr", "bin", "hello"), "r", encoding="utf-8") as f: data = f.read() assert data == "cookie" else: # This cannot be reached assert 0 # Ensure that various versions that should not be accepted raise a # LoadError.INVALID_DATA @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "workspace_cfg", [ # Test loading a negative workspace version {"format-version": -1}, # Test loading version 0 with two sources { "format-version": 0, "alpha.bst": { 0: "/workspaces/bravo", 1: "/workspaces/charlie", }, }, # Test loading a version with decimals {"format-version": 0.5}, # Test loading an unsupported old version {"format-version": 3}, # Test loading a future version {"format-version": BST_WORKSPACE_FORMAT_VERSION + 1}, ], ) def test_list_unsupported_workspace(cli, datafiles, workspace_cfg): project = str(datafiles) os.makedirs(os.path.join(project, ".bst2")) workspace_config_path = os.path.join(project, ".bst2", "workspaces.yml") _yaml.roundtrip_dump(workspace_cfg, workspace_config_path) result = cli.run(project=project, args=["workspace", "list"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) # Ensure that various versions that should be accepted are parsed # correctly. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "workspace_cfg,expected", [ # Test loading version 4 ( { "format-version": 4, "workspaces": {"alpha.bst": {"path": "/workspaces/bravo"}}, }, { "format-version": BST_WORKSPACE_FORMAT_VERSION, "workspaces": {"alpha.bst": {"path": "/workspaces/bravo"}}, }, ), ], ) def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expected): def parse_dict_as_yaml(node): tempfile = os.path.join(str(tmpdir), "yaml_dump") _yaml.roundtrip_dump(node, tempfile) return _yaml.load(tempfile, shortname=None).strip_node_info() project = str(datafiles) os.makedirs(os.path.join(project, ".bst")) workspace_config_path = os.path.join(project, ".bst", "workspaces.yml") _yaml.roundtrip_dump(workspace_cfg, workspace_config_path) # Check that we can still read workspace config that is in old format result = cli.run(project=project, args=["workspace", "list"]) result.assert_success() loaded_config = _yaml.load(workspace_config_path, shortname=None).strip_node_info() # Check that workspace config remains the same if no modifications # to workspaces were made assert loaded_config == parse_dict_as_yaml(workspace_cfg) # Create a test bst file bin_files_path = os.path.join(project, "files", "bin-files") element_path = os.path.join(project, "elements") element_name = "workspace-test.bst" workspace = os.path.join(str(tmpdir), "workspace") # Create our repo object of the given source type with # the bin files, and then collect the initial ref. # repo = create_repo("tar", str(tmpdir)) ref = repo.create(bin_files_path) # Write out our test target element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # Make a change to the workspaces file result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) result.assert_success() result = cli.run(project=project, args=["workspace", "close", "--remove-dir", element_name]) result.assert_success() # Check that workspace config is converted correctly if necessary loaded_config = _yaml.load(workspace_config_path, shortname=None).strip_node_info() assert loaded_config == parse_dict_as_yaml(expected) @pytest.mark.datafiles(DATA_DIR) def test_inconsitent_pipeline_message(cli, tmpdir, datafiles): element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") shutil.rmtree(workspace) result = cli.run(project=project, args=["build", element_name]) result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict): checkout = os.path.join(str(tmpdir), "checkout") element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), "repo-a"), datafiles, "tar") element_path = os.path.join(project, "elements") back_dep_element_name = "workspace-test-back-dep.bst" # Write out our test target element = {"kind": "compose", "depends": [{"filename": element_name, "type": "build"}]} _yaml.roundtrip_dump(element, os.path.join(element_path, back_dep_element_name)) # Modify workspace shutil.rmtree(os.path.join(workspace, "usr", "bin")) os.makedirs(os.path.join(workspace, "etc")) with open(os.path.join(workspace, "etc", "pony.conf"), "w", encoding="utf-8") as f: f.write("PONY='pink'") # Configure strict mode strict_mode = True if strict != "strict": strict_mode = False cli.configure({"projects": {"test": {"strict": strict_mode}}}) # Build artifact with dependency's modified workspace assert cli.get_element_state(project, element_name) == "buildable" key_a1 = cli.get_element_key(project, element_name) assert key_a1 != "{:?<64}".format("") assert cli.get_element_state(project, back_dep_element_name) == "waiting" key_b1 = cli.get_element_key(project, back_dep_element_name) assert key_b1 != "{:?<64}".format("") result = cli.run(project=project, args=["build", back_dep_element_name]) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" key_a2 = cli.get_element_key(project, element_name) assert key_a2 != "{:?<64}".format("") assert cli.get_element_state(project, back_dep_element_name) == "cached" key_b2 = cli.get_element_key(project, back_dep_element_name) assert key_b2 != "{:?<64}".format("") result = cli.run(project=project, args=["build", back_dep_element_name]) result.assert_success() # workspace keys are not recalculated assert key_a1 == key_a2 assert key_b1 == key_b2 # Checkout the result result = cli.run(project=project, args=["artifact", "checkout", back_dep_element_name, "--directory", checkout]) result.assert_success() # Check that the pony.conf from the modified workspace exists filename = os.path.join(checkout, "etc", "pony.conf") assert os.path.exists(filename) # Check that the original /usr/bin/hello is not in the checkout assert not os.path.exists(os.path.join(checkout, "usr", "bin", "hello")) @pytest.mark.datafiles(DATA_DIR) def test_multiple_failed_builds(cli, tmpdir, datafiles): element_config = {"kind": "manual", "config": {"configure-commands": ["unknown_command_that_will_fail"]}} element_name, project, _ = open_workspace(cli, tmpdir, datafiles, "tar", element_attrs=element_config) for _ in range(2): result = cli.run(project=project, args=["build", element_name]) assert "BUG" not in result.stderr assert cli.get_element_state(project, element_name) != "cached" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("subdir", [True, False], ids=["subdir", "no-subdir"]) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_fetch(cli, datafiles, tmpdir_factory, subdir, guess_element): # An element with an open workspace can't be fetched, but we still expect fetches # to fetch any dependencies tmpdir = tmpdir_factory.mktemp(BASE_FILENAME) depend_element = "fetchable.bst" # Create an element to fetch (local sources do not need to fetch) create_element_size(depend_element, str(datafiles), "elements", [], 1024) element_name, project, workspace = open_workspace( cli, tmpdir, datafiles, "tar", no_checkout=True, element_attrs={"depends": [depend_element]} ) arg_elm = [element_name] if not guess_element else [] if subdir: call_dir = os.path.join(workspace, "usr") os.makedirs(call_dir, exist_ok=True) else: call_dir = workspace # Assert that the depended element is not fetched yet assert cli.get_element_state(str(datafiles), depend_element) == "fetch needed" # Fetch the workspaced element result = cli.run(project=project, args=["-C", call_dir, "source", "fetch", "--deps", "all", *arg_elm]) result.assert_success() # Assert that the depended element has now been fetched assert cli.get_element_state(str(datafiles), depend_element) == "buildable" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element): # Pushing and pulling to/from an artifact cache works from an external workspace tmpdir = tmpdir_factory.mktemp(BASE_FILENAME) element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") arg_elm = [element_name] if not guess_element else [] with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: result = cli.run(project=project, args=["-C", workspace, "build", element_name]) result.assert_success() cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["-C", workspace, "artifact", "push", *arg_elm]) result.assert_success() result = cli.run(project=project, args=["-C", workspace, "artifact", "pull", "--deps", "all", *arg_elm]) result.assert_success() # Attempting to track in an open workspace is not a sensible thing and it's not compatible with workspaces as plugin # sources: The new ref (if it differed from the old) would have been ignored regardless. # The user should be expected to simply close the workspace before tracking. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_track(cli, datafiles, tmpdir_factory, guess_element): tmpdir = tmpdir_factory.mktemp(BASE_FILENAME) element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") element_file = os.path.join(str(datafiles), "elements", element_name) arg_elm = [element_name] if not guess_element else [] # Delete the ref from the source so that we can detect if the # element has been tracked after closing the workspace element_contents = _yaml.load(element_file, shortname=None) ref1 = element_contents.get_sequence("sources").mapping_at(0).get_str("ref") del element_contents.get_sequence("sources").mapping_at(0)["ref"] _yaml.roundtrip_dump(element_contents, element_file) result = cli.run(project=project, args=["-C", workspace, "source", "track", *arg_elm]) result.assert_success() # Element is not tracked now element_contents = _yaml.load(element_file, shortname=None) assert "ref" not in element_contents.get_sequence("sources").mapping_at(0) # close the workspace result = cli.run(project=project, args=["-C", workspace, "workspace", "close", *arg_elm]) result.assert_success() # and retrack the element result = cli.run(project=project, args=["source", "track", element_name]) result.assert_success() element_contents = _yaml.load(element_file, shortname=None) ref2 = element_contents.get_sequence("sources").mapping_at(0).get_str("ref") # these values should be equivalent assert ref1 == ref2 @pytest.mark.datafiles(DATA_DIR) def test_external_open_other(cli, datafiles, tmpdir_factory): # From inside an external workspace, open another workspace tmpdir1 = tmpdir_factory.mktemp(BASE_FILENAME) tmpdir2 = tmpdir_factory.mktemp(BASE_FILENAME) # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "tar", suffix="-alpha") beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "tar", suffix="-beta") # Closing the other element first, because I'm too lazy to create an # element without opening it result = cli.run(project=project, args=["workspace", "close", beta_element]) result.assert_success() result = cli.run( project=project, args=["-C", alpha_workspace, "workspace", "open", "--force", "--directory", beta_workspace, beta_element], ) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_external_reset_other(cli, datafiles, tmpdir_factory): tmpdir1 = tmpdir_factory.mktemp(BASE_FILENAME) tmpdir2 = tmpdir_factory.mktemp(BASE_FILENAME) # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "tar", suffix="-alpha") beta_element, _, _ = open_workspace(cli, tmpdir2, datafiles, "tar", suffix="-beta") result = cli.run(project=project, args=["-C", alpha_workspace, "workspace", "reset", beta_element]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) def test_external_reset_self(cli, datafiles, tmpdir, guess_element): element, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") arg_elm = [element] if not guess_element else [] # Command succeeds result = cli.run(project=project, args=["-C", workspace, "workspace", "reset", *arg_elm]) result.assert_success() # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted) result = cli.run(project=project, args=["-C", workspace, "workspace", "list"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_external_list(cli, datafiles, tmpdir_factory): tmpdir = tmpdir_factory.mktemp(BASE_FILENAME) # Making use of the assumption that it's the same project in both invocations of open_workspace _, project, workspace = open_workspace(cli, tmpdir, datafiles, "tar") result = cli.run(project=project, args=["-C", workspace, "workspace", "list"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_multisource_workspace(cli, datafiles, tmpdir): # checks that if an element has multiple sources, then the opened workspace # will contain them project = str(datafiles) element_name = "multisource.bst" element = { "kind": "import", "sources": [{"kind": "local", "path": "files/bin-files"}, {"kind": "local", "path": "files/dev-files"}], } element_path = os.path.join(project, "elements", element_name) _yaml.roundtrip_dump(element, element_path) workspace_dir = os.path.join(str(tmpdir), "multisource") res = cli.run(project=project, args=["workspace", "open", "multisource.bst", "--directory", workspace_dir]) res.assert_success() directories = os.listdir(os.path.join(workspace_dir, "usr")) assert "bin" in directories and "include" in directories # This strange test tests against a regression raised in issue #919, # where opening a workspace on a runtime dependency of a build only # dependency causes `bst build` to not build the specified target # but just successfully builds the workspaced element and happily # exits without completing the build. # TEST_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__))) @pytest.mark.datafiles(TEST_DIR) @pytest.mark.parametrize( ["case", "non_workspaced_elements_state"], [ ("workspaced-build-dep", ["waiting", "waiting", "waiting", "waiting", "waiting"]), ("workspaced-runtime-dep", ["buildable", "buildable", "waiting", "waiting", "waiting"]), ], ) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) def test_build_all(cli, tmpdir, datafiles, case, strict, non_workspaced_elements_state): project = os.path.join(str(datafiles), case) workspace = os.path.join(str(tmpdir), "workspace") non_leaf_elements = ["elem2.bst", "elem3.bst", "stack.bst", "elem4.bst", "elem5.bst"] all_elements = ["elem1.bst", *non_leaf_elements] # Configure strict mode strict_mode = True if strict != "strict": strict_mode = False cli.configure({"projects": {"test": {"strict": strict_mode}}}) # First open the workspace result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, "elem1.bst"]) result.assert_success() # Ensure all elements are waiting build the first assert cli.get_element_states(project, all_elements) == dict( zip(all_elements, ["buildable", *non_workspaced_elements_state]) ) # Now build the targets elem4.bst and elem5.bst result = cli.run(project=project, args=["build", "elem4.bst", "elem5.bst"]) result.assert_success() # Assert that the target is built assert cli.get_element_states(project, all_elements) == {elem: "cached" for elem in all_elements} @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict", ["strict", "non-strict"]) def test_show_workspace_logs(cli, tmpdir, datafiles, strict): project = str(datafiles) workspace = os.path.join(str(tmpdir), "workspace") target = "manual.bst" # Configure strict mode strict_mode = True if strict != "strict": strict_mode = False cli.configure({"projects": {"test": {"strict": strict_mode}}}) # First open the workspace result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target]) result.assert_success() # Build the element result = cli.run(project=project, args=["build", target]) result.assert_task_error(ErrorDomain.SANDBOX, "missing-command") result = cli.run(project=project, args=["artifact", "log", target]) result.assert_success() # Assert that the log is not empty assert result.output != "" apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/000077500000000000000000000000001514607367700246105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/000077500000000000000000000000001514607367700264245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/elem1.bst000066400000000000000000000000711514607367700301370ustar00rootroot00000000000000kind: import sources: - kind: local path: files/file1 apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/elem2.bst000066400000000000000000000001251514607367700301400ustar00rootroot00000000000000kind: import build-depends: - elem1.bst sources: - kind: local path: files/file2 apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/elem3.bst000066400000000000000000000001251514607367700301410ustar00rootroot00000000000000kind: import build-depends: - elem2.bst sources: - kind: local path: files/file3 apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/elem4.bst000066400000000000000000000001251514607367700301420ustar00rootroot00000000000000kind: import build-depends: - stack.bst sources: - kind: local path: files/file4 apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/elem5.bst000066400000000000000000000001251514607367700301430ustar00rootroot00000000000000kind: import build-depends: - elem3.bst sources: - kind: local path: files/file4 apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/elements/stack.bst000066400000000000000000000000421514607367700302370ustar00rootroot00000000000000kind: stack depends: - elem3.bst apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/files/000077500000000000000000000000001514607367700257125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/files/file1000066400000000000000000000000001514607367700266230ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/files/file2000066400000000000000000000000001514607367700266240ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/files/file3000066400000000000000000000000001514607367700266250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/files/file4000066400000000000000000000000001514607367700266260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-build-dep/project.conf000066400000000000000000000002231514607367700271220ustar00rootroot00000000000000# Unique project name name: test # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/000077500000000000000000000000001514607367700251745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/000077500000000000000000000000001514607367700270105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/elem1.bst000066400000000000000000000000711514607367700305230ustar00rootroot00000000000000kind: import sources: - kind: local path: files/file1 apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/elem2.bst000066400000000000000000000001271514607367700305260ustar00rootroot00000000000000kind: import runtime-depends: - elem1.bst sources: - kind: local path: files/file2 apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/elem3.bst000066400000000000000000000001271514607367700305270ustar00rootroot00000000000000kind: import runtime-depends: - elem2.bst sources: - kind: local path: files/file3 apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/elem4.bst000066400000000000000000000001251514607367700305260ustar00rootroot00000000000000kind: import build-depends: - stack.bst sources: - kind: local path: files/file4 apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/elem5.bst000066400000000000000000000001251514607367700305270ustar00rootroot00000000000000kind: import build-depends: - elem3.bst sources: - kind: local path: files/file4 apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/elements/stack.bst000066400000000000000000000000421514607367700306230ustar00rootroot00000000000000kind: stack depends: - elem3.bst apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/files/000077500000000000000000000000001514607367700262765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/files/file1000066400000000000000000000000001514607367700272070ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/files/file2000066400000000000000000000000001514607367700272100ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/files/file3000066400000000000000000000000001514607367700272110ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/files/file4000066400000000000000000000000001514607367700272120ustar00rootroot00000000000000apache-buildstream-27ae392/tests/frontend/workspaced-runtime-dep/project.conf000066400000000000000000000002231514607367700275060ustar00rootroot00000000000000# Unique project name name: test # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/tests/integration/000077500000000000000000000000001514607367700213075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/__init__.py000066400000000000000000000000001514607367700234060ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/artifact.py000066400000000000000000000223221514607367700234570ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: Richard Maw # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) # A test to capture the integration of the cachebuildtrees # behaviour, which by default is to include the buildtree # content of an element on caching. # Dse this really need a sandbox? @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_cache_buildtrees(cli, tmpdir, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing with create_artifact_share(os.path.join(str(tmpdir), "share1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "share2") ) as share2, create_artifact_share(os.path.join(str(tmpdir), "share3")) as share3: cli.configure({"artifacts": {"servers": [{"url": share1.repo, "push": True}]}, "cachedir": str(tmpdir)}) # Build autotools element with the default behavior of caching buildtrees # only when necessary. The artifact should be successfully pushed to the share1 remote # and cached locally with an 'empty' buildtree digest, as it's not a # dangling ref result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share1.get_artifact(cli.get_artifact_name(project, "test", element_name)) # The buildtree dir should not exist, as we set the config to not cache buildtrees. artifact_name = cli.get_artifact_name(project, "test", element_name) assert share1.get_artifact(artifact_name) with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees # that is was cached in share1 as expected without a buildtree dir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) assert cli.get_element_state(project, element_name) != "cached" result = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Assert that the default behaviour of pull to not include buildtrees on the artifact # in share1 which was purposely cached with an empty one behaves as expected. As such the # pulled artifact will have a dangling ref for the buildtree dir, regardless of content, # leading to no buildtreedir being extracted result = cli.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Repeat building the artifacts, this time with cache-buildtrees set to # 'always' via the cli, as such the buildtree dir should not be empty cli.configure({"artifacts": {"servers": [{"url": share2.repo, "push": True}]}, "cachedir": str(tmpdir)}) result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share2.get_artifact(cli.get_artifact_name(project, "test", element_name)) # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees # that it was cached in share2 as expected with a populated buildtree dir shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) assert cli.get_element_state(project, element_name) != "cached" result = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) # Clarify that the user config option for cache-buildtrees works as the cli # main option does. Point to share3 which does not have the artifacts cached to force # a build cli.configure( { "artifacts": {"servers": [{"url": share3.repo, "push": True}]}, "cachedir": str(tmpdir), "cache": {"cache-buildtrees": "always"}, } ) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" with cli.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert os.listdir(buildtreedir) # # This test asserts that the environment variables are indeed preserved in # generated artifacts, and asserts that local project state is not erronously # observed when checking out and integrating an artifact by artifact name. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_preserve_environment(datafiles, cli): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # # First build the target, this will create the echo-env-var.bst artifact # and cache an integration command which uses it's build environment # to create /etc/test.conf # result = cli.run(project=project, args=["build", "echo-target.bst"]) result.assert_success() # # Now preserve the cache key of the toplevel # key = cli.get_element_key(project, "echo-target.bst") # # From here on out, we will augment the project.conf with a modified # environment, causing the definition of the element to change and # consequently the cache key. # project_config = {"environment": {"TEST_VAR": "horsy"}} # # This should changed the cache key such that a different artifact # would be produced, as the environment has changed. # result = cli.run_project_config( project=project, project_config=project_config, args=["show", "--deps", "none", "--format", "%{full-key}", "echo-target.bst"], ) result.assert_success() assert result.output.strip() != key # # Now checkout the originally built artifact and request that it be integrated, # this will run integration commands encoded into the artifact. # checkout_args = [ "artifact", "checkout", "--directory", checkout, "--deps", "build", "--integrate", "test/echo-target/" + key, ] result = cli.run_project_config(project=project, args=checkout_args, project_config=project_config) result.assert_success() # # Now observe the file generated by echo-env-var.bst's integration command. # # Here we assert that the actual environment from the artifact is being # used to run the integration command, and not the irrelevant project # data in the local project directory. # filename = os.path.join(checkout, "etc", "test.conf") assert os.path.exists(filename) with open(filename, "r", encoding="utf-8") as f: data = f.read() data = data.strip() assert data == "pony" apache-buildstream-27ae392/tests/integration/autotools.py000066400000000000000000000072521514607367700237200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that an autotools build 'works' - we use the autotools sample # amhello project for this. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # Check the log result = cli.run(project=project, args=["artifact", "log", element_name]) assert result.exit_code == 0 log = result.output # Verify we get expected output exactly once assert log.count("Making all in src") == 1 # Test that an autotools build 'works' - we use the autotools sample # amhello project for this. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_autotools_confroot_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhelloconfroot.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # Test running an executable built with autotools @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_autotools_run(cli, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) assert result.exit_code == 0 assert result.output == "Hello World!\nThis is amhello 1.0.\n" apache-buildstream-27ae392/tests/integration/base/000077500000000000000000000000001514607367700222215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/base/generate-base.sh000077500000000000000000000054121514607367700252640ustar00rootroot00000000000000#!/bin/sh # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Generate a base sysroot for running the BuildStream integration tests. # # The sysroot is based off the Alpine Linux distribution. The script downloads # a release of Alpine, sets up a cheap container using `bwrap` and installs the # packages that are needed by the integration tests, then outputs a .tar.xz # file. set -eux ALPINE_ARCH=${ARCH:-x86-64} ALPINE_BASE=http://dl-cdn.alpinelinux.org/alpine/v3.7/releases/${ALPINE_ARCH}/alpine-minirootfs-3.7.0-${ALPINE_ARCH}.tar.gz mkdir root wget ${ALPINE_BASE} -O alpine-base.tar.gz tar -x -f ./alpine-base.tar.gz -C ./root --exclude dev/\* run() { # This turns the unpacked rootfs into a container using Bubblewrap. # The Alpine package manager (apk) calls `chroot` when running package # triggers so we need to enable CAP_SYS_CHROOT. We also have to fake # UID 0 (root) inside the container to avoid permissions errors. bwrap --bind ./root / --dev /dev --proc /proc --tmpfs /tmp \ --ro-bind /etc/resolv.conf /etc/resolv.conf \ --setenv PATH "/usr/bin:/usr/sbin:/bin:/sbin" \ --unshare-user --uid 0 --gid 0 \ --cap-add CAP_SYS_CHROOT \ /bin/sh -c "$@" } # Enable testing repo for Tiny C Compiler package run "echo http://dl-cdn.alpinelinux.org/alpine/edge/testing >> /etc/apk/repositories" # Fetch the list of Alpine packages. run "apk update" # There are various random errors from `apk add` to do with ownership, probably # because of our hacked up `bwrap` container. The errors seem harmless so I am # just ignoring them. set +e # Install stuff needed by all integration tests that compile C code. # # Note that we use Tiny C Compiler in preference to GCC. There is a huge # size difference -- 600KB for TinyCC vs. 50MB to 100MB for GCC. TinyCC # supports most of the ISO C99 standard, but has no C++ support at all. run "apk add binutils libc-dev make tcc" run "ln -s /usr/bin/tcc /usr/bin/cc" # Install stuff for tests/integration/autotools run "apk add autoconf automake" # Install stuff for tests/integration/cmake run "apk add cmake" # Install stuff for tests/integration/pip run "apk add python3" set -e # Cleanup the package cache run "rm -R /var/cache/apk" tar -c -v -J -f integration-tests-base.tar.xz -C root . apache-buildstream-27ae392/tests/integration/base/generate-debian-base.sh000077500000000000000000000022671514607367700265110ustar00rootroot00000000000000#!/bin/sh # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Generate a base sysroot for running the BuildStream integration tests. # # The sysroot is based off the Debian Linux distribution. set -eux DOCKER_ARCH=${DOCKER_ARCH:-amd64} export DOCKER_DEFAULT_PLATFORM="linux/%{DOCKER_ARCH}" IMAGE_NAME="integration-tests-debian-base" docker build --tag ${IMAGE_NAME} -f - << "EOF" FROM debian:trixie-slim RUN apt-get update && apt-get install -y --no-install-recommends \ gcc libc6-dev make autoconf automake && \ rm -rf /var/lib/apt/lists/* EOF CONTAINER_NAME="$(docker create ${IMAGE_NAME})" docker export "${CONTAINER_NAME}" | xz > ${IMAGE_NAME}.tar.xz docker rm "${CONTAINER_NAME}" apache-buildstream-27ae392/tests/integration/build-uid.py000066400000000000000000000053711514607367700235450ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Custom UID/GID not supported by userchroot", ) @pytest.mark.datafiles(DATA_DIR) def test_build_uid_overridden(cli, datafiles): project = str(datafiles) element_name = "build-uid/build-uid.bst" project_config = {"name": "build-uid-test", "sandbox": {"build-uid": 800, "build-gid": 900}} result = cli.run_project_config(project=project, project_config=project_config, args=["build", element_name]) assert result.exit_code == 0 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Custom UID/GID not supported by userchroot", ) @pytest.mark.datafiles(DATA_DIR) def test_build_uid_in_project(cli, datafiles): project = str(datafiles) element_name = "build-uid/build-uid-1023.bst" project_config = {"name": "build-uid-test", "sandbox": {"build-uid": 1023, "build-gid": 3490}} result = cli.run_project_config(project=project, project_config=project_config, args=["build", element_name]) assert result.exit_code == 0 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Custom UID/GID not supported by userchroot", ) def test_build_uid_default(cli, datafiles): project = str(datafiles) element_name = "build-uid/build-uid-default.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 apache-buildstream-27ae392/tests/integration/cached-fail/000077500000000000000000000000001514607367700234275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/cached-fail/elements/000077500000000000000000000000001514607367700252435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/cached-fail/elements/base-also-fail.bst000066400000000000000000000001451514607367700305340ustar00rootroot00000000000000kind: script build-depends: - base.bst config: commands: - touch %{install-root}/foo - false apache-buildstream-27ae392/tests/integration/cached-fail/elements/base-fail.bst000066400000000000000000000001451514607367700276000ustar00rootroot00000000000000kind: script build-depends: - base.bst config: commands: - touch %{install-root}/foo - false apache-buildstream-27ae392/tests/integration/cached-fail/elements/base-success.bst000066400000000000000000000001331514607367700303320ustar00rootroot00000000000000kind: script build-depends: - base.bst config: commands: - touch %{install-root}/foo apache-buildstream-27ae392/tests/integration/cached-fail/elements/base.bst000066400000000000000000000010061514607367700266640ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" depends-on-base-fail-expect-foo.bst000066400000000000000000000001361514607367700336220ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/cached-fail/elementskind: script build-depends: - base.bst - base-fail.bst config: commands: - test -e /foo apache-buildstream-27ae392/tests/integration/cached-fail/elements/depends-on-two-failures.bst000066400000000000000000000001501514607367700324240ustar00rootroot00000000000000kind: script build-depends: - base-fail.bst - base-also-fail.bst config: commands: - test -e /foo apache-buildstream-27ae392/tests/integration/cached-fail/project.conf000066400000000000000000000011641514607367700257460ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ project_dir: file://{project_dir} plugins: - origin: pip package-name: sample-plugins elements: - autotools options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture variable: build_arch values: - x86-64 - aarch64 environment: TEST_VAR: pony split-rules: test: - | /tests - | /tests/* apache-buildstream-27ae392/tests/integration/cachedfail.py000066400000000000000000000513301514607367700237260ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os from contextlib import ExitStack import pytest from buildstream import utils, _yaml from buildstream.exceptions import ErrorDomain from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils import ( create_artifact_share, assert_shared, assert_not_shared, ) pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "cached-fail") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_build_checkout_cached_fail(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Try to build it, this should result in a failure that contains the content result = cli.run(project=project, args=["build", "base-fail.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "base-fail.bst") == "failed" # Now check it out result = cli.run(project=project, args=["artifact", "checkout", "base-fail.bst", "--directory", checkout]) result.assert_success() # Check that the checkout contains the file created before failure filename = os.path.join(checkout, "foo") assert os.path.exists(filename) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_build_depend_on_cached_fail(cli, datafiles): project = str(datafiles) # Try to build it, this should result in caching a failure to build dep result = cli.run(project=project, args=["build", "base-fail.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "base-fail.bst") == "failed" # Now we should fail because we've a cached fail of dep result = cli.run(project=project, args=["build", "depends-on-base-fail-expect-foo.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's not yet built, since one of its dependencies isn't ready. assert cli.get_element_state(project, "depends-on-base-fail-expect-foo.bst") == "waiting" @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("on_error", ("continue", "quit")) def test_push_cached_fail(cli, tmpdir, datafiles, on_error): project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "depends": [ { "filename": "base.bst", "type": "build", }, ], "config": { "commands": [ "false", # Ensure unique cache key for different test variants 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Build the element, continuing to finish active jobs on error. result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst")) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("on_error", ("continue", "quit")) def test_push_failed_missing_shell(cli, tmpdir, datafiles, on_error): """Test that we can upload a built artifact that didn't have a valid shell inside. When we don't have a valid shell, the artifact will be empty, not even the root directory. This ensures we handle the case of an entirely empty artifact correctly. """ project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "config": { "commands": [ "false", # Ensure unique cache key for different test variants 'TEST="{}"'.format(os.environ.get("PYTEST_CURRENT_TEST")), ], }, } _yaml.roundtrip_dump(element, element_path) with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Build the element, continuing to finish active jobs on error. result = cli.run(project=project, args=["--on-error={}".format(on_error), "build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # This element should have failed assert cli.get_element_state(project, "element.bst") == "failed" # This element should have been pushed to the remote assert share.get_artifact(cli.get_artifact_name(project, "test", "element.bst")) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path): # Create symlink to buildbox-casd to work with custom PATH buildbox_casd = tmp_path.joinpath("bin/buildbox-casd") buildbox_casd.parent.mkdir() os.symlink(utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox"), str(buildbox_casd)) project = str(datafiles) # Build without access to host tools, this will fail result1 = cli.run( project=project, args=["build", "base-success.bst"], env={"PATH": str(tmp_path.joinpath("bin"))}, ) result1.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") assert cli.get_element_state(project, "base-success.bst") == "buildable" # When rebuilding, this should work result2 = cli.run(project=project, args=["build", "base-success.bst"]) result2.assert_success() assert cli.get_element_state(project, "base-success.bst") == "cached" # Tests that failed builds will be retried if --retry-failed is specified # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.parametrize("use_share", (False, True), ids=["local-cache", "pull-failed-artifact"]) @pytest.mark.parametrize("retry", (True, False), ids=["retry", "no-retry"]) @pytest.mark.parametrize("strict", (True, False), ids=["strict", "non-strict"]) def test_retry_failed(cli, tmpdir, datafiles, use_share, retry, strict): project = str(datafiles) # Use separate cache directories for each iteration of this test # even though we're using cli_integration # # Global nonstrict configuration ensures all commands will be non-strict cli.configure({"cachedir": cli.directory, "projects": {"test": {"strict": strict}}}) with ExitStack() as stack: if use_share: share = stack.enter_context(create_artifact_share(os.path.join(str(tmpdir), "artifactshare"))) cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) # Try to build it, this should result in caching a failure of the target result = cli.run(project=project, args=["build", "base-fail.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "base-fail.bst") == "failed" if use_share: # Delete the local cache, provoke pulling of the failed build cli.remove_artifact_from_cache(project, "base-fail.bst") # Assert that the failed build has been removed assert cli.get_element_state(project, "base-fail.bst") == "buildable" # Even though we are in non-strict mode, the failed build should be retried if retry: result = cli.run(project=project, args=["build", "--retry-failed", "base-fail.bst"]) else: result = cli.run(project=project, args=["build", "base-fail.bst"]) # If we did not modify the cache key, we want to assert that we did not # in fact attempt to rebuild the failed artifact. # # Since the UX is very similar, we'll distinguish this by counting the number of # build logs which were produced. # result.assert_main_error(ErrorDomain.STREAM, None) if retry: assert "base-fail.bst" in result.get_built_elements() assert "base-fail.bst" in result.get_discarded_elements() else: assert "base-fail.bst" not in result.get_built_elements() assert "base-fail.bst" not in result.get_discarded_elements() if use_share: # Assert that we did indeed go through the motions of downloading the failed # build, and possibly discarded the failed artifact if the strong key did not match # assert "base-fail.bst" in result.get_pulled_elements() # Tests that failed builds will be retried in strict mode when dependencies have changed. # # This test ensures: # o Fixing a dependency such that the reverse dependency will succeed, gets automatically retried # o A subsequent retry of the same failed build will not trigger a retry attempt # o The same behavior is observed when a failed build artifact is downloaded from a remote cache # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.parametrize("use_share", (False, True), ids=["local-cache", "pull-failed-artifact"]) @pytest.mark.parametrize("success", (True, False), ids=["success", "no-success"]) def test_nonstrict_retry_failed(cli, tmpdir, datafiles, use_share, success): project = str(datafiles) intermediate_path = os.path.join(project, "elements", "intermediate.bst") dep_path = os.path.join(project, "elements", "dep.bst") target_path = os.path.join(project, "elements", "target.bst") # Use separate cache directories for each iteration of this test # even though we're using cli_integration # # Global nonstrict configuration ensures all commands will be non-strict cli.configure({"cachedir": cli.directory, "projects": {"test": {"strict": False}}}) def generate_dep(filename, dependency): return { "kind": "manual", "depends": [dependency], "config": { "install-commands": [ "touch %{install-root}/" + filename, ], }, } def generate_target(): return { "kind": "manual", "depends": [ "dep.bst", ], "config": { "build-commands": [ "test -e /foo", ], }, } with ExitStack() as stack: if use_share: share = stack.enter_context(create_artifact_share(os.path.join(str(tmpdir), "artifactshare"))) cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) intermediate = generate_dep("pony", "base.bst") dep = generate_dep("bar", "intermediate.bst") target = generate_target() _yaml.roundtrip_dump(intermediate, intermediate_path) _yaml.roundtrip_dump(dep, dep_path) _yaml.roundtrip_dump(target, target_path) # First build the dep / intermediate elements result = cli.run(project=project, args=["build", "dep.bst"]) result.assert_success() # Remove the intermediate element from cache, rebuild the dep, such that only # a weak key for the dep is possible cli.remove_artifact_from_cache(project, "intermediate.bst") intermediate = generate_dep("horsy", "base.bst") _yaml.roundtrip_dump(intermediate, intermediate_path) result = cli.run(project=project, args=["build", "dep.bst"]) result.assert_success() assert "dep.bst" not in result.get_built_elements() # Try to build it, this should result in caching a failure of the target result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "target.bst") == "failed" if use_share: # Delete the local cache, provoke pulling of the failed build cli.remove_artifact_from_cache(project, "target.bst") # Assert that the failed build has been removed assert cli.get_element_state(project, "target.bst") == "buildable" # Regenerate the dependency so that the target would succeed to build if success: dep = generate_dep("foo", "intermediate.bst") _yaml.roundtrip_dump(dep, dep_path) # Even though we are in non-strict mode, the failed build should be retried result = cli.run(project=project, args=["build", "target.bst"]) # Because the intermediate.bst is changed, the failed target.bst will be # retried unconditionally, assert that it gets discarded. # assert "target.bst" in result.get_discarded_elements() if success: result.assert_success() else: result.assert_main_error(ErrorDomain.STREAM, None) if use_share: # Assert that we did indeed go through the motions of downloading the failed # build, and possibly discarded the failed artifact if the strong key did not match # assert "target.bst" in result.get_pulled_elements() # Tests that failed build artifacts in non-strict mode can be deleted. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_nonstrict_delete_failed(cli, tmpdir, datafiles): project = str(datafiles) intermediate_path = os.path.join(project, "elements", "intermediate.bst") dep_path = os.path.join(project, "elements", "dep.bst") target_path = os.path.join(project, "elements", "target.bst") # Global nonstrict configuration ensures all commands will be non-strict cli.configure({"projects": {"test": {"strict": False}}}) def generate_dep(filename, dependency): return { "kind": "manual", "depends": [dependency], "config": { "install-commands": [ "touch %{install-root}/" + filename, ], }, } def generate_target(): return { "kind": "manual", "depends": [ "dep.bst", ], "config": { "build-commands": [ "test -e /foo", ], }, } intermediate = generate_dep("pony", "base.bst") dep = generate_dep("bar", "intermediate.bst") target = generate_target() _yaml.roundtrip_dump(intermediate, intermediate_path) _yaml.roundtrip_dump(dep, dep_path) _yaml.roundtrip_dump(target, target_path) # First build the dep / intermediate elements result = cli.run(project=project, args=["build", "dep.bst"]) result.assert_success() # Remove the intermediate element from cache, rebuild the dep, such that only # a weak key for the dep is possible cli.remove_artifact_from_cache(project, "intermediate.bst") intermediate = generate_dep("horsy", "base.bst") _yaml.roundtrip_dump(intermediate, intermediate_path) result = cli.run(project=project, args=["build", "dep.bst"]) result.assert_success() assert "dep.bst" not in result.get_built_elements() # Try to build it, this should result in caching a failure of the target result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that it's cached in a failed artifact assert cli.get_element_state(project, "target.bst") == "failed" # Delete it result = cli.run(project=project, args=["artifact", "delete", "target.bst"]) # Assert that it's no longer cached, and returns to a buildable state assert cli.get_element_state(project, "target.bst") == "buildable" # Test that we do not keep scheduling builds after one build fails # with `--builders 1` and `--on-error quit`. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_stop_building_after_failed(cli, datafiles): project = str(datafiles) # Since integration tests share a local artifact cache (for test performance, avoiding # downloading a runtime in every test), we reset the local cache state here by # deleting the two failing artifacts we are testing with # cli.remove_artifact_from_cache(project, "base-fail.bst") cli.remove_artifact_from_cache(project, "base-also-fail.bst") # Set only 1 builder, and explicitly configure `--on-error quit` cli.configure({"scheduler": {"builders": 1, "on-error": "quit"}}) # Try to build it, this should result in only one failure result = cli.run(project=project, args=["build", "depends-on-two-failures.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that out of the two elements, only one of them failed, the other one is # buildable because they both depend on base.bst which must have succeeded. states = cli.get_element_states(project, ["base-fail.bst", "base-also-fail.bst"], deps="none") assert "failed" in states.values() assert "buildable" in states.values() # Test that we do push the failed build artifact, but we do not keep scheduling # builds after one build fails with `--builders 1` and `--on-error quit`. # @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_push_but_stop_building_after_failed(cli, tmpdir, datafiles): project = str(datafiles) # Since integration tests share a local artifact cache (for test performance, avoiding # downloading a runtime in every test), we reset the local cache state here by # deleting the two failing artifacts we are testing with # cli.remove_artifact_from_cache(project, "base-fail.bst") cli.remove_artifact_from_cache(project, "base-also-fail.bst") with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: # Set only 1 builder, and explicitly configure `--on-error quit` cli.configure( { "scheduler": {"builders": 1, "on-error": "quit"}, "artifacts": {"servers": [{"url": share.repo, "push": True}]}, } ) # Try to build it, this should result in only one failure result = cli.run(project=project, args=["build", "depends-on-two-failures.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) # Assert that out of the two elements, only one of them failed, the other one is # buildable because they both depend on base.bst which must have succeeded. states = cli.get_element_states(project, ["base-fail.bst", "base-also-fail.bst"], deps="none") assert "failed" in states.values() assert "buildable" in states.values() # Assert that the failed build is cached in a failed artifact, and that the other build # which would have failed, of course never made it to the artifact cache. for element_name, state in states.items(): if state == "buildable": assert_not_shared(cli, share, project, element_name) elif state == "failed": assert_shared(cli, share, project, element_name) else: assert False, "Unreachable code reached !" apache-buildstream-27ae392/tests/integration/compose-symlinks.py000066400000000000000000000033271514607367700252020ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that staging a file inside a directory symlink fails. # # Regression test for https://gitlab.com/BuildStream/buildstream/issues/270 # noinspection PyUnusedLocal @pytest.mark.datafiles(DATA_DIR) def test_compose_symlinks(cli, tmpdir, datafiles): project = str(datafiles) # Symlinks do not survive being placed in a source distribution # ('setup.py sdist'), so we have to create the one we need here. project_files = os.path.join(project, "files", "compose-symlinks", "base") symlink_file = os.path.join(project_files, "sbin") os.symlink(os.path.join("usr", "sbin"), symlink_file, target_is_directory=True) result = cli.run(project=project, args=["build", "compose-symlinks/compose.bst"]) assert result.exit_code == -1 assert "Destination is a symlink, not a directory: /sbin" in result.stderr apache-buildstream-27ae392/tests/integration/compose.py000066400000000000000000000140001514607367700233210ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import walk_dir from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_compose_element(name, path, config=None): if config is None: config = {} element = { "kind": "compose", "depends": [ {"filename": "compose/amhello.bst", "type": "build"}, {"filename": "compose/test.bst", "type": "build"}, ], "config": config, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "include_domains,exclude_domains,expected", [ # Test flat inclusion ( [], [], [ "/usr", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", "/tests", "/tests/test", ], ), # Test only runtime (["runtime"], [], ["/usr", "/usr/share", "/usr/bin", "/usr/bin/hello"]), # Test with runtime and doc ( ["runtime", "doc"], [], [ "/usr", "/usr/share", "/usr/bin", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ), # Test with only runtime excluded ( [], ["runtime"], [ "/usr", "/usr/share", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", "/tests", "/tests/test", ], ), # Test with runtime and doc excluded ([], ["runtime", "doc"], ["/usr", "/usr/share", "/tests", "/tests/test"]), # Test with runtime simultaneously in- and excluded (["runtime"], ["runtime"], ["/usr", "/usr/share"]), # Test with runtime included and doc excluded (["runtime"], ["doc"], ["/usr", "/usr/share", "/usr/bin", "/usr/bin/hello"]), # Test including a custom 'test' domain (["test"], [], ["/usr", "/usr/share", "/tests", "/tests/test"]), # Test excluding a custom 'test' domain ( [], ["test"], [ "/usr", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ), ], ) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_compose_include(cli, datafiles, include_domains, exclude_domains, expected): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "compose/compose-amhello.bst" # Create a yaml configuration from the specified include and # exclude domains config = {"include": include_domains, "exclude": exclude_domains} create_compose_element(element_name, element_path, config=config) result = cli.run(project=project, args=["source", "track", "compose/amhello.bst"]) assert result.exit_code == 0 result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 assert set(walk_dir(checkout)) == set(expected) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Root directory not writable with userchroot", ) def test_compose_run_integration(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "compose/compose-amhello.bst" element = { "kind": "compose", "depends": [ {"filename": "compose/amhello.bst", "type": "build"}, {"filename": "compose/test-integration.bst", "type": "build"}, ], "config": {"include": ["runtime"]}, } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = cli.run(project=project, args=["source", "track", "compose/amhello.bst"]) assert result.exit_code == 0 result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 test_file = os.path.join(checkout, "tests", "test") assert os.path.isfile(test_file) apache-buildstream-27ae392/tests/integration/digest-environment.py000066400000000000000000000140271514607367700255060ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test that the digest environment variable is set correctly during a build @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_base(cli, datafiles): project = str(datafiles) element_name = "digest-environment/base.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name]) assert result.exit_code == 0 # Test that the digest environment variable is not affected by unrelated build dependencies @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_base_plus_extra_dep(cli, datafiles): project = str(datafiles) element_name = "digest-environment/base-plus-extra-dep.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test that multiple dependencies can be merged into a single digest environment variable @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_merge(cli, datafiles): project = str(datafiles) element_name = "digest-environment/merge.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test that multiple digest environment variables can be configured in a single element @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_two(cli, datafiles): project = str(datafiles) element_name = "digest-environment/two.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test that the digest environment variable is also set in a build shell @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_shell(cli, datafiles): project = str(datafiles) element_name = "digest-environment/base.bst" # Ensure artifacts of build dependencies are available for build shell result = cli.run(project=project, args=["build", "--deps", "build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["shell", "--build", element_name, "--", "sh", "-c", "echo $BASE_DIGEST"]) assert result.exit_code == 0 assert result.output.strip() == "63450d93eab71f525d08378fe50960aff92b0ec8f1b0be72b2ac4b8259d09833/1227" # Test that the digest environment variable is also set in a build shell staged from a buildtree @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_shell_buildtree(cli, datafiles): project = str(datafiles) element_name = "digest-environment/base-buildtree.bst" # Generate buildtree result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 result = cli.run( project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "echo $BASE_DIGEST"], ) assert result.exit_code == 0 assert result.output.strip() == "63450d93eab71f525d08378fe50960aff92b0ec8f1b0be72b2ac4b8259d09833/1227" # Test that buildtree push works for elements with a digest environment variable @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_pushed_buildtree(cli, tmpdir, datafiles): project = str(datafiles) element_name = "digest-environment/merge.bst" with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: cli.configure( { "artifacts": {"servers": [{"url": share.repo, "push": True}]}, "cachedir": str(tmpdir), "cache": {"cache-buildtrees": "always"}, } ) # Generate buildtree result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == "cached" assert share.get_artifact(cli.get_artifact_name(project, "test", element_name)) # Clear the local cache to make sure everything can and will be pulled from the remote shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "artifacts")) result = cli.run( project=project, args=[ "--pull-buildtrees", "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "echo $MERGED_DIGEST", ], ) assert result.exit_code == 0 assert result.output.strip() == "469369597f4faa56c4b8338d6a948c8c1d4f29e6ea8f4d4d261cac4182bcef48/1389" apache-buildstream-27ae392/tests/integration/filter.py000066400000000000000000000062401514607367700231500ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Root directory not writable with userchroot", ) def test_filter_pass_integration(datafiles, cli): project = str(datafiles) # Passing integration commands should build nicely result = cli.run(project=project, args=["build", "filter/filter.bst"]) result.assert_success() # Checking out the element should work checkout_dir = os.path.join(project, "filter") result = cli.run( project=project, args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "filter/filter.bst"], ) result.assert_success() # Check that the integration command was run assert_contains(checkout_dir, ["/foo"]) shutil.rmtree(checkout_dir) @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Root directory not writable with userchroot", ) def test_filter_pass_integration_uncached(datafiles, cli): project = str(datafiles) # Passing integration commands should build nicely result = cli.run(project=project, args=["build", "filter/filter.bst"]) result.assert_success() # Delete the build dependency of the filter element. # The built filter element should be usable even if the build dependency # is not available in the local cache. result = cli.run(project=project, args=["artifact", "delete", "filter/parent.bst"]) result.assert_success() # Checking out the element should work checkout_dir = os.path.join(project, "filter") result = cli.run( project=project, args=["artifact", "checkout", "--integrate", "--directory", checkout_dir, "filter/filter.bst"], ) result.assert_success() # Check that the integration command was run assert_contains(checkout_dir, ["/foo"]) shutil.rmtree(checkout_dir) apache-buildstream-27ae392/tests/integration/import.py000066400000000000000000000046551514607367700232050ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import walk_dir pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_import_element(name, path, source, target, source_path): element = { "kind": "import", "sources": [{"kind": "local", "path": source_path}], "config": {"source": source, "target": target}, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "source,target,path,expected", [ ("/", "/", "files/import-source", ["/test.txt", "/subdir", "/subdir/test.txt"]), ("/subdir", "/", "files/import-source", ["/test.txt"]), ("/", "/", "files/import-source/subdir", ["/test.txt"]), ( "/", "/output", "files/import-source", ["/output", "/output/test.txt", "/output/subdir", "/output/subdir/test.txt"], ), ], ) def test_import(cli, datafiles, source, target, path, expected): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "import/import.bst" create_import_element(element_name, element_path, source, target, path) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 assert set(walk_dir(checkout)) == set(expected) apache-buildstream-27ae392/tests/integration/interactive_build.py000066400000000000000000000122231514607367700253550ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pexpect import pytest from buildstream._testing import integration_cache # pylint: disable=unused-import from buildstream._testing import runcli from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils.constants import PEXPECT_TIMEOUT_SHORT, PEXPECT_TIMEOUT_LONG pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # This fixture launches a `bst build` of given element, and returns a # `pexpect.spawn` object for the interactive session. @pytest.fixture def build_session(integration_cache, datafiles, element_name): project = str(datafiles) # Spawn interactive session using `configured()` context manager in order # to get the same config file as the `cli` fixture. with runcli.configured(project, config={"sourcedir": integration_cache.sources}) as config_file: session = pexpect.spawn( "bst", [ "--directory", project, "--config", config_file, "--no-colors", "build", element_name, ], timeout=PEXPECT_TIMEOUT_SHORT, ) yield session # Verify that BuildStream exits cleanly on any of the following choices. # # In our simple test case, there is no practical difference between the # following choices. In future, we'd like to test their behavior separately. # Currently, this just verifies that BuildStream doesn't choke on any of these # choices. @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("element_name", ["interactive/failed-build.bst"]) @pytest.mark.parametrize("choice", ["continue", "quit", "terminate"]) def test_failed_build_quit(element_name, build_session, choice): build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG) build_session.sendline(choice) build_session.expect_exact(pexpect.EOF) build_session.close() assert build_session.exitstatus == 255 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("element_name", ["interactive/failed-build.bst"]) def test_failed_build_retry(element_name, build_session): build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG) build_session.sendline("retry") build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG) build_session.sendline("quit") build_session.expect_exact(pexpect.EOF) build_session.close() assert build_session.exitstatus == 255 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("element_name", ["interactive/failed-build.bst"]) def test_failed_build_log(element_name, build_session): build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG) build_session.sendline("log") # Send a few carriage returns to get to the end of the pager build_session.sendline(os.linesep * 20) # Assert that we got something from the logs build_session.expect("FAILURE interactive/failed-build.bst: Running (build-)?commands") # Quit the pager build_session.send("q") # Quit the session build_session.expect_exact("Choice: [continue]:") build_session.sendline("quit") build_session.expect_exact(pexpect.EOF) build_session.close() assert build_session.exitstatus == 255 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("element_name", ["interactive/failed-build.bst"]) def test_failed_build_shell(element_name, build_session): build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG) build_session.sendline("shell") # Wait for shell prompt build_session.expect_exact("interactive/failed-build.bst:/buildstream/test/interactive/failed-build.bst]") # Verify that we have our sources build_session.sendline("ls") build_session.expect_exact("test.txt") # Quit the shell build_session.sendline("exit") # Quit the session build_session.expect_exact("Choice: [continue]:", timeout=PEXPECT_TIMEOUT_LONG) build_session.sendline("quit") build_session.expect_exact(pexpect.EOF) build_session.close() assert build_session.exitstatus == 255 apache-buildstream-27ae392/tests/integration/manual.py000066400000000000000000000202101514607367700231310ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_manual_element(name, path, config, variables, environment, sources=None): element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "config": config, "variables": variables, "environment": environment, } if sources: element["sources"] = sources os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "import/import.bst" create_manual_element( element_name, element_path, { "configure-commands": ["echo './configure' >> test"], "build-commands": ["echo 'make' >> test"], "install-commands": ["echo 'make install' >> test", "cp test %{install-root}"], "strip-commands": ["echo 'strip' >> %{install-root}/test"], }, {}, {}, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert ( text == """./configure make make install strip """ ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element_environment(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "import/import.bst" create_manual_element( element_name, element_path, {"install-commands": ["echo $V >> test", "cp test %{install-root}"]}, {}, {"V": 2} ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert text == "2\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element_noparallel(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "import/import.bst" create_manual_element( element_name, element_path, {"install-commands": ["echo $MAKEFLAGS >> test", "echo $V >> test", "cp test %{install-root}"]}, {"notparallel": True}, {"MAKEFLAGS": "-j%{max-jobs} -Wall", "V": 2}, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert ( text == """-j1 -Wall 2 """ ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_element_logging(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements") element_name = "import/import.bst" create_manual_element( element_name, element_path, { "configure-commands": ["echo configure"], "build-commands": ["echo build"], "install-commands": ["echo install"], "strip-commands": ["echo strip"], }, {}, {}, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Verify that individual commands are logged assert "echo configure" in res.stderr assert "echo build" in res.stderr assert "echo install" in res.stderr assert "echo strip" in res.stderr # Regression test for https://gitlab.com/BuildStream/buildstream/-/issues/1295. # # Test that the command-subdir variable works as expected. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_command_subdir(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "manual/command-subdir.bst" sources = [{"kind": "local", "path": "files/manual-element/root"}] create_manual_element( element_name, element_path, {"install-commands": ["cp hello %{install-root}"]}, {}, {}, sources=sources, ) # First, verify that element builds, and has the correct expected output. result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() with open(os.path.join(checkout, "hello"), encoding="utf-8") as f: assert f.read() == "hello from root\n" # Now, change element configuration to have a different command-subdir. # This should result in a different cache key. create_manual_element( element_name, element_path, {"install-commands": ["cp hello %{install-root}"]}, {"command-subdir": "subdir"}, {}, sources=sources, ) # Verify that the element needs to be rebuilt. assert cli.get_element_state(project, element_name) == "buildable" # Finally, ensure that the variable actually takes effect. result = cli.run(project=project, args=["build", element_name]) result.assert_success() shutil.rmtree(checkout) result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() with open(os.path.join(checkout, "hello"), encoding="utf-8") as f: assert f.read() == "hello from subdir\n" # Test staging artifacts into subdirectories @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_manual_stage_custom(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") # Verify that the element builds, and has the correct expected output. result = cli.run(project=project, args=["build", "manual/manual-stage-custom.bst"]) result.assert_success() result = cli.run( project=project, args=["artifact", "checkout", "manual/manual-stage-custom.bst", "--directory", checkout] ) result.assert_success() with open(os.path.join(checkout, "test.txt"), encoding="utf-8") as f: assert f.read() == "This is another test\n" apache-buildstream-27ae392/tests/integration/messages.py000066400000000000000000000066121514607367700234750ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: Tristan Maat # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_disable_message_lines(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements") element_name = "message.bst" element = { "kind": "manual", "depends": [{"filename": "base.bst"}], "config": {"build-commands": ['echo "Silly message"'], "strip-commands": []}, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # First we check that we get the "Silly message" result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert 'echo "Silly message"' in result.stderr # Let's now build it again, but with --message-lines 0 cli.remove_artifact_from_cache(project, element_name) result = cli.run(project=project, args=["--message-lines", "0", "build", element_name]) result.assert_success() assert "Message contains " not in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_disable_error_lines(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements") element_name = "message.bst" element = { "kind": "manual", "depends": [{"filename": "base.bst"}], "config": {"build-commands": ["This is a syntax error > >"], "strip-commands": []}, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # First we check that we get the syntax error result = cli.run(project=project, args=["--error-lines", "0", "build", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) assert "This is a syntax error" in result.stderr # Let's now build it again, but with --error-lines 0 cli.remove_artifact_from_cache(project, element_name) result = cli.run(project=project, args=["--error-lines", "0", "build", element_name]) result.assert_main_error(ErrorDomain.STREAM, None) assert "Printing the last" not in result.stderr apache-buildstream-27ae392/tests/integration/project/000077500000000000000000000000001514607367700227555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/000077500000000000000000000000001514607367700245715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/autotools/000077500000000000000000000000001514607367700266225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/autotools/amhello-failure.bst000066400000000000000000000004241514607367700324020ustar00rootroot00000000000000kind: autotools description: An element which is meant to fail config: build-commands: (>): - this-will-fail depends: - base.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 apache-buildstream-27ae392/tests/integration/project/elements/autotools/amhello.bst000066400000000000000000000003101514607367700307470ustar00rootroot00000000000000kind: autotools description: Autotools test depends: - base.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 apache-buildstream-27ae392/tests/integration/project/elements/autotools/amhelloconfroot.bst000066400000000000000000000004541514607367700325320ustar00rootroot00000000000000kind: autotools description: Autotools test depends: - base.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 directory: SourceFile variables: conf-root: "%{build-root}/SourceFile" command-subdir: build apache-buildstream-27ae392/tests/integration/project/elements/base.bst000066400000000000000000000001031514607367700262070ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/integration/project/elements/base/000077500000000000000000000000001514607367700255035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/base/base-alpine.bst000066400000000000000000000010201514607367700303660ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "test-images:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "test-images:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/integration/project/elements/base/base-debian.bst000066400000000000000000000005021514607367700303440ustar00rootroot00000000000000kind: import description: | Debian Linux base for tests Generated using the `tests/integration/base/generate-debian-base.sh` script. sources: - kind: tar base-dir: '' ref: 4964ce8b12f2200a3cf1570ac93756bea27055dad771bf2be97751c7a2a0f92c url: test-images:integration-tests-debian-base.v1.x86_64.tar.xz apache-buildstream-27ae392/tests/integration/project/elements/build-shell/000077500000000000000000000000001514607367700267755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/build-shell/buildtree-fail.bst000066400000000000000000000004641514607367700324030ustar00rootroot00000000000000kind: manual description: | Puts a file in the build tree so that build tree caching and staging can be tested, then deliberately failing to build so we can check the output. depends: - filename: base.bst type: build config: build-commands: - "echo 'Hi' > %{build-root}/test" - "false" apache-buildstream-27ae392/tests/integration/project/elements/build-shell/buildtree.bst000066400000000000000000000003451514607367700314700ustar00rootroot00000000000000kind: manual description: | Puts a file in the build tree so that build tree caching and staging can be tested. depends: - filename: base.bst type: build config: build-commands: - "echo 'Hi' > %{build-root}/test" apache-buildstream-27ae392/tests/integration/project/elements/build-shell/compose-dep-fail.bst000066400000000000000000000002171514607367700326330ustar00rootroot00000000000000kind: manual depends: - base.bst config: build-commands: - "echo 'Goodbye world'" public: bst: integration-commands: - exit 1 apache-buildstream-27ae392/tests/integration/project/elements/build-shell/compose-dep-success.bst000066400000000000000000000002471514607367700333730ustar00rootroot00000000000000kind: manual depends: - base.bst config: build-commands: - "echo 'Hello world'" public: bst: integration-commands: - echo "Hi" > /integration-success apache-buildstream-27ae392/tests/integration/project/elements/build-shell/compose-fail.bst000066400000000000000000000001011514607367700320550ustar00rootroot00000000000000kind: compose build-depends: - build-shell/compose-dep-fail.bst apache-buildstream-27ae392/tests/integration/project/elements/build-shell/compose-success.bst000066400000000000000000000001041514607367700326150ustar00rootroot00000000000000kind: compose build-depends: - build-shell/compose-dep-success.bst apache-buildstream-27ae392/tests/integration/project/elements/build-shell/script.bst000066400000000000000000000001251514607367700310110ustar00rootroot00000000000000kind: script build-depends: - base.bst config: commands: - "echo 'Hi' > /test" apache-buildstream-27ae392/tests/integration/project/elements/build-uid/000077500000000000000000000000001514607367700264475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/build-uid/build-uid-1023.bst000066400000000000000000000002061514607367700314200ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: build-commands: - "[ `id -u` = 1023 -a `id -g` = 3490 ]" apache-buildstream-27ae392/tests/integration/project/elements/build-uid/build-uid-default.bst000066400000000000000000000001771514607367700324660ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: build-commands: - "[ `id -u` = 0 -a `id -g` = 0 ]"apache-buildstream-27ae392/tests/integration/project/elements/build-uid/build-uid.bst000066400000000000000000000002641514607367700310410ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build sandbox: build-uid: 1024 build-gid: 1048 config: build-commands: - "[ `id -u` = 1024 -a `id -g` = 1048 ]" apache-buildstream-27ae392/tests/integration/project/elements/compose-symlinks/000077500000000000000000000000001514607367700301055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/compose-symlinks/base.bst000066400000000000000000000001111514607367700315220ustar00rootroot00000000000000kind: import sources: - kind: local path: files/compose-symlinks/base apache-buildstream-27ae392/tests/integration/project/elements/compose-symlinks/compose.bst000066400000000000000000000002431514607367700322630ustar00rootroot00000000000000kind: compose depends: - filename: compose-symlinks/base.bst type: build - filename: compose-symlinks/overlay.bst type: build config: include: - runtime apache-buildstream-27ae392/tests/integration/project/elements/compose-symlinks/overlay.bst000066400000000000000000000001141514607367700322740ustar00rootroot00000000000000kind: import sources: - kind: local path: files/compose-symlinks/overlay apache-buildstream-27ae392/tests/integration/project/elements/compose/000077500000000000000000000000001514607367700262365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/compose/amhello.bst000066400000000000000000000003521514607367700303710ustar00rootroot00000000000000kind: autotools description: Autotools test depends: - filename: base.bst type: build sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 apache-buildstream-27ae392/tests/integration/project/elements/compose/test-integration.bst000066400000000000000000000002171514607367700322500ustar00rootroot00000000000000kind: stack depends: - base.bst public: bst: integration-commands: - "mkdir -p /tests" - "echo 'This is a test' > /tests/test" apache-buildstream-27ae392/tests/integration/project/elements/compose/test.bst000066400000000000000000000002651514607367700277320ustar00rootroot00000000000000kind: script depends: - filename: base.bst type: build config: commands: - "mkdir -p %{install-root}/tests" - "echo 'This is a test' > %{install-root}/tests/test" apache-buildstream-27ae392/tests/integration/project/elements/digest-environment/000077500000000000000000000000001514607367700304125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/digest-environment/base-buildtree.bst000066400000000000000000000003711514607367700340140ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: digest-environment: BASE_DIGEST config: build-commands: - env - test "$BASE_DIGEST" = "63450d93eab71f525d08378fe50960aff92b0ec8f1b0be72b2ac4b8259d09833/1227" base-plus-extra-dep.bst000066400000000000000000000004561514607367700346340ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/digest-environmentkind: manual depends: - filename: base.bst type: build config: digest-environment: BASE_DIGEST - filename: manual/import-file.bst type: build config: build-commands: - env - test "$BASE_DIGEST" = "63450d93eab71f525d08378fe50960aff92b0ec8f1b0be72b2ac4b8259d09833/1227" apache-buildstream-27ae392/tests/integration/project/elements/digest-environment/base.bst000066400000000000000000000003711514607367700320370ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: digest-environment: BASE_DIGEST config: build-commands: - env - test "$BASE_DIGEST" = "63450d93eab71f525d08378fe50960aff92b0ec8f1b0be72b2ac4b8259d09833/1227" apache-buildstream-27ae392/tests/integration/project/elements/digest-environment/merge.bst000066400000000000000000000005461514607367700322300ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: digest-environment: MERGED_DIGEST - filename: manual/import-file.bst type: build config: digest-environment: MERGED_DIGEST config: build-commands: - env - test "$MERGED_DIGEST" = "469369597f4faa56c4b8338d6a948c8c1d4f29e6ea8f4d4d261cac4182bcef48/1389" apache-buildstream-27ae392/tests/integration/project/elements/digest-environment/two.bst000066400000000000000000000007071514607367700317410ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: digest-environment: BASE_DIGEST - filename: manual/import-file.bst type: build config: digest-environment: IMPORT_DIGEST config: build-commands: - env - test "$BASE_DIGEST" = "63450d93eab71f525d08378fe50960aff92b0ec8f1b0be72b2ac4b8259d09833/1227" - test "$IMPORT_DIGEST" = "eec5ed9053acb296a8e7a30ab7ee173abf4f7392b8228ba7644cd5b51a5cfdeb/162" apache-buildstream-27ae392/tests/integration/project/elements/echo-env-var.bst000066400000000000000000000001721514607367700275750ustar00rootroot00000000000000kind: manual depends: - base.bst public: bst: integration-commands: - | echo $TEST_VAR > /etc/test.conf apache-buildstream-27ae392/tests/integration/project/elements/echo-target.bst000066400000000000000000000000511514607367700275010ustar00rootroot00000000000000kind: manual depends: - echo-env-var.bst apache-buildstream-27ae392/tests/integration/project/elements/filter/000077500000000000000000000000001514607367700260565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/filter/filter.bst000066400000000000000000000006251514607367700300600ustar00rootroot00000000000000kind: filter depends: - filename: filter/parent.bst type: build config: pass-integration: True # `sh` is `/bin/sh`, and `/bin` is not covered by any split rule # As a result, to include `sh` in the image, we need to include orphans # # However we can use this to minimise the size of the artifacts, which # will hopefully reduce test time # include-orphans: True exclude: - runtime apache-buildstream-27ae392/tests/integration/project/elements/filter/parent.bst000066400000000000000000000007041514607367700300620ustar00rootroot00000000000000kind: compose depends: - filename: base.bst type: build public: bst: integration-commands: - touch /foo config: # `sh` is `/bin/sh`, and `/bin` is not covered by any split rule # As a result, to include `sh` in the image, we need to include orphans # # However we can use this to minimise the size of the artifacts, which # will hopefully reduce test time # include-orphans: True include: - runtime exclude: - runtime apache-buildstream-27ae392/tests/integration/project/elements/integration.bst000066400000000000000000000001571514607367700276310ustar00rootroot00000000000000kind: manual depends: - base.bst public: bst: integration-commands: - | echo noise >/dev/null apache-buildstream-27ae392/tests/integration/project/elements/interactive/000077500000000000000000000000001514607367700271065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/interactive/failed-build.bst000066400000000000000000000002361514607367700321420ustar00rootroot00000000000000# This element must fail to build kind: manual sources: - kind: local path: files/import-source depends: - base.bst config: build-commands: - false apache-buildstream-27ae392/tests/integration/project/elements/manual/000077500000000000000000000000001514607367700260465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/manual/import-file.bst000066400000000000000000000001011514607367700307770ustar00rootroot00000000000000kind: import sources: - kind: local path: files/import-source apache-buildstream-27ae392/tests/integration/project/elements/manual/manual-stage-custom.bst000066400000000000000000000003171514607367700324470ustar00rootroot00000000000000kind: manual depends: - base.bst build-depends: - filename: manual/import-file.bst config: location: /flying-ponies config: install-commands: - cp /flying-ponies/subdir/test.txt %{install-root} apache-buildstream-27ae392/tests/integration/project/elements/recc/000077500000000000000000000000001514607367700255055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/recc/cacheonly.bst000066400000000000000000000012311514607367700301610ustar00rootroot00000000000000kind: autotools description: recc test with autotools build-depends: - filename: base/base-debian.bst config: digest-environment: RECC_REMOTE_PLATFORM_chrootRootDigest - recc/recc.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 sandbox: remote-apis-socket: path: /tmp/casd.sock action-cache-enable-update: true environment: CC: recc gcc RECC_LOG_LEVEL: debug RECC_LOG_DIRECTORY: .recc-log RECC_DEPS_GLOBAL_PATHS: 1 RECC_NO_PATH_REWRITE: 1 RECC_LINK: 1 RECC_CACHE_ONLY: 1 RECC_CACHE_UPLOAD_LOCAL_BUILD: 1 RECC_SERVER: unix:/tmp/casd.sock apache-buildstream-27ae392/tests/integration/project/elements/recc/recc.bst000066400000000000000000000003641514607367700271360ustar00rootroot00000000000000kind: import sources: - kind: tar base-dir: '' ref: d621d1641a5e9617d6dbe9bcebdf4c62a28e1354fcdb09c4dc94bd1a773e85d4 url: buildbox:buildbox-integration/-/releases/1.3.39/downloads/recc-x86_64-linux-gnu.tgz directory: usr/bin apache-buildstream-27ae392/tests/integration/project/elements/recc/remoteexecution.bst000066400000000000000000000010741514607367700314400ustar00rootroot00000000000000kind: autotools description: recc test with autotools build-depends: - filename: base/base-debian.bst config: digest-environment: RECC_REMOTE_PLATFORM_chrootRootDigest - recc/recc.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 sandbox: remote-apis-socket: path: /tmp/casd.sock environment: CC: recc gcc RECC_LOG_LEVEL: debug RECC_LOG_DIRECTORY: .recc-log RECC_DEPS_GLOBAL_PATHS: 1 RECC_NO_PATH_REWRITE: 1 RECC_LINK: 1 RECC_SERVER: unix:/tmp/casd.sock apache-buildstream-27ae392/tests/integration/project/elements/sandbox/000077500000000000000000000000001514607367700262275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/sandbox/build-arch.bst000066400000000000000000000002141514607367700307500ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build sandbox: build-arch: '%{build_arch}' config: build-commands: - true remote-apis-socket-ac-update.bst000066400000000000000000000003261514607367700342370ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/sandboxkind: manual depends: - filename: base.bst type: build sandbox: remote-apis-socket: path: /tmp/reapi.sock action-cache-enable-update: true config: build-commands: - test -S /tmp/reapi.sock apache-buildstream-27ae392/tests/integration/project/elements/sandbox/remote-apis-socket.bst000066400000000000000000000002611514607367700324530ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build sandbox: remote-apis-socket: path: /tmp/reapi.sock config: build-commands: - test -S /tmp/reapi.sock apache-buildstream-27ae392/tests/integration/project/elements/sandbox/test-dev-shm.bst000066400000000000000000000002421514607367700312570ustar00rootroot00000000000000kind: manual depends: - base.bst config: build-commands: - cc test_shm.c install-commands: - ./a.out sources: - kind: local path: files/test_shm.c apache-buildstream-27ae392/tests/integration/project/elements/script/000077500000000000000000000000001514607367700260755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/script/corruption-2.bst000066400000000000000000000001661514607367700311550ustar00rootroot00000000000000kind: script build-depends: - base.bst - script/corruption-image.bst config: commands: - echo smashed >>/canary apache-buildstream-27ae392/tests/integration/project/elements/script/corruption-image.bst000066400000000000000000000000711514607367700320710ustar00rootroot00000000000000kind: import sources: - kind: local path: files/canary apache-buildstream-27ae392/tests/integration/project/elements/script/corruption-integration.bst000066400000000000000000000001261514607367700333330ustar00rootroot00000000000000kind: stack public: bst: integration-commands: - echo smashed >>/canary apache-buildstream-27ae392/tests/integration/project/elements/script/corruption.bst000066400000000000000000000002121514607367700310060ustar00rootroot00000000000000kind: script build-depends: - base.bst - script/corruption-image.bst - script/corruption-integration.bst variables: install-root: "/" apache-buildstream-27ae392/tests/integration/project/elements/script/marked-tmpdir.bst000066400000000000000000000001751514607367700313520ustar00rootroot00000000000000kind: compose build-depends: - base.bst public: bst: split-rules: remove: - "/tmp/**" - "/tmp" apache-buildstream-27ae392/tests/integration/project/elements/script/no-tmpdir.bst000066400000000000000000000002031514607367700305130ustar00rootroot00000000000000kind: filter depends: - filename: script/marked-tmpdir.bst type: build config: exclude: - remove include-orphans: True apache-buildstream-27ae392/tests/integration/project/elements/script/script-layout.bst000066400000000000000000000004731514607367700314320ustar00rootroot00000000000000kind: script description: Write to root using a script element variables: install-root: /buildstream/nstall build-root: /buildstream/uild build-depends: - base.bst - filename: script/script.bst config: location: /buildstream/uild config: commands: - "cp %{build-root}/test %{install-root}" apache-buildstream-27ae392/tests/integration/project/elements/script/script.bst000066400000000000000000000001771514607367700301200ustar00rootroot00000000000000kind: script description: Script test build-depends: - base.bst config: commands: - "echo 'Hi' > %{install-root}/test" apache-buildstream-27ae392/tests/integration/project/elements/script/tmpdir.bst000066400000000000000000000001461514607367700301070ustar00rootroot00000000000000kind: script build-depends: - script/no-tmpdir.bst config: commands: - | mkdir -p /tmp/blah apache-buildstream-27ae392/tests/integration/project/elements/sockets/000077500000000000000000000000001514607367700262445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/sockets/make-builddir-socket.bst000066400000000000000000000003601514607367700327540ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: build-commands: - | python3 -c ' from socket import socket, AF_UNIX, SOCK_STREAM s = socket(AF_UNIX, SOCK_STREAM) s.bind("testsocket") ' apache-buildstream-27ae392/tests/integration/project/elements/sockets/make-install-root-socket.bst000066400000000000000000000005131514607367700336050ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: install-commands: - | python3 -c ' from os.path import join from sys import argv from socket import socket, AF_UNIX, SOCK_STREAM s = socket(AF_UNIX, SOCK_STREAM) s.bind(join(argv[1], "testsocket")) ' %{install-root} apache-buildstream-27ae392/tests/integration/project/elements/stack/000077500000000000000000000000001514607367700256765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/stack/another-hi.bst000066400000000000000000000003101514607367700304400ustar00rootroot00000000000000kind: script description: Another hi test depends: - filename: base.bst type: build config: commands: - "mkdir -p %{install-root}" - "echo 'Another hi' > %{install-root}/another-hi" apache-buildstream-27ae392/tests/integration/project/elements/stack/hi.bst000066400000000000000000000002311514607367700270040ustar00rootroot00000000000000kind: script depends: - filename: base.bst type: build config: commands: - "mkdir -p %{install-root}" - "echo 'Hi' > %{install-root}/hi" apache-buildstream-27ae392/tests/integration/project/elements/stack/stack.bst000066400000000000000000000001271514607367700275150ustar00rootroot00000000000000kind: stack description: Stack test depends: - stack/hi.bst - stack/another-hi.bst apache-buildstream-27ae392/tests/integration/project/elements/symlinks/000077500000000000000000000000001514607367700264425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/symlinks/dangling-symlink-overlap.bst000066400000000000000000000022311514607367700340670ustar00rootroot00000000000000kind: manual depends: - base.bst - symlinks/dangling-symlink.bst config: install-commands: # The element that we depend on installs a symlink at `/opt/orgname`, # which points to a non-existant target of `/usr/orgs/orgname`. # BuildStream converts absolute symlink targets into relative ones so it # ends up pointing to ../usr/orgs/orgname, but this resolves to the same # place. # This element creates a directory at `/opt/orgname` and installs files # inside it. When this element is staged on top of the dependency this # directory will be ignored as the symlink will already be there; # BuildStream will then process the files that should be /in/ the # directory. The expected behaviour when installing files within a symlink # is to install them within the symlink's target, so the file # `/opt/orgname/etc/org.conf` should end up at # `/usr/orgs/orgname/etc/org.conf`. And since that directory doesn't exist # BuildStream will also need to create it before installing anything there. # - mkdir -p "%{install-root}"/opt/orgname/etc/ - echo "example" > "%{install-root}"/opt/orgname/etc/org.conf apache-buildstream-27ae392/tests/integration/project/elements/symlinks/dangling-symlink.bst000066400000000000000000000007731514607367700324320ustar00rootroot00000000000000kind: manual depends: - base.bst config: install-commands: # The installed file `/opt/orgname` will be a symlink to a directory that # doesn't exist (`/usr/orgs/orgname`). BuildStream should store this as a # relative symlink; among other reasons, if we ever stage an absolute # symlinks then we risk subsequent operations trying to write outside the # sandbox to paths on the host. - mkdir -p "%{install-root}"/opt/ - ln -s /usr/orgs/orgname "%{install-root}"/opt/orgname apache-buildstream-27ae392/tests/integration/project/elements/symlinks/link-on-path-use.bst000066400000000000000000000001741514607367700322510ustar00rootroot00000000000000kind: manual depends: - filename: symlinks/link-on-path.bst config: build-commands: - touch %{install-root}/foo apache-buildstream-27ae392/tests/integration/project/elements/symlinks/link-on-path.bst000066400000000000000000000002751514607367700314610ustar00rootroot00000000000000kind: manual depends: - filename: base.bst type: build config: install-commands: - | cd "%{install-root}" cp -r /bin /lib . mv bin altbin ln -s altbin bin symlink-to-outside-sandbox-overlap.bst000066400000000000000000000021501514607367700357550ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/symlinkskind: manual depends: - base.bst - symlinks/symlink-to-outside-sandbox.bst config: install-commands: # The element we depend on has installed a relative symlink to # `/opt/escape-hatch` which uses `../` path sections so that its # target points outside of the sandbox. # # This element installs a directory to the same `/opt/escape-hatch` # location and installs a file inside the directory. # # When this element is staged on top of its dependency, the directory will # overlap with the symlink and will thus be ignored. BuildStream will then # try to install the `etc/org.conf` file inside the symlinks target and # will end up with a path like `../../usr/etc/org.conf`. # # This could in theory overwrite something on the host system. In practice # the normal UNIX permissions model should prevent any damage, but we # should still detect this happening and raise an error as it is a sure # sign that something is wrong. # - mkdir -p "%{install-root}"/opt/escape-hatch/etc/ - echo "example" > "%{install-root}"/opt/escape-hatch/etc/org.conf symlink-to-outside-sandbox.bst000066400000000000000000000004441514607367700343130ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/symlinkskind: manual depends: - base.bst config: install-commands: # This symlink could be used by a dependent element to trick BuildStream into # trying to create files outside of the sandbox. - mkdir "%{install-root}/opt/" - ln -s ../../usr "%{install-root}"/opt/escape-hatch apache-buildstream-27ae392/tests/integration/project/elements/workspace/000077500000000000000000000000001514607367700265675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/workspace/workspace-commanddir.bst000066400000000000000000000003651514607367700334160ustar00rootroot00000000000000kind: manual description: Workspace mount test depends: - filename: base.bst type: build sources: - kind: local path: files/workspace-mount-src/ variables: command-subdir: build config: build-commands: - cc -c ../hello.c apache-buildstream-27ae392/tests/integration/project/elements/workspace/workspace-mount-fail.bst000066400000000000000000000003131514607367700333450ustar00rootroot00000000000000kind: manual description: Workspace mount test depends: - filename: base.bst sources: - kind: local path: files/workspace-mount-src/ config: build-commands: - cc -c hello.c - exit 1 apache-buildstream-27ae392/tests/integration/project/elements/workspace/workspace-mount.bst000066400000000000000000000003161514607367700324370ustar00rootroot00000000000000kind: manual description: Workspace mount test depends: - filename: base.bst type: build sources: - kind: local path: files/workspace-mount-src/ config: build-commands: - cc -c hello.c workspace-updated-dependency-failed.bst000066400000000000000000000004701514607367700362030ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/workspacekind: manual depends: - base.bst - workspace/dependency.bst sources: - kind: local path: files/workspace-updated-dependency-failed/ config: build-commands: - make - chmod +x test.sh - mkdir -p %{install-root}/usr/bin/ - cp test.sh %{install-root}/usr/bin/ - ls %{install-root} workspace-updated-dependency-nested.bst000066400000000000000000000004701514607367700362410ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/workspacekind: manual depends: - base.bst - workspace/dependency.bst sources: - kind: local path: files/workspace-updated-dependency-nested/ config: build-commands: - make - chmod +x test.sh - mkdir -p %{install-root}/usr/bin/ - cp test.sh %{install-root}/usr/bin/ - ls %{install-root} workspace-updated-dependency.bst000066400000000000000000000004611514607367700347610ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/elements/workspacekind: manual depends: - base.bst - workspace/dependency.bst sources: - kind: local path: files/workspace-updated-dependency/ config: build-commands: - make - chmod +x test.sh - mkdir -p %{install-root}/usr/bin/ - cp test.sh %{install-root}/usr/bin/ - ls %{install-root} apache-buildstream-27ae392/tests/integration/project/files/000077500000000000000000000000001514607367700240575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/amhello.tar.gz000066400000000000000000000336171514607367700266410ustar00rootroot00000000000000‹ì\k{ÛƱö×òWli=©’”¨‹Ë•Ÿ2%«ÑíˆTmÉ¥!`I¢Â…ÁE2ëø¿Ÿwfw¢m¥'Î9y¦‰(`wvv®ïÌ®jéûÑÚ£¯øYÇçéö6ýl?Ý^/ÿ4ŸGí­ÍÍ'ÛOž´7­·7¶¶6‰í¯É”ùdIjÇB1‚±âÕáàåéÅ@tNÞˆWóóÎÉàMƒ¶@óåûT†©Ú[Jt¡ß¾{nX y+ÕZ^0õ= ¨b;Lg"‰ãÞy÷%(v~8<:¼¡MïNzý¾Ø?=Ì?뜻Gsqvq~vÚïasÁÖÐ…®Õ.;Ýa÷ôdÿð`xÜ鞟÷ÏûoâCð> k—ÃÎñò!oëÅ O‘Y:¹¶ò·ú[ü³ÀGçbpJc‡ÿè÷OOÞ6¿ˆÒv¾ˆ–Kç´úŠzMñ._¡ ­'ÏÞ6Z ½p\»Ls˽ƒÞC™Ò6þA3[•7Q&&ö-A׊ŠÅ­ŒÙN¡ 3@ö†hÏÄ]ß4Ø2”é‰qf“>%ˆ§Q«r83CÆwíË6Khv¨†! –Ø6®3ÏwE2KRX”K†$ 7‚Ë5È&y (:ÒÍàlnäd†*£ãw À8gS,â?–´«Å»PIl<¬•ð{°d²ŸãιÖôÏ:^7?ó¡Ù/½n½À_ŽÔ˶G´&¤˜"GÃÉÛ/,0Ž(ê' ¦v8ƒåέCý›šcÞ-‹Z,`/d}ˆW¤jÐ…dÏ•"·üV½—Ùëí_œÀyîoîyiá=õ†Úäw-J"VÞ5 c¢†¢n UìQƽ!])ç‚ÃÁ#¦ËŸ2Ø2¬´/ô‚,0î—:‹Pñ‘,P•]ò.[Úÿ/WÚ'ˆ1ZTèCñ`¿3èÕ.÷"– B¬¬+o»‡'‡ƒœg¦So™Ø¢\a¨7UŽ)µ 2 “z—ÚKÍ@mŽ7fçÆ“ÑÏ«6"æ"!SP‘Ï !?N#O&˜J$ýhì9”Å4 q7ñœ DªÁ¨¾Ç!\_ñÄd¼”×ΩªÌp?hSbÐþÒï †Ý‹óóÞÉàžò¿$P—Tò)ËYöÃj“[-ü®Jè™’õéRÉâD6JU<*$ÀY¸¼Úê­{ö5ÁŸ 2­J!‡bHùE&ûžïÙñ Zd˜µ¨o˜#â—a²fGÔ Šb9¢0S–Ä}‡rµnœXz,û:‰|Òx¾’n%)Ñ•Sº$th|Ý Õ$¤\Šeã,l‰<\BoiJöF”-v¡#AÄ£¬q;*3‚äƒå²0KX—Ivm¶ˆ`ºcê€4š‚F¡Â€y" ·¯á—ÄIn”d²:¸±}wÂ2w?•ó¡å8,ßR#Ä.ï» ¤¯‚_ÝJ7ÏÓª$¥Q¶¥‹•JÒ0<ËÁ^¹UaÆafçâh ÀÔcÑûÄç¹nUJöýÎqŒù»ïÄôÎ}WÉqÖÜ;|uŽ ²„³ÚÏž=ý6pÖ¼k'ã^C°£7óç_,»JÕÅž #6ÙœËå dÌ™Ri=mGgç=˜Rír£µ½¡,FÌšƒó‹~–ëË•uB ·¶ï¹ÅR;‚ ʼzú~ç¨ßÓxÿÓ™ í_üÐ C^ÿþcE7/øÅðät`jÛÒ¼åïKJ@’Ñð£‹ÞPíaE †`ÁÛë5vÕW&·k=¶*ÒG -^Ó£ÒˆÊÈ«Û==>†K÷I µÊ%H%€kóßð¾šÂGv¾ùçLëc5çûÃÞùù)ùeIÿ˜Ð®rÓOÁ^µQØß…JîºHGHOT†Ën…T5‡·HänÙ¢üYë-¢ÀÈ[Þ@ƒû>ûÜWa4èl1’w„”Æ'H”|@wŠ³tvÄ‘¦”(,ö¸Sh¾ka 1S«<äl \7è›é^:Ò%MC¡©XC—¢­¦v„ÔOÐè†" |‹Uä¼Õ†:ª‘PטJ|÷R‹ÐFÐu:Eê' !É5OÇO¸—Tœ KÁoÄY‚š¦A *Ýí ™:-SG˜{V µ¬Ü¨žX( 1Pfnµ”@Éûözg½“½ÞI÷°×çøéHQJ®‘h5êb±˜$5ƒ]LÉЙ g"°IP…èò-‰n·ÚÀ_¿¦§?ü½k~ª'gô,WðŠf¾¢&ÌL+<•Î$ô~ÊHÜ¡«q‰Mu,—³6•ccŸZ’X<ÇAÊô_QÏ:´ÒRãFï"†•M#U–äÄ¥]¹¸c˜¨ŒWµÑ•·Þc·[^ MÕÜŽ;@t^r’””¹o…z+Õõ4x ÕîÚga©ÇGvHÕ43ª] ÞËIh‘¹{pF5§0P&üåקƒ³‹AAäMF—çÖáI÷èb¯·t!Ðvº?ª·ó}Ön—’Ÿ¸„ˆg·ºƒQ¥€±¶¡ð²[´aõ´×¯9iæÓ`O˜F–G?Í4¶ÈûÓ¬±ãl üÇZF€—ž#A‹ÿ"°ú9æÉ ÀüA÷ïsÓðû²iKÖÿä(s-[·Îø¸Ûé¾ì ño÷ÇÚeÉÏ“tæSY!V˜ÎýE¨ËïÜ"·óYžLŠEÒÙT’UÎeãα2“‹^)$D†&阽›ûP¦#Àn媦>ÍMTHFˆ=¥/wBÕº’"É”©€ëhœ%úp„òÎÄŽ›±Ï¾¦$F ¨Kª.@z?Š™ŒÒ%‡Ï$(³Ç’=ù0ƒ§¾ õëOêžs9‹Ê¯ª N¯‚Q¥0®2”Э¦cq¶š‘Å2%5$ tK¿êÚÉ$HRQ¹ªiªë5tO;¡€] ñ~ž¡<ïþå/…¹‘TÐ|G(‚}cT,ís>X_û»Âbuža«V…^CYßa˜J_ÓÐ'ôdVÔ¾Œ2>BcKTy$?ôÑVHÒA$ßÛÊ€ÐÞëA3ú˜ ¼0Kxhµqä _”8Z­…™Ôs¨S|5€#FÂm/-:%'Öò¦¤ª§i¡ˆp”8R‹Ÿ&¥Âå2©Çë¸F”Uv,8éŒR©ÏS{Üðq x/ n ð”'Ö¦šÚ0gú–ïçTØæÕ];W¹=ê,d/Åsº5"é¤Ìl OmïWdH¡H‡³!] ¨F;z?“‰ÎsZØw©þt Ÿ»ÑO8w•ö$·ÎSñ³ú$³D}÷¨oäL§ú ½XÐK®ÄÆqtC¾ ¬`„ÍÆÆÝ&²‚Hífb'9U—3™ªÆ{"º•\}Šµ2Œ²ñĈ1ÐhèÎÐù‘}“»ôè|ˆ4wmÈ ¦¶S²±äOZñ¢/ZK(ë‚åkQÞ¬hýéÍìæV~eª• çÈÑèh×øI1ó)Ùì.Ä)‘Ó¥¯x—O^áfsa @€²Væ6·Ï B¼ÀäÛµ0¤Ûx‘‘qŒ2ÉŒ£âX é‹ ¿eBßµ?3ýÉ0=—Ü/˜¸òzM*ä>ŠfBõžŠ÷÷&•Á$áyÈÂ2™? | FT­…"šê>£¹cËM/I2ºœÉ÷5 Gà»ß$¢ÆWg9ø6_A¾Q\‡o¨3R%¡Ô%±«*™$öXæ¤Ø :ó² \ ª‹Q|K® ë…ÂÖ( ©žPóø¬ÁÛ{ÚZçŒò´Õf‡÷mvÃ,ñg;ùDACwP¼hÅlqG­Ë×/HTÖs:_°ã«œæºœ[ /T±Ùc ň3´cuy™8ú¾µþù•• wJëYó[.âk%j-ðh‰9ÿùçRkAM˜#7?¼>ï*ÆhÄÎs¡ ò¨òÅÇ„|Ò’@Î?è¨Òµõ©¢[¯þàÚ „ßÖçNcözgǧ{=ÝvRí!MP™?ã ˜#›,yªê¤ŠxP®û3’]¨"›éÎTÂeIêG*iŠk~ªÿ¸Øz¦w^1äµ-õAL-dª®ò‹zêTeá–ìv±ºä–žn„.ï’õ:{‡'ýÓÁâ™™'.5¼Õ—†n”~£Iµ¸ù©¶÷Av=ÏSÑ-U,uΆ½“ÎG½r®iTÃ6AŒõ‡È5gú¯©ö¼l6•F›K§ÑW]d%%?‘ЉJIÄò=†2¤eô]/ùüÉ”/$¡2…a7S/ÐW`øhë0K™^žŠþÑnùݤXB¸ÜKü"ìuú/w­+Kw BxÎt×ÂO‹Ï œƒç‘@ÎÖœ£Î¯ûÉÃÑeà "9ŸŸœ¿0B˜ûý¡}@¹—ôixo‰¹Ïƒn |+GŽÃÏž|ñ®ÀÂ9ÌgNRàܸwêU àÒP¨É"4©Z÷:néæ5!|òZ.c8 2Fí_ø3BPD î#™Zتo£R²Gꦹ$R®þ¶ÄU³”/såå«d–ø\ÏTŽúp}ÿð¨×WåãꕵZglwï‘ʃª ¢cˆÐïçið[ ß“‰7Juï •¿(µ¸ø~Ùñ¾ª;tó¼è¿p–Ð…?ï[UY£Ýwº Œª(ª¸óˆ²=YÛi­®¬­Yï4ù®ê,L$ßl2wôè,Õ\z-þ¨Â¼jPg:D¯t… k ìÔ™(!†ÓÐÏÿï1¼PUlz:·¦2šú¦x?Ô§zN¦^4ý-í]ÆGÁ§Ì•ž0ÃÅ’E J ˆËŸ™F®êvæV›FIÚä?öIL3k%NÙhlÿŠPÞ)Šì˜S:ff(ôEKÙ¹›D¾ÌÃÿ=S¹úÊî÷ÐèÑU8›É{“ˆ ´ÒÆúÖ÷¦Ì%ks-ºþ«NjøäEÆÔ¸w‹&ÝÖúúzËÔ†º÷l%>n­J¥¿m4мÚøŸö®´;,ÉÖWó+²] Õ€Ûe—Êr[eɶ¦eI#¼ÕH*Œ …òH†Kš.ÿ÷‰oÉ$Á®¥ûœ>ÇÇ‚Ì·¯ñ"nÜ÷®>í á2½+0U*¤=hÜ™L·ÞÓNI‚,àUŽ^“æÉ UJ“€‘ñv«zã@*‹¨Q<%š¨›¬®L“°k;>-n6â3yÈžÙÒ’¬ÕÅC/Hf[]¥Ú?’úKù©N±™.S¯x™D½çŸIÙ YœkÉi¸&H?Àì&É<ã¬ÝóFÔìè¥.«˜3g¥åòÕ< OP<» jADýë,þ¢}¹ Mõg…8IÄEÞʦäÎ'™ÃJYÃ+ÁN‹oìî‘L«F°ìó'zÞÿ‡‹í™ù ‹&g6ÛL’ÄfÝKVä Õz$nô†ˆ”ÈM2›X†ŽÝ&SQ»AgHiü$^ ¡z¤‘Žœ:Áyܧ! 00»¡Gêpà¡Ë€TÌ0)xX2 Ó~Ú>ÌøKºè4Ò|ÿ<ó´V:ÉÅwøñ¬7ª:Öõè£CîÁaƒfÒÞÁîY­(ÆÉáÎÅ­³/FHç–(ñ–.ÖìYh±.™öl#›dÀ'à" úµ:žDÖ%¾±Ñ¼¿V«;Ûñ90‘Á˜$EñF†ƒ¯d£> ÖÇÓØ!®°BŽ;©ŠèÔ¨2‡L8n¨Ï+Áæ<=YÐ7¸Æ…pX.{j ‘´˜Ñ=×c6ci£èzê“"jä9ú¤Ä¯Øª,…dW#Ó^P¹vŒÇ˜óuAhÿ^½U1æmé¯:("  qJ S£S:¾¸äYÞø=G粌¢TênèŠgØј£6®žüBÉl7þçìoÏö·Ÿ·*…Ã¥î·ótFí´^mïï[0º¨à«a™ÔKƒ·ˆò{þêÒ^Së6ö*UãÏÃK o§îElª‰²#>„ašo©­9 fæ,@YÄG”rc¯É®Œº(¦Ä¡¶ ýÜóÚÝO˜-PIÖE|8RO·€Ð7žwMÉÙEIX3ŽWJ°FÇ­é,õÎCYÀ|ÞÓ± ùÙYæqmXíQY}øËò–è_I‹më4høè×½éÃÕ¦”A>ýù9ŒÚo]WÁ¡aiiS#|Î~`¿{µ²Im…6ú”‹öpçB@|´výeèþ/¶ôÀÄ!D-¿F“çÇNÌ(•Yœ^Ò"6½ŠhPÑÜmS°‰Wú‹Ýn•Î€UøWÈÔÝÙÛ~~pØ¢qÀ5)šFyô&;L¯’†®,´ë72J(µ»Ó:C}‰1Ù4¼,’Õ]Ãâ‚)([Hõd”4dO3(ÞÖ5›ý@ÖçÞzíðA=²‰·ª6š´[¯"¯qÌ2Ï3ɵ’ï÷†ÿ¡ˆŽq¾7®õ8o ɇ³Úfa¯¬Ä ðäÃ&}¯‡ý‹Î­dŠ¤;“#Õà2=`¸al,oÁÐ+Ϋm>#…ËÚ4œ/~hr`ÕóÞ³‚N¬ëhbé¡ýúà¿_¾ÚÝɤ\Ñeü:PÕœÖ 3¹(¾W°rEpü7Ú £Ù𜦻Ÿ’–uEä;ø™§noewèæÜæ°Mmÿ³ › 7˜\”BVß¿l«Ÿ'?G ýçpñ NÔQ¤!F Ü䓘ßr•Ycê–¹(œì¹Õù Y¼ØÝÞÙ=6‘/i­&Ea‘ÅÞÁ³Cfp¢,@à'& ¿ÁüͱíY,‡yu¼w´xŸÕºñŸÔ–Ö4à|àúIŒ›¼IT>´"âhÉ^FJãÓ­íåõæÍkxY$‹S$0¨Æ;„_RîOplawx?ºœNÇ›«« llöG³f2鯡IšÕËép°jš~•ŽQµ«Ã´ÏütM¼~ü‡Y¿§‰¸™ÉXÜö˜gf¥ªÍ„¼äÀ«VÄ,óy†vI`5§Ì’UnŸ«ÚùÕöÉzÀ`ÕçOq¡}*SBe–£;,/Š:xЙò"ß,ì¸í·ÅFz+)-Ì ²9cjn¡Z4˜xÏl!œ,-cÕyŒåÿŽîSqÇëù˜xÈñîÜqÝë΄Z¬h±sFm³äQÄ7{;´a<ÛÝoíæNhÈaÎý¦ $rͼ]t¸3¥+<ä-JÓys.,#”•‹^.-˜F^T2D_˜ìçË&®8‹ Ço—•ÎF_PÀ´ 3’±ÏÈ»ÐwM…à,¼/(ïc Ã0'ZT s §a!—‰j$)ë6ë䲑£Îdp˲8u6ùhåñ$ùHÂ2;‘”FEÀó^µ~Ð’WäYÙ53­G‡­½w‚ƒ)‹}K.fS(²<Ƨ7cT)õsQV Ó¬DOzø 6gcÃÎr.¶2=èÃÝ3MgCNA1:›X+Þ»ñ˜OЭĒzôû“ˆ{eþ‹Äš#¦çYAÒä§Ì!¦ 2 £ ÊŠð¤÷ʆ» ÛÆSõŽ¡|˜Þ f7 Îgý•õµ‡…£50N v¼ë~Œ±EuÁs&¦ÑëŽiˆ­ÀG“¤3°sâ„“aè¨5™ÍÂâÿÔÐJÍ%®“¢;äCè6àZÉ,eSt'&–µ:ÇþeÇEîAïùËÄ|›\w£óèÜ<þv#xô(¤ù–“qú—ˆ'Ù¢Ò˜B3j:G\£êóñ8šÐØ ÙCEg7ÓR22^ YWVOqW@s4.Bƒ²3ºÐÑÔ£-‚Ùdҙܔ”æî<¢ãqŒâ‘èbz2èðK\<Ó[´ô@Ì8šÑÇx’Œ`kû`q?•L÷ÔUÝÀ"áñþèË«Ôæå±—O&7MO[f…‚I'fKÚ(º@ˆï^ú´Ì_ âs†FÔqààdŠÏg¢+JDë² )FhtR´œA}Á©tÔ§cž5“­"bþiì6ß?¼·Þd’yL ±",+B,P.Ð>ŒØ¤î çA ,žR ‘}2¢ŽØÑtYL %UçX.&X>Uê!0D8÷ël¤>ª`VÏdÝ)ñWæ\~¾¨èuÍŽŠ%#%a¼yFsk¡M@݈º ±iþfãž!¦ÐÃqãRñU¾Õö ¸'Ž¸â…5Ìè{So‚i%…ÌÇ_²²@q6†­^>¬÷é•—Ã^|q±¬Øæåì' D 92iQˆ´_þ#¨òFªD-ÒA ’dŒMvåǘ´Q){ZWV5SzJ‚7-¯ðâ„žA.(4›0àÇLÓ¶¢){Ñ5û}JqÚšw³Ô•Ô¤¿UY篜c»›ÌFÓ­õ”x*¡ âÏ( AóeÂòJSÑôƒ_í×ÍïŸÈøܸÇù‚c[%÷8ø[°þÞÃû1nBà@kÛ%›q9xœ|˜gµ÷«:OÎòù, žþJFçuv…6\g¨èf܆¯gƘ×ýÍãäsìiã HÿtIÿ¤˜O¯ oV*’aèØÑ ¤”ׯôçÎw2Ä\:>a_è3öiFzZÈé’¸ÅQËflfì6.……¼æw¿ŽÁ•ÇÉ*ârÀtâ¨TC5ŒzÀ”šò:m ѱ ‘½dš#[^¶;‡-s¾+ðØðõˆ4Õå¦9¥ñæ™íJBl0÷ØÚ{üÎ÷+ð]8¶šžßNæEc2,L¦¾LrvAø5ü£ª'ZŠyÙZóøc‚u öi¹€›…ŒPý…ôÙÞi– Aɱ§Ô‡JÌÑQ3~Q¶DÝFœ6˜©Wþ/ûâš×7[.JÙ“Ûü%«A{»}L=êË0H»“x,®IˆH€L“«;6üE4GŽâBžª†7(õbÌö­çÖ 6 ZÏ÷w±}9 iÞ§2“J<-ùgûlx¯}>&ovžNi!©''¿Xj±öãŒ,žC©-+Û¾¦¦³hc±»0Ô‚yþ7ÊË'ŠTf»túŠ×€§ÅXødzUr…8“kÅT 8ßTŒuÃíDÅuhUõï²µœ^i(¹(wºtf‹Æqƒ’›«Å×(¡ÆâR·´Ø0²tº—í+õ~åx ì›Gº 5¿6„!hÜ?kP4J€¾¾j0Æä·ÒKp¯‹ºÌ’öI_ïFál4u·¼çël §5Ž/(ê’~ò2(ò«¦r^É{9Ìw=ÙûœÃ½ œøys—žúƼ…ÒYXÝ\,(•óžägv¹loz9ý‘ÝY /lˆýmHE¯[/ª'OíCŸMø£?©€¡Ùz¬¦ ÂŽØÀÕ\}"xáÚ€ÀÛîvÛÝv¢0Í åW»LB` R]õ„Ç^ÖÌ8 8oØyûå`”~ˆ£bÐå ½Ѓa'K—,ÝÃÇÕÀ®æÜ(ãø–g:ì‚q‰GÉJÖk7Í$°n«ÔuA¨³™^ü4ÙVªT€tmZ^¯e¶nØ%ž‡Ë°mxö¬g"Í|ê]’ÈÚû€@¢â®cl›4‰}¶ÁAuMÇŸœÉcª- œq„ ï <_ö‘qŸtŒ3Žh&ÌImÈð;z<©¼û¼•¹äYFt¤ešw?I!Ä– ‹“°Þõ"$5% ;üm‚C×¢L°0G£t(ø¨9Ü¥bO§aU`àe»a¼£Ý&Áµs°¡2»/ÀšÊ6/1Ѥ1¥e‹ãÚ^cÍAÖJÈÈ:nRŸã&„຤”q–á4´Õ(<¤ØÌ•3Ú=~ú´ÆÇa„ò%\¤Õz–©wxÄS_®ÉB\«]Ú‡O=­Y“‚üE$øÕÛ7ÌüU„Øy€žÓÉMÑÐÆhQ5Â4Pó_kØ–b…$}*°»Ó\¬tz±½TÓHô¡µDz›dî2L}É´†³“¶‡Ñfeï`÷à÷s”ƒÇßÂy·Õ ŸÂ·Ÿ±ÎGâ¢w`!(Œ`ë³Uù»Õ5ÍåxZù{°Å÷ÖHàÅùss¹€µƒO9}öÐóõVð™P6ûxˆZ‘8¶Þú¯fƳï”P e ¾ƒ.Y­¯.öËÏÇBûD{8aHÊývi˜ŒÕLæö&#ˆ l·1¦}ørw†èÒmF ÊqB–Úι¹ä„‘kø¤ÁÅVX U£ 7:UMœœœžž–OWN+§ßž†§¬÷\œ©Æ! Ñ‚/.—ì¬f·¢1«-Éï48½³s]ÃJð‚Ï~J–«ˆìIÄv㣽CÝsÕ‰e´Úd2`;Hs=‰±Ç'gïsƒnÁÁæ+Ñ“åœzÀ2ªWM.ÕwB½)N|‰,ÞÏá-âßd6ÀßnóÆ·B.„4lôŒCV^ ?¡zP%eT/Ž§7B_Öòß)ižÂ4hÚàB} wi6<^TQÑUêËdSIÞl­—kŸ#.OýK^+;—,*•(T’ksrX‘ŸüuóIP­"SjÚšÀrsÃÛú§Ã‰ñ[kbO% Þ¯¨å‚÷™ë´n¤U×ÎÄH`ÄQËÒÁת)æ(‹­v³ß¬ɨV¿ÃoÖÑõ´5Æ5N@c ùɈyóÎÝâ«S¡-˜ƒpÿðÃÆCNV¼‡6î>àåuf“S™:}©ABXtY® Jí¤ Smkж„lçCB!Øjï>}qX=9 ™Å©Rýiû¸R}S«•èËš^2C_×ùª– ® â§RE´Ú°'<;«EkátÔ]¸=|A‰yûÎðI/N²õ…¯¬|A¤ «!D7ʵ"aåMhy(i; Ñ@¡È$i°w¼÷ŽµØh ¾Ù’÷òË[<-Æ5{±'Ç­…É'U(ƒIAbÖ‡Ž‡×¾óÃÛ—ġ)ŒoJdÓÉRg"} kp>ÐW¯œçX´KÂæ9ŠÌ%(ÀLj–ôI*®7Túÿç¯köîéq[ áÈ)zâUoÙíØ:™ ,’(;Äí,;¦öbYcŽ/­ø=Èî Üa±ç™šÚ ’È|ÒÁ«´Bb—ÁÖÀN\œ¶—¼Ô}BS©UŠ„ˆ›sÀ2Ý©6›Õs;¶tlæíW‡‡ûÕíÂà„›Œ¾lò±/ÓÃÆŠÉÖÁ¯Áb×HËÞZm¡ä+Ñ+Ïo#o¶÷ ˆ?Ao&X=¢:Vʈ@̶Z“ ’'æûŒi4zþ‚pˆ:ÿ8‘r¸¿Àø{ªZÌÎq7Heïç¹òÇ—Lîµ’ɼò$æ¿”{¨Õ(ÝÜöœ†éRöÎ/„Ýûz©%dyvxürûÕÖÒ6Çèƒß:íÝ’oƒüÍãÚ‚©âëÇa=™Z&äK¨Ãqç:”m@Ogœº]L+Uö¾f];ÆS¿$WÇb —;çè­äÞ0tªñ ÉL±+†~£Èøȯ-gÌøö[?ÿÇÀ¦›SÜ0ý™ÏF®Ìz«Ÿº—ØämÛ‘O&9^¦.>™es-€ÓP'Š+ã¶H† õ%½à=f!ÅÓÏßÙÄÒ68HW¥•Àã¦6UÆýh_Š„ú”Ü»§ÌtØ­pñ¹/gsé@®Tù'}E¯| Õ'6 )OvÅ~)Ž |ŸÔžI/Í·@9žhîPqyP1¦F¼3™ˆl+ìfš9¢[Š#¾Ó‰yxAw'|­Y˜»"ûã Bßoù¥ÇõW8S–+:|Êa`û+òšÂáJh¥¢s—¥²vÁK4{'Bà*Lë?<|h扙^qjÑÕ¼ \Ä×Q¯‘’àG_£Aϲ{sBVmÞ9g¾Üu’Ãèè.¤þ†ÍêõÞŽhÏ÷vpI#Mg«u›‹;˜¥<úáz.FÜ 5 oŸºŽÒY¥ø(sÚ‡‡wïݵiY-ÄÝûÖ¼ké†ëö,îmm¬ýð`ýþ:Ýø…JÜ0út¦­­ ïW8¨˜ÊTPw¶6wâäíIÛÞ=§Á¸ú ¸Šû¡!´¯ +îU×k¦¨™vAza]¯ÃcÁÚ&ü:ÊÛÈ.þ8ÌFæ®yáÿç}•vSã:E†6ª®z´¸Lâž¹m©wR2òÛmøžB5f0m°Oïh{ïõM þ¢@ ­Ûhbvÿœ¡Æ°l[F/x¼êpµÔï¾d†\QàuaÖØ2§¸„¼Ê\)–È÷Ù—n=™kQ½Û V﹩^ÿwU¯Ÿ«^ÿ÷WoYíTNצd“õÅ­ýçÝÏda&ŽåÃ¥m‰Dÿ⻞ëa³•½xÒ‰Y}´rò“4¾6¯tx»ä=+ôv­lºí?œpAº²'z cG4·=¨’ÝìÉî®F^¤â1û—Ñ«Mõ²÷ÛàMAÍ[†x𘵩ɴ5"—XÜÍ’YÍ4ÿÜu†÷aX‰1µî‹›4jaûp(–r-Ó¹®å³…ØÒا†ÄŒl\-Í|>ìâ"pØI®æJÀ" Ë6hŒ'ÔÁ¯"M5èß Îo¾@õ7ÅÔòIØéåJÊwÓæJj®©Ï”!ÿP’÷ŸúWÛ–dí]x ˜÷d1£MVZReYc/aÐy´)š±&dnâ\±ÒÉ@tâù5ëSÙ­ š ½X•:z-¸=²ˆN‹¥Dn†ÕL%e¤Ptc&ž‹ûþÜc–p¥ÔËÈGüØbQ?þº¦Ç(? ªÀ~fÓ»X—åñ¥³6ô4ðce6×ÌÊêÚøQA6ÙÀð.Ÿ«” ɸ­ü\¨¢õb1ˆ<_3ÕϱÉí&fûžÙ„ó·ìnÄ…Ì»÷Î'Â[|Ö7XÚ"ûl&£-êY A©ôÍíç_ýé BLV­&¯3üÓóò÷÷îáïúƒûkþ_þܽ·þÍúƽûk÷7¾¿ÿýýoÖÖïÝ{pÿ›`íO/IÁG¶éà›é$šma¸Ï½ÿýü~¹¨"½qó¯ÐHÖ}]$¶Ö<Þ0d¿D‹ÈÎÞq+ØH·„€í^Òmïl¿Ú¦gÇ»Û;/wo×å3ÿ a§ûgçñ¹ùÿýÚZ~þ?ظÿÿ–Ïÿþ¸ñÜëÿ­ë€Þ±q‚3#¢š Œ*¢•ÛrW`5Þâ”Öx+¬²0ÃÆý‘È*ÆßÔ6…̯P†™^J@ÿ¢ÛêIÉ‘:c}²;¬²{Ë]s·ËÓíçösû¹ýÜ~n?·Ÿûçÿ3h_Èapache-buildstream-27ae392/tests/integration/project/files/base-with-tmp/000077500000000000000000000000001514607367700265405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/base-with-tmp/tmp/000077500000000000000000000000001514607367700273405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/base-with-tmp/tmp/dummy000066400000000000000000000000061514607367700304120ustar00rootroot00000000000000dummy!apache-buildstream-27ae392/tests/integration/project/files/canary000066400000000000000000000000061514607367700252530ustar00rootroot00000000000000alive apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/000077500000000000000000000000001514607367700273735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/base/000077500000000000000000000000001514607367700303055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/base/usr/000077500000000000000000000000001514607367700311165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/base/usr/sbin/000077500000000000000000000000001514607367700320515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/base/usr/sbin/dummy000066400000000000000000000000061514607367700331230ustar00rootroot00000000000000dummy apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/overlay/000077500000000000000000000000001514607367700310545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/overlay/sbin/000077500000000000000000000000001514607367700320075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/compose-symlinks/overlay/sbin/init000066400000000000000000000000041514607367700326670ustar00rootroot00000000000000foo apache-buildstream-27ae392/tests/integration/project/files/hello.tar.xz000066400000000000000000000011641514607367700263340ustar00rootroot00000000000000ý7zXZæÖ´F!t/å£à'ÿ3]4Iîñ' Î:ðô(3õ?XéeUJæÊœ•/"`°ÞðšøÈ3» ìjçèBýYH,Ò&tB¦üä7ÜIË|%àñàö?æVÇȤd§â‘„,KÓ-3.tÁ¼H˜}p(R—"Fíî[²ÇÖëØöËp }Jêøë§Ëψ™ÿò7@±µsˆ“5·z2?ÜLhBV˜é’vųqHÄ„¡Äëí„cÄ£ýIܻИëûIUêO̺ øvÌñî]·Â€ƒ¡mœžxB>h![ûU¨ª£ÀÇÔ5‘(¡Ì&¬lÝi¬-Ö¿U_VŽmˆ&džUÚz¯òGý¨¨Äc埼ÚÄ«ÉX,ï #þÓz)´¬5S7;ຢPçMÿ'Ñ_¢×Lê ùN/ÛÅe“¥_¬tÈ6íjÿ‹`þ6“þg¨S!48ˆ±N†WU#wy4HÁY2¨O\]ïOh‡UkwΨò>µ±±±å—öø‡ôû#Ò§ –ZhF¯S‘ %C®‹Ý±4þgIãVòÔ ¥þžví×xhÈîüÈÐQa̹âï-ý РU<ç?Ÿø­K˜ <(¢ê›úRÙ|>Øm·º$B<ªñî›w=’€Z¤îܨû«[½(­èÚèGvì’ŸÝħ…›»'…¥ý¶&²±°$JªÎ,>Âþk‚|QÝ®Y"r/ß-=%{_¡(7€qŒiª£ªÏ€P÷FŽW±ÄgûYZapache-buildstream-27ae392/tests/integration/project/files/import-source/000077500000000000000000000000001514607367700266675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/import-source/subdir/000077500000000000000000000000001514607367700301575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/import-source/subdir/test.txt000066400000000000000000000000251514607367700316740ustar00rootroot00000000000000This is another test apache-buildstream-27ae392/tests/integration/project/files/import-source/test.txt000066400000000000000000000000171514607367700304050ustar00rootroot00000000000000This is a test apache-buildstream-27ae392/tests/integration/project/files/manual-element/000077500000000000000000000000001514607367700267635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/manual-element/root/000077500000000000000000000000001514607367700277465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/manual-element/root/hello000066400000000000000000000000201514607367700307640ustar00rootroot00000000000000hello from root apache-buildstream-27ae392/tests/integration/project/files/manual-element/root/subdir/000077500000000000000000000000001514607367700312365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/manual-element/root/subdir/hello000066400000000000000000000000221514607367700322560ustar00rootroot00000000000000hello from subdir apache-buildstream-27ae392/tests/integration/project/files/shell-mount/000077500000000000000000000000001514607367700263265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/shell-mount/pony.txt000066400000000000000000000000051514607367700300470ustar00rootroot00000000000000pony apache-buildstream-27ae392/tests/integration/project/files/test_shm.c000066400000000000000000000010041514607367700260440ustar00rootroot00000000000000#include #include #include #include #include #include #include #include int main () { int fd = shm_open ("/foo", O_RDONLY | O_CREAT, S_IRWXU); if (fd < 0) { fprintf (stderr, "Failed to open shm: %s\n", strerror (errno)); exit(1); } int success = shm_unlink ("/foo"); if (success < 0) { fprintf (stderr, "Failed to close shm: %s\n", strerror (errno)); exit(2); } close (fd); return 0; } apache-buildstream-27ae392/tests/integration/project/files/workspace-configure-only-once/000077500000000000000000000000001514607367700317355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-configure-only-once/configure000077500000000000000000000001551514607367700336450ustar00rootroot00000000000000#!/usr/bin/env sh set -eu if [ -f "./prepared" ]; then touch "./prepared-again" fi touch "./prepared" apache-buildstream-27ae392/tests/integration/project/files/workspace-incremental/000077500000000000000000000000001514607367700303545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-incremental/Makefile000066400000000000000000000001511514607367700320110ustar00rootroot00000000000000all: random copy random: dd if=/dev/urandom count=8 | sha256sum > random copy: source cp source copy apache-buildstream-27ae392/tests/integration/project/files/workspace-incremental/source000066400000000000000000000000011514607367700315660ustar00rootroot000000000000001apache-buildstream-27ae392/tests/integration/project/files/workspace-mount-src/000077500000000000000000000000001514607367700300025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-mount-src/hello.c000066400000000000000000000001311514607367700312440ustar00rootroot00000000000000#include int main() { fprintf(stdout, "Hello world!\n"); return 0; } apache-buildstream-27ae392/tests/integration/project/files/workspace-partial/000077500000000000000000000000001514607367700275075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-partial/Makefile000066400000000000000000000002171514607367700311470ustar00rootroot00000000000000all: copy1 copy2 random: dd if=/dev/urandom count=8 | sha256sum > random copy1: source1 cp source1 copy1 copy2: source2 cp source2 copy2 apache-buildstream-27ae392/tests/integration/project/files/workspace-partial/source1000066400000000000000000000000011514607367700310020ustar00rootroot000000000000001apache-buildstream-27ae392/tests/integration/project/files/workspace-partial/source2000066400000000000000000000000011514607367700310030ustar00rootroot000000000000001apache-buildstream-27ae392/tests/integration/project/files/workspace-updated-dependency-failed/000077500000000000000000000000001514607367700330375ustar00rootroot00000000000000Makefile000066400000000000000000000003121514607367700344140ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-updated-dependency-failedall: test.sh hello: /etc/test/hello.txt cp $^ $@ brazil: /etc/test/brazil.txt cp $^ $@ test.sh: hello brazil echo "#!/usr/bin/env sh" > $@ echo -n "echo '" >> $@ cat $^ >> $@ echo -n "'" >> $@ apache-buildstream-27ae392/tests/integration/project/files/workspace-updated-dependency-nested/000077500000000000000000000000001514607367700330755ustar00rootroot00000000000000Makefile000066400000000000000000000003111514607367700344510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-updated-dependency-nestedall: test.sh hello: /etc/test/hello.txt cp $^ $@ tests: /etc/test/tests/*.txt cp $^ $@ test.sh: hello tests echo "#!/usr/bin/env sh" > $@ echo -n "echo '" >> $@ cat $^ >> $@ echo -n "'" >> $@ apache-buildstream-27ae392/tests/integration/project/files/workspace-updated-dependency/000077500000000000000000000000001514607367700316155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/files/workspace-updated-dependency/Makefile000066400000000000000000000001651514607367700332570ustar00rootroot00000000000000test.sh: /etc/test/hello.txt echo "#!/usr/bin/env sh" > $@ echo -n "echo '" >> $@ cat $^ >> $@ echo -n "'" >> $@ apache-buildstream-27ae392/tests/integration/project/keys/000077500000000000000000000000001514607367700237305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/integration/project/keys/gnome-sdk.gpg000066400000000000000000000011651514607367700263160ustar00rootroot00000000000000™ Ug¿+ÕžWŒBhª÷4¤j=w˜æjZ\rjÂê ÚÃÇÝ–RîÒú®!4©ª,èwgoÍÏ×O·¬|ˆÆc_võ¡¡Êu;å 8h•\—-ž‚È—ï |;N¾³™£õn­ô}0ˆAÔp˜Ã>ÖÊ:je‰m<Òø10¡ø‡%ç¢C˜›ÊËMo鉟îø›ˆ9ûöŽÄŽx #³Ôð¾Éw %k\Ú°!0(íÈòsÅÝUzˆÌ'µÅÉé»`(Œn#6 V\©æaãá12| t£ ÚÜa I¼êìLîN Eß Ë€díR·«n«Z1œþ×—´(Gnome SDK 3.16 ‰8"Ug  €  äÏ¥UÑRk—ÿK×k}X€HÍ“¿ Uê ]U˜ÎÎcâ×ÏeâP<ÓaL^0™È3ªuÞA…-ÀoGï¿™•ÑPý ÁïµÍ2r¥M‰2pé“x4­U?¯í„€>¼çÙ›ï Ji`jQö}Z­sƒ{ ¾©ñ1«ë¬Ê’ÁiÔ¢ùñ:ãZÐ{ÑrÜä6ô©€-‘òÅ{A¯’Àiâðfòá°ßÃñÛ„ˆÐö—¤O Áû‘° ó¿µ(©ÿVÐ|$¾\|ö´àv,gjYÖ¼!a†þIùfKü¼ÅâðL”Ç3mv¬ÏkЖ³ÐºkžÌI§† AË XDÊ"Iל<Žfapache-buildstream-27ae392/tests/integration/project/project.conf000066400000000000000000000012541514607367700252740ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements aliases: test-images: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ buildbox: https://gitlab.com/BuildGrid/buildbox/ project_dir: file://{project_dir} plugins: - origin: pip package-name: sample-plugins elements: - autotools options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture variable: build_arch values: - x86-64 - aarch64 environment: TEST_VAR: pony split-rules: test: - | /tests - | /tests/* apache-buildstream-27ae392/tests/integration/pullbuildtrees.py000066400000000000000000000240631514607367700247250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli, cli_integration as cli2 # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from buildstream.exceptions import ErrorDomain, LoadErrorReason from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Remove artifact cache & set cli.config value of pull-buildtrees # to false, which is the default user context. The cache has to be # cleared as just forcefully removing the refpath leaves dangling objects. def default_state(cli, tmpdir, share): shutil.rmtree(os.path.join(str(tmpdir), "cas")) shutil.rmtree(os.path.join(str(tmpdir), "assets")) cli.configure( { "artifacts": {"servers": [{"url": share.repo, "push": False}]}, "cachedir": str(tmpdir), "cache": {"pull-buildtrees": False}, } ) # A test to capture the integration of the pullbuildtrees # behaviour, which by default is to not include the buildtree # directory of an element. @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_pullbuildtrees(cli2, tmpdir, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cwd = str(tmpdir) # Create artifact shares for pull & push testing with create_artifact_share(os.path.join(str(tmpdir), "share1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "share2") ) as share2, create_artifact_share(os.path.join(str(tmpdir), "share3")) as share3: cli2.configure( { "artifacts": {"servers": [{"url": share1.repo, "push": True}]}, "cachedir": str(tmpdir), "cache": {"cache-buildtrees": "always"}, } ) # Build autotools element, checked pushed, delete local result = cli2.run(project=project, args=["build", element_name]) assert result.exit_code == 0 assert cli2.get_element_state(project, element_name) == "cached" assert share1.get_artifact(cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Pull artifact with default config, assert that pulling again # doesn't create a pull job, then assert with buildtrees user # config set creates a pull job. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config, then assert that pulling # with buildtrees cli flag set creates a pull job. # Also assert that the buildtree is added to the local CAS. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() artifact_name = cli2.get_artifact_name(project, "test", element_name) with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) default_state(cli2, tmpdir, share1) # Pull artifact with pullbuildtrees set in user config, then assert # that pulling with the same user config doesn't creates a pull job, # or when buildtrees cli flag is set. cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Pull artifact with default config and buildtrees cli flag set, then assert # that pulling with pullbuildtrees set in user config doesn't create a pull # job. result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure({"cache": {"pull-buildtrees": True}}) result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name not in result.get_pulled_elements() default_state(cli2, tmpdir, share1) # Assert that a partial build element (not containing a populated buildtree dir) # can't be pushed to an artifact share, then assert that a complete build element # can be. This will attempt a partial pull from share1 and then a partial push # to share2 result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure({"artifacts": {"servers": [{"url": share2.repo, "push": True}]}}) result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name not in result.get_pushed_elements() assert not share2.get_artifact(cli2.get_artifact_name(project, "test", element_name)) # Assert that after pulling the missing buildtree the element artifact can be # successfully pushed to the remote. This will attempt to pull the buildtree # from share1 and then a 'complete' push to share2 cli2.configure({"artifacts": {"servers": [{"url": share1.repo, "push": False}]}}) result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure({"artifacts": {"servers": [{"url": share2.repo, "push": True}]}}) result = cli2.run(project=project, args=["artifact", "push", element_name]) assert element_name in result.get_pushed_elements() assert share2.get_artifact(cli2.get_artifact_name(project, "test", element_name)) default_state(cli2, tmpdir, share1) # Assert that bst artifact push will automatically attempt to pull a missing buildtree # if pull-buildtrees is set, however as share3 is the only defined remote and is empty, # assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the # artifact cannot be pushed. result = cli2.run(project=project, args=["artifact", "pull", element_name]) assert element_name in result.get_pulled_elements() cli2.configure({"artifacts": {"servers": [{"url": share3.repo, "push": True}]}}) result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert element_name not in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert not buildtreedir assert element_name not in result.get_pushed_elements() assert not share3.get_artifact(cli2.get_artifact_name(project, "test", element_name)) # Assert that if we add an extra remote that has the buildtree artfact cached, bst artifact push will # automatically attempt to pull it and will be successful, leading to the full artifact being pushed # to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote, # without exlipictly requiring a bst artifact pull. cli2.configure( {"artifacts": {"servers": [{"url": share1.repo, "push": False}, {"url": share3.repo, "push": True}]}} ) result = cli2.run(project=project, args=["--pull-buildtrees", "artifact", "push", element_name]) assert element_name in result.get_pulled_elements() with cli2.artifact.extract_buildtree(cwd, cwd, artifact_name) as buildtreedir: assert os.path.isdir(buildtreedir) assert element_name in result.get_pushed_elements() assert share3.get_artifact(cli2.get_artifact_name(project, "test", element_name)) # Ensure that only valid pull-buildtrees boolean options make it through the loading # process. @pytest.mark.parametrize("value,success", [(True, True), (False, True), ("pony", False), ("2", False), ("1", True)]) @pytest.mark.datafiles(DATA_DIR) def test_invalid_cache_pullbuildtrees(cli, datafiles, value, success): project = str(datafiles) cli.configure( { "cache": { "pull-buildtrees": value, } } ) res = cli.run(project=project, args=["workspace", "list"]) if success: res.assert_success() else: res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/integration/recc.py000066400000000000000000000173661514607367700226120ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_remote_execution(cli, datafiles): project = str(datafiles) checkout1 = os.path.join(cli.directory, "checkout1") checkout2 = os.path.join(cli.directory, "checkout2") element_name = "recc/remoteexecution.bst" # Always cache buildtrees to be able to check recc logs result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) if result.exit_code != 0: # Output recc logs in case of failure cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "cat config.log .recc-log/* */.recc-log/*", ], ) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout1]) assert result.exit_code == 0 assert_contains( checkout1, [ "/usr", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # Check the main build log result = cli.run(project=project, args=["artifact", "log", element_name]) assert result.exit_code == 0 log = result.output # Verify we get expected output exactly once assert log.count("Making all in src") == 1 result = cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "cat src/.recc-log/recc.buildbox*", ], ) assert result.exit_code == 0 recc_log = result.output # Verify recc is successfully using remote execution for both, compiling and linking assert recc_log.count("Executing action remotely") == 2 assert recc_log.count("Remote execution finished with exit code 0") == 2 # Delete artifact from BuildStream cache to trigger a BuildStream rebuild with action cache hits for recc result = cli.run(project=project, args=["artifact", "delete", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout2]) assert result.exit_code == 0 assert_contains( checkout2, [ "/usr", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) result = cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "cat src/.recc-log/recc.buildbox*", ], ) assert result.exit_code == 0 recc_log = result.output # Verify recc is getting action cache hits for both, compiling and linking assert recc_log.count("Action Cache hit") == 2 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_cache_only(cli, datafiles): project = str(datafiles) checkout1 = os.path.join(cli.directory, "checkout1") checkout2 = os.path.join(cli.directory, "checkout2") element_name = "recc/cacheonly.bst" # Always cache buildtrees to be able to check recc logs result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) if result.exit_code != 0: # Output recc logs in case of failure cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "cat config.log .recc-log/* */.recc-log/*", ], ) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout1]) assert result.exit_code == 0 assert_contains( checkout1, [ "/usr", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # Check the main build log result = cli.run(project=project, args=["artifact", "log", element_name]) assert result.exit_code == 0 log = result.output # Verify we get expected output exactly once assert log.count("Making all in src") == 1 result = cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "cat src/.recc-log/recc.buildbox*", ], ) assert result.exit_code == 0 recc_log = result.output # Verify recc is using local execution for both, compiling and linking assert recc_log.count("Action not cached and running in cache-only mode, executing locally") == 2 # Delete artifact from BuildStream cache to trigger a BuildStream rebuild with action cache hits for recc result = cli.run(project=project, args=["artifact", "delete", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout2]) assert result.exit_code == 0 assert_contains( checkout2, [ "/usr", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) result = cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", element_name, "--", "sh", "-c", "cat src/.recc-log/recc.buildbox*", ], ) assert result.exit_code == 0 recc_log = result.output # Verify recc is getting action cache hits for both, compiling and linking assert recc_log.count("Action Cache hit") == 2 apache-buildstream-27ae392/tests/integration/sandbox.py000066400000000000000000000142031514607367700233170ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_sandbox_shm(cli, datafiles): project = str(datafiles) element_name = "sandbox/test-dev-shm.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test that variable expansion works in build-arch sandbox config. # Regression test for https://gitlab.com/BuildStream/buildstream/-/issues/1303 @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_build_arch(cli, datafiles): project = str(datafiles) element_name = "sandbox/build-arch.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test that the REAPI socket is created in the sandbox. @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_remote_apis_socket(cli, datafiles): project = str(datafiles) element_name = "sandbox/remote-apis-socket.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Verify that loading the artifact succeeds artifact_name = cli.get_artifact_name(project, "test", element_name) result = cli.run(project=project, args=["artifact", "show", artifact_name]) assert result.exit_code == 0 # Test configuration with remote action cache for nested REAPI. @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_remote_apis_socket_with_action_cache(cli, tmpdir, datafiles): project = str(datafiles) element_name = "sandbox/remote-apis-socket.bst" with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( { "remote-execution": { "storage-service": {"url": share.repo}, "action-cache-service": {"url": share.repo, "push": True}, } } ) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test configuration with remote action cache for nested REAPI with updates enabled. @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_remote_apis_socket_with_action_cache_update(cli, tmpdir, datafiles): project = str(datafiles) element_name = "sandbox/remote-apis-socket-ac-update.bst" with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( { "remote-execution": { "storage-service": {"url": share.repo}, "action-cache-service": {"url": share.repo, "push": True}, } } ) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Verify that loading the artifact succeeds artifact_name = cli.get_artifact_name(project, "test", element_name) result = cli.run(project=project, args=["artifact", "show", artifact_name]) assert result.exit_code == 0 # Test configuration with cache storage-service and remote action cache for nested REAPI. @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_remote_apis_socket_with_cache_storage_service_and_action_cache(cli, tmpdir, datafiles): project = str(datafiles) element_name = "sandbox/remote-apis-socket.bst" with create_artifact_share(os.path.join(str(tmpdir), "remote")) as share: cli.configure( { "cache": { "storage-service": {"url": share.repo}, }, "remote-execution": { "action-cache-service": {"url": share.repo, "push": True}, }, } ) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # Test configuration with two different storage-services and remote action cache for nested REAPI. @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.datafiles(DATA_DIR) def test_remote_apis_socket_with_two_storage_services_and_action_cache(cli, tmpdir, datafiles): project = str(datafiles) element_name = "sandbox/remote-apis-socket.bst" with create_artifact_share(os.path.join(str(tmpdir), "remote1")) as share1, create_artifact_share( os.path.join(str(tmpdir), "remote2") ) as share2: cli.configure( { "cache": { "storage-service": {"url": share1.repo}, }, "remote-execution": { "storage-service": {"url": share2.repo}, "action-cache-service": {"url": share2.repo, "push": True}, }, } ) result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 apache-buildstream-27ae392/tests/integration/script.py000066400000000000000000000215541514607367700231740ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_script_element(name, path, config=None, variables=None): if config is None: config = {} if variables is None: variables = {} element = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build"}], "config": config, "variables": variables, } os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "script/script-layout.bst" create_script_element( element_name, element_path, config={ "commands": ["mkdir -p %{install-root}", "echo 'Hi' > %{install-root}/test"], }, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert text == "Hi\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Root directory not writable with userchroot", ) def test_script_root(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "script/script-layout.bst" create_script_element( element_name, element_path, config={ # Root-read only is False by default, we # want to check the default here # 'root-read-only': False, "commands": ["mkdir -p %{install-root}", "echo 'I can write to root' > /test", "cp /test %{install-root}"], }, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert text == "I can write to root\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script_no_root(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements") element_name = "script/script-layout.bst" create_script_element( element_name, element_path, config={ "root-read-only": True, "commands": [ "mkdir -p %{install-root}", "echo 'I can not write to root' > /test", "cp /test %{install-root}", ], }, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code != 0 assert "/test: Read-only file system" in res.stderr or "/test: Permission denied" in res.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script_cwd(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_path = os.path.join(project, "elements") element_name = "script/script-layout.bst" create_script_element( element_name, element_path, config={ "commands": ["echo 'test' > test", "cp /buildstream/test %{install-root}"], }, variables={"cwd": "/buildstream"}, ) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert text == "test\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_script_layout(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "script/script-layout.bst" res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "test"), encoding="utf-8") as f: text = f.read() assert text == "Hi\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Root directory not writable with userchroot", ) def test_regression_cache_corruption(cli, datafiles): project = str(datafiles) checkout_original = os.path.join(cli.directory, "checkout-original") checkout_after = os.path.join(cli.directory, "checkout-after") element_name = "script/corruption.bst" canary_element_name = "script/corruption-image.bst" res = cli.run(project=project, args=["build", canary_element_name]) assert res.exit_code == 0 res = cli.run( project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original] ) assert res.exit_code == 0 with open(os.path.join(checkout_original, "canary"), encoding="utf-8") as f: assert f.read() == "alive\n" res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after]) assert res.exit_code == 0 with open(os.path.join(checkout_after, "canary"), encoding="utf-8") as f: assert f.read() == "alive\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_regression_tmpdir(cli, datafiles): project = str(datafiles) element_name = "script/tmpdir.bst" res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="Root directory not writable with userchroot", ) def test_regression_cache_corruption_2(cli, datafiles): project = str(datafiles) checkout_original = os.path.join(cli.directory, "checkout-original") checkout_after = os.path.join(cli.directory, "checkout-after") element_name = "script/corruption-2.bst" canary_element_name = "script/corruption-image.bst" res = cli.run(project=project, args=["build", canary_element_name]) assert res.exit_code == 0 res = cli.run( project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_original] ) assert res.exit_code == 0 with open(os.path.join(checkout_original, "canary"), encoding="utf-8") as f: assert f.read() == "alive\n" res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["artifact", "checkout", canary_element_name, "--directory", checkout_after]) assert res.exit_code == 0 with open(os.path.join(checkout_after, "canary"), encoding="utf-8") as f: assert f.read() == "alive\n" apache-buildstream-27ae392/tests/integration/shell.py000066400000000000000000000435731514607367700230040ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import uuid import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # execute_shell() # # Helper to run `bst shell` and first ensure that the element is built # # Args: # cli (Cli): The cli runner fixture # project (str): The project directory # command (list): The command argv list # config (dict): A project.conf dictionary to composite over the default # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` # def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 args = ["shell"] if isolate: args += ["--isolate"] if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] args += [element, "--", *command] return cli.run_project_config(project=project, project_config=config, args=args) # Test running something through a shell, allowing it to find the # executable @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_shell(cli, datafiles): project = str(datafiles) result = execute_shell(cli, project, ["echo", "Ponies!"]) assert result.exit_code == 0 assert result.output == "Ponies!\n" # Test running an executable directly @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_executable(cli, datafiles): project = str(datafiles) result = execute_shell(cli, project, ["/bin/echo", "Horseys!"]) assert result.exit_code == 0 assert result.output == "Horseys!\n" # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") # This test seems to fail or pass depending on if this file is run or the hole test suite def test_env_assign(cli, datafiles, animal): project = str(datafiles) expected = animal + "\n" result = execute_shell( cli, project, ["/bin/sh", "-c", "echo ${ANIMAL}"], config={"shell": {"environment": {"ANIMAL": animal}}} ) assert result.exit_code == 0 assert result.output == expected # Test shell environment variable explicit assignments with host env var expansion @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") # This test seems to fail or pass depending on if this file is run or the hole test suite def test_env_assign_expand_host_environ(cli, datafiles, animal): project = str(datafiles) expected = "The animal is: {}\n".format(animal) os.environ["BEAST"] = animal result = execute_shell( cli, project, ["/bin/sh", "-c", "echo ${ANIMAL}"], config={"shell": {"environment": {"ANIMAL": "The animal is: ${BEAST}"}}}, ) assert result.exit_code == 0 assert result.output == expected # Test that shell environment variable explicit assignments are discarded # when running an isolated shell @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") # This test seems to faili or pass depending on if this file is run or the hole test suite def test_env_assign_isolated(cli, datafiles, animal): project = str(datafiles) result = execute_shell( cli, project, ["/bin/sh", "-c", "echo ${ANIMAL}"], isolate=True, config={"shell": {"environment": {"ANIMAL": animal}}}, ) assert result.exit_code == 0 assert result.output == "\n" # Test running an executable in a runtime with no shell (i.e., no # /bin/sh) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN == "buildbox-run-userchroot", reason="buildbox-run-userchroot requires a shell", ) def test_no_shell(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements") element_name = "shell/no-shell.bst" # Create an element that removes /bin/sh from the base runtime element = { "kind": "script", "depends": [{"filename": "base.bst", "type": "build"}], "variables": {"install-root": "/"}, "config": {"commands": ["rm /bin/sh"]}, } os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True) _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) result = execute_shell(cli, project, ["/bin/echo", "Pegasissies!"], element=element_name) assert result.exit_code == 0 assert result.output == "Pegasissies!\n" # Test that bind mounts defined in project.conf work @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt"), (None)]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN != "buildbox-run-bubblewrap", reason="Only available with bubblewrap", ) def test_host_files(cli, datafiles, path): project = str(datafiles) ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt") if path is None: result = execute_shell(cli, project, ["cat", ponyfile], config={"shell": {"host-files": [ponyfile]}}) else: result = execute_shell( cli, project, ["cat", path], config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}} ) assert result.exit_code == 0 assert result.output == "pony\n" # Test that bind mounts defined in project.conf work @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN != "buildbox-run-bubblewrap", reason="Only available with bubblewrap", ) def test_host_files_expand_environ(cli, datafiles, path): project = str(datafiles) hostpath = os.path.join(project, "files", "shell-mount") fullpath = os.path.join(path, "pony.txt") os.environ["BASE_PONY"] = path os.environ["HOST_PONY_PATH"] = hostpath result = execute_shell( cli, project, ["cat", fullpath], config={ "shell": {"host-files": [{"host_path": "${HOST_PONY_PATH}/pony.txt", "path": "${BASE_PONY}/pony.txt"}]} }, ) assert result.exit_code == 0 assert result.output == "pony\n" # Test that bind mounts defined in project.conf dont mount in isolation @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_isolated_no_mount(cli, datafiles, path): project = str(datafiles) ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt") result = execute_shell( cli, project, ["cat", path], isolate=True, config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}}, ) assert result.exit_code != 0 assert path in result.stderr assert "No such file or directory" in result.stderr # Test that we warn about non-existing files on the host if the mount is not # declared as optional, and that there is no warning if it is optional @pytest.mark.parametrize("optional", [("mandatory"), ("optional")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_host_files_missing(cli, datafiles, optional): project = str(datafiles) ponyfile = os.path.join(project, "files", "shell-mount", "horsy.txt") option = optional == "optional" # Assert that we did successfully run something in the shell anyway result = execute_shell( cli, project, ["echo", "Hello"], config={"shell": {"host-files": [{"host_path": ponyfile, "path": "/etc/pony.conf", "optional": option}]}}, ) assert result.exit_code == 0 assert result.output == "Hello\n" if option: # Assert that there was no warning about the mount assert ponyfile not in result.stderr else: # Assert that there was a warning about the mount assert ponyfile in result.stderr # Test that bind mounts defined in project.conf work @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")]) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.xfail( HAVE_SANDBOX == "buildbox-run" and BUILDBOX_RUN != "buildbox-run-bubblewrap", reason="Only available with bubblewrap", ) def test_cli_mount(cli, datafiles, path): project = str(datafiles) ponyfile = os.path.join(project, "files", "shell-mount", "pony.txt") result = execute_shell(cli, project, ["cat", path], mount=(ponyfile, path)) assert result.exit_code == 0 assert result.output == "pony\n" # Test that we can see the workspace files in a shell @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_visible(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_name = "workspace/workspace-mount-fail.bst" # Open a workspace on our build failing element # res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # Ensure the dependencies of our build failing element are built result = cli.run(project=project, args=["build", "base.bst"]) assert result.exit_code == 0 # Obtain a copy of the hello.c content from the workspace # workspace_hello_path = os.path.join(cli.directory, "workspace", "hello.c") assert os.path.exists(workspace_hello_path) with open(workspace_hello_path, "r", encoding="utf-8") as f: workspace_hello = f.read() # Cat the hello.c file from a bst shell command, and assert # that we got the same content here # result = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "hello.c"]) assert result.exit_code == 0 assert result.output == workspace_hello # Test system integration commands can access devices in /dev @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_integration_devices(cli, datafiles): project = str(datafiles) element_name = "integration.bst" result = execute_shell(cli, project, ["true"], element=element_name) assert result.exit_code == 0 # Test that a shell can be opened from an external workspace @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("build_shell", [("build"), ("nobuild")]) @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"]) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element): tmpdir = tmpdir_factory.mktemp(os.path.basename(__file__)) project = str(datafiles) element_name = "autotools/amhello.bst" workspace_dir = os.path.join(str(tmpdir), "workspace") if guess_element: # Mutate the project.conf to use a default shell command project_file = os.path.join(project, "project.conf") config_text = "shell:\n command: ['true']\n" with open(project_file, "a", encoding="utf-8") as f: f.write(config_text) result = cli.run(project=project, args=["workspace", "open", "--directory", workspace_dir, element_name]) result.assert_success() result = cli.run(project=project, args=["-C", workspace_dir, "build", element_name]) result.assert_success() command = ["-C", workspace_dir, "shell"] if build_shell == "build": command.append("--build") if not guess_element: command.extend([element_name, "--", "true"]) result = cli.run(project=project, cwd=workspace_dir, args=command) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache): project = str(datafiles) element_name = "autotools/amhello.bst" # push to an artifact server so we can pull from it later. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["build", element_name]) result.assert_success() # If the build is cached then it might not push to the artifact cache result = cli.run(project=project, args=["artifact", "push", element_name]) result.assert_success() result = cli.run(project=project, args=["shell", element_name]) result.assert_success() # do a checkout and get the digest of the hello binary. result = cli.run( project=project, args=[ "artifact", "checkout", "--deps", "none", "--directory", os.path.join(str(tmpdir), "tmp"), "autotools/amhello.bst", ], ) result.assert_success() digest = utils.sha256sum(os.path.join(str(tmpdir), "tmp", "usr", "bin", "hello")) # Remove the binary from the CAS cachedir = cli.config["cachedir"] objpath = os.path.join(cachedir, "cas", "objects", digest[:2], digest[2:]) os.unlink(objpath) # check shell doesn't work when it cannot pull the missing bits cli.configure({"artifacts": {}}) result = cli.run(project=project, args=["shell", element_name, "--", "hello"]) result.assert_main_error(ErrorDomain.APP, "shell-missing-deps") # check the artifact gets completed with access to the remote cli.configure({"artifacts": {"servers": [{"url": share.repo, "push": True}]}}) result = cli.run(project=project, args=["shell", element_name, "--", "hello"]) result.assert_success() assert "autotools/amhello.bst" in result.get_pulled_elements() # Test that the sources are fetched automatically when opening a build shell @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_build_shell_fetch(cli, datafiles): project = str(datafiles) element_name = "build-shell-fetch.bst" # Create a file with unique contents such that it cannot be in the cache already test_filepath = os.path.join(project, "files", "hello.txt") test_message = "Hello World! {}".format(uuid.uuid4()) with open(test_filepath, "w", encoding="utf-8") as f: f.write(test_message) checksum = utils.sha256sum(test_filepath) # Create an element that has this unique file as a source element = { "kind": "manual", "depends": ["base.bst"], "sources": [{"kind": "remote", "url": "project_dir:/files/hello.txt", "ref": checksum}], } _yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name)) # Ensure our dependencies are cached result = cli.run(project=project, args=["build", "base.bst"]) result.assert_success() # Ensure our sources are not cached assert cli.get_element_state(project, element_name) == "fetch needed" # Launching a shell should fetch any uncached sources result = cli.run(project=project, args=["shell", "--build", element_name, "cat", "hello.txt"]) result.assert_success() assert result.output == test_message @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_shell_artifact_failure(cli, datafiles): project = str(datafiles) # This happens to be the artifact name of "autotools/amhello.bst" if we built it, but we don't # need to build it for this test. artifact_name = "test/autotools-amhello/847d23cd0e61c54a6beca9071b64643388ad2ab783191358b2a499fe77e86563" result = cli.run(project=project, args=["shell", artifact_name, "--", "hello"]) result.assert_main_error(ErrorDomain.APP, "only-buildtrees-supported") apache-buildstream-27ae392/tests/integration/shellbuildtrees.py000066400000000000000000000426651514607367700250700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli, cli_integration, Cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils import ArtifactShare pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # # Ensure that we didn't get a build tree if we didn't ask for one # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_unused(cli_integration, datafiles): # We can only test the non interacitve case # The non interactive case defaults to not using buildtrees # for `bst shell --build` project = str(datafiles) element_name = "build-shell/buildtree.bst" res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() res = cli_integration.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() # # Ensure we can use a buildtree from a successful build # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_from_success(cli_integration, datafiles): # Test that if we ask for a build tree it is there. project = str(datafiles) element_name = "build-shell/buildtree.bst" res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() res = cli_integration.run( project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "cat", "test"] ) res.assert_success() assert "Hi" in res.output # # Ensure we can use a buildtree from a failed build # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_buildtree_from_failure(cli_integration, datafiles): # Test that we can use a build tree after a failure project = str(datafiles) element_name = "build-shell/buildtree-fail.bst" res = cli_integration.run(project=project, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) # Assert that file has expected contents res = cli_integration.run( project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"] ) res.assert_success() assert "WARNING using a buildtree from a failed build" in res.stderr assert "Hi" in res.output ########################################################################### # Custom fixture ahead # ########################################################################### # # There are a lot of scenarios to test with launching shells with various states # of local cache, which all require that artifacts be built in an artifact share. # # We want to use @pytest.mark.parametrize() here so that we can more coherently test # specific scenarios, but testing each of these in a separate test is very expensive. # # For this reason, we use some module scope fixtures which will prepare the # ArtifactShare() object by building and pushing to it, and the same ArtifactShare() # object is shared across all tests which need the ArtifactShare() to be in that # given state. # # This means we only need to download (fetch) the external alpine runtime and # push it to our internal ArtifactShare() once, but we can reuse it for many # parametrized tests. # # It is important that none of the tests using these fixtures access the # module scope ArtifactShare() instances with "push" access, as tests # should not be modifying the state of the shared data. # ########################################################################### # create_built_artifact_share() # # A helper function to create an ArtifactShare object with artifacts # prebuilt, this can be shared across multiple tests which access # the artifact share in a read-only fashion. # # Args: # tmpdir (str): The temp directory to be used # cache_buildtrees (bool): Whether to cache buildtrees when building # integration_cache (IntegrationCache): The session wide integration cache so that we # can reuse the sources from previous runs # def create_built_artifact_share(tmpdir, cache_buildtrees, integration_cache): element_name = "build-shell/buildtree.bst" # Replicate datafiles behavior and do work entirely in the temp directory project = os.path.join(tmpdir, "project") shutil.copytree(DATA_DIR, project) # Create the share to be hosted from this temp directory share = ArtifactShare(os.path.join(tmpdir, "artifactcache")) # Create a Cli instance to build and populate the share cli = Cli(os.path.join(tmpdir, "cache")) cli.configure( {"artifacts": {"servers": [{"url": share.repo, "push": True}]}, "sourcedir": integration_cache.sources} ) # Optionally cache build trees args = [] if cache_buildtrees: args += ["--cache-buildtrees", "always"] args += ["build", element_name] # Build result = cli.run(project=project, args=args) result.assert_success() # Assert that the artifact is indeed in the share assert cli.get_element_state(project, element_name) == "cached" artifact_name = cli.get_artifact_name(project, "test", element_name) assert share.get_artifact(artifact_name) return share # share_with_buildtrees() # # A module scope fixture which prepares an ArtifactShare() instance # which will have all dependencies of "build-shell/buildtree.bst" built and # cached with buildtrees also cached. # @pytest.fixture(scope="module") def share_with_buildtrees(tmp_path_factory, integration_cache): # Get a temporary directory for this module scope fixture tmpdir = tmp_path_factory.mktemp("artifact_share_with_buildtrees") # Create our ArtifactShare instance which will persist for the duration of # the class scope fixture. share = create_built_artifact_share(tmpdir, True, integration_cache) try: yield share finally: share.close() # share_without_buildtrees() # # A module scope fixture which prepares an ArtifactShare() instance # which will have all dependencies of "build-shell/buildtree.bst" built # but without caching any buildtrees. # @pytest.fixture(scope="module") def share_without_buildtrees(tmp_path_factory, integration_cache): # Get a temporary directory for this module scope fixture tmpdir = tmp_path_factory.mktemp("artifact_share_without_buildtrees") # Create our ArtifactShare instance which will persist for the duration of # the class scope fixture. share = create_built_artifact_share(tmpdir, False, integration_cache) try: yield share finally: share.close() # maybe_pull_deps() # # Convenience function for optionally pulling element dependencies # in the following parametrized tests. # # Args: # cli (Cli): The Cli object # project (str): The project path # element_name (str): The element name # pull_deps (str): The argument for `--deps`, or None # pull_buildtree (bool): Whether to also pull buildtrees # def maybe_pull_deps(cli, project, element_name, pull_deps, pull_buildtree): # Optionally pull the buildtree along with `bst artifact pull` if pull_deps: args = [] if pull_buildtree: args += ["--pull-buildtrees"] args += ["artifact", "pull", "--deps", pull_deps, element_name] # Pull from cache result = cli.run(project=project, args=args) result.assert_success() # # Test behavior of launching a shell and requesting to use a buildtree, with # various states of local cache (ranging from nothing cached to everything cached) # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.parametrize( "pull_deps,pull_buildtree,expect_error", [ # Don't pull at all (None, False, "missing-buildtree-artifact-not-cached"), # Pull only dependencies ("build", False, "missing-buildtree-artifact-not-cached"), # Pull all elements including the shell element, but without the buildtree ("all", False, "missing-buildtree-artifact-buildtree-not-cached"), # Pull all elements including the shell element, and pull buildtrees ("all", True, None), # Pull only the artifact, but without the buildtree ("none", False, "missing-buildtree-artifact-buildtree-not-cached"), # Pull only the artifact with its buildtree ("none", True, None), ], ids=[ "no-pull", "pull-only-deps", "pull-without-buildtree", "pull-with-buildtree", "pull-target-without-buildtree", "pull-target-with-buildtree", ], ) def test_shell_use_cached_buildtree(share_with_buildtrees, datafiles, cli, pull_deps, pull_buildtree, expect_error): project = str(datafiles) element_name = "build-shell/buildtree.bst" cli.configure({"artifacts": {"servers": [{"url": share_with_buildtrees.repo}]}}) # Optionally pull the buildtree along with `bst artifact pull` maybe_pull_deps(cli, project, element_name, pull_deps, pull_buildtree) # Disable access to the artifact server after pulling, so that `bst shell` cannot automatically # pull the missing bits, this should be equivalent to the missing bits being missing in a # remote server cli.configure({"artifacts": {}}) # Run the shell without asking it to pull any buildtree, just asking to use a buildtree result = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]) if expect_error: result.assert_main_error(ErrorDomain.APP, expect_error) else: result.assert_success() assert "Hi" in result.output # # Test behavior of launching a shell and requesting to use a buildtree, while allowing # BuildStream to download any missing bits from the artifact server on the fly (which # it will do by default) again with various states of local cache (ranging from nothing # cached to everything cached) # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.parametrize( "pull_deps,pull_buildtree", [ # Don't pull at all (None, False), # Pull only dependencies ("build", False), # Pull all elements including the shell element, but without the buildtree ("all", False), # Pull all elements including the shell element, and pull buildtrees ("all", True), ], ids=["no-pull", "pull-only-deps", "pull-without-buildtree", "pull-with-buildtree"], ) def test_shell_pull_cached_buildtree(share_with_buildtrees, datafiles, cli, pull_deps, pull_buildtree): project = str(datafiles) element_name = "build-shell/buildtree.bst" cli.configure({"artifacts": {"servers": [{"url": share_with_buildtrees.repo}]}}) # Optionally pull the buildtree along with `bst artifact pull` maybe_pull_deps(cli, project, element_name, pull_deps, pull_buildtree) # Run the shell and request that required artifacts and buildtrees should be pulled result = cli.run( project=project, args=[ "--pull-buildtrees", "shell", "--build", element_name, "--use-buildtree", "--", "cat", "test", ], ) # In this case, we should succeed every time, regardless of what was # originally available in the local cache. # result.assert_success() assert "Hi" in result.output # Check that the working directory is writable result = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "touch", "foo", ], ) result.assert_success() # # Test behavior of shelling into a buildtree by its artifact name # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_shell_pull_artifact_cached_buildtree(share_with_buildtrees, datafiles, cli): project = str(datafiles) artifact_name = "test/build-shell-buildtree/f2f21aa730dcc49f682bb8406ef4705a51213ccf9739dc43b8b4fdc2d310e3f6" cli.configure({"artifacts": {"servers": [{"url": share_with_buildtrees.repo}]}}) # Run the shell and request that required artifacts and buildtrees should be pulled result = cli.run( project=project, args=[ "--pull-buildtrees", "shell", "--build", "--use-buildtree", artifact_name, "--", "cat", "test", ], ) # In this case, we should succeed every time, regardless of what was # originally available in the local cache. # result.assert_success() assert "Hi" in result.output # Check that the working directory is writable result = cli.run( project=project, args=[ "shell", "--build", "--use-buildtree", artifact_name, "--", "touch", "foo", ], ) result.assert_success() # # Test behavior of launching a shell and requesting to use a buildtree. # # In this case we download everything we need first, but the buildtree was never cached at build time # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_shell_use_uncached_buildtree(share_without_buildtrees, datafiles, cli): project = str(datafiles) element_name = "build-shell/buildtree.bst" cli.configure({"artifacts": {"servers": [{"url": share_without_buildtrees.repo}]}}) # Pull everything we would need maybe_pull_deps(cli, project, element_name, "all", True) # Run the shell without asking it to pull any buildtree, just asking to use a buildtree result = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]) # Sorry, a buildtree was never cached for this element result.assert_main_error(ErrorDomain.APP, "missing-buildtree-artifact-created-without-buildtree") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_shell_script_element(datafiles, cli_integration): project = str(datafiles) element_name = "build-shell/script.bst" result = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) result.assert_success() # Run the shell and use the cached buildtree on this script element result = cli_integration.run( project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "/test"] ) result.assert_success() assert "Hi" in result.output @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") @pytest.mark.parametrize( "element_name,expect_success", [ # Build shell into a compose element which succeeded ("build-shell/compose-success.bst", True), # Build shell into a compose element with failed integration commands ("build-shell/compose-fail.bst", False), ], ids=["integration-success", "integration-fail"], ) def test_shell_compose_element(datafiles, cli_integration, element_name, expect_success): project = str(datafiles) # Build the element so it's in the local cache, ensure caching of buildtrees at build time result = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) if expect_success: result.assert_success() else: result.assert_main_error(ErrorDomain.STREAM, None) # Ensure that the shell works regardless of success expectations # result = cli_integration.run( project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "echo", "Hi"] ) result.assert_success() assert "Hi" in result.output # Check the file created with integration commands # result = cli_integration.run( project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "/integration-success"], ) if expect_success: result.assert_success() assert "Hi" in result.output else: # Here the exit code is determined by `cat`, and will be non-zero. # # We cannot use result.assert_main_error() because that explicitly expects -1 assert result.exit_code != 0 apache-buildstream-27ae392/tests/integration/sockets.py000066400000000000000000000032401514607367700233330ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_builddir_socket_ignored(cli, datafiles): project = str(datafiles) element_name = "sockets/make-builddir-socket.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_install_root_socket_ignored(cli, datafiles): project = str(datafiles) element_name = "sockets/make-install-root-socket.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 apache-buildstream-27ae392/tests/integration/source-determinism.py000066400000000000000000000073311514607367700255030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX, CASD_SEPARATE_USER DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_test_file(*path, mode=0o644, content="content\n"): path = os.path.join(*path) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write(content) os.fchmod(f.fileno(), mode) def create_test_directory(*path, mode=0o644): create_test_file(*path, ".keep", content="") path = os.path.join(*path) os.chmod(path, mode) @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_deterministic_source_local(cli, tmpdir, datafiles): """Only user rights should be considered for local source.""" project = str(datafiles) element_name = "test.bst" element_path = os.path.join(project, "elements", element_name) sourcedir = os.path.join(project, "source") element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "sources": [{"kind": "local", "path": "source"}], "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']}, } _yaml.roundtrip_dump(element, element_path) def get_value_for_mask(mask): checkoutdir = os.path.join(str(tmpdir), "checkout-{}".format(mask)) create_test_file(sourcedir, "a.txt", mode=0o644 & mask) create_test_file(sourcedir, "b.txt", mode=0o755 & mask) create_test_file(sourcedir, "c.txt", mode=0o4755 & mask) create_test_file(sourcedir, "d.txt", mode=0o2755 & mask) create_test_file(sourcedir, "e.txt", mode=0o1755 & mask) create_test_directory(sourcedir, "dir-a", mode=0o0755 & mask) create_test_directory(sourcedir, "dir-b", mode=0o4755 & mask) create_test_directory(sourcedir, "dir-c", mode=0o2755 & mask) create_test_directory(sourcedir, "dir-d", mode=0o1755 & mask) try: test_values = [] result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkoutdir]) result.assert_success() with open(os.path.join(checkoutdir, "ls-l"), "r", encoding="utf-8") as f: for line in f.readlines(): test_values.append(line.split()[0] + " " + line.split()[-1]) return test_values finally: cli.remove_artifact_from_cache(project, element_name) if CASD_SEPARATE_USER: # buildbox-casd running as separate user of the same group can't # read files with too restrictive permissions. assert get_value_for_mask(0o7777) == get_value_for_mask(0o0770) else: assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700) apache-buildstream-27ae392/tests/integration/stack.py000066400000000000000000000033061514607367700227700ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_stack(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "stack/stack.bst" res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert res.exit_code == 0 with open(os.path.join(checkout, "hi"), encoding="utf-8") as f: hi = f.read() with open(os.path.join(checkout, "another-hi"), encoding="utf-8") as f: another_hi = f.read() assert hi == "Hi\n" assert another_hi == "Another hi\n" apache-buildstream-27ae392/tests/integration/symlinks.py000066400000000000000000000102701514607367700235320ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_absolute_symlinks(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "symlinks/dangling-symlink.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == 0 symlink = os.path.join(checkout, "opt", "orgname") assert os.path.islink(symlink) # The symlink is created to point to /usr/orgs/orgname and BuildStream # should not mangle symlinks. assert os.readlink(symlink) == "/usr/orgs/orgname" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_disallow_overlaps_inside_symlink_with_dangling_target(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "symlinks/dangling-symlink-overlap.bst" result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == -1 assert "Destination is a symlink, not a directory: /opt/orgname" in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_detect_symlink_overlaps_pointing_outside_sandbox(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "symlinks/symlink-to-outside-sandbox-overlap.bst" # Building the two elements should succeed... result = cli.run(project=project, args=["build", element_name]) assert result.exit_code == 0 # ...but when we compose them together, the overlaps create paths that # point outside the sandbox which BuildStream needs to detect before it # tries to actually write there. result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) assert result.exit_code == -1 assert "Destination is a symlink, not a directory: /opt/escape-hatch" in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_symlink_in_sandbox_path(cli, datafiles): project = str(datafiles) element_name = "symlinks/link-on-path-use.bst" base_element_name = "symlinks/link-on-path.bst" # This test is inspired by how freedesktop-SDK has /bin -> /usr/bin # Create a element that has sh in altbin and a link from bin to altbin result1 = cli.run(project=project, args=["build", base_element_name]) result1.assert_success() # Build a element that uses the element that has sh in altbin. result2 = cli.run(project=project, args=["build", element_name]) result2.assert_success() # When this element is built it demonstrates that the virtual sandbox # can detect sh across links and that the sandbox can find sh accross # the link from its PATH. apache-buildstream-27ae392/tests/integration/workspace.py000066400000000000000000000526621514607367700236720ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from buildstream.exceptions import ErrorDomain from buildstream.utils import BST_ARBITRARY_TIMESTAMP from tests.testutils import wait_for_cache_granularity pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_stages_once(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_name = "workspace/workspace-mount.bst" res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 assert cli.get_element_key(project, element_name) != "{:?<64}".format("") res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_mount(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_name = "workspace/workspace-mount.bst" res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 assert os.path.exists(os.path.join(cli.directory, "workspace")) @pytest.mark.datafiles(DATA_DIR) def test_workspace_mount_on_read_only_directory(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") os.makedirs(workspace) element_name = "workspace/workspace-mount.bst" # make directory RO os.chmod(workspace, 0o555) res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_commanddir(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_name = "workspace/workspace-commanddir.bst" res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Check that the object file was created in the command-subdir `build` # using the cached buildtree. res = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "find", "..", "-mindepth", "1", ], ) res.assert_success() files = res.output.splitlines() assert "../build/hello.o" in files @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_updated_dependency(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_path = os.path.join(project, "elements") element_name = "workspace/workspace-updated-dependency.bst" dep_name = "workspace/dependency.bst" dependency = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "config": { "build-commands": [ "mkdir -p %{install-root}/etc/test/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', ] }, } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # We build the workspaced element, so that we have an artifact # with specific built dependencies res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Now we update a dependency of our element. dependency["config"]["build-commands"] = [ "mkdir -p %{install-root}/etc/test/", 'echo "Hello china!" > %{install-root}/etc/test/hello.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # `Make` would look at timestamps and normally not realize that # our dependency's header files changed. BuildStream must # therefore ensure that we change the mtimes of any files touched # since the last successful build of this element, otherwise this # build will fail. res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"]) assert res.exit_code == 0 assert res.output == "Hello china!\n\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_update_dependency_failed(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_path = os.path.join(project, "elements") element_name = "workspace/workspace-updated-dependency-failed.bst" dep_name = "workspace/dependency.bst" dependency = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "config": { "build-commands": [ "mkdir -p %{install-root}/etc/test/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt', ] }, } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # We build the workspaced element, so that we have an artifact # with specific built dependencies res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Now we update a dependency of our element. dependency["config"]["build-commands"] = [ "mkdir -p %{install-root}/etc/test/", 'echo "Hello china!" > %{install-root}/etc/test/hello.txt', 'echo "Hello brazil!" > %{install-root}/etc/test/brazil.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # And our build fails! with open(os.path.join(workspace, "Makefile"), "a", encoding="utf-8") as f: f.write("\texit 1") res = cli.run(project=project, args=["build", element_name]) assert res.exit_code != 0 # We update our dependency again... dependency["config"]["build-commands"] = [ "mkdir -p %{install-root}/etc/test/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', 'echo "Hello spain!" > %{install-root}/etc/test/brazil.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # And fix the source with open(os.path.join(workspace, "Makefile"), "r", encoding="utf-8") as f: makefile = f.readlines() with open(os.path.join(workspace, "Makefile"), "w", encoding="utf-8") as f: f.write("\n".join(makefile[:-1])) # Since buildstream thinks hello.txt did not change, we could end # up not rebuilding a file! We need to make sure that a case like # this can't blind-side us. res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"]) assert res.exit_code == 0 assert res.output == "Hello world!\nHello spain!\n\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_updated_dependency_nested(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_path = os.path.join(project, "elements") element_name = "workspace/workspace-updated-dependency-nested.bst" dep_name = "workspace/dependency.bst" dependency = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "config": { "build-commands": [ "mkdir -p %{install-root}/etc/test/tests/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', 'echo "Hello brazil!" > %{install-root}/etc/test/tests/brazil.txt', ] }, } os.makedirs(os.path.dirname(os.path.join(element_path, dep_name)), exist_ok=True) _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) # First open the workspace res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # We build the workspaced element, so that we have an artifact # with specific built dependencies res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Now we update a dependency of our element. dependency["config"]["build-commands"] = [ "mkdir -p %{install-root}/etc/test/tests/", 'echo "Hello world!" > %{install-root}/etc/test/hello.txt', 'echo "Hello test!" > %{install-root}/etc/test/tests/tests.txt', ] _yaml.roundtrip_dump(dependency, os.path.join(element_path, dep_name)) res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Buildstream should pick up the newly added element, and pick up # the lack of the newly removed element res = cli.run(project=project, args=["shell", element_name, "/usr/bin/test.sh"]) assert res.exit_code == 0 assert res.output == "Hello world!\nHello test!\n\n" @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_incremental_configure_commands_run_only_once(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_path = os.path.join(project, "elements") element_name = "workspace/incremental.bst" element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "sources": [{"kind": "local", "path": "files/workspace-configure-only-once"}], "config": {"configure-commands": ["$SHELL configure"]}, } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # We open a workspace on the above element res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Then we build, and check whether the configure step succeeded res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() # check that the workspace was not configured outside the sandbox assert not os.path.exists(os.path.join(workspace, "prepared")) # the configure should have been run in the sandbox, so check the buildtree res = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1", ], ) res.assert_success() files = res.output.splitlines() assert "./prepared" in files assert not "./prepared-again" in files # Add file to workspace to trigger an (incremental) build with open(os.path.join(workspace, "newfile"), "w", encoding="utf-8"): pass # When we build again, the configure commands should not be # called, and we should therefore exit cleanly (the configure # commands are set to always fail after the first run) res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() assert not os.path.exists(os.path.join(workspace, "prepared-again")) res = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1", ], ) res.assert_success() files = res.output.splitlines() assert "./prepared" in files assert not "./prepared-again" in files # Test that rebuilding an already built workspaced element does # not crash after the last successfully built artifact is removed # from the cache # # A user can remove their artifact cache, or manually remove the # artifact with `bst artifact delete`, or BuildStream can delete # the last successfully built artifact for this workspace as a # part of a cleanup job. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_missing_last_successful(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_name = "workspace/workspace-commanddir.bst" # Open workspace res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) assert res.exit_code == 0 # Build first, this will record the last successful build in local state res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Remove the artifact from the cache, invalidating the last successful build res = cli.run(project=project, args=["artifact", "delete", element_name]) assert res.exit_code == 0 # Build again, ensure we dont crash just because the artifact went missing res = cli.run(project=project, args=["build", element_name]) assert res.exit_code == 0 # Check that we can still read failed workspace logs @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_workspace_failed_logs(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "failing_amhello") element_name = "autotools/amhello-failure.bst" # Open workspace res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Try to build and ensure the build fails res = cli.run(project=project, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) assert cli.get_element_state(project, element_name) == "failed" res = cli.run(project=project, args=["artifact", "log", element_name]) res.assert_success() log = res.output # Assert that we can get the log assert log != "" fail_str = "FAILURE {}: Running build-commands".format(element_name) batch_fail_str = "FAILURE {}: Running commands".format(element_name) assert fail_str in log or batch_fail_str in log def get_buildtree_file_contents(cli, project, element_name, filename): res = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "cat", filename, ], ) res.assert_success() return res.output @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_incremental(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_path = os.path.join(project, "elements") element_name = "workspace/incremental.bst" element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "sources": [{"kind": "local", "path": "files/workspace-incremental"}], "config": {"build-commands": ["make"]}, } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # We open a workspace on the above element res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Initial (non-incremental) build of the workspace res = cli.run(project=project, args=["build", element_name]) res.assert_success() # Save the random hash random_hash = get_buildtree_file_contents(cli, project, element_name, "random") # Verify the expected output file of the initial build assert get_buildtree_file_contents(cli, project, element_name, "copy") == "1" wait_for_cache_granularity() # Replace source file contents with '2' with open(os.path.join(workspace, "source"), "w", encoding="utf-8") as f: f.write("2") # Perform incremental build of the workspace res = cli.run(project=project, args=["build", element_name]) res.assert_success() # Verify that this was an incremental build by comparing the random hash assert get_buildtree_file_contents(cli, project, element_name, "random") == random_hash # Verify that the output file matches the new source file assert get_buildtree_file_contents(cli, project, element_name, "copy") == "2" wait_for_cache_granularity() # Replace source file contents with '3', however, set an old mtime such # that `make` will not pick up the change with open(os.path.join(workspace, "source"), "w", encoding="utf-8") as f: f.write("3") os.utime(os.path.join(workspace, "source"), (BST_ARBITRARY_TIMESTAMP, BST_ARBITRARY_TIMESTAMP)) # Perform incremental build of the workspace res = cli.run(project=project, args=["build", element_name]) res.assert_success() # Verify that this was an incremental build by comparing the random hash assert get_buildtree_file_contents(cli, project, element_name, "random") == random_hash # Verify that the output file still matches the previous content '2' assert get_buildtree_file_contents(cli, project, element_name, "copy") == "2" # Test incremental build after partial build / build failure @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_incremental_partial(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, "workspace") element_path = os.path.join(project, "elements") element_name = "workspace/incremental.bst" element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "sources": [{"kind": "local", "path": "files/workspace-partial"}], "config": {"build-commands": ["make random", "make copy1", "make copy2"]}, } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # We open a workspace on the above element res = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) res.assert_success() # Initial (non-incremental) build of the workspace res = cli.run(project=project, args=["build", element_name]) res.assert_success() # Save the random hash random_hash = get_buildtree_file_contents(cli, project, element_name, "random") # Verify the expected output files of the initial build assert get_buildtree_file_contents(cli, project, element_name, "copy1") == "1" assert get_buildtree_file_contents(cli, project, element_name, "copy2") == "1" wait_for_cache_granularity() # Delete source1 and replace source2 file contents with '2' os.unlink(os.path.join(workspace, "source1")) with open(os.path.join(workspace, "source2"), "w", encoding="utf-8") as f: f.write("2") # Perform incremental build of the workspace # This should fail because of the missing source1 file. res = cli.run(project=project, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) wait_for_cache_granularity() # Recreate source1 file with open(os.path.join(workspace, "source1"), "w", encoding="utf-8") as f: f.write("2") # Perform incremental build of the workspace res = cli.run(project=project, args=["build", element_name]) res.assert_success() # Verify that this was an incremental build by comparing the random hash assert get_buildtree_file_contents(cli, project, element_name, "random") == random_hash # Verify that both files got rebuilt assert get_buildtree_file_contents(cli, project, element_name, "copy1") == "2" assert get_buildtree_file_contents(cli, project, element_name, "copy2") == "2" apache-buildstream-27ae392/tests/internals/000077500000000000000000000000001514607367700207635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/__init__.py000066400000000000000000000000001514607367700230620ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/cascache.py000066400000000000000000000112231514607367700230660ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import time from unittest.mock import MagicMock from buildstream._cas import casdprocessmanager from buildstream._messenger import Messenger from tests.testutils import casd_cache # # A dummy CASD script placeholder which supports the --version argument # DUMMY_CASD_SCRIPT_FMT = ( "#!/usr/bin/env sh\n" + "\n" + 'if test "$1" = "--version"; then\n' + ' echo "buildbox-casd 2.0.0"\n' + " exit 0\n" + "fi\n" + "{}\n" + "exit 0\n" ) def test_report_when_cascache_dies_before_asked_to(tmp_path, monkeypatch): dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd") dummy_buildbox_casd.write_text(DUMMY_CASD_SCRIPT_FMT.format("")) dummy_buildbox_casd.chmod(0o777) monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) messenger = MagicMock(spec_set=Messenger) with casd_cache(tmp_path.joinpath("casd"), messenger): time.sleep(1) assert messenger.bug.call_count == 1 message = messenger.bug.call_args[0][0] assert "0" in message assert "died" in message def test_report_when_cascache_exits_not_cleanly(tmp_path, monkeypatch): dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd") dummy_buildbox_casd.write_text(DUMMY_CASD_SCRIPT_FMT.format("while :\ndo\nsleep 60\ndone")) dummy_buildbox_casd.chmod(0o777) monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) # FIXME: this is a hack, we should instead have a socket be created nicely # on the fake casd script. This whole test suite probably would # need some cleanup monkeypatch.setattr(casdprocessmanager, "_CASD_TIMEOUT", 0.1) messenger = MagicMock(spec_set=Messenger) with casd_cache(tmp_path.joinpath("casd"), messenger): time.sleep(1) assert messenger.bug.call_count == 1 message = messenger.bug.call_args[0][0] assert "-15" in message assert "cleanly" in message def test_report_when_cascache_is_forcefully_killed(tmp_path, monkeypatch): dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd") dummy_buildbox_casd.write_text(DUMMY_CASD_SCRIPT_FMT.format("trap 'echo hello' TERM\nwhile :\ndo\nsleep 60\ndone")) dummy_buildbox_casd.chmod(0o777) monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) # FIXME: this is a hack, we should instead have a socket be created nicely # on the fake casd script. This whole test suite probably would # need some cleanup monkeypatch.setattr(casdprocessmanager, "_CASD_TIMEOUT", 0.1) messenger = MagicMock(spec_set=Messenger) with casd_cache(tmp_path.joinpath("casd"), messenger): time.sleep(1) assert messenger.warn.call_count == 1 message = messenger.warn.call_args[0][0] assert "killed" in message def test_casd_redirects_stderr_to_file_and_rotate(tmp_path, monkeypatch): n_max_log_files = 10 dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd") dummy_buildbox_casd.write_text(DUMMY_CASD_SCRIPT_FMT.format("printf '%s\n' hello")) dummy_buildbox_casd.chmod(0o777) monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep) casd_files_path = tmp_path.joinpath("casd") casd_parent_logs_path = tmp_path.joinpath("logs") casd_logs_path = casd_parent_logs_path.joinpath("_casd") # Ensure we don't have any files in the log directory assert not casd_logs_path.exists() existing_log_files = [] # Let's create the first `n_max_log_files` log files for i in range(1, n_max_log_files + 1): with casd_cache(casd_files_path): time.sleep(0.5) existing_log_files = sorted(casd_logs_path.iterdir()) assert len(existing_log_files) == i assert existing_log_files[-1].read_text() == "hello\n" # Ensure the oldest log files get removed first for _ in range(3): evicted_file = existing_log_files.pop(0) with casd_cache(casd_files_path): time.sleep(0.5) existing_log_files = sorted(casd_logs_path.iterdir()) assert len(existing_log_files) == n_max_log_files assert evicted_file not in existing_log_files assert existing_log_files[-1].read_text() == "hello\n" apache-buildstream-27ae392/tests/internals/context.py000066400000000000000000000133451514607367700230270ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._context import Context from buildstream import _yaml, utils from buildstream._exceptions import LoadError from buildstream.exceptions import LoadErrorReason DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "context", ) # Simple fixture to create a Context object. @pytest.fixture() def context_fixture(): if os.environ.get("XDG_CACHE_HOME"): cache_home = os.environ["XDG_CACHE_HOME"] else: cache_home = os.path.expanduser("~/.cache") with Context() as context: yield {"xdg-cache": cache_home, "context": context} ####################################### # Test instantiation # ####################################### def test_context_create(context_fixture): context = context_fixture["context"] assert isinstance(context, Context) ####################################### # Test configuration loading # ####################################### def test_context_load(context_fixture): context = context_fixture["context"] cache_home = os.path.normpath(context_fixture["xdg-cache"]) assert isinstance(context, Context) context.load(config=os.devnull) assert context.sourcedir == os.path.join(cache_home, "buildstream", "sources") assert context.builddir == os.path.join(cache_home, "buildstream", "build") assert context.cachedir == os.path.join(cache_home, "buildstream") assert context.logdir == os.path.join(cache_home, "buildstream", "logs") # Assert that a changed XDG_CACHE_HOME doesn't cause issues def test_context_load_envvar(context_fixture): os.environ["XDG_CACHE_HOME"] = "/some/path/" context = context_fixture["context"] assert isinstance(context, Context) context.load(config=os.devnull) assert context.sourcedir == os.path.join("/", "some", "path", "buildstream", "sources") assert context.builddir == os.path.join("/", "some", "path", "buildstream", "build") assert context.cachedir == os.path.join("/", "some", "path", "buildstream") assert context.logdir == os.path.join("/", "some", "path", "buildstream", "logs") # Reset the environment variable del os.environ["XDG_CACHE_HOME"] # Test that values in a user specified config file # override the defaults @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_user_config(context_fixture, datafiles): context = context_fixture["context"] cache_home = context_fixture["xdg-cache"] assert isinstance(context, Context) conf_file = os.path.join(datafiles, "userconf.yaml") context.load(conf_file) assert context.sourcedir == os.path.expanduser("~/pony") assert context.builddir == os.path.join(cache_home, "buildstream", "build") assert context.cachedir == os.path.join(cache_home, "buildstream") assert context.logdir == os.path.join(cache_home, "buildstream", "logs") @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_priority(datafiles): confdir = os.path.join(str(datafiles), "config") os.makedirs(confdir) # The fallback (usual) config file bst_conf_path = os.path.join(confdir, "buildstream.conf") bst_conf = {"sourcedir": "/sources"} _yaml.roundtrip_dump(bst_conf, bst_conf_path) # The version specific config file major_version, _ = utils._get_bst_api_version() bst_conf_path = os.path.join(confdir, "buildstream{}.conf".format(major_version)) bst_conf = {"sourcedir": "/other_sources"} _yaml.roundtrip_dump(bst_conf, bst_conf_path) # Load the Context() object and assert that we've chosen # the version specific one. # os.environ["XDG_CONFIG_HOME"] = confdir with Context() as context: context.load() assert context.sourcedir == "/other_sources" del os.environ["XDG_CONFIG_HOME"] ####################################### # Test failure modes # ####################################### @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_missing_config(context_fixture, datafiles): context = context_fixture["context"] assert isinstance(context, Context) conf_file = os.path.join(datafiles, "nonexistant.yaml") with pytest.raises(LoadError) as exc: context.load(conf_file) assert exc.value.reason == LoadErrorReason.MISSING_FILE @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_malformed_config(context_fixture, datafiles): context = context_fixture["context"] assert isinstance(context, Context) conf_file = os.path.join(datafiles, "malformed.yaml") with pytest.raises(LoadError) as exc: context.load(conf_file) assert exc.value.reason == LoadErrorReason.INVALID_YAML @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_context_load_notdict_config(context_fixture, datafiles): context = context_fixture["context"] assert isinstance(context, Context) conf_file = os.path.join(datafiles, "notdict.yaml") with pytest.raises(LoadError) as exc: context.load(conf_file) # XXX Should this be a different LoadErrorReason ? assert exc.value.reason == LoadErrorReason.INVALID_YAML apache-buildstream-27ae392/tests/internals/context/000077500000000000000000000000001514607367700224475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/context/malformed.yaml000066400000000000000000000000421514607367700252750ustar00rootroot00000000000000- | this is malformed yaml. ** apache-buildstream-27ae392/tests/internals/context/notdict.yaml000066400000000000000000000000241514607367700247730ustar00rootroot00000000000000This is not a dict. apache-buildstream-27ae392/tests/internals/context/userconf.yaml000066400000000000000000000000561514607367700251600ustar00rootroot00000000000000# Try overriding something sourcedir: ~/pony apache-buildstream-27ae392/tests/internals/loader.py000066400000000000000000000067341514607367700226150ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from contextlib import contextmanager import os import pytest from buildstream.exceptions import LoadErrorReason from buildstream._exceptions import LoadError from buildstream._project import Project from buildstream._loader import LoadElement from tests.testutils import dummy_context DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "loader", ) @contextmanager def make_loader(basedir): with dummy_context() as context: project = Project(basedir, context) yield project.loader ############################################################## # Basics: Test behavior loading the simplest of projects # ############################################################## @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_one_file(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader: element = loader.load(["elements/onefile.bst"])[0] assert isinstance(element, LoadElement) assert element.kind == "pony" @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_missing_file(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load(["elements/missing.bst"]) assert exc.value.reason == LoadErrorReason.MISSING_FILE @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_reference(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load(["elements/badreference.bst"]) assert exc.value.reason == LoadErrorReason.INVALID_YAML @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_yaml(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load(["elements/badfile.bst"]) assert exc.value.reason == LoadErrorReason.INVALID_YAML @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_fail_fullpath_target(datafiles): basedir = str(datafiles) fullpath = os.path.join(basedir, "elements", "onefile.bst") with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load([fullpath]) assert exc.value.reason == LoadErrorReason.INVALID_DATA @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_key(datafiles): basedir = str(datafiles) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load(["elements/invalidkey.bst"]) assert exc.value.reason == LoadErrorReason.INVALID_DATA @pytest.mark.datafiles(os.path.join(DATA_DIR, "onefile")) def test_invalid_directory_load(datafiles): basedir = str(datafiles) os.makedirs(os.path.join(basedir, "element.bst")) with make_loader(basedir) as loader, pytest.raises(LoadError) as exc: loader.load(["element.bst"]) assert exc.value.reason == LoadErrorReason.LOADING_DIRECTORY apache-buildstream-27ae392/tests/internals/loader/000077500000000000000000000000001514607367700222315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/loader/onefile/000077500000000000000000000000001514607367700236525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/loader/onefile/elements/000077500000000000000000000000001514607367700254665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/loader/onefile/elements/badfile.bst000066400000000000000000000000751514607367700275700ustar00rootroot00000000000000# This is just invalid YAML - | this is malformed yaml. ) apache-buildstream-27ae392/tests/internals/loader/onefile/elements/badreference.bst000066400000000000000000000001251514607367700306030ustar00rootroot00000000000000# This bad YAML file makes a reference to an undefined entity name: pony pony: *pony apache-buildstream-27ae392/tests/internals/loader/onefile/elements/invalidkey.bst000066400000000000000000000000651514607367700303400ustar00rootroot00000000000000kind: pony description: This is the pony wings: blue apache-buildstream-27ae392/tests/internals/loader/onefile/elements/invalidsourcekey.bst000066400000000000000000000001641514607367700315610ustar00rootroot00000000000000kind: pony description: This is the pony sources: - kind: ponyland url: ptp://pw.ponies.p/ weather: great apache-buildstream-27ae392/tests/internals/loader/onefile/elements/onefile.bst000066400000000000000000000000511514607367700276150ustar00rootroot00000000000000kind: pony description: This is the pony apache-buildstream-27ae392/tests/internals/loader/onefile/project.conf000066400000000000000000000000531514607367700261650ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/internals/storage.py000066400000000000000000000311521514607367700230030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from contextlib import contextmanager import os import pprint import shutil import glob import hashlib from pathlib import Path from typing import List, Optional import pytest from buildstream import DirectoryError, FileType from buildstream.storage._casbaseddirectory import CasBasedDirectory from buildstream.storage._filebaseddirectory import FileBasedDirectory from tests.testutils import casd_cache DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "storage") @contextmanager def setup_backend(backend_class, tmpdir): if backend_class == FileBasedDirectory: path = os.path.join(tmpdir, "vdir") os.mkdir(path) yield backend_class(path) else: with casd_cache(os.path.join(tmpdir, "cas")) as cas_cache: yield backend_class(cas_cache) @pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_import(tmpdir, datafiles, backend): original = os.path.join(str(datafiles), "original") with setup_backend(backend, str(tmpdir)) as c: c.import_files(original) assert "bin/bash" in c.list_relative_paths() assert "bin/hello" in c.list_relative_paths() @pytest.mark.parametrize( "directories", [ ("merge-base", "merge-base"), ("empty", "empty"), ], ) @pytest.mark.datafiles(DATA_DIR) def test_merge_same_casdirs(tmpdir, datafiles, directories): buildtree = os.path.join(str(datafiles), "merge-buildtree") before = os.path.join(str(datafiles), directories[0]) after = os.path.join(str(datafiles), directories[1]) # Bring the directories into a canonical state for directory in (buildtree, before, after): clear_gitkeeps(directory) utime_recursively(directory, (100, 100)) with setup_backend(CasBasedDirectory, str(tmpdir)) as c, setup_backend( CasBasedDirectory, str(tmpdir) ) as a, setup_backend(CasBasedDirectory, str(tmpdir)) as b: a.import_files(before) b.import_files(after) c.import_files(buildtree) assert a._get_digest() == b._get_digest(), "{}\n{}".format( pprint.pformat(list_relative_paths(a)), pprint.pformat(list_relative_paths(b)) ) old_digest = c._get_digest() c._apply_changes(a, b) # Assert that the build tree stays the same (since there were # no changes between a and b) assert c._get_digest() == old_digest @pytest.mark.parametrize( "directories", [ ("merge-base", "merge-replace"), ("merge-base", "merge-remove"), ("merge-base", "merge-add"), ("merge-base", "merge-link"), ("merge-base", "merge-subdirectory-replace"), ("merge-base", "merge-subdirectory-remove"), ("merge-base", "merge-subdirectory-add"), ("merge-base", "merge-subdirectory-link"), ("merge-link", "merge-link-change"), ("merge-subdirectory-link", "merge-link-change"), ("merge-base", "merge-override-with-file"), ("merge-base", "merge-override-with-directory"), ("merge-base", "merge-override-in-subdir-with-file"), ("merge-base", "merge-override-in-subdir-with-directory"), ("merge-base", "merge-override-subdirectory"), ("merge-override-with-new-subdirectory", "merge-subdirectory-add"), ("empty", "merge-subdirectory-add"), ], ) @pytest.mark.datafiles(DATA_DIR) def test_merge_casdirs(tmpdir, datafiles, directories): buildtree = os.path.join(str(datafiles), "merge-buildtree") before = os.path.join(str(datafiles), directories[0]) after = os.path.join(str(datafiles), directories[1]) # Bring the directories into a canonical state for directory in (buildtree, before, after): clear_gitkeeps(directory) utime_recursively(directory, (100, 100)) _test_merge_dirs(before, after, buildtree, str(tmpdir)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("modification", ["executable", "time"]) def test_merge_casdir_properties(tmpdir, datafiles, modification): buildtree = os.path.join(str(datafiles), "merge-buildtree") before = os.path.join(str(datafiles), "merge-base") after = os.path.join(str(tmpdir), "merge-modified") shutil.copytree(before, after, symlinks=True) # Bring the directories into a canonical state for directory in (buildtree, before, after): clear_gitkeeps(directory) utime_recursively(directory, (100, 100)) if modification == "executable": os.chmod(os.path.join(after, "root-file"), 0o755) elif modification == "time": os.utime(os.path.join(after, "root-file"), (200, 200)) _test_merge_dirs(before, after, buildtree, str(tmpdir), properties=["mtime"]) def _test_merge_dirs( before: str, after: str, buildtree: str, tmpdir: str, properties: Optional[List[str]] = None ) -> bool: with setup_backend(CasBasedDirectory, tmpdir) as c, setup_backend( CasBasedDirectory, tmpdir ) as copy, setup_backend(CasBasedDirectory, tmpdir) as a, setup_backend(CasBasedDirectory, tmpdir) as b: a._import_files_internal(before, properties=properties) b._import_files_internal(after, properties=properties) c._import_files_internal(buildtree, properties=properties) copy._import_files_internal(buildtree, properties=properties) assert c._get_digest() == copy._get_digest() assert a._get_digest() != b._get_digest(), "{}\n{}".format( pprint.pformat(list_relative_paths(a)), pprint.pformat(list_relative_paths(b)) ) c._apply_changes(a, b) # The files in c now should contain changes from b, so these # shouldn't be the same anymore assert c._get_digest() != copy._get_digest(), "{}\n{}".format( pprint.pformat(list_relative_paths(c)), pprint.pformat(list_relative_paths(copy)) ) # This is the set of paths that should have been removed removed = [path for path in list_paths_with_properties(a) if path not in list_paths_with_properties(b)] # This is the set of paths that were added in the new set added = [path for path in list_paths_with_properties(b) if path not in list_paths_with_properties(a)] # We need to strip some types of values, since they're more # than our little list comparisons can handle def make_info(entry, list_props=None): ret = {k: v for k, v in vars(entry).items() if k not in ("directory", "cas_cache")} if entry.type == FileType.REGULAR_FILE: # Only file digests make sense here (directory digests # need to be re-calculated taking into account their # contents). ret["digest"] = entry.get_digest() else: ret["digest"] = None return ret combined = [path for path in list_paths_with_properties(copy) if path not in removed] # Add the new list, overriding any old entries that already # exist. for path in added: if path.name in (o.name for o in combined): # Any paths that already exist must be removed # first combined = [o for o in combined if o.name != path.name] combined.append(path) else: combined.append(path) # If any paths don't have a parent directory, we need to # remove them now for e in combined: path = Path(e.name) for parent in list(path.parents)[:-1]: if not str(parent) in (e.name for e in combined if e.type == FileType.DIRECTORY): # If not all parent directories are existing # directories combined = [e for e in combined if e.name != str(path)] assert sorted(list(make_info(e) for e in combined), key=lambda x: x["name"]) == sorted( list(make_info(e) for e in list_paths_with_properties(c)), key=lambda x: x["name"] ) @pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_file_types(tmpdir, datafiles, backend): with setup_backend(backend, str(tmpdir)) as c: c.import_files(os.path.join(str(datafiles), "merge-link")) # Test __iter__ assert set(c) == {"link", "root-file", "subdirectory"} assert c.exists("root-file") assert c.isfile("root-file") assert not c.isdir("root-file") assert not c.islink("root-file") stat = c.stat("root-file") assert stat.file_type == FileType.REGULAR_FILE assert c.exists("link") assert c.islink("link") assert not c.isfile("link") assert c.readlink("link") == "root-file" stat = c.stat("link") assert stat.file_type == FileType.SYMLINK assert c.exists("subdirectory") assert c.isdir("subdirectory") assert not c.isfile("subdirectory") subdir = c.open_directory("subdirectory") assert set(subdir) == {"subdir-file"} stat = c.stat("subdirectory") assert stat.file_type == FileType.DIRECTORY @pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_open_file(tmpdir, datafiles, backend): with setup_backend(backend, str(tmpdir)) as c: assert not c.isfile("hello") with c.open_file("hello", mode="w") as f: f.write("world") assert c.isfile("hello") assert c.file_digest("hello") == hashlib.sha256(b"world").hexdigest() with c.open_file("hello", mode="r") as f: assert f.read() == "world" @pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_remove(tmpdir, datafiles, backend): with setup_backend(backend, str(tmpdir)) as c: c.import_files(os.path.join(str(datafiles), "merge-link")) with pytest.raises(DirectoryError): c.remove("subdirectory") with pytest.raises(DirectoryError): c.remove("subdirectory/does-not-exist") # Check that `remove()` doesn't follow symlinks c.remove("link") assert not c.exists("link") assert c.exists("root-file") c.remove("subdirectory", recursive=True) assert not c.exists("subdirectory") # Removing an empty directory does not require recursive=True c.open_directory("empty-directory", create=True) c.remove("empty-directory") @pytest.mark.parametrize("backend", [FileBasedDirectory, CasBasedDirectory]) @pytest.mark.datafiles(DATA_DIR) def test_rename(tmpdir, datafiles, backend): with setup_backend(backend, str(tmpdir)) as c: c.import_files(os.path.join(str(datafiles), "original")) c.rename("bin/hello", "bin/hello2") c.rename("bin", "bin2") assert c.isfile("bin2/hello2") # This is purely for error output; lists relative paths and # their digests so differences are human-grokkable def list_relative_paths(directory): def entry_output(entry): if entry.type == FileType.DIRECTORY: return list_relative_paths(entry.get_directory(directory)) elif entry.type == FileType.SYMLINK: return "-> " + entry.target else: return entry.get_digest().hash return {name: entry_output(entry) for name, entry in directory._CasBasedDirectory__index.items()} def list_paths_with_properties(directory, prefix=""): for leaf in directory._CasBasedDirectory__index.keys(): entry = directory._CasBasedDirectory__index[leaf].clone() if directory._CasBasedDirectory__filename: entry.name = directory._CasBasedDirectory__filename + os.path.sep + entry.name yield entry if entry.type == FileType.DIRECTORY: subdir = entry.get_directory(directory) yield from list_paths_with_properties(subdir) def utime_recursively(directory, time): for f in glob.glob(os.path.join(directory, "**"), recursive=True): os.utime(f, time) def clear_gitkeeps(directory): for f in glob.glob(os.path.join(directory, "**", ".gitkeep"), recursive=True): os.remove(f) apache-buildstream-27ae392/tests/internals/storage/000077500000000000000000000000001514607367700224275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/empty/000077500000000000000000000000001514607367700235655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/empty/.gitkeep000066400000000000000000000000001514607367700252040ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-add/000077500000000000000000000000001514607367700242545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-add/added000066400000000000000000000000001514607367700252260ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-add/root-file000066400000000000000000000000001514607367700260650ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-add/subdirectory/000077500000000000000000000000001514607367700267725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-add/subdirectory/subdir-file000066400000000000000000000000001514607367700311100ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-base/000077500000000000000000000000001514607367700244365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-base/root-file000066400000000000000000000000001514607367700262470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-base/subdirectory/000077500000000000000000000000001514607367700271545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-base/subdirectory/subdir-file000066400000000000000000000000001514607367700312720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-buildtree/000077500000000000000000000000001514607367700255035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-buildtree/root-file000066400000000000000000000000001514607367700273140ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-buildtree/root-file.o000066400000000000000000000000001514607367700275510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-buildtree/subdirectory/000077500000000000000000000000001514607367700302215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-buildtree/subdirectory/subdir-file000066400000000000000000000000001514607367700323370ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-buildtree/subdirectory/subdir-file.o000066400000000000000000000000001514607367700325740ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link-change/000077500000000000000000000000001514607367700257045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link-change/link000077700000000000000000000000001514607367700334252subdirectory/subdir-fileustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link-change/root-file000066400000000000000000000000001514607367700275150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link-change/subdirectory/000077500000000000000000000000001514607367700304225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link-change/subdirectory/link000077700000000000000000000000001514607367700333332../root-fileustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link-change/subdirectory/subdir-file000066400000000000000000000000001514607367700325400ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link/000077500000000000000000000000001514607367700244615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link/link000077700000000000000000000000001514607367700271572root-fileustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link/root-file000066400000000000000000000000001514607367700262720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link/subdirectory/000077500000000000000000000000001514607367700271775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-link/subdirectory/subdir-file000066400000000000000000000000001514607367700313150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directory/000077500000000000000000000000001514607367700322105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directory/root-file000066400000000000000000000000001514607367700340210ustar00rootroot00000000000000root-file.o000066400000000000000000000000001514607367700341770ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directorysubdirectory/000077500000000000000000000000001514607367700346475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directorysubdir-file000066400000000000000000000000001514607367700367650ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directory/subdirectorysubdir-file.o/000077500000000000000000000000001514607367700373115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directory/subdirectorytest000066400000000000000000000000001514607367700402010ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-directory/subdirectory/subdir-file.oapache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-file/000077500000000000000000000000001514607367700311235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-file/root-file000066400000000000000000000000001514607367700327340ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-file/root-file.o000066400000000000000000000000051514607367700331760ustar00rootroot00000000000000test apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-file/subdirectory/000077500000000000000000000000001514607367700336415ustar00rootroot00000000000000subdir-file000066400000000000000000000000001514607367700357000ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-file/subdirectorysubdir-file.o000066400000000000000000000000001514607367700361350ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-in-subdir-with-file/subdirectoryapache-buildstream-27ae392/tests/internals/storage/merge-override-subdirectory/000077500000000000000000000000001514607367700300575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-subdirectory/root-file000066400000000000000000000000001514607367700316700ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-subdirectory/root-file.o000066400000000000000000000000001514607367700321250ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-subdirectory/subdirectory000066400000000000000000000000001514607367700325060ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/000077500000000000000000000000001514607367700303165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/root-file000066400000000000000000000000001514607367700321270ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/root-file.o/000077500000000000000000000000001514607367700324535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/root-file.o/test000066400000000000000000000000001514607367700333430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/subdirectory/000077500000000000000000000000001514607367700330345ustar00rootroot00000000000000subdir-file000066400000000000000000000000001514607367700350730ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/subdirectorysubdir-file.o000066400000000000000000000000001514607367700353300ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-directory/subdirectoryapache-buildstream-27ae392/tests/internals/storage/merge-override-with-file/000077500000000000000000000000001514607367700272315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-file/root-file000066400000000000000000000000001514607367700310420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-file/root-file.o000066400000000000000000000000051514607367700313040ustar00rootroot00000000000000test apache-buildstream-27ae392/tests/internals/storage/merge-override-with-file/subdirectory/000077500000000000000000000000001514607367700317475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-file/subdirectory/subdir-file000066400000000000000000000000001514607367700340650ustar00rootroot00000000000000subdir-file.o000066400000000000000000000000001514607367700342430ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-file/subdirectoryapache-buildstream-27ae392/tests/internals/storage/merge-override-with-new-subdirectory/000077500000000000000000000000001514607367700316175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-new-subdirectory/root-file000066400000000000000000000000001514607367700334300ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-override-with-new-subdirectory/subdirectory000066400000000000000000000000001514607367700342460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-properties/000077500000000000000000000000001514607367700257205ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-properties/root-file000077500000000000000000000000001514607367700275340ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-properties/subdirectory/000077500000000000000000000000001514607367700304365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-properties/subdirectory/subdir-file000066400000000000000000000000001514607367700325540ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-remove/000077500000000000000000000000001514607367700250215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-remove/subdirectory/000077500000000000000000000000001514607367700275375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-remove/subdirectory/subdir-file000066400000000000000000000000001514607367700316550ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-replace/000077500000000000000000000000001514607367700251375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-replace/root-file000066400000000000000000000000021514607367700267520ustar00rootroot00000000000000b apache-buildstream-27ae392/tests/internals/storage/merge-replace/subdirectory/000077500000000000000000000000001514607367700276555ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-replace/subdirectory/subdir-file000066400000000000000000000000001514607367700317730ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-add/000077500000000000000000000000001514607367700267705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-add/root-file000066400000000000000000000000001514607367700306010ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-add/subdirectory/000077500000000000000000000000001514607367700315065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-add/subdirectory/added000066400000000000000000000000001514607367700324600ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-add/subdirectory/subdir-file000066400000000000000000000000021514607367700336260ustar00rootroot00000000000000b apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-link/000077500000000000000000000000001514607367700271755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-link/root-file000066400000000000000000000000001514607367700310060ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-link/subdirectory/000077500000000000000000000000001514607367700317135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-link/subdirectory/link000077700000000000000000000000001514607367700347162subdir-fileustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-link/subdirectory/subdir-file000066400000000000000000000000001514607367700340310ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-remove/000077500000000000000000000000001514607367700275355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-remove/root-file000066400000000000000000000000001514607367700313460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-remove/subdirectory/000077500000000000000000000000001514607367700322535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-remove/subdirectory/.gitkeep000066400000000000000000000000001514607367700336720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-replace/000077500000000000000000000000001514607367700276535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-replace/root-file000066400000000000000000000000001514607367700314640ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-replace/subdirectory/000077500000000000000000000000001514607367700323715ustar00rootroot00000000000000subdir-file000066400000000000000000000000021514607367700344320ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/merge-subdirectory-replace/subdirectoryb apache-buildstream-27ae392/tests/internals/storage/original/000077500000000000000000000000001514607367700242335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/original/bin/000077500000000000000000000000001514607367700250035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/original/bin/bash000066400000000000000000000000401514607367700256350ustar00rootroot00000000000000This is the original /bin/bash. apache-buildstream-27ae392/tests/internals/storage/original/bin/hello000066400000000000000000000000411514607367700260240ustar00rootroot00000000000000This is the original /bin/hello. apache-buildstream-27ae392/tests/internals/storage/overlay/000077500000000000000000000000001514607367700241105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/overlay/bin/000077500000000000000000000000001514607367700246605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/storage/overlay/bin/bash000066400000000000000000000000431514607367700255150ustar00rootroot00000000000000This is the replacement /bin/bash. apache-buildstream-27ae392/tests/internals/storage_vdir_import.py000066400000000000000000000367741514607367700254400ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from hashlib import sha256 import os import random import pytest from buildstream import DirectoryError from buildstream.storage._casbaseddirectory import CasBasedDirectory from buildstream.storage._filebaseddirectory import FileBasedDirectory from buildstream.utils import _set_file_mtime from buildstream._testing._utils.site import have_subsecond_mtime from tests.testutils import casd_cache # These are comparitive tests that check that FileBasedDirectory and # CasBasedDirectory act identically. # This is a set of example file system contents. It's a set of trees # which are either expected to be problematic or were found to be # problematic during random testing. # The test attempts to import each on top of each other to test # importing works consistently. Each tuple is defined as (, # , ). Type can be 'F' (file), 'S' (symlink) or 'D' # (directory) with content being the contents for a file or the # destination for a symlink. root_filesets = [ [("a/b/c/textfile1", "F", "This is textfile 1\n")], [("a/b/c/textfile1", "F", "This is the replacement textfile 1\n")], [("a/b/f", "S", "/a/b/c")], [("a/b/c", "D", ""), ("a/b/f", "S", "/a/b/c")], [("a/b/f", "F", "This is textfile 1\n")], ] empty_hash_ref = sha256().hexdigest() RANDOM_SEED = 69105 NUM_RANDOM_TESTS = 4 MTIME = 1576486144.0120000 def generate_import_roots(rootno, directory): rootname = "root{}".format(rootno) rootdir = os.path.join(directory, "content", rootname) generate_import_root(rootdir, root_filesets[rootno - 1]) def generate_import_root(rootdir, filelist): if os.path.exists(rootdir): return for (path, typesymbol, content) in filelist: if typesymbol == "F": (dirnames, filename) = os.path.split(path) os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True) fullpath = os.path.join(rootdir, dirnames, filename) with open(fullpath, "wt", encoding="utf-8") as f: f.write(content) # set file mtime to arbitrary _set_file_mtime(fullpath, MTIME) elif typesymbol == "D": os.makedirs(os.path.join(rootdir, path), exist_ok=True) elif typesymbol == "S": (dirnames, filename) = os.path.split(path) os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True) os.symlink(content, os.path.join(rootdir, path)) # Set deterministic mtime for all directories for (dirpath, _, _) in os.walk(rootdir): _set_file_mtime(dirpath, MTIME) def generate_random_root(rootno, directory): # By seeding the random number generator, we ensure these tests # will be repeatable, at least until Python changes the random # number algorithm. random.seed(RANDOM_SEED + rootno) rootname = "root{}".format(rootno) rootdir = os.path.join(directory, "content", rootname) if os.path.exists(rootdir): return things = [] locations = ["."] os.makedirs(rootdir) for i in range(0, 100): location = random.choice(locations) thingname = "node{}".format(i) thing = random.choice(["dir", "link", "file"]) if thing == "dir": thingname = "dir" + thingname target = os.path.join(rootdir, location, thingname) if thing == "dir": os.makedirs(target) locations.append(os.path.join(location, thingname)) elif thing == "file": with open(target, "wt", encoding="utf-8") as f: f.write("This is node {}\n".format(i)) _set_file_mtime(target, MTIME) elif thing == "link": symlink_type = random.choice(["absolute", "relative", "broken"]) if symlink_type == "broken" or not things: os.symlink("/broken", target) elif symlink_type == "absolute": symlink_destination = random.choice(things) os.symlink(symlink_destination, target) else: symlink_destination = random.choice(things) relative_link = os.path.relpath(symlink_destination, start=location) os.symlink(relative_link, target) things.append(os.path.join(location, thingname)) # Set deterministic mtime for all directories for (dirpath, _, _) in os.walk(rootdir): _set_file_mtime(dirpath, MTIME) def file_contents(path): with open(path, "r", encoding="utf-8") as f: result = f.read() return result def file_contents_are(path, contents): return file_contents(path) == contents def create_new_casdir(root_number, cas_cache, tmpdir): d = CasBasedDirectory(cas_cache) d._import_files_internal(os.path.join(tmpdir, "content", "root{}".format(root_number)), properties=["mtime"]) digest = d._get_digest() assert digest.hash != empty_hash_ref return d def create_new_filedir(root_number, tmpdir): root = os.path.join(tmpdir, "vdir") os.makedirs(root) d = FileBasedDirectory(root) d._import_files_internal(os.path.join(tmpdir, "content", "root{}".format(root_number))) return d def combinations(integer_range): for x in integer_range: for y in integer_range: yield (x, y) def resolve_symlinks(path, root): """A function to resolve symlinks inside 'path' components apart from the last one. For example, resolve_symlinks('/a/b/c/d', '/a/b') will return '/a/b/f/d' if /a/b/c is a symlink to /a/b/f. The final component of 'path' is not resolved, because we typically want to inspect the symlink found at that path, not its target. """ components = path.split(os.path.sep) location = root for i in range(0, len(components) - 1): location = os.path.join(location, components[i]) if os.path.islink(location): # Resolve the link, add on all the remaining components target = os.path.join(os.readlink(location)) tail = os.path.sep.join(components[i + 1 :]) if target.startswith(os.path.sep): # Absolute link - relative to root location = os.path.join(root, target, tail) else: # Relative link - relative to symlink location location = os.path.join(location, target) return resolve_symlinks(location, root) # If we got here, no symlinks were found. Add on the final component and return. location = os.path.join(location, components[-1]) return location def directory_not_empty(path): return os.listdir(path) def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False): # Skip this test if we do not have support for subsecond precision mtimes # if not have_subsecond_mtime(str(tmpdir)): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(str(tmpdir))) with casd_cache(os.path.join(tmpdir, "casd")) as cas_cache: # Create some fake content generator_function(original, tmpdir) if original != overlay: generator_function(overlay, tmpdir) d = create_new_casdir(original, cas_cache, tmpdir) duplicate_cas = create_new_casdir(original, cas_cache, tmpdir) assert duplicate_cas._get_digest().hash == d._get_digest().hash d2 = create_new_casdir(overlay, cas_cache, tmpdir) d._import_files_internal(d2, properties=["mtime"]) export_dir = os.path.join(tmpdir, "output-{}-{}".format(original, overlay)) roundtrip_dir = os.path.join(tmpdir, "roundtrip-{}-{}".format(original, overlay)) d2._export_files(roundtrip_dir) d._export_files(export_dir) if verify_contents: for item in root_filesets[overlay - 1]: (path, typename, content) = item realpath = resolve_symlinks(path, export_dir) if typename == "F": if os.path.isdir(realpath) and directory_not_empty(realpath): # The file should not have overwritten the directory in this case. pass else: assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format( path ) assert file_contents_are(realpath, content) roundtrip = os.path.join(roundtrip_dir, path) assert os.path.getmtime(roundtrip) == MTIME assert os.path.getmtime(realpath) == MTIME elif typename == "S": if os.path.isdir(realpath) and directory_not_empty(realpath): # The symlink should not have overwritten the directory in this case. pass else: assert os.path.islink(realpath) assert os.readlink(realpath) == content elif typename == "D": # We can't do any more tests than this because it # depends on things present in the original. Blank # directories here will be ignored and the original # left in place. assert os.path.lexists(realpath) # Now do the same thing with filebaseddirectories and check the contents match duplicate_cas._import_files_internal(roundtrip_dir, properties=["mtime"]) assert duplicate_cas._get_digest().hash == d._get_digest().hash @pytest.mark.parametrize("original", range(1, len(root_filesets) + 1)) @pytest.mark.parametrize("overlay", range(1, len(root_filesets) + 1)) def test_fixed_cas_import(tmpdir, original, overlay): _import_test(str(tmpdir), original, overlay, generate_import_roots, verify_contents=True) @pytest.mark.parametrize("original", range(1, NUM_RANDOM_TESTS + 1)) @pytest.mark.parametrize("overlay", range(1, NUM_RANDOM_TESTS + 1)) def test_random_cas_import(tmpdir, original, overlay): _import_test(str(tmpdir), original, overlay, generate_random_root, verify_contents=False) def _listing_test(tmpdir, root, generator_function): with casd_cache(os.path.join(tmpdir, "casd")) as cas_cache: # Create some fake content generator_function(root, tmpdir) d = create_new_filedir(root, tmpdir) filelist = list(d.list_relative_paths()) d2 = create_new_casdir(root, cas_cache, tmpdir) filelist2 = list(d2.list_relative_paths()) assert filelist == filelist2 @pytest.mark.parametrize("root", range(1, NUM_RANDOM_TESTS + 1)) def test_random_directory_listing(tmpdir, root): _listing_test(str(tmpdir), root, generate_random_root) @pytest.mark.parametrize("root", range(1, len(root_filesets) + 1)) def test_fixed_directory_listing(tmpdir, root): _listing_test(str(tmpdir), root, generate_import_roots) # Check that the vdir is decending and readable def test_open_directory(tmpdir): cas_dir = os.path.join(str(tmpdir), "cas") with casd_cache(cas_dir) as cas_cache: d = CasBasedDirectory(cas_cache) Content_to_check = "You got me" test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [("a", "D", ""), ("a/l", "D", ""), ("a/l/g", "F", Content_to_check)] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) digest = d.open_directory("a/l")._CasBasedDirectory__index["g"].get_digest() with open(cas_cache.objpath(digest), encoding="utf-8") as fp: content = fp.read() assert Content_to_check == content # Check symlink logic for edgecases # Make sure the correct erros are raised when trying # to decend in to files or links to files def test_bad_symlinks(tmpdir): cas_dir = os.path.join(str(tmpdir), "cas") with casd_cache(cas_dir) as cas_cache: d = CasBasedDirectory(cas_cache) test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [("a", "D", ""), ("a/l", "S", "../target"), ("target", "F", "You got me")] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) exp_reason = "not-a-directory" with pytest.raises(DirectoryError) as error: d.open_directory("a/l", follow_symlinks=True) assert error.reason == exp_reason with pytest.raises(DirectoryError) as error: d.open_directory("a/l") assert error.reason == exp_reason with pytest.raises(DirectoryError) as error: d.open_directory("a/f") assert error.reason == exp_reason # Check symlink logic for edgecases # Check decend accross relitive link def test_relative_symlink(tmpdir): cas_dir = os.path.join(str(tmpdir), "cas") with casd_cache(cas_dir) as cas_cache: d = CasBasedDirectory(cas_cache) Content_to_check = "You got me" test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [ ("a", "D", ""), ("a/l", "S", "../target"), ("target", "D", ""), ("target/file", "F", Content_to_check), ] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) digest = d.open_directory("a/l", follow_symlinks=True)._CasBasedDirectory__index["file"].get_digest() with open(cas_cache.objpath(digest), encoding="utf-8") as fp: content = fp.read() assert Content_to_check == content # Check symlink logic for edgecases # Check deccend accross abs link def test_abs_symlink(tmpdir): cas_dir = os.path.join(str(tmpdir), "cas") with casd_cache(cas_dir) as cas_cache: d = CasBasedDirectory(cas_cache) Content_to_check = "two step file" test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [ ("a", "D", ""), ("a/l", "S", "/target"), ("target", "D", ""), ("target/file", "F", Content_to_check), ] generate_import_root(test_dir, filesys_discription) d.import_files(test_dir) digest = d.open_directory("a/l", follow_symlinks=True)._CasBasedDirectory__index["file"].get_digest() with open(cas_cache.objpath(digest), encoding="utf-8") as fp: content = fp.read() assert Content_to_check == content # Check symlink logic for edgecases # Check symlink can not escape root def test_bad_sym_escape(tmpdir): cas_dir = os.path.join(str(tmpdir), "cas") with casd_cache(cas_dir) as cas_cache: d = CasBasedDirectory(cas_cache) test_dir = os.path.join(str(tmpdir), "importfrom") filesys_discription = [ ("jail", "D", ""), ("jail/a", "D", ""), ("jail/a/l", "S", "../../target"), ("target", "D", ""), ("target/file", "F", "two step file"), ] generate_import_root(test_dir, filesys_discription) d.import_files(os.path.join(test_dir, "jail")) with pytest.raises(DirectoryError) as error: d.open_directory("a/l", follow_symlinks=True) assert error.reason == "directory-not-found" apache-buildstream-27ae392/tests/internals/utils_move_atomic.py000066400000000000000000000063471514607367700250710ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name from os.path import getmtime import pytest from buildstream.utils import ( move_atomic, DirectoryExistsError, _set_file_mtime, ) from buildstream._testing._utils.site import have_subsecond_mtime @pytest.fixture def src(tmp_path): src = tmp_path.joinpath("src") src.mkdir() with src.joinpath("test").open("w") as fp: fp.write("test") return src def test_move_to_empty_dir(src, tmp_path): dst = tmp_path.joinpath("dst") move_atomic(src, dst) assert dst.joinpath("test").exists() def test_move_to_empty_dir_create_parents(src, tmp_path): dst = tmp_path.joinpath("nested/dst") move_atomic(src, dst) assert dst.joinpath("test").exists() def test_move_to_empty_dir_no_create_parents(src, tmp_path): dst = tmp_path.joinpath("nested/dst") with pytest.raises(FileNotFoundError): move_atomic(src, dst, ensure_parents=False) def test_move_non_existing_dir(tmp_path): dst = tmp_path.joinpath("dst") src = tmp_path.joinpath("src") with pytest.raises(FileNotFoundError): move_atomic(src, dst) def test_move_to_existing_empty_dir(src, tmp_path): dst = tmp_path.joinpath("dst") dst.mkdir() move_atomic(src, dst) assert dst.joinpath("test").exists() def test_move_to_existing_file(src, tmp_path): dst = tmp_path.joinpath("dst") with dst.open("w") as fp: fp.write("error") with pytest.raises(NotADirectoryError): move_atomic(src, dst) def test_move_file_to_existing_file(tmp_path): dst = tmp_path.joinpath("dst") src = tmp_path.joinpath("src") with src.open("w") as fp: fp.write("src") with dst.open("w") as fp: fp.write("dst") move_atomic(src, dst) with dst.open() as fp: assert fp.read() == "src" def test_move_to_existing_non_empty_dir(src, tmp_path): dst = tmp_path.joinpath("dst") dst.mkdir() with dst.joinpath("existing").open("w") as fp: fp.write("already there") with pytest.raises(DirectoryExistsError): move_atomic(src, dst) def test_move_to_empty_dir_set_mtime(src, tmp_path): # Skip this test if we do not have support for subsecond precision mtimes # if not have_subsecond_mtime(str(tmp_path)): pytest.skip("Filesystem does not support subsecond mtime precision: {}".format(str(tmp_path))) dst = tmp_path.joinpath("dst") move_atomic(src, dst) assert dst.joinpath("test").exists() _dst = str(dst) # set the mtime via stamp timestamp1 = 1578481550.832123 _set_file_mtime(_dst, timestamp1) assert timestamp1 == getmtime(_dst) apache-buildstream-27ae392/tests/internals/utils_save_atomic.py000066400000000000000000000047171514607367700250600ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import pytest from buildstream.utils import save_file_atomic def test_save_new_file(tmpdir): filename = os.path.join(str(tmpdir), "savefile-success.test") with save_file_atomic(filename, "w") as f: f.write("foo\n") assert os.listdir(str(tmpdir)) == ["savefile-success.test"] with open(filename, encoding="utf-8") as f: assert f.read() == "foo\n" def test_save_over_existing_file(tmpdir): filename = os.path.join(str(tmpdir), "savefile-overwrite.test") with open(filename, "w", encoding="utf-8") as f: f.write("existing contents\n") with save_file_atomic(filename, "w") as f: f.write("overwritten contents\n") assert os.listdir(str(tmpdir)) == ["savefile-overwrite.test"] with open(filename, encoding="utf-8") as f: assert f.read() == "overwritten contents\n" def test_exception_new_file(tmpdir): filename = os.path.join(str(tmpdir), "savefile-exception.test") with pytest.raises(RuntimeError): with save_file_atomic(filename, "w") as f: f.write("Some junk\n") raise RuntimeError("Something goes wrong") assert os.listdir(str(tmpdir)) == [] def test_exception_existing_file(tmpdir): filename = os.path.join(str(tmpdir), "savefile-existing.test") with open(filename, "w", encoding="utf-8") as f: f.write("existing contents\n") with pytest.raises(RuntimeError): with save_file_atomic(filename, "w") as f: f.write("Some junk\n") raise RuntimeError("Something goes wrong") assert os.listdir(str(tmpdir)) == ["savefile-existing.test"] with open(filename, encoding="utf-8") as f: assert f.read() == "existing contents\n" def test_attributes(tmpdir): filename = os.path.join(str(tmpdir), "savefile-attributes.test") with save_file_atomic(filename, "w") as f: assert f.real_filename == filename assert f.name != filename apache-buildstream-27ae392/tests/internals/yaml.py000066400000000000000000000523501514607367700223040ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from io import StringIO import pytest from buildstream import _yaml, Node, ProvenanceInformation, SequenceNode from buildstream.exceptions import LoadErrorReason from buildstream._exceptions import LoadError DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "yaml", ) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_load_yaml(datafiles): filename = os.path.join(datafiles, "basics.yaml") loaded = _yaml.load(filename, shortname=None) assert loaded.get_str("kind") == "pony" def assert_provenance(filename, line, col, node): provenance = node.get_provenance() assert isinstance(provenance, ProvenanceInformation) assert provenance._shortname == filename assert provenance._line == line assert provenance._col == col @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_basic_provenance(datafiles): filename = os.path.join(datafiles, "basics.yaml") loaded = _yaml.load(filename, shortname=None) assert loaded.get_str("kind") == "pony" assert_provenance(filename, 1, 0, loaded) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_member_provenance(datafiles): filename = os.path.join(datafiles, "basics.yaml") loaded = _yaml.load(filename, shortname=None) assert loaded.get_str("kind") == "pony" assert_provenance(filename, 2, 13, loaded.get_scalar("description")) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_element_provenance(datafiles): filename = os.path.join(datafiles, "basics.yaml") loaded = _yaml.load(filename, shortname=None) assert loaded.get_str("kind") == "pony" assert_provenance(filename, 5, 2, loaded.get_sequence("moods").scalar_at(1)) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_mapping_validate_keys(datafiles): valid = os.path.join(datafiles, "basics.yaml") invalid = os.path.join(datafiles, "invalid.yaml") base = _yaml.load(valid, shortname=None) base.validate_keys(["kind", "description", "moods", "children", "extra"]) base = _yaml.load(invalid, shortname=None) with pytest.raises(LoadError) as exc: base.validate_keys(["kind", "description", "moods", "children", "extra"]) assert exc.value.reason == LoadErrorReason.INVALID_DATA @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_get(datafiles): filename = os.path.join(datafiles, "basics.yaml") base = _yaml.load(filename, shortname=None) assert base.get_str("kind") == "pony" children = base.get_sequence("children") assert isinstance(children, SequenceNode) assert len(children) == 7 child = base.get_sequence("children").mapping_at(6) assert_provenance(filename, 20, 8, child.get_scalar("mood")) extra = base.get_mapping("extra") with pytest.raises(LoadError) as exc: extra.get_mapping("old") assert exc.value.reason == LoadErrorReason.INVALID_DATA @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_set(datafiles): filename = os.path.join(datafiles, "basics.yaml") base = _yaml.load(filename, shortname=None) assert "mother" not in base base["mother"] = "snow white" assert base.get_str("mother") == "snow white" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_set_overwrite(datafiles): filename = os.path.join(datafiles, "basics.yaml") base = _yaml.load(filename, shortname=None) # Overwrite a string assert base.get_str("kind") == "pony" base["kind"] = "cow" assert base.get_str("kind") == "cow" # Overwrite a list as a string assert base.get_str_list("moods") == ["happy", "sad"] base["moods"] = "unemotional" assert base.get_str("moods") == "unemotional" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_set_list_element(datafiles): filename = os.path.join(datafiles, "basics.yaml") base = _yaml.load(filename, shortname=None) assert base.get_str_list("moods") == ["happy", "sad"] base.get_sequence("moods")[0] = "confused" assert base.get_str_list("moods") == ["confused", "sad"] # Really this is testing _yaml.node_copy(), we want to # be sure that compositing values still preserves the original # values in the copied dict. # @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_composite_preserve_originals(datafiles): filename = os.path.join(datafiles, "basics.yaml") overlayfile = os.path.join(datafiles, "composite.yaml") base = _yaml.load(filename, shortname=None) overlay = _yaml.load(overlayfile, shortname=None) base_copy = base.clone() overlay._composite(base_copy) copy_extra = base_copy.get_mapping("extra") orig_extra = base.get_mapping("extra") # Test that the node copy has the overridden value... assert copy_extra.get_str("old") == "override" # But the original node is not effected by the override. assert orig_extra.get_str("old") == "new" # Tests for list composition # # Each test composits a filename on top of basics.yaml, and tests # the toplevel children list at the specified index # # Parameters: # filename: The file to composite on top of basics.yaml # index: The index in the children list # length: The expected length of the children list # mood: The expected value of the mood attribute of the dictionary found at index in children # prov_file: The expected provenance filename of "mood" # prov_line: The expected provenance line of "mood" # prov_col: The expected provenance column of "mood" # @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize( "filename,index,length,mood,prov_file,prov_line,prov_col", [ # Test results of compositing with the (<) prepend directive ("listprepend.yaml", 0, 9, "prepended1", "listprepend.yaml", 5, 10), ("listprepend.yaml", 1, 9, "prepended2", "listprepend.yaml", 7, 10), ("listprepend.yaml", 2, 9, "silly", "basics.yaml", 8, 8), ("listprepend.yaml", 8, 9, "sleepy", "basics.yaml", 20, 8), # Test results of compositing with the (>) append directive ("listappend.yaml", 7, 9, "appended1", "listappend.yaml", 5, 10), ("listappend.yaml", 8, 9, "appended2", "listappend.yaml", 7, 10), ("listappend.yaml", 0, 9, "silly", "basics.yaml", 8, 8), ("listappend.yaml", 6, 9, "sleepy", "basics.yaml", 20, 8), # Test results of compositing with both (<) and (>) directives ("listappendprepend.yaml", 0, 11, "prepended1", "listappendprepend.yaml", 5, 10), ("listappendprepend.yaml", 1, 11, "prepended2", "listappendprepend.yaml", 7, 10), ("listappendprepend.yaml", 2, 11, "silly", "basics.yaml", 8, 8), ("listappendprepend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8), ("listappendprepend.yaml", 9, 11, "appended1", "listappendprepend.yaml", 10, 10), ("listappendprepend.yaml", 10, 11, "appended2", "listappendprepend.yaml", 12, 10), # Test results of compositing with the (=) overwrite directive ("listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10), ("listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10), # Test results of compositing without any directive, implicitly overwriting ("implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8), ("implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8), ], ) def test_list_composition(datafiles, filename, tmpdir, index, length, mood, prov_file, prov_line, prov_col): base_file = os.path.join(datafiles, "basics.yaml") overlay_file = os.path.join(datafiles, filename) base = _yaml.load(base_file, shortname="basics.yaml") overlay = _yaml.load(overlay_file, shortname=filename) overlay._composite(base) children = base.get_sequence("children") assert len(children) == length child = children.mapping_at(index) assert child.get_str("mood") == mood assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood")) # Test that overwriting a list with an empty list works as expected. @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_list_deletion(datafiles): base = os.path.join(datafiles, "basics.yaml") overlay = os.path.join(datafiles, "listoverwriteempty.yaml") base = _yaml.load(base, shortname="basics.yaml") overlay = _yaml.load(overlay, shortname="listoverwriteempty.yaml") overlay._composite(base) children = base.get_sequence("children") assert not children # Tests for deep list composition # # Same as test_list_composition(), but adds an additional file # in between so that lists are composited twice. # # This test will to two iterations for each parameter # specification, expecting the same results # # First iteration: # composited = basics.yaml & filename1 # composited = composited & filename2 # # Second iteration: # composited = filename1 & filename2 # composited = basics.yaml & composited # # Parameters: # filename1: The file to composite on top of basics.yaml # filename2: The file to composite on top of filename1 # index: The index in the children list # length: The expected length of the children list # mood: The expected value of the mood attribute of the dictionary found at index in children # prov_file: The expected provenance filename of "mood" # prov_line: The expected provenance line of "mood" # prov_col: The expected provenance column of "mood" # @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize( "filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [ # Test results of compositing literal list with (>) and then (<) ("listprepend.yaml", "listappend.yaml", 0, 11, "prepended1", "listprepend.yaml", 5, 10), ("listprepend.yaml", "listappend.yaml", 1, 11, "prepended2", "listprepend.yaml", 7, 10), ("listprepend.yaml", "listappend.yaml", 2, 11, "silly", "basics.yaml", 8, 8), ("listprepend.yaml", "listappend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8), ("listprepend.yaml", "listappend.yaml", 9, 11, "appended1", "listappend.yaml", 5, 10), ("listprepend.yaml", "listappend.yaml", 10, 11, "appended2", "listappend.yaml", 7, 10), # Test results of compositing literal list with (<) and then (>) ("listappend.yaml", "listprepend.yaml", 0, 11, "prepended1", "listprepend.yaml", 5, 10), ("listappend.yaml", "listprepend.yaml", 1, 11, "prepended2", "listprepend.yaml", 7, 10), ("listappend.yaml", "listprepend.yaml", 2, 11, "silly", "basics.yaml", 8, 8), ("listappend.yaml", "listprepend.yaml", 8, 11, "sleepy", "basics.yaml", 20, 8), ("listappend.yaml", "listprepend.yaml", 9, 11, "appended1", "listappend.yaml", 5, 10), ("listappend.yaml", "listprepend.yaml", 10, 11, "appended2", "listappend.yaml", 7, 10), # Test results of compositing literal list with (>) and then (>) ("listappend.yaml", "secondappend.yaml", 0, 11, "silly", "basics.yaml", 8, 8), ("listappend.yaml", "secondappend.yaml", 6, 11, "sleepy", "basics.yaml", 20, 8), ("listappend.yaml", "secondappend.yaml", 7, 11, "appended1", "listappend.yaml", 5, 10), ("listappend.yaml", "secondappend.yaml", 8, 11, "appended2", "listappend.yaml", 7, 10), ("listappend.yaml", "secondappend.yaml", 9, 11, "secondappend1", "secondappend.yaml", 5, 10), ("listappend.yaml", "secondappend.yaml", 10, 11, "secondappend2", "secondappend.yaml", 7, 10), # Test results of compositing literal list with (>) and then (>) ("listprepend.yaml", "secondprepend.yaml", 0, 11, "secondprepend1", "secondprepend.yaml", 5, 10), ("listprepend.yaml", "secondprepend.yaml", 1, 11, "secondprepend2", "secondprepend.yaml", 7, 10), ("listprepend.yaml", "secondprepend.yaml", 2, 11, "prepended1", "listprepend.yaml", 5, 10), ("listprepend.yaml", "secondprepend.yaml", 3, 11, "prepended2", "listprepend.yaml", 7, 10), ("listprepend.yaml", "secondprepend.yaml", 4, 11, "silly", "basics.yaml", 8, 8), ("listprepend.yaml", "secondprepend.yaml", 10, 11, "sleepy", "basics.yaml", 20, 8), # Test results of compositing literal list with (>) or (<) and then another literal list ("listappend.yaml", "implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8), ("listappend.yaml", "implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8), ("listprepend.yaml", "implicitoverwrite.yaml", 0, 2, "overwrite1", "implicitoverwrite.yaml", 4, 8), ("listprepend.yaml", "implicitoverwrite.yaml", 1, 2, "overwrite2", "implicitoverwrite.yaml", 6, 8), # Test results of compositing literal list with (>) or (<) and then an explicit (=) overwrite ("listappend.yaml", "listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10), ("listappend.yaml", "listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10), ("listprepend.yaml", "listoverwrite.yaml", 0, 2, "overwrite1", "listoverwrite.yaml", 5, 10), ("listprepend.yaml", "listoverwrite.yaml", 1, 2, "overwrite2", "listoverwrite.yaml", 7, 10), # Test results of compositing literal list an explicit overwrite (=) and then with (>) or (<) ("listoverwrite.yaml", "listappend.yaml", 0, 4, "overwrite1", "listoverwrite.yaml", 5, 10), ("listoverwrite.yaml", "listappend.yaml", 1, 4, "overwrite2", "listoverwrite.yaml", 7, 10), ("listoverwrite.yaml", "listappend.yaml", 2, 4, "appended1", "listappend.yaml", 5, 10), ("listoverwrite.yaml", "listappend.yaml", 3, 4, "appended2", "listappend.yaml", 7, 10), ("listoverwrite.yaml", "listprepend.yaml", 0, 4, "prepended1", "listprepend.yaml", 5, 10), ("listoverwrite.yaml", "listprepend.yaml", 1, 4, "prepended2", "listprepend.yaml", 7, 10), ("listoverwrite.yaml", "listprepend.yaml", 2, 4, "overwrite1", "listoverwrite.yaml", 5, 10), ("listoverwrite.yaml", "listprepend.yaml", 3, 4, "overwrite2", "listoverwrite.yaml", 7, 10), ], ) def test_list_composition_twice( datafiles, tmpdir, filename1, filename2, index, length, mood, prov_file, prov_line, prov_col ): file_base = os.path.join(datafiles, "basics.yaml") file1 = os.path.join(datafiles, filename1) file2 = os.path.join(datafiles, filename2) ##################### # Round 1 - Fight ! ##################### base = _yaml.load(file_base, shortname="basics.yaml") overlay1 = _yaml.load(file1, shortname=filename1) overlay2 = _yaml.load(file2, shortname=filename2) overlay1._composite(base) overlay2._composite(base) children = base.get_sequence("children") assert len(children) == length child = children.mapping_at(index) assert child.get_str("mood") == mood assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood")) ##################### # Round 2 - Fight ! ##################### base = _yaml.load(file_base, shortname="basics.yaml") overlay1 = _yaml.load(file1, shortname=filename1) overlay2 = _yaml.load(file2, shortname=filename2) overlay2._composite(overlay1) overlay1._composite(base) children = base.get_sequence("children") assert len(children) == length child = children.mapping_at(index) assert child.get_str("mood") == mood assert_provenance(prov_file, prov_line, prov_col, child.get_node("mood")) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_convert_value_to_string(datafiles): conf_file = os.path.join(datafiles, "convert_value_to_str.yaml") # Run file through yaml to convert it test_dict = _yaml.load(conf_file, shortname=None) user_config = test_dict.get_str("Test1") assert isinstance(user_config, str) assert user_config == "1_23_4" user_config = test_dict.get_str("Test2") assert isinstance(user_config, str) assert user_config == "1.23.4" user_config = test_dict.get_str("Test3") assert isinstance(user_config, str) assert user_config == "1.20" user_config = test_dict.get_str("Test4") assert isinstance(user_config, str) assert user_config == "OneTwoThree" @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_value_doesnt_match_expected(datafiles): conf_file = os.path.join(datafiles, "convert_value_to_str.yaml") # Run file through yaml to convert it test_dict = _yaml.load(conf_file, shortname=None) with pytest.raises(LoadError) as exc: test_dict.get_int("Test4") assert exc.value.reason == LoadErrorReason.INVALID_DATA @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_roundtrip_dump(datafiles): filename = os.path.join(datafiles, "roundtrip-test.yaml") with open(filename, "r", encoding="utf-8") as fh: rt_raw = fh.read() rt_loaded = _yaml.roundtrip_load(filename) # Now walk the loaded data structure, checking for ints etc. def walk_node(node): for v in node.values(): if isinstance(v, list): walk_list(v) elif isinstance(v, dict): walk_node(v) else: assert isinstance(v, str) def walk_list(l): for v in l: if isinstance(v, list): walk_list(v) elif isinstance(v, dict): walk_node(v) else: assert isinstance(v, str) walk_node(rt_loaded) outfile = StringIO() _yaml.roundtrip_dump(rt_loaded, file=outfile) rt_back = outfile.getvalue() assert rt_raw == rt_back @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize( "case", [ ["a", "b", "c"], ["foo", 1], ["stuff", 0, "colour"], ["bird", 0, 1], ], ) def test_node_find_target(datafiles, case): filename = os.path.join(datafiles, "traversal.yaml") # We set copy_tree in order to ensure that the nodes in `loaded` # are not the same nodes as in `prov.toplevel` loaded = _yaml.load(filename, shortname=None, copy_tree=True) prov = loaded.get_provenance() toplevel = prov._toplevel assert toplevel is not loaded # Walk down the node tree, with insider knowledge of how nodes are # laid out. Client code should never do this. def _walk(node, entry, rest): if rest: if isinstance(entry, int): new_node = node.node_at(entry) else: new_node = node.get_node(entry) return _walk(new_node, rest[0], rest[1:]) else: if isinstance(entry, int): return node.node_at(entry) return node.get_node(entry) want = _walk(loaded, case[0], case[1:]) found_path = toplevel._find(want) assert case == found_path @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_node_find_target_fails(datafiles): filename = os.path.join(datafiles, "traversal.yaml") loaded = _yaml.load(filename, shortname=None, copy_tree=True) brand_new = Node.from_dict({}) assert loaded._find(brand_new) is None @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize( "filename, provenance", [ ("list-of-dict.yaml", "list-of-dict.yaml [line 2 column 2]"), ("list-of-list.yaml", "list-of-list.yaml [line 2 column 2]"), ], ids=["list-of-dict", "list-of-list"], ) def test_get_str_list_invalid(datafiles, filename, provenance): conf_file = os.path.join(datafiles, filename) base = _yaml.load(conf_file, shortname=None) with pytest.raises(LoadError) as exc: base.get_str_list("list") assert exc.value.reason == LoadErrorReason.INVALID_DATA assert provenance in str(exc.value) @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_get_str_list_default_none(datafiles): conf_file = os.path.join(datafiles, "list-of-dict.yaml") base = _yaml.load(conf_file, shortname=None) # There is no "pony" key here, assert that the default return is smooth strings = base.get_str_list("pony", None) assert strings is None @pytest.mark.datafiles(os.path.join(DATA_DIR)) def test_mapping_node_assign_none(datafiles): conf_file = os.path.join(datafiles, "dictionary.yaml") dump_file = os.path.join(datafiles, "dictionary-dump.yaml") base = _yaml.load(conf_file, shortname=None) nested = base.get_mapping("nested") nested["ref"] = None # Check that we have successfully set the ref to None value = nested.get_scalar("ref") assert value.is_none() # Without saving and loading, our None value is retained in memory stripped = base.strip_node_info() assert stripped["nested"]["ref"] is None # Save and load _yaml.roundtrip_dump(base, dump_file) loaded = _yaml.load(dump_file, shortname=None) loaded_nested = loaded.get_mapping("nested") value = loaded_nested.get_scalar("ref") # The loaded value will be an empty string, because we don't recognize None # value representations in YAML assert value.as_str() == "" apache-buildstream-27ae392/tests/internals/yaml/000077500000000000000000000000001514607367700217255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/internals/yaml/basics.yaml000066400000000000000000000005171514607367700240600ustar00rootroot00000000000000kind: pony description: The vehicle of choice for rainbow travel moods: - happy - sad children: - name: dopey mood: silly - name: grumpy mood: grumpy - name: happy mood: happy - name: doc mood: happy - name: sneezy mood: curious - name: bashful mood: bashful - name: sleepy mood: sleepy extra: this: that old: new apache-buildstream-27ae392/tests/internals/yaml/composite.yaml000066400000000000000000000002161514607367700246120ustar00rootroot00000000000000kind: horse description: The horse you ride children: - name: extra mood: funny extra: old: override another: one: one two: two apache-buildstream-27ae392/tests/internals/yaml/convert_value_to_str.yaml000066400000000000000000000000731514607367700270570ustar00rootroot00000000000000Test1: 1_23_4 Test2: 1.23.4 Test3: 1.20 Test4: OneTwoThree apache-buildstream-27ae392/tests/internals/yaml/dictionary.yaml000066400000000000000000000001301514607367700247500ustar00rootroot00000000000000 nested: foo: "One foo over the coocoos nest" bar: "And was barred from the asylum" apache-buildstream-27ae392/tests/internals/yaml/implicitoverwrite.yaml000066400000000000000000000002301514607367700263650ustar00rootroot00000000000000# Composited on top of basics.yaml, overwriting its children list children: - name: overwrite1 mood: overwrite1 - name: overwrite2 mood: overwrite2 apache-buildstream-27ae392/tests/internals/yaml/invalid.yaml000066400000000000000000000002031514607367700242320ustar00rootroot00000000000000kind: pony description: The vehicle of choice for rainbow travel mods: - happy - sad children: - naam: dopey mood: silly apache-buildstream-27ae392/tests/internals/yaml/list-of-dict.yaml000066400000000000000000000000661514607367700251110ustar00rootroot00000000000000list: - key1: one key2: two - key1: one key2: two apache-buildstream-27ae392/tests/internals/yaml/list-of-list.yaml000066400000000000000000000000541514607367700251360ustar00rootroot00000000000000list: - [ "one", "two" ] - [ "one", "two" ] apache-buildstream-27ae392/tests/internals/yaml/listappend.yaml000066400000000000000000000002441514607367700247540ustar00rootroot00000000000000# Composited on top of basics.yaml, appending to its children list children: (>): - name: appended1 mood: appended1 - name: appended2 mood: appended2 apache-buildstream-27ae392/tests/internals/yaml/listappendprepend.yaml000066400000000000000000000004161514607367700263330ustar00rootroot00000000000000# Composited on top of basics.yaml, prepending and appending to its children list children: (<): - name: prepended1 mood: prepended1 - name: prepended2 mood: prepended2 (>): - name: appended1 mood: appended1 - name: appended2 mood: appended2 apache-buildstream-27ae392/tests/internals/yaml/listoverwrite.yaml000066400000000000000000000002471514607367700255360ustar00rootroot00000000000000# Composited on top of basics.yaml, overwriting its children list children: (=): - name: overwrite1 mood: overwrite1 - name: overwrite2 mood: overwrite2 apache-buildstream-27ae392/tests/internals/yaml/listoverwriteempty.yaml000066400000000000000000000001371514607367700266130ustar00rootroot00000000000000# Composited on top of basics.yaml, effectively deleting its children list children: (=): [] apache-buildstream-27ae392/tests/internals/yaml/listprepend.yaml000066400000000000000000000002511514607367700251400ustar00rootroot00000000000000# Composited on top of basics.yaml, prepending to its children list children: (<): - name: prepended1 mood: prepended1 - name: prepended2 mood: prepended2 apache-buildstream-27ae392/tests/internals/yaml/roundtrip-test.yaml000066400000000000000000000021031514607367700256100ustar00rootroot00000000000000# This is the roundtrip test YAML document # It contains comments such as this # The test which uses this YAML file will also ensure that # any interesting values we load (such as floats, ints, etc) # are actually loaded as strings. # Blank lines such as the above this: is: a: # Double-quoted - "nested" - mapping # Single-quoted - 'with' - a - list # Comments can go with mappings too such: as: this # We roundtrip integers and floats as strings, to prevent truncation. trunctest: simple-int: 123 small-float: 123.456 int: 000000000000000000000001 float: 1.000000000000000000000 # We also roundtrip booleans in various forms as strings to prevent # normalisation to 'True' and 'False' boolcheck: - True - False - true - false # These are valid YAML booleans, though BuildStream doesn't support them - TRUE - FALSE # The following are YAML 1.1 only - yes - no - on - off nullcheck: null timestamp: 2019-03-14 long-string: Sometimes very long strings get truncated by ruamel-yaml because the default width is set to 80. # That is all apache-buildstream-27ae392/tests/internals/yaml/secondappend.yaml000066400000000000000000000002701514607367700252530ustar00rootroot00000000000000# Composited on top of listappend.yaml, appending to its children list children: (>): - name: secondappend1 mood: secondappend1 - name: secondappend2 mood: secondappend2 apache-buildstream-27ae392/tests/internals/yaml/secondprepend.yaml000066400000000000000000000002761514607367700254470ustar00rootroot00000000000000# Composited on top of listprepend.yaml, prepending to its children list children: (<): - name: secondprepend1 mood: secondprepend1 - name: secondprepend2 mood: secondprepend2 apache-buildstream-27ae392/tests/internals/yaml/traversal.yaml000066400000000000000000000003651514607367700246200ustar00rootroot00000000000000# This document is used to test the target finding algorithm # employed during tracking. a: b: c: fish foo: - bar - baz - meta stuff: - kind: banana colour: yellow - kind: orange colour: orange bird: - [ 'a', 'b' ] apache-buildstream-27ae392/tests/plugins/000077500000000000000000000000001514607367700204455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/__init__.py000066400000000000000000000000001514607367700225440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/junction-with-junction/000077500000000000000000000000001514607367700250765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/junction-with-junction/element.bst000066400000000000000000000000151514607367700272350ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/plugins/junction-with-junction/project.conf000066400000000000000000000002201514607367700274050ustar00rootroot00000000000000name: test min-version: 2.0 plugins: - origin: junction junction: sample-plugins.bst sources: - git (@): - subproject.bst:variables.yml apache-buildstream-27ae392/tests/plugins/junction-with-junction/subproject/000077500000000000000000000000001514607367700272565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/junction-with-junction/subproject/project.conf000066400000000000000000000000471514607367700315740ustar00rootroot00000000000000name: subproject-test min-version: 2.0 apache-buildstream-27ae392/tests/plugins/junction-with-junction/subproject/target.bst000066400000000000000000000000151514607367700312520ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/plugins/junction-with-junction/subproject/variables.yml000066400000000000000000000000341514607367700317460ustar00rootroot00000000000000 variables: animal: pony apache-buildstream-27ae392/tests/plugins/loading.py000066400000000000000000000702051514607367700224400ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name # # This test case tests the failure modes of loading a plugin # after it has already been discovered via it's origin. # import os import shutil import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from buildstream import _yaml from tests.testutils.repo.git import Git from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "loading") def update_project(project_path, updated_configuration): project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf.update(updated_configuration) _yaml.roundtrip_dump(project_conf, project_conf_path) # Sets up the element.bst file so that it requires a source # or element plugin. # def setup_element(project_path, plugin_type, plugin_name): element_dir = os.path.join(project_path, "elements") element_path = os.path.join(element_dir, "element.bst") os.makedirs(element_dir, exist_ok=True) if plugin_type == "elements": element = {"kind": plugin_name} else: element = {"kind": "manual", "sources": [{"kind": plugin_name}]} _yaml.roundtrip_dump(element, element_path) #################################################### # Tests # #################################################### @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_nosetup(cli, datafiles, plugin_type): project = str(datafiles) update_project(project, {"plugins": [{"origin": "local", "path": "plugins/nosetup", plugin_type: ["nosetup"]}]}) setup_element(project, plugin_type, "nosetup") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "missing-setup-function") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_setup_not_function(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, {"plugins": [{"origin": "local", "path": "plugins/setupnotfunction", plugin_type: ["setupnotfunction"]}]}, ) setup_element(project, plugin_type, "setupnotfunction") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "setup-is-not-function") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_setup_returns_not_type(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ {"origin": "local", "path": "plugins/setupreturnsnottype", plugin_type: ["setupreturnsnottype"]} ] }, ) setup_element(project, plugin_type, "setupreturnsnottype") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "setup-returns-not-type") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_setup_returns_bad_type(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ {"origin": "local", "path": "plugins/setupreturnsbadtype", plugin_type: ["setupreturnsbadtype"]} ] }, ) setup_element(project, plugin_type, "setupreturnsbadtype") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "setup-returns-bad-type") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_missing_min_version(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "nominversion"), plugin_type: ["nominversion"], } ] }, ) setup_element(project, plugin_type, "nominversion") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "missing-min-version") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.parametrize("plugin", [("badstring"), ("number"), ("dict"), ("list")]) def test_malformed_min_version(cli, datafiles, plugin_type, plugin): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "malformedminversion"), plugin_type: [plugin], } ] }, ) setup_element(project, plugin_type, plugin) result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "malformed-min-version") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_incompatible_major_version(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "incompatiblemajor"), plugin_type: ["incompatiblemajor"], } ] }, ) setup_element(project, plugin_type, "incompatiblemajor") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "incompatible-major-version") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_incompatible_minor_version(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "incompatibleminor"), plugin_type: ["incompatibleminor"], } ] }, ) setup_element(project, plugin_type, "incompatibleminor") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "incompatible-minor-version") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_plugin_not_found(cli, datafiles, plugin_type): project = str(datafiles) setup_element(project, plugin_type, "notfound") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "plugin-not-found") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_plugin_found(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "found") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_deprecation_warnings(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "deprecated"), plugin_type: ["deprecated"], } ] }, ) setup_element(project, plugin_type, "deprecated") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() assert "Here is some detail." in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_deprecation_warning_suppressed_by_origin(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "deprecated"), "allow-deprecated": True, plugin_type: ["deprecated"], } ] }, ) setup_element(project, plugin_type, "deprecated") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() assert "Here is some detail." not in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_deprecation_warning_suppressed_specifically(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "deprecated"), plugin_type: [{"kind": "deprecated", "allow-deprecated": True}], } ] }, ) setup_element(project, plugin_type, "deprecated") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() assert "Here is some detail." not in result.stderr @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_pip_origin_load_success(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_pip_origin_with_constraints(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins>=1.0,<1.2.5,!=1.1.3", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_pip_origin_package_not_found(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "pip", "package-name": "not-a-package", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "package-not-found") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_pip_origin_plugin_not_found(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins", plugin_type: ["notfound"], } ] }, ) setup_element(project, plugin_type, "notfound") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "plugin-not-found") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_pip_origin_version_conflict(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins>=1.4", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "package-version-conflict") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_pip_origin_malformed_constraints(cli, datafiles, plugin_type): project = str(datafiles) update_project( project, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins>1.4,A", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "package-malformed-requirement") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_junction_plugin_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subproject, "plugins")) update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst", plugin_type: ["found"], } ] }, ) update_project( subproject, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "found") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_junction_plugin_not_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subproject, "plugins")) # The toplevel says to search for the "notfound" plugin in the subproject # update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst", plugin_type: ["notfound"], } ] }, ) # The subproject only configures the "found" plugin # update_project( subproject, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "notfound") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "junction-plugin-not-found") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_junction_deep_plugin_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") subsubproject = os.path.join(subproject, "subsubproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst", plugin_type: ["found"], } ] }, ) update_project( subproject, { "plugins": [ { "origin": "junction", "junction": "subsubproject-junction.bst", plugin_type: ["found"], } ] }, ) update_project( subsubproject, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "found") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_junction_deep_plugin_not_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") subsubproject = os.path.join(subproject, "subsubproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) # The toplevel says to search for the "notfound" plugin in the subproject # update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst", plugin_type: ["notfound"], } ] }, ) # The subproject says to search for the "notfound" plugin in the subproject # update_project( subproject, { "plugins": [ { "origin": "junction", "junction": "subsubproject-junction.bst", plugin_type: ["notfound"], } ] }, ) # The subsubproject only configures the "found" plugin # update_project( subsubproject, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "notfound") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "junction-plugin-load-error") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_junction_pip_plugin_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subproject, "plugins")) update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst", plugin_type: ["sample"], } ] }, ) update_project( subproject, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_junction_pip_plugin_version_conflict(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subproject, "plugins")) update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst", plugin_type: ["sample"], } ] }, ) update_project( subproject, { "plugins": [ { "origin": "pip", "package-name": "sample-plugins>=1.4", plugin_type: ["sample"], } ] }, ) setup_element(project, plugin_type, "sample") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "junction-plugin-load-error") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_junction_full_path_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") subsubproject = os.path.join(subproject, "subsubproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst:subsubproject-junction.bst", plugin_type: ["found"], } ] }, ) update_project( subsubproject, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "found") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_success() @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("plugin_type", [("elements"), ("sources")]) def test_junction_full_path_not_found(cli, datafiles, plugin_type): project = str(datafiles) subproject = os.path.join(project, "subproject") subsubproject = os.path.join(subproject, "subsubproject") shutil.copytree(os.path.join(project, "plugins"), os.path.join(subsubproject, "plugins")) # The toplevel says to search for the "notfound" plugin in the subproject # update_project( project, { "plugins": [ { "origin": "junction", "junction": "subproject-junction.bst:subsubproject-junction.bst", plugin_type: ["notfound"], } ] }, ) # The subsubproject only configures the "found" plugin # update_project( subsubproject, { "plugins": [ { "origin": "local", "path": os.path.join("plugins", plugin_type, "found"), plugin_type: ["found"], } ] }, ) setup_element(project, plugin_type, "notfound") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.PLUGIN, "junction-plugin-not-found") @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "plugin_type,provenance", [("elements", "project.conf [line 12 column 2]"), ("sources", "project.conf [line 12 column 2]")], ) def test_junction_invalid_full_path(cli, datafiles, plugin_type, provenance): project = str(datafiles) shutil.copy(os.path.join(project, "not-found-{}.conf".format(plugin_type)), os.path.join(project, "project.conf")) setup_element(project, plugin_type, "notfound") result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) assert provenance in result.stderr # Test scenario for junction plugin origins # ========================================= # # This is a regression test which ensures that cross junction includes # at the project.conf level continues to work even in conjunction with # complex cross junction plugin loading scenarios. # # main project # / \ # | | # junction (tar) | # | | include a file across this junction # | | # / | # git plugin \ # \ # junction (git) # | # | # subproject # # # `bst source track subproject.bst` # # JUNCTION_DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__))) @pytest.mark.datafiles(JUNCTION_DATA_DIR) def test_load_junction_via_junctioned_plugin(cli, datafiles, tmpdir): sample_plugins_dir = os.path.join(str(datafiles), "sample-plugins") project = os.path.join(str(datafiles), "junction-with-junction") subproject = os.path.join(str(datafiles), "junction-with-junction", "subproject") # Create a tar repo containing the sample plugins # repo = create_repo("tar", str(tmpdir)) ref = repo.create(sample_plugins_dir) # Generate the junction to the sample plugins # element = {"kind": "junction", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(project, "sample-plugins.bst")) # Create a git repo containing the subproject # subproject_repo = Git(str(tmpdir)) subproject_repo.create(subproject) # Generate the subproject junction pointing to the git repo with the subproject # element = {"kind": "junction", "sources": [subproject_repo.source_config()]} _yaml.roundtrip_dump(element, os.path.join(project, "subproject.bst")) # Track the subproject # result = cli.run(project=project, args=["source", "track", "subproject.bst"]) result.assert_success() # Check the included variable resolves in the element # result = cli.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{vars}", "element.bst"], ) result.assert_success() loaded = _yaml.load_data(result.output) assert loaded.get_str("animal") == "pony" # Try a subproject element access on the command line, as this project # has the potential to make this break. # result = cli.run(project=project, args=["show", "subproject.bst:target.bst"]) result.assert_success() apache-buildstream-27ae392/tests/plugins/loading/000077500000000000000000000000001514607367700220625ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/elements/000077500000000000000000000000001514607367700236765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/elements/subproject-junction.bst000066400000000000000000000000721514607367700304160ustar00rootroot00000000000000kind: junction sources: - kind: local path: subproject apache-buildstream-27ae392/tests/plugins/loading/not-found-elements.conf000066400000000000000000000005401514607367700264530ustar00rootroot00000000000000# This project.conf gets rewritten for each plugin loading test name: test # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Load non-existant element across junction boundaries plugins: - origin: junction junction: subproject-junction.bst:pony-junction.bst elements: - notfound apache-buildstream-27ae392/tests/plugins/loading/not-found-sources.conf000066400000000000000000000005361514607367700263270ustar00rootroot00000000000000# This project.conf gets rewritten for each plugin loading test name: test # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements # Load non-existant source across junction boundaries plugins: - origin: junction junction: subproject-junction.bst:pony-junction.bst sources: - notfound apache-buildstream-27ae392/tests/plugins/loading/plugins/000077500000000000000000000000001514607367700235435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/000077500000000000000000000000001514607367700253575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/deprecated/000077500000000000000000000000001514607367700274575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/deprecated/deprecated.py000066400000000000000000000005651514607367700321370ustar00rootroot00000000000000from buildstream import Element class Deprecated(Element): BST_MIN_VERSION = "2.0" BST_PLUGIN_DEPRECATED = True BST_PLUGIN_DEPRECATION_MESSAGE = "Here is some detail." def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return Deprecated apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/found/000077500000000000000000000000001514607367700264725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/found/found.py000066400000000000000000000004161514607367700301600ustar00rootroot00000000000000from buildstream import Element class Found(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return Found apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/incompatiblemajor/000077500000000000000000000000001514607367700310565ustar00rootroot00000000000000incompatiblemajor.py000066400000000000000000000002151514607367700350460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/incompatiblemajorfrom buildstream import Element class IncompatibleMajor(Element): BST_MIN_VERSION = "1.0" def setup(): return IncompatibleMajor apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/incompatibleminor/000077500000000000000000000000001514607367700310725ustar00rootroot00000000000000incompatibleminor.py000066400000000000000000000002201514607367700350720ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/incompatibleminorfrom buildstream import Element class IncompatibleMinor(Element): BST_MIN_VERSION = "2.1000" def setup(): return IncompatibleMinor apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/malformedminversion/000077500000000000000000000000001514607367700314375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/malformedminversion/badstring.py000066400000000000000000000003121514607367700337620ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Element class MalformedMinVersion(Element): BST_MIN_VERSION = "1.pony" def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/malformedminversion/dict.py000066400000000000000000000003321514607367700327320ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Element class MalformedMinVersion(Element): BST_MIN_VERSION = {"major": 2, "minor": 0} def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/malformedminversion/list.py000066400000000000000000000003101514607367700327560ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Element class MalformedMinVersion(Element): BST_MIN_VERSION = [2, 0] def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/malformedminversion/number.py000066400000000000000000000003051514607367700332770ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Element class MalformedMinVersion(Element): BST_MIN_VERSION = 2.0 def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/nominversion/000077500000000000000000000000001514607367700301055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/elements/nominversion/nominversion.py000066400000000000000000000002451514607367700332060ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Element class NoMinVersion(Element): pass def setup(): return NoMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/nosetup/000077500000000000000000000000001514607367700252405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/nosetup/nosetup.py000066400000000000000000000002551514607367700273110ustar00rootroot00000000000000# A plugin is supposed to define a setup function # which returns the type that the plugin provides # # This plugin fails to do so def useless(): print("Hello World") apache-buildstream-27ae392/tests/plugins/loading/plugins/setupnotfunction/000077500000000000000000000000001514607367700271725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/setupnotfunction/setupnotfunction.py000066400000000000000000000002321514607367700331700ustar00rootroot00000000000000# A plugin's setup symbol is supposed to be a function # which returns the plugin type. # # This plugin's setup symbol is not such a function. setup = 9 apache-buildstream-27ae392/tests/plugins/loading/plugins/setupreturnsbadtype/000077500000000000000000000000001514607367700276775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/setupreturnsbadtype/setupreturnsbadtype.py000066400000000000000000000005111514607367700344020ustar00rootroot00000000000000# A plugin's setup symbol is supposed to be a function # which returns the plugin type, which should be a subclass # of Source or Element depending on the plugin type. # # This plugin's setup function returns a different kind # of type. class Pony: def __init__(self): self.pony = 12 def setup(): return Pony apache-buildstream-27ae392/tests/plugins/loading/plugins/setupreturnsnottype/000077500000000000000000000000001514607367700277515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/setupreturnsnottype/setupreturnsnottype.py000066400000000000000000000002561514607367700345340ustar00rootroot00000000000000# A plugin's setup symbol is supposed to be a function # which returns the plugin type. # # This plugin's setup function returns a number instead def setup(): return 5 apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/000077500000000000000000000000001514607367700252265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/deprecated/000077500000000000000000000000001514607367700273265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/deprecated/deprecated.py000066400000000000000000000010511514607367700317750ustar00rootroot00000000000000from buildstream import Source class Deprecated(Source): BST_MIN_VERSION = "2.0" BST_PLUGIN_DEPRECATED = True BST_PLUGIN_DEPRECATION_MESSAGE = "Here is some detail." def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False # Plugin entry point def setup(): return Deprecated apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/found/000077500000000000000000000000001514607367700263415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/found/found.py000066400000000000000000000007021514607367700300250ustar00rootroot00000000000000from buildstream import Source class Found(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False # Plugin entry point def setup(): return Found apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/incompatiblemajor/000077500000000000000000000000001514607367700307255ustar00rootroot00000000000000incompatiblemajor.py000066400000000000000000000002131514607367700347130ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/incompatiblemajorfrom buildstream import Source class IncompatibleMajor(Source): BST_MIN_VERSION = "1.0" def setup(): return IncompatibleMajor apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/incompatibleminor/000077500000000000000000000000001514607367700307415ustar00rootroot00000000000000incompatibleminor.py000066400000000000000000000002161514607367700347460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/incompatibleminorfrom buildstream import Source class IncompatibleMinor(Source): BST_MIN_VERSION = "2.1000" def setup(): return IncompatibleMinor apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/malformedminversion/000077500000000000000000000000001514607367700313065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/malformedminversion/badstring.py000066400000000000000000000003101514607367700336270ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Source class MalformedMinVersion(Source): BST_MIN_VERSION = "1.pony" def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/malformedminversion/dict.py000066400000000000000000000003301514607367700325770ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Source class MalformedMinVersion(Source): BST_MIN_VERSION = {"major": 2, "minor": 0} def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/malformedminversion/list.py000066400000000000000000000003061514607367700326320ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Source class MalformedMinVersion(Source): BST_MIN_VERSION = [2, 0] def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/malformedminversion/number.py000066400000000000000000000003031514607367700331440ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Source class MalformedMinVersion(Source): BST_MIN_VERSION = 2.0 def setup(): return MalformedMinVersion apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/nominversion/000077500000000000000000000000001514607367700277545ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/plugins/sources/nominversion/nominversion.py000066400000000000000000000002431514607367700330530ustar00rootroot00000000000000# Plugins are required to specify the BST_MIN_VERSION from buildstream import Source class NoMinVersion(Source): pass def setup(): return NoMinVersion apache-buildstream-27ae392/tests/plugins/loading/project.conf000066400000000000000000000002751514607367700244030ustar00rootroot00000000000000# This project.conf gets rewritten for each plugin loading test name: test # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/tests/plugins/loading/subproject/000077500000000000000000000000001514607367700242425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/subproject/elements/000077500000000000000000000000001514607367700260565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/subproject/elements/subsubproject-junction.bst000066400000000000000000000000751514607367700333130ustar00rootroot00000000000000kind: junction sources: - kind: local path: subsubproject apache-buildstream-27ae392/tests/plugins/loading/subproject/project.conf000066400000000000000000000002261514607367700265570ustar00rootroot00000000000000# The subproject test name: subtest # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/tests/plugins/loading/subproject/subsubproject/000077500000000000000000000000001514607367700271345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/loading/subproject/subsubproject/project.conf000066400000000000000000000001301514607367700314430ustar00rootroot00000000000000# The subproject test name: subsubtest # Required BuildStream version min-version: 2.0 apache-buildstream-27ae392/tests/plugins/sample-plugins/000077500000000000000000000000001514607367700234055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/MANIFEST.in000066400000000000000000000000261514607367700251410ustar00rootroot00000000000000global-include *.yaml apache-buildstream-27ae392/tests/plugins/sample-plugins/project.conf000066400000000000000000000004621514607367700257240ustar00rootroot00000000000000name: sample-plugins min-version: 2.0 plugins: - origin: local path: src/sample_plugins/elements elements: - autotools - sample - origin: local path: src/sample_plugins/sources sources: - git - sample - origin: local path: src/sample_plugins/sourcemirrors source-mirrors: - mirror apache-buildstream-27ae392/tests/plugins/sample-plugins/setup.py000077500000000000000000000026341514607367700251270ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from setuptools import setup, find_packages setup( name="sample-plugins", version="1.2.3", description="A collection of sample plugins for testing.", license="Apache License Version 2.0", url="https://example.com/sample-plugins", package_dir={"": "src"}, packages=find_packages(where="src"), include_package_data=True, entry_points={ "buildstream.plugins.elements": [ "sample = sample_plugins.elements.sample", "autotools = sample_plugins.elements.autotools", ], "buildstream.plugins.sources": [ "sample = sample_plugins.sources.sample", "git = sample_plugins.sources.git", ], "buildstream.plugins.sourcemirrors": [ "mirror = sample_plugins.sourcemirrors.mirror", ], }, zip_safe=False, ) # eof setup() apache-buildstream-27ae392/tests/plugins/sample-plugins/src/000077500000000000000000000000001514607367700241745ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/000077500000000000000000000000001514607367700272165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/__init__.py000066400000000000000000000000001514607367700313150ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/elements/000077500000000000000000000000001514607367700310325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/elements/__init__.py000066400000000000000000000000001514607367700331310ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/elements/autotools.py000066400000000000000000000041231514607367700334350ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom """ autotools - Autotools build element =================================== This is a :mod:`BuildElement ` implementation for using Autotools build scripts (also known as the `GNU Build System `_). You will often want to pass additional arguments to ``configure``. This should be done on a per-element basis by setting the ``conf-local`` variable. Here is an example: .. code:: yaml variables: conf-local: | --disable-foo --enable-bar If you want to pass extra options to ``configure`` for every element in your project, set the ``conf-global`` variable in your project.conf file. Here is an example of that: .. code:: yaml elements: autotools: variables: conf-global: | --disable-gtk-doc --disable-static Here is the default configuration for the ``autotools`` element in full: .. literalinclude:: ../../../src/buildstream_plugins/elements/autotools.yaml :language: yaml See `built-in functionality documentation `_ for details on common configuration options for build elements. """ from buildstream import BuildElement # Element implementation for the 'autotools' kind. class AutotoolsElement(BuildElement): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" # Plugin entry point def setup(): return AutotoolsElement apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/elements/autotools.yaml000066400000000000000000000060401514607367700337470ustar00rootroot00000000000000# Autotools default configurations variables: autogen: | export NOCONFIGURE=1; if [ -x %{conf-cmd} ]; then true; elif [ -x %{conf-root}/autogen ]; then %{conf-root}/autogen; elif [ -x %{conf-root}/autogen.sh ]; then %{conf-root}/autogen.sh; elif [ -x %{conf-root}/bootstrap ]; then %{conf-root}/bootstrap; elif [ -x %{conf-root}/bootstrap.sh ]; then %{conf-root}/bootstrap.sh; else autoreconf -ivf %{conf-root}; fi # Project-wide extra arguments to be passed to `configure` conf-global: '' # Element-specific extra arguments to be passed to `configure`. conf-local: '' # For backwards compatibility only, do not use. conf-extra: '' conf-cmd: "%{conf-root}/configure" conf-args: | --prefix=%{prefix} \ --exec-prefix=%{exec_prefix} \ --bindir=%{bindir} \ --sbindir=%{sbindir} \ --sysconfdir=%{sysconfdir} \ --datadir=%{datadir} \ --includedir=%{includedir} \ --libdir=%{libdir} \ --libexecdir=%{libexecdir} \ --localstatedir=%{localstatedir} \ --sharedstatedir=%{sharedstatedir} \ --mandir=%{mandir} \ --infodir=%{infodir} %{conf-extra} %{conf-global} %{conf-local} configure: | %{conf-cmd} %{conf-args} make: make make-install: make -j1 DESTDIR="%{install-root}" install # Set this if the sources cannot handle parallelization. # # notparallel: True # Automatically remove libtool archive files # # Set remove-libtool-modules to "true" to remove .la files for # modules intended to be opened with lt_dlopen() # # Set remove-libtool-libraries to "true" to remove .la files for # libraries # # Value must be "true" or "false" remove-libtool-modules: "false" remove-libtool-libraries: "false" delete-libtool-archives: | if %{remove-libtool-modules} || %{remove-libtool-libraries}; then find "%{install-root}" -name "*.la" -print0 | while read -d '' -r file; do if grep '^shouldnotlink=yes$' "${file}" &>/dev/null; then if %{remove-libtool-modules}; then echo "Removing ${file}." rm "${file}" else echo "Not removing ${file}." fi else if %{remove-libtool-libraries}; then echo "Removing ${file}." rm "${file}" else echo "Not removing ${file}." fi fi done fi config: # Commands for configuring the software # configure-commands: - | %{autogen} - | %{configure} # Commands for building the software # build-commands: - | %{make} # Commands for installing the software into a # destination folder # install-commands: - | %{make-install} - | %{delete-libtool-archives} # Commands for stripping debugging information out of # installed binaries # strip-commands: - | %{strip-binaries} # Use max-jobs CPUs for building and enable verbosity environment: MAKEFLAGS: -j%{max-jobs} V: 1 # And dont consider MAKEFLAGS or V as something which may # affect build output. environment-nocache: - MAKEFLAGS - V apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/elements/sample.py000066400000000000000000000004201514607367700326610ustar00rootroot00000000000000from buildstream import Element class Sample(Element): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return Sample apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/elements/sample.yaml000066400000000000000000000001741514607367700332010ustar00rootroot00000000000000# Set a variable so that we can test that the yaml # was actually loaded using bst show. # variables: sample-loaded: True apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sourcemirrors/000077500000000000000000000000001514607367700321345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sourcemirrors/__init__.py000066400000000000000000000000001514607367700342330ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sourcemirrors/mirror.py000066400000000000000000000016141514607367700340220ustar00rootroot00000000000000from typing import Optional, Dict, Any from buildstream import SourceMirror, MappingNode # This mirror plugin basically implements the default behavior # by loading the alias definitions as custom "config" configuration # instead, and implementing the translate_url method. # class Sample(SourceMirror): def configure(self, node): node.validate_keys(["aliases"]) self.aliases = {} aliases = node.get_mapping("aliases") for alias_name, url_list in aliases.items(): self.aliases[alias_name] = url_list.as_str_list() self.set_supported_aliases(self.aliases.keys()) def translate_url( self, *, alias: str, alias_url: str, source_url: str, extra_data: Optional[Dict[str, Any]], ) -> str: return self.aliases[alias][0] + source_url # Plugin entry point def setup(): return Sample apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sources/000077500000000000000000000000001514607367700307015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sources/__init__.py000066400000000000000000000000001514607367700330000ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sources/git.py000066400000000000000000001146661514607367700320540ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tristan Van Berkom # Chandan Singh # Tom Mewett """ git - stage files from a git repository ======================================= **Host dependencies:** * git .. attention:: Note that this plugin **will checkout git submodules by default**; even if they are not specified in the `.bst` file. **Usage:** .. code:: yaml # Specify the git source kind kind: git # Specify the repository url, using an alias defined # in your project configuration is recommended. url: upstream:foo.git # Optionally specify a symbolic tracking branch or tag, this # will be used to update the 'ref' when refreshing the pipeline. track: master # Optionally specify the ref format used for tracking. # The default is 'sha1' for the raw commit hash. # If you specify 'git-describe', the commit hash will be prefixed # with the closest tag. ref-format: sha1 # Specify the commit ref, this must be specified in order to # checkout sources and build, but can be automatically updated # if the 'track' attribute was specified. ref: d63cbb6fdc0bbdadc4a1b92284826a6d63a7ebcd # Optionally specify whether submodules should be checked-out. # This is done recursively, as with `git clone --recurse-submodules`. # If not set, this will default to 'True' checkout-submodules: True # If your repository has submodules, explicitly specifying the # url from which they are to be fetched allows you to easily # rebuild the same sources from a different location. This is # especially handy when used with project defined aliases which # can be redefined at a later time. # You may also explicitly specify whether to check out this # submodule. If 'checkout' is set, it will control whether to # checkout that submodule and recurse into it. It defaults to the # value of 'checkout-submodules'. submodules: plugins/bar: url: upstream:bar.git checkout: True plugins/bar/quux: checkout: False plugins/baz: url: upstream:baz.git checkout: False # Enable tag tracking. # # This causes the `tags` metadata to be populated automatically # as a result of tracking the git source. # # By default this is 'False'. # track-tags: True # If the list of tags below is set, then a lightweight dummy # git repository will be staged along with the content at # build time. # # This is useful for a growing number of modules which use # `git describe` at build time in order to determine the version # which will be encoded into the built software. # # The 'tags' below is considered as a part of the git source # reference and will be stored in the 'project.refs' file if # that has been selected as your project's ref-storage. # # Migration notes: # # If you are upgrading from BuildStream 1.2, which used to # stage the entire repository by default, you will notice that # some modules which use `git describe` are broken, and will # need to enable this feature in order to fix them. # # If you need to enable this feature without changing the # the specific commit that you are building, then we recommend # the following migration steps for any git sources where # `git describe` is required: # # o Enable `track-tags` feature # o Set the `track` parameter to the desired commit sha which # the current `ref` points to # o Run `bst source track` for these elements, this will result in # populating the `tags` portion of the refs without changing # the refs # o Restore the `track` parameter to the branches which you have # previously been tracking afterwards. # tags: - tag: lightweight-example commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78 annotated: false - tag: annotated-example commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a annotated: true See `built-in functionality doumentation `_ for details on common configuration options for sources. **Configurable Warnings:** This plugin provides the following `configurable warnings `_: - ``git:inconsistent-submodule`` - A submodule present in the git repository's .gitmodules was never added with `git submodule add`. - ``git:unlisted-submodule`` - A submodule is present in the git repository but was not specified in the source configuration and was not disabled for checkout. - ``git:invalid-submodule`` - A submodule is specified in the source configuration but does not exist in the repository. This plugin also utilises the following configurable `core warnings `_: - `ref-not-in-track `_ - The provided ref was not found in the provided track in the element's git repository. """ import os import re import shutil from io import StringIO from tempfile import TemporaryFile from configparser import RawConfigParser from buildstream import Source, SourceError, SourceFetcher from buildstream import CoreWarnings, FastEnum from buildstream import utils from buildstream.utils import DirectoryExistsError GIT_MODULES = ".gitmodules" EXACT_TAG_PATTERN = r"(?P.*)-0-g(?P.*)" # Warnings WARN_INCONSISTENT_SUBMODULE = "inconsistent-submodule" WARN_UNLISTED_SUBMODULE = "unlisted-submodule" WARN_INVALID_SUBMODULE = "invalid-submodule" class _RefFormat(FastEnum): SHA1 = "sha1" GIT_DESCRIBE = "git-describe" def _strip_tag(rev): return rev.split("-g")[-1] # This class represents a single Git repository. The Git source needs to account for # submodules, but we don't want to cache them all under the umbrella of the # superproject - so we use this class which caches them independently, according # to their URL. Instances keep reference to their "parent" GitSource, # and if applicable, where in the superproject they are found. # # Args: # source (GitSource): The parent source # path (str): The relative location of the submodule in the superproject; # the empty string for the superproject itself # url (str): Where to clone the repo from # ref (str): Specified 'ref' from the source configuration # primary (bool): Whether this is the primary URL for the source # tags (list): Tag configuration; see GitSource._load_tags # class GitMirror(SourceFetcher): def __init__(self, source, path, url, ref, *, primary=False, tags=None): super().__init__() self.source = source self.path = path self.url = url self.ref = ref self.tags = tags or [] self.primary = primary self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url)) # _ensure_repo(): # # Ensures that the Git repository exists at the mirror location and is configured # to fetch from the given URL # def _ensure_repo(self): if not os.path.exists(self.mirror): with self.source.tempdir() as tmpdir: self.source.call( [self.source.host_git, "init", "--bare", tmpdir], fail="Failed to initialise repository", ) try: utils.move_atomic(tmpdir, self.mirror) except DirectoryExistsError: # Another process was quicker to download this repository. # Let's discard our own self.source.status("{}: Discarding duplicate repository".format(self.source)) except OSError as e: raise SourceError( "{}: Failed to move created repository from '{}' to '{}': {}".format( self.source, tmpdir, self.mirror, e ) ) from e def _fetch(self, url, fetch_all=False): self._ensure_repo() # Work out whether we can fetch a specific tag: are we given a ref which # 1. is in git-describe format # 2. refers to an exact tag (is "...-0-g...") # 3. is available on the remote and tags the specified commit? # And lastly: are we on a new-enough Git which allows cloning from our potentially shallow cache? if fetch_all: pass # Fetching from a shallow-cloned repo was first supported in v1.9.0 elif not self.ref or self.source.host_git_version is not None and self.source.host_git_version < (1, 9, 0): fetch_all = True else: m = re.match(EXACT_TAG_PATTERN, self.ref) if m is None: fetch_all = True else: tag = m.group("tag") commit = m.group("commit") if not self.remote_has_tag(url, tag, commit): self.source.status( "{}: {} is not advertised on {}. Fetching all Git refs".format(self.source, self.ref, url) ) fetch_all = True else: exit_code = self.source.call( [ self.source.host_git, "fetch", "--depth=1", url, "+refs/tags/{tag}:refs/tags/{tag}".format(tag=tag), ], cwd=self.mirror, ) if exit_code != 0: self.source.status( "{}: Failed to fetch tag '{}' from {}. Fetching all Git refs".format(self.source, tag, url) ) fetch_all = True if fetch_all: self.source.call( [ self.source.host_git, "fetch", "--prune", url, "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*", ], fail="Failed to fetch from remote git repository: {}".format(url), fail_temporarily=True, cwd=self.mirror, ) def fetch(self, alias_override=None): # pylint: disable=arguments-differ resolved_url = self.source.translate_url(self.url, alias_override=alias_override, primary=self.primary) with self.source.timed_activity("Fetching from {}".format(resolved_url), silent_nested=True): if not self.has_ref(): self._fetch(resolved_url) self.assert_ref() def has_ref(self): if not self.ref: return False # If the mirror doesnt exist, we also dont have the ref if not os.path.exists(self.mirror): return False # Check if the ref is really there rc = self.source.call([self.source.host_git, "cat-file", "-t", self.ref], cwd=self.mirror) return rc == 0 def assert_ref(self): if not self.has_ref(): raise SourceError( "{}: expected ref '{}' was not found in git repository: '{}'".format(self.source, self.ref, self.url) ) # remote_has_tag(): # # Args: # url (str) # tag (str) # commit (str) # # Returns: # (bool): Whether the remote at `url` has the tag `tag` attached to `commit` # def remote_has_tag(self, url, tag, commit): _, ls_remote = self.source.check_output( [self.source.host_git, "ls-remote", url], cwd=self.mirror, fail="Failed to list advertised remote refs from git repository {}".format(url), ) line = "{commit}\trefs/tags/{tag}".format(commit=commit, tag=tag) return line in ls_remote or line + "^{}" in ls_remote # to_commit(): # # Args: # rev (str): A Git "commit-ish" rev # # Returns: # (str): The full revision ID of the commit # def to_commit(self, rev): _, output = self.source.check_output( [self.source.host_git, "rev-list", "-n", "1", rev], fail="Unable to find revision {}".format(rev), cwd=self.mirror, ) return output.strip() # describe(): # # Args: # rev (str): A Git "commit-ish" rev # # Returns: # (str): The full revision ID of the commit given by rev, prepended # with tag information as given by git-describe (where available) # def describe(self, rev): _, output = self.source.check_output( [ self.source.host_git, "describe", "--tags", "--abbrev=40", "--long", "--always", rev, ], fail="Unable to find revision {}".format(rev), cwd=self.mirror, ) return output.strip() # reachable_tags(): # # Args: # rev (str): A Git "commit-ish" rev # # Returns: # (list): A list of tags in the ancestry of rev. Each entry is a triple of the form # (tag name (str), commit ref (str), annotated (bool)) describing a tag, # its tagged commit and whether it's annotated # def reachable_tags(self, rev): tags = set() for options in [ [], ["--first-parent"], ["--tags"], ["--tags", "--first-parent"], ]: exit_code, output = self.source.check_output( [ self.source.host_git, "describe", "--abbrev=0", rev, *options, ], cwd=self.mirror, ) if exit_code == 0: tag = output.strip() _, commit_ref = self.source.check_output( [self.source.host_git, "rev-parse", tag + "^{commit}"], fail="Unable to resolve tag '{}'".format(tag), cwd=self.mirror, ) exit_code = self.source.call( [self.source.host_git, "cat-file", "tag", tag], cwd=self.mirror, ) annotated = exit_code == 0 tags.add((tag, commit_ref.strip(), annotated)) return list(tags) def stage(self, directory): fullpath = os.path.join(directory, self.path) # Using --shared here avoids copying the objects into the checkout, in any # case we're just checking out a specific commit and then removing the .git/ # directory. self.source.call( [ self.source.host_git, "clone", "--no-checkout", "--shared", self.mirror, fullpath, ], fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath), fail_temporarily=True, ) self.source.call( [self.source.host_git, "checkout", "--force", self.ref], fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath, ) # Remove .git dir shutil.rmtree(os.path.join(fullpath, ".git")) self._rebuild_git(fullpath) def init_workspace(self, directory): fullpath = os.path.join(directory, self.path) url = self.source.translate_url(self.url) self.source.call( [ self.source.host_git, "clone", "--no-checkout", self.mirror, fullpath, ], fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath), fail_temporarily=True, ) self.source.call( [self.source.host_git, "remote", "set-url", "origin", url], fail='Failed to add remote origin "{}"'.format(url), cwd=fullpath, ) self.source.call( [self.source.host_git, "checkout", "--force", self.ref], fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath, ) # get_submodule_mirrors(): # # Returns: # An iterator through new instances of this class, one of each submodule # in the repo # def get_submodule_mirrors(self): for path, url in self.submodule_list(): ref = self.submodule_ref(path) if ref is not None: mirror = self.__class__(self.source, os.path.join(self.path, path), url, ref) yield mirror # List the submodules (path/url tuples) present at the given ref of this repo def submodule_list(self): modules = "{}:{}".format(self.ref, GIT_MODULES) exit_code, output = self.source.check_output([self.source.host_git, "show", modules], cwd=self.mirror) # If git show reports error code 128 here, we take it to mean there is # no .gitmodules file to display for the given revision. if exit_code == 128: return elif exit_code != 0: raise SourceError("{plugin}: Failed to show gitmodules at ref {ref}".format(plugin=self, ref=self.ref)) content = "\n".join([l.strip() for l in output.splitlines()]) io = StringIO(content) parser = RawConfigParser() parser.read_file(io) for section in parser.sections(): # validate section name against the 'submodule "foo"' pattern if re.match(r'submodule "(.*)"', section): path = parser.get(section, "path") url = parser.get(section, "url") yield (path, url) # Fetch the ref which this mirror requires its submodule to have, # at the given ref of this mirror. def submodule_ref(self, submodule, ref=None): if not ref: ref = self.ref # list objects in the parent repo tree to find the commit # object that corresponds to the submodule _, output = self.source.check_output( [self.source.host_git, "ls-tree", ref, submodule], fail="ls-tree failed for commit {} and submodule: {}".format(ref, submodule), cwd=self.mirror, ) # read the commit hash from the output fields = output.split() if len(fields) >= 2 and fields[1] == "commit": submodule_commit = output.split()[2] # fail if the commit hash is invalid if len(submodule_commit) != 40: raise SourceError( "{}: Error reading commit information for submodule '{}'".format(self.source, submodule) ) return submodule_commit else: detail = ( "The submodule '{}' is defined either in the BuildStream source\n".format(submodule) + "definition, or in a .gitmodules file. But the submodule was never added to the\n" + "underlying git repository with `git submodule add`." ) self.source.warn( "{}: Ignoring inconsistent submodule '{}'".format(self.source, submodule), detail=detail, warning_token=WARN_INCONSISTENT_SUBMODULE, ) return None def _rebuild_git(self, fullpath): if not self.tags: return with self.source.tempdir() as tmpdir: included = set() shallow = set() for _, commit_ref, _ in self.tags: if commit_ref == self.ref: # rev-list does not work in case of same rev shallow.add(self.ref) else: _, out = self.source.check_output( [ self.source.host_git, "rev-list", "--ancestry-path", "--boundary", "{}..{}".format(commit_ref, self.ref), ], fail="Failed to get git history {}..{} in directory: {}".format( commit_ref, self.ref, fullpath ), fail_temporarily=True, cwd=self.mirror, ) self.source.warn("refs {}..{}: {}".format(commit_ref, self.ref, out.splitlines())) for line in out.splitlines(): rev = line.lstrip("-") if line[0] == "-": shallow.add(rev) else: included.add(rev) shallow -= included included |= shallow self.source.call( [self.source.host_git, "init"], fail="Cannot initialize git repository: {}".format(fullpath), cwd=fullpath, ) for rev in included: with TemporaryFile(dir=tmpdir) as commit_file: self.source.call( [self.source.host_git, "cat-file", "commit", rev], stdout=commit_file, fail="Failed to get commit {}".format(rev), cwd=self.mirror, ) commit_file.seek(0, 0) self.source.call( [ self.source.host_git, "hash-object", "-w", "-t", "commit", "--stdin", ], stdin=commit_file, fail="Failed to add commit object {}".format(rev), cwd=fullpath, ) with open( os.path.join(fullpath, ".git", "shallow"), "w", encoding="utf-8", ) as shallow_file: for rev in shallow: shallow_file.write("{}\n".format(rev)) for tag, commit_ref, annotated in self.tags: if annotated: with TemporaryFile(dir=tmpdir) as tag_file: tag_data = "object {}\ntype commit\ntag {}\n".format(commit_ref, tag) tag_file.write(tag_data.encode("ascii")) tag_file.seek(0, 0) _, tag_ref = self.source.check_output( [ self.source.host_git, "hash-object", "-w", "-t", "tag", "--stdin", ], stdin=tag_file, fail="Failed to add tag object {}".format(tag), cwd=fullpath, ) self.source.call( [self.source.host_git, "tag", tag, tag_ref.strip()], fail="Failed to tag: {}".format(tag), cwd=fullpath, ) else: self.source.call( [self.source.host_git, "tag", tag, commit_ref], fail="Failed to tag: {}".format(tag), cwd=fullpath, ) with open(os.path.join(fullpath, ".git", "HEAD"), "w", encoding="utf-8") as head: self.source.call( [self.source.host_git, "rev-parse", self.ref], stdout=head, fail="Failed to parse commit {}".format(self.ref), cwd=self.mirror, ) class GitSource(Source): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" def configure(self, node): ref = node.get_str("ref", None) config_keys = [ "url", "track", "ref", "submodules", "checkout-submodules", "ref-format", "track-tags", "tags", ] node.validate_keys(config_keys + Source.COMMON_CONFIG_KEYS) tags_node = node.get_sequence("tags", []) for tag_node in tags_node: tag_node.validate_keys(["tag", "commit", "annotated"]) tags = self._load_tags(node) self.track_tags = node.get_bool("track-tags", default=False) self.original_url = node.get_str("url") self.mirror = GitMirror(self, "", self.original_url, ref, tags=tags, primary=True) self.tracking = node.get_str("track", None) self.ref_format = node.get_enum("ref-format", _RefFormat, _RefFormat.SHA1) # At this point we now know if the source has a ref and/or a track. # If it is missing both then we will be unable to track or build. if self.mirror.ref is None and self.tracking is None: raise SourceError( "{}: Git sources require a ref and/or track".format(self), reason="missing-track-and-ref", ) self.checkout_submodules = node.get_bool("checkout-submodules", default=True) # Parse a dict of submodule overrides, stored in the submodule_overrides # and submodule_checkout_overrides dictionaries. self.submodule_overrides = {} self.submodule_checkout_overrides = {} modules = node.get_mapping("submodules", {}) for path in modules.keys(): submodule = modules.get_mapping(path) url = submodule.get_str("url", None) # Make sure to mark all URLs that are specified in the configuration if url: self.mark_download_url(url, primary=False) self.submodule_overrides[path] = url if "checkout" in submodule: checkout = submodule.get_bool("checkout") self.submodule_checkout_overrides[path] = checkout self.mark_download_url(self.original_url) def preflight(self): # Check if git is installed, get the binary at the same time self.host_git = utils.get_host_tool("git") rc, version_str = self.check_output([self.host_git, "--version"]) # e.g. on Git for Windows we get "git version 2.21.0.windows.1". # e.g. on Mac via Homebrew we get "git version 2.19.0". if rc == 0: self.host_git_version = tuple(int(x) for x in version_str.split(" ")[2].split(".")[:3]) else: self.host_git_version = None def get_unique_key(self): ref = self.mirror.ref if ref is not None: # Strip any (arbitary) tag information, leaving just the commit ID ref = _strip_tag(ref) # Here we want to encode the local name of the repository and # the ref, if the user changes the alias to fetch the same sources # from another location, it should not affect the cache key. key = [self.original_url, ref] if self.mirror.tags: tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags} key.append({"tags": tags}) # Only modify the cache key with checkout_submodules if it's something # other than the default behaviour. if self.checkout_submodules is False: key.append({"checkout_submodules": self.checkout_submodules}) # We want the cache key to change if the source was # configured differently, and submodules count. if self.submodule_overrides: key.append(self.submodule_overrides) if self.submodule_checkout_overrides: key.append({"submodule_checkout_overrides": self.submodule_checkout_overrides}) return key def is_resolved(self): return self.mirror.ref is not None def is_cached(self): return self._have_all_refs() def load_ref(self, node): self.mirror.ref = node.get_str("ref", None) self.mirror.tags = self._load_tags(node) def get_ref(self): if self.mirror.ref is None: return None return self.mirror.ref, self.mirror.tags def set_ref(self, ref, node): if not ref: self.mirror.ref = None if "ref" in node: del node["ref"] self.mirror.tags = [] if "tags" in node: del node["tags"] else: actual_ref, tags = ref node["ref"] = self.mirror.ref = actual_ref self.mirror.tags = tags if tags: node["tags"] = [] for tag, commit_ref, annotated in tags: data = { "tag": tag, "commit": commit_ref, "annotated": annotated, } node["tags"].append(data) else: if "tags" in node: del node["tags"] def track(self): # pylint: disable=arguments-differ # If self.tracking is not specified it's not an error, just silently return if not self.tracking: # Is there a better way to check if a ref is given. if self.mirror.ref is None: detail = "Without a tracking branch ref can not be updated. Please " + "provide a ref or a track." raise SourceError( "{}: No track or ref".format(self), detail=detail, reason="track-attempt-no-track", ) return None # Resolve the URL for the message resolved_url = self.translate_url(self.mirror.url) with self.timed_activity( "Tracking {} from {}".format(self.tracking, resolved_url), silent_nested=True, ): self.mirror._fetch(resolved_url, fetch_all=True) ref = self.mirror.to_commit(self.tracking) tags = self.mirror.reachable_tags(ref) if self.track_tags else [] if self.ref_format == _RefFormat.GIT_DESCRIBE: ref = self.mirror.describe(ref) return ref, tags def init_workspace(self, directory): with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True): self.mirror.init_workspace(directory) for mirror in self._recurse_submodules(configure=True): mirror.init_workspace(directory) def stage(self, directory): # Stage the main repo in the specified directory # with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True): self.mirror.stage(directory) for mirror in self._recurse_submodules(configure=True): mirror.stage(directory) def get_source_fetchers(self): self.mirror.mark_download_url(self.mirror.url) yield self.mirror # _recurse_submodules only iterates those which are known at the current # cached state - but fetch is called on each result as we go, so this will # yield all configured submodules for submodule in self._recurse_submodules(configure=True): submodule.mark_download_url(submodule.url) yield submodule def validate_cache(self): discovered_submodules = {} unlisted_submodules = [] invalid_submodules = [] for submodule in self._recurse_submodules(configure=False): discovered_submodules[submodule.path] = submodule.url if self._ignoring_submodule(submodule.path): continue if submodule.path not in self.submodule_overrides: unlisted_submodules.append((submodule.path, submodule.url)) # Warn about submodules which are explicitly configured but do not exist for path, url in self.submodule_overrides.items(): if path not in discovered_submodules: invalid_submodules.append((path, url)) if invalid_submodules: detail = [] for path, url in invalid_submodules: detail.append(" Submodule URL '{}' at path '{}'".format(url, path)) self.warn( "{}: Invalid submodules specified".format(self), warning_token=WARN_INVALID_SUBMODULE, detail="The following submodules are specified in the source " "description but do not exist according to the repository\n\n" + "\n".join(detail), ) # Warn about submodules which exist but have not been explicitly configured if unlisted_submodules: detail = [] for path, url in unlisted_submodules: detail.append(" Submodule URL '{}' at path '{}'".format(url, path)) self.warn( "{}: Unlisted submodules exist".format(self), warning_token=WARN_UNLISTED_SUBMODULE, detail="The following submodules exist but are not specified " + "in the source description\n\n" + "\n".join(detail), ) # Assert that the ref exists in the track tag/branch, if track has been specified. # Also don't do this check if an exact tag ref is given, as we probably didn't fetch # any branch refs ref_in_track = False if not re.match(EXACT_TAG_PATTERN, self.mirror.ref) and self.tracking: _, branch = self.check_output( [ self.host_git, "branch", "--list", self.tracking, "--contains", self.mirror.ref, ], cwd=self.mirror.mirror, ) if branch: ref_in_track = True else: _, tag = self.check_output( [ self.host_git, "tag", "--list", self.tracking, "--contains", self.mirror.ref, ], cwd=self.mirror.mirror, ) if tag: ref_in_track = True if not ref_in_track: detail = ( "The ref provided for the element does not exist locally " + "in the provided track branch / tag '{}'.\n".format(self.tracking) + "You may wish to track the element to update the ref from '{}' ".format(self.tracking) + "with `bst source track`,\n" + "or examine the upstream at '{}' for the specific ref.".format(self.mirror.url) ) self.warn( "{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n".format( self, self.mirror.ref, self.tracking, self.mirror.url ), detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK, ) ########################################################### # Local Functions # ########################################################### def _have_all_refs(self): return self.mirror.has_ref() and all( submodule.has_ref() for submodule in self._recurse_submodules(configure=True) ) # _configure_submodules(): # # Args: # submodules: An iterator of GitMirror (or similar) objects for submodules # # Returns: # An iterator through `submodules` but filtered of any ignored submodules # and modified to use any custom URLs configured in the source # def _configure_submodules(self, submodules): for submodule in submodules: if self._ignoring_submodule(submodule.path): continue # Allow configuration to override the upstream location of the submodules. submodule.url = self.submodule_overrides.get(submodule.path, submodule.url) yield submodule # _recurse_submodules(): # # Recursively iterates through GitMirrors for submodules of the main repo. Only # submodules that are cached are recursed into - but this is decided at # iteration time, so you can fetch in a for loop over this function to fetch # all submodules. # # Args: # configure (bool): Whether to apply the 'submodule' config while recursing # (URL changing and 'checkout' overrides) # def _recurse_submodules(self, configure): def recurse(mirror): submodules = mirror.get_submodule_mirrors() if configure: submodules = self._configure_submodules(submodules) for submodule in submodules: yield submodule if submodule.has_ref(): yield from recurse(submodule) yield from recurse(self.mirror) def _load_tags(self, node): tags = [] tags_node = node.get_sequence("tags", []) for tag_node in tags_node: tag = tag_node.get_str("tag") commit_ref = tag_node.get_str("commit") annotated = tag_node.get_bool("annotated") tags.append((tag, commit_ref, annotated)) return tags # _ignoring_submodule(): # # Args: # path (str): The path of a submodule in the superproject # # Returns: # (bool): Whether to not clone/checkout this submodule # def _ignoring_submodule(self, path): return not self.submodule_checkout_overrides.get(path, self.checkout_submodules) # Plugin entry point def setup(): return GitSource apache-buildstream-27ae392/tests/plugins/sample-plugins/src/sample_plugins/sources/sample.py000066400000000000000000000007041514607367700325350ustar00rootroot00000000000000from buildstream import Source class Sample(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False # Plugin entry point def setup(): return Sample apache-buildstream-27ae392/tests/plugins/shadow.py000066400000000000000000000040411514607367700223030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name # # This test case ensures that user provided plugins appropriately shadow # core defined plugins, which is a behavior that is required in order to # make it safe for BuildStream to add more plugins in the future without # stomping on plugin namespace. # import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream import _yaml DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "shadow") def update_project(project_path, updated_configuration): project_conf_path = os.path.join(project_path, "project.conf") project_conf = _yaml.roundtrip_load(project_conf_path) project_conf.update(updated_configuration) _yaml.roundtrip_dump(project_conf, project_conf_path) # # Run the test with and without shadowing the "manual" plugin. # @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("shadow", [True, False], ids=["shadowed", "not-shadowed"]) def test_manual(cli, datafiles, shadow): project = str(datafiles) if shadow: update_project(project, {"plugins": [{"origin": "local", "path": "plugins", "elements": ["manual"]}]}) result = cli.run(project=project, args=["show", "manual.bst"]) result.assert_success() if shadow: assert "This is an overridden manual element" in result.stderr else: assert "This is an overridden manual element" not in result.stderr apache-buildstream-27ae392/tests/plugins/shadow/000077500000000000000000000000001514607367700217325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/shadow/elements/000077500000000000000000000000001514607367700235465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/shadow/elements/manual.bst000066400000000000000000000000151514607367700255310ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/plugins/shadow/plugins/000077500000000000000000000000001514607367700234135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/plugins/shadow/plugins/manual.py000066400000000000000000000004751514607367700252500ustar00rootroot00000000000000from buildstream import Element class Manual(Element): BST_MIN_VERSION = "2.0" def configure(self, node): self.info("This is an overridden manual element") def preflight(self): pass def get_unique_key(self): return {} # Plugin entry point def setup(): return Manual apache-buildstream-27ae392/tests/plugins/shadow/project.conf000066400000000000000000000002751514607367700242530ustar00rootroot00000000000000# This project.conf gets rewritten for each plugin loading test name: test # Required BuildStream version min-version: 2.0 # Subdirectory where elements are stored element-path: elements apache-buildstream-27ae392/tests/remotecache/000077500000000000000000000000001514607367700212435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/__init__.py000066400000000000000000000000001514607367700233420ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/000077500000000000000000000000001514607367700227115ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/elements/000077500000000000000000000000001514607367700245255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/elements/autotools/000077500000000000000000000000001514607367700265565ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/elements/autotools/amhello.bst000066400000000000000000000006161514607367700307140ustar00rootroot00000000000000kind: autotools description: Autotools test depends: - base.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 config: configure-commands: - | %{autogen} - | %{configure} - | date +%s > config-time build-commands: - | %{make} - | date +%s > build-time apache-buildstream-27ae392/tests/remotecache/project/elements/base.bst000066400000000000000000000001031514607367700261430ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/remotecache/project/elements/base/000077500000000000000000000000001514607367700254375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/elements/base/base-alpine.bst000066400000000000000000000010061514607367700303260ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/remotecache/project/elements/build-shell/000077500000000000000000000000001514607367700267315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/elements/build-shell/buildtree-fail.bst000066400000000000000000000004641514607367700323370ustar00rootroot00000000000000kind: manual description: | Puts a file in the build tree so that build tree caching and staging can be tested, then deliberately failing to build so we can check the output. depends: - filename: base.bst type: build config: build-commands: - "echo 'Hi' > %{build-root}/test" - "false" apache-buildstream-27ae392/tests/remotecache/project/elements/build-shell/buildtree.bst000066400000000000000000000003451514607367700314240ustar00rootroot00000000000000kind: manual description: | Puts a file in the build tree so that build tree caching and staging can be tested. depends: - filename: base.bst type: build config: build-commands: - "echo 'Hi' > %{build-root}/test" apache-buildstream-27ae392/tests/remotecache/project/elements/no-runtime-deps.bst000066400000000000000000000002561514607367700302700ustar00rootroot00000000000000kind: manual description: Test element without runtime dependencies build-depends: - autotools/amhello.bst config: install-commands: - echo Test > %{install-root}/test apache-buildstream-27ae392/tests/remotecache/project/files/000077500000000000000000000000001514607367700240135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/files/amhello.tar.gz000066400000000000000000000336171514607367700265750ustar00rootroot00000000000000‹ì\k{ÛƱö×òWli=©’”¨‹Ë•Ÿ2%«ÑíˆTmÉ¥!`I¢Â…ÁE2ëø¿Ÿwfw¢m¥'Î9y¦‰(`wvv®ïÌ®jéûÑÚ£¯øYÇçéö6ýl?Ý^/ÿ4ŸGí­ÍÍ'ÛOž´7­·7¶¶6‰í¯É”ùdIjÇB1‚±âÕáàåéÅ@tNÞˆWóóÎÉàMƒ¶@óåûT†©Ú[Jt¡ß¾{nX y+ÕZ^0õ= ¨b;Lg"‰ãÞy÷%(v~8<:¼¡MïNzý¾Ø?=Ì?뜻Gsqvq~vÚïasÁÖÐ…®Õ.;Ýa÷ôdÿð`xÜ鞟÷ÏûoâCð> k—ÃÎñò!oëÅ O‘Y:¹¶ò·ú[ü³ÀGçbpJc‡ÿè÷OOÞ6¿ˆÒv¾ˆ–Kç´úŠzMñ._¡ ­'ÏÞ6Z ½p\»Ls˽ƒÞC™Ò6þA3[•7Q&&ö-A׊ŠÅ­ŒÙN¡ 3@ö†hÏÄ]ß4Ø2”é‰qf“>%ˆ§Q«r83CÆwíË6Khv¨†! –Ø6®3ÏwE2KRX”K†$ 7‚Ë5È&y (:ÒÍàlnäd†*£ãw À8gS,â?–´«Å»PIl<¬•ð{°d²ŸãιÖôÏ:^7?ó¡Ù/½n½À_ŽÔ˶G´&¤˜"GÃÉÛ/,0Ž(ê' ¦v8ƒåέCý›šcÞ-‹Z,`/d}ˆW¤jÐ…dÏ•"·üV½—Ùëí_œÀyîoîyiá=õ†Úäw-J"VÞ5 c¢†¢n UìQƽ!])ç‚ÃÁ#¦ËŸ2Ø2¬´/ô‚,0î—:‹Pñ‘,P•]ò.[Úÿ/WÚ'ˆ1ZTèCñ`¿3èÕ.÷"– B¬¬+o»‡'‡ƒœg¦So™Ø¢\a¨7UŽ)µ 2 “z—ÚKÍ@mŽ7fçÆ“ÑÏ«6"æ"!SP‘Ï !?N#O&˜J$ýhì9”Å4 q7ñœ DªÁ¨¾Ç!\_ñÄd¼”×ΩªÌp?hSbÐþÒï †Ý‹óóÞÉàžò¿$P—Tò)ËYöÃj“[-ü®Jè™’õéRÉâD6JU<*$ÀY¸¼Úê­{ö5ÁŸ 2­J!‡bHùE&ûžïÙñ Zd˜µ¨o˜#â—a²fGÔ Šb9¢0S–Ä}‡rµnœXz,û:‰|Òx¾’n%)Ñ•Sº$th|Ý Õ$¤\Šeã,l‰<\BoiJöF”-v¡#AÄ£¬q;*3‚äƒå²0KX—Ivm¶ˆ`ºcê€4š‚F¡Â€y" ·¯á—ÄIn”d²:¸±}wÂ2w?•ó¡å8,ßR#Ä.ï» ¤¯‚_ÝJ7ÏÓª$¥Q¶¥‹•JÒ0<ËÁ^¹UaÆafçâh ÀÔcÑûÄç¹nUJöýÎqŒù»ïÄôÎ}WÉqÖÜ;|uŽ ²„³ÚÏž=ý6pÖ¼k'ã^C°£7óç_,»JÕÅž #6ÙœËå dÌ™Ri=mGgç=˜Rír£µ½¡,FÌšƒó‹~–ëË•uB ·¶ï¹ÅR;‚ ʼzú~ç¨ßÓxÿÓ™ í_üÐ C^ÿþcE7/øÅðät`jÛÒ¼åïKJ@’Ñð£‹ÞPíaE †`ÁÛë5vÕW&·k=¶*ÒG -^Ó£ÒˆÊÈ«Û==>†K÷I µÊ%H%€kóßð¾šÂGv¾ùçLëc5çûÃÞùù)ùeIÿ˜Ð®rÓOÁ^µQØß…JîºHGHOT†Ën…T5‡·HänÙ¢üYë-¢ÀÈ[Þ@ƒû>ûÜWa4èl1’w„”Æ'H”|@wŠ³tvÄ‘¦”(,ö¸Sh¾ka 1S«<äl \7è›é^:Ò%MC¡©XC—¢­¦v„ÔOÐè†" |‹Uä¼Õ†:ª‘PטJ|÷R‹ÐFÐu:Eê' !É5OÇO¸—Tœ KÁoÄY‚š¦A *Ýí ™:-SG˜{V µ¬Ü¨žX( 1Pfnµ”@Éûözg½“½ÞI÷°×çøéHQJ®‘h5êb±˜$5ƒ]LÉЙ g"°IP…èò-‰n·ÚÀ_¿¦§?ü½k~ª'gô,WðŠf¾¢&ÌL+<•Î$ô~ÊHÜ¡«q‰Mu,—³6•ccŸZ’X<ÇAÊô_QÏ:´ÒRãFï"†•M#U–äÄ¥]¹¸c˜¨ŒWµÑ•·Þc·[^ MÕÜŽ;@t^r’””¹o…z+Õõ4x ÕîÚga©ÇGvHÕ43ª] ÞËIh‘¹{pF5§0P&üåקƒ³‹AAäMF—çÖáI÷èb¯·t!Ðvº?ª·ó}Ön—’Ÿ¸„ˆg·ºƒQ¥€±¶¡ð²[´aõ´×¯9iæÓ`O˜F–G?Í4¶ÈûÓ¬±ãl üÇZF€—ž#A‹ÿ"°ú9æÉ ÀüA÷ïsÓðû²iKÖÿä(s-[·Îø¸Ûé¾ì ño÷ÇÚeÉÏ“tæSY!V˜ÎýE¨ËïÜ"·óYžLŠEÒÙT’UÎeãα2“‹^)$D†&阽›ûP¦#Àn媦>ÍMTHFˆ=¥/wBÕº’"É”©€ëhœ%úp„òÎÄŽ›±Ï¾¦$F ¨Kª.@z?Š™ŒÒ%‡Ï$(³Ç’=ù0ƒ§¾ õëOêžs9‹Ê¯ª N¯‚Q¥0®2”Э¦cq¶š‘Å2%5$ tK¿êÚÉ$HRQ¹ªiªë5tO;¡€] ñ~ž¡<ïþå/…¹‘TÐ|G(‚}cT,ís>X_û»Âbuža«V…^CYßa˜J_ÓÐ'ôdVÔ¾Œ2>BcKTy$?ôÑVHÒA$ßÛÊ€ÐÞëA3ú˜ ¼0Kxhµqä _”8Z­…™Ôs¨S|5€#FÂm/-:%'Öò¦¤ª§i¡ˆp”8R‹Ÿ&¥Âå2©Çë¸F”Uv,8éŒR©ÏS{Üðq x/ n ð”'Ö¦šÚ0gú–ïçTØæÕ];W¹=ê,d/Åsº5"é¤Ìl OmïWdH¡H‡³!] ¨F;z?“‰ÎsZØw©þt Ÿ»ÑO8w•ö$·ÎSñ³ú$³D}÷¨oäL§ú ½XÐK®ÄÆqtC¾ ¬`„ÍÆÆÝ&²‚Hífb'9U—3™ªÆ{"º•\}Šµ2Œ²ñĈ1ÐhèÎÐù‘}“»ôè|ˆ4wmÈ ¦¶S²±äOZñ¢/ZK(ë‚åkQÞ¬hýéÍìæV~eª• çÈÑèh×øI1ó)Ùì.Ä)‘Ó¥¯x—O^áfsa @€²Væ6·Ï B¼ÀäÛµ0¤Ûx‘‘qŒ2ÉŒ£âX é‹ ¿eBßµ?3ýÉ0=—Ü/˜¸òzM*ä>ŠfBõžŠ÷÷&•Á$áyÈÂ2™? | FT­…"šê>£¹cËM/I2ºœÉ÷5 Gà»ß$¢ÆWg9ø6_A¾Q\‡o¨3R%¡Ô%±«*™$öXæ¤Ø :ó² \ ª‹Q|K® ë…ÂÖ( ©žPóø¬ÁÛ{ÚZçŒò´Õf‡÷mvÃ,ñg;ùDACwP¼hÅlqG­Ë×/HTÖs:_°ã«œæºœ[ /T±Ùc ň3´cuy™8ú¾µþù•• wJëYó[.âk%j-ðh‰9ÿùçRkAM˜#7?¼>ï*ÆhÄÎs¡ ò¨òÅÇ„|Ò’@Î?è¨Òµõ©¢[¯þàÚ „ßÖçNcözgǧ{=ÝvRí!MP™?ã ˜#›,yªê¤ŠxP®û3’]¨"›éÎTÂeIêG*iŠk~ªÿ¸Øz¦w^1äµ-õAL-dª®ò‹zêTeá–ìv±ºä–žn„.ï’õ:{‡'ýÓÁâ™™'.5¼Õ—†n”~£Iµ¸ù©¶÷Av=ÏSÑ-U,uΆ½“ÎG½r®iTÃ6AŒõ‡È5gú¯©ö¼l6•F›K§ÑW]d%%?‘ЉJIÄò=†2¤eô]/ùüÉ”/$¡2…a7S/ÐW`øhë0K™^žŠþÑnùݤXB¸ÜKü"ìuú/w­+Kw BxÎt×ÂO‹Ï œƒç‘@ÎÖœ£Î¯ûÉÃÑeà "9ŸŸœ¿0B˜ûý¡}@¹—ôixo‰¹Ïƒn |+GŽÃÏž|ñ®ÀÂ9ÌgNRàܸwêU àÒP¨É"4©Z÷:néæ5!|òZ.c8 2Fí_ø3BPD î#™Zتo£R²Gꦹ$R®þ¶ÄU³”/såå«d–ø\ÏTŽúp}ÿð¨×WåãꕵZglwï‘ʃª ¢cˆÐïçið[ ß“‰7Juï •¿(µ¸ø~Ùñ¾ª;tó¼è¿p–Ð…?ï[UY£Ýwº Œª(ª¸óˆ²=YÛi­®¬­Yï4ù®ê,L$ßl2wôè,Õ\z-þ¨Â¼jPg:D¯t… k ìÔ™(!†ÓÐÏÿï1¼PUlz:·¦2šú¦x?Ô§zN¦^4ý-í]ÆGÁ§Ì•ž0ÃÅ’E J ˆËŸ™F®êvæV›FIÚä?öIL3k%NÙhlÿŠPÞ)Šì˜S:ff(ôEKÙ¹›D¾ÌÃÿ=S¹úÊî÷ÐèÑU8›É{“ˆ ´ÒÆúÖ÷¦Ì%ks-ºþ«NjøäEÆÔ¸w‹&ÝÖúúzËÔ†º÷l%>n­J¥¿m4мÚøŸö®´;,ÉÖWó+²] Õ€Ûe—Êr[eɶ¦eI#¼ÕH*Œ …òH†Kš.ÿ÷‰oÉ$Á®¥ûœ>ÇÇ‚Ì·¯ñ"nÜ÷®>í á2½+0U*¤=hÜ™L·ÞÓNI‚,àUŽ^“æÉ UJ“€‘ñv«zã@*‹¨Q<%š¨›¬®L“°k;>-n6â3yÈžÙÒ’¬ÕÅC/Hf[]¥Ú?’úKù©N±™.S¯x™D½çŸIÙ YœkÉi¸&H?Àì&É<ã¬ÝóFÔìè¥.«˜3g¥åòÕ< OP<» jADýë,þ¢}¹ Mõg…8IÄEÞʦäÎ'™ÃJYÃ+ÁN‹oìî‘L«F°ìó'zÞÿ‡‹í™ù ‹&g6ÛL’ÄfÝKVä Õz$nô†ˆ”ÈM2›X†ŽÝ&SQ»AgHiü$^ ¡z¤‘Žœ:Áyܧ! 00»¡Gêpà¡Ë€TÌ0)xX2 Ó~Ú>ÌøKºè4Ò|ÿ<ó´V:ÉÅwøñ¬7ª:Öõè£CîÁaƒfÒÞÁîY­(ÆÉáÎÅ­³/FHç–(ñ–.ÖìYh±.™öl#›dÀ'à" úµ:žDÖ%¾±Ñ¼¿V«;Ûñ90‘Á˜$EñF†ƒ¯d£> ÖÇÓØ!®°BŽ;©ŠèÔ¨2‡L8n¨Ï+Áæ<=YÐ7¸Æ…pX.{j ‘´˜Ñ=×c6ci£èzê“"jä9ú¤Ä¯Øª,…dW#Ó^P¹vŒÇ˜óuAhÿ^½U1æmé¯:("  qJ S£S:¾¸äYÞø=G粌¢TênèŠgØј£6®žüBÉl7þçìoÏö·Ÿ·*…Ã¥î·ótFí´^mïï[0º¨à«a™ÔKƒ·ˆò{þêÒ^Së6ö*UãÏÃK o§îElª‰²#>„ašo©­9 fæ,@YÄG”rc¯É®Œº(¦Ä¡¶ ýÜóÚÝO˜-PIÖE|8RO·€Ð7žwMÉÙEIX3ŽWJ°FÇ­é,õÎCYÀ|ÞÓ± ùÙYæqmXíQY}øËò–è_I‹më4høè×½éÃÕ¦”A>ýù9ŒÚo]WÁ¡aiiS#|Î~`¿{µ²Im…6ú”‹öpçB@|´výeèþ/¶ôÀÄ!D-¿F“çÇNÌ(•Yœ^Ò"6½ŠhPÑÜmS°‰Wú‹Ýn•Î€UøWÈÔÝÙÛ~~pØ¢qÀ5)šFyô&;L¯’†®,´ë72J(µ»Ó:C}‰1Ù4¼,’Õ]Ãâ‚)([Hõd”4dO3(ÞÖ5›ý@ÖçÞzíðA=²‰·ª6š´[¯"¯qÌ2Ï3ɵ’ï÷†ÿ¡ˆŽq¾7®õ8o ɇ³Úfa¯¬Ä ðäÃ&}¯‡ý‹Î­dŠ¤;“#Õà2=`¸al,oÁÐ+Ϋm>#…ËÚ4œ/~hr`ÕóÞ³‚N¬ëhbé¡ýúà¿_¾ÚÝɤ\Ñeü:PÕœÖ 3¹(¾W°rEpü7Ú £Ù𜦻Ÿ’–uEä;ø™§noewèæÜæ°Mmÿ³ › 7˜\”BVß¿l«Ÿ'?G ýçpñ NÔQ¤!F Ü䓘ßr•Ycê–¹(œì¹Õù Y¼ØÝÞÙ=6‘/i­&Ea‘ÅÞÁ³Cfp¢,@à'& ¿ÁüͱíY,‡yu¼w´xŸÕºñŸÔ–Ö4à|àúIŒ›¼IT>´"âhÉ^FJãÓ­íåõæÍkxY$‹S$0¨Æ;„_RîOplawx?ºœNÇ›«« llöG³f2鯡IšÕËép°jš~•ŽQµ«Ã´ÏütM¼~ü‡Y¿§‰¸™ÉXÜö˜gf¥ªÍ„¼äÀ«VÄ,óy†vI`5§Ì’UnŸ«ÚùÕöÉzÀ`ÕçOq¡}*SBe–£;,/Š:xЙò"ß,ì¸í·ÅFz+)-Ì ²9cjn¡Z4˜xÏl!œ,-cÕyŒåÿŽîSqÇëù˜xÈñîÜqÝë΄Z¬h±sFm³äQÄ7{;´a<ÛÝoíæNhÈaÎý¦ $rͼ]t¸3¥+<ä-JÓys.,#”•‹^.-˜F^T2D_˜ìçË&®8‹ Ço—•ÎF_PÀ´ 3’±ÏÈ»ÐwM…à,¼/(ïc Ã0'ZT s §a!—‰j$)ë6ë䲑£Îdp˲8u6ùhåñ$ùHÂ2;‘”FEÀó^µ~Ð’WäYÙ53­G‡­½w‚ƒ)‹}K.fS(²<Ƨ7cT)õsQV Ó¬DOzø 6gcÃÎr.¶2=èÃÝ3MgCNA1:›X+Þ»ñ˜OЭĒzôû“ˆ{eþ‹Äš#¦çYAÒä§Ì!¦ 2 £ ÊŠð¤÷ʆ» ÛÆSõŽ¡|˜Þ f7 Îgý•õµ‡…£50N v¼ë~Œ±EuÁs&¦ÑëŽiˆ­ÀG“¤3°sâ„“aè¨5™ÍÂâÿÔÐJÍ%®“¢;äCè6àZÉ,eSt'&–µ:ÇþeÇEîAïùËÄ|›\w£óèÜ<þv#xô(¤ù–“qú—ˆ'Ù¢Ò˜B3j:G\£êóñ8šÐØ ÙCEg7ÓR22^ YWVOqW@s4.Bƒ²3ºÐÑÔ£-‚Ùdҙܔ”æî<¢ãqŒâ‘èbz2èðK\<Ó[´ô@Ì8šÑÇx’Œ`kû`q?•L÷ÔUÝÀ"áñþèË«Ôæå±—O&7MO[f…‚I'fKÚ(º@ˆï^ú´Ì_ âs†FÔqààdŠÏg¢+JDë² )FhtR´œA}Á©tÔ§cž5“­"bþiì6ß?¼·Þd’yL ±",+B,P.Ð>ŒØ¤î çA ,žR ‘}2¢ŽØÑtYL %UçX.&X>Uê!0D8÷ël¤>ª`VÏdÝ)ñWæ\~¾¨èuÍŽŠ%#%a¼yFsk¡M@݈º ±iþfãž!¦ÐÃqãRñU¾Õö ¸'Ž¸â…5Ìè{So‚i%…ÌÇ_²²@q6†­^>¬÷é•—Ã^|q±¬Øæåì' D 92iQˆ´_þ#¨òFªD-ÒA ’dŒMvåǘ´Q){ZWV5SzJ‚7-¯ðâ„žA.(4›0àÇLÓ¶¢){Ñ5û}JqÚšw³Ô•Ô¤¿UY篜c»›ÌFÓ­õ”x*¡ âÏ( AóeÂòJSÑôƒ_í×ÍïŸÈøܸÇù‚c[%÷8ø[°þÞÃû1nBà@kÛ%›q9xœ|˜gµ÷«:OÎòù, žþJFçuv…6\g¨èf܆¯gƘ×ýÍãäsìiã HÿtIÿ¤˜O¯ oV*’aèØÑ ¤”ׯôçÎw2Ä\:>a_è3öiFzZÈé’¸ÅQËflfì6.……¼æw¿ŽÁ•ÇÉ*ârÀtâ¨TC5ŒzÀ”šò:m ѱ ‘½dš#[^¶;‡-s¾+ðØðõˆ4Õå¦9¥ñæ™íJBl0÷ØÚ{üÎ÷+ð]8¶šžßNæEc2,L¦¾LrvAø5ü£ª'ZŠyÙZóøc‚u öi¹€›…ŒPý…ôÙÞi– Aɱ§Ô‡JÌÑQ3~Q¶DÝFœ6˜©Wþ/ûâš×7[.JÙ“Ûü%«A{»}L=êË0H»“x,®IˆH€L“«;6üE4GŽâBžª†7(õbÌö­çÖ 6 ZÏ÷w±}9 iÞ§2“J<-ùgûlx¯}>&ovžNi!©''¿Xj±öãŒ,žC©-+Û¾¦¦³hc±»0Ô‚yþ7ÊË'ŠTf»túŠ×€§ÅXødzUr…8“kÅT 8ßTŒuÃíDÅuhUõï²µœ^i(¹(wºtf‹Æqƒ’›«Å×(¡ÆâR·´Ø0²tº—í+õ~åx ì›Gº 5¿6„!hÜ?kP4J€¾¾j0Æä·ÒKp¯‹ºÌ’öI_ïFál4u·¼çël §5Ž/(ê’~ò2(ò«¦r^É{9Ìw=ÙûœÃ½ œøys—žúƼ…ÒYXÝ\,(•óžägv¹loz9ý‘ÝY /lˆýmHE¯[/ª'OíCŸMø£?©€¡Ùz¬¦ ÂŽØÀÕ\}"xáÚ€ÀÛîvÛÝv¢0Í åW»LB` R]õ„Ç^ÖÌ8 8oØyûå`”~ˆ£bÐå ½Ѓa'K—,ÝÃÇÕÀ®æÜ(ãø–g:ì‚q‰GÉJÖk7Í$°n«ÔuA¨³™^ü4ÙVªT€tmZ^¯e¶nØ%ž‡Ë°mxö¬g"Í|ê]’ÈÚû€@¢â®cl›4‰}¶ÁAuMÇŸœÉcª- œq„ ï <_ö‘qŸtŒ3Žh&ÌImÈð;z<©¼û¼•¹äYFt¤ešw?I!Ä– ‹“°Þõ"$5% ;üm‚C×¢L°0G£t(ø¨9Ü¥bO§aU`àe»a¼£Ý&Áµs°¡2»/ÀšÊ6/1Ѥ1¥e‹ãÚ^cÍAÖJÈÈ:nRŸã&„຤”q–á4´Õ(<¤ØÌ•3Ú=~ú´ÆÇa„ò%\¤Õz–©wxÄS_®ÉB\«]Ú‡O=­Y“‚üE$øÕÛ7ÌüU„Øy€žÓÉMÑÐÆhQ5Â4Pó_kØ–b…$}*°»Ó\¬tz±½TÓHô¡µDz›dî2L}É´†³“¶‡Ñfeï`÷à÷s”ƒÇßÂy·Õ ŸÂ·Ÿ±ÎGâ¢w`!(Œ`ë³Uù»Õ5ÍåxZù{°Å÷ÖHàÅùss¹€µƒO9}öÐóõVð™P6ûxˆZ‘8¶Þú¯fƳï”P e ¾ƒ.Y­¯.öËÏÇBûD{8aHÊývi˜ŒÕLæö&#ˆ l·1¦}ørw†èÒmF ÊqB–Úι¹ä„‘kø¤ÁÅVX U£ 7:UMœœœžž–OWN+§ßž†§¬÷\œ©Æ! Ñ‚/.—ì¬f·¢1«-Éï48½³s]ÃJð‚Ï~J–«ˆìIÄv㣽CÝsÕ‰e´Úd2`;Hs=‰±Ç'gïsƒnÁÁæ+Ñ“åœzÀ2ªWM.ÕwB½)N|‰,ÞÏá-âßd6ÀßnóÆ·B.„4lôŒCV^ ?¡zP%eT/Ž§7B_Öòß)ižÂ4hÚàB} wi6<^TQÑUêËdSIÞl­—kŸ#.OýK^+;—,*•(T’ksrX‘ŸüuóIP­"SjÚšÀrsÃÛú§Ã‰ñ[kbO% Þ¯¨å‚÷™ë´n¤U×ÎÄH`ÄQËÒÁת)æ(‹­v³ß¬ɨV¿ÃoÖÑõ´5Æ5N@c ùɈyóÎÝâ«S¡-˜ƒpÿðÃÆCNV¼‡6î>àåuf“S™:}©ABXtY® Jí¤ Smkж„lçCB!Øjï>}qX=9 ™Å©Rýiû¸R}S«•èËš^2C_×ùª– ® â§RE´Ú°'<;«EkátÔ]¸=|A‰yûÎðI/N²õ…¯¬|A¤ «!D7ʵ"aåMhy(i; Ñ@¡È$i°w¼÷ŽµØh ¾Ù’÷òË[<-Æ5{±'Ç­…É'U(ƒIAbÖ‡Ž‡×¾óÃÛ—ġ)ŒoJdÓÉRg"} kp>ÐW¯œçX´KÂæ9ŠÌ%(ÀLj–ôI*®7Túÿç¯köîéq[ áÈ)zâUoÙíØ:™ ,’(;Äí,;¦öbYcŽ/­ø=Èî Üa±ç™šÚ ’È|ÒÁ«´Bb—ÁÖÀN\œ¶—¼Ô}BS©UŠ„ˆ›sÀ2Ý©6›Õs;¶tlæíW‡‡ûÕíÂà„›Œ¾lò±/ÓÃÆŠÉÖÁ¯Áb×HËÞZm¡ä+Ñ+Ïo#o¶÷ ˆ?Ao&X=¢:Vʈ@̶Z“ ’'æûŒi4zþ‚pˆ:ÿ8‘r¸¿Àø{ªZÌÎq7Heïç¹òÇ—Lîµ’ɼò$æ¿”{¨Õ(ÝÜöœ†éRöÎ/„Ýûz©%dyvxürûÕÖÒ6Çèƒß:íÝ’oƒüÍãÚ‚©âëÇa=™Z&äK¨Ãqç:”m@Ogœº]L+Uö¾f];ÆS¿$WÇb —;çè­äÞ0tªñ ÉL±+†~£Èøȯ-gÌøö[?ÿÇÀ¦›SÜ0ý™ÏF®Ìz«Ÿº—ØämÛ‘O&9^¦.>™es-€ÓP'Š+ã¶H† õ%½à=f!ÅÓÏßÙÄÒ68HW¥•Àã¦6UÆýh_Š„ú”Ü»§ÌtØ­pñ¹/gsé@®Tù'}E¯| Õ'6 )OvÅ~)Ž |ŸÔžI/Í·@9žhîPqyP1¦F¼3™ˆl+ìfš9¢[Š#¾Ó‰yxAw'|­Y˜»"ûã Bßoù¥ÇõW8S–+:|Êa`û+òšÂáJh¥¢s—¥²vÁK4{'Bà*Lë?<|h扙^qjÑÕ¼ \Ä×Q¯‘’àG_£Aϲ{sBVmÞ9g¾Üu’Ãèè.¤þ†ÍêõÞŽhÏ÷vpI#Mg«u›‹;˜¥<úáz.FÜ 5 oŸºŽÒY¥ø(sÚ‡‡wïݵiY-ÄÝûÖ¼ké†ëö,îmm¬ýð`ýþ:Ýø…JÜ0út¦­­ ïW8¨˜ÊTPw¶6wâäíIÛÞ=§Á¸ú ¸Šû¡!´¯ +îU×k¦¨™vAza]¯ÃcÁÚ&ü:ÊÛÈ.þ8ÌFæ®yáÿç}•vSã:E†6ª®z´¸Lâž¹m©wR2òÛmøžB5f0m°Oïh{ïõM þ¢@ ­Ûhbvÿœ¡Æ°l[F/x¼êpµÔï¾d†\QàuaÖØ2§¸„¼Ê\)–È÷Ù—n=™kQ½Û V﹩^ÿwU¯Ÿ«^ÿ÷WoYíTNצd“õÅ­ýçÝÏda&ŽåÃ¥m‰Dÿ⻞ëa³•½xÒ‰Y}´rò“4¾6¯tx»ä=+ôv­lºí?œpAº²'z cG4·=¨’ÝìÉî®F^¤â1û—Ñ«Mõ²÷ÛàMAÍ[†x𘵩ɴ5"—XÜÍ’YÍ4ÿÜu†÷aX‰1µî‹›4jaûp(–r-Ó¹®å³…ØÒا†ÄŒl\-Í|>ìâ"pØI®æJÀ" Ë6hŒ'ÔÁ¯"M5èß Îo¾@õ7ÅÔòIØéåJÊwÓæJj®©Ï”!ÿP’÷ŸúWÛ–dí]x ˜÷d1£MVZReYc/aÐy´)š±&dnâ\±ÒÉ@tâù5ëSÙ­ š ½X•:z-¸=²ˆN‹¥Dn†ÕL%e¤Ptc&ž‹ûþÜc–p¥ÔËÈGüØbQ?þº¦Ç(? ªÀ~fÓ»X—åñ¥³6ô4ðce6×ÌÊêÚøQA6ÙÀð.Ÿ«” ɸ­ü\¨¢õb1ˆ<_3ÕϱÉí&fûžÙ„ó·ìnÄ…Ì»÷Î'Â[|Ö7XÚ"ûl&£-êY A©ôÍíç_ýé BLV­&¯3üÓóò÷÷îáïúƒûkþ_þܽ·þÍúƽûk÷7¾¿ÿýýoÖÖïÝ{pÿ›`íO/IÁG¶éà›é$šma¸Ï½ÿýü~¹¨"½qó¯ÐHÖ}]$¶Ö<Þ0d¿D‹ÈÎÞq+ØH·„€í^Òmïl¿Ú¦gÇ»Û;/wo×å3ÿ a§ûgçñ¹ùÿýÚZ~þ?ظÿÿ–Ïÿþ¸ñÜëÿ­ë€Þ±q‚3#¢š Œ*¢•ÛrW`5Þâ”Öx+¬²0ÃÆý‘È*ÆßÔ6…̯P†™^J@ÿ¢ÛêIÉ‘:c}²;¬²{Ë]s·ËÓíçösû¹ýÜ~n?·Ÿûçÿ3h_Èapache-buildstream-27ae392/tests/remotecache/project/files/dev-files/000077500000000000000000000000001514607367700256715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/files/dev-files/usr/000077500000000000000000000000001514607367700265025ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/files/dev-files/usr/include/000077500000000000000000000000001514607367700301255ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remotecache/project/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700312640ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/remotecache/project/project.conf000066400000000000000000000010711514607367700252250ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ project_dir: file://{project_dir} plugins: - origin: pip package-name: sample-plugins elements: - autotools options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture values: - x86-64 - aarch64 split-rules: test: - | /tests - | /tests/* apache-buildstream-27ae392/tests/remotecache/simple.py000066400000000000000000000060531514607367700231120ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remotecache DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test building an executable with a remote cache: @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_remote_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert element_name in result.get_pushed_elements() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # then remove it locally result = cli.run(project=project, args=["artifact", "delete", element_name]) result.assert_success() result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert element_name in result.get_pulled_elements() # Test building an executable with a remote cache: @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_remote_autotools_build_no_cache(cli, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" cli.configure({"artifacts": {"servers": [{"url": "http://fake.url.service", "push": True}]}}) result = cli.run(project=project, args=["build", element_name]) result.assert_success() assert """WARNING Failed to initialize remote""" in result.stderr assert """Remote initialisation failed with status UNAVAILABLE""" in result.stderr assert """DNS resolution failed""" in result.stderr or """address lookup failed""" in result.stderr apache-buildstream-27ae392/tests/remoteexecution/000077500000000000000000000000001514607367700222035ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/__init__.py000066400000000000000000000000001514607367700243020ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/buildfail.py000066400000000000000000000047161514607367700245200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain from buildstream import _yaml from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remoteexecution # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_build_remote_failure(cli, datafiles): project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") checkout_path = os.path.join(cli.directory, "checkout") # Write out our test target element = { "kind": "script", "depends": [ { "filename": "base.bst", "type": "build", }, ], "config": { "commands": [ "touch %{install-root}/foo", "false", ], }, } _yaml.roundtrip_dump(element, element_path) services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # Try to build it, this should result in a failure that contains the content result = cli.run(project=project, args=["build", "element.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result = cli.run(project=project, args=["artifact", "checkout", "element.bst", "--directory", checkout_path]) result.assert_success() # check that the file created before the failure exists filename = os.path.join(checkout_path, "foo") assert os.path.isfile(filename) apache-buildstream-27ae392/tests/remoteexecution/buildtree.py000066400000000000000000000053541514607367700245430ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from tests.testutils import create_artifact_share from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remoteexecution # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_buildtree_remote(cli, tmpdir, datafiles): project = str(datafiles) element_name = "build-shell/buildtree.bst" share_path = os.path.join(str(tmpdir), "share") services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) with create_artifact_share(share_path) as share: cli.configure( {"artifacts": {"servers": [{"url": share.repo, "push": True}]}, "cache": {"pull-buildtrees": False}} ) res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) res.assert_success() # remove local cache shutil.rmtree(os.path.join(str(tmpdir), "cache", "cas")) shutil.rmtree(os.path.join(str(tmpdir), "cache", "artifacts")) # pull without buildtree res = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name]) res.assert_success() # check shell doesn't work res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() # pull with buildtree res = cli.run(project=project, args=["--pull-buildtrees", "artifact", "pull", "--deps", "all", element_name]) res.assert_success() # check it works this time res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]) res.assert_success() assert "Hi" in res.output apache-buildstream-27ae392/tests/remoteexecution/junction.py000066400000000000000000000101551514607367700244100ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from buildstream._testing import create_repo from buildstream import _yaml from tests.testutils import generate_junction from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remoteexecution # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) def configure_project(path, config): config["name"] = "test" config["element-path"] = "elements" _yaml.roundtrip_dump(config, os.path.join(path, "project.conf")) def create_element(repo, name, path, dependencies, ref=None): element = {"kind": "import", "sources": [repo.source_config(ref=ref)], "depends": dependencies} _yaml.roundtrip_dump(element, os.path.join(path, name)) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_junction_build_remote(cli, tmpdir, datafiles): project = str(datafiles) subproject_path = os.path.join(project, "files", "sub-project") subproject_element_path = os.path.join(subproject_path, "elements") amhello_files_path = os.path.join(subproject_path, "files") element_path = os.path.join(project, "elements") junction_path = os.path.join(element_path, "junction.bst") # We need a repo for real trackable elements repo = create_repo("tar", str(tmpdir)) ref = repo.create(amhello_files_path) # ensure that the correct project directory is also listed in the junction subproject_conf = os.path.join(subproject_path, "project.conf") with open(subproject_conf, encoding="utf-8") as f: config = f.read() config = config.format(project_dir=subproject_path) with open(subproject_conf, "w", encoding="utf-8") as f: f.write(config) # Create a trackable element to depend on the cross junction element, # this one has it's ref resolved already create_element(repo, "sub-target.bst", subproject_element_path, ["autotools/amhello.bst"], ref=ref) # Create a trackable element to depend on the cross junction element create_element(repo, "target.bst", element_path, [{"junction": "junction.bst", "filename": "sub-target.bst"}]) # Create a repo to hold the subproject and generate a junction element for it generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) # Now create a compose element at the top level element = {"kind": "compose", "depends": [{"filename": "target.bst", "type": "build"}]} _yaml.roundtrip_dump(element, os.path.join(element_path, "composed.bst")) # We're doing remote execution so ensure services are available services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # track the junction first to ensure we have refs result = cli.run(project=project, args=["source", "track", "junction.bst"]) result.assert_success() # track target to ensure we have refs result = cli.run(project=project, args=["source", "track", "--deps", "all", "composed.bst"]) result.assert_success() # build result = cli.run(project=project, silent=True, args=["build", "composed.bst"]) result.assert_success() # Assert that the main target is cached as a result assert cli.get_element_state(project, "composed.bst") == "cached" apache-buildstream-27ae392/tests/remoteexecution/project/000077500000000000000000000000001514607367700236515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/elements/000077500000000000000000000000001514607367700254655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/elements/autotools/000077500000000000000000000000001514607367700275165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/elements/autotools/amhello.bst000066400000000000000000000006161514607367700316540ustar00rootroot00000000000000kind: autotools description: Autotools test depends: - base.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 config: configure-commands: - | %{autogen} - | %{configure} - | date +%s > config-time build-commands: - | %{make} - | date +%s > build-time apache-buildstream-27ae392/tests/remoteexecution/project/elements/base.bst000066400000000000000000000001031514607367700271030ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/remoteexecution/project/elements/base/000077500000000000000000000000001514607367700263775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/elements/base/base-alpine.bst000066400000000000000000000010061514607367700312660ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/remoteexecution/project/elements/build-shell/000077500000000000000000000000001514607367700276715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/elements/build-shell/buildtree-fail.bst000066400000000000000000000004641514607367700332770ustar00rootroot00000000000000kind: manual description: | Puts a file in the build tree so that build tree caching and staging can be tested, then deliberately failing to build so we can check the output. depends: - filename: base.bst type: build config: build-commands: - "echo 'Hi' > %{build-root}/test" - "false" apache-buildstream-27ae392/tests/remoteexecution/project/elements/build-shell/buildtree.bst000066400000000000000000000003451514607367700323640ustar00rootroot00000000000000kind: manual description: | Puts a file in the build tree so that build tree caching and staging can be tested. depends: - filename: base.bst type: build config: build-commands: - "echo 'Hi' > %{build-root}/test" apache-buildstream-27ae392/tests/remoteexecution/project/elements/no-runtime-deps.bst000066400000000000000000000002561514607367700312300ustar00rootroot00000000000000kind: manual description: Test element without runtime dependencies build-depends: - autotools/amhello.bst config: install-commands: - echo Test > %{install-root}/test apache-buildstream-27ae392/tests/remoteexecution/project/files/000077500000000000000000000000001514607367700247535ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/amhello.tar.gz000066400000000000000000000336171514607367700275350ustar00rootroot00000000000000‹ì\k{ÛƱö×òWli=©’”¨‹Ë•Ÿ2%«ÑíˆTmÉ¥!`I¢Â…ÁE2ëø¿Ÿwfw¢m¥'Î9y¦‰(`wvv®ïÌ®jéûÑÚ£¯øYÇçéö6ýl?Ý^/ÿ4ŸGí­ÍÍ'ÛOž´7­·7¶¶6‰í¯É”ùdIjÇB1‚±âÕáàåéÅ@tNÞˆWóóÎÉàMƒ¶@óåûT†©Ú[Jt¡ß¾{nX y+ÕZ^0õ= ¨b;Lg"‰ãÞy÷%(v~8<:¼¡MïNzý¾Ø?=Ì?뜻Gsqvq~vÚïasÁÖÐ…®Õ.;Ýa÷ôdÿð`xÜ鞟÷ÏûoâCð> k—ÃÎñò!oëÅ O‘Y:¹¶ò·ú[ü³ÀGçbpJc‡ÿè÷OOÞ6¿ˆÒv¾ˆ–Kç´úŠzMñ._¡ ­'ÏÞ6Z ½p\»Ls˽ƒÞC™Ò6þA3[•7Q&&ö-A׊ŠÅ­ŒÙN¡ 3@ö†hÏÄ]ß4Ø2”é‰qf“>%ˆ§Q«r83CÆwíË6Khv¨†! –Ø6®3ÏwE2KRX”K†$ 7‚Ë5È&y (:ÒÍàlnäd†*£ãw À8gS,â?–´«Å»PIl<¬•ð{°d²ŸãιÖôÏ:^7?ó¡Ù/½n½À_ŽÔ˶G´&¤˜"GÃÉÛ/,0Ž(ê' ¦v8ƒåέCý›šcÞ-‹Z,`/d}ˆW¤jÐ…dÏ•"·üV½—Ùëí_œÀyîoîyiá=õ†Úäw-J"VÞ5 c¢†¢n UìQƽ!])ç‚ÃÁ#¦ËŸ2Ø2¬´/ô‚,0î—:‹Pñ‘,P•]ò.[Úÿ/WÚ'ˆ1ZTèCñ`¿3èÕ.÷"– B¬¬+o»‡'‡ƒœg¦So™Ø¢\a¨7UŽ)µ 2 “z—ÚKÍ@mŽ7fçÆ“ÑÏ«6"æ"!SP‘Ï !?N#O&˜J$ýhì9”Å4 q7ñœ DªÁ¨¾Ç!\_ñÄd¼”×ΩªÌp?hSbÐþÒï †Ý‹óóÞÉàžò¿$P—Tò)ËYöÃj“[-ü®Jè™’õéRÉâD6JU<*$ÀY¸¼Úê­{ö5ÁŸ 2­J!‡bHùE&ûžïÙñ Zd˜µ¨o˜#â—a²fGÔ Šb9¢0S–Ä}‡rµnœXz,û:‰|Òx¾’n%)Ñ•Sº$th|Ý Õ$¤\Šeã,l‰<\BoiJöF”-v¡#AÄ£¬q;*3‚äƒå²0KX—Ivm¶ˆ`ºcê€4š‚F¡Â€y" ·¯á—ÄIn”d²:¸±}wÂ2w?•ó¡å8,ßR#Ä.ï» ¤¯‚_ÝJ7ÏÓª$¥Q¶¥‹•JÒ0<ËÁ^¹UaÆafçâh ÀÔcÑûÄç¹nUJöýÎqŒù»ïÄôÎ}WÉqÖÜ;|uŽ ²„³ÚÏž=ý6pÖ¼k'ã^C°£7óç_,»JÕÅž #6ÙœËå dÌ™Ri=mGgç=˜Rír£µ½¡,FÌšƒó‹~–ëË•uB ·¶ï¹ÅR;‚ ʼzú~ç¨ßÓxÿÓ™ í_üÐ C^ÿþcE7/øÅðät`jÛÒ¼åïKJ@’Ñð£‹ÞPíaE †`ÁÛë5vÕW&·k=¶*ÒG -^Ó£ÒˆÊÈ«Û==>†K÷I µÊ%H%€kóßð¾šÂGv¾ùçLëc5çûÃÞùù)ùeIÿ˜Ð®rÓOÁ^µQØß…JîºHGHOT†Ën…T5‡·HänÙ¢üYë-¢ÀÈ[Þ@ƒû>ûÜWa4èl1’w„”Æ'H”|@wŠ³tvÄ‘¦”(,ö¸Sh¾ka 1S«<äl \7è›é^:Ò%MC¡©XC—¢­¦v„ÔOÐè†" |‹Uä¼Õ†:ª‘PטJ|÷R‹ÐFÐu:Eê' !É5OÇO¸—Tœ KÁoÄY‚š¦A *Ýí ™:-SG˜{V µ¬Ü¨žX( 1Pfnµ”@Éûözg½“½ÞI÷°×çøéHQJ®‘h5êb±˜$5ƒ]LÉЙ g"°IP…èò-‰n·ÚÀ_¿¦§?ü½k~ª'gô,WðŠf¾¢&ÌL+<•Î$ô~ÊHÜ¡«q‰Mu,—³6•ccŸZ’X<ÇAÊô_QÏ:´ÒRãFï"†•M#U–äÄ¥]¹¸c˜¨ŒWµÑ•·Þc·[^ MÕÜŽ;@t^r’””¹o…z+Õõ4x ÕîÚga©ÇGvHÕ43ª] ÞËIh‘¹{pF5§0P&üåקƒ³‹AAäMF—çÖáI÷èb¯·t!Ðvº?ª·ó}Ön—’Ÿ¸„ˆg·ºƒQ¥€±¶¡ð²[´aõ´×¯9iæÓ`O˜F–G?Í4¶ÈûÓ¬±ãl üÇZF€—ž#A‹ÿ"°ú9æÉ ÀüA÷ïsÓðû²iKÖÿä(s-[·Îø¸Ûé¾ì ño÷ÇÚeÉÏ“tæSY!V˜ÎýE¨ËïÜ"·óYžLŠEÒÙT’UÎeãα2“‹^)$D†&阽›ûP¦#Àn媦>ÍMTHFˆ=¥/wBÕº’"É”©€ëhœ%úp„òÎÄŽ›±Ï¾¦$F ¨Kª.@z?Š™ŒÒ%‡Ï$(³Ç’=ù0ƒ§¾ õëOêžs9‹Ê¯ª N¯‚Q¥0®2”Э¦cq¶š‘Å2%5$ tK¿êÚÉ$HRQ¹ªiªë5tO;¡€] ñ~ž¡<ïþå/…¹‘TÐ|G(‚}cT,ís>X_û»Âbuža«V…^CYßa˜J_ÓÐ'ôdVÔ¾Œ2>BcKTy$?ôÑVHÒA$ßÛÊ€ÐÞëA3ú˜ ¼0Kxhµqä _”8Z­…™Ôs¨S|5€#FÂm/-:%'Öò¦¤ª§i¡ˆp”8R‹Ÿ&¥Âå2©Çë¸F”Uv,8éŒR©ÏS{Üðq x/ n ð”'Ö¦šÚ0gú–ïçTØæÕ];W¹=ê,d/Åsº5"é¤Ìl OmïWdH¡H‡³!] ¨F;z?“‰ÎsZØw©þt Ÿ»ÑO8w•ö$·ÎSñ³ú$³D}÷¨oäL§ú ½XÐK®ÄÆqtC¾ ¬`„ÍÆÆÝ&²‚Hífb'9U—3™ªÆ{"º•\}Šµ2Œ²ñĈ1ÐhèÎÐù‘}“»ôè|ˆ4wmÈ ¦¶S²±äOZñ¢/ZK(ë‚åkQÞ¬hýéÍìæV~eª• çÈÑèh×øI1ó)Ùì.Ä)‘Ó¥¯x—O^áfsa @€²Væ6·Ï B¼ÀäÛµ0¤Ûx‘‘qŒ2ÉŒ£âX é‹ ¿eBßµ?3ýÉ0=—Ü/˜¸òzM*ä>ŠfBõžŠ÷÷&•Á$áyÈÂ2™? | FT­…"šê>£¹cËM/I2ºœÉ÷5 Gà»ß$¢ÆWg9ø6_A¾Q\‡o¨3R%¡Ô%±«*™$öXæ¤Ø :ó² \ ª‹Q|K® ë…ÂÖ( ©žPóø¬ÁÛ{ÚZçŒò´Õf‡÷mvÃ,ñg;ùDACwP¼hÅlqG­Ë×/HTÖs:_°ã«œæºœ[ /T±Ùc ň3´cuy™8ú¾µþù•• wJëYó[.âk%j-ðh‰9ÿùçRkAM˜#7?¼>ï*ÆhÄÎs¡ ò¨òÅÇ„|Ò’@Î?è¨Òµõ©¢[¯þàÚ „ßÖçNcözgǧ{=ÝvRí!MP™?ã ˜#›,yªê¤ŠxP®û3’]¨"›éÎTÂeIêG*iŠk~ªÿ¸Øz¦w^1äµ-õAL-dª®ò‹zêTeá–ìv±ºä–žn„.ï’õ:{‡'ýÓÁâ™™'.5¼Õ—†n”~£Iµ¸ù©¶÷Av=ÏSÑ-U,uΆ½“ÎG½r®iTÃ6AŒõ‡È5gú¯©ö¼l6•F›K§ÑW]d%%?‘ЉJIÄò=†2¤eô]/ùüÉ”/$¡2…a7S/ÐW`øhë0K™^žŠþÑnùݤXB¸ÜKü"ìuú/w­+Kw BxÎt×ÂO‹Ï œƒç‘@ÎÖœ£Î¯ûÉÃÑeà "9ŸŸœ¿0B˜ûý¡}@¹—ôixo‰¹Ïƒn |+GŽÃÏž|ñ®ÀÂ9ÌgNRàܸwêU àÒP¨É"4©Z÷:néæ5!|òZ.c8 2Fí_ø3BPD î#™Zتo£R²Gꦹ$R®þ¶ÄU³”/såå«d–ø\ÏTŽúp}ÿð¨×WåãꕵZglwï‘ʃª ¢cˆÐïçið[ ß“‰7Juï •¿(µ¸ø~Ùñ¾ª;tó¼è¿p–Ð…?ï[UY£Ýwº Œª(ª¸óˆ²=YÛi­®¬­Yï4ù®ê,L$ßl2wôè,Õ\z-þ¨Â¼jPg:D¯t… k ìÔ™(!†ÓÐÏÿï1¼PUlz:·¦2šú¦x?Ô§zN¦^4ý-í]ÆGÁ§Ì•ž0ÃÅ’E J ˆËŸ™F®êvæV›FIÚä?öIL3k%NÙhlÿŠPÞ)Šì˜S:ff(ôEKÙ¹›D¾ÌÃÿ=S¹úÊî÷ÐèÑU8›É{“ˆ ´ÒÆúÖ÷¦Ì%ks-ºþ«NjøäEÆÔ¸w‹&ÝÖúúzËÔ†º÷l%>n­J¥¿m4мÚøŸö®´;,ÉÖWó+²] Õ€Ûe—Êr[eɶ¦eI#¼ÕH*Œ …òH†Kš.ÿ÷‰oÉ$Á®¥ûœ>ÇÇ‚Ì·¯ñ"nÜ÷®>í á2½+0U*¤=hÜ™L·ÞÓNI‚,àUŽ^“æÉ UJ“€‘ñv«zã@*‹¨Q<%š¨›¬®L“°k;>-n6â3yÈžÙÒ’¬ÕÅC/Hf[]¥Ú?’úKù©N±™.S¯x™D½çŸIÙ YœkÉi¸&H?Àì&É<ã¬ÝóFÔìè¥.«˜3g¥åòÕ< OP<» jADýë,þ¢}¹ Mõg…8IÄEÞʦäÎ'™ÃJYÃ+ÁN‹oìî‘L«F°ìó'zÞÿ‡‹í™ù ‹&g6ÛL’ÄfÝKVä Õz$nô†ˆ”ÈM2›X†ŽÝ&SQ»AgHiü$^ ¡z¤‘Žœ:Áyܧ! 00»¡Gêpà¡Ë€TÌ0)xX2 Ó~Ú>ÌøKºè4Ò|ÿ<ó´V:ÉÅwøñ¬7ª:Öõè£CîÁaƒfÒÞÁîY­(ÆÉáÎÅ­³/FHç–(ñ–.ÖìYh±.™öl#›dÀ'à" úµ:žDÖ%¾±Ñ¼¿V«;Ûñ90‘Á˜$EñF†ƒ¯d£> ÖÇÓØ!®°BŽ;©ŠèÔ¨2‡L8n¨Ï+Áæ<=YÐ7¸Æ…pX.{j ‘´˜Ñ=×c6ci£èzê“"jä9ú¤Ä¯Øª,…dW#Ó^P¹vŒÇ˜óuAhÿ^½U1æmé¯:("  qJ S£S:¾¸äYÞø=G粌¢TênèŠgØј£6®žüBÉl7þçìoÏö·Ÿ·*…Ã¥î·ótFí´^mïï[0º¨à«a™ÔKƒ·ˆò{þêÒ^Së6ö*UãÏÃK o§îElª‰²#>„ašo©­9 fæ,@YÄG”rc¯É®Œº(¦Ä¡¶ ýÜóÚÝO˜-PIÖE|8RO·€Ð7žwMÉÙEIX3ŽWJ°FÇ­é,õÎCYÀ|ÞÓ± ùÙYæqmXíQY}øËò–è_I‹më4høè×½éÃÕ¦”A>ýù9ŒÚo]WÁ¡aiiS#|Î~`¿{µ²Im…6ú”‹öpçB@|´výeèþ/¶ôÀÄ!D-¿F“çÇNÌ(•Yœ^Ò"6½ŠhPÑÜmS°‰Wú‹Ýn•Î€UøWÈÔÝÙÛ~~pØ¢qÀ5)šFyô&;L¯’†®,´ë72J(µ»Ó:C}‰1Ù4¼,’Õ]Ãâ‚)([Hõd”4dO3(ÞÖ5›ý@ÖçÞzíðA=²‰·ª6š´[¯"¯qÌ2Ï3ɵ’ï÷†ÿ¡ˆŽq¾7®õ8o ɇ³Úfa¯¬Ä ðäÃ&}¯‡ý‹Î­dŠ¤;“#Õà2=`¸al,oÁÐ+Ϋm>#…ËÚ4œ/~hr`ÕóÞ³‚N¬ëhbé¡ýúà¿_¾ÚÝɤ\Ñeü:PÕœÖ 3¹(¾W°rEpü7Ú £Ù𜦻Ÿ’–uEä;ø™§noewèæÜæ°Mmÿ³ › 7˜\”BVß¿l«Ÿ'?G ýçpñ NÔQ¤!F Ü䓘ßr•Ycê–¹(œì¹Õù Y¼ØÝÞÙ=6‘/i­&Ea‘ÅÞÁ³Cfp¢,@à'& ¿ÁüͱíY,‡yu¼w´xŸÕºñŸÔ–Ö4à|àúIŒ›¼IT>´"âhÉ^FJãÓ­íåõæÍkxY$‹S$0¨Æ;„_RîOplawx?ºœNÇ›«« llöG³f2鯡IšÕËép°jš~•ŽQµ«Ã´ÏütM¼~ü‡Y¿§‰¸™ÉXÜö˜gf¥ªÍ„¼äÀ«VÄ,óy†vI`5§Ì’UnŸ«ÚùÕöÉzÀ`ÕçOq¡}*SBe–£;,/Š:xЙò"ß,ì¸í·ÅFz+)-Ì ²9cjn¡Z4˜xÏl!œ,-cÕyŒåÿŽîSqÇëù˜xÈñîÜqÝë΄Z¬h±sFm³äQÄ7{;´a<ÛÝoíæNhÈaÎý¦ $rͼ]t¸3¥+<ä-JÓys.,#”•‹^.-˜F^T2D_˜ìçË&®8‹ Ço—•ÎF_PÀ´ 3’±ÏÈ»ÐwM…à,¼/(ïc Ã0'ZT s §a!—‰j$)ë6ë䲑£Îdp˲8u6ùhåñ$ùHÂ2;‘”FEÀó^µ~Ð’WäYÙ53­G‡­½w‚ƒ)‹}K.fS(²<Ƨ7cT)õsQV Ó¬DOzø 6gcÃÎr.¶2=èÃÝ3MgCNA1:›X+Þ»ñ˜OЭĒzôû“ˆ{eþ‹Äš#¦çYAÒä§Ì!¦ 2 £ ÊŠð¤÷ʆ» ÛÆSõŽ¡|˜Þ f7 Îgý•õµ‡…£50N v¼ë~Œ±EuÁs&¦ÑëŽiˆ­ÀG“¤3°sâ„“aè¨5™ÍÂâÿÔÐJÍ%®“¢;äCè6àZÉ,eSt'&–µ:ÇþeÇEîAïùËÄ|›\w£óèÜ<þv#xô(¤ù–“qú—ˆ'Ù¢Ò˜B3j:G\£êóñ8šÐØ ÙCEg7ÓR22^ YWVOqW@s4.Bƒ²3ºÐÑÔ£-‚Ùdҙܔ”æî<¢ãqŒâ‘èbz2èðK\<Ó[´ô@Ì8šÑÇx’Œ`kû`q?•L÷ÔUÝÀ"áñþèË«Ôæå±—O&7MO[f…‚I'fKÚ(º@ˆï^ú´Ì_ âs†FÔqààdŠÏg¢+JDë² )FhtR´œA}Á©tÔ§cž5“­"bþiì6ß?¼·Þd’yL ±",+B,P.Ð>ŒØ¤î çA ,žR ‘}2¢ŽØÑtYL %UçX.&X>Uê!0D8÷ël¤>ª`VÏdÝ)ñWæ\~¾¨èuÍŽŠ%#%a¼yFsk¡M@݈º ±iþfãž!¦ÐÃqãRñU¾Õö ¸'Ž¸â…5Ìè{So‚i%…ÌÇ_²²@q6†­^>¬÷é•—Ã^|q±¬Øæåì' D 92iQˆ´_þ#¨òFªD-ÒA ’dŒMvåǘ´Q){ZWV5SzJ‚7-¯ðâ„žA.(4›0àÇLÓ¶¢){Ñ5û}JqÚšw³Ô•Ô¤¿UY篜c»›ÌFÓ­õ”x*¡ âÏ( AóeÂòJSÑôƒ_í×ÍïŸÈøܸÇù‚c[%÷8ø[°þÞÃû1nBà@kÛ%›q9xœ|˜gµ÷«:OÎòù, žþJFçuv…6\g¨èf܆¯gƘ×ýÍãäsìiã HÿtIÿ¤˜O¯ oV*’aèØÑ ¤”ׯôçÎw2Ä\:>a_è3öiFzZÈé’¸ÅQËflfì6.……¼æw¿ŽÁ•ÇÉ*ârÀtâ¨TC5ŒzÀ”šò:m ѱ ‘½dš#[^¶;‡-s¾+ðØðõˆ4Õå¦9¥ñæ™íJBl0÷ØÚ{üÎ÷+ð]8¶šžßNæEc2,L¦¾LrvAø5ü£ª'ZŠyÙZóøc‚u öi¹€›…ŒPý…ôÙÞi– Aɱ§Ô‡JÌÑQ3~Q¶DÝFœ6˜©Wþ/ûâš×7[.JÙ“Ûü%«A{»}L=êË0H»“x,®IˆH€L“«;6üE4GŽâBžª†7(õbÌö­çÖ 6 ZÏ÷w±}9 iÞ§2“J<-ùgûlx¯}>&ovžNi!©''¿Xj±öãŒ,žC©-+Û¾¦¦³hc±»0Ô‚yþ7ÊË'ŠTf»túŠ×€§ÅXødzUr…8“kÅT 8ßTŒuÃíDÅuhUõï²µœ^i(¹(wºtf‹Æqƒ’›«Å×(¡ÆâR·´Ø0²tº—í+õ~åx ì›Gº 5¿6„!hÜ?kP4J€¾¾j0Æä·ÒKp¯‹ºÌ’öI_ïFál4u·¼çël §5Ž/(ê’~ò2(ò«¦r^É{9Ìw=ÙûœÃ½ œøys—žúƼ…ÒYXÝ\,(•óžägv¹loz9ý‘ÝY /lˆýmHE¯[/ª'OíCŸMø£?©€¡Ùz¬¦ ÂŽØÀÕ\}"xáÚ€ÀÛîvÛÝv¢0Í åW»LB` R]õ„Ç^ÖÌ8 8oØyûå`”~ˆ£bÐå ½Ѓa'K—,ÝÃÇÕÀ®æÜ(ãø–g:ì‚q‰GÉJÖk7Í$°n«ÔuA¨³™^ü4ÙVªT€tmZ^¯e¶nØ%ž‡Ë°mxö¬g"Í|ê]’ÈÚû€@¢â®cl›4‰}¶ÁAuMÇŸœÉcª- œq„ ï <_ö‘qŸtŒ3Žh&ÌImÈð;z<©¼û¼•¹äYFt¤ešw?I!Ä– ‹“°Þõ"$5% ;üm‚C×¢L°0G£t(ø¨9Ü¥bO§aU`àe»a¼£Ý&Áµs°¡2»/ÀšÊ6/1Ѥ1¥e‹ãÚ^cÍAÖJÈÈ:nRŸã&„຤”q–á4´Õ(<¤ØÌ•3Ú=~ú´ÆÇa„ò%\¤Õz–©wxÄS_®ÉB\«]Ú‡O=­Y“‚üE$øÕÛ7ÌüU„Øy€žÓÉMÑÐÆhQ5Â4Pó_kØ–b…$}*°»Ó\¬tz±½TÓHô¡µDz›dî2L}É´†³“¶‡Ñfeï`÷à÷s”ƒÇßÂy·Õ ŸÂ·Ÿ±ÎGâ¢w`!(Œ`ë³Uù»Õ5ÍåxZù{°Å÷ÖHàÅùss¹€µƒO9}öÐóõVð™P6ûxˆZ‘8¶Þú¯fƳï”P e ¾ƒ.Y­¯.öËÏÇBûD{8aHÊývi˜ŒÕLæö&#ˆ l·1¦}ørw†èÒmF ÊqB–Úι¹ä„‘kø¤ÁÅVX U£ 7:UMœœœžž–OWN+§ßž†§¬÷\œ©Æ! Ñ‚/.—ì¬f·¢1«-Éï48½³s]ÃJð‚Ï~J–«ˆìIÄv㣽CÝsÕ‰e´Úd2`;Hs=‰±Ç'gïsƒnÁÁæ+Ñ“åœzÀ2ªWM.ÕwB½)N|‰,ÞÏá-âßd6ÀßnóÆ·B.„4lôŒCV^ ?¡zP%eT/Ž§7B_Öòß)ižÂ4hÚàB} wi6<^TQÑUêËdSIÞl­—kŸ#.OýK^+;—,*•(T’ksrX‘ŸüuóIP­"SjÚšÀrsÃÛú§Ã‰ñ[kbO% Þ¯¨å‚÷™ë´n¤U×ÎÄH`ÄQËÒÁת)æ(‹­v³ß¬ɨV¿ÃoÖÑõ´5Æ5N@c ùɈyóÎÝâ«S¡-˜ƒpÿðÃÆCNV¼‡6î>àåuf“S™:}©ABXtY® Jí¤ Smkж„lçCB!Øjï>}qX=9 ™Å©Rýiû¸R}S«•èËš^2C_×ùª– ® â§RE´Ú°'<;«EkátÔ]¸=|A‰yûÎðI/N²õ…¯¬|A¤ «!D7ʵ"aåMhy(i; Ñ@¡È$i°w¼÷ŽµØh ¾Ù’÷òË[<-Æ5{±'Ç­…É'U(ƒIAbÖ‡Ž‡×¾óÃÛ—ġ)ŒoJdÓÉRg"} kp>ÐW¯œçX´KÂæ9ŠÌ%(ÀLj–ôI*®7Túÿç¯köîéq[ áÈ)zâUoÙíØ:™ ,’(;Äí,;¦öbYcŽ/­ø=Èî Üa±ç™šÚ ’È|ÒÁ«´Bb—ÁÖÀN\œ¶—¼Ô}BS©UŠ„ˆ›sÀ2Ý©6›Õs;¶tlæíW‡‡ûÕíÂà„›Œ¾lò±/ÓÃÆŠÉÖÁ¯Áb×HËÞZm¡ä+Ñ+Ïo#o¶÷ ˆ?Ao&X=¢:Vʈ@̶Z“ ’'æûŒi4zþ‚pˆ:ÿ8‘r¸¿Àø{ªZÌÎq7Heïç¹òÇ—Lîµ’ɼò$æ¿”{¨Õ(ÝÜöœ†éRöÎ/„Ýûz©%dyvxürûÕÖÒ6Çèƒß:íÝ’oƒüÍãÚ‚©âëÇa=™Z&äK¨Ãqç:”m@Ogœº]L+Uö¾f];ÆS¿$WÇb —;çè­äÞ0tªñ ÉL±+†~£Èøȯ-gÌøö[?ÿÇÀ¦›SÜ0ý™ÏF®Ìz«Ÿº—ØämÛ‘O&9^¦.>™es-€ÓP'Š+ã¶H† õ%½à=f!ÅÓÏßÙÄÒ68HW¥•Àã¦6UÆýh_Š„ú”Ü»§ÌtØ­pñ¹/gsé@®Tù'}E¯| Õ'6 )OvÅ~)Ž |ŸÔžI/Í·@9žhîPqyP1¦F¼3™ˆl+ìfš9¢[Š#¾Ó‰yxAw'|­Y˜»"ûã Bßoù¥ÇõW8S–+:|Êa`û+òšÂáJh¥¢s—¥²vÁK4{'Bà*Lë?<|h扙^qjÑÕ¼ \Ä×Q¯‘’àG_£Aϲ{sBVmÞ9g¾Üu’Ãèè.¤þ†ÍêõÞŽhÏ÷vpI#Mg«u›‹;˜¥<úáz.FÜ 5 oŸºŽÒY¥ø(sÚ‡‡wïݵiY-ÄÝûÖ¼ké†ëö,îmm¬ýð`ýþ:Ýø…JÜ0út¦­­ ïW8¨˜ÊTPw¶6wâäíIÛÞ=§Á¸ú ¸Šû¡!´¯ +îU×k¦¨™vAza]¯ÃcÁÚ&ü:ÊÛÈ.þ8ÌFæ®yáÿç}•vSã:E†6ª®z´¸Lâž¹m©wR2òÛmøžB5f0m°Oïh{ïõM þ¢@ ­Ûhbvÿœ¡Æ°l[F/x¼êpµÔï¾d†\QàuaÖØ2§¸„¼Ê\)–È÷Ù—n=™kQ½Û V﹩^ÿwU¯Ÿ«^ÿ÷WoYíTNצd“õÅ­ýçÝÏda&ŽåÃ¥m‰Dÿ⻞ëa³•½xÒ‰Y}´rò“4¾6¯tx»ä=+ôv­lºí?œpAº²'z cG4·=¨’ÝìÉî®F^¤â1û—Ñ«Mõ²÷ÛàMAÍ[†x𘵩ɴ5"—XÜÍ’YÍ4ÿÜu†÷aX‰1µî‹›4jaûp(–r-Ó¹®å³…ØÒا†ÄŒl\-Í|>ìâ"pØI®æJÀ" Ë6hŒ'ÔÁ¯"M5èß Îo¾@õ7ÅÔòIØéåJÊwÓæJj®©Ï”!ÿP’÷ŸúWÛ–dí]x ˜÷d1£MVZReYc/aÐy´)š±&dnâ\±ÒÉ@tâù5ëSÙ­ š ½X•:z-¸=²ˆN‹¥Dn†ÕL%e¤Ptc&ž‹ûþÜc–p¥ÔËÈGüØbQ?þº¦Ç(? ªÀ~fÓ»X—åñ¥³6ô4ðce6×ÌÊêÚøQA6ÙÀð.Ÿ«” ɸ­ü\¨¢õb1ˆ<_3ÕϱÉí&fûžÙ„ó·ìnÄ…Ì»÷Î'Â[|Ö7XÚ"ûl&£-êY A©ôÍíç_ýé BLV­&¯3üÓóò÷÷îáïúƒûkþ_þܽ·þÍúƽûk÷7¾¿ÿýýoÖÖïÝ{pÿ›`íO/IÁG¶éà›é$šma¸Ï½ÿýü~¹¨"½qó¯ÐHÖ}]$¶Ö<Þ0d¿D‹ÈÎÞq+ØH·„€í^Òmïl¿Ú¦gÇ»Û;/wo×å3ÿ a§ûgçñ¹ùÿýÚZ~þ?ظÿÿ–Ïÿþ¸ñÜëÿ­ë€Þ±q‚3#¢š Œ*¢•ÛrW`5Þâ”Öx+¬²0ÃÆý‘È*ÆßÔ6…̯P†™^J@ÿ¢ÛêIÉ‘:c}²;¬²{Ë]s·ËÓíçösû¹ýÜ~n?·Ÿûçÿ3h_Èapache-buildstream-27ae392/tests/remoteexecution/project/files/dev-files/000077500000000000000000000000001514607367700266315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/dev-files/usr/000077500000000000000000000000001514607367700274425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/dev-files/usr/include/000077500000000000000000000000001514607367700310655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700322240ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/000077500000000000000000000000001514607367700272105ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/elements/000077500000000000000000000000001514607367700310245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/elements/autotools/000077500000000000000000000000001514607367700330555ustar00rootroot00000000000000amhello.bst000066400000000000000000000003101514607367700351230ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/elements/autotoolskind: autotools description: Autotools test depends: - base.bst sources: - kind: tar url: project_dir:/files/amhello.tar.gz ref: 534a884bc1974ffc539a9c215e35c4217b6f666a134cd729e786b9c84af99650 apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/elements/base.bst000066400000000000000000000001031514607367700324420ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/elements/base/000077500000000000000000000000001514607367700317365ustar00rootroot00000000000000base-alpine.bst000066400000000000000000000010061514607367700345460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/elements/basekind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/files/000077500000000000000000000000001514607367700303125ustar00rootroot00000000000000apache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/files/amhello.tar.gz000066400000000000000000000336171514607367700330740ustar00rootroot00000000000000‹ì\k{ÛƱö×òWli=©’”¨‹Ë•Ÿ2%«ÑíˆTmÉ¥!`I¢Â…ÁE2ëø¿Ÿwfw¢m¥'Î9y¦‰(`wvv®ïÌ®jéûÑÚ£¯øYÇçéö6ýl?Ý^/ÿ4ŸGí­ÍÍ'ÛOž´7­·7¶¶6‰í¯É”ùdIjÇB1‚±âÕáàåéÅ@tNÞˆWóóÎÉàMƒ¶@óåûT†©Ú[Jt¡ß¾{nX y+ÕZ^0õ= ¨b;Lg"‰ãÞy÷%(v~8<:¼¡MïNzý¾Ø?=Ì?뜻Gsqvq~vÚïasÁÖÐ…®Õ.;Ýa÷ôdÿð`xÜ鞟÷ÏûoâCð> k—ÃÎñò!oëÅ O‘Y:¹¶ò·ú[ü³ÀGçbpJc‡ÿè÷OOÞ6¿ˆÒv¾ˆ–Kç´úŠzMñ._¡ ­'ÏÞ6Z ½p\»Ls˽ƒÞC™Ò6þA3[•7Q&&ö-A׊ŠÅ­ŒÙN¡ 3@ö†hÏÄ]ß4Ø2”é‰qf“>%ˆ§Q«r83CÆwíË6Khv¨†! –Ø6®3ÏwE2KRX”K†$ 7‚Ë5È&y (:ÒÍàlnäd†*£ãw À8gS,â?–´«Å»PIl<¬•ð{°d²ŸãιÖôÏ:^7?ó¡Ù/½n½À_ŽÔ˶G´&¤˜"GÃÉÛ/,0Ž(ê' ¦v8ƒåέCý›šcÞ-‹Z,`/d}ˆW¤jÐ…dÏ•"·üV½—Ùëí_œÀyîoîyiá=õ†Úäw-J"VÞ5 c¢†¢n UìQƽ!])ç‚ÃÁ#¦ËŸ2Ø2¬´/ô‚,0î—:‹Pñ‘,P•]ò.[Úÿ/WÚ'ˆ1ZTèCñ`¿3èÕ.÷"– B¬¬+o»‡'‡ƒœg¦So™Ø¢\a¨7UŽ)µ 2 “z—ÚKÍ@mŽ7fçÆ“ÑÏ«6"æ"!SP‘Ï !?N#O&˜J$ýhì9”Å4 q7ñœ DªÁ¨¾Ç!\_ñÄd¼”×ΩªÌp?hSbÐþÒï †Ý‹óóÞÉàžò¿$P—Tò)ËYöÃj“[-ü®Jè™’õéRÉâD6JU<*$ÀY¸¼Úê­{ö5ÁŸ 2­J!‡bHùE&ûžïÙñ Zd˜µ¨o˜#â—a²fGÔ Šb9¢0S–Ä}‡rµnœXz,û:‰|Òx¾’n%)Ñ•Sº$th|Ý Õ$¤\Šeã,l‰<\BoiJöF”-v¡#AÄ£¬q;*3‚äƒå²0KX—Ivm¶ˆ`ºcê€4š‚F¡Â€y" ·¯á—ÄIn”d²:¸±}wÂ2w?•ó¡å8,ßR#Ä.ï» ¤¯‚_ÝJ7ÏÓª$¥Q¶¥‹•JÒ0<ËÁ^¹UaÆafçâh ÀÔcÑûÄç¹nUJöýÎqŒù»ïÄôÎ}WÉqÖÜ;|uŽ ²„³ÚÏž=ý6pÖ¼k'ã^C°£7óç_,»JÕÅž #6ÙœËå dÌ™Ri=mGgç=˜Rír£µ½¡,FÌšƒó‹~–ëË•uB ·¶ï¹ÅR;‚ ʼzú~ç¨ßÓxÿÓ™ í_üÐ C^ÿþcE7/øÅðät`jÛÒ¼åïKJ@’Ñð£‹ÞPíaE †`ÁÛë5vÕW&·k=¶*ÒG -^Ó£ÒˆÊÈ«Û==>†K÷I µÊ%H%€kóßð¾šÂGv¾ùçLëc5çûÃÞùù)ùeIÿ˜Ð®rÓOÁ^µQØß…JîºHGHOT†Ën…T5‡·HänÙ¢üYë-¢ÀÈ[Þ@ƒû>ûÜWa4èl1’w„”Æ'H”|@wŠ³tvÄ‘¦”(,ö¸Sh¾ka 1S«<äl \7è›é^:Ò%MC¡©XC—¢­¦v„ÔOÐè†" |‹Uä¼Õ†:ª‘PטJ|÷R‹ÐFÐu:Eê' !É5OÇO¸—Tœ KÁoÄY‚š¦A *Ýí ™:-SG˜{V µ¬Ü¨žX( 1Pfnµ”@Éûözg½“½ÞI÷°×çøéHQJ®‘h5êb±˜$5ƒ]LÉЙ g"°IP…èò-‰n·ÚÀ_¿¦§?ü½k~ª'gô,WðŠf¾¢&ÌL+<•Î$ô~ÊHÜ¡«q‰Mu,—³6•ccŸZ’X<ÇAÊô_QÏ:´ÒRãFï"†•M#U–äÄ¥]¹¸c˜¨ŒWµÑ•·Þc·[^ MÕÜŽ;@t^r’””¹o…z+Õõ4x ÕîÚga©ÇGvHÕ43ª] ÞËIh‘¹{pF5§0P&üåקƒ³‹AAäMF—çÖáI÷èb¯·t!Ðvº?ª·ó}Ön—’Ÿ¸„ˆg·ºƒQ¥€±¶¡ð²[´aõ´×¯9iæÓ`O˜F–G?Í4¶ÈûÓ¬±ãl üÇZF€—ž#A‹ÿ"°ú9æÉ ÀüA÷ïsÓðû²iKÖÿä(s-[·Îø¸Ûé¾ì ño÷ÇÚeÉÏ“tæSY!V˜ÎýE¨ËïÜ"·óYžLŠEÒÙT’UÎeãα2“‹^)$D†&阽›ûP¦#Àn媦>ÍMTHFˆ=¥/wBÕº’"É”©€ëhœ%úp„òÎÄŽ›±Ï¾¦$F ¨Kª.@z?Š™ŒÒ%‡Ï$(³Ç’=ù0ƒ§¾ õëOêžs9‹Ê¯ª N¯‚Q¥0®2”Э¦cq¶š‘Å2%5$ tK¿êÚÉ$HRQ¹ªiªë5tO;¡€] ñ~ž¡<ïþå/…¹‘TÐ|G(‚}cT,ís>X_û»Âbuža«V…^CYßa˜J_ÓÐ'ôdVÔ¾Œ2>BcKTy$?ôÑVHÒA$ßÛÊ€ÐÞëA3ú˜ ¼0Kxhµqä _”8Z­…™Ôs¨S|5€#FÂm/-:%'Öò¦¤ª§i¡ˆp”8R‹Ÿ&¥Âå2©Çë¸F”Uv,8éŒR©ÏS{Üðq x/ n ð”'Ö¦šÚ0gú–ïçTØæÕ];W¹=ê,d/Åsº5"é¤Ìl OmïWdH¡H‡³!] ¨F;z?“‰ÎsZØw©þt Ÿ»ÑO8w•ö$·ÎSñ³ú$³D}÷¨oäL§ú ½XÐK®ÄÆqtC¾ ¬`„ÍÆÆÝ&²‚Hífb'9U—3™ªÆ{"º•\}Šµ2Œ²ñĈ1ÐhèÎÐù‘}“»ôè|ˆ4wmÈ ¦¶S²±äOZñ¢/ZK(ë‚åkQÞ¬hýéÍìæV~eª• çÈÑèh×øI1ó)Ùì.Ä)‘Ó¥¯x—O^áfsa @€²Væ6·Ï B¼ÀäÛµ0¤Ûx‘‘qŒ2ÉŒ£âX é‹ ¿eBßµ?3ýÉ0=—Ü/˜¸òzM*ä>ŠfBõžŠ÷÷&•Á$áyÈÂ2™? | FT­…"šê>£¹cËM/I2ºœÉ÷5 Gà»ß$¢ÆWg9ø6_A¾Q\‡o¨3R%¡Ô%±«*™$öXæ¤Ø :ó² \ ª‹Q|K® ë…ÂÖ( ©žPóø¬ÁÛ{ÚZçŒò´Õf‡÷mvÃ,ñg;ùDACwP¼hÅlqG­Ë×/HTÖs:_°ã«œæºœ[ /T±Ùc ň3´cuy™8ú¾µþù•• wJëYó[.âk%j-ðh‰9ÿùçRkAM˜#7?¼>ï*ÆhÄÎs¡ ò¨òÅÇ„|Ò’@Î?è¨Òµõ©¢[¯þàÚ „ßÖçNcözgǧ{=ÝvRí!MP™?ã ˜#›,yªê¤ŠxP®û3’]¨"›éÎTÂeIêG*iŠk~ªÿ¸Øz¦w^1äµ-õAL-dª®ò‹zêTeá–ìv±ºä–žn„.ï’õ:{‡'ýÓÁâ™™'.5¼Õ—†n”~£Iµ¸ù©¶÷Av=ÏSÑ-U,uΆ½“ÎG½r®iTÃ6AŒõ‡È5gú¯©ö¼l6•F›K§ÑW]d%%?‘ЉJIÄò=†2¤eô]/ùüÉ”/$¡2…a7S/ÐW`øhë0K™^žŠþÑnùݤXB¸ÜKü"ìuú/w­+Kw BxÎt×ÂO‹Ï œƒç‘@ÎÖœ£Î¯ûÉÃÑeà "9ŸŸœ¿0B˜ûý¡}@¹—ôixo‰¹Ïƒn |+GŽÃÏž|ñ®ÀÂ9ÌgNRàܸwêU àÒP¨É"4©Z÷:néæ5!|òZ.c8 2Fí_ø3BPD î#™Zتo£R²Gꦹ$R®þ¶ÄU³”/såå«d–ø\ÏTŽúp}ÿð¨×WåãꕵZglwï‘ʃª ¢cˆÐïçið[ ß“‰7Juï •¿(µ¸ø~Ùñ¾ª;tó¼è¿p–Ð…?ï[UY£Ýwº Œª(ª¸óˆ²=YÛi­®¬­Yï4ù®ê,L$ßl2wôè,Õ\z-þ¨Â¼jPg:D¯t… k ìÔ™(!†ÓÐÏÿï1¼PUlz:·¦2šú¦x?Ô§zN¦^4ý-í]ÆGÁ§Ì•ž0ÃÅ’E J ˆËŸ™F®êvæV›FIÚä?öIL3k%NÙhlÿŠPÞ)Šì˜S:ff(ôEKÙ¹›D¾ÌÃÿ=S¹úÊî÷ÐèÑU8›É{“ˆ ´ÒÆúÖ÷¦Ì%ks-ºþ«NjøäEÆÔ¸w‹&ÝÖúúzËÔ†º÷l%>n­J¥¿m4мÚøŸö®´;,ÉÖWó+²] Õ€Ûe—Êr[eɶ¦eI#¼ÕH*Œ …òH†Kš.ÿ÷‰oÉ$Á®¥ûœ>ÇÇ‚Ì·¯ñ"nÜ÷®>í á2½+0U*¤=hÜ™L·ÞÓNI‚,àUŽ^“æÉ UJ“€‘ñv«zã@*‹¨Q<%š¨›¬®L“°k;>-n6â3yÈžÙÒ’¬ÕÅC/Hf[]¥Ú?’úKù©N±™.S¯x™D½çŸIÙ YœkÉi¸&H?Àì&É<ã¬ÝóFÔìè¥.«˜3g¥åòÕ< OP<» jADýë,þ¢}¹ Mõg…8IÄEÞʦäÎ'™ÃJYÃ+ÁN‹oìî‘L«F°ìó'zÞÿ‡‹í™ù ‹&g6ÛL’ÄfÝKVä Õz$nô†ˆ”ÈM2›X†ŽÝ&SQ»AgHiü$^ ¡z¤‘Žœ:Áyܧ! 00»¡Gêpà¡Ë€TÌ0)xX2 Ó~Ú>ÌøKºè4Ò|ÿ<ó´V:ÉÅwøñ¬7ª:Öõè£CîÁaƒfÒÞÁîY­(ÆÉáÎÅ­³/FHç–(ñ–.ÖìYh±.™öl#›dÀ'à" úµ:žDÖ%¾±Ñ¼¿V«;Ûñ90‘Á˜$EñF†ƒ¯d£> ÖÇÓØ!®°BŽ;©ŠèÔ¨2‡L8n¨Ï+Áæ<=YÐ7¸Æ…pX.{j ‘´˜Ñ=×c6ci£èzê“"jä9ú¤Ä¯Øª,…dW#Ó^P¹vŒÇ˜óuAhÿ^½U1æmé¯:("  qJ S£S:¾¸äYÞø=G粌¢TênèŠgØј£6®žüBÉl7þçìoÏö·Ÿ·*…Ã¥î·ótFí´^mïï[0º¨à«a™ÔKƒ·ˆò{þêÒ^Së6ö*UãÏÃK o§îElª‰²#>„ašo©­9 fæ,@YÄG”rc¯É®Œº(¦Ä¡¶ ýÜóÚÝO˜-PIÖE|8RO·€Ð7žwMÉÙEIX3ŽWJ°FÇ­é,õÎCYÀ|ÞÓ± ùÙYæqmXíQY}øËò–è_I‹më4høè×½éÃÕ¦”A>ýù9ŒÚo]WÁ¡aiiS#|Î~`¿{µ²Im…6ú”‹öpçB@|´výeèþ/¶ôÀÄ!D-¿F“çÇNÌ(•Yœ^Ò"6½ŠhPÑÜmS°‰Wú‹Ýn•Î€UøWÈÔÝÙÛ~~pØ¢qÀ5)šFyô&;L¯’†®,´ë72J(µ»Ó:C}‰1Ù4¼,’Õ]Ãâ‚)([Hõd”4dO3(ÞÖ5›ý@ÖçÞzíðA=²‰·ª6š´[¯"¯qÌ2Ï3ɵ’ï÷†ÿ¡ˆŽq¾7®õ8o ɇ³Úfa¯¬Ä ðäÃ&}¯‡ý‹Î­dŠ¤;“#Õà2=`¸al,oÁÐ+Ϋm>#…ËÚ4œ/~hr`ÕóÞ³‚N¬ëhbé¡ýúà¿_¾ÚÝɤ\Ñeü:PÕœÖ 3¹(¾W°rEpü7Ú £Ù𜦻Ÿ’–uEä;ø™§noewèæÜæ°Mmÿ³ › 7˜\”BVß¿l«Ÿ'?G ýçpñ NÔQ¤!F Ü䓘ßr•Ycê–¹(œì¹Õù Y¼ØÝÞÙ=6‘/i­&Ea‘ÅÞÁ³Cfp¢,@à'& ¿ÁüͱíY,‡yu¼w´xŸÕºñŸÔ–Ö4à|àúIŒ›¼IT>´"âhÉ^FJãÓ­íåõæÍkxY$‹S$0¨Æ;„_RîOplawx?ºœNÇ›«« llöG³f2鯡IšÕËép°jš~•ŽQµ«Ã´ÏütM¼~ü‡Y¿§‰¸™ÉXÜö˜gf¥ªÍ„¼äÀ«VÄ,óy†vI`5§Ì’UnŸ«ÚùÕöÉzÀ`ÕçOq¡}*SBe–£;,/Š:xЙò"ß,ì¸í·ÅFz+)-Ì ²9cjn¡Z4˜xÏl!œ,-cÕyŒåÿŽîSqÇëù˜xÈñîÜqÝë΄Z¬h±sFm³äQÄ7{;´a<ÛÝoíæNhÈaÎý¦ $rͼ]t¸3¥+<ä-JÓys.,#”•‹^.-˜F^T2D_˜ìçË&®8‹ Ço—•ÎF_PÀ´ 3’±ÏÈ»ÐwM…à,¼/(ïc Ã0'ZT s §a!—‰j$)ë6ë䲑£Îdp˲8u6ùhåñ$ùHÂ2;‘”FEÀó^µ~Ð’WäYÙ53­G‡­½w‚ƒ)‹}K.fS(²<Ƨ7cT)õsQV Ó¬DOzø 6gcÃÎr.¶2=èÃÝ3MgCNA1:›X+Þ»ñ˜OЭĒzôû“ˆ{eþ‹Äš#¦çYAÒä§Ì!¦ 2 £ ÊŠð¤÷ʆ» ÛÆSõŽ¡|˜Þ f7 Îgý•õµ‡…£50N v¼ë~Œ±EuÁs&¦ÑëŽiˆ­ÀG“¤3°sâ„“aè¨5™ÍÂâÿÔÐJÍ%®“¢;äCè6àZÉ,eSt'&–µ:ÇþeÇEîAïùËÄ|›\w£óèÜ<þv#xô(¤ù–“qú—ˆ'Ù¢Ò˜B3j:G\£êóñ8šÐØ ÙCEg7ÓR22^ YWVOqW@s4.Bƒ²3ºÐÑÔ£-‚Ùdҙܔ”æî<¢ãqŒâ‘èbz2èðK\<Ó[´ô@Ì8šÑÇx’Œ`kû`q?•L÷ÔUÝÀ"áñþèË«Ôæå±—O&7MO[f…‚I'fKÚ(º@ˆï^ú´Ì_ âs†FÔqààdŠÏg¢+JDë² )FhtR´œA}Á©tÔ§cž5“­"bþiì6ß?¼·Þd’yL ±",+B,P.Ð>ŒØ¤î çA ,žR ‘}2¢ŽØÑtYL %UçX.&X>Uê!0D8÷ël¤>ª`VÏdÝ)ñWæ\~¾¨èuÍŽŠ%#%a¼yFsk¡M@݈º ±iþfãž!¦ÐÃqãRñU¾Õö ¸'Ž¸â…5Ìè{So‚i%…ÌÇ_²²@q6†­^>¬÷é•—Ã^|q±¬Øæåì' D 92iQˆ´_þ#¨òFªD-ÒA ’dŒMvåǘ´Q){ZWV5SzJ‚7-¯ðâ„žA.(4›0àÇLÓ¶¢){Ñ5û}JqÚšw³Ô•Ô¤¿UY篜c»›ÌFÓ­õ”x*¡ âÏ( AóeÂòJSÑôƒ_í×ÍïŸÈøܸÇù‚c[%÷8ø[°þÞÃû1nBà@kÛ%›q9xœ|˜gµ÷«:OÎòù, žþJFçuv…6\g¨èf܆¯gƘ×ýÍãäsìiã HÿtIÿ¤˜O¯ oV*’aèØÑ ¤”ׯôçÎw2Ä\:>a_è3öiFzZÈé’¸ÅQËflfì6.……¼æw¿ŽÁ•ÇÉ*ârÀtâ¨TC5ŒzÀ”šò:m ѱ ‘½dš#[^¶;‡-s¾+ðØðõˆ4Õå¦9¥ñæ™íJBl0÷ØÚ{üÎ÷+ð]8¶šžßNæEc2,L¦¾LrvAø5ü£ª'ZŠyÙZóøc‚u öi¹€›…ŒPý…ôÙÞi– Aɱ§Ô‡JÌÑQ3~Q¶DÝFœ6˜©Wþ/ûâš×7[.JÙ“Ûü%«A{»}L=êË0H»“x,®IˆH€L“«;6üE4GŽâBžª†7(õbÌö­çÖ 6 ZÏ÷w±}9 iÞ§2“J<-ùgûlx¯}>&ovžNi!©''¿Xj±öãŒ,žC©-+Û¾¦¦³hc±»0Ô‚yþ7ÊË'ŠTf»túŠ×€§ÅXødzUr…8“kÅT 8ßTŒuÃíDÅuhUõï²µœ^i(¹(wºtf‹Æqƒ’›«Å×(¡ÆâR·´Ø0²tº—í+õ~åx ì›Gº 5¿6„!hÜ?kP4J€¾¾j0Æä·ÒKp¯‹ºÌ’öI_ïFál4u·¼çël §5Ž/(ê’~ò2(ò«¦r^É{9Ìw=ÙûœÃ½ œøys—žúƼ…ÒYXÝ\,(•óžägv¹loz9ý‘ÝY /lˆýmHE¯[/ª'OíCŸMø£?©€¡Ùz¬¦ ÂŽØÀÕ\}"xáÚ€ÀÛîvÛÝv¢0Í åW»LB` R]õ„Ç^ÖÌ8 8oØyûå`”~ˆ£bÐå ½Ѓa'K—,ÝÃÇÕÀ®æÜ(ãø–g:ì‚q‰GÉJÖk7Í$°n«ÔuA¨³™^ü4ÙVªT€tmZ^¯e¶nØ%ž‡Ë°mxö¬g"Í|ê]’ÈÚû€@¢â®cl›4‰}¶ÁAuMÇŸœÉcª- œq„ ï <_ö‘qŸtŒ3Žh&ÌImÈð;z<©¼û¼•¹äYFt¤ešw?I!Ä– ‹“°Þõ"$5% ;üm‚C×¢L°0G£t(ø¨9Ü¥bO§aU`àe»a¼£Ý&Áµs°¡2»/ÀšÊ6/1Ѥ1¥e‹ãÚ^cÍAÖJÈÈ:nRŸã&„຤”q–á4´Õ(<¤ØÌ•3Ú=~ú´ÆÇa„ò%\¤Õz–©wxÄS_®ÉB\«]Ú‡O=­Y“‚üE$øÕÛ7ÌüU„Øy€žÓÉMÑÐÆhQ5Â4Pó_kØ–b…$}*°»Ó\¬tz±½TÓHô¡µDz›dî2L}É´†³“¶‡Ñfeï`÷à÷s”ƒÇßÂy·Õ ŸÂ·Ÿ±ÎGâ¢w`!(Œ`ë³Uù»Õ5ÍåxZù{°Å÷ÖHàÅùss¹€µƒO9}öÐóõVð™P6ûxˆZ‘8¶Þú¯fƳï”P e ¾ƒ.Y­¯.öËÏÇBûD{8aHÊývi˜ŒÕLæö&#ˆ l·1¦}ørw†èÒmF ÊqB–Úι¹ä„‘kø¤ÁÅVX U£ 7:UMœœœžž–OWN+§ßž†§¬÷\œ©Æ! Ñ‚/.—ì¬f·¢1«-Éï48½³s]ÃJð‚Ï~J–«ˆìIÄv㣽CÝsÕ‰e´Úd2`;Hs=‰±Ç'gïsƒnÁÁæ+Ñ“åœzÀ2ªWM.ÕwB½)N|‰,ÞÏá-âßd6ÀßnóÆ·B.„4lôŒCV^ ?¡zP%eT/Ž§7B_Öòß)ižÂ4hÚàB} wi6<^TQÑUêËdSIÞl­—kŸ#.OýK^+;—,*•(T’ksrX‘ŸüuóIP­"SjÚšÀrsÃÛú§Ã‰ñ[kbO% Þ¯¨å‚÷™ë´n¤U×ÎÄH`ÄQËÒÁת)æ(‹­v³ß¬ɨV¿ÃoÖÑõ´5Æ5N@c ùɈyóÎÝâ«S¡-˜ƒpÿðÃÆCNV¼‡6î>àåuf“S™:}©ABXtY® Jí¤ Smkж„lçCB!Øjï>}qX=9 ™Å©Rýiû¸R}S«•èËš^2C_×ùª– ® â§RE´Ú°'<;«EkátÔ]¸=|A‰yûÎðI/N²õ…¯¬|A¤ «!D7ʵ"aåMhy(i; Ñ@¡È$i°w¼÷ŽµØh ¾Ù’÷òË[<-Æ5{±'Ç­…É'U(ƒIAbÖ‡Ž‡×¾óÃÛ—ġ)ŒoJdÓÉRg"} kp>ÐW¯œçX´KÂæ9ŠÌ%(ÀLj–ôI*®7Túÿç¯köîéq[ áÈ)zâUoÙíØ:™ ,’(;Äí,;¦öbYcŽ/­ø=Èî Üa±ç™šÚ ’È|ÒÁ«´Bb—ÁÖÀN\œ¶—¼Ô}BS©UŠ„ˆ›sÀ2Ý©6›Õs;¶tlæíW‡‡ûÕíÂà„›Œ¾lò±/ÓÃÆŠÉÖÁ¯Áb×HËÞZm¡ä+Ñ+Ïo#o¶÷ ˆ?Ao&X=¢:Vʈ@̶Z“ ’'æûŒi4zþ‚pˆ:ÿ8‘r¸¿Àø{ªZÌÎq7Heïç¹òÇ—Lîµ’ɼò$æ¿”{¨Õ(ÝÜöœ†éRöÎ/„Ýûz©%dyvxürûÕÖÒ6Çèƒß:íÝ’oƒüÍãÚ‚©âëÇa=™Z&äK¨Ãqç:”m@Ogœº]L+Uö¾f];ÆS¿$WÇb —;çè­äÞ0tªñ ÉL±+†~£Èøȯ-gÌøö[?ÿÇÀ¦›SÜ0ý™ÏF®Ìz«Ÿº—ØämÛ‘O&9^¦.>™es-€ÓP'Š+ã¶H† õ%½à=f!ÅÓÏßÙÄÒ68HW¥•Àã¦6UÆýh_Š„ú”Ü»§ÌtØ­pñ¹/gsé@®Tù'}E¯| Õ'6 )OvÅ~)Ž |ŸÔžI/Í·@9žhîPqyP1¦F¼3™ˆl+ìfš9¢[Š#¾Ó‰yxAw'|­Y˜»"ûã Bßoù¥ÇõW8S–+:|Êa`û+òšÂáJh¥¢s—¥²vÁK4{'Bà*Lë?<|h扙^qjÑÕ¼ \Ä×Q¯‘’àG_£Aϲ{sBVmÞ9g¾Üu’Ãèè.¤þ†ÍêõÞŽhÏ÷vpI#Mg«u›‹;˜¥<úáz.FÜ 5 oŸºŽÒY¥ø(sÚ‡‡wïݵiY-ÄÝûÖ¼ké†ëö,îmm¬ýð`ýþ:Ýø…JÜ0út¦­­ ïW8¨˜ÊTPw¶6wâäíIÛÞ=§Á¸ú ¸Šû¡!´¯ +îU×k¦¨™vAza]¯ÃcÁÚ&ü:ÊÛÈ.þ8ÌFæ®yáÿç}•vSã:E†6ª®z´¸Lâž¹m©wR2òÛmøžB5f0m°Oïh{ïõM þ¢@ ­Ûhbvÿœ¡Æ°l[F/x¼êpµÔï¾d†\QàuaÖØ2§¸„¼Ê\)–È÷Ù—n=™kQ½Û V﹩^ÿwU¯Ÿ«^ÿ÷WoYíTNצd“õÅ­ýçÝÏda&ŽåÃ¥m‰Dÿ⻞ëa³•½xÒ‰Y}´rò“4¾6¯tx»ä=+ôv­lºí?œpAº²'z cG4·=¨’ÝìÉî®F^¤â1û—Ñ«Mõ²÷ÛàMAÍ[†x𘵩ɴ5"—XÜÍ’YÍ4ÿÜu†÷aX‰1µî‹›4jaûp(–r-Ó¹®å³…ØÒا†ÄŒl\-Í|>ìâ"pØI®æJÀ" Ë6hŒ'ÔÁ¯"M5èß Îo¾@õ7ÅÔòIØéåJÊwÓæJj®©Ï”!ÿP’÷ŸúWÛ–dí]x ˜÷d1£MVZReYc/aÐy´)š±&dnâ\±ÒÉ@tâù5ëSÙ­ š ½X•:z-¸=²ˆN‹¥Dn†ÕL%e¤Ptc&ž‹ûþÜc–p¥ÔËÈGüØbQ?þº¦Ç(? ªÀ~fÓ»X—åñ¥³6ô4ðce6×ÌÊêÚøQA6ÙÀð.Ÿ«” ɸ­ü\¨¢õb1ˆ<_3ÕϱÉí&fûžÙ„ó·ìnÄ…Ì»÷Î'Â[|Ö7XÚ"ûl&£-êY A©ôÍíç_ýé BLV­&¯3üÓóò÷÷îáïúƒûkþ_þܽ·þÍúƽûk÷7¾¿ÿýýoÖÖïÝ{pÿ›`íO/IÁG¶éà›é$šma¸Ï½ÿýü~¹¨"½qó¯ÐHÖ}]$¶Ö<Þ0d¿D‹ÈÎÞq+ØH·„€í^Òmïl¿Ú¦gÇ»Û;/wo×å3ÿ a§ûgçñ¹ùÿýÚZ~þ?ظÿÿ–Ïÿþ¸ñÜëÿ­ë€Þ±q‚3#¢š Œ*¢•ÛrW`5Þâ”Öx+¬²0ÃÆý‘È*ÆßÔ6…̯P†™^J@ÿ¢ÛêIÉ‘:c}²;¬²{Ë]s·ËÓíçösû¹ýÜ~n?·Ÿûçÿ3h_Èapache-buildstream-27ae392/tests/remoteexecution/project/files/sub-project/project.conf000066400000000000000000000010771514607367700315320ustar00rootroot00000000000000# Project config for frontend build test name: subtest min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ project_dir: file://{project_dir} plugins: - origin: pip package-name: sample-plugins elements: - autotools options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture values: - x86-64 - aarch64 split-rules: test: - | /tests - | /tests/* apache-buildstream-27ae392/tests/remoteexecution/project/project.conf000066400000000000000000000010701514607367700261640ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ project_dir: file://{project_dir} plugins: - origin: pip package-name: sample-plugins elements: - autotools options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture values: - x86-64 - aarch64 split-rules: test: - | /tests - | /tests/* apache-buildstream-27ae392/tests/remoteexecution/remotecache.py000066400000000000000000000060341514607367700250370ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import copy import os import pytest from buildstream.exceptions import ErrorDomain from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remoteexecution DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test building an executable with remote-execution and remote-cache enabled @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_remote_autotools_build(cli, datafiles, remote_services): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # Enable remote cache and remove explicit remote execution CAS configuration. config_without_remote_cache = copy.deepcopy(cli.config) cli.configure({"cache": {"storage-service": {"url": remote_services.storage_service}}}) del cli.config["remote-execution"]["storage-service"] config_with_remote_cache = cli.config # Build element with remote execution. result = cli.run(project=project, args=["build", element_name]) result.assert_success() # Attempt checkout from local cache by temporarily disabling remote cache. # This should fail as the build result shouldn't have been downloaded to the local cache. cli.config = config_without_remote_cache result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_main_error(ErrorDomain.STREAM, "uncached-checkout-attempt") cli.config = config_with_remote_cache # Attempt checkout again with remote cache. result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) apache-buildstream-27ae392/tests/remoteexecution/simple.py000066400000000000000000000053201514607367700240460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remoteexecution DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") # Test building an executable with remote-execution: @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_remote_autotools_build(cli, datafiles): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") element_name = "autotools/amhello.bst" services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout]) result.assert_success() assert_contains( checkout, [ "/usr", "/usr/lib", "/usr/bin", "/usr/share", "/usr/bin/hello", "/usr/share/doc", "/usr/share/doc/amhello", "/usr/share/doc/amhello/README", ], ) # Test running an executable built with remote-execution: @pytest.mark.datafiles(DATA_DIR) def test_remote_autotools_run(cli, datafiles): project = str(datafiles) element_name = "autotools/amhello.bst" services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) services = cli.ensure_services() result = cli.run(project=project, args=["build", element_name]) result.assert_success() result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"]) result.assert_success() assert result.output == "Hello World!\nThis is amhello 1.0.\n" apache-buildstream-27ae392/tests/remoteexecution/workspace.py000066400000000000000000000255551514607367700245670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import re import shutil import pytest from buildstream._testing import cli_remote_execution as cli # pylint: disable=unused-import from buildstream._testing.integration import assert_contains from tests.testutils.site import pip_sample_packages # pylint: disable=unused-import from tests.testutils.site import SAMPLE_PACKAGES_SKIP_REASON pytestmark = pytest.mark.remoteexecution # subdirectories of the buildtree SRC = "src" DEPS = os.path.join(SRC, ".deps") AUTO = "autom4te.cache" DIRS = [os.sep + SRC, os.sep + DEPS, os.sep + AUTO] DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") MAIN = os.path.join(SRC, "main.c") MAINO = os.path.join(SRC, "main.o") CFGMARK = "config-time" BLDMARK = "build-time" def files(): _input_files = [ ".bstproject.yaml", "aclocal.m4", "README", "configure.ac", SRC, MAIN, os.path.join(SRC, "Makefile.am"), "Makefile.am", ] input_files = [os.sep + fname for fname in _input_files] _generated_files = [ "Makefile", "Makefile.in", AUTO, os.path.join(AUTO, "traces.1"), os.path.join(AUTO, "traces.0"), os.path.join(AUTO, "requests"), os.path.join(AUTO, "output.0"), os.path.join(AUTO, "output.1"), "compile", "config.h", "config.h.in", "config.log", "config.status", "configure", "configure.lineno", "depcomp", "install-sh", "missing", os.path.join(SRC, "hello"), DEPS, os.path.join(DEPS, "main.Po"), os.path.join(SRC, "Makefile"), MAINO, CFGMARK, BLDMARK, os.path.join(SRC, "Makefile.in"), "stamp-h1", ] generated_files = [os.sep + fname for fname in _generated_files] _artifacts = [ "usr", os.path.join("usr", "lib"), os.path.join("usr", "bin"), os.path.join("usr", "share"), os.path.join("usr", "bin", "hello"), os.path.join("usr", "share", "doc"), os.path.join("usr", "share", "doc", "amhello"), os.path.join("usr", "share", "doc", "amhello", "README"), ] artifacts = [os.sep + fname for fname in _artifacts] return input_files, generated_files, artifacts def _get_mtimes(root): assert os.path.exists(root) # timestamps on subdirs are not currently semantically meaningful for dirname, _, filenames in os.walk(root): filenames.sort() for filename in filenames: fname = os.path.join(dirname, filename) yield fname[len(root) :], os.stat(fname).st_mtime def get_mtimes(root): return dict(set(_get_mtimes(root))) def check_buildtree( cli, project, element_name, input_files, generated_files, incremental=False, ): # check modified workspace dir was cached # - generated files are present # - generated files are newer than inputs # - check the date recorded in the marker file # - check that the touched file mtime is preserved from before assert cli and project and element_name and input_files and generated_files result = cli.run( project=project, args=[ "shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1", "-exec", "stat", "-c", "%n::%Y", "{}", ";", ], ) result.assert_success() buildtree = {} output = result.output.splitlines() typ_inptime = None typ_gentime = None for line in output: assert "::" in line fname, mtime = line.split("::") # remove the symbolic dir fname = fname[1:] mtime = int(mtime) buildtree[fname] = mtime if incremental: # directory timestamps are not meaningful if fname in DIRS: continue if fname in input_files: if fname != os.sep + MAIN and not typ_inptime: typ_inptime = mtime if fname in generated_files: if fname != os.sep + MAINO and not typ_gentime: typ_gentime = mtime # all expected files should have been found for filename in input_files + generated_files: assert filename in buildtree if incremental: # the source file was changed so should be more recent than other input files # it should be older than the main object. # The main object should be more recent than generated files. assert buildtree[os.sep + MAIN] > typ_inptime assert buildtree[os.sep + MAINO] > buildtree[os.sep + MAIN] assert buildtree[os.sep + MAINO] > typ_gentime for fname in DIRS: del buildtree[fname] return buildtree def get_timemark(cli, project, element_name, marker): result = cli.run( project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", marker[1:]], ) result.assert_success() marker_time = int(result.output) return marker_time @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize( "modification", [ pytest.param("content"), pytest.param("time"), ], ) @pytest.mark.skipif("not pip_sample_packages()", reason=SAMPLE_PACKAGES_SKIP_REASON) def test_workspace_build(cli, tmpdir, datafiles, modification): project = str(datafiles) checkout = os.path.join(cli.directory, "checkout") workspace = os.path.join(cli.directory, "workspace") element_name = "autotools/amhello.bst" # cli args artifact_checkout = ["artifact", "checkout", element_name, "--directory", checkout] build = ["--cache-buildtrees", "always", "build", element_name] input_files, generated_files, artifacts = files() services = cli.ensure_services() assert set(services) == set(["action-cache", "execution", "storage"]) # open a workspace for the element in the workspace directory result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, element_name]) result.assert_success() # check that the workspace path exists assert os.path.exists(workspace) # add a file (asserting later that this is in the buildtree) newfile = "newfile.cfg" newfile_path = os.path.join(workspace, newfile) with open(newfile_path, "w", encoding="utf-8") as fdata: fdata.write("somestring") input_files.append(os.sep + newfile) # check that the workspace *only* contains the expected input files assert_contains(workspace, input_files, strict=True) # save the mtimes for later comparison ws_times = get_mtimes(workspace) # build the element and cache the buildtree result = cli.run(project=project, args=build) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" build_key = cli.get_element_key(project, element_name) # check that the local workspace is unchanged assert_contains(workspace, input_files, strict=True) assert ws_times == get_mtimes(workspace) # check modified workspace dir was cached and save the time # build was run. Incremental build conditions do not apply since the workspace # was initially opened using magic timestamps. build_times = check_buildtree(cli, project, element_name, input_files, generated_files, incremental=False) build_timemark = get_timemark(cli, project, element_name, (os.sep + BLDMARK)) # check that the artifacts are available result = cli.run(project=project, args=artifact_checkout) result.assert_success() assert_contains(checkout, artifacts) shutil.rmtree(checkout) # rebuild the element result = cli.run(project=project, args=build) result.assert_success() assert cli.get_element_state(project, element_name) == "cached" rebuild_key = cli.get_element_key(project, element_name) assert rebuild_key == build_key rebuild_times = check_buildtree(cli, project, element_name, input_files, generated_files, incremental=False) rebuild_timemark = get_timemark(cli, project, element_name, (os.sep + BLDMARK)) # buildmark time should be the same assert build_timemark == rebuild_timemark assert all(rebuild_time == build_times[fname] for fname, rebuild_time in rebuild_times.items()), "{}\n{}".format( rebuild_times, build_times ) # modify the open workspace and rebuild main_path = os.path.join(workspace, MAIN) assert os.path.exists(main_path) if modification == "time": # touch a file in the workspace and save the mtime os.utime(main_path) touched_time = int(os.stat(main_path).st_mtime) elif modification == "content": # change a source file (there's a race here but it's not serious) with open(main_path, "r", encoding="utf-8") as fdata: data = fdata.readlines() with open(main_path, "w", encoding="utf-8") as fdata: for line in data: fdata.write(re.sub(r"Hello", "Goodbye", line)) touched_time = int(os.stat(main_path).st_mtime) else: assert False, "unreachable" # refresh input times ws_times = get_mtimes(workspace) # rebuild the element result = cli.run(project=project, args=build) result.assert_success() rebuild_times = check_buildtree(cli, project, element_name, input_files, generated_files, incremental=True) rebuild_timemark = get_timemark(cli, project, element_name, (os.sep + BLDMARK)) assert rebuild_timemark > build_timemark # check the times of the changed files assert rebuild_times[os.sep + MAIN] == touched_time del rebuild_times[os.sep + MAIN] del rebuild_times[os.sep + MAINO] del rebuild_times[os.sep + SRC + os.sep + "hello"] del rebuild_times[os.sep + DEPS + os.sep + "main.Po"] del rebuild_times[os.sep + BLDMARK] # check the times of the unmodified files assert all(rebuild_time == build_times[fname] for fname, rebuild_time in rebuild_times.items()), "{}\n{}".format( rebuild_times, build_times ) # Check workspace is unchanged assert_contains(workspace, input_files, strict=True) assert ws_times == get_mtimes(workspace) apache-buildstream-27ae392/tests/sandboxes/000077500000000000000000000000001514607367700207525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/__init__.py000066400000000000000000000000001514607367700230510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-command.py000066400000000000000000000023141514607367700244110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "missing-command") @pytest.mark.datafiles(DATA_DIR) def test_missing_command(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "no-runtime.bst"]) result.assert_task_error(ErrorDomain.SANDBOX, "missing-command") assert cli.get_element_state(project, "no-runtime.bst") == "failed" apache-buildstream-27ae392/tests/sandboxes/missing-command/000077500000000000000000000000001514607367700240375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-command/no-runtime.bst000066400000000000000000000000151514607367700266420ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/sandboxes/missing-command/project.conf000066400000000000000000000000341514607367700263510ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/000077500000000000000000000000001514607367700250475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/elements/000077500000000000000000000000001514607367700266635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/elements/base.bst000077500000000000000000000000701514607367700303070ustar00rootroot00000000000000kind: import sources: - kind: local path: files/base/ apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/files/000077500000000000000000000000001514607367700261515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/files/base/000077500000000000000000000000001514607367700270635ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/files/base/bin/000077500000000000000000000000001514607367700276335ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/files/base/bin/sh000077500000000000000000000000341514607367700301700ustar00rootroot00000000000000# This is the original bash apache-buildstream-27ae392/tests/sandboxes/missing-dependencies/project.conf000077500000000000000000000001421514607367700273640ustar00rootroot00000000000000# Project config for missing dependencies test name: test min-version: 2.0 element-path: elements apache-buildstream-27ae392/tests/sandboxes/missing_dependencies.py000066400000000000000000000044531514607367700255110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import utils, _yaml from buildstream.exceptions import ErrorDomain from buildstream._testing._utils.site import IS_LINUX from buildstream._testing import cli # pylint: disable=unused-import # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "missing-dependencies", ) def _symlink_host_tools_to_dir(host_tools, dir_): dir_.mkdir(exist_ok=True) for tool in host_tools: target_path = dir_ / tool os.symlink(utils.get_host_tool(tool), str(target_path)) @pytest.mark.skipif(not IS_LINUX, reason="Only available on Linux") @pytest.mark.datafiles(DATA_DIR) def test_missing_buildbox_run_has_nice_error_message(cli, datafiles, tmp_path): # Create symlink to buildbox-casd and git to work with custom PATH bin_dir = tmp_path / "bin" _symlink_host_tools_to_dir(["buildbox-casd", "git"], bin_dir) project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "depends": [ { "filename": "base.bst", "type": "build", }, ], "config": { "commands": [ "false", ], }, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this should fail with a nice error result = cli.run(project=project, args=["build", "element.bst"], env={"PATH": str(bin_dir)}) result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") assert "not found" in result.stderr apache-buildstream-27ae392/tests/sandboxes/project/000077500000000000000000000000001514607367700224205ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/project/elements/000077500000000000000000000000001514607367700242345ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/project/elements/base.bst000066400000000000000000000001031514607367700256520ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/sandboxes/project/elements/base/000077500000000000000000000000001514607367700251465ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/project/elements/base/base-alpine.bst000066400000000000000000000010061514607367700300350ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/sandboxes/project/elements/import-file1.bst000066400000000000000000000001421514607367700272530ustar00rootroot00000000000000kind: import description: This is the test element sources: - kind: local path: files/file1.txt apache-buildstream-27ae392/tests/sandboxes/project/files/000077500000000000000000000000001514607367700235225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/project/files/file1.txt000066400000000000000000000000001514607367700252510ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/project/project.conf000066400000000000000000000007501514607367700247370ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements aliases: alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ project_dir: file://{project_dir} options: linux: type: bool description: Whether to expect a linux platform default: True arch: type: arch description: Current architecture values: - x86-64 - aarch64 split-rules: test: - | /tests - | /tests/* apache-buildstream-27ae392/tests/sandboxes/remote-exec-config.py000066400000000000000000000037451514607367700250150ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "remote-exec-config") # Assert that if either the client key or client cert is specified # without specifying its counterpart, we get a comprehensive LoadError # instead of an unhandled exception. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) def test_missing_certs(cli, datafiles, config_key, config_value): project = os.path.join(datafiles, "missing-certs") cli.configure( { "remote-execution": { "execution-service": {"url": "http://localhost:8088"}, "storage-service": { "url": "http://charactron:11001", config_key: config_value, }, }, } ) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. result = cli.run(project=project, args=["show", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing") apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/000077500000000000000000000000001514607367700244325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/missing-certs/000077500000000000000000000000001514607367700272215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/missing-certs/certificates/000077500000000000000000000000001514607367700316665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/missing-certs/certificates/client.crt000066400000000000000000000000001514607367700336440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/missing-certs/certificates/client.key000066400000000000000000000000001514607367700336440ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/missing-certs/element.bst000066400000000000000000000000151514607367700313600ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/sandboxes/remote-exec-config/missing-certs/project.conf000066400000000000000000000000341514607367700315330ustar00rootroot00000000000000name: test min-version: 2.0 apache-buildstream-27ae392/tests/sandboxes/selection.py000066400000000000000000000045131514607367700233140ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import utils, _yaml from buildstream.exceptions import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import pytestmark = pytest.mark.integration DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_dummy_sandbox_fallback(cli, datafiles, tmp_path): # Create symlink to buildbox-casd to work with custom PATH buildbox_casd = tmp_path.joinpath("bin/buildbox-casd") buildbox_casd.parent.mkdir() os.symlink(utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox"), str(buildbox_casd)) project = str(datafiles) element_path = os.path.join(project, "elements", "element.bst") # Write out our test target element = { "kind": "script", "depends": [ { "filename": "base.bst", "type": "build", }, ], "config": { "commands": [ "true", ], }, } _yaml.roundtrip_dump(element, element_path) # Build without access to host tools, this will fail result = cli.run( project=project, args=["build", "element.bst"], env={"PATH": str(tmp_path.joinpath("bin"))}, ) # But if we dont spesify a sandbox then we fall back to dummy, we still # fail early but only once we know we need a facny sandbox and that # dumy is not enough, there for element gets fetched and so is buildable result.assert_task_error(ErrorDomain.SANDBOX, "unavailable-local-sandbox") assert cli.get_element_state(project, "element.bst") == "buildable" apache-buildstream-27ae392/tests/sourcecache/000077500000000000000000000000001514607367700212505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/__init__.py000066400000000000000000000000001514607367700233470ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/cache.py000066400000000000000000000121351514607367700226670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing.runcli import cli # pylint: disable=unused-import from buildstream import _yaml DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") @pytest.mark.datafiles(DATA_DIR) def test_patch_sources_cached_1(cli, datafiles): project_dir = str(datafiles) res = cli.run(project=project_dir, args=["build", "source-with-patches-1.bst"]) res.assert_success() source_protos = os.path.join(project_dir, "cache", "source_protos") elementsources_protos = os.path.join(project_dir, "cache", "elementsources") # The two local sources can be cached individually, # the patch source cannot be cached on its own assert len(os.listdir(os.path.join(source_protos, "local"))) == 2 assert not os.path.exists(os.path.join(source_protos, "patch")) assert len(os.listdir(elementsources_protos)) == 1 @pytest.mark.datafiles(DATA_DIR) def test_patch_sources_cached_2(cli, datafiles): project_dir = str(datafiles) res = cli.run(project=project_dir, args=["build", "source-with-patches-2.bst"]) res.assert_success() source_protos = os.path.join(project_dir, "cache", "source_protos") elementsources_protos = os.path.join(project_dir, "cache", "elementsources") # The three local sources can be cached individually, # the patch source cannot be cached on its own assert len(os.listdir(os.path.join(source_protos, "local"))) == 3 assert not os.path.exists(os.path.join(source_protos, "patch")) assert len(os.listdir(elementsources_protos)) == 1 @pytest.mark.datafiles(DATA_DIR) def test_sources_without_patch(cli, datafiles): project_dir = str(datafiles) res = cli.run(project=project_dir, args=["build", "source-without-patches.bst"]) res.assert_success() # No patches so everything should be cached seperately source_protos = os.path.join(project_dir, "cache", "source_protos") elementsources_protos = os.path.join(project_dir, "cache", "elementsources") assert len(os.listdir(os.path.join(source_protos, "local"))) == 3 assert len(os.listdir(elementsources_protos)) == 1 @pytest.mark.datafiles(DATA_DIR) def test_source_cache_key(cli, datafiles): project_dir = str(datafiles) file_path = os.path.join(project_dir, "files") file_url = "file://" + file_path element_path = os.path.join(project_dir, "elements") element_name = "key_check.bst" element = { "kind": "import", "sources": [ { "kind": "remote", "url": os.path.join(file_url, "bin-files", "usr", "bin", "hello"), "directory": "usr/bin", }, { "kind": "remote", "url": os.path.join(file_url, "dev-files", "usr", "include", "pony.h"), "directory": "usr/include", }, {"kind": "patch", "path": "files/hello-patch.diff"}, ], } _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=["source", "track", element_name]) res.assert_success() res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() # Should have source refs for the two remote sources remote_protos = os.path.join(project_dir, "cache", "source_protos", "remote") assert len(os.listdir(remote_protos)) == 2 # Should not have any source refs for the patch source # as that is a transformation of the previous sources, # not cacheable on its own patch_protos = os.path.join(project_dir, "cache", "source_protos", "patch") assert not os.path.exists(patch_protos) # Should have one element sources ref elementsources_protos = os.path.join(project_dir, "cache", "elementsources") assert len(os.listdir(elementsources_protos)) == 1 # modify hello-patch file and check tracking updates refs with open(os.path.join(file_path, "dev-files", "usr", "include", "pony.h"), "a", encoding="utf-8") as f: f.write("\nappending nonsense") res = cli.run(project=project_dir, args=["source", "track", element_name]) res.assert_success() assert "Found new revision: " in res.stderr res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() # We should have a new element sources ref assert len(os.listdir(elementsources_protos)) == 2 apache-buildstream-27ae392/tests/sourcecache/capabilities.py000066400000000000000000000044471514607367700242640ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._project import Project from buildstream import _yaml from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils import dummy_context from tests.testutils.artifactshare import create_dummy_artifact_share DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project", ) @pytest.mark.datafiles(DATA_DIR) def test_artifact_cache_with_missing_capabilities_is_skipped(cli, tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. with create_dummy_artifact_share() as share: # Configure artifact share cache_dir = os.path.join(str(tmpdir), "cache") user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) with dummy_context(config=user_config_file) as context: # Load the project project = Project(project_dir, context) project.ensure_fully_loaded() # Initialize remotes context.initialize_remotes(True, True, None, None) # Create a local artifact cache handle sourcecache = context.sourcecache assert ( not sourcecache.has_fetch_remotes() ), "System didn't realize the source cache didn't support BuildStream" apache-buildstream-27ae392/tests/sourcecache/config.py000066400000000000000000000040331514607367700230670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # Assert that if either the client key or client cert is specified # without specifying its counterpart, we get a comprehensive LoadError # instead of an unhandled exception. @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) def test_missing_certs(cli, datafiles, config_key, config_value): project = os.path.join(datafiles, "missing-certs") project_conf = { "name": "test", "min-version": "2.0", "source-caches": {"url": "https://cache.example.com:12345", "push": "true", config_key: config_value}, } project_conf_file = os.path.join(project, "project.conf") _yaml.roundtrip_dump(project_conf, project_conf_file) # Use `pull` here to ensure we try to initialize the remotes, triggering the error # # This does not happen for a simple `bst show`. result = cli.run(project=project, args=["source", "fetch", "element.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) apache-buildstream-27ae392/tests/sourcecache/fetch.py000066400000000000000000000237041514607367700227210ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name from contextlib import contextmanager import os import shutil import pytest from buildstream.exceptions import ErrorDomain from buildstream._project import Project from buildstream import _yaml from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from tests.testutils import create_artifact_share, dummy_context DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def create_test_element(tmpdir, project_dir): repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) element_path = os.path.join(project_dir, "elements") element_name = "fetch.bst" element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) return element_name, repo, ref @contextmanager def context_with_source_cache(cli, cache, share, tmpdir): user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, } ] }, "cachedir": cache, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) with dummy_context(config=user_config_file) as context: yield context @pytest.mark.datafiles(DATA_DIR) def test_source_fetch(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, _repo, _ref = create_test_element(tmpdir, project_dir) cache_dir = os.path.join(str(tmpdir), "cache") # use artifact cache for sources for now, they should work the same with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements([element_name])[0] element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] assert not share.get_source_proto(source._get_source_name()) # Just check that we sensibly fetch and build the element res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "tar")) != [] # get root digest of source sourcecache = context.sourcecache digest = sourcecache.export(source)._get_digest() # Push the source to the remote res = cli.run(project=project_dir, args=["source", "push", "--source-remote", share.repo, element_name]) res.assert_success() # check the share has the proto and the object assert share.get_source_proto(source._get_source_name()) assert share.has_object(digest) # Delete the source locally shutil.rmtree(os.path.join(str(cache_dir), "sources")) shutil.rmtree(os.path.join(str(cache_dir), "cas")) state = cli.get_element_state(project_dir, element_name) assert state == "fetch needed" # Now fetch the source and check res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() assert "Pulled source" in res.stderr with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements([element_name])[0] # check that we have the source in the cas now and it's not fetched element._query_source_cache() assert element._cached_sources() assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "tar")) == [] @pytest.mark.datafiles(DATA_DIR) def test_fetch_fallback(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, repo, ref = create_test_element(tmpdir, project_dir) cache_dir = os.path.join(str(tmpdir), "cache") # use artifact cache for sources for now, they should work the same with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements([element_name])[0] element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] assert not share.get_source_proto(source._get_source_name()) assert not os.path.exists(os.path.join(cache_dir, "sources")) # Now check if it falls back to the source fetch method. res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() brief_key = source._get_brief_display_key() assert ( "Remote source service ({}) does not have source {} cached".format(share.repo, brief_key) ) in res.stderr assert ("SUCCESS Fetching {}".format(repo.source_config(ref=ref)["url"])) in res.stderr # Check that the source in both in the source dir and the local CAS project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements([element_name])[0] element._query_source_cache() assert element._cached_sources() @pytest.mark.datafiles(DATA_DIR) def test_pull_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, repo, _ref = create_test_element(tmpdir, project_dir) cache_dir = os.path.join(str(tmpdir), "cache") with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements([element_name])[0] element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] # remove files and check that it doesn't build shutil.rmtree(repo.repo) # Should fail in stream, with a plugin task causing the error res = cli.run(project=project_dir, args=["build", element_name]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.SOURCE, None) assert ( "Remote source service ({}) does not have source {} cached".format( share.repo, source._get_brief_display_key() ) in res.stderr ) @pytest.mark.datafiles(DATA_DIR) def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles): project_dir = str(datafiles) element_name, repo, ref = create_test_element(tmpdir, project_dir) cache_dir = os.path.join(str(tmpdir), "cache") # use artifact cache for sources for now, they should work the same with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: with context_with_source_cache(cli, cache_dir, share, tmpdir) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements([element_name])[0] element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] assert not share.get_artifact_proto(source._get_source_name()) # Just check that we sensibly fetch and build the element res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() assert os.listdir(os.path.join(str(tmpdir), "cache", "sources", "tar")) != [] # get root digest of source sourcecache = context.sourcecache digest = sourcecache.export(source)._get_digest() # Push the source to the remote res = cli.run(project=project_dir, args=["source", "push", "--source-remote", share.repo, element_name]) res.assert_success() # Remove the cas content, only keep the proto and such around shutil.rmtree(os.path.join(str(tmpdir), "sourceshare", "repo", "cas", "objects")) # check the share doesn't have the object assert not share.has_object(digest) # Delete the source locally shutil.rmtree(os.path.join(str(cache_dir), "sources")) shutil.rmtree(os.path.join(str(cache_dir), "cas")) state = cli.get_element_state(project_dir, element_name) assert state == "fetch needed" # Now fetch the source and check res = cli.run(project=project_dir, args=["source", "fetch", element_name]) res.assert_success() assert ("SUCCESS Fetching {}".format(repo.source_config(ref=ref)["url"])) in res.stderr apache-buildstream-27ae392/tests/sourcecache/missing-certs/000077500000000000000000000000001514607367700240375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/missing-certs/certificates/000077500000000000000000000000001514607367700265045ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/missing-certs/certificates/client.crt000066400000000000000000000000001514607367700304620ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/missing-certs/certificates/client.key000066400000000000000000000000001514607367700304620ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/missing-certs/element.bst000066400000000000000000000000151514607367700261760ustar00rootroot00000000000000kind: manual apache-buildstream-27ae392/tests/sourcecache/project/000077500000000000000000000000001514607367700227165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/elements/000077500000000000000000000000001514607367700245325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/elements/compose-all.bst000066400000000000000000000003441514607367700274600ustar00rootroot00000000000000kind: compose depends: - filename: import-bin.bst type: build - filename: import-dev.bst type: build config: # Dont try running the sandbox, we dont have a # runtime to run anything in this context. integrate: False apache-buildstream-27ae392/tests/sourcecache/project/elements/import-bin.bst000066400000000000000000000000741514607367700273250ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files apache-buildstream-27ae392/tests/sourcecache/project/elements/import-dev.bst000066400000000000000000000000741514607367700273330ustar00rootroot00000000000000kind: import sources: - kind: local path: files/dev-files apache-buildstream-27ae392/tests/sourcecache/project/elements/source-with-patches-1.bst000066400000000000000000000002171514607367700313000ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files - kind: patch path: files/hello-patch.diff - kind: local path: files/dev-files apache-buildstream-27ae392/tests/sourcecache/project/elements/source-with-patches-2.bst000066400000000000000000000002741514607367700313040ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files - kind: local path: files/dev-files - kind: local path: files/hello-patch.diff - kind: patch path: files/hello-patch.diff apache-buildstream-27ae392/tests/sourcecache/project/elements/source-without-patches.bst000066400000000000000000000002171514607367700316720ustar00rootroot00000000000000kind: import sources: - kind: local path: files/bin-files - kind: local path: files/dev-files - kind: local path: files/hello-patch.diff apache-buildstream-27ae392/tests/sourcecache/project/elements/target.bst000066400000000000000000000002051514607367700265270ustar00rootroot00000000000000kind: stack description: | Main stack target for the bst build test depends: - import-bin.bst - import-dev.bst - compose-all.bst apache-buildstream-27ae392/tests/sourcecache/project/files/000077500000000000000000000000001514607367700240205ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/bin-files/000077500000000000000000000000001514607367700256705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/bin-files/usr/000077500000000000000000000000001514607367700265015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/bin-files/usr/bin/000077500000000000000000000000001514607367700272515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/bin-files/usr/bin/hello000077500000000000000000000000341514607367700302770ustar00rootroot00000000000000#!/bin/bash echo "Hello !" apache-buildstream-27ae392/tests/sourcecache/project/files/dev-files/000077500000000000000000000000001514607367700256765ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/dev-files/usr/000077500000000000000000000000001514607367700265075ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/dev-files/usr/include/000077500000000000000000000000001514607367700301325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/files/dev-files/usr/include/pony.h000066400000000000000000000003711514607367700312710ustar00rootroot00000000000000#ifndef __PONY_H__ #define __PONY_H__ #define PONY_BEGIN "Once upon a time, there was a pony." #define PONY_END "And they lived happily ever after, the end." #define MAKE_PONY(story) \ PONY_BEGIN \ story \ PONY_END #endif /* __PONY_H__ */ apache-buildstream-27ae392/tests/sourcecache/project/files/hello-patch.diff000066400000000000000000000001561514607367700270540ustar00rootroot00000000000000--- a/usr/bin/hello +++ b/usr/bin/hello @@ -1,3 +1,3 @@ #!/bin/bash -echo "Hello !" +echo "Patched hello !" apache-buildstream-27ae392/tests/sourcecache/project/plugins/000077500000000000000000000000001514607367700243775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/plugins/elements/000077500000000000000000000000001514607367700262135ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/plugins/elements/always_fail.py000066400000000000000000000016221514607367700310610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # from buildstream.element import ElementError from buildstream.buildelement import BuildElement class AlwaysFail(BuildElement): BST_MIN_VERSION = "2.0" def assemble(self, sandbox): raise ElementError("Always fails") def setup(): return AlwaysFail apache-buildstream-27ae392/tests/sourcecache/project/plugins/elements/always_fail.yaml000066400000000000000000000006371514607367700314000ustar00rootroot00000000000000# always-fail build element does not provide any default # build commands config: # Commands for configuring the software # configure-commands: [] # Commands for building the software # build-commands: [] # Commands for installing the software into a # destination folder # install-commands: [] # Commands for stripping installed binaries # strip-commands: - | %{strip-binaries} apache-buildstream-27ae392/tests/sourcecache/project/plugins/sources/000077500000000000000000000000001514607367700260625ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sourcecache/project/plugins/sources/patch.py000066400000000000000000000064211514607367700275360ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Chandan Singh # Tiago Gomes """ patch - apply locally stored patches ==================================== **Host dependencies:** * patch **Usage:** .. code:: yaml # Specify the local source kind kind: patch # Specify the project relative path to a patch file path: files/somefile.diff # Optionally specify the strip level, defaults to 1 strip-level: 1 See `built-in functionality doumentation `_ for details on common configuration options for sources. """ import os from buildstream import Source, SourceError from buildstream import utils class PatchSource(Source): # pylint: disable=attribute-defined-outside-init BST_MIN_VERSION = "2.0" BST_REQUIRES_PREVIOUS_SOURCES_STAGE = True def configure(self, node): node.validate_keys(["path", "strip-level", *Source.COMMON_CONFIG_KEYS]) self.path = self.node_get_project_path(node.get_scalar("path"), check_is_file=True) self.strip_level = node.get_int("strip-level", default=1) self.fullpath = os.path.join(self.get_project_directory(), self.path) def preflight(self): # Check if patch is installed, get the binary at the same time self.host_patch = utils.get_host_tool("patch") def get_unique_key(self): return [self.path, utils.sha256sum(self.fullpath), self.strip_level] def is_resolved(self): return True def is_cached(self): return True def load_ref(self, node): pass def get_ref(self): return None # pragma: nocover def set_ref(self, ref, node): pass # pragma: nocover def fetch(self): # pylint: disable=arguments-differ # Nothing to do here for a local source pass # pragma: nocover def stage(self, directory): with self.timed_activity("Applying local patch: {}".format(self.path)): # Bail out with a comprehensive message if the target directory is empty if not os.listdir(directory): raise SourceError( "Nothing to patch in directory '{}'".format(directory), reason="patch-no-files", ) strip_level_option = "-p{}".format(self.strip_level) self.call( [ self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory, ], fail="Failed to apply patch {}".format(self.path), ) # Plugin entry point def setup(): return PatchSource apache-buildstream-27ae392/tests/sourcecache/project/project.conf000066400000000000000000000003521514607367700252330ustar00rootroot00000000000000# Project config for frontend build test name: test min-version: 2.0 element-path: elements plugins: - origin: local path: plugins/elements elements: - always_fail - origin: local path: plugins/sources sources: - patch apache-buildstream-27ae392/tests/sourcecache/push.py000066400000000000000000000321661514607367700226110ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil from contextlib import contextmanager, ExitStack import pytest from buildstream.exceptions import ErrorDomain from buildstream._project import Project from buildstream import _yaml from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from tests.testutils import create_artifact_share, dummy_context DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def message_handler(message, is_silenced): pass # Args: # tmpdir: A temporary directory to use as root. # directories: Directory names to use as cache directories. # @contextmanager def _configure_caches(tmpdir, *directories): with ExitStack() as stack: def create_share(directory): return create_artifact_share(os.path.join(str(tmpdir), directory)) yield (stack.enter_context(create_share(remote)) for remote in directories) @pytest.mark.datafiles(DATA_DIR) def test_source_push_split(cli, tmpdir, datafiles): cache_dir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) with _configure_caches(tmpdir, "indexshare", "storageshare") as (index, storage): user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ {"url": index.repo, "push": True, "type": "index"}, {"url": storage.repo, "push": True, "type": "storage"}, ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) element_path = os.path.join(project_dir, "elements") element_name = "push.bst" element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # get the source object with dummy_context(config=user_config_file) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements(["push.bst"])[0] element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] # check we don't have it in the current cache assert not index.get_source_proto(source._get_source_name()) # build the element, this should fetch and then push the source to the # remote res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() assert "Pushed source" in res.stderr # check that we've got the remote locally now sourcecache = context.sourcecache assert sourcecache.contains(source) # check that the remote CAS now has it digest = sourcecache.export(source)._get_digest() assert storage.has_object(digest) @pytest.mark.datafiles(DATA_DIR) def test_source_push(cli, tmpdir, datafiles): cache_dir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) element_path = os.path.join(project_dir, "elements") element_name = "push.bst" element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # get the source object with dummy_context(config=user_config_file) as context: project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements(["push.bst"])[0] element._query_source_cache() assert not element._cached_sources() source = list(element.sources())[0] # check we don't have it in the current cache assert not share.get_source_proto(source._get_source_name()) # build the element, this should fetch and then push the source to the # remote res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() assert "Pushed source" in res.stderr # check that we've got the remote locally now sourcecache = context.sourcecache assert sourcecache.contains(source) # check that the remote CAS now has it digest = sourcecache.export(source)._get_digest() assert share.has_object(digest) @pytest.mark.datafiles(DATA_DIR) def test_push_pull(cli, datafiles, tmpdir): project_dir = str(datafiles) cache_dir = os.path.join(str(tmpdir), "cache") with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) # create repo to pull from repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) element_path = os.path.join(project_dir, "elements") element_name = "push.bst" element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() # remove local cache dir, and repo files and check it all works shutil.rmtree(cache_dir) os.makedirs(cache_dir) shutil.rmtree(repo.repo) # check it's pulls from the share res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_push_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) cache_dir = os.path.join(str(tmpdir), "cache") # set up config with remote that we'll take down with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: remote = share.repo user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) # create repo to pull from repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) element_path = os.path.join(project_dir, "elements") element_name = "push.bst" element = {"kind": "import", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) # build and check that it fails to set up the remote res = cli.run(project=project_dir, args=["build", "push.bst"]) res.assert_success() assert "Failed to initialize remote {}".format(remote) in res.stderr assert "Pushing" not in res.stderr assert "Pushed" not in res.stderr @pytest.mark.datafiles(DATA_DIR) def test_source_push_build_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) cache_dir = os.path.join(str(tmpdir), "cache") with create_artifact_share(os.path.join(str(tmpdir), "share")) as share: user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } cli.configure(user_config) repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project_dir, "files")) element_path = os.path.join(project_dir, "elements") element_name = "always-fail.bst" element = {"kind": "always_fail", "sources": [repo.source_config(ref=ref)]} _yaml.roundtrip_dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=["build", "always-fail.bst"]) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.ELEMENT, None) # Sources are still pushed after failing a build assert "Pushed source " in res.stderr # Test that source push succeeds if the source needs to be fetched # even if the artifact of the corresponding element is already cached. @pytest.mark.datafiles(DATA_DIR) def test_push_missing_source_after_build(cli, tmpdir, datafiles): cache_dir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) element_name = "import-bin.bst" res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() # Delete source but keep artifact in cache shutil.rmtree(os.path.join(cache_dir, "elementsources")) shutil.rmtree(os.path.join(cache_dir, "source_protos")) with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) res = cli.run(project=project_dir, args=["source", "push", element_name]) res.assert_success() assert "fetch:{}".format(element_name) in res.stderr assert "Pushed source" in res.stderr # Regression test for https://github.com/apache/buildstream/issues/1456 # Test that a build pipeline with source push enabled doesn't fail if an # element is already cached. @pytest.mark.datafiles(DATA_DIR) def test_build_push_source_twice(cli, tmpdir, datafiles): cache_dir = os.path.join(str(tmpdir), "cache") project_dir = str(datafiles) element_name = "import-bin.bst" with create_artifact_share(os.path.join(str(tmpdir), "sourceshare")) as share: user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, "cachedir": cache_dir, } _yaml.roundtrip_dump(user_config, file=user_config_file) cli.configure(user_config) res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() assert "fetch:{}".format(element_name) in res.stderr assert "Pushed source" in res.stderr # The second build pipeline is a no-op as everything is already cached. # However, this verifies that the pipeline behaves as expected. res = cli.run(project=project_dir, args=["build", element_name]) res.assert_success() assert "fetch:{}".format(element_name) not in res.stderr assert "Pushed source" not in res.stderr apache-buildstream-27ae392/tests/sourcecache/source-checkout.py000066400000000000000000000045341514607367700247330ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream.exceptions import ErrorDomain from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils.element_generators import create_element_size DATA_DIR = os.path.dirname(os.path.realpath(__file__)) @pytest.mark.datafiles(DATA_DIR) def test_source_checkout(tmpdir, datafiles, cli): project_dir = os.path.join(str(tmpdir), "project") element_path = "elements" cache_dir = os.path.join(str(tmpdir), "cache") source_dir = os.path.join(cache_dir, "sources") cli.configure( { "cachedir": cache_dir, } ) target_dir = os.path.join(str(tmpdir), "target") repo = create_element_size("target.bst", project_dir, element_path, [], 100000) # check implicit fetching res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"]) res.assert_success() assert "Fetching" in res.stderr # remove the directory and check source checkout works with sources only in # the CAS shutil.rmtree(repo.repo) shutil.rmtree(target_dir) shutil.rmtree(source_dir) res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"]) res.assert_success() assert "Fetching" not in res.stderr # remove the CAS and check it doesn't work again shutil.rmtree(target_dir) shutil.rmtree(os.path.join(cache_dir, "cas")) res = cli.run(project=project_dir, args=["source", "checkout", "--directory", target_dir, "target.bst"]) res.assert_task_error(ErrorDomain.SOURCE, None) apache-buildstream-27ae392/tests/sourcecache/staging.py000066400000000000000000000142431514607367700232620ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._project import Project from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils import dummy_context from tests.testutils.element_generators import create_element_size DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # walk that removes the root directory from roots def relative_walk(rootdir): for root, dirnames, filenames in os.walk(rootdir): relative_root = root.split(rootdir)[1] yield (relative_root, dirnames, filenames) @pytest.mark.datafiles(DATA_DIR) def test_source_staged(tmpdir, cli, datafiles): project_dir = os.path.join(datafiles, "project") cachedir = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": cachedir}) res = cli.run(project=project_dir, args=["build", "import-bin.bst"]) res.assert_success() with dummy_context() as context: context.cachedir = cachedir # load project and sourcecache project = Project(project_dir, context) project.ensure_fully_loaded() sourcecache = context.sourcecache cas = context.get_cascache() # now check that the source is in the refs file, this is pretty messy but # seems to be the only way to get the sources? element = project.load_elements(["import-bin.bst"])[0] element._query_source_cache() source = list(element.sources())[0] assert element._cached_sources() assert sourcecache.contains(source) # Extract the file and check it's the same as the one we imported digest = sourcecache.export(source)._get_digest() extractdir = os.path.join(str(tmpdir), "extract") cas.checkout(extractdir, digest) dir1 = extractdir dir2 = os.path.join(project_dir, "files", "bin-files") assert list(relative_walk(dir1)) == list(relative_walk(dir2)) # Check sources are staged during a fetch @pytest.mark.datafiles(DATA_DIR) def test_source_fetch(tmpdir, cli, datafiles): project_dir = os.path.join(datafiles, "project") cachedir = os.path.join(str(tmpdir), "cache") cli.configure({"cachedir": cachedir}) res = cli.run(project=project_dir, args=["source", "fetch", "import-dev.bst"]) res.assert_success() with dummy_context() as context: context.cachedir = cachedir # load project and sourcecache project = Project(project_dir, context) project.ensure_fully_loaded() cas = context.get_cascache() sourcecache = context.sourcecache element = project.load_elements(["import-dev.bst"])[0] element._query_source_cache() source = list(element.sources())[0] assert element._cached_sources() # check that the directory structures are identical digest = sourcecache.export(source)._get_digest() extractdir = os.path.join(str(tmpdir), "extract") cas.checkout(extractdir, digest) dir1 = extractdir dir2 = os.path.join(project_dir, "files", "dev-files") assert list(relative_walk(dir1)) == list(relative_walk(dir2)) # Check that with sources only in the CAS build successfully completes @pytest.mark.datafiles(DATA_DIR) def test_staged_source_build(tmpdir, datafiles, cli): project_dir = os.path.join(datafiles, "project") cachedir = os.path.join(str(tmpdir), "cache") element_path = "elements" source_protos = os.path.join(str(tmpdir), "cache", "source_protos") elementsources = os.path.join(str(tmpdir), "cache", "elementsources") source_dir = os.path.join(str(tmpdir), "cache", "sources") cli.configure({"cachedir": cachedir}) create_element_size("target.bst", project_dir, element_path, [], 10000) with dummy_context() as context: context.cachedir = cachedir project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements(["import-dev.bst"])[0] # check consistency of the source element._query_source_cache() assert not element._cached_sources() res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() # delete artifacts check state is buildable cli.remove_artifact_from_cache(project_dir, "target.bst") states = cli.get_element_states(project_dir, ["target.bst"]) assert states["target.bst"] == "buildable" # delete source dir and check that state is still buildable shutil.rmtree(source_dir) states = cli.get_element_states(project_dir, ["target.bst"]) assert states["target.bst"] == "buildable" # build and check that no fetching was done. res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() assert "Fetching" not in res.stderr # assert the source directory is still empty (though there may be # directories from staging etc.) files = [] for _, _, filename in os.walk(source_dir): files.extend(filename) assert not files, files # Now remove the source refs and check the state shutil.rmtree(source_protos) shutil.rmtree(elementsources) cli.remove_artifact_from_cache(project_dir, "target.bst") states = cli.get_element_states(project_dir, ["target.bst"]) assert states["target.bst"] == "fetch needed" # Check that it now fetches from when building the target res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() assert "Fetching" in res.stderr apache-buildstream-27ae392/tests/sourcecache/workspace.py000066400000000000000000000071201514607367700236200ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._testing.runcli import cli # pylint: disable=unused-import from tests.testutils.artifactshare import create_artifact_share from tests.testutils.element_generators import create_element_size DATA_DIR = os.path.dirname(os.path.realpath(__file__)) # Test that when we have sources only in the local CAS buildstream fetches them # for opening a workspace @pytest.mark.datafiles(DATA_DIR) def test_workspace_source_fetch(tmpdir, datafiles, cli): project_dir = os.path.join(str(tmpdir), "project") element_path = "elements" source_dir = os.path.join(str(tmpdir), "cache", "sources") workspace = os.path.join(cli.directory, "workspace") cli.configure({"cachedir": os.path.join(str(tmpdir), "cache")}) create_element_size("target.bst", project_dir, element_path, [], 10000) res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() assert "Fetching" in res.stderr # remove the original sources shutil.rmtree(source_dir) # Open a workspace and check that fetches the original sources res = cli.run(project=project_dir, args=["workspace", "open", "target.bst", "--directory", workspace]) res.assert_success() assert "Fetching" in res.stderr assert os.listdir(workspace) != [] @pytest.mark.datafiles(DATA_DIR) def test_workspace_open_no_source_push(tmpdir, datafiles, cli): project_dir = os.path.join(str(tmpdir), "project") element_path = "elements" cache_dir = os.path.join(str(tmpdir), "cache") share_dir = os.path.join(str(tmpdir), "share") workspace = os.path.join(cli.directory, "workspace") with create_artifact_share(share_dir) as share: cli.configure( { "cachedir": cache_dir, "scheduler": {"pushers": 1}, "source-caches": { "servers": [ { "url": share.repo, "push": True, } ] }, } ) # Fetch as in previous test and check it pushes the source create_element_size("target.bst", project_dir, element_path, [], 10000) res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() assert "Fetching" in res.stderr assert "Pushed source" in res.stderr # clear the cas and open a workspace shutil.rmtree(os.path.join(cache_dir, "cas")) res = cli.run(project=project_dir, args=["workspace", "open", "target.bst", "--directory", workspace]) res.assert_success() # Check that this time it does not push the sources res = cli.run(project=project_dir, args=["build", "target.bst"]) res.assert_success() assert "Pushed source" not in res.stderr apache-buildstream-27ae392/tests/sources/000077500000000000000000000000001514607367700204475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/__init__.py000066400000000000000000000014241514607367700225610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os def list_dir_contents(srcdir): contents = set() for _, dirs, files in os.walk(srcdir): for d in dirs: contents.add(d) for f in files: contents.add(f) return contents apache-buildstream-27ae392/tests/sources/keytest.py000066400000000000000000000034651514607367700225210ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import ErrorDomain from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project_key_test") # using the key-test plugin to ensure get_unique_key is never called before # refs are resolved @pytest.mark.datafiles(DATA_DIR) def test_generate_key(cli, datafiles): project_dir = str(datafiles) # check that we don't fail if not tracking due to get_unique_key res = cli.run(project=project_dir, args=["build", "key-test.bst"]) res.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline") assert cli.get_element_state(project_dir, "key-test.bst") == "no reference" res = cli.run(project=project_dir, args=["source", "track", "key-test.bst"]) res.assert_success() assert cli.get_element_state(project_dir, "key-test.bst") == "fetch needed" res = cli.run(project=project_dir, args=["build", "key-test.bst"]) res.assert_success() assert cli.get_element_state(project_dir, "key-test.bst") == "cached" apache-buildstream-27ae392/tests/sources/local.py000066400000000000000000000204131514607367700221130ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream import _yaml from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_SANDBOX from tests.testutils import filetypegenerator DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "local", ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_missing_path(cli, datafiles): project = str(datafiles) # Removing the local file causes preflight to fail localfile = os.path.join(project, "file.txt") os.remove(localfile) result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_non_regular_file_or_directory(cli, datafiles): project = str(datafiles) localfile = os.path.join(project, "file.txt") for _file_type in filetypegenerator.generate_file_types(localfile): result = cli.run(project=project, args=["show", "target.bst"]) if os.path.isdir(localfile) and not os.path.islink(localfile): result.assert_success() elif os.path.isfile(localfile) and not os.path.islink(localfile): result.assert_success() else: result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID_KIND) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_invalid_absolute_path(cli, datafiles): project = str(datafiles) with open(os.path.join(project, "target.bst"), "r", encoding="utf-8") as f: old_yaml = f.read() new_yaml = old_yaml.replace("file.txt", os.path.join(project, "file.txt")) assert old_yaml != new_yaml with open(os.path.join(project, "target.bst"), "w", encoding="utf-8") as f: f.write(new_yaml) result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID) @pytest.mark.datafiles(os.path.join(DATA_DIR, "invalid-relative-path")) def test_invalid_relative_path(cli, datafiles): project = str(datafiles) result = cli.run(project=project, args=["show", "target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.PROJ_PATH_INVALID) @pytest.mark.datafiles(os.path.join(DATA_DIR, "basic")) def test_stage_file(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file assert os.path.exists(os.path.join(checkoutdir, "file.txt")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "directory")) def test_stage_directory(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file and directory and other file assert os.path.exists(os.path.join(checkoutdir, "file.txt")) assert os.path.exists(os.path.join(checkoutdir, "subdir", "anotherfile.txt")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "symlink")) def test_stage_symlink(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Workaround datafiles bug: # # https://github.com/omarkohl/pytest-datafiles/issues/1 # # Create the symlink by hand. symlink = os.path.join(project, "files", "symlink-to-file.txt") os.symlink("file.txt", symlink) # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file and directory and other file assert os.path.exists(os.path.join(checkoutdir, "file.txt")) assert os.path.exists(os.path.join(checkoutdir, "symlink-to-file.txt")) assert os.path.islink(os.path.join(checkoutdir, "symlink-to-file.txt")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "file-exists")) def test_stage_file_exists(cli, datafiles): project = str(datafiles) # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.ELEMENT, "stage-sources-fail") @pytest.mark.datafiles(os.path.join(DATA_DIR, "directory")) def test_stage_directory_symlink(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") symlink = os.path.join(project, "files", "symlink-to-subdir") os.symlink("subdir", symlink) # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected directory and directory symlink assert os.path.exists(os.path.join(checkoutdir, "subdir", "anotherfile.txt")) assert os.path.exists(os.path.join(checkoutdir, "symlink-to-subdir", "anotherfile.txt")) assert os.path.islink(os.path.join(checkoutdir, "symlink-to-subdir")) @pytest.mark.integration @pytest.mark.datafiles(os.path.join(DATA_DIR, "deterministic-umask")) @pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") def test_deterministic_source_umask(cli, tmpdir, datafiles): def create_test_file(*path, mode=0o644, content="content\n"): path = os.path.join(*path) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w", encoding="utf-8") as f: f.write(content) os.fchmod(f.fileno(), mode) def create_test_directory(*path, mode=0o644): create_test_file(*path, ".keep", content="") path = os.path.join(*path) os.chmod(path, mode) project = str(datafiles) element_name = "list.bst" element_path = os.path.join(project, "elements", element_name) sourcedir = os.path.join(project, "source") create_test_file(sourcedir, "a.txt", mode=0o700) create_test_file(sourcedir, "b.txt", mode=0o755) create_test_file(sourcedir, "c.txt", mode=0o600) create_test_file(sourcedir, "d.txt", mode=0o400) create_test_file(sourcedir, "e.txt", mode=0o644) create_test_file(sourcedir, "f.txt", mode=0o4755) create_test_file(sourcedir, "g.txt", mode=0o2755) create_test_file(sourcedir, "h.txt", mode=0o1755) create_test_directory(sourcedir, "dir-a", mode=0o0700) create_test_directory(sourcedir, "dir-c", mode=0o0755) create_test_directory(sourcedir, "dir-d", mode=0o4755) create_test_directory(sourcedir, "dir-e", mode=0o2755) create_test_directory(sourcedir, "dir-f", mode=0o1755) source = {"kind": "local", "path": "source"} element = { "kind": "manual", "depends": [{"filename": "base.bst", "type": "build"}], "sources": [source], "config": {"install-commands": ['ls -l >"%{install-root}/ls-l"']}, } _yaml.roundtrip_dump(element, element_path) apache-buildstream-27ae392/tests/sources/local/000077500000000000000000000000001514607367700215415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/basic/000077500000000000000000000000001514607367700226225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/basic/file.txt000066400000000000000000000000241514607367700242760ustar00rootroot00000000000000This is a text file apache-buildstream-27ae392/tests/sources/local/basic/project.conf000066400000000000000000000000531514607367700251350ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/sources/local/basic/target.bst000066400000000000000000000001231514607367700246160ustar00rootroot00000000000000kind: import description: This is the pony sources: - kind: local path: file.txt apache-buildstream-27ae392/tests/sources/local/deterministic-umask/000077500000000000000000000000001514607367700255225ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/deterministic-umask/elements/000077500000000000000000000000001514607367700273365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/deterministic-umask/elements/base.bst000066400000000000000000000001031514607367700307540ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/sources/local/deterministic-umask/elements/base/000077500000000000000000000000001514607367700302505ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/deterministic-umask/elements/base/base-alpine.bst000066400000000000000000000010061514607367700331370ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/sources/local/directory/000077500000000000000000000000001514607367700235455ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/directory/files/000077500000000000000000000000001514607367700246475ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/directory/files/file.txt000066400000000000000000000000201514607367700263170ustar00rootroot00000000000000I'm a text file apache-buildstream-27ae392/tests/sources/local/directory/files/subdir/000077500000000000000000000000001514607367700261375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/directory/files/subdir/anotherfile.txt000066400000000000000000000000261514607367700311760ustar00rootroot00000000000000I'm another text file apache-buildstream-27ae392/tests/sources/local/directory/project.conf000066400000000000000000000000531514607367700260600ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/sources/local/directory/target.bst000066400000000000000000000001201514607367700255360ustar00rootroot00000000000000kind: import description: This is the pony sources: - kind: local path: files apache-buildstream-27ae392/tests/sources/local/file-exists/000077500000000000000000000000001514607367700237755ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/file-exists/files/000077500000000000000000000000001514607367700250775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/file-exists/files/file.txt000066400000000000000000000000201514607367700265470ustar00rootroot00000000000000I'm a text file apache-buildstream-27ae392/tests/sources/local/file-exists/project.conf000066400000000000000000000000531514607367700263100ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/sources/local/file-exists/target.bst000066400000000000000000000005651514607367700260030ustar00rootroot00000000000000kind: import description: | This is the regular file staged twice, second time we stage into a subdir of the staging area. The subdir we specify is the file we already staged, provoking a plausible error where the user tries to stage something unreasonable. sources: - kind: local path: files/file.txt - kind: local path: files/file.txt directory: file.txt apache-buildstream-27ae392/tests/sources/local/invalid-relative-path/000077500000000000000000000000001514607367700257325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/invalid-relative-path/file.txt000066400000000000000000000000241514607367700274060ustar00rootroot00000000000000This is a text file apache-buildstream-27ae392/tests/sources/local/invalid-relative-path/project.conf000066400000000000000000000000531514607367700302450ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/sources/local/invalid-relative-path/target.bst000066400000000000000000000001541514607367700277320ustar00rootroot00000000000000kind: import description: This is the pony sources: - kind: local path: ../invalid-relative-path/file.txt apache-buildstream-27ae392/tests/sources/local/symlink/000077500000000000000000000000001514607367700232275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/symlink/files/000077500000000000000000000000001514607367700243315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/local/symlink/files/file.txt000066400000000000000000000000201514607367700260010ustar00rootroot00000000000000I'm a text file apache-buildstream-27ae392/tests/sources/local/symlink/project.conf000066400000000000000000000000531514607367700255420ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/sources/local/symlink/target.bst000066400000000000000000000001201514607367700252200ustar00rootroot00000000000000kind: import description: This is the pony sources: - kind: local path: files apache-buildstream-27ae392/tests/sources/no-fetch-cached/000077500000000000000000000000001514607367700233575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/no-fetch-cached/files/000077500000000000000000000000001514607367700244615ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/no-fetch-cached/files/file000066400000000000000000000000151514607367700253170ustar00rootroot00000000000000Hello World! apache-buildstream-27ae392/tests/sources/no-fetch-cached/plugins/000077500000000000000000000000001514607367700250405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/no-fetch-cached/plugins/sources/000077500000000000000000000000001514607367700265235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/no-fetch-cached/plugins/sources/always_cached.py000066400000000000000000000014771514607367700316750ustar00rootroot00000000000000""" always_cached ============= This is a test source plugin that is always cached. Used to test that BuildStream core does not call fetch() for cached sources. """ from buildstream import Source class AlwaysCachedSource(Source): BST_MIN_VERSION = "2.0" def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return None def is_resolved(self): return True def is_cached(self): return True def load_ref(self, node): pass def get_ref(self): return None def set_ref(self, ref, node): pass def fetch(self): # Source is always cached, so fetch() should never be called assert False def stage(self, directory): pass def setup(): return AlwaysCachedSource apache-buildstream-27ae392/tests/sources/no-fetch-cached/project.conf000066400000000000000000000002321514607367700256710ustar00rootroot00000000000000# Project with local source plugins name: no-fetch-cached min-version: 2.0 plugins: - origin: local path: plugins/sources sources: - always_cached apache-buildstream-27ae392/tests/sources/no_fetch_cached.py000066400000000000000000000035011514607367700240740ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing import create_repo from buildstream._testing import generate_element DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "no-fetch-cached") ################################################################## # Tests # ################################################################## # Test that fetch() is not called for cached sources @pytest.mark.datafiles(DATA_DIR) def test_no_fetch_cached(cli, tmpdir, datafiles): project = str(datafiles) # Create the repo from 'files' subdir repo = create_repo("tar", str(tmpdir)) ref = repo.create(os.path.join(project, "files")) # Write out test target with a cached and a non-cached source element = {"kind": "import", "sources": [repo.source_config(ref=ref), {"kind": "always_cached"}]} generate_element(project, "target.bst", element) # Test fetch of target with a cached and a non-cached source result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() apache-buildstream-27ae392/tests/sources/previous_source_access.py000066400000000000000000000051331514607367700256000ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import generate_project, load_yaml from buildstream._testing import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "previous_source_access") ################################################################## # Tests # ################################################################## # Test that plugins can access data from previous sources @pytest.mark.datafiles(DATA_DIR) def test_custom_transform_source(cli, datafiles): project = str(datafiles) # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = load_yaml(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) generate_project(project, project_config) # Ensure we can track result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() # Ensure we can fetch result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() # Ensure we get correct output from foo_transform cli.run(project=project, args=["build", "target.bst"]) destpath = os.path.join(cli.directory, "checkout") result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", destpath]) result.assert_success() # Assert that files from both sources exist, and that they have # the same content assert os.path.exists(os.path.join(destpath, "file")) assert os.path.exists(os.path.join(destpath, "filetransform")) with open(os.path.join(destpath, "file"), encoding="utf-8") as file1: with open(os.path.join(destpath, "filetransform"), encoding="utf-8") as file2: assert file1.read() == file2.read() apache-buildstream-27ae392/tests/sources/previous_source_access/000077500000000000000000000000001514607367700252245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/previous_source_access/elements/000077500000000000000000000000001514607367700270405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/previous_source_access/elements/target.bst000066400000000000000000000001331514607367700310350ustar00rootroot00000000000000kind: import sources: - kind: remote url: project_dir:/files/file - kind: foo_transform apache-buildstream-27ae392/tests/sources/previous_source_access/files/000077500000000000000000000000001514607367700263265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/previous_source_access/files/file000066400000000000000000000000151514607367700271640ustar00rootroot00000000000000Hello World! apache-buildstream-27ae392/tests/sources/previous_source_access/plugins/000077500000000000000000000000001514607367700267055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/previous_source_access/plugins/sources/000077500000000000000000000000001514607367700303705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/previous_source_access/plugins/sources/foo_transform.py000066400000000000000000000052761514607367700336320ustar00rootroot00000000000000""" foo_transform - transform "file" from previous sources into "filetransform" =========================================================================== This is a test source plugin that looks for a file named "file" staged by previous sources, and copies its contents to a file called "filetransform". """ import os import hashlib from buildstream import Source, SourceError, utils class FooTransformSource(Source): BST_MIN_VERSION = "2.0" # We need access to previous both at track time and fetch time BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True BST_REQUIRES_PREVIOUS_SOURCES_FETCH = True BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True @property def mirror(self): """Directory where this source should stage its files""" path = os.path.join(self.get_mirror_directory(), self.name, self.ref.strip()) os.makedirs(path, exist_ok=True) return path def configure(self, node): node.validate_keys(["ref", *Source.COMMON_CONFIG_KEYS]) self.ref = node.get_str("ref", None) def preflight(self): pass def get_unique_key(self): return (self.ref,) def is_cached(self): # If we have a file called "filetransform", verify that its checksum # matches our ref. Otherwise, it is not cached. fpath = os.path.join(self.mirror, "filetransform") try: with open(fpath, "rb") as f: if hashlib.sha256(f.read()).hexdigest() == self.ref.strip(): return True except FileNotFoundError: pass return False def get_ref(self): return self.ref def set_ref(self, ref, node): self.ref = node["ref"] = ref def track(self, previous_sources_dir): # Store the checksum of the file from previous source as our ref fpath = os.path.join(previous_sources_dir, "file") with open(fpath, "rb") as f: return hashlib.sha256(f.read()).hexdigest() def fetch(self, previous_sources_dir): fpath = os.path.join(previous_sources_dir, "file") # Verify that the checksum of the file from previous source matches # our ref with open(fpath, "rb") as f: if hashlib.sha256(f.read()).hexdigest() != self.ref.strip(): raise SourceError("Element references do not match") # Copy "file" as "filetransform" newfpath = os.path.join(self.mirror, "filetransform") utils.safe_copy(fpath, newfpath) def stage(self, directory): # Simply stage the "filetransform" file utils.safe_copy(os.path.join(self.mirror, "filetransform"), os.path.join(directory, "filetransform")) def setup(): return FooTransformSource apache-buildstream-27ae392/tests/sources/previous_source_access/project.conf000066400000000000000000000003241514607367700275400ustar00rootroot00000000000000# Project with local source plugins name: foo min-version: 2.0 element-path: elements aliases: project_dir: file://{project_dir} plugins: - origin: local path: plugins/sources sources: - foo_transform apache-buildstream-27ae392/tests/sources/project/000077500000000000000000000000001514607367700221155ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project/elements/000077500000000000000000000000001514607367700237315ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project/elements/base.bst000066400000000000000000000001031514607367700253470ustar00rootroot00000000000000# elements/base.bst kind: stack depends: - base/base-alpine.bst apache-buildstream-27ae392/tests/sources/project/elements/base/000077500000000000000000000000001514607367700246435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project/elements/base/base-alpine.bst000066400000000000000000000010061514607367700275320ustar00rootroot00000000000000kind: import description: | Alpine Linux base for tests Generated using the `tests/integration-tests/base/generate-base.sh` script. sources: - kind: tar base-dir: '' (?): - arch == "x86-64": ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - arch == "aarch64": ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a url: "alpine:integration-tests-base.v1.aarch64.tar.xz" apache-buildstream-27ae392/tests/sources/project_key_test/000077500000000000000000000000001514607367700240245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project_key_test/elements/000077500000000000000000000000001514607367700256405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project_key_test/elements/key-test.bst000066400000000000000000000000541514607367700301160ustar00rootroot00000000000000kind: import sources: - kind: key-test apache-buildstream-27ae392/tests/sources/project_key_test/plugins/000077500000000000000000000000001514607367700255055ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project_key_test/plugins/sources/000077500000000000000000000000001514607367700271705ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/project_key_test/plugins/sources/key-test.py000066400000000000000000000020141514607367700313040ustar00rootroot00000000000000import os from buildstream import Source class KeyTest(Source): """This plugin should fail if get_unique_key is called before track""" BST_MIN_VERSION = "2.0" def preflight(self): pass def configure(self, node): if node.get_scalar("ref", None).is_none(): self.ref = None else: self.ref = True def get_unique_key(self): assert self.ref return "abcdefg" def is_cached(self): return False def load_ref(self, node): pass def get_ref(self): return self.ref def set_ref(self, ref, node): node["ref"] = self.ref = ref def track(self, **kwargs): return True def fetch(self, **kwargs): pass def stage(self, directory): # Create a dummy file as output, as import elements do not allow empty # output. Its existence is a statement that we have staged ourselves. open(os.path.join(directory, "output"), "w").close() def setup(): return KeyTest apache-buildstream-27ae392/tests/sources/project_key_test/project.conf000066400000000000000000000002031514607367700263340ustar00rootroot00000000000000name: key-test min-version: 2.0 element-path: elements plugins: - origin: local path: plugins/sources sources: - key-test apache-buildstream-27ae392/tests/sources/remote.py000066400000000000000000000174411514607367700223230ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import stat import pytest from buildstream import utils from buildstream._testing import ErrorDomain from buildstream._testing import generate_project from buildstream._testing import cli # pylint: disable=unused-import from tests.testutils.file_server import create_file_server DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "remote", ) # Test that without ref, consistency is set appropriately. @pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) assert cli.get_element_state(project, "target.bst") == "no reference" # Here we are doing a fetch on a file that doesn't exist. target.bst # refers to 'file' but that file is not present. @pytest.mark.datafiles(os.path.join(DATA_DIR, "missing-file")) def test_missing_file(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) @pytest.mark.datafiles(os.path.join(DATA_DIR, "path-in-filename")) def test_path_in_filename(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) # The bst file has a / in the filename param result.assert_main_error(ErrorDomain.SOURCE, "filename-contains-directory") @pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file")) def test_simple_file_build(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Note that the url of the file in target.bst is actually /dir/file # but this tests confirms we take the basename checkout_file = os.path.join(checkoutdir, "file") assert os.path.exists(checkout_file) mode = os.stat(checkout_file).st_mode # Assert not executable by anyone assert not mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) # Assert not writeable by anyone other than me (unless umask allows it) if utils.get_umask() & stat.S_IWGRP: assert not mode & stat.S_IWGRP if utils.get_umask() & stat.S_IWOTH: assert not mode & stat.S_IWOTH @pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file-custom-name")) def test_simple_file_custom_name_build(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() assert not os.path.exists(os.path.join(checkoutdir, "file")) assert os.path.exists(os.path.join(checkoutdir, "custom-file")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "unique-keys")) def test_unique_key(cli, tmpdir, datafiles): """This test confirms that the 'filename' parameter is honoured when it comes to generating a cache key for the source. """ project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) states = cli.get_element_states(project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) assert states["target.bst"] == "fetch needed" assert states["target-custom.bst"] == "fetch needed" assert states["target-custom-executable.bst"] == "fetch needed" # Try to fetch it cli.run(project=project, args=["source", "fetch", "target.bst"]) # We should download_yaml the file only once states = cli.get_element_states(project, ["target.bst", "target-custom.bst", "target-custom-executable.bst"]) assert states["target.bst"] == "buildable" assert states["target-custom.bst"] == "buildable" assert states["target-custom-executable.bst"] == "buildable" # But the cache key is different because the 'filename' is different. assert ( cli.get_element_key(project, "target.bst") != cli.get_element_key(project, "target-custom.bst") != cli.get_element_key(project, "target-custom-executable.bst") ) @pytest.mark.datafiles(os.path.join(DATA_DIR, "unique-keys")) def test_executable(cli, tmpdir, datafiles): """This test confirms that the 'ecxecutable' parameter is honoured.""" project = str(datafiles) generate_project(project, {"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") assert cli.get_element_state(project, "target-custom-executable.bst") == "fetch needed" # Try to fetch it cli.run(project=project, args=["build", "target-custom-executable.bst"]) cli.run(project=project, args=["artifact", "checkout", "target-custom-executable.bst", "--directory", checkoutdir]) mode = os.stat(os.path.join(checkoutdir, "some-custom-file")).st_mode assert mode & stat.S_IEXEC # Assert executable by anyone assert mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) @pytest.mark.parametrize("server_type", ("FTP", "HTTP")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "single-file")) def test_use_netrc(cli, datafiles, server_type, tmpdir): fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(fake_home, exist_ok=True) project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"login testuser\n") f.write(b"password 12345\n") with create_file_server(server_type) as server: server.add_user("testuser", "12345", project) generate_project(project, {"aliases": {"tmpdir": server.base_url()}}) server.start() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() checkout_file = os.path.join(checkoutdir, "file") assert os.path.exists(checkout_file) apache-buildstream-27ae392/tests/sources/remote/000077500000000000000000000000001514607367700217425ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/missing-file/000077500000000000000000000000001514607367700243305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/missing-file/target.bst000066400000000000000000000002211514607367700263230ustar00rootroot00000000000000kind: manual description: The kind of this element is irrelevant. sources: - kind: remote url: tmpdir:/file ref: abcdef filename: filename apache-buildstream-27ae392/tests/sources/remote/no-ref/000077500000000000000000000000001514607367700231305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/no-ref/file000066400000000000000000000000141514607367700237650ustar00rootroot00000000000000filecontent apache-buildstream-27ae392/tests/sources/remote/no-ref/target.bst000066400000000000000000000001561514607367700251320ustar00rootroot00000000000000kind: manual description: The kind of this element is irrelevant. sources: - kind: remote url: tmpdir:/file apache-buildstream-27ae392/tests/sources/remote/path-in-filename/000077500000000000000000000000001514607367700250605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/path-in-filename/dir/000077500000000000000000000000001514607367700256365ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/path-in-filename/dir/file000066400000000000000000000000001514607367700264660ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/path-in-filename/target.bst000066400000000000000000000002601514607367700270560ustar00rootroot00000000000000kind: import description: test sources: - kind: remote url: tmpdir:/dir/file ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 filename: path/to/file apache-buildstream-27ae392/tests/sources/remote/single-file-custom-name/000077500000000000000000000000001514607367700263665ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/single-file-custom-name/dir/000077500000000000000000000000001514607367700271445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/single-file-custom-name/dir/file000066400000000000000000000000001514607367700277740ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/single-file-custom-name/target.bst000066400000000000000000000002571514607367700303720ustar00rootroot00000000000000kind: import description: test sources: - kind: remote url: tmpdir:/dir/file ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 filename: custom-file apache-buildstream-27ae392/tests/sources/remote/single-file/000077500000000000000000000000001514607367700241405ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/single-file/dir/000077500000000000000000000000001514607367700247165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/single-file/dir/file000066400000000000000000000000001514607367700255460ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/single-file/target.bst000066400000000000000000000002271514607367700261410ustar00rootroot00000000000000kind: import description: test sources: - kind: remote url: tmpdir:/dir/file ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 apache-buildstream-27ae392/tests/sources/remote/unique-keys/000077500000000000000000000000001514607367700242215ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/unique-keys/dir/000077500000000000000000000000001514607367700247775ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/unique-keys/dir/file000066400000000000000000000000001514607367700256270ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/remote/unique-keys/target-custom-executable.bst000066400000000000000000000003071514607367700316500ustar00rootroot00000000000000kind: import description: test sources: - kind: remote url: tmpdir:/dir/file ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 filename: some-custom-file executable: true apache-buildstream-27ae392/tests/sources/remote/unique-keys/target-custom.bst000066400000000000000000000002641514607367700275330ustar00rootroot00000000000000kind: import description: test sources: - kind: remote url: tmpdir:/dir/file ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 filename: some-custom-file apache-buildstream-27ae392/tests/sources/remote/unique-keys/target.bst000066400000000000000000000002271514607367700262220ustar00rootroot00000000000000kind: import description: test sources: - kind: remote url: tmpdir:/dir/file ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 apache-buildstream-27ae392/tests/sources/source_provenance_attributes.py000066400000000000000000000136771514607367700270250ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream._testing import generate_project, load_yaml from buildstream._testing import cli # pylint: disable=unused-import from buildstream.exceptions import ErrorDomain, LoadErrorReason DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "source_provenance_attributes") ################################################################## # Tests # ################################################################## # Test that no defined source provenance attributes blocks all source provenance data @pytest.mark.datafiles(DATA_DIR) def test_source_provenance_disallow_top_level(cli, datafiles): project = str(datafiles) # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = load_yaml(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) source_provenance_attrs = project_config.get_mapping("source-provenance-attributes") source_provenance_attrs["homepage"] = "Testing" generate_project(project, project_config) # Make sure disallowed usage of top-level source proveance fails result = cli.run( project=project, args=["show", "target.bst"], ) result.assert_main_error(ErrorDomain.SOURCE, "top-level-provenance-on-custom-implementation") @pytest.mark.datafiles(DATA_DIR) def test_source_provenance_no_defined_attributes(cli, datafiles): project = str(datafiles) # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = load_yaml(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) generate_project(project, project_config) # Make sure a non-default attribute fails result = cli.run( project=project, args=["show", "--format", "%{source-info}", "target_a.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE) # Make sure a default attribute fails result = cli.run( project=project, args=["show", "--format", "%{source-info}", "target_b.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE) # Test that no defined source provenance attributes blocks all source provenance data @pytest.mark.datafiles(DATA_DIR) def test_source_provenance_default_attributes(cli, datafiles): project = str(datafiles) # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = load_yaml(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) # Edit config to fallback to default source provenance attributes project_config.safe_del("source-provenance-attributes") generate_project(project, project_config) # Make sure defined attributes are available result = cli.run( project=project, args=["show", "--format", "%{source-info}", "target_b.bst"], ) result.assert_success() provenance_result = "" for line in result.output.split("\n"): if "provenance:" in line or " " in line: provenance_result += line assert provenance_result == " provenance: homepage: foo" # Make sure undefined attributes fail result = cli.run( project=project, args=["show", "--format", "%{source-info}", "target_a.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE) # Test that no defined source provenance attributes blocks all source provenance data @pytest.mark.datafiles(DATA_DIR) def test_source_provenance_project_defined_attributes(cli, datafiles): project = str(datafiles) # Set the project_dir alias in project.conf to the path to the tested project project_config_path = os.path.join(project, "project.conf") project_config = load_yaml(project_config_path) aliases = project_config.get_mapping("aliases") aliases["project_dir"] = "file://{}".format(project) # Edit config to only use project specified source provenance attributes source_provenance_attrs = project_config.get_mapping("source-provenance-attributes") source_provenance_attrs["originator"] = "Testing" generate_project(project, project_config) # Make sure defined attributes are available result = cli.run( project=project, args=["show", "--format", "%{source-info}", "target_a.bst"], ) result.assert_success() provenance_result = "" for line in result.output.split("\n"): if "provenance:" in line or " " in line: provenance_result += line assert provenance_result == " provenance: originator: bar" # Make sure undefined attributes fail result = cli.run( project=project, args=["show", "--format", "%{source-info}", "target_b.bst"], ) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE) apache-buildstream-27ae392/tests/sources/source_provenance_attributes/000077500000000000000000000000001514607367700264355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/source_provenance_attributes/elements/000077500000000000000000000000001514607367700302515ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/source_provenance_attributes/elements/target.bst000066400000000000000000000001611514607367700322470ustar00rootroot00000000000000kind: import sources: - kind: multisource-plugin url: project_dir:/files/file provenance: homepage: bar apache-buildstream-27ae392/tests/sources/source_provenance_attributes/elements/target_a.bst000066400000000000000000000001471514607367700325530ustar00rootroot00000000000000kind: import sources: - kind: remote url: project_dir:/files/file provenance: originator: bar apache-buildstream-27ae392/tests/sources/source_provenance_attributes/elements/target_b.bst000066400000000000000000000001451514607367700325520ustar00rootroot00000000000000kind: import sources: - kind: remote url: project_dir:/files/file provenance: homepage: foo apache-buildstream-27ae392/tests/sources/source_provenance_attributes/files/000077500000000000000000000000001514607367700275375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/source_provenance_attributes/files/file000066400000000000000000000000151514607367700303750ustar00rootroot00000000000000Hello World! apache-buildstream-27ae392/tests/sources/source_provenance_attributes/plugins/000077500000000000000000000000001514607367700301165ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/source_provenance_attributes/plugins/multisource-plugin.py000066400000000000000000000010621514607367700343360ustar00rootroot00000000000000from buildstream import Node, Source class MultiSource(Source): BST_MIN_VERSION = "2.0" BST_CUSTOM_SOURCE_PROVENANCE = True def configure(self, node): pass def preflight(self): pass def get_unique_key(self): return {} def load_ref(self, node): pass def get_ref(self): return {} def set_ref(self, ref, node): pass def is_cached(self): return False def collect_source_info(self): return [] # Plugin entry point def setup(): return MultiSource apache-buildstream-27ae392/tests/sources/source_provenance_attributes/project.conf000066400000000000000000000004301514607367700307470ustar00rootroot00000000000000# Project with source provenance attributes name: foo min-version: 2.0 element-path: elements aliases: project_dir: file://{project_dir} plugins: - origin: local path: plugins sources: - multisource-plugin # Don't allow any provenance source-provenance-attributes: {} apache-buildstream-27ae392/tests/sources/tar.py000066400000000000000000000621441514607367700216160ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os from shutil import copyfile import subprocess import tarfile import tempfile import urllib.parse import pytest from buildstream import utils from buildstream.exceptions import ErrorDomain from buildstream._testing import generate_project, generate_element from buildstream._testing import cli # pylint: disable=unused-import from buildstream._testing._utils.site import HAVE_LZIP from tests.testutils.file_server import create_file_server, create_bearer_http_server from . import list_dir_contents DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "tar", ) def _assemble_tar(workingdir, srcdir, dstfile): old_dir = os.getcwd() os.chdir(workingdir) with tarfile.open(dstfile, "w:gz") as tar: tar.add(srcdir) os.chdir(old_dir) def _assemble_tar_lz(workingdir, srcdir, dstfile): old_dir = os.getcwd() os.chdir(workingdir) with tempfile.TemporaryFile() as uncompressed: with tarfile.open(fileobj=uncompressed, mode="w:") as tar: tar.add(srcdir) uncompressed.seek(0, 0) with open(dstfile, "wb") as dst: subprocess.call(["lzip"], stdin=uncompressed, stdout=dst) os.chdir(old_dir) # Test that without ref, consistency is set appropriately. @pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref")) def test_no_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) assert cli.get_element_state(project, "target.bst") == "no reference" # Test that when I fetch a nonexistent URL, errors are handled gracefully and a retry is performed. @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_url(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) assert "FAILURE Try #" in result.stderr result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when I fetch with an invalid ref, it fails. @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_fetch_bad_ref(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Try to fetch it result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that when tracking with a ref set, there is a warning @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_track_warning(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Track it result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() assert "Potential man-in-the-middle attack!" in result.stderr # Test that a staged checkout matches what was tarred up, with the default first subdir @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents # Test that a staged checkout matches what was tarred up, with an empty base-dir @pytest.mark.datafiles(os.path.join(DATA_DIR, "no-basedir")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the full content of the tarball is checked out (base-dir: '') original_dir = os.path.join(str(datafiles), "content") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents # Test that a staged checkout matches what was tarred up, with an explicit basedir @pytest.mark.datafiles(os.path.join(DATA_DIR, "explicit-basedir")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents # Test that we succeed to extract tarballs with hardlinks when stripping the # leading paths @pytest.mark.datafiles(os.path.join(DATA_DIR, "contains-links")) def test_stage_contains_links(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") # Create a hardlink, we wont trust git to store that info for us os.makedirs(os.path.join(str(datafiles), "content", "base-directory", "subdir2"), exist_ok=True) file1 = os.path.join(str(datafiles), "content", "base-directory", "subdir1", "file.txt") file2 = os.path.join(str(datafiles), "content", "base-directory", "subdir2", "file.txt") os.link(file1, file2) _assemble_tar(os.path.join(str(datafiles), "content"), "base-directory", src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') original_dir = os.path.join(str(datafiles), "content", "base-directory") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents @pytest.mark.skipif(not HAVE_LZIP, reason="lzip is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) @pytest.mark.parametrize("srcdir", ["a", "./a"]) def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.lz") _assemble_tar_lz(os.path.join(str(datafiles), "content"), srcdir, src_tar) # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target-lz.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target-lz.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target-lz.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target-lz.bst", "--directory", checkoutdir]) result.assert_success() # Check that the content of the first directory is checked out (base-dir: '*') original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents # Test that tarballs with read-only files work # a - contains read-only files in a writable directory # b - root directory has read-only permission # c - contains one file that has no read nor write permissions. Base-dir set to '' to extract root of tarball @pytest.mark.datafiles(os.path.join(DATA_DIR, "read-only")) @pytest.mark.parametrize("tar_name, base_dir", [("a", "*"), ("b", "*"), ("c", "")]) def test_read_only_dir(cli, tmpdir, datafiles, tar_name, base_dir): try: project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) tar_file = "{}.tar.gz".format(tar_name) generate_element( project, "target.bst", { "kind": "import", "sources": [{"kind": "tar", "url": "tmpdir:/{}".format(tar_file), "ref": "foo", "base-dir": base_dir}], }, ) # Get the tarball in tests/sources/tar/read-only/content # # NOTE that we need to do this because tarfile.open and tar.add() # are packing the tar up with writeable files and dirs tarball = os.path.join(str(datafiles), "content", tar_file) if not os.path.exists(tarball): raise FileNotFoundError("{} does not exist".format(tarball)) copyfile(tarball, os.path.join(str(tmpdir), tar_file)) # Because this test can potentially leave directories behind # which are difficult to remove, ask buildstream to use # our temp directory, so we can clean up. tmpdir_str = str(tmpdir) if not tmpdir_str.endswith(os.path.sep): tmpdir_str += os.path.sep env = {"TMP": tmpdir_str} # Track, fetch, build, checkout result = cli.run(project=project, args=["source", "track", "target.bst"], env=env) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"], env=env) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"], env=env) result.assert_success() finally: utils._force_rmtree(str(tmpdir)) @pytest.mark.parametrize("server_type", ("FTP", "HTTP")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_use_netrc(cli, datafiles, server_type, tmpdir): file_server_files = os.path.join(str(tmpdir), "file_server") fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"login testuser\n") f.write(b"password 12345\n") with create_file_server(server_type) as server: server.add_user("testuser", "12345", file_server_files) generate_project(project, config={"aliases": {"tmpdir": server.base_url()}}) src_tar = os.path.join(file_server_files, "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_use_netrc_bearer_auth(cli, datafiles, tmpdir): file_server_files = os.path.join(str(tmpdir), "file_server") fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"password 12345\n") # # Enable using mirrors for source tracking # cli.configure({"track": {"source": "mirrors"}}) # # Create a file server which uses bearer authentication # with create_bearer_http_server() as server: server.set_directory(file_server_files) server.add_token("12345") # # Configure the project to load our source mirror plugin which # reports the "http-auth" extra data # additional_config = { "aliases": {"tmpdir": server.base_url()}, "mirrors": [ { "name": "middle-earth", "kind": "bearermirror", "config": { "aliases": { "tmpdir": [server.base_url()], }, }, }, ], "plugins": [ {"origin": "local", "path": "sourcemirrors", "source-mirrors": ["bearermirror"]}, ], } generate_project(project, config=additional_config) src_tar = os.path.join(file_server_files, "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() original_dir = os.path.join(str(datafiles), "content", "a") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents @pytest.mark.parametrize("server_type", ("FTP", "HTTP")) @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_netrc_already_specified_user(cli, datafiles, server_type, tmpdir): file_server_files = os.path.join(str(tmpdir), "file_server") fake_home = os.path.join(str(tmpdir), "fake_home") os.makedirs(file_server_files, exist_ok=True) os.makedirs(fake_home, exist_ok=True) project = str(datafiles) os.environ["HOME"] = fake_home with open(os.path.join(fake_home, ".netrc"), "wb") as f: os.fchmod(f.fileno(), 0o700) f.write(b"machine 127.0.0.1\n") f.write(b"login testuser\n") f.write(b"password 12345\n") with create_file_server(server_type) as server: server.add_user("otheruser", "12345", file_server_files) parts = urllib.parse.urlsplit(server.base_url()) base_url = urllib.parse.urlunsplit([parts[0], "otheruser@{}".format(parts[1]), *parts[2:]]) generate_project(project, config={"aliases": {"tmpdir": base_url}}) src_tar = os.path.join(file_server_files, "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) server.start() result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) # Test that BuildStream doesnt crash if HOME is unset while # the netrc module is trying to find it's ~/.netrc file. @pytest.mark.datafiles(os.path.join(DATA_DIR, "fetch")) def test_homeless_environment(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Create a local tar src_tar = os.path.join(str(tmpdir), "a.tar.gz") _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar) # Use a track, make sure the plugin tries to find a ~/.netrc result = cli.run(project=project, args=["source", "track", "target.bst"], env={"HOME": None}) result.assert_success() @pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks")) def test_out_of_basedir_hardlinks(cli, tmpdir, datafiles): def ensure_link(member): # By default, python will simply duplicate files - we want # hardlinks! if member.path == "contents/to_extract/a": member.type = tarfile.LNKTYPE member.linkname = "contents/elsewhere/a" return member project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create a tarball with an odd hardlink src_tar = os.path.join(str(tmpdir), "contents.tar.gz") old_dir = os.getcwd() os.chdir(str(tmpdir)) with tarfile.open(src_tar, "w:gz") as tar: # Don't recursively add `contents` as the order is not guaranteed. # We need to add `elsewhere` before `to_extract` as the latter # references the former in `linkname`. tar.add("contents", recursive=False) tar.add("contents/elsewhere") tar.add("contents/to_extract", filter=ensure_link) os.chdir(old_dir) # Make sure our tarfile is actually created with the desired # attributes set with tarfile.open(src_tar, "r:gz") as tar: assert any( member.islnk() and member.path == "contents/to_extract/a" and member.linkname == "contents/elsewhere/a" for member in tar.getmembers() ) # Assert that we will actually create a singular copy of the file result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() original_dir = os.path.join(str(datafiles), "contents", "to_extract") original_contents = list_dir_contents(original_dir) checkout_contents = list_dir_contents(checkoutdir) assert checkout_contents == original_contents @pytest.mark.datafiles(os.path.join(DATA_DIR, "out-of-basedir-hardlinks")) def test_malicious_out_of_basedir_hardlinks(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) # Create a maliciously-hardlinked tarball def ensure_link(member): # By default, python will simply duplicate files - we want # hardlinks! if member.path == "contents/elsewhere/malicious": member.type = tarfile.LNKTYPE # This should not be allowed member.linkname = "../../../malicious_target.bst" return member src_tar = os.path.join(str(tmpdir), "contents.tar.gz") old_dir = os.getcwd() os.chdir(str(tmpdir)) with tarfile.open(src_tar, "w:gz") as tar: tar.add("contents", filter=ensure_link) os.chdir(old_dir) # Make sure our tarfile is actually created with the desired # attributes set with tarfile.open(src_tar, "r:gz") as tar: assert any( member.islnk() and member.path == "contents/elsewhere/malicious" and member.linkname == "../../../malicious_target.bst" for member in tar.getmembers() ) # Try to execute the exploit result = cli.run(project=project, args=["source", "track", "malicious_target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "malicious_target.bst"]) result.assert_main_error(ErrorDomain.STREAM, None) @pytest.mark.datafiles(os.path.join(DATA_DIR, "symlinks")) def test_symlinks(cli, tmpdir, datafiles): project = str(datafiles) generate_project(project, config={"aliases": {"tmpdir": "file:///" + str(tmpdir)}}) checkoutdir = os.path.join(str(tmpdir), "checkout") absolute_target = "/tmp/foo" relative_target = "foo/../bar" # Create a tarball with an absolute symlink src_tar = os.path.join(str(tmpdir), "contents.tar.gz") old_dir = os.getcwd() os.chdir(str(tmpdir)) os.mkdir("contents") os.symlink(absolute_target, "contents/absolute-symlink") os.symlink(relative_target, "contents/relative-symlink") with tarfile.open(src_tar, "w:gz") as tar: tar.add("contents") os.chdir(old_dir) # Make sure our tarfile is actually created with the desired attributes set with tarfile.open(src_tar, "r:gz") as tar: assert any( member.issym() and member.path == "contents/absolute-symlink" and member.linkname == absolute_target for member in tar.getmembers() ) assert any( member.issym() and member.path == "contents/relative-symlink" and member.linkname == relative_target for member in tar.getmembers() ) # Assert that we will allow and not mangle symlinks with relative and absolute target paths result = cli.run(project=project, args=["source", "track", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["source", "fetch", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() assert os.readlink(checkoutdir + "/absolute-symlink") == absolute_target assert os.readlink(checkoutdir + "/relative-symlink") == relative_target apache-buildstream-27ae392/tests/sources/tar/000077500000000000000000000000001514607367700212355ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/contains-links/000077500000000000000000000000001514607367700241715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/contains-links/content/000077500000000000000000000000001514607367700256435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/contains-links/content/base-directory/000077500000000000000000000000001514607367700305575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/contains-links/content/base-directory/subdir1/000077500000000000000000000000001514607367700321305ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/contains-links/content/base-directory/subdir1/file.txt000066400000000000000000000000051514607367700336030ustar00rootroot00000000000000pony apache-buildstream-27ae392/tests/sources/tar/contains-links/target.bst000066400000000000000000000001721514607367700261710ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.gz ref: foo apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/000077500000000000000000000000001514607367700244655ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/content/000077500000000000000000000000001514607367700261375ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/content/a/000077500000000000000000000000001514607367700263575ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/content/a/b/000077500000000000000000000000001514607367700266005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/content/a/b/d000066400000000000000000000000021514607367700267360ustar00rootroot00000000000000d apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/content/a/c000066400000000000000000000000021514607367700265140ustar00rootroot00000000000000c apache-buildstream-27ae392/tests/sources/tar/explicit-basedir/target.bst000066400000000000000000000002121514607367700264600ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.gz ref: foo base-dir: 'a' apache-buildstream-27ae392/tests/sources/tar/fetch/000077500000000000000000000000001514607367700223265ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/fetch/content/000077500000000000000000000000001514607367700240005ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/fetch/content/a/000077500000000000000000000000001514607367700242205ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/fetch/content/a/b/000077500000000000000000000000001514607367700244415ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/fetch/content/a/b/d000066400000000000000000000000021514607367700245770ustar00rootroot00000000000000d apache-buildstream-27ae392/tests/sources/tar/fetch/content/a/c000066400000000000000000000000021514607367700243550ustar00rootroot00000000000000c apache-buildstream-27ae392/tests/sources/tar/fetch/sourcemirrors/000077500000000000000000000000001514607367700252445ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/fetch/sourcemirrors/bearermirror.py000066400000000000000000000014451514607367700303150ustar00rootroot00000000000000from typing import Optional, Dict, Any from buildstream import SourceMirror, MappingNode class Sample(SourceMirror): def configure(self, node): node.validate_keys(["aliases"]) self.aliases = {} aliases = node.get_mapping("aliases") for alias_name, url_list in aliases.items(): self.aliases[alias_name] = url_list.as_str_list() self.set_supported_aliases(self.aliases.keys()) def translate_url( self, *, alias: str, alias_url: str, source_url: str, extra_data: Optional[Dict[str, Any]], ) -> str: if extra_data is not None: extra_data["http-auth"] = "bearer" return self.aliases[alias][0] + source_url # Plugin entry point def setup(): return Sample apache-buildstream-27ae392/tests/sources/tar/fetch/target-lz.bst000066400000000000000000000001721514607367700247510ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.lz ref: foo apache-buildstream-27ae392/tests/sources/tar/fetch/target.bst000066400000000000000000000001721514607367700243260ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.gz ref: foo apache-buildstream-27ae392/tests/sources/tar/no-basedir/000077500000000000000000000000001514607367700232605ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-basedir/content/000077500000000000000000000000001514607367700247325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-basedir/content/a/000077500000000000000000000000001514607367700251525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-basedir/content/a/b/000077500000000000000000000000001514607367700253735ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-basedir/content/a/b/d000066400000000000000000000000021514607367700255310ustar00rootroot00000000000000d apache-buildstream-27ae392/tests/sources/tar/no-basedir/content/a/c000066400000000000000000000000021514607367700253070ustar00rootroot00000000000000c apache-buildstream-27ae392/tests/sources/tar/no-basedir/target.bst000066400000000000000000000002111514607367700252520ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.gz ref: foo base-dir: '' apache-buildstream-27ae392/tests/sources/tar/no-ref/000077500000000000000000000000001514607367700224235ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-ref/a/000077500000000000000000000000001514607367700226435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-ref/a/b/000077500000000000000000000000001514607367700230645ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/no-ref/a/b/d000066400000000000000000000000021514607367700232220ustar00rootroot00000000000000d apache-buildstream-27ae392/tests/sources/tar/no-ref/a/c000066400000000000000000000000021514607367700230000ustar00rootroot00000000000000c apache-buildstream-27ae392/tests/sources/tar/no-ref/target.bst000066400000000000000000000001571514607367700244260ustar00rootroot00000000000000kind: manual description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.gz apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/000077500000000000000000000000001514607367700260325ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/contents/000077500000000000000000000000001514607367700276675ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/000077500000000000000000000000001514607367700316525ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/a000066400000000000000000000000061514607367700320110ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/contents/elsewhere/malicious000066400000000000000000000000001514607367700335500ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/000077500000000000000000000000001514607367700320435ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/contents/to_extract/a000066400000000000000000000000061514607367700322020ustar00rootroot00000000000000hello apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/malicious_target.bst000066400000000000000000000001661514607367700321020ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/contents.tar.gz apache-buildstream-27ae392/tests/sources/tar/out-of-basedir-hardlinks/target.bst000066400000000000000000000002261514607367700300320ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/contents.tar.gz base-dir: contents/to_extract apache-buildstream-27ae392/tests/sources/tar/read-only/000077500000000000000000000000001514607367700231275ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/read-only/content/000077500000000000000000000000001514607367700246015ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/read-only/content/a.tar.gz000066400000000000000000000240001514607367700261440ustar00rootroot00000000000000a/0000755000175000017310000000000013337516601010513 5ustar jennisusersa/c0000644000175000017310000000000213337516601010650 0ustar jennisusersc a/b/0000555000175000017310000000000013337516601010732 5ustar jennisusersa/b/d0000644000175000017310000000000213337516601011072 0ustar jennisusersd apache-buildstream-27ae392/tests/sources/tar/read-only/content/b.tar.gz000066400000000000000000000002241514607367700261470ustar00rootroot00000000000000‹ŸC]íÑ1Â0 …a%'h46â8‘P§NîŠ $† Uÿ/Ã[¬ÄŽ‡QºSU3KÏ<¹¿RK}ç&åÉÔ뤚=iÎQ'Éú·&r_oí­¬—¶´¥œ¿ÕEÙ<ÿ¸g›ã“;1Œ­ûñ^ë?û/q$i÷Îäðû°_ú)È(apache-buildstream-27ae392/tests/sources/tar/read-only/content/c.tar.gz000066400000000000000000000002001514607367700261420ustar00rootroot00000000000000‹E@]íÏ» €0 EQFÉ&?Ä8F@QðÙHÐ@—îžæI¶eÙÛ4Úk—†ªÉÌ™MŒ9Åzsׯfí‚ñÖo¤vÑIe¤ÜImYu>NYzMšlû5wŒãϞד9(hŽ2ëQ(apache-buildstream-27ae392/tests/sources/tar/read-only/target.bst000066400000000000000000000001721514607367700251270ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/a.tar.gz ref: foo apache-buildstream-27ae392/tests/sources/tar/symlinks/000077500000000000000000000000001514607367700231065ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/tar/symlinks/target.bst000066400000000000000000000001661514607367700251110ustar00rootroot00000000000000kind: import description: The kind of this element is irrelevant. sources: - kind: tar url: tmpdir:/contents.tar.gz apache-buildstream-27ae392/tests/sources/variables.py000066400000000000000000000033631514607367700227760ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import pytest from buildstream.exceptions import ErrorDomain, LoadErrorReason from buildstream._testing.runcli import cli # pylint: disable=unused-import DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "variables") @pytest.mark.datafiles(DATA_DIR) def test_variables_are_resolved(cli, tmpdir, datafiles): project = str(datafiles) checkoutdir = os.path.join(str(tmpdir), "checkout") # Build, checkout result = cli.run(project=project, args=["build", "target.bst"]) result.assert_success() result = cli.run(project=project, args=["artifact", "checkout", "target.bst", "--directory", checkoutdir]) result.assert_success() # Check that the checkout contains the expected file assert os.path.exists(os.path.join(checkoutdir, "file.txt")) @pytest.mark.datafiles(DATA_DIR) def test_handles_unresolved_variables(cli, tmpdir, datafiles): project = str(datafiles) result = cli.run(project=project, args=["build", "unresolveable-target.bst"]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.UNRESOLVED_VARIABLE) apache-buildstream-27ae392/tests/sources/variables/000077500000000000000000000000001514607367700224175ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/variables/folder/000077500000000000000000000000001514607367700236725ustar00rootroot00000000000000apache-buildstream-27ae392/tests/sources/variables/folder/file.txt000066400000000000000000000000241514607367700253460ustar00rootroot00000000000000This is a text file apache-buildstream-27ae392/tests/sources/variables/project.conf000066400000000000000000000000531514607367700247320ustar00rootroot00000000000000# Basic project name: foo min-version: 2.0 apache-buildstream-27ae392/tests/sources/variables/target.bst000066400000000000000000000002031514607367700244120ustar00rootroot00000000000000kind: import description: This is the pony sources: - kind: local path: "%{my-folder}/file.txt" variables: my-folder: folder apache-buildstream-27ae392/tests/sources/variables/unresolveable-target.bst000066400000000000000000000001431514607367700272610ustar00rootroot00000000000000kind: import description: This is the pony sources: - kind: local path: "%{my-folder}/file.txt" apache-buildstream-27ae392/tests/testutils/000077500000000000000000000000001514607367700210245ustar00rootroot00000000000000apache-buildstream-27ae392/tests/testutils/__init__.py000066400000000000000000000024661514607367700231450ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: Tristan Van Berkom # Tristan Maat # Sam Thursfield # James Ennis # Valentin David # William Salmon # from .artifactshare import create_artifact_share, create_split_share, assert_shared, assert_not_shared, ArtifactShare from .casd import casd_cache from .context import dummy_context from .element_generators import create_element_size from .junction import generate_junction from .runner_integration import wait_for_cache_granularity from .python_repo import setup_pypi_repo from .platform import override_platform_uname apache-buildstream-27ae392/tests/testutils/artifactshare.py000066400000000000000000000252371514607367700242270ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import multiprocessing import os import shutil import signal from collections import namedtuple from contextlib import ExitStack, contextmanager from concurrent import futures from urllib.parse import urlparse import grpc from buildstream._cas import CASCache from buildstream._cas.casserver import create_server from buildstream._exceptions import CASError from buildstream._protos.build.bazel.remote.asset.v1 import remote_asset_pb2, remote_asset_pb2_grpc from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 from buildstream._protos.buildstream.v2 import artifact_pb2 from buildstream._protos.google.rpc import code_pb2 REMOTE_ASSET_ARTIFACT_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:artifact:{}" REMOTE_ASSET_SOURCE_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:source:{}" class ArtifactSharror(Exception): pass class BaseArtifactShare: def __init__(self): multiprocessing_context = multiprocessing.get_context("forkserver") q = multiprocessing_context.Queue() self.process = multiprocessing_context.Process(target=self.run, args=(q,)) self.process.start() # Retrieve port from server subprocess port = q.get() if port is None: raise ArtifactSharror("Error occurred when starting artifact server.") self.repo = "http://localhost:{}".format(port) # run(): # # Run the artifact server. # def run(self, q): # Block SIGTERM and SIGINT to allow graceful shutdown and cleanup after initialization signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGTERM, signal.SIGINT]) with ExitStack() as stack: try: server = stack.enter_context(self._create_server()) port = server.add_insecure_port("127.0.0.1:0") server.start() except Exception: q.put(None) raise # Send port to parent q.put(port) # Sleep until termination by signal signal.sigwait([signal.SIGTERM, signal.SIGINT]) server.stop(0) # _create_server() # # Create the server that will be run in the process # def _create_server(self): raise NotImplementedError() # close(): # # Remove the artifact share. # def close(self): self.process.terminate() self.process.join() assert self.process.exitcode == 0 # DummyArtifactShare() # # A dummy artifact share without any capabilities # class DummyArtifactShare(BaseArtifactShare): @contextmanager def _create_server(self): max_workers = (os.cpu_count() or 1) * 5 server = grpc.server(futures.ThreadPoolExecutor(max_workers)) yield server # ArtifactShare() # # Abstract class providing scaffolding for # generating data to be used with various sources # # Args: # directory (str): The base temp directory for the test # cache_quota (int): Maximum amount of disk space to use # enable_push (bool): Whether the share should allow pushes # class ArtifactShare(BaseArtifactShare): def __init__(self, directory, *, quota=None, index_only=False): # The working directory for the artifact share (in case it # needs to do something outside of its backend's storage folder). # self.directory = os.path.abspath(directory) # The directory the actual repo will be stored in. # # Unless this gets more complicated, just use this directly # in tests as a remote artifact push/pull configuration # self.repodir = os.path.join(self.directory, "repo") os.makedirs(self.repodir) self.quota = quota self.index_only = index_only super().__init__() # Set after subprocess creation as it's not picklable self.cas = CASCache(self.repodir, casd=None) def _create_server(self): return create_server( self.repodir, quota=self.quota, enable_push=True, index_only=self.index_only, ) # has_object(): # # Checks whether the object is present in the share # # Args: # digest (str): The object's digest # # Returns: # (bool): True if the object exists in the share, otherwise false. def has_object(self, digest): assert isinstance(digest, remote_execution_pb2.Digest) object_path = self.cas.objpath(digest) return os.path.exists(object_path) def get_artifact_proto(self, artifact_name): url = urlparse(self.repo) channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) try: fetch_service = remote_asset_pb2_grpc.FetchStub(channel) uri = REMOTE_ASSET_ARTIFACT_URN_TEMPLATE.format(artifact_name) request = remote_asset_pb2.FetchBlobRequest() request.uris.append(uri) try: response = fetch_service.FetchBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: return None raise if response.status.code != code_pb2.OK: return None return response.blob_digest finally: channel.close() def get_source_proto(self, source_name): url = urlparse(self.repo) channel = grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) try: fetch_service = remote_asset_pb2_grpc.FetchStub(channel) uri = REMOTE_ASSET_SOURCE_URN_TEMPLATE.format(source_name) request = remote_asset_pb2.FetchDirectoryRequest() request.uris.append(uri) try: response = fetch_service.FetchDirectory(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: return None raise if response.status.code != code_pb2.OK: return None return response.root_directory_digest finally: channel.close() def get_cas_files(self, artifact_proto_digest): reachable = set() def reachable_dir(digest): self._reachable_refs_dir(reachable, digest) try: artifact_proto_path = self.cas.objpath(artifact_proto_digest) if not os.path.exists(artifact_proto_path): return None artifact_proto = artifact_pb2.Artifact() try: with open(artifact_proto_path, "rb") as f: artifact_proto.ParseFromString(f.read()) except FileNotFoundError: return None if str(artifact_proto.files): reachable_dir(artifact_proto.files) if str(artifact_proto.buildtree): reachable_dir(artifact_proto.buildtree) if str(artifact_proto.public_data): if not os.path.exists(self.cas.objpath(artifact_proto.public_data)): return None for log_file in artifact_proto.logs: if not os.path.exists(self.cas.objpath(log_file.digest)): return None return artifact_proto.files except CASError: return None except FileNotFoundError: return None # has_artifact(): # # Checks whether the artifact is present in the share # # Args: # artifact_name (str): The composed complete artifact name # # Returns: # (ArtifactProto): artifact digest if the artifact exists in the share, otherwise None. def get_artifact(self, artifact_name): artifact_proto = self.get_artifact_proto(artifact_name) if not artifact_proto: return None return self.get_cas_files(artifact_proto) # close(): # # Remove the artifact share. # def close(self): super().close() shutil.rmtree(self.directory) def _reachable_refs_dir(self, reachable, tree): if tree.hash in reachable: return reachable.add(tree.hash) directory = remote_execution_pb2.Directory() with open(self.cas.objpath(tree), "rb") as f: directory.ParseFromString(f.read()) for filenode in directory.files: if not os.path.exists(self.cas.objpath(filenode.digest)): raise FileNotFoundError reachable.add(filenode.digest.hash) for dirnode in directory.directories: self._reachable_refs_dir(reachable, dirnode.digest) # create_artifact_share() # # Create an ArtifactShare for use in a test case # @contextmanager def create_artifact_share(directory, *, quota=None): share = ArtifactShare(directory, quota=quota) try: yield share finally: share.close() @contextmanager def create_split_share(directory1, directory2, *, quota=None): index = ArtifactShare(directory1, quota=quota, index_only=True) storage = ArtifactShare(directory2, quota=quota) try: yield index, storage finally: index.close() storage.close() # create_dummy_artifact_share() # # Create a dummy artifact share that doesn't have any capabilities # @contextmanager def create_dummy_artifact_share(): share = DummyArtifactShare() try: yield share finally: share.close() statvfs_result = namedtuple("statvfs_result", "f_blocks f_bfree f_bsize f_bavail") # Assert that a given artifact is in the share # def assert_shared(cli, share, project, element_name, *, project_name="test"): if not share.get_artifact(cli.get_artifact_name(project, project_name, element_name)): raise AssertionError( "Artifact share at {} does not contain the expected element {}".format(share.repo, element_name) ) # Assert that a given artifact is not in the share # def assert_not_shared(cli, share, project, element_name, *, project_name="test"): if share.get_artifact(cli.get_artifact_name(project, project_name, element_name)): raise AssertionError( "Artifact share at {} unexpectedly contains the element {}".format(share.repo, element_name) ) apache-buildstream-27ae392/tests/testutils/bearer_http_server.py000066400000000000000000000067461514607367700253000ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import threading import os import posixpath import html from http.server import SimpleHTTPRequestHandler, HTTPServer, HTTPStatus class Unauthorized(Exception): pass class BearerRequestHandler(SimpleHTTPRequestHandler): def get_root_dir(self): authorization = self.headers.get("authorization") if not authorization: raise Unauthorized("unauthorized") authorization = authorization.split() if len(authorization) != 2 or authorization[0].lower() != "bearer": raise Unauthorized("unauthorized") token = authorization[1] if token not in self.server.tokens: raise Unauthorized("unauthorized") return self.server.directory def unauthorized(self): shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED] self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg) self.send_header("Connection", "close") content = self.error_message_format % { "code": HTTPStatus.UNAUTHORIZED, "message": html.escape(longmsg, quote=False), "explain": html.escape(longmsg, quote=False), } body = content.encode("UTF-8", "replace") self.send_header("Content-Type", self.error_content_type) self.send_header("Content-Length", str(len(body))) self.send_header("WWW-Authenticate", 'Bearer realm="{}"'.format(self.server.realm)) self.end_headers() self.end_headers() if self.command != "HEAD" and body: self.wfile.write(body) def do_GET(self): try: super().do_GET() except Unauthorized: self.unauthorized() def do_HEAD(self): try: super().do_HEAD() except Unauthorized: self.unauthorized() def translate_path(self, path): path = path.split("?", 1)[0] path = path.split("#", 1)[0] path = posixpath.normpath(path) assert posixpath.isabs(path) path = posixpath.relpath(path, "/") return os.path.join(self.get_root_dir(), path) class BearerHTTPServer(HTTPServer): def __init__(self, *args, **kwargs): self.tokens = set() self.directory = None self.realm = "Realm" super().__init__(*args, **kwargs) class BearerHttpServer(threading.Thread): def __init__(self): super().__init__() self.server = BearerHTTPServer(("127.0.0.1", 0), BearerRequestHandler) self.started = False def start(self): self.started = True super().start() def run(self): self.server.serve_forever() def stop(self): if not self.started: return self.server.shutdown() self.join() def set_directory(self, directory): self.server.directory = directory def add_token(self, token): self.server.tokens.add(token) def base_url(self): return "http://127.0.0.1:{}".format(self.server.server_port) apache-buildstream-27ae392/tests/testutils/casd.py000066400000000000000000000022011514607367700223030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from contextlib import contextmanager from buildstream._cas import CASCache, CASDProcessManager, CASLogLevel @contextmanager def casd_cache(path, messenger=None): casd = CASDProcessManager( str(path), os.path.join(str(path), "..", "logs", "_casd"), CASLogLevel.WARNING, 16 * 1024 * 1024, None, True, None, ) try: cascache = CASCache(str(path), casd=casd) try: yield cascache finally: cascache.release_resources() finally: casd.release_resources(messenger) apache-buildstream-27ae392/tests/testutils/constants.py000066400000000000000000000020041514607367700234060ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Constants used during BuildStream tests. # Timeout for short interactive operations (in seconds). # # Use this for operations that are expected to finish within a short amount of # time. Like `bst init`, `bst show` on a small project. PEXPECT_TIMEOUT_SHORT = 30 # Timeout for longer interactive operations (in seconds). # # Use this for operations that are expected to take longer amounts of time, # like `bst build` on a small project. PEXPECT_TIMEOUT_LONG = 300 apache-buildstream-27ae392/tests/testutils/context.py000066400000000000000000000041371514607367700230670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os from types import MethodType from contextlib import contextmanager from buildstream._context import Context # Handle messages from the pipeline def _dummy_message_handler(message, is_silenced): pass class _DummyTask: # Note that unittest.mock.MagicMock doesn't pickle, so we must make our # _DummyTask manually here. def __init__(self, state, action_name, full_name, elapsed_offset): self._state = state self.action_name = action_name self.full_name = full_name self.elapsed_offset = elapsed_offset self.current_progress = None self.maximum_progress = None def set_render_cb(self, callback): pass def set_current_progress(self, progress): pass def set_maximum_progress(self, progress): pass def add_current_progress(self): pass def add_maximum_progress(self): pass @contextmanager def _get_dummy_task(self, activity_name, *, element_name=None, full_name=None, silent_nested=False): yield _DummyTask("state", activity_name, full_name, 0) # dummy_context() # # Context manager to create minimal context for tests. # # Args: # config (filename): The configuration file, if any # @contextmanager def dummy_context(*, config=None): with Context() as context: if not config: config = os.devnull context.load(config=config) context.messenger.set_message_handler(_dummy_message_handler) context.messenger.simple_task = MethodType(_get_dummy_task, context.messenger) yield context apache-buildstream-27ae392/tests/testutils/element_generators.py000066400000000000000000000050661514607367700252670ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from buildstream import _yaml from buildstream import utils from buildstream._testing import create_repo # create_element_size() # # Creates an import element with a git repo, using random # data to create a file in that repo of the specified size, # such that building it will add an artifact of the specified # size to the artifact cache. # # Args: # name: (str) of the element name (e.g. target.bst) # project_dir (str): The path to the project # element_path (str): The element path within the project # dependencies: A list of strings (can also be an empty list) # size: (int) size of the element in bytes # # Returns: # (Repo): A repo representing the sized element # def create_element_size(name, project_dir, elements_path, dependencies, size): full_elements_path = os.path.join(project_dir, elements_path) os.makedirs(full_elements_path, exist_ok=True) # Create a repo repodir = os.path.join(project_dir, "repos") repo = create_repo("tar", repodir, subdir=name) with utils._tempdir(dir=project_dir) as tmp: # We use a data/ subdir in the git repo we create, # and we set the import element to only extract that # part; this ensures we never include a .git/ directory # in the cached artifacts for these sized elements. # datadir = os.path.join(tmp, "data") os.makedirs(datadir) # Use /dev/urandom to create the sized file in the datadir with open(os.path.join(datadir, name), "wb+") as f: f.write(os.urandom(size)) # Create the git repo from the temp directory ref = repo.create(tmp) element = { "kind": "import", "sources": [repo.source_config(ref=ref)], "config": { # Extract only the data directory "source": "data" }, "depends": dependencies, } _yaml.roundtrip_dump(element, os.path.join(project_dir, elements_path, name)) # Return the repo, so that it can later be used to add commits return repo apache-buildstream-27ae392/tests/testutils/file_server.py000066400000000000000000000025741514607367700237130ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from contextlib import contextmanager from .ftp_server import SimpleFtpServer from .http_server import SimpleHttpServer from .bearer_http_server import BearerHttpServer @contextmanager def create_file_server(file_server_type): if file_server_type == "FTP": server = SimpleFtpServer() elif file_server_type == "HTTP": server = SimpleHttpServer() else: assert False try: yield server finally: server.stop() # # We use a separate function here in order to avoid # confusing the linter (which thinks that anything # yielded by `create_file_server()` is a SimpleFtpServer). # # And no, type annotations with Union[...] does not fix this. # @contextmanager def create_bearer_http_server(): server = BearerHttpServer() try: yield server finally: server.stop() apache-buildstream-27ae392/tests/testutils/filetypegenerator.py000066400000000000000000000032141514607367700251260ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Tiago Gomes import os import socket # generate_file_types() # # Generator that creates a regular file directory, symbolic link, fifo # and socket at the specified path. # # Args: # path: (str) path where to create each different type of file # def generate_file_types(path): def clean(): if os.path.exists(path): if os.path.isdir(path): os.rmdir(path) else: os.remove(path) clean() with open(path, "w", encoding="utf-8"): pass yield clean() os.makedirs(path) yield clean() os.symlink("project.conf", path) yield clean() os.mkfifo(path) yield clean() # Change directory because the full path may be longer than the ~100 # characters permitted for a unix socket old_dir = os.getcwd() parent, child = os.path.split(path) os.chdir(parent) s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: s.bind(child) os.chdir(old_dir) yield finally: s.close() clean() apache-buildstream-27ae392/tests/testutils/ftp_server.py000066400000000000000000000026061514607367700235610ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import threading from pyftpdlib.authorizers import DummyAuthorizer from pyftpdlib.handlers import FTPHandler from pyftpdlib.servers import ThreadedFTPServer class SimpleFtpServer(threading.Thread): def __init__(self): super().__init__() self.authorizer = DummyAuthorizer() handler = FTPHandler handler.authorizer = self.authorizer self.server = ThreadedFTPServer(("127.0.0.1", 0), handler) def run(self): self.server.serve_forever() def stop(self): self.server.close_all() self.join() def allow_anonymous(self, cwd): self.authorizer.add_anonymous(cwd) def add_user(self, user, password, cwd): self.authorizer.add_user(user, password, cwd, perm="elradfmwMT") def base_url(self): return "ftp://127.0.0.1:{}".format(self.server.address[1]) apache-buildstream-27ae392/tests/testutils/http_server.py000066400000000000000000000075631514607367700237560ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import threading import os import posixpath import html import base64 from http.server import SimpleHTTPRequestHandler, HTTPServer, HTTPStatus class Unauthorized(Exception): pass class RequestHandler(SimpleHTTPRequestHandler): def get_root_dir(self): authorization = self.headers.get("authorization") if not authorization: if not self.server.anonymous_dir: raise Unauthorized("unauthorized") return self.server.anonymous_dir else: authorization = authorization.split() if len(authorization) != 2 or authorization[0].lower() != "basic": raise Unauthorized("unauthorized") try: decoded = base64.decodebytes(authorization[1].encode("ascii")) user, password = decoded.decode("ascii").split(":") expected_password, directory = self.server.users[user] if password == expected_password: return directory except: # noqa raise Unauthorized("unauthorized") return None def unauthorized(self): shortmsg, longmsg = self.responses[HTTPStatus.UNAUTHORIZED] self.send_response(HTTPStatus.UNAUTHORIZED, shortmsg) self.send_header("Connection", "close") content = self.error_message_format % { "code": HTTPStatus.UNAUTHORIZED, "message": html.escape(longmsg, quote=False), "explain": html.escape(longmsg, quote=False), } body = content.encode("UTF-8", "replace") self.send_header("Content-Type", self.error_content_type) self.send_header("Content-Length", str(len(body))) self.send_header("WWW-Authenticate", 'Basic realm="{}"'.format(self.server.realm)) self.end_headers() self.end_headers() if self.command != "HEAD" and body: self.wfile.write(body) def do_GET(self): try: super().do_GET() except Unauthorized: self.unauthorized() def do_HEAD(self): try: super().do_HEAD() except Unauthorized: self.unauthorized() def translate_path(self, path): path = path.split("?", 1)[0] path = path.split("#", 1)[0] path = posixpath.normpath(path) assert posixpath.isabs(path) path = posixpath.relpath(path, "/") return os.path.join(self.get_root_dir(), path) class AuthHTTPServer(HTTPServer): def __init__(self, *args, **kwargs): self.users = {} self.anonymous_dir = None self.realm = "Realm" super().__init__(*args, **kwargs) class SimpleHttpServer(threading.Thread): def __init__(self): super().__init__() self.server = AuthHTTPServer(("127.0.0.1", 0), RequestHandler) self.started = False def start(self): self.started = True super().start() def run(self): self.server.serve_forever() def stop(self): if not self.started: return self.server.shutdown() self.join() def allow_anonymous(self, cwd): self.server.anonymous_dir = cwd def add_user(self, user, password, cwd): self.server.users[user] = (password, cwd) def base_url(self): return "http://127.0.0.1:{}".format(self.server.server_port) apache-buildstream-27ae392/tests/testutils/junction.py000066400000000000000000000032071514607367700232310ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os from buildstream import _yaml from buildstream._testing import create_repo # generate_junction() # # Generates a junction element with a git repository # # Args: # tmpdir: The tmpdir fixture, for storing the generated git repo # subproject_path: The path for the subproject, to add to the git repo # junction_path: The location to store the generated junction element # store_ref: Whether to store the ref in the junction.bst file # # Returns: # (str): The ref # def generate_junction(tmpdir, subproject_path, junction_path, *, store_ref=True, options=None): # Create a repo to hold the subproject and generate # a junction element for it # repodir = os.path.join(tmpdir, "junction-repo") os.makedirs(repodir) repo = create_repo("tar", repodir) source_ref = ref = repo.create(subproject_path) if not store_ref: source_ref = None element = {"kind": "junction", "sources": [repo.source_config(ref=source_ref)]} if options: element["config"] = {"options": options} _yaml.roundtrip_dump(element, junction_path) return ref apache-buildstream-27ae392/tests/testutils/patch.py000066400000000000000000000021671514607367700225030ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import subprocess def apply(file, patch): try: subprocess.check_output(["patch", file, patch]) except subprocess.CalledProcessError as e: message = "Patch failed with exit code {}\n Output:\n {}".format(e.returncode, e.output) print(message) raise def remove(file, patch): try: subprocess.check_output(["patch", "--reverse", file, patch]) except subprocess.CalledProcessError as e: message = "patch --reverse failed with exit code {}\n Output:\n {}".format(e.returncode, e.output) print(message) raise apache-buildstream-27ae392/tests/testutils/platform.py000066400000000000000000000035301514607367700232230ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Angelos Evripiotis import collections from contextlib import contextmanager import platform # override_platform_uname() # # Context manager to override the reported value of `platform.uname()`. # # Args: # system (str): Optional str to replace the 1st entry of uname with. # machine (str): Optional str to replace the 5th entry of uname with. # @contextmanager def override_platform_uname(*, system=None, machine=None): orig_func = platform.uname orig_system, node, release, version, orig_machine, processor = platform.uname() system = system or orig_system machine = machine or orig_machine def override_func(): # NOTE: # 1. We can't use `_replace` here because of this bug in # Python 3.9.0 - https://bugs.python.org/issue42163. # 2. We need to create a new subclass because the constructor of # `platform.uname_result` doesn't share the same interface between # Python 3.8 and 3.9. uname_result = collections.namedtuple("uname_result", "system node release version machine processor") return uname_result(system, node, release, version, machine, processor) platform.uname = override_func try: yield finally: platform.uname = orig_func apache-buildstream-27ae392/tests/testutils/python_repo.py000066400000000000000000000103441514607367700237460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import re import shutil import subprocess import sys import pytest SETUP_TEMPLATE = """\ from setuptools import setup setup( name='{name}', version='{version}', description='{name}', packages=['{pkgdirname}'], install_requires={pkgdeps}, entry_points={{ 'console_scripts': [ '{pkgdirname}={pkgdirname}:main' ] }} ) """ # All packages generated via generate_pip_package will have the functions below INIT_TEMPLATE = """\ def main(): print('This is {name}') def hello(actor='world'): print('Hello {{}}! This is {name}'.format(actor)) """ HTML_TEMPLATE = """\ Links for {name} {name}-{version}.tar.gz
""" # Creates a simple python source distribution and copies this into a specified # directory which is to serve as a mock python repository # # Args: # tmpdir (str): Directory in which the source files will be created # pypi (str): Directory serving as a mock python repository # name (str): The name of the package to be created # version (str): The version of the package to be created # # Returns: # None # def generate_pip_package(tmpdir, pypi, name, version="0.1", dependencies=None): if dependencies is None: dependencies = [] # check if package already exists in pypi pypi_package = os.path.join(pypi, re.sub("[^0-9a-zA-Z]+", "-", name)) if os.path.exists(pypi_package): return # create the package source files in tmpdir resulting in a directory # tree resembling the following structure: # # tmpdir # |-- setup.py # `-- package # `-- __init__.py # setup_file = os.path.join(tmpdir, "setup.py") pkgdirname = re.sub("[^0-9a-zA-Z]+", "", name) with open(setup_file, "w", encoding="utf-8") as f: f.write(SETUP_TEMPLATE.format(name=name, version=version, pkgdirname=pkgdirname, pkgdeps=dependencies)) os.chmod(setup_file, 0o755) package = os.path.join(tmpdir, pkgdirname) os.makedirs(package) main_file = os.path.join(package, "__init__.py") with open(main_file, "w", encoding="utf-8") as f: f.write(INIT_TEMPLATE.format(name=name)) os.chmod(main_file, 0o644) # Run sdist with a fresh process subprocess.run([sys.executable, "setup.py", "sdist"], cwd=tmpdir, check=True) # create directory for this package in pypi resulting in a directory # tree resembling the following structure: # # pypi # `-- pypi_package # |-- index.html # `-- foo-0.1.tar.gz # os.makedirs(pypi_package) # add an index html page index_html = os.path.join(pypi_package, "index.html") with open(index_html, "w", encoding="utf-8") as f: f.write(HTML_TEMPLATE.format(name=name, version=version)) # copy generated tarfile to pypi package dist_dir = os.path.join(tmpdir, "dist") for tar in os.listdir(dist_dir): tarpath = os.path.join(dist_dir, tar) shutil.copy(tarpath, pypi_package) @pytest.fixture def setup_pypi_repo(tmpdir): def create_pkgdir(package): pkgdirname = re.sub("[^0-9a-zA-Z]+", "", package) pkgdir = os.path.join(str(tmpdir), pkgdirname) os.makedirs(pkgdir) return pkgdir def add_packages(packages, pypi_repo): for package, dependencies in packages.items(): pkgdir = create_pkgdir(package) generate_pip_package(pkgdir, pypi_repo, package, dependencies=list(dependencies.keys())) for dependency, dependency_dependencies in dependencies.items(): add_packages({dependency: dependency_dependencies}, pypi_repo) return add_packages apache-buildstream-27ae392/tests/testutils/repo/000077500000000000000000000000001514607367700217715ustar00rootroot00000000000000apache-buildstream-27ae392/tests/testutils/repo/__init__.py000066400000000000000000000000001514607367700240700ustar00rootroot00000000000000apache-buildstream-27ae392/tests/testutils/repo/git.py000066400000000000000000000104741514607367700231340ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import shutil import subprocess import pytest from buildstream._testing import Repo from buildstream._testing._utils.site import GIT, GIT_ENV, HAVE_GIT class Git(Repo): def __init__(self, directory, subdir="repo"): if not HAVE_GIT: pytest.skip("git is not available") self.submodules = {} super().__init__(directory, subdir) self.env = os.environ.copy() self.env.update(GIT_ENV) def _run_git(self, *args, **kwargs): argv = [GIT] argv.extend(args) if "env" not in kwargs: kwargs["env"] = dict(self.env, PWD=self.repo) kwargs.setdefault("cwd", self.repo) kwargs.setdefault("check", True) return subprocess.run(argv, **kwargs) # pylint: disable=subprocess-run-check def create(self, directory): self.copy_directory(directory, self.repo) self._run_git("init", ".") self._run_git("checkout", "-b", "master") self._run_git("add", ".") self._run_git("commit", "-m", "Initial commit") return self.latest_commit() def add_tag(self, tag): self._run_git("tag", tag) def add_annotated_tag(self, tag, message): self._run_git("tag", "-a", tag, "-m", message) def add_commit(self): self._run_git("commit", "--allow-empty", "-m", "Additional commit") return self.latest_commit() def add_file(self, filename): shutil.copy(filename, self.repo) self._run_git("add", os.path.basename(filename)) self._run_git("commit", "-m", "Added {}".format(os.path.basename(filename))) return self.latest_commit() def modify_file(self, new_file, path): shutil.copy(new_file, os.path.join(self.repo, path)) self._run_git("commit", path, "-m", "Modified {}".format(os.path.basename(path))) return self.latest_commit() def add_submodule(self, subdir, url=None, checkout=None): submodule = {} if checkout is not None: submodule["checkout"] = checkout if url is not None: submodule["url"] = url self.submodules[subdir] = submodule self._run_git("-c", "protocol.file.allow=always", "submodule", "add", url, subdir) self._run_git("commit", "-m", "Added the submodule") return self.latest_commit() # This can also be used to a file or a submodule def remove_path(self, path): self._run_git("rm", path) self._run_git("commit", "-m", "Removing {}".format(path)) return self.latest_commit() def source_config(self, ref=None): return self.source_config_extra(ref) def source_config_extra(self, ref=None, checkout_submodules=None): config = {"kind": "git", "url": "file://" + self.repo, "track": "master"} if ref is not None: config["ref"] = ref if checkout_submodules is not None: config["checkout-submodules"] = checkout_submodules if self.submodules: config["submodules"] = dict(self.submodules) return config def latest_commit(self): return self._run_git( "rev-parse", "HEAD", stdout=subprocess.PIPE, universal_newlines=True, ).stdout.strip() def branch(self, branch_name): self._run_git("checkout", "-b", branch_name) def delete_tag(self, tag_name): self._run_git("tag", "-d", tag_name) def checkout(self, commit): self._run_git("checkout", commit) def merge(self, commit): self._run_git("merge", "-m", "Merge", commit) return self.latest_commit() def rev_parse(self, rev): return self._run_git( "rev-parse", rev, stdout=subprocess.PIPE, universal_newlines=True, ).stdout.strip() apache-buildstream-27ae392/tests/testutils/repo/tar.py000066400000000000000000000023311514607367700231300ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os import tarfile from buildstream.utils import sha256sum from buildstream._testing import Repo class Tar(Repo): def create(self, directory): tarball = os.path.join(self.repo, "file.tar.gz") old_dir = os.getcwd() os.chdir(directory) with tarfile.open(tarball, "w:gz") as tar: tar.add(".") os.chdir(old_dir) return sha256sum(tarball) def source_config(self, ref=None): tarball = os.path.join(self.repo, "file.tar.gz") config = {"kind": "tar", "url": "file://" + tarball, "directory": "", "base-dir": ""} if ref is not None: config["ref"] = ref return config apache-buildstream-27ae392/tests/testutils/runner_integration.py000066400000000000000000000020331514607367700253100ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Authors: # Will Salmon import time def wait_for_cache_granularity(): # This isn't called very often so has minimal impact on test runtime. # If this changes it may be worth while adding a more sophisticated approach. """ Mitigate the coarse granularity of the gitlab runners mtime This function waits for the mtime to increment so that the cache can sort by mtime and get the most recent results. """ time.sleep(1.1) apache-buildstream-27ae392/tests/testutils/site.py000066400000000000000000000027441514607367700223510ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # This function is used for pytest skipif() expressions. # # Tests which require our plugins in tests/plugins/pip-samples need # to check if these plugins are installed, they are only guaranteed # to be installed when running tox, but not when using pytest directly # to test that BuildStream works when integrated in your system. # def pip_sample_packages(): import importlib.metadata required = {"sample-plugins"} installed = {dist.name for dist in importlib.metadata.distributions()} missing = required - installed if missing: return False return True SAMPLE_PACKAGES_SKIP_REASON = """ The sample plugins package used to test pip plugin origins is not installed. This is usually tested automatically with `tox`, if you are running `pytest` directly then you can install these plugins directly using pip. The plugins are located in the tests/plugins/sample-plugins directory of your BuildStream checkout. """ apache-buildstream-27ae392/tox.ini000066400000000000000000000172361514607367700171460ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # # Tox global configuration # [tox] envlist = py{310,311,312,313,314} skip_missing_interpreters = true isolated_build = true # Configuration variables to share across environments [config] BST_PLUGINS_COMMUNITY_VERSION = 637e2d95e656b9d46575da5737b65fd0cc25dce2 # 2.1.0 BST_PLUGINS_VERSION = 79649529cffb695d0d22195ed9a4910c80ca6907 # 2.5.0 # # Defaults for all environments # # Anything specified here is inherited by the sections # [testenv] usedevelop = # This is required by Cython in order to get coverage for cython files. py{310,311,312,313,314}-!nocover: True commands = # Running with coverage reporting enabled py{310,311,312,313,314}-!plugins-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs} # Running with coverage reporting disabled py{310,311,312,313,314}-!plugins-nocover: pytest --basetemp {envtmpdir} {posargs} # Running external plugins tests with coverage reporting enabled py{310,311,312,313,314}-plugins-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc --plugins {posargs} # Running external plugins tests with coverage disabled py{310,311,312,313,314}-plugins-nocover: pytest --basetemp {envtmpdir} --plugins {posargs} commands_post: py{310,311,312,313,314}-!nocover: mkdir -p .coverage-reports py{310,311,312,313,314}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} deps = py{310,311,312,313,314}: -rrequirements/requirements.txt py{310,311,312,313,314}: -rrequirements/dev-requirements.txt py{310,311,312,313,314}: git+https://github.com/apache/buildstream-plugins.git@{env:BST_PLUGINS_VERSION:{[config]BST_PLUGINS_VERSION}} # Install local sample plugins for testing pip plugin origins py{310,311,312,313,314}: {toxinidir}/tests/plugins/sample-plugins # Install external plugins for plugin tests py{310,311,312,313,314}-plugins: git+https://gitlab.com/buildstream/buildstream-plugins-community.git@{env:BST_PLUGINS_COMMUNITY_VERSION:{[config]BST_PLUGINS_COMMUNITY_VERSION}}#egg=bst_plugins_community[deb] # Only require coverage and pytest-cov when using it !nocover: -rrequirements/cov-requirements.txt # Install pytest-random-order for '-randomized' randomized: pytest-random-order passenv = ARTIFACT_CACHE_SERVICE ARTIFACT_INDEX_SERVICE ARTIFACT_STORAGE_SERVICE BST_CAS_STAGING_ROOT GI_TYPELIB_PATH INTEGRATION_CACHE http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY REMOTE_EXECUTION_SERVICE SOURCE_CACHE_SERVICE SSL_CERT_FILE BST_PLUGINS_COMMUNITY_VERSION # # These keys are not inherited by any other sections # setenv = py{310,311,312,313,314}: COVERAGE_FILE = {envtmpdir}/.coverage py{310,311,312,313,314}: BST_TEST_HOME = {envtmpdir} py{310,311,312,313,314}: BST_TEST_XDG_CACHE_HOME = {envtmpdir}/cache py{310,311,312,313,314}: BST_TEST_XDG_CONFIG_HOME = {envtmpdir}/config py{310,311,312,313,314}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share # This is required to get coverage for Cython py{310,311,312,313,314}-!nocover: BST_CYTHON_TRACE = 1 randomized: PYTEST_ADDOPTS="--random-order-bucket=global" allowlist_externals = py{310,311,312,313,314}: mv mkdir # # Environment for native windows # [testenv:win32] commands = bst help cmd /C md testdir bst init testdir --project-name="test" deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt colorama windows-curses cython . allowlist_externals = bst cmd # # Coverage reporting # [testenv:coverage] # This is required by Cython in order to get coverage for cython files. usedevelop = True commands = coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/ coverage html --rcfile={toxinidir}/.coveragerc --directory={toxinidir}/.coverage-reports/ coverage report --rcfile={toxinidir}/.coveragerc --show-missing deps = -rrequirements/cov-requirements.txt setenv = COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage # # Code formatters # [testenv:format] skip_install = True deps = black==22.3.0 commands = black {posargs: src tests doc/source/conf.py setup.py} # # Code format checkers # [testenv:format-check] skip_install = True deps = black==22.3.0 commands = black --check --diff {posargs: src tests doc/source/conf.py setup.py} # # Running linters # [testenv:lint] commands_pre = # Build C extensions to allow Pylint to analyse them {envpython} setup.py build_ext --inplace commands = pylint {posargs: buildstream tests doc/source/conf.py setup.py} deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt setuptools Cython # # Running static type checkers # [testenv:mypy] skip_install = True commands = mypy --namespace-packages {posargs} deps = mypy==1.13.0 types-protobuf==5.28.3.20241030 types-ujson==5.10.0.20240515 -rrequirements/requirements.txt -rrequirements/dev-requirements.txt # # Building documentation # [testenv:docs] commands = make -C doc # sphinx_rtd_theme < 0.4.2 breaks search functionality for Sphinx >= 1.8 deps = sphinx == 8.1.3 sphinx-click == 6.0.0 sphinx_rtd_theme == 3.0.2 pytest -rrequirements/requirements.txt git+https://github.com/apache/buildstream-plugins.git@{env:BST_PLUGINS_VERSION:{[config]BST_PLUGINS_VERSION}} passenv = BST_FORCE_SESSION_REBUILD BST_SOURCE_CACHE HOME LANG LC_ALL allowlist_externals = make # # (re-)Generating man pages # [testenv:man] commands = # We set the version to 0, because click-man would now # otherwise want to pull a hardcoded version from pyproject.toml # that somehow gets magically transported to the EntryPoint.version click-man --man-version {posargs} --target man bst deps = click-man >= 0.5.0 -rrequirements/requirements.txt -rrequirements/dev-requirements.txt Cython # # Usefull for running arbitrary scripts in a BuildStream virtual env # [testenv:venv] commands = {posargs} deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt allowlist_externals = * # # Convenience environment for running individual tests from the # battery of templated source tests. # # You should pass this the part of a test node's id after "::". For # example, to run the test # buildstream/testing/_sourcetests/fetch.py::test_fetch_cross_junction[git-inline] # you would do tox -e sourcetests -- test_fetch_cross_junction[git-inline] # # This does rely on the fact that none of the tests in # buildstream.testing have the same name. # [testenv:sourcetests] commands = pytest --basetemp {envtmpdir} --ignore tests -k "{posargs}" deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt # When building using PEP518 and 517, we don't want default dependencies # installed by the base environment. [testenv:.package] deps = # # Regenerate protobuf and grpc code # [testenv:build-grpc] skip_install = True commands = python setup.py build_grpc deps = grpcio-tools==1.69.0 # Requires protobuf >= 5.29 Cython apache-buildstream-27ae392/versioneer.py000066400000000000000000002432271514607367700203670ustar00rootroot00000000000000 # Version: 0.28 """The Versioneer - like a rocketeer, but for versions. The Versioneer ============== * like a rocketeer, but for versions! * https://github.com/python-versioneer/python-versioneer * Brian Warner * License: Public Domain (Unlicense) * Compatible with: Python 3.7, 3.8, 3.9, 3.10 and pypy3 * [![Latest Version][pypi-image]][pypi-url] * [![Build Status][travis-image]][travis-url] This is a tool for managing a recorded version number in setuptools-based python projects. The goal is to remove the tedious and error-prone "update the embedded version string" step from your release process. Making a new release should be as easy as recording a new tag in your version-control system, and maybe making new tarballs. ## Quick Install Versioneer provides two installation modes. The "classic" vendored mode installs a copy of versioneer into your repository. The experimental build-time dependency mode is intended to allow you to skip this step and simplify the process of upgrading. ### Vendored mode * `pip install versioneer` to somewhere in your $PATH * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is available, so you can also use `conda install -c conda-forge versioneer` * add a `[tool.versioneer]` section to your `pyproject.toml` or a `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) * Note that you will need to add `tomli; python_version < "3.11"` to your build-time dependencies if you use `pyproject.toml` * run `versioneer install --vendor` in your source tree, commit the results * verify version information with `python setup.py version` ### Build-time dependency mode * `pip install versioneer` to somewhere in your $PATH * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is available, so you can also use `conda install -c conda-forge versioneer` * add a `[tool.versioneer]` section to your `pyproject.toml` or a `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) * add `versioneer` (with `[toml]` extra, if configuring in `pyproject.toml`) to the `requires` key of the `build-system` table in `pyproject.toml`: ```toml [build-system] requires = ["setuptools", "versioneer[toml]"] build-backend = "setuptools.build_meta" ``` * run `versioneer install --no-vendor` in your source tree, commit the results * verify version information with `python setup.py version` ## Version Identifiers Source trees come from a variety of places: * a version-control system checkout (mostly used by developers) * a nightly tarball, produced by build automation * a snapshot tarball, produced by a web-based VCS browser, like github's "tarball from tag" feature * a release tarball, produced by "setup.py sdist", distributed through PyPI Within each source tree, the version identifier (either a string or a number, this tool is format-agnostic) can come from a variety of places: * ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows about recent "tags" and an absolute revision-id * the name of the directory into which the tarball was unpacked * an expanded VCS keyword ($Id$, etc) * a `_version.py` created by some earlier build step For released software, the version identifier is closely related to a VCS tag. Some projects use tag names that include more than just the version string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool needs to strip the tag prefix to extract the version identifier. For unreleased software (between tags), the version identifier should provide enough information to help developers recreate the same tree, while also giving them an idea of roughly how old the tree is (after version 1.2, before version 1.3). Many VCS systems can report a description that captures this, for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has uncommitted changes). The version identifier is used for multiple purposes: * to allow the module to self-identify its version: `myproject.__version__` * to choose a name and prefix for a 'setup.py sdist' tarball ## Theory of Operation Versioneer works by adding a special `_version.py` file into your source tree, where your `__init__.py` can import it. This `_version.py` knows how to dynamically ask the VCS tool for version information at import time. `_version.py` also contains `$Revision$` markers, and the installation process marks `_version.py` to have this marker rewritten with a tag name during the `git archive` command. As a result, generated tarballs will contain enough information to get the proper version. To allow `setup.py` to compute a version too, a `versioneer.py` is added to the top level of your source tree, next to `setup.py` and the `setup.cfg` that configures it. This overrides several distutils/setuptools commands to compute the version when invoked, and changes `setup.py build` and `setup.py sdist` to replace `_version.py` with a small static file that contains just the generated version data. ## Installation See [INSTALL.md](./INSTALL.md) for detailed installation instructions. ## Version-String Flavors Code which uses Versioneer can learn about its version string at runtime by importing `_version` from your main `__init__.py` file and running the `get_versions()` function. From the "outside" (e.g. in `setup.py`), you can import the top-level `versioneer.py` and run `get_versions()`. Both functions return a dictionary with different flavors of version information: * `['version']`: A condensed version string, rendered using the selected style. This is the most commonly used value for the project's version string. The default "pep440" style yields strings like `0.11`, `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section below for alternative styles. * `['full-revisionid']`: detailed revision identifier. For Git, this is the full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". * `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the commit date in ISO 8601 format. This will be None if the date is not available. * `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that this is only accurate if run in a VCS checkout, otherwise it is likely to be False or None * `['error']`: if the version string could not be computed, this will be set to a string describing the problem, otherwise it will be None. It may be useful to throw an exception in setup.py if this is set, to avoid e.g. creating tarballs with a version string of "unknown". Some variants are more useful than others. Including `full-revisionid` in a bug report should allow developers to reconstruct the exact code being tested (or indicate the presence of local changes that should be shared with the developers). `version` is suitable for display in an "about" box or a CLI `--version` output: it can be easily compared against release notes and lists of bugs fixed in various releases. The installer adds the following text to your `__init__.py` to place a basic version in `YOURPROJECT.__version__`: from ._version import get_versions __version__ = get_versions()['version'] del get_versions ## Styles The setup.cfg `style=` configuration controls how the VCS information is rendered into a version string. The default style, "pep440", produces a PEP440-compliant string, equal to the un-prefixed tag name for actual releases, and containing an additional "local version" section with more detail for in-between builds. For Git, this is TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags --dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and that this commit is two revisions ("+2") beyond the "0.11" tag. For released software (exactly equal to a known tag), the identifier will only contain the stripped tag, e.g. "0.11". Other styles are available. See [details.md](details.md) in the Versioneer source tree for descriptions. ## Debugging Versioneer tries to avoid fatal errors: if something goes wrong, it will tend to return a version of "0+unknown". To investigate the problem, run `setup.py version`, which will run the version-lookup code in a verbose mode, and will display the full contents of `get_versions()` (including the `error` string, which may help identify what went wrong). ## Known Limitations Some situations are known to cause problems for Versioneer. This details the most significant ones. More can be found on Github [issues page](https://github.com/python-versioneer/python-versioneer/issues). ### Subprojects Versioneer has limited support for source trees in which `setup.py` is not in the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are two common reasons why `setup.py` might not be in the root: * Source trees which contain multiple subprojects, such as [Buildbot](https://github.com/buildbot/buildbot), which contains both "master" and "slave" subprojects, each with their own `setup.py`, `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI distributions (and upload multiple independently-installable tarballs). * Source trees whose main purpose is to contain a C library, but which also provide bindings to Python (and perhaps other languages) in subdirectories. Versioneer will look for `.git` in parent directories, and most operations should get the right version string. However `pip` and `setuptools` have bugs and implementation details which frequently cause `pip install .` from a subproject directory to fail to find a correct version string (so it usually defaults to `0+unknown`). `pip install --editable .` should work correctly. `setup.py install` might work too. Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in some later version. [Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking this issue. The discussion in [PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the issue from the Versioneer side in more detail. [pip PR#3176](https://github.com/pypa/pip/pull/3176) and [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve pip to let Versioneer work correctly. Versioneer-0.16 and earlier only looked for a `.git` directory next to the `setup.cfg`, so subprojects were completely unsupported with those releases. ### Editable installs with setuptools <= 18.5 `setup.py develop` and `pip install --editable .` allow you to install a project into a virtualenv once, then continue editing the source code (and test) without re-installing after every change. "Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a convenient way to specify executable scripts that should be installed along with the python package. These both work as expected when using modern setuptools. When using setuptools-18.5 or earlier, however, certain operations will cause `pkg_resources.DistributionNotFound` errors when running the entrypoint script, which must be resolved by re-installing the package. This happens when the install happens with one version, then the egg_info data is regenerated while a different version is checked out. Many setup.py commands cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into a different virtualenv), so this can be surprising. [Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes this one, but upgrading to a newer version of setuptools should probably resolve it. ## Updating Versioneer To upgrade your project to a new release of Versioneer, do the following: * install the new Versioneer (`pip install -U versioneer` or equivalent) * edit `setup.cfg` and `pyproject.toml`, if necessary, to include any new configuration settings indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. * re-run `versioneer install --[no-]vendor` in your source tree, to replace `SRC/_version.py` * commit any changed files ## Future Directions This tool is designed to make it easily extended to other version-control systems: all VCS-specific components are in separate directories like src/git/ . The top-level `versioneer.py` script is assembled from these components by running make-versioneer.py . In the future, make-versioneer.py will take a VCS name as an argument, and will construct a version of `versioneer.py` that is specific to the given VCS. It might also take the configuration arguments that are currently provided manually during installation by editing setup.py . Alternatively, it might go the other direction and include code from all supported VCS systems, reducing the number of intermediate scripts. ## Similar projects * [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time dependency * [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of versioneer * [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools plugin ## License To make Versioneer easier to embed, all its code is dedicated to the public domain. The `_version.py` that it creates is also in the public domain. Specifically, both are released under the "Unlicense", as described in https://unlicense.org/. [pypi-image]: https://img.shields.io/pypi/v/versioneer.svg [pypi-url]: https://pypi.python.org/pypi/versioneer/ [travis-image]: https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg [travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer """ # pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring # pylint:disable=missing-class-docstring,too-many-branches,too-many-statements # pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error # pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with # pylint:disable=attribute-defined-outside-init,too-many-arguments import configparser import errno import json import os import re import subprocess import sys from pathlib import Path from typing import Callable, Dict import functools have_tomllib = True if sys.version_info >= (3, 11): import tomllib else: try: import tomli as tomllib except ImportError: have_tomllib = False class VersioneerConfig: """Container for Versioneer configuration parameters.""" def get_root(): """Get the project root directory. We require that all commands are run from the project root, i.e. the directory that contains setup.py, setup.cfg, and versioneer.py . """ root = os.path.realpath(os.path.abspath(os.getcwd())) setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): # allow 'python path/to/setup.py COMMAND' root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) setup_py = os.path.join(root, "setup.py") versioneer_py = os.path.join(root, "versioneer.py") if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): err = ("Versioneer was unable to run the project root directory. " "Versioneer requires setup.py to be executed from " "its immediate directory (like 'python setup.py COMMAND'), " "or in a way that lets it use sys.argv[0] to find the root " "(like 'python path/to/setup.py COMMAND').") raise VersioneerBadRootError(err) try: # Certain runtime workflows (setup.py install/develop in a setuptools # tree) execute all dependencies in a single python process, so # "versioneer" may be imported multiple times, and python's shared # module-import table will cache the first one. So we can't use # os.path.dirname(__file__), as that will find whichever # versioneer.py was first imported, even in later projects. my_path = os.path.realpath(os.path.abspath(__file__)) me_dir = os.path.normcase(os.path.splitext(my_path)[0]) vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): print("Warning: build in %s is using versioneer.py from %s" % (os.path.dirname(my_path), versioneer_py)) except NameError: pass return root def get_config_from_root(root): """Read the project setup.cfg file to determine Versioneer config.""" # This might raise OSError (if setup.cfg is missing), or # configparser.NoSectionError (if it lacks a [versioneer] section), or # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . root = Path(root) pyproject_toml = root / "pyproject.toml" setup_cfg = root / "setup.cfg" section = None if pyproject_toml.exists() and have_tomllib: try: with open(pyproject_toml, 'rb') as fobj: pp = tomllib.load(fobj) section = pp['tool']['versioneer'] except (tomllib.TOMLDecodeError, KeyError): pass if not section: parser = configparser.ConfigParser() with open(setup_cfg) as cfg_file: parser.read_file(cfg_file) parser.get("versioneer", "VCS") # raise error if missing section = parser["versioneer"] cfg = VersioneerConfig() cfg.VCS = section['VCS'] cfg.style = section.get("style", "") cfg.versionfile_source = section.get("versionfile_source") cfg.versionfile_build = section.get("versionfile_build") cfg.tag_prefix = section.get("tag_prefix") if cfg.tag_prefix in ("''", '""', None): cfg.tag_prefix = "" cfg.parentdir_prefix = section.get("parentdir_prefix") cfg.verbose = section.get("verbose") return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" # these dictionaries contain VCS-specific tools LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" HANDLERS.setdefault(vcs, {})[method] = f return f return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue if verbose: print("unable to run %s" % dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %s" % (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %s (error)" % dispcmd) print("stdout was %s" % stdout) return None, process.returncode return stdout, process.returncode LONG_VERSION_PY['git'] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. # Generated by versioneer-0.28 # https://github.com/python-versioneer/python-versioneer """Git implementation of _version.py.""" import errno import os import re import subprocess import sys from typing import Callable, Dict import functools def get_keywords(): """Get the keywords needed to look up the version information.""" # these strings will be replaced by git during git-archive. # setup.py/versioneer.py will grep for the variable names, so they must # each be defined on a line of their own. _version.py will just call # get_keywords(). git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} return keywords class VersioneerConfig: """Container for Versioneer configuration parameters.""" def get_config(): """Create, populate and return the VersioneerConfig() object.""" # these strings are filled in when 'setup.py versioneer' creates # _version.py cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "%(STYLE)s" cfg.tag_prefix = "%(TAG_PREFIX)s" cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" cfg.verbose = False return cfg class NotThisMethod(Exception): """Exception raised if a method is not valid for the current scenario.""" LONG_VERSION_PY: Dict[str, str] = {} HANDLERS: Dict[str, Dict[str, Callable]] = {} def register_vcs_handler(vcs, method): # decorator """Create decorator to mark a method as the handler of a VCS.""" def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) process = None popen_kwargs = {} if sys.platform == "win32": # This hides the console window if pythonw.exe is used startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW popen_kwargs["startupinfo"] = startupinfo for command in commands: try: dispcmd = str([command] + args) # remember shell=False, so use git.cmd on windows, not just git process = subprocess.Popen([command] + args, cwd=cwd, env=env, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr else None), **popen_kwargs) break except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue if verbose: print("unable to run %%s" %% dispcmd) print(e) return None, None else: if verbose: print("unable to find command, tried %%s" %% (commands,)) return None, None stdout = process.communicate()[0].strip().decode() if process.returncode != 0: if verbose: print("unable to run %%s (error)" %% dispcmd) print("stdout was %%s" %% stdout) return None, process.returncode return stdout, process.returncode def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %%s but none started with prefix %%s" %% (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %%d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%%s', no digits" %% ",".join(refs - tags)) if verbose: print("likely tags: %%s" %% ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %%s" %% r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %%s not under git control" %% root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%%s'" %% describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%%s' doesn't start with prefix '%%s'" print(fmt %% (full_tag, tag_prefix)) pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" %% (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def plus_or_dot(pieces): """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces): """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver): """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces): """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%%d" %% (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%%d" %% pieces["distance"] return rendered def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%%s" %% pieces["short"] else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%%s" %% pieces["short"] return rendered def render_pep440_post_branch(pieces): """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%%s" %% pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%%s" %% pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%%d" %% pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces): """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces): """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%%s'" %% style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} def get_versions(): """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which # case we can only use expanded keywords. cfg = get_config() verbose = cfg.verbose try: return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass try: root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. for _ in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to find root of source tree", "date": None} try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) return render(pieces, cfg.style) except NotThisMethod: pass try: if cfg.parentdir_prefix: return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) except NotThisMethod: pass return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} ''' @register_vcs_handler("git", "get_keywords") def git_get_keywords(versionfile_abs): """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these # keywords. When used from setup.py, we don't want to import _version.py, # so we do it with a regexp instead. This function is not used from # _version.py. keywords = {} try: with open(versionfile_abs, "r") as fobj: for line in fobj: if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["full"] = mo.group(1) if line.strip().startswith("git_date ="): mo = re.search(r'=\s*"(.*)"', line) if mo: keywords["date"] = mo.group(1) except OSError: pass return keywords @register_vcs_handler("git", "keywords") def git_versions_from_keywords(keywords, tag_prefix, verbose): """Get version information from git keywords.""" if "refnames" not in keywords: raise NotThisMethod("Short version file found") date = keywords.get("date") if date is not None: # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 # -like" string, which we must then edit to make compliant), because # it's been around since git-1.5.3, and it's too difficult to # discover which version we're using, or to work around using an # older one. date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d # expansion behaves like git log --decorate=short and strips out the # refs/heads/ and refs/tags/ prefixes that would let us distinguish # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". tags = {r for r in refs if re.search(r'\d', r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] # Filter out refs that exactly match prefix or that don't start # with a number once the prefix is stripped (mostly a concern # when prefix is '') if not re.match(r'\d', r): continue if verbose: print("picking %s" % r) return {"version": r, "full-revisionid": keywords["full"].strip(), "dirty": False, "error": None, "date": date} # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") return {"version": "0+unknown", "full-revisionid": keywords["full"].strip(), "dirty": False, "error": "no suitable tags", "date": None} @register_vcs_handler("git", "pieces_from_vcs") def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): """Get version from 'git describe' in the root of the source tree. This only gets called if the git-archive 'subst' keywords were *not* expanded, and _version.py hasn't already been rewritten with a short version string, meaning we're inside a checked out source tree. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] # GIT_DIR can interfere with correct operation of Versioneer. # It may be intended to be passed to the Versioneer-versioned project, # but that should not change where we get our version from. env = os.environ.copy() env.pop("GIT_DIR", None) runner = functools.partial(runner, env=env) _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose) if rc != 0: if verbose: print("Directory %s not under git control" % root) raise NotThisMethod("'git rev-parse --git-dir' returned error") # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) describe_out, rc = runner(GITS, [ "describe", "--tags", "--dirty", "--always", "--long", "--match", f"{tag_prefix}[[:digit:]]*" ], cwd=root) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") describe_out = describe_out.strip() full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) if full_out is None: raise NotThisMethod("'git rev-parse' failed") full_out = full_out.strip() pieces = {} pieces["long"] = full_out pieces["short"] = full_out[:7] # maybe improved later pieces["error"] = None branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) # --abbrev-ref was added in git-1.6.3 if rc != 0 or branch_name is None: raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") branch_name = branch_name.strip() if branch_name == "HEAD": # If we aren't exactly on a branch, pick a branch which represents # the current commit. If all else fails, we are on a branchless # commit. branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) # --contains was added in git-1.5.4 if rc != 0 or branches is None: raise NotThisMethod("'git branch --contains' returned error") branches = branches.split("\n") # Remove the first line if we're running detached if "(" in branches[0]: branches.pop(0) # Strip off the leading "* " from the list of branches. branches = [branch[2:] for branch in branches] if "master" in branches: branch_name = "master" elif not branches: branch_name = None else: # Pick the first branch that is returned. Good or bad. branch_name = branches[0] pieces["branch"] = branch_name # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] # TAG might have hyphens. git_describe = describe_out # look for -dirty suffix dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: git_describe = git_describe[:git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) if not mo: # unparsable. Maybe git-describe is misbehaving? pieces["error"] = ("unable to parse git-describe output: '%s'" % describe_out) return pieces # tag full_tag = mo.group(1) if not full_tag.startswith(tag_prefix): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" % (full_tag, tag_prefix)) return pieces pieces["closest-tag"] = full_tag[len(tag_prefix):] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) # commit: short hex revision ID pieces["short"] = mo.group(3) else: # HEX: no tags pieces["closest-tag"] = None out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) pieces["distance"] = len(out.split()) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() # Use only the last line. Previous lines may contain GPG signature # information. date = date.splitlines()[-1] pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces def do_vcs_install(versionfile_source, ipy): """Git-specific installation logic for Versioneer. For Git, this means creating/changing .gitattributes to mark _version.py for export-subst keyword substitution. """ GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] files = [versionfile_source] if ipy: files.append(ipy) if "VERSIONEER_PEP518" not in globals(): try: my_path = __file__ if my_path.endswith((".pyc", ".pyo")): my_path = os.path.splitext(my_path)[0] + ".py" versioneer_file = os.path.relpath(my_path) except NameError: versioneer_file = "versioneer.py" files.append(versioneer_file) present = False try: with open(".gitattributes", "r") as fobj: for line in fobj: if line.strip().startswith(versionfile_source): if "export-subst" in line.strip().split()[1:]: present = True break except OSError: pass if not present: with open(".gitattributes", "a+") as fobj: fobj.write(f"{versionfile_source} export-subst\n") files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) def versions_from_parentdir(parentdir_prefix, root, verbose): """Try to determine the version from the parent directory name. Source tarballs conventionally unpack into a directory that includes both the project name and a version string. We will also support searching up two directory levels for an appropriately named parent directory """ rootdirs = [] for _ in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): return {"version": dirname[len(parentdir_prefix):], "full-revisionid": None, "dirty": False, "error": None, "date": None} rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: print("Tried directories %s but none started with prefix %s" % (str(rootdirs), parentdir_prefix)) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") SHORT_VERSION_PY = """ # This file was generated by 'versioneer.py' (0.28) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. import json version_json = ''' %s ''' # END VERSION_JSON def get_versions(): return json.loads(version_json) """ def versions_from_file(filename): """Try to determine the version from _version.py if present.""" try: with open(filename) as f: contents = f.read() except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) if not mo: mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", contents, re.M | re.S) if not mo: raise NotThisMethod("no version_json in _version.py") return json.loads(mo.group(1)) def write_to_version_file(filename, versions): """Write the given version number to the given _version.py file.""" os.unlink(filename) contents = json.dumps(versions, sort_keys=True, indent=1, separators=(",", ": ")) with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) print("set %s to '%s'" % (filename, versions["version"])) def plus_or_dot(pieces): """Return a + if we don't already have one, else return a .""" if "+" in pieces.get("closest-tag", ""): return "." return "+" def render_pep440(pieces): """Build up version string, with post-release "local version identifier". Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty Exceptions: 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_branch(pieces): """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . The ".dev0" means not master branch. Note that .dev0 sorts backwards (a feature branch will appear "older" than the master branch). Exceptions: 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0" if pieces["branch"] != "master": rendered += ".dev0" rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered def pep440_split_post(ver): """Split pep440 version string at the post-release segment. Returns the release segments before the post-release and the post-release version number (or -1 if no post-release segment is present). """ vc = str.split(ver, ".post") return vc[0], int(vc[1] or 0) if len(vc) == 2 else None def render_pep440_pre(pieces): """TAG[.postN.devDISTANCE] -- No -dirty. Exceptions: 1: no tags. 0.post0.devDISTANCE """ if pieces["closest-tag"]: if pieces["distance"]: # update the post release segment tag_version, post_version = pep440_split_post(pieces["closest-tag"]) rendered = tag_version if post_version is not None: rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) else: rendered += ".post0.dev%d" % (pieces["distance"]) else: # no commits, use the tag as the version rendered = pieces["closest-tag"] else: # exception #1 rendered = "0.post0.dev%d" % pieces["distance"] return rendered def render_pep440_post(pieces): """TAG[.postDISTANCE[.dev0]+gHEX] . The ".dev0" means dirty. Note that .dev0 sorts backwards (a dirty tree will appear "older" than the corresponding clean one), but you shouldn't be releasing software with -dirty anyways. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" rendered += "+g%s" % pieces["short"] return rendered def render_pep440_post_branch(pieces): """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . The ".dev0" means not master branch. Exceptions: 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += plus_or_dot(pieces) rendered += "g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["branch"] != "master": rendered += ".dev0" rendered += "+g%s" % pieces["short"] if pieces["dirty"]: rendered += ".dirty" return rendered def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . The ".dev0" means dirty. Exceptions: 1: no tags. 0.postDISTANCE[.dev0] """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"] or pieces["dirty"]: rendered += ".post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" else: # exception #1 rendered = "0.post%d" % pieces["distance"] if pieces["dirty"]: rendered += ".dev0" return rendered def render_git_describe(pieces): """TAG[-DISTANCE-gHEX][-dirty]. Like 'git describe --tags --dirty --always'. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] if pieces["distance"]: rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render_git_describe_long(pieces): """TAG-DISTANCE-gHEX[-dirty]. Like 'git describe --tags --dirty --always -long'. The distance/hash is unconditional. Exceptions: 1: no tags. HEX[-dirty] (note: no 'g' prefix) """ if pieces["closest-tag"]: rendered = pieces["closest-tag"] rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: # exception #1 rendered = pieces["short"] if pieces["dirty"]: rendered += "-dirty" return rendered def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: return {"version": "unknown", "full-revisionid": pieces.get("long"), "dirty": None, "error": pieces["error"], "date": None} if not style or style == "default": style = "pep440" # the default if style == "pep440": rendered = render_pep440(pieces) elif style == "pep440-branch": rendered = render_pep440_branch(pieces) elif style == "pep440-pre": rendered = render_pep440_pre(pieces) elif style == "pep440-post": rendered = render_pep440_post(pieces) elif style == "pep440-post-branch": rendered = render_pep440_post_branch(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": rendered = render_git_describe_long(pieces) else: raise ValueError("unknown style '%s'" % style) return {"version": rendered, "full-revisionid": pieces["long"], "dirty": pieces["dirty"], "error": None, "date": pieces.get("date")} class VersioneerBadRootError(Exception): """The project root directory is unknown or missing key files.""" def get_versions(verbose=False): """Get the project version from whatever source is available. Returns dict with two keys: 'version' and 'full'. """ if "versioneer" in sys.modules: # see the discussion in cmdclass.py:get_cmdclass() del sys.modules["versioneer"] root = get_root() cfg = get_config_from_root(root) assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" handlers = HANDLERS.get(cfg.VCS) assert handlers, "unrecognized VCS '%s'" % cfg.VCS verbose = verbose or cfg.verbose assert cfg.versionfile_source is not None, \ "please set versioneer.versionfile_source" assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" versionfile_abs = os.path.join(root, cfg.versionfile_source) # extract version from first of: _version.py, VCS command (e.g. 'git # describe'), parentdir. This is meant to work for developers using a # source checkout, for users of a tarball created by 'setup.py sdist', # and for users of a tarball/zipball created by 'git archive' or github's # download-from-tag feature or the equivalent in other VCSes. get_keywords_f = handlers.get("get_keywords") from_keywords_f = handlers.get("keywords") if get_keywords_f and from_keywords_f: try: keywords = get_keywords_f(versionfile_abs) ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) if verbose: print("got version from expanded keyword %s" % ver) return ver except NotThisMethod: pass try: ver = versions_from_file(versionfile_abs) if verbose: print("got version from file %s %s" % (versionfile_abs, ver)) return ver except NotThisMethod: pass from_vcs_f = handlers.get("pieces_from_vcs") if from_vcs_f: try: pieces = from_vcs_f(cfg.tag_prefix, root, verbose) ver = render(pieces, cfg.style) if verbose: print("got version from VCS %s" % ver) return ver except NotThisMethod: pass try: if cfg.parentdir_prefix: ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) if verbose: print("got version from parentdir %s" % ver) return ver except NotThisMethod: pass if verbose: print("unable to compute version") return {"version": "0+unknown", "full-revisionid": None, "dirty": None, "error": "unable to compute version", "date": None} def get_version(): """Get the short version string for this project.""" return get_versions()["version"] def get_cmdclass(cmdclass=None): """Get the custom setuptools subclasses used by Versioneer. If the package uses a different cmdclass (e.g. one from numpy), it should be provide as an argument. """ if "versioneer" in sys.modules: del sys.modules["versioneer"] # this fixes the "python setup.py develop" case (also 'install' and # 'easy_install .'), in which subdependencies of the main project are # built (using setup.py bdist_egg) in the same python process. Assume # a main project A and a dependency B, which use different versions # of Versioneer. A's setup.py imports A's Versioneer, leaving it in # sys.modules by the time B's setup.py is executed, causing B to run # with the wrong versioneer. Setuptools wraps the sub-dep builds in a # sandbox that restores sys.modules to it's pre-build state, so the # parent is protected against the child's "import versioneer". By # removing ourselves from sys.modules here, before the child build # happens, we protect the child from the parent's versioneer too. # Also see https://github.com/python-versioneer/python-versioneer/issues/52 cmds = {} if cmdclass is None else cmdclass.copy() # we add "version" to setuptools from setuptools import Command class cmd_version(Command): description = "report generated version string" user_options = [] boolean_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self): vers = get_versions(verbose=True) print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) print(" dirty: %s" % vers.get("dirty")) print(" date: %s" % vers.get("date")) if vers["error"]: print(" error: %s" % vers["error"]) cmds["version"] = cmd_version # we override "build_py" in setuptools # # most invocation pathways end up running build_py: # distutils/build -> build_py # distutils/install -> distutils/build ->.. # setuptools/bdist_wheel -> distutils/install ->.. # setuptools/bdist_egg -> distutils/install_lib -> build_py # setuptools/install -> bdist_egg ->.. # setuptools/develop -> ? # pip install: # copies source tree to a tempdir before running egg_info/etc # if .git isn't copied too, 'git describe' will fail # then does setup.py bdist_wheel, or sometimes setup.py install # setup.py egg_info -> ? # pip install -e . and setuptool/editable_wheel will invoke build_py # but the build_py command is not expected to copy any files. # we override different "build_py" commands for both environments if 'build_py' in cmds: _build_py = cmds['build_py'] else: from setuptools.command.build_py import build_py as _build_py class cmd_build_py(_build_py): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_py.run(self) if getattr(self, "editable_mode", False): # During editable installs `.py` and data files are # not copied to build_lib return # now locate _version.py in the new build/ directory and replace # it with an updated value if cfg.versionfile_build: target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_py"] = cmd_build_py if 'build_ext' in cmds: _build_ext = cmds['build_ext'] else: from setuptools.command.build_ext import build_ext as _build_ext class cmd_build_ext(_build_ext): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() _build_ext.run(self) if self.inplace: # build_ext --inplace will only build extensions in # build/lib<..> dir with no _version.py to write to. # As in place builds will already have a _version.py # in the module dir, we do not need to write one. return # now locate _version.py in the new build/ directory and replace # it with an updated value if not cfg.versionfile_build: return target_versionfile = os.path.join(self.build_lib, cfg.versionfile_build) if not os.path.exists(target_versionfile): print(f"Warning: {target_versionfile} does not exist, skipping " "version update. This can happen if you are running build_ext " "without first running build_py.") return print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) cmds["build_ext"] = cmd_build_ext if "cx_Freeze" in sys.modules: # cx_freeze enabled? from cx_Freeze.dist import build_exe as _build_exe # nczeczulin reports that py2exe won't like the pep440-style string # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. # setup(console=[{ # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION # "product_version": versioneer.get_version(), # ... class cmd_build_exe(_build_exe): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() target_versionfile = cfg.versionfile_source print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) _build_exe.run(self) os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) cmds["build_exe"] = cmd_build_exe del cmds["build_py"] if 'py2exe' in sys.modules: # py2exe enabled? try: from py2exe.setuptools_buildexe import py2exe as _py2exe except ImportError: from py2exe.distutils_buildexe import py2exe as _py2exe class cmd_py2exe(_py2exe): def run(self): root = get_root() cfg = get_config_from_root(root) versions = get_versions() target_versionfile = cfg.versionfile_source print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, versions) _py2exe.run(self) os.unlink(target_versionfile) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) cmds["py2exe"] = cmd_py2exe # sdist farms its file list building out to egg_info if 'egg_info' in cmds: _egg_info = cmds['egg_info'] else: from setuptools.command.egg_info import egg_info as _egg_info class cmd_egg_info(_egg_info): def find_sources(self): # egg_info.find_sources builds the manifest list and writes it # in one shot super().find_sources() # Modify the filelist and normalize it root = get_root() cfg = get_config_from_root(root) self.filelist.append('versioneer.py') if cfg.versionfile_source: # There are rare cases where versionfile_source might not be # included by default, so we must be explicit self.filelist.append(cfg.versionfile_source) self.filelist.sort() self.filelist.remove_duplicates() # The write method is hidden in the manifest_maker instance that # generated the filelist and was thrown away # We will instead replicate their final normalization (to unicode, # and POSIX-style paths) from setuptools import unicode_utils normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') for f in self.filelist.files] manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') with open(manifest_filename, 'w') as fobj: fobj.write('\n'.join(normalized)) cmds['egg_info'] = cmd_egg_info # we override different "sdist" commands for both environments if 'sdist' in cmds: _sdist = cmds['sdist'] else: from setuptools.command.sdist import sdist as _sdist class cmd_sdist(_sdist): def run(self): versions = get_versions() self._versioneer_generated_versions = versions # unless we update this, the command will keep using the old # version self.distribution.metadata.version = versions["version"] return _sdist.run(self) def make_release_tree(self, base_dir, files): root = get_root() cfg = get_config_from_root(root) _sdist.make_release_tree(self, base_dir, files) # now locate _version.py in the new base_dir directory # (remembering that it may be a hardlink) and replace it with an # updated value target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) write_to_version_file(target_versionfile, self._versioneer_generated_versions) cmds["sdist"] = cmd_sdist return cmds CONFIG_ERROR = """ setup.cfg is missing the necessary Versioneer configuration. You need a section like: [versioneer] VCS = git style = pep440 versionfile_source = src/myproject/_version.py versionfile_build = myproject/_version.py tag_prefix = parentdir_prefix = myproject- You will also need to edit your setup.py to use the results: import versioneer setup(version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), ...) Please read the docstring in ./versioneer.py for configuration instructions, edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. """ SAMPLE_CONFIG = """ # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the # resulting files. [versioneer] #VCS = git #style = pep440 #versionfile_source = #versionfile_build = #tag_prefix = #parentdir_prefix = """ OLD_SNIPPET = """ from ._version import get_versions __version__ = get_versions()['version'] del get_versions """ INIT_PY_SNIPPET = """ from . import {0} __version__ = {0}.get_versions()['version'] """ def do_setup(): """Do main VCS-independent setup function for installing Versioneer.""" root = get_root() try: cfg = get_config_from_root(root) except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (OSError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: f.write(SAMPLE_CONFIG) print(CONFIG_ERROR, file=sys.stderr) return 1 print(" creating %s" % cfg.versionfile_source) with open(cfg.versionfile_source, "w") as f: LONG = LONG_VERSION_PY[cfg.VCS] f.write(LONG % {"DOLLAR": "$", "STYLE": cfg.style, "TAG_PREFIX": cfg.tag_prefix, "PARENTDIR_PREFIX": cfg.parentdir_prefix, "VERSIONFILE_SOURCE": cfg.versionfile_source, }) ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: with open(ipy, "r") as f: old = f.read() except OSError: old = "" module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] snippet = INIT_PY_SNIPPET.format(module) if OLD_SNIPPET in old: print(" replacing boilerplate in %s" % ipy) with open(ipy, "w") as f: f.write(old.replace(OLD_SNIPPET, snippet)) elif snippet not in old: print(" appending to %s" % ipy) with open(ipy, "a") as f: f.write(snippet) else: print(" %s unmodified" % ipy) else: print(" %s doesn't exist, ok" % ipy) ipy = None # Make VCS-specific changes. For git, this means creating/changing # .gitattributes to mark _version.py for export-subst keyword # substitution. do_vcs_install(cfg.versionfile_source, ipy) return 0 def scan_setup_py(): """Validate the contents of setup.py against Versioneer's expectations.""" found = set() setters = False errors = 0 with open("setup.py", "r") as f: for line in f.readlines(): if "import versioneer" in line: found.add("import") if "versioneer.get_cmdclass()" in line: found.add("cmdclass") if "versioneer.get_version()" in line: found.add("get_version") if "versioneer.VCS" in line: setters = True if "versioneer.versionfile_source" in line: setters = True if len(found) != 3: print("") print("Your setup.py appears to be missing some important items") print("(but I might be wrong). Please make sure it has something") print("roughly like the following:") print("") print(" import versioneer") print(" setup( version=versioneer.get_version(),") print(" cmdclass=versioneer.get_cmdclass(), ...)") print("") errors += 1 if setters: print("You should remove lines like 'versioneer.VCS = ' and") print("'versioneer.versionfile_source = ' . This configuration") print("now lives in setup.cfg, and should be removed from setup.py") print("") errors += 1 return errors def setup_command(): """Set up Versioneer and exit with appropriate error code.""" errors = do_setup() errors += scan_setup_py() sys.exit(1 if errors else 0) if __name__ == "__main__": cmd = sys.argv[1] if cmd == "setup": setup_command()