pax_global_header00006660000000000000000000000064150002553010014501gustar00rootroot0000000000000052 comment=c007969f0f8558debe2f2bcb1776fada4e32bdd5 caio-0.9.22/000077500000000000000000000000001500025530100125065ustar00rootroot00000000000000caio-0.9.22/.editorconfig000066400000000000000000000004351500025530100151650ustar00rootroot00000000000000root = true [*] end_of_line = lf insert_final_newline = true charset = utf-8 trim_trailing_whitespace = true [*.{py,yml}] indent_style = space [*.py] indent_size = 4 [docs/**.py] max_line_length = 80 [*.rst] indent_size = 3 [Makefile] indent_style = tab [*.yml] indent_size = 2 caio-0.9.22/.github/000077500000000000000000000000001500025530100140465ustar00rootroot00000000000000caio-0.9.22/.github/ISSUE_TEMPLATE.md000066400000000000000000000017201500025530100165530ustar00rootroot00000000000000## Long story short ## Expected behavior ## Actual behavior ## Steps to reproduce ## Environment info Kernel version: `replace here to "uname -a" output` File system: `your file system` I have been produced this problem with implementations: * [] `export CAIO_IMPL=linux` - Native linux implementation * [] `export CAIO_IMPL=thread` - Thread implementation * [] `export CAIO_IMPL=python` - Pure Python implementation ## Additional info caio-0.9.22/.github/workflows/000077500000000000000000000000001500025530100161035ustar00rootroot00000000000000caio-0.9.22/.github/workflows/publish.yml000066400000000000000000000060361500025530100203010ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: publish on: release: types: - created jobs: sdist: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: submodules: recursive - name: Setup python3.9 uses: actions/setup-python@v2 with: python-version: "3.9" - name: Install requires run: python -m pip install twine wheel setuptools - name: Build source package run: python setup.py sdist - name: Publishing to pypi run: twine upload --skip-existing --disable-progress-bar dist/*.tar.gz env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} wheel: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: # MacOS - python: '3.8' os: macos-latest - python: '3.9' os: macos-latest - python: '3.10' os: macos-latest - python: '3.11' os: macos-latest - python: '3.12' os: macos-latest - python: '3.13' os: macos-latest # Windows - python: '3.8' os: windows-latest - python: '3.9' os: windows-latest - python: '3.10' os: windows-latest - python: '3.11' os: windows-latest - python: '3.12' os: windows-latest - python: '3.13' os: windows-latest steps: - uses: actions/checkout@v2 with: submodules: recursive - name: Setup python${{ matrix.python }} uses: actions/setup-python@v2 with: python-version: "${{ matrix.python }}" - name: Install requires run: python -m pip install twine wheel setuptools - name: Build wheel for python "${{ matrix.python }}" run: python setup.py bdist_wheel - name: Publishing to pypi run: twine upload --skip-existing --disable-progress-bar dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} linux-wheels: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: submodules: recursive - name: Building manylinux2014 wheels uses: docker://quay.io/pypa/manylinux2014_x86_64 with: args: /bin/bash scripts/make-wheels.sh - name: Setup python${{ matrix.python }} uses: actions/setup-python@v2 with: python-version: "3.10" - name: Install requires run: python -m pip install twine - name: Publishing to pypi run: twine upload --skip-existing --disable-progress-bar dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} caio-0.9.22/.github/workflows/tox.yml000066400000000000000000000051261500025530100174440ustar00rootroot00000000000000name: tox on: push: branches: [ master ] pull_request: branches: [ master ] jobs: lint: runs-on: ubuntu-latest strategy: matrix: toxenv: - lint - mypy steps: - uses: actions/checkout@v2 - name: Setup python${{ matrix.python }} uses: actions/setup-python@v2 with: python-version: "3.10" - name: Install tox run: python -m pip install tox - name: tox ${{ matrix.toxenv }} run: tox env: TOXENV: ${{ matrix.toxenv }} FORCE_COLOR: 1 COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} tests: needs: lint runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - toxenv: py38 python: "3.8" os: ubuntu-latest - toxenv: py39 python: "3.9" os: ubuntu-latest - toxenv: py310 python: "3.10" os: ubuntu-latest - toxenv: py311 python: "3.11" os: ubuntu-latest - toxenv: py312 python: "3.12" os: ubuntu-latest - toxenv: py313 python: "3.13" os: ubuntu-latest - toxenv: py38 python: "3.8" os: windows-latest - toxenv: py39 python: "3.9" os: windows-latest - toxenv: py310 python: "3.10" os: windows-latest - toxenv: py311 python: "3.11" os: windows-latest - toxenv: py313 python: "3.13" os: windows-latest - toxenv: py38 python: "3.8" os: macos-latest - toxenv: py39 python: "3.9" os: macos-latest - toxenv: py310 python: "3.10" os: macos-latest - toxenv: py311 python: "3.11" os: macos-latest - toxenv: py312 python: "3.12" os: macos-latest - toxenv: py313 python: "3.13" os: macos-latest steps: - uses: actions/checkout@v2 - name: Setup python${{ matrix.python }} uses: actions/setup-python@v2 with: python-version: "${{ matrix.python }}" - name: Install tox run: python -m pip install tox - name: tox ${{ matrix.toxenv }} run: tox env: TOXENV: ${{ matrix.toxenv }} FORCE_COLOR: 1 COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} caio-0.9.22/.gitignore000066400000000000000000000034151500025530100145010ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ *.bin caio-0.9.22/LICENSE000066400000000000000000000261351500025530100135220ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. caio-0.9.22/MANIFEST.in000066400000000000000000000001661500025530100142470ustar00rootroot00000000000000include caio/*.pyi include caio/py.typed recursive-include caio/src/threadpool *.* include README.rst include LICENSE caio-0.9.22/Makefile000066400000000000000000000012101500025530100141400ustar00rootroot00000000000000build: sdist mac_wheel linux_wheel sdist: python3 setup.py sdist mac_wheel: python3.8 setup.py bdist_wheel python3.9 setup.py bdist_wheel python3.10 setup.py bdist_wheel python3.11 setup.py bdist_wheel python3.12 setup.py bdist_wheel python3.13 setup.py bdist_wheel linux_wheel: docker run -it --rm \ -v `pwd`:/mnt \ --entrypoint /bin/bash \ --workdir /mnt \ --platform linux/amd64 \ quay.io/pypa/manylinux_2_34_x86_64 \ scripts/make-wheels.sh docker run -it --rm \ -v `pwd`:/mnt \ --entrypoint /bin/bash \ --platform linux/arm64 \ --workdir /mnt \ quay.io/pypa/manylinux_2_34_aarch64 \ scripts/make-wheels.sh caio-0.9.22/README.md000066400000000000000000000036611500025530100137730ustar00rootroot00000000000000Python wrapper for AIO ====================== > **NOTE:** Native Linux aio implementation supports since 4.18 kernel version. Python bindings for Linux AIO API and simple asyncio wrapper. Example ------- ```python import asyncio from caio import AsyncioContext loop = asyncio.get_event_loop() async def main(): # max_requests=128 by default ctx = AsyncioContext(max_requests=128) with open("test.file", "wb+") as fp: fd = fp.fileno() # Execute one write operation await ctx.write(b"Hello world", fd, offset=0) # Execute one read operation print(await ctx.read(32, fd, offset=0)) # Execute one fdsync operation await ctx.fdsync(fd) op1 = ctx.write(b"Hello from ", fd, offset=0) op2 = ctx.write(b"async world", fd, offset=11) await asyncio.gather(op1, op2) print(await ctx.read(32, fd, offset=0)) # Hello from async world loop.run_until_complete(main()) ``` Troubleshooting --------------- The `linux` implementation works normal for modern linux kernel versions and file systems. So you may have problems specific for your environment. It's not a bug and might be resolved some ways: 1. Upgrade the kernel 2. Use compatible file system 3. Use threads based or pure python implementation. The caio since version 0.7.0 contains some ways to do this. 1. In runtime use the environment variable `CAIO_IMPL` with possible values: * `linux` - use native linux kernels aio mechanism * `thread` - use thread based implementation written in C * `python` - use pure python implementation 2. File ``default_implementation`` located near ``__init__.py`` in caio installation path. It's useful for distros package maintainers. This file might contains comments (lines starts with ``#`` symbol) and the first line should be one of ``linux`` ``thread`` or ``python``. Previous versions allows direct import of the target implementation. caio-0.9.22/benchmark/000077500000000000000000000000001500025530100144405ustar00rootroot00000000000000caio-0.9.22/benchmark/benchmark_read_common.py000066400000000000000000000035241500025530100213130ustar00rootroot00000000000000import asyncio import os import time from functools import lru_cache from caio.asyncio_base import AsyncioContextBase chunk_size = 16 * 1024 # 1024 context_max_requests = 512 @lru_cache(1024) def open_file_by_id(file_id): fname = f"data/{file_id}.bin" return open(fname, "rb"), os.stat(fname).st_size async def read_file(ctx: AsyncioContextBase, file_id): offset = 0 fp, file_size = open_file_by_id(file_id) fd = fp.fileno() c = 0 futures = [] while offset < file_size: futures.append(ctx.read(chunk_size, fd, offset)) offset += chunk_size c += 1 await asyncio.gather(*futures) return c async def timer(future): await asyncio.sleep(0) delta = time.monotonic() return await future, time.monotonic() - delta async def main(context_maker): print("files nr min madian max op/s total #ops chunk") for generation in range(1, 129): context = context_maker(context_max_requests) futures = [] for file_id in range(generation): futures.append(read_file(context, file_id)) stat = [] total = -time.monotonic() nops = 0 for ops, delta in await asyncio.gather(*map(timer, futures)): stat.append(delta) nops += ops total += time.monotonic() stat = sorted(stat) ops_sec = nops / total dmin = stat[0] dmedian = stat[int(len(stat) / 2)] dmax = stat[-1] print( "%5d %4d %2.6f %2.6f %2.6f %6d %-3.6f %5d %d" % ( generation, context_max_requests, dmin, dmedian, dmax, ops_sec, total, nops, chunk_size, ), ) context.close() caio-0.9.22/benchmark/benchmark_read_linux_aio.py000066400000000000000000000002541500025530100220070ustar00rootroot00000000000000import asyncio from benchmark_read_common import main from caio.linux_aio_asyncio import AsyncioContext if __name__ == "__main__": asyncio.run(main(AsyncioContext)) caio-0.9.22/benchmark/benchmark_read_python_aio.py000066400000000000000000000002551500025530100221720ustar00rootroot00000000000000import asyncio from benchmark_read_common import main from caio.python_aio_asyncio import AsyncioContext if __name__ == "__main__": asyncio.run(main(AsyncioContext)) caio-0.9.22/benchmark/benchmark_read_thread_aio.py000066400000000000000000000002551500025530100221200ustar00rootroot00000000000000import asyncio from benchmark_read_common import main from caio.thread_aio_asyncio import AsyncioContext if __name__ == "__main__": asyncio.run(main(AsyncioContext)) caio-0.9.22/benchmark/benchmark_write_common.py000066400000000000000000000017111500025530100215260ustar00rootroot00000000000000import asyncio import os import time import typing from tempfile import NamedTemporaryFile from caio.asyncio_base import AsyncioContextBase data = os.urandom(1024) async def main(context_maker: typing.Type[AsyncioContextBase]): async with context_maker() as context: with NamedTemporaryFile(mode="wb+") as fp: async def writer(offset=0): timer = - time.monotonic() fileno = fp.file.fileno() futures = [] for i in range(1, 2 ** 15): futures.append( context.write(data, fileno, offset * i * len(data)), ) await asyncio.gather(*futures) timer += time.monotonic() print("Done", timer) return timer timers = [] for i in range(10): timers.append(await writer(i)) print(sum(timers) / len(timers)) caio-0.9.22/benchmark/benchmark_write_linux_aio.py000066400000000000000000000002551500025530100222270ustar00rootroot00000000000000import asyncio from benchmark_write_common import main from caio.linux_aio_asyncio import AsyncioContext if __name__ == "__main__": asyncio.run(main(AsyncioContext)) caio-0.9.22/benchmark/benchmark_write_python_aio.py000066400000000000000000000002561500025530100224120ustar00rootroot00000000000000import asyncio from benchmark_write_common import main from caio.python_aio_asyncio import AsyncioContext if __name__ == "__main__": asyncio.run(main(AsyncioContext)) caio-0.9.22/benchmark/benchmark_write_thread_aio.py000066400000000000000000000002561500025530100223400ustar00rootroot00000000000000import asyncio from benchmark_write_common import main from caio.thread_aio_asyncio import AsyncioContext if __name__ == "__main__": asyncio.run(main(AsyncioContext)) caio-0.9.22/benchmark/gen_data.py000066400000000000000000000010731500025530100165550ustar00rootroot00000000000000import hashlib import os from multiprocessing.pool import ThreadPool import tqdm POOL = ThreadPool(32) def gen_data(file_id): seed = os.urandom(64) hasher = hashlib.sha512() with open(f"data/{file_id}.bin", "wb+") as fp: for _ in range(100000): hasher.update(seed) seed = hasher.digest() fp.write(seed) def main(): files = 128 iterator = tqdm.tqdm( POOL.imap_unordered(gen_data, range(files)), total=files, ) for _ in iterator: pass if __name__ == "__main__": main() caio-0.9.22/caio/000077500000000000000000000000001500025530100134215ustar00rootroot00000000000000caio-0.9.22/caio/__init__.py000066400000000000000000000050641500025530100155370ustar00rootroot00000000000000import os import warnings from . import python_aio, python_aio_asyncio from .abstract import AbstractContext, AbstractOperation from .version import __author__, __version__ try: from . import linux_aio, linux_aio_asyncio except ImportError: linux_aio = None # type: ignore linux_aio_asyncio = None # type: ignore try: from . import thread_aio, thread_aio_asyncio except ImportError: thread_aio = None # type: ignore thread_aio_asyncio = None # type: ignore variants = tuple(filter(None, [linux_aio, thread_aio, python_aio])) variants_asyncio = tuple( filter( None, [ linux_aio_asyncio, thread_aio_asyncio, python_aio_asyncio, ], ), ) preferred = variants[0] preferred_asyncio = variants_asyncio[0] def __select_implementation(): global preferred global preferred_asyncio implementations = { "linux": (linux_aio, linux_aio_asyncio), "thread": (thread_aio, thread_aio_asyncio), "python": (python_aio, python_aio_asyncio), } implementations = {k: v for k, v in implementations.items() if all(v)} default_implementation = os.path.join( os.path.dirname(os.path.abspath(__file__)), "default_implementation", ) requested = os.getenv("CAIO_IMPL") if not requested and os.path.isfile(default_implementation): with open(default_implementation, "r") as fp: for line in fp: line = line.strip() if line.startswith("#") or not line: continue if line in implementations: requested = line break elif requested and requested not in implementations: warnings.warn( "CAIO_IMPL contains unsupported value %r. Use one of %r" % ( requested, tuple(implementations), ), RuntimeWarning, ) return preferred, preferred_asyncio = implementations.get( requested, (preferred, preferred_asyncio), ) __select_implementation() Context = preferred.Context # type: ignore Operation = preferred.Operation # type: ignore AsyncioContext = preferred_asyncio.AsyncioContext # type: ignore __all__ = ( "Context", "Operation", "AsyncioContext", "AbstractContext", "AbstractOperation", "python_aio", "python_aio_asyncio", "linux_aio", "linux_aio_asyncio", "thread_aio", "thread_aio_asyncio", "__version__", "__author__", "variants", "variants_asyncio", ) caio-0.9.22/caio/abstract.py000066400000000000000000000042121500025530100155750ustar00rootroot00000000000000import abc from typing import Any, Callable, Optional, Union class AbstractContext(abc.ABC): @property def max_requests(self) -> int: raise NotImplementedError def submit(self, *aio_operations) -> int: raise NotImplementedError(aio_operations) def cancel(self, *aio_operations) -> int: raise NotImplementedError(aio_operations) class AbstractOperation(abc.ABC): @classmethod @abc.abstractmethod def read( cls, nbytes: int, fd: int, offset: int, priority=0, ) -> "AbstractOperation": """ Creates a new instance of AIOOperation on read mode. """ raise NotImplementedError @classmethod @abc.abstractmethod def write( cls, payload_bytes: bytes, fd: int, offset: int, priority=0, ) -> "AbstractOperation": """ Creates a new instance of AIOOperation on write mode. """ raise NotImplementedError @classmethod @abc.abstractmethod def fsync(cls, fd: int, priority=0) -> "AbstractOperation": """ Creates a new instance of AIOOperation on fsync mode. """ raise NotImplementedError @classmethod @abc.abstractmethod def fdsync(cls, fd: int, priority=0) -> "AbstractOperation": """ Creates a new instance of AIOOperation on fdsync mode. """ raise NotImplementedError @abc.abstractmethod def get_value(self) -> Union[bytes, int]: """ Method returns a bytes value of AIOOperation's result or None. """ raise NotImplementedError @property @abc.abstractmethod def fileno(self) -> int: raise NotImplementedError @property @abc.abstractmethod def offset(self) -> int: raise NotImplementedError @property @abc.abstractmethod def payload(self) -> Optional[Union[bytes, memoryview]]: raise NotImplementedError @property @abc.abstractmethod def nbytes(self) -> int: raise NotImplementedError @abc.abstractmethod def set_callback(self, callback: Callable[[int], Any]) -> bool: raise NotImplementedError caio-0.9.22/caio/asyncio_base.py000066400000000000000000000053231500025530100164350ustar00rootroot00000000000000import abc import asyncio import typing from functools import partial from . import abstract ContextType = typing.Type[abstract.AbstractContext] OperationType = typing.Type[abstract.AbstractOperation] class AsyncioContextBase(abc.ABC): MAX_REQUESTS_DEFAULT = 512 CONTEXT_CLASS = None # type: ContextType OPERATION_CLASS = None # type: OperationType def __init__(self, max_requests=None, loop=None, **kwargs): max_requests = max_requests or self.MAX_REQUESTS_DEFAULT self.loop = loop or asyncio.get_event_loop() self.semaphore = asyncio.BoundedSemaphore(max_requests) self.context = self._create_context(max_requests, **kwargs) def _create_context(self, max_requests, **kwargs): return self.CONTEXT_CLASS(max_requests=max_requests, **kwargs) def _destroy_context(self): return async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): self.close() def close(self): self._destroy_context() async def submit(self, op): if not isinstance(op, self.OPERATION_CLASS): raise ValueError("Operation object expected") future = self.loop.create_future() op.set_callback(partial(self._on_done, future)) async with self.semaphore: if self.context.submit(op) != 1: raise IOError("Operation was not submitted") try: await future except asyncio.CancelledError: try: self.context.cancel(op) except ValueError: pass raise return op.get_value() def _on_done(self, future, result): """ In general case impossible predict current thread and the thread of event loop. So have to use `call_soon_threadsave` the result setter. """ if future.done(): return self.loop.call_soon_threadsafe( lambda: future.done() or future.set_result(True), ) def read( self, nbytes: int, fd: int, offset: int, priority: int = 0, ) -> typing.Awaitable[bytes]: return self.submit( self.OPERATION_CLASS.read(nbytes, fd, offset, priority), ) def write( self, payload: bytes, fd: int, offset: int, priority: int = 0, ) -> typing.Awaitable[int]: return self.submit( self.OPERATION_CLASS.write(payload, fd, offset, priority), ) def fsync(self, fd: int) -> typing.Awaitable: return self.submit(self.OPERATION_CLASS.fsync(fd)) def fdsync(self, fd: int) -> typing.Awaitable: return self.submit(self.OPERATION_CLASS.fdsync(fd)) caio-0.9.22/caio/linux_aio.c000066400000000000000000000602071500025530100155610ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #define PY_SSIZE_T_CLEAN #include #include #include static const unsigned CTX_MAX_REQUESTS_DEFAULT = 32; static const unsigned EV_MAX_REQUESTS_DEFAULT = 512; static int kernel_support = -1; inline static int io_setup(unsigned nr, aio_context_t *ctxp) { return syscall(__NR_io_setup, nr, ctxp); } inline static int io_destroy(aio_context_t ctx) { return syscall(__NR_io_destroy, ctx); } inline static int io_getevents( aio_context_t ctx, long min_nr, long max_nr, struct io_event *events, struct timespec *timeout ) { return syscall(__NR_io_getevents, ctx, min_nr, max_nr, events, timeout); } inline static int io_submit(aio_context_t ctx, long nr, struct iocb **iocbpp) { return syscall(__NR_io_submit, ctx, nr, iocbpp); } inline static long io_cancel(aio_context_t ctx, struct iocb *aiocb, struct io_event *res) { return syscall(__NR_io_cancel, ctx, aiocb, res); } inline int io_cancel_error(int result) { if (result == 0) return result; switch (errno) { case EAGAIN: PyErr_SetString( PyExc_SystemError, "Specified operation was not canceled [EAGAIN]" ); break; case EFAULT: PyErr_SetString( PyExc_RuntimeError, "One of the data structures points to invalid data [EFAULT]" ); break; case EINVAL: PyErr_SetString( PyExc_ValueError, "The AIO context specified by ctx_id is invalid [EINVAL]" ); break; case ENOSYS: PyErr_SetString( PyExc_NotImplementedError, "io_cancel() is not implemented on this architecture [ENOSYS]" ); break; default: PyErr_SetFromErrno(PyExc_SystemError); break; } return result; } inline int io_submit_error(int result) { if (result >= 0) return result; switch (errno) { case EAGAIN: PyErr_SetString( PyExc_OverflowError, "Insufficient resources are available to queue any iocbs [EAGAIN]" ); break; case EBADF: PyErr_SetString( PyExc_ValueError, "The file descriptor specified in the first iocb is invalid [EBADF]" ); break; case EFAULT: PyErr_SetString( PyExc_ValueError, "One of the data structures points to invalid data [EFAULT]" ); break; case EINVAL: PyErr_SetString( PyExc_ValueError, "The AIO context specified by ctx_id is invalid. nr is less " "than 0. The iocb at *iocbpp[0] is not properly initialized, " "the operation specified is invalid for the file descriptor in " "the iocb, or the value in the aio_reqprio field is invalid. " "[EINVAL]" ); break; default: PyErr_SetFromErrno(PyExc_SystemError); break; } return result; } typedef struct { PyObject_HEAD aio_context_t ctx; int32_t fileno; uint32_t max_requests; } AIOContext; typedef struct { PyObject_HEAD AIOContext* context; PyObject* py_buffer; PyObject* callback; char* buffer; int error; struct iocb iocb; } AIOOperation; static PyTypeObject* AIOOperationTypeP = NULL; static PyTypeObject* AIOContextTypeP = NULL; static void AIOContext_dealloc(AIOContext *self) { if (self->ctx != 0) { aio_context_t ctx = self->ctx; self->ctx = 0; io_destroy(ctx); } if (self->fileno >= 0) { close(self->fileno); self->fileno = -1; } Py_TYPE(self)->tp_free((PyObject *) self); } /* AIOContext.__new__ classmethod definition */ static PyObject * AIOContext_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { AIOContext *self; self = (AIOContext *) type->tp_alloc(type, 0); return (PyObject *) self; } static int AIOContext_init(AIOContext *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"max_requests", NULL}; self->ctx = 0; self->max_requests = 0; self->fileno = eventfd(0, 0); if (self->fileno < 0) { PyErr_SetFromErrno(PyExc_SystemError); return -1; } if (!PyArg_ParseTupleAndKeywords(args, kwds, "|I", kwlist, &self->max_requests)) { return -1; } if (self->max_requests <= 0) { self->max_requests = CTX_MAX_REQUESTS_DEFAULT; } if (io_setup(self->max_requests, &self->ctx) < 0) { PyErr_SetFromErrno(PyExc_SystemError); return -1; } return 0; } static PyObject* AIOContext_repr(AIOContext *self) { if (self->ctx == 0) { PyErr_SetNone(PyExc_RuntimeError); return NULL; } return PyUnicode_FromFormat( "<%s as %p: max_requests=%i, ctx=%lli>", Py_TYPE(self)->tp_name, self, self->max_requests, self->ctx ); } PyDoc_STRVAR(AIOContext_submit_docstring, "Accepts multiple Operations. Returns \n\n" " Operation.submit(aio_op1, aio_op2, aio_opN, ...) -> int" ); static PyObject* AIOContext_submit(AIOContext *self, PyObject *args) { if (self == 0) { PyErr_SetString(PyExc_RuntimeError, "self is NULL"); return NULL; } if (self->ctx == 0) { PyErr_SetString(PyExc_RuntimeError, "self->ctx is NULL"); return NULL; } if (!PyTuple_Check(args)) { PyErr_SetNone(PyExc_ValueError); return NULL; } int result = 0; uint32_t nr = PyTuple_Size(args); PyObject* obj; AIOOperation* op; struct iocb** iocbpp = PyMem_Calloc(nr, sizeof(struct iocb*)); uint32_t i; for (i=0; i < nr; i++) { obj = PyTuple_GetItem(args, i); if (PyObject_TypeCheck(obj, AIOOperationTypeP) == 0) { PyErr_Format( PyExc_TypeError, "Wrong type for argument %d -> %r", i, obj ); PyMem_Free(iocbpp); return NULL; } op = (AIOOperation*) obj; op->context = self; Py_INCREF(self); Py_INCREF(op); op->iocb.aio_flags |= IOCB_FLAG_RESFD; op->iocb.aio_resfd = self->fileno; iocbpp[i] = &op->iocb; } result = io_submit(self->ctx, nr, iocbpp); if (io_submit_error(result) < 0) { PyMem_Free(iocbpp); return NULL; } PyMem_Free(iocbpp); return (PyObject*) PyLong_FromSsize_t(result); } PyDoc_STRVAR(AIOContext_cancel_docstring, "Cancels multiple Operations. Returns \n\n" " Operation.cancel(aio_op1, aio_op2, aio_opN, ...) -> int" ); static PyObject* AIOContext_cancel(AIOContext *self, PyObject *args, PyObject *kwds) { if (self == 0) { PyErr_SetString(PyExc_RuntimeError, "self is NULL"); return NULL; } if (self->ctx == 0) { PyErr_SetString(PyExc_RuntimeError, "self->ctx is NULL"); return NULL; } static char *kwlist[] = {"operation", NULL}; AIOOperation* op = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &op)) return NULL; if (PyObject_TypeCheck(op, AIOOperationTypeP) == 0) { PyErr_Format(PyExc_TypeError, "Operation required not %r", op); return NULL; } struct io_event ev; if (io_cancel_error(io_cancel(self->ctx, &op->iocb, &ev))) { return NULL; } if (ev.res >= 0) { op->iocb.aio_nbytes = ev.res; } else { op->error = -ev.res; } if (op->callback != NULL) { if (PyObject_CallFunction(op->callback, "K", ev.res) == NULL) { return NULL; } } return (PyObject*) PyLong_FromSsize_t(ev.res); } PyDoc_STRVAR(AIOContext_process_events_docstring, "Gather events for Context. \n\n" " Operation.process_events(max_events, min_events) -> Tuple[Tuple[]]" ); static PyObject* AIOContext_process_events( AIOContext *self, PyObject *args, PyObject *kwds ) { if (self->ctx == 0) { PyErr_SetNone(PyExc_RuntimeError); return NULL; } uint32_t min_requests = 0; uint32_t max_requests = 0; int32_t tv_sec = 0; struct timespec timeout = {0, 0}; static char *kwlist[] = {"max_requests", "min_requests", "timeout", NULL}; if (!PyArg_ParseTupleAndKeywords( args, kwds, "|IIi", kwlist, &max_requests, &min_requests, &tv_sec )) { return NULL; } timeout.tv_sec = tv_sec; if (max_requests == 0) { max_requests = EV_MAX_REQUESTS_DEFAULT; } if (min_requests > max_requests) { PyErr_Format( PyExc_ValueError, "min_requests \"%d\" must be lower then max_requests \"%d\"", min_requests, max_requests ); return NULL; } struct io_event events[max_requests]; int result = io_getevents( self->ctx, min_requests, max_requests, events, &timeout ); if (result < 0) { PyErr_SetFromErrno(PyExc_SystemError); return NULL; } AIOOperation* op; struct io_event* ev; int32_t i; for (i = 0; i < result; i++) { ev = &events[i]; op = (AIOOperation*)(uintptr_t) ev->data; if (ev->res >= 0) { op->iocb.aio_nbytes = ev->res; } else { op->error = -ev->res; } if (op->callback == NULL) { continue; } if (PyObject_CallFunction(op->callback, "K", ev->res) == NULL) { return NULL; } Py_XDECREF(op); } return (PyObject*) PyLong_FromSsize_t(i); } PyDoc_STRVAR(AIOContext_poll_docstring, "Read value from context file descriptor.\n\n" " Context().poll() -> int" ); static PyObject* AIOContext_poll( AIOContext *self, PyObject *args ) { if (self->ctx == 0) { PyErr_SetNone(PyExc_RuntimeError); return NULL; } if (self->fileno < 0) { PyErr_SetNone(PyExc_RuntimeError); return NULL; } uint64_t result = 0; int size = read(self->fileno, &result, sizeof(uint64_t)); if (size != sizeof(uint64_t)) { PyErr_SetNone(PyExc_BlockingIOError); return NULL; } return (PyObject*) PyLong_FromUnsignedLongLong(result); } /* AIOContext properties */ static PyMemberDef AIOContext_members[] = { { "fileno", T_INT, offsetof(AIOContext, fileno), READONLY, "fileno" }, { "max_requests", T_USHORT, offsetof(AIOContext, max_requests), READONLY, "max requests" }, {NULL} /* Sentinel */ }; static PyMethodDef AIOContext_methods[] = { { "submit", (PyCFunction) AIOContext_submit, METH_VARARGS, AIOContext_submit_docstring }, { "cancel", (PyCFunction) AIOContext_cancel, METH_VARARGS | METH_KEYWORDS, AIOContext_cancel_docstring }, { "process_events", (PyCFunction) AIOContext_process_events, METH_VARARGS | METH_KEYWORDS, AIOContext_process_events_docstring }, { "poll", (PyCFunction) AIOContext_poll, METH_NOARGS, AIOContext_poll_docstring }, {NULL} /* Sentinel */ }; static PyTypeObject AIOContextType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "Context", .tp_doc = "linux aio context representation", .tp_basicsize = sizeof(AIOContext), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = AIOContext_new, .tp_init = (initproc) AIOContext_init, .tp_dealloc = (destructor) AIOContext_dealloc, .tp_members = AIOContext_members, .tp_methods = AIOContext_methods, .tp_repr = (reprfunc) AIOContext_repr }; static void AIOOperation_dealloc(AIOOperation *self) { Py_CLEAR(self->context); Py_CLEAR(self->callback); if (self->iocb.aio_lio_opcode == IOCB_CMD_PREAD && self->buffer != NULL) { PyMem_Free(self->buffer); self->buffer = NULL; } Py_CLEAR(self->py_buffer); Py_TYPE(self)->tp_free((PyObject *) self); } static PyObject* AIOOperation_repr(AIOOperation *self) { char* mode; switch (self->iocb.aio_lio_opcode) { case IOCB_CMD_PREAD: mode = "read"; break; case IOCB_CMD_PWRITE: mode = "write"; break; case IOCB_CMD_FSYNC: mode = "fsync"; break; case IOCB_CMD_FDSYNC: mode = "fdsync"; break; default: mode = "noop"; break; } return PyUnicode_FromFormat( "<%s at %p: mode=\"%s\", fd=%i, offset=%i, buffer=%p>", Py_TYPE(self)->tp_name, self, mode, self->iocb.aio_fildes, self->iocb.aio_offset, self->iocb.aio_buf ); } /* AIOOperation.read classmethod definition */ PyDoc_STRVAR(AIOOperation_read_docstring, "Creates a new instance of Operation on read mode.\n\n" " Operation.read(\n" " nbytes: int,\n" " aio_context: Context,\n" " fd: int, \n" " offset: int,\n" " priority=0\n" " )" ); static PyObject* AIOOperation_read( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"nbytes", "fd", "offset", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } memset(&self->iocb, 0, sizeof(struct iocb)); self->iocb.aio_data = (uint64_t)(uintptr_t) self; self->context = NULL; self->buffer = NULL; self->py_buffer = NULL; uint64_t nbytes = 0; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "KI|Lh", kwlist, &nbytes, &(self->iocb.aio_fildes), &(self->iocb.aio_offset), &(self->iocb.aio_reqprio) ); if (!argIsOk) return NULL; self->buffer = PyMem_Calloc(nbytes, sizeof(char)); self->iocb.aio_buf = (uint64_t) self->buffer; self->iocb.aio_nbytes = nbytes; self->py_buffer = PyMemoryView_FromMemory(self->buffer, nbytes, PyBUF_READ); self->iocb.aio_lio_opcode = IOCB_CMD_PREAD; return (PyObject*) self; } /* AIOOperation.write classmethod definition */ PyDoc_STRVAR(AIOOperation_write_docstring, "Creates a new instance of Operation on write mode.\n\n" " Operation.write(\n" " payload_bytes: bytes,\n" " fd: int, \n" " offset: int,\n" " priority=0\n" " )" ); static PyObject* AIOOperation_write( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"payload_bytes", "fd", "offset", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } memset(&self->iocb, 0, sizeof(struct iocb)); self->iocb.aio_data = (uint64_t)(uintptr_t) self; self->context = NULL; self->buffer = NULL; self->py_buffer = NULL; Py_ssize_t nbytes = 0; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "OI|Lh", kwlist, &(self->py_buffer), &(self->iocb.aio_fildes), &(self->iocb.aio_offset), &(self->iocb.aio_reqprio) ); if (!argIsOk) return NULL; if (!PyBytes_Check(self->py_buffer)) { Py_XDECREF(self); PyErr_SetString( PyExc_ValueError, "payload_bytes argument must be bytes" ); return NULL; } self->iocb.aio_lio_opcode = IOCB_CMD_PWRITE; if (PyBytes_AsStringAndSize( self->py_buffer, &self->buffer, &nbytes )) { Py_XDECREF(self); PyErr_SetString( PyExc_RuntimeError, "Can not convert bytes to c string" ); return NULL; } Py_INCREF(self->py_buffer); self->iocb.aio_nbytes = nbytes; self->iocb.aio_buf = (uint64_t) self->buffer; return (PyObject*) self; } /* AIOOperation.fsync classmethod definition */ PyDoc_STRVAR(AIOOperation_fsync_docstring, "Creates a new instance of Operation on fsync mode.\n\n" " Operation.fsync(\n" " aio_context: AIOContext,\n" " fd: int, \n" " priority=0\n" " )" ); static PyObject* AIOOperation_fsync( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"fd", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } memset(&self->iocb, 0, sizeof(struct iocb)); self->iocb.aio_data = (uint64_t)(uintptr_t) self; self->context = NULL; self->buffer = NULL; self->py_buffer = NULL; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "I|h", kwlist, &(self->iocb.aio_fildes), &(self->iocb.aio_reqprio) ); if (!argIsOk) return NULL; self->iocb.aio_lio_opcode = IOCB_CMD_FSYNC; return (PyObject*) self; } /* AIOOperation.fdsync classmethod definition */ PyDoc_STRVAR(AIOOperation_fdsync_docstring, "Creates a new instance of Operation on fdsync mode.\n\n" " Operation.fdsync(\n" " aio_context: AIOContext,\n" " fd: int, \n" " priority=0\n" " )" ); static PyObject* AIOOperation_fdsync( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"fd", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } memset(&self->iocb, 0, sizeof(struct iocb)); self->iocb.aio_data = (uint64_t)(uintptr_t) self; self->buffer = NULL; self->py_buffer = NULL; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "I|h", kwlist, &(self->iocb.aio_fildes), &(self->iocb.aio_reqprio) ); if (!argIsOk) return NULL; self->iocb.aio_lio_opcode = IOCB_CMD_FDSYNC; return (PyObject*) self; } /* AIOOperation.get_value method definition */ PyDoc_STRVAR(AIOOperation_get_value_docstring, "Method returns a bytes value of Operation's result or None.\n\n" " Operation.get_value() -> Optional[bytes]" ); static PyObject* AIOOperation_get_value( AIOOperation *self, PyObject *args, PyObject *kwds ) { if (self->error != 0) { PyErr_SetString( PyExc_SystemError, strerror(self->error) ); return NULL; } switch (self->iocb.aio_lio_opcode) { case IOCB_CMD_PREAD: return PyBytes_FromStringAndSize( self->buffer, self->iocb.aio_nbytes ); case IOCB_CMD_PWRITE: return PyLong_FromSsize_t(self->iocb.aio_nbytes); } return Py_None; } /* AIOOperation.set_callback method definition */ PyDoc_STRVAR(AIOOperation_set_callback_docstring, "Set callback which will be called after Operation will be finished.\n\n" " Operation.get_value() -> Optional[bytes]" ); static PyObject* AIOOperation_set_callback( AIOOperation *self, PyObject *args, PyObject *kwds ) { static char *kwlist[] = {"callback", NULL}; PyObject* callback; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "O", kwlist, &callback ); if (!argIsOk) return NULL; if (!PyCallable_Check(callback)) { PyErr_Format( PyExc_ValueError, "object %r is not callable", callback ); return NULL; } Py_INCREF(callback); self->callback = callback; Py_RETURN_TRUE; } /* AIOOperation properties */ static PyMemberDef AIOOperation_members[] = { { "context", T_OBJECT, offsetof(AIOOperation, context), READONLY, "context object" }, { "fileno", T_UINT, offsetof(AIOOperation, iocb.aio_fildes), READONLY, "file descriptor" }, { "priority", T_USHORT, offsetof(AIOOperation, iocb.aio_reqprio), READONLY, "request priority" }, { "offset", T_ULONGLONG, offsetof(AIOOperation, iocb.aio_offset), READONLY, "offset" }, { "payload", T_OBJECT, offsetof(AIOOperation, py_buffer), READONLY, "payload" }, { "nbytes", T_ULONGLONG, offsetof(AIOOperation, iocb.aio_nbytes), READONLY, "nbytes" }, {NULL} /* Sentinel */ }; /* AIOOperation methods */ static PyMethodDef AIOOperation_methods[] = { { "read", (PyCFunction) AIOOperation_read, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_read_docstring }, { "write", (PyCFunction) AIOOperation_write, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_write_docstring }, { "fsync", (PyCFunction) AIOOperation_fsync, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_fsync_docstring }, { "fdsync", (PyCFunction) AIOOperation_fdsync, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_fdsync_docstring }, { "get_value", (PyCFunction) AIOOperation_get_value, METH_NOARGS, AIOOperation_get_value_docstring }, { "set_callback", (PyCFunction) AIOOperation_set_callback, METH_VARARGS | METH_KEYWORDS, AIOOperation_set_callback_docstring }, {NULL} /* Sentinel */ }; /* AIOOperation class */ static PyTypeObject AIOOperationType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "aio.AIOOperation", .tp_doc = "linux aio operation representation", .tp_basicsize = sizeof(AIOOperation), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_dealloc = (destructor) AIOOperation_dealloc, .tp_members = AIOOperation_members, .tp_methods = AIOOperation_methods, .tp_repr = (reprfunc) AIOOperation_repr }; static PyModuleDef linux_aio_module = { PyModuleDef_HEAD_INIT, .m_name = "linux_aio", .m_doc = "Linux AIO c API bindings.", .m_size = -1, }; PyMODINIT_FUNC PyInit_linux_aio(void) { Py_Initialize(); struct utsname uname_data; if (uname(&uname_data)) { PyErr_SetString(PyExc_ImportError, "Can not detect linux kernel version"); return NULL; } int release[2] = {0}; sscanf(uname_data.release, "%d.%d", &release[0], &release[1]); kernel_support = (release[0] > 4) || (release[0] == 4 && release[1] >= 18); if (!kernel_support) { PyErr_Format( PyExc_ImportError, "Linux kernel supported since 4.18 but current kernel is %s.", uname_data.release ); return NULL; } aio_context_t temp_ctx = 0; if (io_setup(1, &temp_ctx) < 0) { PyErr_Format( PyExc_ImportError, "Error on io_setup with code %d", errno ); return NULL; } if (io_destroy(temp_ctx)) { PyErr_Format( PyExc_ImportError, "Error on io_destroy with code %d", errno ); return NULL; } AIOContextTypeP = &AIOContextType; AIOOperationTypeP = &AIOOperationType; PyObject *m; m = PyModule_Create(&linux_aio_module); if (m == NULL) return NULL; if (PyType_Ready(AIOContextTypeP) < 0) return NULL; Py_INCREF(AIOContextTypeP); if (PyModule_AddObject(m, "Context", (PyObject *) AIOContextTypeP) < 0) { Py_XDECREF(AIOContextTypeP); Py_XDECREF(m); return NULL; } if (PyType_Ready(AIOOperationTypeP) < 0) return NULL; Py_INCREF(AIOOperationTypeP); if (PyModule_AddObject(m, "Operation", (PyObject *) AIOOperationTypeP) < 0) { Py_XDECREF(AIOOperationTypeP); Py_XDECREF(m); return NULL; } return m; } caio-0.9.22/caio/linux_aio.pyi000066400000000000000000000021061500025530100161320ustar00rootroot00000000000000from typing import Union, Optional, Callable, Any from .abstract import AbstractContext, AbstractOperation # noinspection PyPropertyDefinition class Context(AbstractContext): def __init__(self, max_requests: int = 32): ... # noinspection PyPropertyDefinition class Operation(AbstractOperation): @classmethod def read( cls, nbytes: int, fd: int, offset: int, priority=0 ) -> "AbstractOperation": ... @classmethod def write( cls, payload_bytes: bytes, fd: int, offset: int, priority=0, ) -> "AbstractOperation": ... @classmethod def fsync(cls, fd: int, priority=0) -> "AbstractOperation": ... @classmethod def fdsync(cls, fd: int, priority=0) -> "AbstractOperation": ... def get_value(self) -> Union[bytes, int]: ... @property def fileno(self) -> int: ... @property def offset(self) -> int: ... @property def payload(self) -> Optional[Union[bytes, memoryview]]: ... @property def nbytes(self) -> int: ... def set_callback(self, callback: Callable[[int], Any]) -> bool: ... caio-0.9.22/caio/linux_aio_asyncio.py000066400000000000000000000014411500025530100175070ustar00rootroot00000000000000from .asyncio_base import AsyncioContextBase from .linux_aio import Context, Operation class AsyncioContext(AsyncioContextBase): OPERATION_CLASS = Operation CONTEXT_CLASS = Context def _create_context(self, max_requests): context = super()._create_context(max_requests) self.loop.add_reader(context.fileno, self._on_read_event) return context def _on_done(self, future, result): """ Allow to set result directly. Cause process_events running in the same thread """ if future.done(): return future.set_result(True) def _destroy_context(self): self.loop.remove_reader(self.context.fileno) def _on_read_event(self): self.context.poll() self.context.process_events() caio-0.9.22/caio/py.typed000066400000000000000000000000001500025530100151060ustar00rootroot00000000000000caio-0.9.22/caio/python_aio.py000066400000000000000000000160701500025530100161500ustar00rootroot00000000000000import os from collections import defaultdict from enum import IntEnum, unique from io import BytesIO from multiprocessing.pool import ThreadPool from threading import Lock, RLock from types import MappingProxyType from typing import Any, Callable, Optional, Union from .abstract import AbstractContext, AbstractOperation fdsync = getattr(os, "fdatasync", os.fsync) NATIVE_PREAD_PWRITE = hasattr(os, "pread") and hasattr(os, "pwrite") @unique class OpCode(IntEnum): READ = 0 WRITE = 1 FSYNC = 2 FDSYNC = 3 NOOP = -1 class Context(AbstractContext): """ python aio context implementation """ MAX_POOL_SIZE = 128 def __init__(self, max_requests: int = 32, pool_size: int = 8): assert pool_size < self.MAX_POOL_SIZE self.__max_requests = max_requests self.pool = ThreadPool(pool_size) self._in_progress = 0 self._closed = False self._closed_lock = Lock() if not NATIVE_PREAD_PWRITE: self._locks_cleaner = RLock() # type: ignore self._locks = defaultdict(RLock) # type: ignore @property def max_requests(self) -> int: return self.__max_requests def _execute(self, operation: "Operation"): handler = self._OP_MAP[operation.opcode] def on_error(exc): self._in_progress -= 1 operation.exception = exc operation.written = 0 operation.callback(None) def on_success(result): self._in_progress -= 1 operation.written = result operation.callback(result) if self._in_progress > self.__max_requests: raise RuntimeError( "Maximum simultaneous requests have been reached", ) self._in_progress += 1 self.pool.apply_async( handler, args=(self, operation), callback=on_success, error_callback=on_error, ) if NATIVE_PREAD_PWRITE: def __pread(self, fd, size, offset): return os.pread(fd, size, offset) def __pwrite(self, fd, bytes, offset): return os.pwrite(fd, bytes, offset) else: def __pread(self, fd, size, offset): with self._locks[fd]: os.lseek(fd, 0, os.SEEK_SET) os.lseek(fd, offset, os.SEEK_SET) return os.read(fd, size) def __pwrite(self, fd, bytes, offset): with self._locks[fd]: os.lseek(fd, 0, os.SEEK_SET) os.lseek(fd, offset, os.SEEK_SET) return os.write(fd, bytes) def _handle_read(self, operation: "Operation"): return operation.buffer.write( self.__pread( operation.fileno, operation.nbytes, operation.offset, ), ) def _handle_write(self, operation: "Operation"): return self.__pwrite( operation.fileno, operation.buffer.getvalue(), operation.offset, ) def _handle_fsync(self, operation: "Operation"): return os.fsync(operation.fileno) def _handle_fdsync(self, operation: "Operation"): return fdsync(operation.fileno) def _handle_noop(self, operation: "Operation"): return def submit(self, *aio_operations) -> int: operations = [] for operation in aio_operations: if not isinstance(operation, Operation): raise ValueError("Invalid Operation %r", operation) operations.append(operation) count = 0 for operation in operations: self._execute(operation) count += 1 return count def cancel(self, *aio_operations) -> int: """ Cancels multiple Operations. Returns Operation.cancel(aio_op1, aio_op2, aio_opN, ...) -> int (Always returns zero, this method exists for compatibility reasons) """ return 0 def close(self): if self._closed: return with self._closed_lock: self.pool.close() self._closed = True def __del__(self): if self.pool.close(): self.close() _OP_MAP = MappingProxyType({ OpCode.READ: _handle_read, OpCode.WRITE: _handle_write, OpCode.FSYNC: _handle_fsync, OpCode.FDSYNC: _handle_fdsync, OpCode.NOOP: _handle_noop, }) # noinspection PyPropertyDefinition class Operation(AbstractOperation): """ python aio operation implementation """ def __init__( self, fd: int, nbytes: Optional[int], offset: Optional[int], opcode: OpCode, payload: Optional[bytes] = None, priority: Optional[int] = None, ): self.callback = None # type: Optional[Callable[[int], Any]] self.buffer = BytesIO() if opcode == OpCode.WRITE and payload: self.buffer = BytesIO(payload) self.opcode = opcode self.__fileno = fd self.__offset = offset or 0 self.__opcode = opcode self.__nbytes = nbytes or 0 self.__priority = priority or 0 self.exception = None self.written = 0 @classmethod def read( cls, nbytes: int, fd: int, offset: int, priority=0, ) -> "Operation": """ Creates a new instance of Operation on read mode. """ return cls(fd, nbytes, offset, opcode=OpCode.READ, priority=priority) @classmethod def write( cls, payload_bytes: bytes, fd: int, offset: int, priority=0, ) -> "Operation": """ Creates a new instance of AIOOperation on write mode. """ return cls( fd, len(payload_bytes), offset, payload=payload_bytes, opcode=OpCode.WRITE, priority=priority, ) @classmethod def fsync(cls, fd: int, priority=0) -> "Operation": """ Creates a new instance of AIOOperation on fsync mode. """ return cls(fd, None, None, opcode=OpCode.FSYNC, priority=priority) @classmethod def fdsync(cls, fd: int, priority=0) -> "Operation": """ Creates a new instance of AIOOperation on fdsync mode. """ return cls(fd, None, None, opcode=OpCode.FDSYNC, priority=priority) def get_value(self) -> Union[bytes, int]: """ Method returns a bytes value of AIOOperation's result or None. """ if self.exception: raise self.exception if self.opcode == OpCode.WRITE: return self.written if self.buffer is None: return return self.buffer.getvalue() @property def fileno(self) -> int: return self.__fileno @property def offset(self) -> int: return self.__offset @property def payload(self) -> Optional[memoryview]: return self.buffer.getbuffer() @property def nbytes(self) -> int: return self.__nbytes def set_callback(self, callback: Callable[[int], Any]) -> bool: self.callback = callback return True caio-0.9.22/caio/python_aio_asyncio.py000066400000000000000000000003761500025530100176770ustar00rootroot00000000000000from .asyncio_base import AsyncioContextBase from .python_aio import Context, Operation class AsyncioContext(AsyncioContextBase): OPERATION_CLASS = Operation CONTEXT_CLASS = Context def _destroy_context(self): self.context.close() caio-0.9.22/caio/src/000077500000000000000000000000001500025530100142105ustar00rootroot00000000000000caio-0.9.22/caio/src/threadpool/000077500000000000000000000000001500025530100163515ustar00rootroot00000000000000caio-0.9.22/caio/src/threadpool/LICENSE000066400000000000000000000024361500025530100173630ustar00rootroot00000000000000Copyright (c) 2016, Mathias Brossard. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. caio-0.9.22/caio/src/threadpool/LINK000066400000000000000000000000501500025530100170240ustar00rootroot00000000000000https://github.com/mbrossard/threadpool caio-0.9.22/caio/src/threadpool/README.md000066400000000000000000000016001500025530100176250ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/mbrossard/threadpool.svg?branch=master)](https://travis-ci.org/mbrossard/threadpool) A simple C thread pool implementation ===================================== Currently, the implementation: * Works with pthreads only, but API is intentionally opaque to allow other implementations (Windows for instance). * Starts all threads on creation of the thread pool. * Reserves one task for signaling the queue is full. * Stops and joins all worker threads on destroy. Possible enhancements ===================== The API contains addtional unused 'flags' parameters that would allow some additional options: * Lazy creation of threads (easy) * Reduce number of threads automatically (hard) * Unlimited queue size (medium) * Kill worker threads on destroy (hard, dangerous) * Support Windows API (medium) * Reduce locking contention (medium/hard) caio-0.9.22/caio/src/threadpool/threadpool.c000066400000000000000000000204421500025530100206600ustar00rootroot00000000000000/* * Copyright (c) 2016, Mathias Brossard . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** * @file threadpool.c * @brief Threadpool implementation file */ #include #include #include #include "threadpool.h" typedef enum { immediate_shutdown = 1, graceful_shutdown = 2 } threadpool_shutdown_t; /** * @struct threadpool_task * @brief the work struct * * @var function Pointer to the function that will perform the task. * @var argument Argument to be passed to the function. */ typedef struct { void (*function)(void *); void *argument; } threadpool_task_t; /** * @struct threadpool * @brief The threadpool struct * * @var notify Condition variable to notify worker threads. * @var threads Array containing worker threads ID. * @var thread_count Number of threads * @var queue Array containing the task queue. * @var queue_size Size of the task queue. * @var head Index of the first element. * @var tail Index of the next element. * @var count Number of pending tasks * @var shutdown Flag indicating if the pool is shutting down * @var started Number of started threads */ struct threadpool_t { pthread_mutex_t lock; pthread_cond_t notify; pthread_t *threads; threadpool_task_t *queue; int thread_count; int queue_size; int head; int tail; int count; int shutdown; int started; }; /** * @function void *threadpool_thread(void *threadpool) * @brief the worker thread * @param threadpool the pool which own the thread */ static void *threadpool_thread(void *threadpool); int threadpool_free(threadpool_t *pool); threadpool_t *threadpool_create(int thread_count, int queue_size, int flags) { threadpool_t *pool; int i; (void) flags; if(thread_count <= 0 || thread_count > MAX_THREADS || queue_size <= 0 || queue_size > MAX_QUEUE) { return NULL; } if((pool = (threadpool_t *)malloc(sizeof(threadpool_t))) == NULL) { goto err; } /* Initialize */ pool->thread_count = 0; pool->queue_size = queue_size; pool->head = pool->tail = pool->count = 0; pool->shutdown = pool->started = 0; /* Allocate thread and task queue */ pool->threads = (pthread_t *)malloc(sizeof(pthread_t) * thread_count); pool->queue = (threadpool_task_t *)malloc (sizeof(threadpool_task_t) * queue_size); /* Initialize mutex and conditional variable first */ if((pthread_mutex_init(&(pool->lock), NULL) != 0) || (pthread_cond_init(&(pool->notify), NULL) != 0) || (pool->threads == NULL) || (pool->queue == NULL)) { goto err; } /* Start worker threads */ for(i = 0; i < thread_count; i++) { if(pthread_create(&(pool->threads[i]), NULL, threadpool_thread, (void*)pool) != 0) { threadpool_destroy(pool, 0); return NULL; } pool->thread_count++; pool->started++; } return pool; err: if(pool) { threadpool_free(pool); } return NULL; } int threadpool_add(threadpool_t *pool, void (*function)(void *), void *argument, int flags) { int err = 0; int next; (void) flags; if(pool == NULL || function == NULL) { return threadpool_invalid; } if(pthread_mutex_lock(&(pool->lock)) != 0) { return threadpool_lock_failure; } next = (pool->tail + 1) % pool->queue_size; do { /* Are we full ? */ if(pool->count == pool->queue_size) { err = threadpool_queue_full; break; } /* Are we shutting down ? */ if(pool->shutdown) { err = threadpool_shutdown; break; } /* Add task to queue */ pool->queue[pool->tail].function = function; pool->queue[pool->tail].argument = argument; pool->tail = next; pool->count += 1; /* pthread_cond_broadcast */ if(pthread_cond_signal(&(pool->notify)) != 0) { err = threadpool_lock_failure; break; } } while(0); if(pthread_mutex_unlock(&pool->lock) != 0) { err = threadpool_lock_failure; } return err; } int threadpool_destroy(threadpool_t *pool, int flags) { int i, err = 0; if(pool == NULL) { return threadpool_invalid; } if(pthread_mutex_lock(&(pool->lock)) != 0) { return threadpool_lock_failure; } do { /* Already shutting down */ if(pool->shutdown) { err = threadpool_shutdown; break; } pool->shutdown = (flags & threadpool_graceful) ? graceful_shutdown : immediate_shutdown; /* Wake up all worker threads */ if((pthread_cond_broadcast(&(pool->notify)) != 0) || (pthread_mutex_unlock(&(pool->lock)) != 0)) { err = threadpool_lock_failure; break; } /* Join all worker thread */ for(i = 0; i < pool->thread_count; i++) { if(pthread_join(pool->threads[i], NULL) != 0) { err = threadpool_thread_failure; } } } while(0); /* Only if everything went well do we deallocate the pool */ if(!err) { threadpool_free(pool); } return err; } int threadpool_free(threadpool_t *pool) { if(pool == NULL || pool->started > 0) { return -1; } /* Did we manage to allocate ? */ if(pool->threads) { free(pool->threads); free(pool->queue); /* Because we allocate pool->threads after initializing the mutex and condition variable, we're sure they're initialized. Let's lock the mutex just in case. */ pthread_mutex_lock(&(pool->lock)); pthread_mutex_destroy(&(pool->lock)); pthread_cond_destroy(&(pool->notify)); } free(pool); return 0; } static void *threadpool_thread(void *threadpool) { threadpool_t *pool = (threadpool_t *)threadpool; threadpool_task_t task; for(;;) { /* Lock must be taken to wait on conditional variable */ pthread_mutex_lock(&(pool->lock)); /* Wait on condition variable, check for spurious wakeups. When returning from pthread_cond_wait(), we own the lock. */ while((pool->count == 0) && (!pool->shutdown)) { pthread_cond_wait(&(pool->notify), &(pool->lock)); } if((pool->shutdown == immediate_shutdown) || ((pool->shutdown == graceful_shutdown) && (pool->count == 0))) { break; } /* Grab our task */ task.function = pool->queue[pool->head].function; task.argument = pool->queue[pool->head].argument; pool->head = (pool->head + 1) % pool->queue_size; pool->count -= 1; /* Unlock */ pthread_mutex_unlock(&(pool->lock)); /* Get to work */ (*(task.function))(task.argument); } pool->started--; pthread_mutex_unlock(&(pool->lock)); pthread_exit(NULL); return(NULL); } caio-0.9.22/caio/src/threadpool/threadpool.h000066400000000000000000000064441500025530100206730ustar00rootroot00000000000000/* * Copyright (c) 2016, Mathias Brossard . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _THREADPOOL_H_ #define _THREADPOOL_H_ #ifdef __cplusplus extern "C" { #endif /** * @file threadpool.h * @brief Threadpool Header File */ /** * Increase this constants at your own risk * Large values might slow down your system */ #define MAX_THREADS 128 #define MAX_QUEUE 65536 typedef struct threadpool_t threadpool_t; typedef enum { threadpool_invalid = -1, threadpool_lock_failure = -2, threadpool_queue_full = -3, threadpool_shutdown = -4, threadpool_thread_failure = -5 } threadpool_error_t; typedef enum { threadpool_graceful = 1 } threadpool_destroy_flags_t; /** * @function threadpool_create * @brief Creates a threadpool_t object. * @param thread_count Number of worker threads. * @param queue_size Size of the queue. * @param flags Unused parameter. * @return a newly created thread pool or NULL */ threadpool_t *threadpool_create(int thread_count, int queue_size, int flags); /** * @function threadpool_add * @brief add a new task in the queue of a thread pool * @param pool Thread pool to which add the task. * @param function Pointer to the function that will perform the task. * @param argument Argument to be passed to the function. * @param flags Unused parameter. * @return 0 if all goes well, negative values in case of error (@see * threadpool_error_t for codes). */ int threadpool_add(threadpool_t *pool, void (*routine)(void *), void *arg, int flags); /** * @function threadpool_destroy * @brief Stops and destroys a thread pool. * @param pool Thread pool to destroy. * @param flags Flags for shutdown * * Known values for flags are 0 (default) and threadpool_graceful in * which case the thread pool doesn't accept any new tasks but * processes all pending tasks before shutdown. */ int threadpool_destroy(threadpool_t *pool, int flags); #ifdef __cplusplus } #endif #endif /* _THREADPOOL_H_ */ caio-0.9.22/caio/thread_aio.c000066400000000000000000000452571500025530100157010ustar00rootroot00000000000000#include #include #include #define PY_SSIZE_T_CLEAN #include #include #include "src/threadpool/threadpool.h" static const unsigned CTX_POOL_SIZE_DEFAULT = 8; static const unsigned CTX_MAX_REQUESTS_DEFAULT = 512; static PyTypeObject AIOOperationType; static PyTypeObject AIOContextType; typedef struct { PyObject_HEAD threadpool_t* pool; uint16_t max_requests; uint8_t pool_size; } AIOContext; typedef struct { PyObject_HEAD PyObject* py_buffer; PyObject* callback; int opcode; unsigned int fileno; off_t offset; int result; uint8_t error; uint8_t in_progress; Py_ssize_t buf_size; char* buf; PyObject* ctx; } AIOOperation; enum THAIO_OP_CODE { THAIO_READ, THAIO_WRITE, THAIO_FSYNC, THAIO_FDSYNC, THAIO_NOOP, }; static void AIOContext_dealloc(AIOContext *self) { if (self->pool != 0) { threadpool_t* pool = self->pool; self->pool = 0; threadpool_destroy(pool, 0); } Py_TYPE(self)->tp_free((PyObject *) self); } /* AIOContext.__new__ classmethod definition */ static PyObject * AIOContext_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { AIOContext *self; self = (AIOContext *) type->tp_alloc(type, 0); return (PyObject *) self; } static int AIOContext_init(AIOContext *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"max_requests", "pool_size", NULL}; self->pool = NULL; self->max_requests = 0; if (!PyArg_ParseTupleAndKeywords( args, kwds, "|HH", kwlist, &self->max_requests, &self->pool_size )) return -1; if (self->max_requests <= 0) { self->max_requests = CTX_MAX_REQUESTS_DEFAULT; } if (self->pool_size <= 0) { self->pool_size = CTX_POOL_SIZE_DEFAULT; } if (self->pool_size > MAX_THREADS) { PyErr_Format( PyExc_ValueError, "pool_size too large. Allowed lower then %d", MAX_THREADS ); return -1; } if (self->max_requests >= (MAX_QUEUE - 1)) { PyErr_Format( PyExc_ValueError, "max_requests too large. Allowed lower then %d", MAX_QUEUE - 1 ); return -1; } self->pool = threadpool_create(self->pool_size, self->max_requests, 0); if (self->pool == NULL) { PyErr_Format( PyExc_RuntimeError, "Pool initialization failed size=%d max_requests=%d", self->pool_size, self->max_requests ); return -1; } return 0; } static PyObject* AIOContext_repr(AIOContext *self) { if (self->pool == NULL) { PyErr_SetString(PyExc_RuntimeError, "Pool not initialized"); return NULL; } return PyUnicode_FromFormat( "<%s as %p: max_requests=%i, pool_size=%i, ctx=%lli>", Py_TYPE(self)->tp_name, self, self->max_requests, self->pool_size, self->pool ); } void worker(void *arg) { PyGILState_STATE state; AIOOperation* op = arg; PyObject* ctx = op->ctx; op->ctx = NULL; op->error = 0; if (op->opcode == THAIO_NOOP) { state = PyGILState_Ensure(); op->ctx = NULL; Py_DECREF(ctx); Py_DECREF(op); PyGILState_Release(state); return; } int fileno = op->fileno; off_t offset = op->offset; int buf_size = op->buf_size; char* buf = op->buf; int result; switch (op->opcode) { case THAIO_WRITE: result = pwrite(fileno, (const char*) buf, buf_size, offset); break; case THAIO_FSYNC: result = fsync(fileno); break; case THAIO_FDSYNC: #ifdef HAVE_FDATASYNC result = fdatasync(fileno); #else result = fsync(fileno); #endif break; case THAIO_READ: result = pread(fileno, buf, buf_size, offset); break; } op->ctx = NULL; op->result = result; if (result < 0) op->error = errno; if (op->opcode == THAIO_READ) { op->buf_size = result; } state = PyGILState_Ensure(); if (op->callback != NULL) { PyObject_CallFunction(op->callback, "i", result); } if (op->opcode == THAIO_WRITE) { Py_DECREF(op->py_buffer); op->py_buffer = NULL; } Py_DECREF(ctx); Py_DECREF(op); PyGILState_Release(state); } inline int process_pool_error(int code) { switch (code) { case threadpool_invalid: PyErr_SetString( PyExc_RuntimeError, "Thread pool pointer is invalid" ); return code; case threadpool_lock_failure: PyErr_SetString( PyExc_RuntimeError, "Failed to lock thread pool" ); return code; case threadpool_queue_full: PyErr_Format( PyExc_RuntimeError, "Thread pool queue full" ); return code; case threadpool_shutdown: PyErr_SetString( PyExc_RuntimeError, "Thread pool is shutdown" ); return code; case threadpool_thread_failure: PyErr_SetString( PyExc_RuntimeError, "Thread failure" ); return code; } if (code < 0) PyErr_SetString(PyExc_RuntimeError, "Unknown error"); return code; } PyDoc_STRVAR(AIOContext_submit_docstring, "Accepts multiple Operations. Returns \n\n" " Operation.submit(aio_op1, aio_op2, aio_opN, ...) -> int" ); static PyObject* AIOContext_submit( AIOContext *self, PyObject *args ) { if (self == NULL) { PyErr_SetString(PyExc_RuntimeError, "self is NULL"); return NULL; } if (self->pool == NULL) { PyErr_SetString(PyExc_RuntimeError, "self->pool is NULL"); return NULL; } if (!PyTuple_Check(args)) { PyErr_SetNone(PyExc_ValueError); return NULL; } unsigned int nr = PyTuple_Size(args); PyObject* obj; AIOOperation* ops[nr]; unsigned int i; for (i=0; i < nr; i++) { obj = PyTuple_GetItem(args, i); if (PyObject_TypeCheck(obj, &AIOOperationType) == 0) { PyErr_Format( PyExc_TypeError, "Wrong type for argument %d", i ); return NULL; } ops[i] = (AIOOperation*) obj; ops[i]->ctx = (void*) self; } unsigned int j=0; int result = 0; for (i=0; i < nr; i++) { if (ops[i]->in_progress) continue; ops[i]->in_progress = 1; Py_INCREF(ops[i]); Py_INCREF(ops[i]->ctx); result = threadpool_add(self->pool, worker, (void*) ops[i], 0); if (process_pool_error(result) < 0) return NULL; j++; } return (PyObject*) PyLong_FromSsize_t(j); } PyDoc_STRVAR(AIOContext_cancel_docstring, "Cancels multiple Operations. Returns \n\n" " Operation.cancel(aio_op1, aio_op2, aio_opN, ...) -> int\n\n" "(Always returns zero, this method exists for compatibility reasons)" ); static PyObject* AIOContext_cancel( AIOContext *self, PyObject *args ) { return (PyObject*) PyLong_FromSsize_t(0); } /* AIOContext properties */ static PyMemberDef AIOContext_members[] = { { "pool_size", T_INT, offsetof(AIOContext, pool_size), READONLY, "pool_size" }, { "max_requests", T_USHORT, offsetof(AIOContext, max_requests), READONLY, "max requests" }, {NULL} /* Sentinel */ }; static PyMethodDef AIOContext_methods[] = { { "submit", (PyCFunction) AIOContext_submit, METH_VARARGS, AIOContext_submit_docstring }, { "cancel", (PyCFunction) AIOContext_cancel, METH_VARARGS, AIOContext_cancel_docstring }, {NULL} /* Sentinel */ }; static PyTypeObject AIOContextType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "Context", .tp_doc = "thread aio context", .tp_basicsize = sizeof(AIOContext), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = AIOContext_new, .tp_init = (initproc) AIOContext_init, .tp_dealloc = (destructor) AIOContext_dealloc, .tp_members = AIOContext_members, .tp_methods = AIOContext_methods, .tp_repr = (reprfunc) AIOContext_repr }; static void AIOOperation_dealloc(AIOOperation *self) { Py_CLEAR(self->callback); if ((self->opcode == THAIO_READ) && self->buf != NULL) { PyMem_Free(self->buf); self->buf = NULL; } Py_CLEAR(self->py_buffer); Py_TYPE(self)->tp_free((PyObject *) self); } static PyObject* AIOOperation_repr(AIOOperation *self) { char* mode; switch (self->opcode) { case THAIO_READ: mode = "read"; break; case THAIO_WRITE: mode = "write"; break; case THAIO_FSYNC: mode = "fsync"; break; case THAIO_FDSYNC: mode = "fdsync"; break; default: mode = "noop"; break; } return PyUnicode_FromFormat( "<%s at %p: mode=\"%s\", fd=%i, offset=%i, result=%i, buffer=%p>", Py_TYPE(self)->tp_name, self, mode, self->fileno, self->offset, self->result, self->buf ); } /* AIOOperation.read classmethod definition */ PyDoc_STRVAR(AIOOperation_read_docstring, "Creates a new instance of Operation on read mode.\n\n" " Operation.read(\n" " nbytes: int,\n" " aio_context: Context,\n" " fd: int, \n" " offset: int,\n" " priority=0\n" " )" ); static PyObject* AIOOperation_read( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"nbytes", "fd", "offset", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } self->buf = NULL; self->py_buffer = NULL; self->in_progress = 0; uint64_t nbytes = 0; uint16_t priority; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "KI|LH", kwlist, &nbytes, &(self->fileno), &(self->offset), &priority ); if (!argIsOk) return NULL; self->buf = PyMem_Calloc(nbytes, sizeof(char)); self->buf_size = nbytes; self->py_buffer = PyMemoryView_FromMemory( self->buf, self->buf_size, PyBUF_READ ); self->opcode = THAIO_READ; return (PyObject*) self; } /* AIOOperation.write classmethod definition */ PyDoc_STRVAR(AIOOperation_write_docstring, "Creates a new instance of Operation on write mode.\n\n" " Operation.write(\n" " payload_bytes: bytes,\n" " fd: int, \n" " offset: int,\n" " priority=0\n" " )" ); static PyObject* AIOOperation_write( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"payload_bytes", "fd", "offset", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } // unused uint16_t priority; self->buf = NULL; self->py_buffer = NULL; self->in_progress = 0; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "OI|LH", kwlist, &(self->py_buffer), &(self->fileno), &(self->offset), &priority ); if (!argIsOk) return NULL; if (!PyBytes_Check(self->py_buffer)) { Py_XDECREF(self); PyErr_SetString( PyExc_ValueError, "payload_bytes argument must be bytes" ); return NULL; } self->opcode = THAIO_WRITE; if (PyBytes_AsStringAndSize( self->py_buffer, &self->buf, &self->buf_size )) { Py_XDECREF(self); PyErr_SetString( PyExc_RuntimeError, "Can not convert bytes to c string" ); return NULL; } Py_INCREF(self->py_buffer); return (PyObject*) self; } /* AIOOperation.fsync classmethod definition */ PyDoc_STRVAR(AIOOperation_fsync_docstring, "Creates a new instance of Operation on fsync mode.\n\n" " Operation.fsync(\n" " aio_context: AIOContext,\n" " fd: int, \n" " priority=0\n" " )" ); static PyObject* AIOOperation_fsync( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"fd", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } uint16_t priority; self->buf = NULL; self->py_buffer = NULL; self->in_progress = 0; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "I|H", kwlist, &(self->fileno), &priority ); if (!argIsOk) return NULL; self->opcode = THAIO_FSYNC; return (PyObject*) self; } /* AIOOperation.fdsync classmethod definition */ PyDoc_STRVAR(AIOOperation_fdsync_docstring, "Creates a new instance of Operation on fdsync mode.\n\n" " Operation.fdsync(\n" " aio_context: AIOContext,\n" " fd: int, \n" " priority=0\n" " )" ); static PyObject* AIOOperation_fdsync( PyTypeObject *type, PyObject *args, PyObject *kwds ) { AIOOperation *self = (AIOOperation *) type->tp_alloc(type, 0); static char *kwlist[] = {"fd", "priority", NULL}; if (self == NULL) { PyErr_SetString(PyExc_MemoryError, "can not allocate memory"); return NULL; } self->buf = NULL; self->py_buffer = NULL; self->in_progress = 0; uint16_t priority; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "I|H", kwlist, &(self->fileno), &priority ); if (!argIsOk) return NULL; self->opcode = THAIO_FDSYNC; return (PyObject*) self; } /* AIOOperation.get_value method definition */ PyDoc_STRVAR(AIOOperation_get_value_docstring, "Method returns a bytes value of Operation's result or None.\n\n" " Operation.get_value() -> Optional[bytes]" ); static PyObject* AIOOperation_get_value( AIOOperation *self, PyObject *args, PyObject *kwds ) { if (self->error != 0) { PyErr_SetString( PyExc_SystemError, strerror(self->error) ); return NULL; } switch (self->opcode) { case THAIO_READ: return PyBytes_FromStringAndSize( self->buf, self->buf_size ); case THAIO_WRITE: return PyLong_FromSsize_t(self->result); } return Py_None; } /* AIOOperation.get_value method definition */ PyDoc_STRVAR(AIOOperation_set_callback_docstring, "Set callback which will be called after Operation will be finished.\n\n" " Operation.get_value() -> Optional[bytes]" ); static PyObject* AIOOperation_set_callback( AIOOperation *self, PyObject *args, PyObject *kwds ) { static char *kwlist[] = {"callback", NULL}; PyObject* callback; int argIsOk = PyArg_ParseTupleAndKeywords( args, kwds, "O", kwlist, &callback ); if (!argIsOk) return NULL; if (!PyCallable_Check(callback)) { PyErr_Format( PyExc_ValueError, "object %r is not callable", callback ); return NULL; } Py_INCREF(callback); self->callback = callback; Py_RETURN_TRUE; } /* AIOOperation properties */ static PyMemberDef AIOOperation_members[] = { { "fileno", T_UINT, offsetof(AIOOperation, fileno), READONLY, "file descriptor" }, { "offset", T_ULONGLONG, offsetof(AIOOperation, offset), READONLY, "offset" }, { "payload", T_OBJECT, offsetof(AIOOperation, py_buffer), READONLY, "payload" }, { "nbytes", T_ULONGLONG, offsetof(AIOOperation, buf_size), READONLY, "nbytes" }, { "result", T_INT, offsetof(AIOOperation, result), READONLY, "result" }, { "error", T_INT, offsetof(AIOOperation, error), READONLY, "error" }, {NULL} /* Sentinel */ }; /* AIOOperation methods */ static PyMethodDef AIOOperation_methods[] = { { "read", (PyCFunction) AIOOperation_read, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_read_docstring }, { "write", (PyCFunction) AIOOperation_write, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_write_docstring }, { "fsync", (PyCFunction) AIOOperation_fsync, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_fsync_docstring }, { "fdsync", (PyCFunction) AIOOperation_fdsync, METH_CLASS | METH_VARARGS | METH_KEYWORDS, AIOOperation_fdsync_docstring }, { "get_value", (PyCFunction) AIOOperation_get_value, METH_NOARGS, AIOOperation_get_value_docstring }, { "set_callback", (PyCFunction) AIOOperation_set_callback, METH_VARARGS | METH_KEYWORDS, AIOOperation_set_callback_docstring }, {NULL} /* Sentinel */ }; /* AIOOperation class */ static PyTypeObject AIOOperationType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "aio.AIOOperation", .tp_doc = "thread aio operation representation", .tp_basicsize = sizeof(AIOOperation), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_dealloc = (destructor) AIOOperation_dealloc, .tp_members = AIOOperation_members, .tp_methods = AIOOperation_methods, .tp_repr = (reprfunc) AIOOperation_repr }; static PyModuleDef thread_aio_module = { PyModuleDef_HEAD_INIT, .m_name = "thread_aio", .m_doc = "Thread based AIO.", .m_size = -1, }; PyMODINIT_FUNC PyInit_thread_aio(void) { Py_Initialize(); PyObject *m; m = PyModule_Create(&thread_aio_module); if (m == NULL) return NULL; if (PyType_Ready(&AIOContextType) < 0) return NULL; Py_INCREF(&AIOContextType); if (PyModule_AddObject(m, "Context", (PyObject *) &AIOContextType) < 0) { Py_XDECREF(&AIOContextType); Py_XDECREF(m); return NULL; } if (PyType_Ready(&AIOOperationType) < 0) return NULL; Py_INCREF(&AIOOperationType); if (PyModule_AddObject(m, "Operation", (PyObject *) &AIOOperationType) < 0) { Py_XDECREF(&AIOOperationType); Py_XDECREF(m); return NULL; } return m; } caio-0.9.22/caio/thread_aio.pyi000066400000000000000000000021221500025530100162400ustar00rootroot00000000000000from typing import Callable, Any, Union, Optional from .abstract import AbstractContext, AbstractOperation # noinspection PyPropertyDefinition class Context(AbstractContext): def __init__(self, max_requests: int = 32, pool_size=8): ... # noinspection PyPropertyDefinition class Operation(AbstractOperation): @classmethod def read( cls, nbytes: int, fd: int, offset: int, priority=0 ) -> "AbstractOperation": ... @classmethod def write( cls, payload_bytes: bytes, fd: int, offset: int, priority=0, ) -> "AbstractOperation": ... @classmethod def fsync(cls, fd: int, priority=0) -> "AbstractOperation": ... @classmethod def fdsync(cls, fd: int, priority=0) -> "AbstractOperation": ... def get_value(self) -> Union[bytes, int]: ... @property def fileno(self) -> int: ... @property def offset(self) -> int: ... @property def payload(self) -> Optional[Union[bytes, memoryview]]: ... @property def nbytes(self) -> int: ... def set_callback(self, callback: Callable[[int], Any]) -> bool: ... caio-0.9.22/caio/thread_aio_asyncio.py000066400000000000000000000003371500025530100176220ustar00rootroot00000000000000from .asyncio_base import AsyncioContextBase from .thread_aio import Context, Operation class AsyncioContext(AsyncioContextBase): MAX_REQUESTS_DEFAULT = 512 OPERATION_CLASS = Operation CONTEXT_CLASS = Context caio-0.9.22/caio/version.py000066400000000000000000000007571500025530100154710ustar00rootroot00000000000000author_info = (("Dmitry Orlov", "me@mosquito.su"),) package_info = "Asynchronous file IO for Linux MacOS or Windows." package_license = "Apache Software License" team_email = author_info[0][1] version_info = (0, 9, 22) __author__ = ", ".join("{} <{}>".format(*info) for info in author_info) __version__ = ".".join(map(str, version_info)) __all__ = ( "author_info", "package_info", "package_license", "team_email", "version_info", "__author__", "__version__", ) caio-0.9.22/example.py000066400000000000000000000013671500025530100145220ustar00rootroot00000000000000import asyncio from caio import AsyncioContext loop = asyncio.get_event_loop() async def main(): # max_requests=128 by default ctx = AsyncioContext(max_requests=128) with open("test.file", "wb+") as fp: fd = fp.fileno() # Execute one write operation await ctx.write(b"Hello world", fd, offset=0) # Execute one read operation print(await ctx.read(32, fd, offset=0)) # Execute one fdsync operation await ctx.fdsync(fd) op1 = ctx.write(b"Hello from ", fd, offset=0) op2 = ctx.write(b"async world", fd, offset=11) await asyncio.gather(op1, op2) print(await ctx.read(32, fd, offset=0)) # Hello from async world loop.run_until_complete(main()) caio-0.9.22/pylava.ini000066400000000000000000000001371500025530100145040ustar00rootroot00000000000000[pylava] ignore=C901,E252 skip = env*,.tox*,*build* [pylava:pycodestyle] max_line_length = 80 caio-0.9.22/pyproject.toml000066400000000000000000000001251500025530100154200ustar00rootroot00000000000000[build-system] requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" caio-0.9.22/scripts/000077500000000000000000000000001500025530100141755ustar00rootroot00000000000000caio-0.9.22/scripts/make-wheels.sh000066400000000000000000000006721500025530100167400ustar00rootroot00000000000000set -ex mkdir -p dist MACHINE=$(/opt/python/cp311-cp311/bin/python3 -c 'import platform; print(platform.machine())') function build_wheel() { /opt/python/$1/bin/pip wheel . -f . -w dist } build_wheel cp38-cp38 build_wheel cp39-cp39 build_wheel cp310-cp310 build_wheel cp311-cp311 build_wheel cp312-cp312 build_wheel cp313-cp313 cd dist for f in ./*-linux*_${MACHINE}*; do if [ -f $f ]; then auditwheel repair $f -w . ; rm $f; fi; done caio-0.9.22/setup.py000066400000000000000000000055421500025530100142260ustar00rootroot00000000000000import os import platform from importlib.machinery import SourceFileLoader from setuptools import Extension, setup module_name = "caio" module = SourceFileLoader( "version", os.path.join(module_name, "version.py"), ).load_module() OS_NAME = platform.system().lower() extensions = [] if "linux" in OS_NAME: extensions.append( Extension( "{}.thread_aio".format(module_name), [ "{}/thread_aio.c".format(module_name), "{}/src/threadpool/threadpool.c".format(module_name), ], extra_compile_args=["-g", "-DHAVE_FDATASYNC"], ), ) elif "darwin" in OS_NAME: extensions.append( Extension( "{}.thread_aio".format(module_name), [ "{}/thread_aio.c".format(module_name), "{}/src/threadpool/threadpool.c".format(module_name), ], extra_compile_args=["-g"], ), ) if "linux" in OS_NAME: extensions.append( Extension( "{}.linux_aio".format(module_name), ["{}/linux_aio.c".format(module_name)], extra_compile_args=["-g"], ), ) setup( name=module_name, version=module.__version__, ext_modules=extensions, include_package_data=True, description=module.package_info, long_description=open("README.md").read(), long_description_content_type="text/markdown", license=module.package_license, author=module.__author__, author_email=module.team_email, package_data={ module_name: [ "{}/linux_aio.pyi".format(module_name), "{}/thread_aio.pyi".format(module_name), "py.typed", ], }, project_urls={ "Documentation": "https://github.com/mosquito/caio/", "Source": "https://github.com/mosquito/caio", }, packages=[module_name], classifiers=[ "License :: OSI Approved :: Apache Software License", "Topic :: Software Development", "Topic :: Software Development :: Libraries", "Intended Audience :: Developers", "Natural Language :: English", "Operating System :: MacOS", "Operating System :: POSIX", "Operating System :: Microsoft", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", ], python_requires=">=3.7, <4", extras_require={ "develop": [ "aiomisc-pytest", "pytest", "pytest-cov", ], }, ) caio-0.9.22/tests/000077500000000000000000000000001500025530100136505ustar00rootroot00000000000000caio-0.9.22/tests/conftest.py000066400000000000000000000004201500025530100160430ustar00rootroot00000000000000import pytest from caio import variants, variants_asyncio @pytest.fixture(params=variants) def context_maker(request): return request.param.Context @pytest.fixture(params=variants_asyncio) def async_context_maker(request): return request.param.AsyncioContext caio-0.9.22/tests/test_aio_context.py000066400000000000000000000003151500025530100175740ustar00rootroot00000000000000 def test_aio_context(context_maker): ctx = context_maker() assert ctx is not None ctx = context_maker(1) assert ctx is not None ctx = context_maker(32218) assert ctx is not None caio-0.9.22/tests/test_asyncio_adapter.py000066400000000000000000000055231500025530100204330ustar00rootroot00000000000000import asyncio import hashlib import os from unittest.mock import Mock import aiomisc import pytest @aiomisc.timeout(5) async def test_adapter(tmp_path, async_context_maker): async with async_context_maker() as context: with open(str(tmp_path / "temp.bin"), "wb+") as fp: fd = fp.fileno() assert await context.read(32, fd, 0) == b"" s = b"Hello world" assert await context.write(s, fd, 0) == len(s) assert await context.read(32, fd, 0) == s s = b"Hello real world" assert await context.write(s, fd, 0) == len(s) assert await context.read(32, fd, 0) == s part = b"\x00\x01\x02\x03" limit = 32 expected_hash = hashlib.md5(part * limit).hexdigest() await asyncio.gather( *[context.write(part, fd, len(part) * i) for i in range(limit)] ) await context.fdsync(fd) data = await context.read(limit * len(part), fd, 0) assert data == part * limit assert hashlib.md5(bytes(data)).hexdigest() == expected_hash @aiomisc.timeout(3) async def test_bad_file_descritor(tmp_path, async_context_maker): async with async_context_maker() as context: with open(str(tmp_path / "temp.bin"), "wb+") as fp: fd = fp.fileno() with pytest.raises((SystemError, OSError, AssertionError, ValueError)): assert await context.read(1, fd, 0) == b"" with pytest.raises((SystemError, OSError, AssertionError, ValueError)): assert await context.write(b"hello", fd, 0) @pytest.fixture async def asyncio_exception_handler(event_loop): handler = Mock( side_effect=lambda _loop, ctx: _loop.default_exception_handler(ctx) ) current_handler = event_loop.get_exception_handler() event_loop.set_exception_handler(handler=handler) yield handler event_loop.set_exception_handler(current_handler) @aiomisc.timeout(3) async def test_operations_cancel_cleanly( tmp_path, async_context_maker, asyncio_exception_handler ): async with async_context_maker() as context: with open(str(tmp_path / "temp.bin"), "wb+") as fp: fd = fp.fileno() await context.write(b"\x00", fd, 1024**2 - 1) assert os.stat(fd).st_size == 1024**2 for _ in range(50): reads = [ asyncio.create_task(context.read(2**16, fd, 2**16 * i)) for i in range(16) ] _, pending = await asyncio.wait( reads, return_when=asyncio.FIRST_COMPLETED ) for read in pending: read.cancel() if pending: await asyncio.wait(pending) asyncio_exception_handler.assert_not_called() caio-0.9.22/tests/test_impl_selector.py000066400000000000000000000027371500025530100201330ustar00rootroot00000000000000import os import platform import sys from subprocess import check_output import caio import pytest @pytest.fixture(params=caio.variants) def implementation(request): if request.param is caio.linux_aio: return "linux" if request.param is caio.thread_aio: return "thread" if request.param is caio.python_aio: return "python" raise RuntimeError("Unknown variant %r" % (request.param,)) @pytest.mark.skipif(platform.system() == 'Windows', reason="Windows skip") def test_env_selector(implementation): output = check_output( [ sys.executable, "-c", "import caio, inspect; print(caio.Context.__doc__)" ], env={"CAIO_IMPL": implementation} ).decode() assert implementation in output, output @pytest.fixture() def implementation_file(implementation): path = os.path.dirname(caio.__file__) fname = os.path.join(path, "default_implementation") try: with open(fname, "w") as fp: fp.write("# NEWER COMMIT THIS FILE") fp.write("\nwrong string\n") fp.write(implementation) fp.write("\n\n") yield implementation finally: os.remove(fname) def test_file_selector(implementation_file): output = check_output( [ sys.executable, "-c", "import caio, inspect; print(caio.Context.__doc__)" ], ).decode() assert implementation_file in output, output caio-0.9.22/tox.ini000066400000000000000000000010651500025530100140230ustar00rootroot00000000000000[tox] envlist = lint,py3{8-13} [testenv] passenv = COVERALLS_*, FORCE_COLOR usedevelop = true extras = develop commands= py.test --cov=caio --cov-report=term-missing -sv tests [testenv:lint] usedevelop = true deps = pyflakes~=2.4.0 pylava commands= pylava -o pylava.ini . [testenv:checkdoc] deps = collective.checkdocs pygments commands = python setup.py checkdocs [testenv:mypy] usedevelop = true deps = mypy commands = mypy \ --allow-untyped-calls \ --allow-untyped-defs \ --allow-untyped-decorators \ caio