"
__repr__ = __str__
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def reopen(fs, path, mode, blocksize, loc, size, autocommit, cache_type, kwargs):
file = fs.open(
path,
mode=mode,
block_size=blocksize,
autocommit=autocommit,
cache_type=cache_type,
size=size,
**kwargs,
)
if loc > 0:
file.seek(loc)
return file
filesystem_spec-2025.3.2/fsspec/tests/ 0000775 0000000 0000000 00000000000 14772531734 0017605 5 ustar 00root root 0000000 0000000 filesystem_spec-2025.3.2/fsspec/tests/__init__.py 0000664 0000000 0000000 00000000000 14772531734 0021704 0 ustar 00root root 0000000 0000000 filesystem_spec-2025.3.2/fsspec/tests/abstract/ 0000775 0000000 0000000 00000000000 14772531734 0021410 5 ustar 00root root 0000000 0000000 filesystem_spec-2025.3.2/fsspec/tests/abstract/__init__.py 0000664 0000000 0000000 00000023705 14772531734 0023530 0 ustar 00root root 0000000 0000000 import os
from hashlib import md5
import pytest
from fsspec.implementations.local import LocalFileSystem
from fsspec.tests.abstract.copy import AbstractCopyTests # noqa: F401
from fsspec.tests.abstract.get import AbstractGetTests # noqa: F401
from fsspec.tests.abstract.open import AbstractOpenTests # noqa: F401
from fsspec.tests.abstract.pipe import AbstractPipeTests # noqa: F401
from fsspec.tests.abstract.put import AbstractPutTests # noqa: F401
class BaseAbstractFixtures:
"""
Abstract base class containing fixtures that are used by but never need to
be overridden in derived filesystem-specific classes to run the abstract
tests on such filesystems.
"""
@pytest.fixture
def fs_bulk_operations_scenario_0(self, fs, fs_join, fs_path):
"""
Scenario on remote filesystem that is used for many cp/get/put tests.
Cleans up at the end of each test it which it is used.
"""
source = self._bulk_operations_scenario_0(fs, fs_join, fs_path)
yield source
fs.rm(source, recursive=True)
@pytest.fixture
def fs_glob_edge_cases_files(self, fs, fs_join, fs_path):
"""
Scenario on remote filesystem that is used for glob edge cases cp/get/put tests.
Cleans up at the end of each test it which it is used.
"""
source = self._glob_edge_cases_files(fs, fs_join, fs_path)
yield source
fs.rm(source, recursive=True)
@pytest.fixture
def fs_dir_and_file_with_same_name_prefix(self, fs, fs_join, fs_path):
"""
Scenario on remote filesystem that is used to check cp/get/put on directory
and file with the same name prefixes.
Cleans up at the end of each test it which it is used.
"""
source = self._dir_and_file_with_same_name_prefix(fs, fs_join, fs_path)
yield source
fs.rm(source, recursive=True)
@pytest.fixture
def fs_10_files_with_hashed_names(self, fs, fs_join, fs_path):
"""
Scenario on remote filesystem that is used to check cp/get/put files order
when source and destination are lists.
Cleans up at the end of each test it which it is used.
"""
source = self._10_files_with_hashed_names(fs, fs_join, fs_path)
yield source
fs.rm(source, recursive=True)
@pytest.fixture
def fs_target(self, fs, fs_join, fs_path):
"""
Return name of remote directory that does not yet exist to copy into.
Cleans up at the end of each test it which it is used.
"""
target = fs_join(fs_path, "target")
yield target
if fs.exists(target):
fs.rm(target, recursive=True)
@pytest.fixture
def local_bulk_operations_scenario_0(self, local_fs, local_join, local_path):
"""
Scenario on local filesystem that is used for many cp/get/put tests.
Cleans up at the end of each test it which it is used.
"""
source = self._bulk_operations_scenario_0(local_fs, local_join, local_path)
yield source
local_fs.rm(source, recursive=True)
@pytest.fixture
def local_glob_edge_cases_files(self, local_fs, local_join, local_path):
"""
Scenario on local filesystem that is used for glob edge cases cp/get/put tests.
Cleans up at the end of each test it which it is used.
"""
source = self._glob_edge_cases_files(local_fs, local_join, local_path)
yield source
local_fs.rm(source, recursive=True)
@pytest.fixture
def local_dir_and_file_with_same_name_prefix(
self, local_fs, local_join, local_path
):
"""
Scenario on local filesystem that is used to check cp/get/put on directory
and file with the same name prefixes.
Cleans up at the end of each test it which it is used.
"""
source = self._dir_and_file_with_same_name_prefix(
local_fs, local_join, local_path
)
yield source
local_fs.rm(source, recursive=True)
@pytest.fixture
def local_10_files_with_hashed_names(self, local_fs, local_join, local_path):
"""
Scenario on local filesystem that is used to check cp/get/put files order
when source and destination are lists.
Cleans up at the end of each test it which it is used.
"""
source = self._10_files_with_hashed_names(local_fs, local_join, local_path)
yield source
local_fs.rm(source, recursive=True)
@pytest.fixture
def local_target(self, local_fs, local_join, local_path):
"""
Return name of local directory that does not yet exist to copy into.
Cleans up at the end of each test it which it is used.
"""
target = local_join(local_path, "target")
yield target
if local_fs.exists(target):
local_fs.rm(target, recursive=True)
def _glob_edge_cases_files(self, some_fs, some_join, some_path):
"""
Scenario that is used for glob edge cases cp/get/put tests.
Creates the following directory and file structure:
📠source
├── 📄 file1
├── 📄 file2
├── 📠subdir0
│ ├── 📄 subfile1
│ ├── 📄 subfile2
│ └── 📠nesteddir
│ └── 📄 nestedfile
└── 📠subdir1
├── 📄 subfile1
├── 📄 subfile2
└── 📠nesteddir
└── 📄 nestedfile
"""
source = some_join(some_path, "source")
some_fs.touch(some_join(source, "file1"))
some_fs.touch(some_join(source, "file2"))
for subdir_idx in range(2):
subdir = some_join(source, f"subdir{subdir_idx}")
nesteddir = some_join(subdir, "nesteddir")
some_fs.makedirs(nesteddir)
some_fs.touch(some_join(subdir, "subfile1"))
some_fs.touch(some_join(subdir, "subfile2"))
some_fs.touch(some_join(nesteddir, "nestedfile"))
return source
def _bulk_operations_scenario_0(self, some_fs, some_join, some_path):
"""
Scenario that is used for many cp/get/put tests. Creates the following
directory and file structure:
📠source
├── 📄 file1
├── 📄 file2
└── 📠subdir
├── 📄 subfile1
├── 📄 subfile2
└── 📠nesteddir
└── 📄 nestedfile
"""
source = some_join(some_path, "source")
subdir = some_join(source, "subdir")
nesteddir = some_join(subdir, "nesteddir")
some_fs.makedirs(nesteddir)
some_fs.touch(some_join(source, "file1"))
some_fs.touch(some_join(source, "file2"))
some_fs.touch(some_join(subdir, "subfile1"))
some_fs.touch(some_join(subdir, "subfile2"))
some_fs.touch(some_join(nesteddir, "nestedfile"))
return source
def _dir_and_file_with_same_name_prefix(self, some_fs, some_join, some_path):
"""
Scenario that is used to check cp/get/put on directory and file with
the same name prefixes. Creates the following directory and file structure:
📠source
├── 📄 subdir.txt
└── 📠subdir
└── 📄 subfile.txt
"""
source = some_join(some_path, "source")
subdir = some_join(source, "subdir")
file = some_join(source, "subdir.txt")
subfile = some_join(subdir, "subfile.txt")
some_fs.makedirs(subdir)
some_fs.touch(file)
some_fs.touch(subfile)
return source
def _10_files_with_hashed_names(self, some_fs, some_join, some_path):
"""
Scenario that is used to check cp/get/put files order when source and
destination are lists. Creates the following directory and file structure:
📠source
└── 📄 {hashed([0-9])}.txt
"""
source = some_join(some_path, "source")
for i in range(10):
hashed_i = md5(str(i).encode("utf-8")).hexdigest()
path = some_join(source, f"{hashed_i}.txt")
some_fs.pipe(path=path, value=f"{i}".encode())
return source
class AbstractFixtures(BaseAbstractFixtures):
"""
Abstract base class containing fixtures that may be overridden in derived
filesystem-specific classes to run the abstract tests on such filesystems.
For any particular filesystem some of these fixtures must be overridden,
such as ``fs`` and ``fs_path``, and others may be overridden if the
default functions here are not appropriate, such as ``fs_join``.
"""
@pytest.fixture
def fs(self):
raise NotImplementedError("This function must be overridden in derived classes")
@pytest.fixture
def fs_join(self):
"""
Return a function that joins its arguments together into a path.
Most fsspec implementations join paths in a platform-dependent way,
but some will override this to always use a forward slash.
"""
return os.path.join
@pytest.fixture
def fs_path(self):
raise NotImplementedError("This function must be overridden in derived classes")
@pytest.fixture(scope="class")
def local_fs(self):
# Maybe need an option for auto_mkdir=False? This is only relevant
# for certain implementations.
return LocalFileSystem(auto_mkdir=True)
@pytest.fixture
def local_join(self):
"""
Return a function that joins its arguments together into a path, on
the local filesystem.
"""
return os.path.join
@pytest.fixture
def local_path(self, tmpdir):
return tmpdir
@pytest.fixture
def supports_empty_directories(self):
"""
Return whether this implementation supports empty directories.
"""
return True
@pytest.fixture
def fs_sanitize_path(self):
return lambda x: x
filesystem_spec-2025.3.2/fsspec/tests/abstract/common.py 0000664 0000000 0000000 00000011555 14772531734 0023261 0 ustar 00root root 0000000 0000000 GLOB_EDGE_CASES_TESTS = {
"argnames": ("path", "recursive", "maxdepth", "expected"),
"argvalues": [
("fil?1", False, None, ["file1"]),
("fil?1", True, None, ["file1"]),
("file[1-2]", False, None, ["file1", "file2"]),
("file[1-2]", True, None, ["file1", "file2"]),
("*", False, None, ["file1", "file2"]),
(
"*",
True,
None,
[
"file1",
"file2",
"subdir0/subfile1",
"subdir0/subfile2",
"subdir0/nesteddir/nestedfile",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
("*", True, 1, ["file1", "file2"]),
(
"*",
True,
2,
[
"file1",
"file2",
"subdir0/subfile1",
"subdir0/subfile2",
"subdir1/subfile1",
"subdir1/subfile2",
],
),
("*1", False, None, ["file1"]),
(
"*1",
True,
None,
[
"file1",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
("*1", True, 2, ["file1", "subdir1/subfile1", "subdir1/subfile2"]),
(
"**",
False,
None,
[
"file1",
"file2",
"subdir0/subfile1",
"subdir0/subfile2",
"subdir0/nesteddir/nestedfile",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
(
"**",
True,
None,
[
"file1",
"file2",
"subdir0/subfile1",
"subdir0/subfile2",
"subdir0/nesteddir/nestedfile",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
("**", True, 1, ["file1", "file2"]),
(
"**",
True,
2,
[
"file1",
"file2",
"subdir0/subfile1",
"subdir0/subfile2",
"subdir0/nesteddir/nestedfile",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
(
"**",
False,
2,
[
"file1",
"file2",
"subdir0/subfile1",
"subdir0/subfile2",
"subdir1/subfile1",
"subdir1/subfile2",
],
),
("**/*1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
(
"**/*1",
True,
None,
[
"file1",
"subdir0/subfile1",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
("**/*1", True, 1, ["file1"]),
(
"**/*1",
True,
2,
["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"],
),
("**/*1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]),
("**/subdir0", False, None, []),
("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]),
("**/subdir0/nested*", False, 2, []),
("**/subdir0/nested*", True, 2, ["nestedfile"]),
("subdir[1-2]", False, None, []),
("subdir[1-2]", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]),
("subdir[1-2]", True, 2, ["subfile1", "subfile2"]),
("subdir[0-1]", False, None, []),
(
"subdir[0-1]",
True,
None,
[
"subdir0/subfile1",
"subdir0/subfile2",
"subdir0/nesteddir/nestedfile",
"subdir1/subfile1",
"subdir1/subfile2",
"subdir1/nesteddir/nestedfile",
],
),
(
"subdir[0-1]/*fil[e]*",
False,
None,
[
"subdir0/subfile1",
"subdir0/subfile2",
"subdir1/subfile1",
"subdir1/subfile2",
],
),
(
"subdir[0-1]/*fil[e]*",
True,
None,
[
"subdir0/subfile1",
"subdir0/subfile2",
"subdir1/subfile1",
"subdir1/subfile2",
],
),
],
}
filesystem_spec-2025.3.2/fsspec/tests/abstract/copy.py 0000664 0000000 0000000 00000046777 14772531734 0022761 0 ustar 00root root 0000000 0000000 from hashlib import md5
from itertools import product
import pytest
from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
class AbstractCopyTests:
def test_copy_file_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
fs_target,
supports_empty_directories,
):
# Copy scenario 1a
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
fs.touch(fs_join(target, "dummy"))
assert fs.isdir(target)
target_file2 = fs_join(target, "file2")
target_subfile1 = fs_join(target, "subfile1")
# Copy from source directory
fs.cp(fs_join(source, "file2"), target)
assert fs.isfile(target_file2)
# Copy from sub directory
fs.cp(fs_join(source, "subdir", "subfile1"), target)
assert fs.isfile(target_subfile1)
# Remove copied files
fs.rm([target_file2, target_subfile1])
assert not fs.exists(target_file2)
assert not fs.exists(target_subfile1)
# Repeat with trailing slash on target
fs.cp(fs_join(source, "file2"), target + "/")
assert fs.isdir(target)
assert fs.isfile(target_file2)
fs.cp(fs_join(source, "subdir", "subfile1"), target + "/")
assert fs.isfile(target_subfile1)
def test_copy_file_to_new_directory(
self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
):
# Copy scenario 1b
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
fs.cp(
fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir/")
) # Note trailing slash
assert fs.isdir(target)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
def test_copy_file_to_file_in_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
fs_target,
supports_empty_directories,
):
# Copy scenario 1c
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
fs.touch(fs_join(target, "dummy"))
assert fs.isdir(target)
fs.cp(fs_join(source, "subdir", "subfile1"), fs_join(target, "newfile"))
assert fs.isfile(fs_join(target, "newfile"))
def test_copy_file_to_file_in_new_directory(
self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
):
# Copy scenario 1d
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
fs.cp(
fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir", "newfile")
)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "newfile"))
def test_copy_directory_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
fs_target,
supports_empty_directories,
):
# Copy scenario 1e
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
dummy = fs_join(target, "dummy")
fs.touch(dummy)
assert fs.isdir(target)
for source_slash, target_slash in zip([False, True], [False, True]):
s = fs_join(source, "subdir")
if source_slash:
s += "/"
t = target + "/" if target_slash else target
# Without recursive does nothing
fs.cp(s, t)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# With recursive
fs.cp(s, t, recursive=True)
if source_slash:
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert fs.isdir(fs_join(target, "nesteddir"))
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
fs_join(target, "nesteddir"),
],
recursive=True,
)
else:
assert fs.isdir(fs_join(target, "subdir"))
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
assert fs.isfile(fs_join(target, "subdir", "subfile2"))
assert fs.isdir(fs_join(target, "subdir", "nesteddir"))
assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile"))
fs.rm(fs_join(target, "subdir"), recursive=True)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# Limit recursive by maxdepth
fs.cp(s, t, recursive=True, maxdepth=1)
if source_slash:
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert not fs.exists(fs_join(target, "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
],
recursive=True,
)
else:
assert fs.isdir(fs_join(target, "subdir"))
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
assert fs.isfile(fs_join(target, "subdir", "subfile2"))
assert not fs.exists(fs_join(target, "subdir", "nesteddir"))
fs.rm(fs_join(target, "subdir"), recursive=True)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
def test_copy_directory_to_new_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
fs_target,
supports_empty_directories,
):
# Copy scenario 1f
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
for source_slash, target_slash in zip([False, True], [False, True]):
s = fs_join(source, "subdir")
if source_slash:
s += "/"
t = fs_join(target, "newdir")
if target_slash:
t += "/"
# Without recursive does nothing
fs.cp(s, t)
if supports_empty_directories:
assert fs.ls(target) == []
else:
with pytest.raises(FileNotFoundError):
fs.ls(target)
# With recursive
fs.cp(s, t, recursive=True)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
# Limit recursive by maxdepth
fs.cp(s, t, recursive=True, maxdepth=1)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
def test_copy_glob_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
fs_target,
supports_empty_directories,
):
# Copy scenario 1g
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
dummy = fs_join(target, "dummy")
fs.touch(dummy)
assert fs.isdir(target)
for target_slash in [False, True]:
t = target + "/" if target_slash else target
# Without recursive
fs.cp(fs_join(source, "subdir", "*"), t)
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert not fs.isdir(fs_join(target, "nesteddir"))
assert not fs.exists(fs_join(target, "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# With recursive
for glob, recursive in zip(["*", "**"], [True, False]):
fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive)
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert fs.isdir(fs_join(target, "nesteddir"))
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
fs_join(target, "nesteddir"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# Limit recursive by maxdepth
fs.cp(
fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
)
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert not fs.exists(fs_join(target, "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
def test_copy_glob_to_new_directory(
self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
):
# Copy scenario 1h
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
for target_slash in [False, True]:
t = fs_join(target, "newdir")
if target_slash:
t += "/"
# Without recursive
fs.cp(fs_join(source, "subdir", "*"), t)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
assert not fs.exists(fs_join(target, "newdir", "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
# With recursive
for glob, recursive in zip(["*", "**"], [True, False]):
fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
assert not fs.exists(fs_join(target, "newdir", "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
# Limit recursive by maxdepth
fs.cp(
fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
assert not fs.exists(fs_join(target, "newdir", "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
@pytest.mark.parametrize(
GLOB_EDGE_CASES_TESTS["argnames"],
GLOB_EDGE_CASES_TESTS["argvalues"],
)
def test_copy_glob_edge_cases(
self,
path,
recursive,
maxdepth,
expected,
fs,
fs_join,
fs_glob_edge_cases_files,
fs_target,
fs_sanitize_path,
):
# Copy scenario 1g
source = fs_glob_edge_cases_files
target = fs_target
for new_dir, target_slash in product([True, False], [True, False]):
fs.mkdir(target)
t = fs_join(target, "newdir") if new_dir else target
t = t + "/" if target_slash else t
fs.copy(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
output = fs.find(target)
if new_dir:
prefixed_expected = [
fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected
]
else:
prefixed_expected = [
fs_sanitize_path(fs_join(target, p)) for p in expected
]
assert sorted(output) == sorted(prefixed_expected)
try:
fs.rm(target, recursive=True)
except FileNotFoundError:
pass
def test_copy_list_of_files_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
fs_target,
supports_empty_directories,
):
# Copy scenario 2a
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
dummy = fs_join(target, "dummy")
fs.touch(dummy)
assert fs.isdir(target)
source_files = [
fs_join(source, "file1"),
fs_join(source, "file2"),
fs_join(source, "subdir", "subfile1"),
]
for target_slash in [False, True]:
t = target + "/" if target_slash else target
fs.cp(source_files, t)
assert fs.isfile(fs_join(target, "file1"))
assert fs.isfile(fs_join(target, "file2"))
assert fs.isfile(fs_join(target, "subfile1"))
fs.rm(
[
fs_join(target, "file1"),
fs_join(target, "file2"),
fs_join(target, "subfile1"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
def test_copy_list_of_files_to_new_directory(
self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
):
# Copy scenario 2b
source = fs_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
source_files = [
fs_join(source, "file1"),
fs_join(source, "file2"),
fs_join(source, "subdir", "subfile1"),
]
fs.cp(source_files, fs_join(target, "newdir") + "/") # Note trailing slash
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "file1"))
assert fs.isfile(fs_join(target, "newdir", "file2"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
def test_copy_two_files_new_directory(
self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target
):
# This is a duplicate of test_copy_list_of_files_to_new_directory and
# can eventually be removed.
source = fs_bulk_operations_scenario_0
target = fs_target
assert not fs.exists(target)
fs.cp([fs_join(source, "file1"), fs_join(source, "file2")], target)
assert fs.isdir(target)
assert fs.isfile(fs_join(target, "file1"))
assert fs.isfile(fs_join(target, "file2"))
def test_copy_directory_without_files_with_same_name_prefix(
self,
fs,
fs_join,
fs_target,
fs_dir_and_file_with_same_name_prefix,
supports_empty_directories,
):
# Create the test dirs
source = fs_dir_and_file_with_same_name_prefix
target = fs_target
# Test without glob
fs.cp(fs_join(source, "subdir"), target, recursive=True)
assert fs.isfile(fs_join(target, "subfile.txt"))
assert not fs.isfile(fs_join(target, "subdir.txt"))
fs.rm([fs_join(target, "subfile.txt")])
if supports_empty_directories:
assert fs.ls(target) == []
else:
assert not fs.exists(target)
# Test with glob
fs.cp(fs_join(source, "subdir*"), target, recursive=True)
assert fs.isdir(fs_join(target, "subdir"))
assert fs.isfile(fs_join(target, "subdir", "subfile.txt"))
assert fs.isfile(fs_join(target, "subdir.txt"))
def test_copy_with_source_and_destination_as_list(
self, fs, fs_target, fs_join, fs_10_files_with_hashed_names
):
# Create the test dir
source = fs_10_files_with_hashed_names
target = fs_target
# Create list of files for source and destination
source_files = []
destination_files = []
for i in range(10):
hashed_i = md5(str(i).encode("utf-8")).hexdigest()
source_files.append(fs_join(source, f"{hashed_i}.txt"))
destination_files.append(fs_join(target, f"{hashed_i}.txt"))
# Copy and assert order was kept
fs.copy(path1=source_files, path2=destination_files)
for i in range(10):
file_content = fs.cat(destination_files[i]).decode("utf-8")
assert file_content == str(i)
filesystem_spec-2025.3.2/fsspec/tests/abstract/get.py 0000664 0000000 0000000 00000050423 14772531734 0022545 0 ustar 00root root 0000000 0000000 from hashlib import md5
from itertools import product
import pytest
from fsspec.implementations.local import make_path_posix
from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
class AbstractGetTests:
def test_get_file_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1a
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
assert local_fs.isdir(target)
target_file2 = local_join(target, "file2")
target_subfile1 = local_join(target, "subfile1")
# Copy from source directory
fs.get(fs_join(source, "file2"), target)
assert local_fs.isfile(target_file2)
# Copy from sub directory
fs.get(fs_join(source, "subdir", "subfile1"), target)
assert local_fs.isfile(target_subfile1)
# Remove copied files
local_fs.rm([target_file2, target_subfile1])
assert not local_fs.exists(target_file2)
assert not local_fs.exists(target_subfile1)
# Repeat with trailing slash on target
fs.get(fs_join(source, "file2"), target + "/")
assert local_fs.isdir(target)
assert local_fs.isfile(target_file2)
fs.get(fs_join(source, "subdir", "subfile1"), target + "/")
assert local_fs.isfile(target_subfile1)
def test_get_file_to_new_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1b
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
fs.get(
fs_join(source, "subdir", "subfile1"), local_join(target, "newdir/")
) # Note trailing slash
assert local_fs.isdir(target)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
def test_get_file_to_file_in_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1c
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
fs.get(fs_join(source, "subdir", "subfile1"), local_join(target, "newfile"))
assert local_fs.isfile(local_join(target, "newfile"))
def test_get_file_to_file_in_new_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1d
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
fs.get(
fs_join(source, "subdir", "subfile1"),
local_join(target, "newdir", "newfile"),
)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "newfile"))
def test_get_directory_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1e
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
assert local_fs.isdir(target)
for source_slash, target_slash in zip([False, True], [False, True]):
s = fs_join(source, "subdir")
if source_slash:
s += "/"
t = target + "/" if target_slash else target
# Without recursive does nothing
fs.get(s, t)
assert local_fs.ls(target) == []
# With recursive
fs.get(s, t, recursive=True)
if source_slash:
assert local_fs.isfile(local_join(target, "subfile1"))
assert local_fs.isfile(local_join(target, "subfile2"))
assert local_fs.isdir(local_join(target, "nesteddir"))
assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile"))
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(
[
local_join(target, "subfile1"),
local_join(target, "subfile2"),
local_join(target, "nesteddir"),
],
recursive=True,
)
else:
assert local_fs.isdir(local_join(target, "subdir"))
assert local_fs.isfile(local_join(target, "subdir", "subfile1"))
assert local_fs.isfile(local_join(target, "subdir", "subfile2"))
assert local_fs.isdir(local_join(target, "subdir", "nesteddir"))
assert local_fs.isfile(
local_join(target, "subdir", "nesteddir", "nestedfile")
)
local_fs.rm(local_join(target, "subdir"), recursive=True)
assert local_fs.ls(target) == []
# Limit recursive by maxdepth
fs.get(s, t, recursive=True, maxdepth=1)
if source_slash:
assert local_fs.isfile(local_join(target, "subfile1"))
assert local_fs.isfile(local_join(target, "subfile2"))
assert not local_fs.exists(local_join(target, "nesteddir"))
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(
[
local_join(target, "subfile1"),
local_join(target, "subfile2"),
],
recursive=True,
)
else:
assert local_fs.isdir(local_join(target, "subdir"))
assert local_fs.isfile(local_join(target, "subdir", "subfile1"))
assert local_fs.isfile(local_join(target, "subdir", "subfile2"))
assert not local_fs.exists(local_join(target, "subdir", "nesteddir"))
local_fs.rm(local_join(target, "subdir"), recursive=True)
assert local_fs.ls(target) == []
def test_get_directory_to_new_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1f
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
for source_slash, target_slash in zip([False, True], [False, True]):
s = fs_join(source, "subdir")
if source_slash:
s += "/"
t = local_join(target, "newdir")
if target_slash:
t += "/"
# Without recursive does nothing
fs.get(s, t)
assert local_fs.ls(target) == []
# With recursive
fs.get(s, t, recursive=True)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
assert local_fs.isdir(local_join(target, "newdir", "nesteddir"))
assert local_fs.isfile(
local_join(target, "newdir", "nesteddir", "nestedfile")
)
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(local_join(target, "newdir"), recursive=True)
assert local_fs.ls(target) == []
# Limit recursive by maxdepth
fs.get(s, t, recursive=True, maxdepth=1)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(local_join(target, "newdir"), recursive=True)
assert not local_fs.exists(local_join(target, "newdir"))
def test_get_glob_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1g
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
for target_slash in [False, True]:
t = target + "/" if target_slash else target
# Without recursive
fs.get(fs_join(source, "subdir", "*"), t)
assert local_fs.isfile(local_join(target, "subfile1"))
assert local_fs.isfile(local_join(target, "subfile2"))
assert not local_fs.isdir(local_join(target, "nesteddir"))
assert not local_fs.exists(local_join(target, "nesteddir", "nestedfile"))
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(
[
local_join(target, "subfile1"),
local_join(target, "subfile2"),
],
recursive=True,
)
assert local_fs.ls(target) == []
# With recursive
for glob, recursive in zip(["*", "**"], [True, False]):
fs.get(fs_join(source, "subdir", glob), t, recursive=recursive)
assert local_fs.isfile(local_join(target, "subfile1"))
assert local_fs.isfile(local_join(target, "subfile2"))
assert local_fs.isdir(local_join(target, "nesteddir"))
assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile"))
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(
[
local_join(target, "subfile1"),
local_join(target, "subfile2"),
local_join(target, "nesteddir"),
],
recursive=True,
)
assert local_fs.ls(target) == []
# Limit recursive by maxdepth
fs.get(
fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
)
assert local_fs.isfile(local_join(target, "subfile1"))
assert local_fs.isfile(local_join(target, "subfile2"))
assert not local_fs.exists(local_join(target, "nesteddir"))
assert not local_fs.exists(local_join(target, "subdir"))
local_fs.rm(
[
local_join(target, "subfile1"),
local_join(target, "subfile2"),
],
recursive=True,
)
assert local_fs.ls(target) == []
def test_get_glob_to_new_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 1h
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
for target_slash in [False, True]:
t = fs_join(target, "newdir")
if target_slash:
t += "/"
# Without recursive
fs.get(fs_join(source, "subdir", "*"), t)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
assert not local_fs.exists(
local_join(target, "newdir", "nesteddir", "nestedfile")
)
assert not local_fs.exists(local_join(target, "subdir"))
assert not local_fs.exists(local_join(target, "newdir", "subdir"))
local_fs.rm(local_join(target, "newdir"), recursive=True)
assert local_fs.ls(target) == []
# With recursive
for glob, recursive in zip(["*", "**"], [True, False]):
fs.get(fs_join(source, "subdir", glob), t, recursive=recursive)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
assert local_fs.isdir(local_join(target, "newdir", "nesteddir"))
assert local_fs.isfile(
local_join(target, "newdir", "nesteddir", "nestedfile")
)
assert not local_fs.exists(local_join(target, "subdir"))
assert not local_fs.exists(local_join(target, "newdir", "subdir"))
local_fs.rm(local_join(target, "newdir"), recursive=True)
assert not local_fs.exists(local_join(target, "newdir"))
# Limit recursive by maxdepth
fs.get(
fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1
)
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
assert local_fs.isfile(local_join(target, "newdir", "subfile2"))
assert not local_fs.exists(local_join(target, "newdir", "nesteddir"))
assert not local_fs.exists(local_join(target, "subdir"))
assert not local_fs.exists(local_join(target, "newdir", "subdir"))
local_fs.rm(local_fs.ls(target, detail=False), recursive=True)
assert not local_fs.exists(local_join(target, "newdir"))
@pytest.mark.parametrize(
GLOB_EDGE_CASES_TESTS["argnames"],
GLOB_EDGE_CASES_TESTS["argvalues"],
)
def test_get_glob_edge_cases(
self,
path,
recursive,
maxdepth,
expected,
fs,
fs_join,
fs_glob_edge_cases_files,
local_fs,
local_join,
local_target,
):
# Copy scenario 1g
source = fs_glob_edge_cases_files
target = local_target
for new_dir, target_slash in product([True, False], [True, False]):
local_fs.mkdir(target)
t = local_join(target, "newdir") if new_dir else target
t = t + "/" if target_slash else t
fs.get(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
output = local_fs.find(target)
if new_dir:
prefixed_expected = [
make_path_posix(local_join(target, "newdir", p)) for p in expected
]
else:
prefixed_expected = [
make_path_posix(local_join(target, p)) for p in expected
]
assert sorted(output) == sorted(prefixed_expected)
try:
local_fs.rm(target, recursive=True)
except FileNotFoundError:
pass
def test_get_list_of_files_to_existing_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 2a
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
source_files = [
fs_join(source, "file1"),
fs_join(source, "file2"),
fs_join(source, "subdir", "subfile1"),
]
for target_slash in [False, True]:
t = target + "/" if target_slash else target
fs.get(source_files, t)
assert local_fs.isfile(local_join(target, "file1"))
assert local_fs.isfile(local_join(target, "file2"))
assert local_fs.isfile(local_join(target, "subfile1"))
local_fs.rm(
[
local_join(target, "file1"),
local_join(target, "file2"),
local_join(target, "subfile1"),
],
recursive=True,
)
assert local_fs.ls(target) == []
def test_get_list_of_files_to_new_directory(
self,
fs,
fs_join,
fs_bulk_operations_scenario_0,
local_fs,
local_join,
local_target,
):
# Copy scenario 2b
source = fs_bulk_operations_scenario_0
target = local_target
local_fs.mkdir(target)
source_files = [
fs_join(source, "file1"),
fs_join(source, "file2"),
fs_join(source, "subdir", "subfile1"),
]
fs.get(source_files, local_join(target, "newdir") + "/") # Note trailing slash
assert local_fs.isdir(local_join(target, "newdir"))
assert local_fs.isfile(local_join(target, "newdir", "file1"))
assert local_fs.isfile(local_join(target, "newdir", "file2"))
assert local_fs.isfile(local_join(target, "newdir", "subfile1"))
def test_get_directory_recursive(
self, fs, fs_join, fs_path, local_fs, local_join, local_target
):
# https://github.com/fsspec/filesystem_spec/issues/1062
# Recursive cp/get/put of source directory into non-existent target directory.
src = fs_join(fs_path, "src")
src_file = fs_join(src, "file")
fs.mkdir(src)
fs.touch(src_file)
target = local_target
# get without slash
assert not local_fs.exists(target)
for loop in range(2):
fs.get(src, target, recursive=True)
assert local_fs.isdir(target)
if loop == 0:
assert local_fs.isfile(local_join(target, "file"))
assert not local_fs.exists(local_join(target, "src"))
else:
assert local_fs.isfile(local_join(target, "file"))
assert local_fs.isdir(local_join(target, "src"))
assert local_fs.isfile(local_join(target, "src", "file"))
local_fs.rm(target, recursive=True)
# get with slash
assert not local_fs.exists(target)
for loop in range(2):
fs.get(src + "/", target, recursive=True)
assert local_fs.isdir(target)
assert local_fs.isfile(local_join(target, "file"))
assert not local_fs.exists(local_join(target, "src"))
def test_get_directory_without_files_with_same_name_prefix(
self,
fs,
fs_join,
local_fs,
local_join,
local_target,
fs_dir_and_file_with_same_name_prefix,
):
# Create the test dirs
source = fs_dir_and_file_with_same_name_prefix
target = local_target
# Test without glob
fs.get(fs_join(source, "subdir"), target, recursive=True)
assert local_fs.isfile(local_join(target, "subfile.txt"))
assert not local_fs.isfile(local_join(target, "subdir.txt"))
local_fs.rm([local_join(target, "subfile.txt")])
assert local_fs.ls(target) == []
# Test with glob
fs.get(fs_join(source, "subdir*"), target, recursive=True)
assert local_fs.isdir(local_join(target, "subdir"))
assert local_fs.isfile(local_join(target, "subdir", "subfile.txt"))
assert local_fs.isfile(local_join(target, "subdir.txt"))
def test_get_with_source_and_destination_as_list(
self,
fs,
fs_join,
local_fs,
local_join,
local_target,
fs_10_files_with_hashed_names,
):
# Create the test dir
source = fs_10_files_with_hashed_names
target = local_target
# Create list of files for source and destination
source_files = []
destination_files = []
for i in range(10):
hashed_i = md5(str(i).encode("utf-8")).hexdigest()
source_files.append(fs_join(source, f"{hashed_i}.txt"))
destination_files.append(
make_path_posix(local_join(target, f"{hashed_i}.txt"))
)
# Copy and assert order was kept
fs.get(rpath=source_files, lpath=destination_files)
for i in range(10):
file_content = local_fs.cat(destination_files[i]).decode("utf-8")
assert file_content == str(i)
filesystem_spec-2025.3.2/fsspec/tests/abstract/mv.py 0000664 0000000 0000000 00000003676 14772531734 0022420 0 ustar 00root root 0000000 0000000 import os
import pytest
import fsspec
def test_move_raises_error_with_tmpdir(tmpdir):
# Create a file in the temporary directory
source = tmpdir.join("source_file.txt")
source.write("content")
# Define a destination that simulates a protected or invalid path
destination = tmpdir.join("non_existent_directory/destination_file.txt")
# Instantiate the filesystem (assuming the local file system interface)
fs = fsspec.filesystem("file")
# Use the actual file paths as string
with pytest.raises(FileNotFoundError):
fs.mv(str(source), str(destination))
@pytest.mark.parametrize("recursive", (True, False))
def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir):
# Create a file in the temporary directory
source = tmpdir.join("source_file.txt")
source.write("content")
# Create a protected directory (non-writable)
protected_dir = tmpdir.mkdir("protected_directory")
protected_path = str(protected_dir)
# Set the directory to read-only
if os.name == "nt":
os.system(f'icacls "{protected_path}" /deny Everyone:(W)')
else:
os.chmod(protected_path, 0o555) # Sets the directory to read-only
# Define a destination inside the protected directory
destination = protected_dir.join("destination_file.txt")
# Instantiate the filesystem (assuming the local file system interface)
fs = fsspec.filesystem("file")
# Try to move the file to the read-only directory, expecting a permission error
with pytest.raises(PermissionError):
fs.mv(str(source), str(destination), recursive=recursive)
# Assert the file was not created in the destination
assert not os.path.exists(destination)
# Cleanup: Restore permissions so the directory can be cleaned up
if os.name == "nt":
os.system(f'icacls "{protected_path}" /remove:d Everyone')
else:
os.chmod(protected_path, 0o755) # Restore write permission for cleanup
filesystem_spec-2025.3.2/fsspec/tests/abstract/open.py 0000664 0000000 0000000 00000000511 14772531734 0022720 0 ustar 00root root 0000000 0000000 import pytest
class AbstractOpenTests:
def test_open_exclusive(self, fs, fs_target):
with fs.open(fs_target, "wb") as f:
f.write(b"data")
with fs.open(fs_target, "rb") as f:
assert f.read() == b"data"
with pytest.raises(FileExistsError):
fs.open(fs_target, "xb")
filesystem_spec-2025.3.2/fsspec/tests/abstract/pipe.py 0000664 0000000 0000000 00000000622 14772531734 0022717 0 ustar 00root root 0000000 0000000 import pytest
class AbstractPipeTests:
def test_pipe_exclusive(self, fs, fs_target):
fs.pipe_file(fs_target, b"data")
assert fs.cat_file(fs_target) == b"data"
with pytest.raises(FileExistsError):
fs.pipe_file(fs_target, b"data", mode="create")
fs.pipe_file(fs_target, b"new data", mode="overwrite")
assert fs.cat_file(fs_target) == b"new data"
filesystem_spec-2025.3.2/fsspec/tests/abstract/put.py 0000664 0000000 0000000 00000051321 14772531734 0022574 0 ustar 00root root 0000000 0000000 from hashlib import md5
from itertools import product
import pytest
from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS
class AbstractPutTests:
def test_put_file_to_existing_directory(
self,
fs,
fs_join,
fs_target,
local_join,
local_bulk_operations_scenario_0,
supports_empty_directories,
):
# Copy scenario 1a
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
fs.touch(fs_join(target, "dummy"))
assert fs.isdir(target)
target_file2 = fs_join(target, "file2")
target_subfile1 = fs_join(target, "subfile1")
# Copy from source directory
fs.put(local_join(source, "file2"), target)
assert fs.isfile(target_file2)
# Copy from sub directory
fs.put(local_join(source, "subdir", "subfile1"), target)
assert fs.isfile(target_subfile1)
# Remove copied files
fs.rm([target_file2, target_subfile1])
assert not fs.exists(target_file2)
assert not fs.exists(target_subfile1)
# Repeat with trailing slash on target
fs.put(local_join(source, "file2"), target + "/")
assert fs.isdir(target)
assert fs.isfile(target_file2)
fs.put(local_join(source, "subdir", "subfile1"), target + "/")
assert fs.isfile(target_subfile1)
def test_put_file_to_new_directory(
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
):
# Copy scenario 1b
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
fs.put(
local_join(source, "subdir", "subfile1"), fs_join(target, "newdir/")
) # Note trailing slash
assert fs.isdir(target)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
def test_put_file_to_file_in_existing_directory(
self,
fs,
fs_join,
fs_target,
local_join,
supports_empty_directories,
local_bulk_operations_scenario_0,
):
# Copy scenario 1c
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
fs.touch(fs_join(target, "dummy"))
assert fs.isdir(target)
fs.put(local_join(source, "subdir", "subfile1"), fs_join(target, "newfile"))
assert fs.isfile(fs_join(target, "newfile"))
def test_put_file_to_file_in_new_directory(
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
):
# Copy scenario 1d
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
fs.put(
local_join(source, "subdir", "subfile1"),
fs_join(target, "newdir", "newfile"),
)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "newfile"))
def test_put_directory_to_existing_directory(
self,
fs,
fs_join,
fs_target,
local_bulk_operations_scenario_0,
supports_empty_directories,
):
# Copy scenario 1e
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
dummy = fs_join(target, "dummy")
fs.touch(dummy)
assert fs.isdir(target)
for source_slash, target_slash in zip([False, True], [False, True]):
s = fs_join(source, "subdir")
if source_slash:
s += "/"
t = target + "/" if target_slash else target
# Without recursive does nothing
fs.put(s, t)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# With recursive
fs.put(s, t, recursive=True)
if source_slash:
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert fs.isdir(fs_join(target, "nesteddir"))
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
fs_join(target, "nesteddir"),
],
recursive=True,
)
else:
assert fs.isdir(fs_join(target, "subdir"))
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
assert fs.isfile(fs_join(target, "subdir", "subfile2"))
assert fs.isdir(fs_join(target, "subdir", "nesteddir"))
assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile"))
fs.rm(fs_join(target, "subdir"), recursive=True)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# Limit recursive by maxdepth
fs.put(s, t, recursive=True, maxdepth=1)
if source_slash:
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert not fs.exists(fs_join(target, "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
],
recursive=True,
)
else:
assert fs.isdir(fs_join(target, "subdir"))
assert fs.isfile(fs_join(target, "subdir", "subfile1"))
assert fs.isfile(fs_join(target, "subdir", "subfile2"))
assert not fs.exists(fs_join(target, "subdir", "nesteddir"))
fs.rm(fs_join(target, "subdir"), recursive=True)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
def test_put_directory_to_new_directory(
self,
fs,
fs_join,
fs_target,
local_bulk_operations_scenario_0,
supports_empty_directories,
):
# Copy scenario 1f
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
for source_slash, target_slash in zip([False, True], [False, True]):
s = fs_join(source, "subdir")
if source_slash:
s += "/"
t = fs_join(target, "newdir")
if target_slash:
t += "/"
# Without recursive does nothing
fs.put(s, t)
if supports_empty_directories:
assert fs.ls(target) == []
else:
with pytest.raises(FileNotFoundError):
fs.ls(target)
# With recursive
fs.put(s, t, recursive=True)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
# Limit recursive by maxdepth
fs.put(s, t, recursive=True, maxdepth=1)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
def test_put_glob_to_existing_directory(
self,
fs,
fs_join,
fs_target,
local_join,
supports_empty_directories,
local_bulk_operations_scenario_0,
):
# Copy scenario 1g
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
dummy = fs_join(target, "dummy")
fs.touch(dummy)
assert fs.isdir(target)
for target_slash in [False, True]:
t = target + "/" if target_slash else target
# Without recursive
fs.put(local_join(source, "subdir", "*"), t)
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert not fs.isdir(fs_join(target, "nesteddir"))
assert not fs.exists(fs_join(target, "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# With recursive
for glob, recursive in zip(["*", "**"], [True, False]):
fs.put(local_join(source, "subdir", glob), t, recursive=recursive)
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert fs.isdir(fs_join(target, "nesteddir"))
assert fs.isfile(fs_join(target, "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
fs_join(target, "nesteddir"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
# Limit recursive by maxdepth
fs.put(
local_join(source, "subdir", glob),
t,
recursive=recursive,
maxdepth=1,
)
assert fs.isfile(fs_join(target, "subfile1"))
assert fs.isfile(fs_join(target, "subfile2"))
assert not fs.exists(fs_join(target, "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
fs.rm(
[
fs_join(target, "subfile1"),
fs_join(target, "subfile2"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
def test_put_glob_to_new_directory(
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
):
# Copy scenario 1h
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
for target_slash in [False, True]:
t = fs_join(target, "newdir")
if target_slash:
t += "/"
# Without recursive
fs.put(local_join(source, "subdir", "*"), t)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
assert not fs.exists(fs_join(target, "newdir", "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
# With recursive
for glob, recursive in zip(["*", "**"], [True, False]):
fs.put(local_join(source, "subdir", glob), t, recursive=recursive)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert fs.isdir(fs_join(target, "newdir", "nesteddir"))
assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile"))
assert not fs.exists(fs_join(target, "subdir"))
assert not fs.exists(fs_join(target, "newdir", "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
# Limit recursive by maxdepth
fs.put(
local_join(source, "subdir", glob),
t,
recursive=recursive,
maxdepth=1,
)
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
assert fs.isfile(fs_join(target, "newdir", "subfile2"))
assert not fs.exists(fs_join(target, "newdir", "nesteddir"))
assert not fs.exists(fs_join(target, "subdir"))
assert not fs.exists(fs_join(target, "newdir", "subdir"))
fs.rm(fs_join(target, "newdir"), recursive=True)
assert not fs.exists(fs_join(target, "newdir"))
@pytest.mark.parametrize(
GLOB_EDGE_CASES_TESTS["argnames"],
GLOB_EDGE_CASES_TESTS["argvalues"],
)
def test_put_glob_edge_cases(
self,
path,
recursive,
maxdepth,
expected,
fs,
fs_join,
fs_target,
local_glob_edge_cases_files,
local_join,
fs_sanitize_path,
):
# Copy scenario 1g
source = local_glob_edge_cases_files
target = fs_target
for new_dir, target_slash in product([True, False], [True, False]):
fs.mkdir(target)
t = fs_join(target, "newdir") if new_dir else target
t = t + "/" if target_slash else t
fs.put(local_join(source, path), t, recursive=recursive, maxdepth=maxdepth)
output = fs.find(target)
if new_dir:
prefixed_expected = [
fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected
]
else:
prefixed_expected = [
fs_sanitize_path(fs_join(target, p)) for p in expected
]
assert sorted(output) == sorted(prefixed_expected)
try:
fs.rm(target, recursive=True)
except FileNotFoundError:
pass
def test_put_list_of_files_to_existing_directory(
self,
fs,
fs_join,
fs_target,
local_join,
local_bulk_operations_scenario_0,
supports_empty_directories,
):
# Copy scenario 2a
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
if not supports_empty_directories:
# Force target directory to exist by adding a dummy file
dummy = fs_join(target, "dummy")
fs.touch(dummy)
assert fs.isdir(target)
source_files = [
local_join(source, "file1"),
local_join(source, "file2"),
local_join(source, "subdir", "subfile1"),
]
for target_slash in [False, True]:
t = target + "/" if target_slash else target
fs.put(source_files, t)
assert fs.isfile(fs_join(target, "file1"))
assert fs.isfile(fs_join(target, "file2"))
assert fs.isfile(fs_join(target, "subfile1"))
fs.rm(
[
fs_join(target, "file1"),
fs_join(target, "file2"),
fs_join(target, "subfile1"),
],
recursive=True,
)
assert fs.ls(target, detail=False) == (
[] if supports_empty_directories else [dummy]
)
def test_put_list_of_files_to_new_directory(
self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0
):
# Copy scenario 2b
source = local_bulk_operations_scenario_0
target = fs_target
fs.mkdir(target)
source_files = [
local_join(source, "file1"),
local_join(source, "file2"),
local_join(source, "subdir", "subfile1"),
]
fs.put(source_files, fs_join(target, "newdir") + "/") # Note trailing slash
assert fs.isdir(fs_join(target, "newdir"))
assert fs.isfile(fs_join(target, "newdir", "file1"))
assert fs.isfile(fs_join(target, "newdir", "file2"))
assert fs.isfile(fs_join(target, "newdir", "subfile1"))
def test_put_directory_recursive(
self, fs, fs_join, fs_target, local_fs, local_join, local_path
):
# https://github.com/fsspec/filesystem_spec/issues/1062
# Recursive cp/get/put of source directory into non-existent target directory.
src = local_join(local_path, "src")
src_file = local_join(src, "file")
local_fs.mkdir(src)
local_fs.touch(src_file)
target = fs_target
# put without slash
assert not fs.exists(target)
for loop in range(2):
fs.put(src, target, recursive=True)
assert fs.isdir(target)
if loop == 0:
assert fs.isfile(fs_join(target, "file"))
assert not fs.exists(fs_join(target, "src"))
else:
assert fs.isfile(fs_join(target, "file"))
assert fs.isdir(fs_join(target, "src"))
assert fs.isfile(fs_join(target, "src", "file"))
fs.rm(target, recursive=True)
# put with slash
assert not fs.exists(target)
for loop in range(2):
fs.put(src + "/", target, recursive=True)
assert fs.isdir(target)
assert fs.isfile(fs_join(target, "file"))
assert not fs.exists(fs_join(target, "src"))
def test_put_directory_without_files_with_same_name_prefix(
self,
fs,
fs_join,
fs_target,
local_join,
local_dir_and_file_with_same_name_prefix,
supports_empty_directories,
):
# Create the test dirs
source = local_dir_and_file_with_same_name_prefix
target = fs_target
# Test without glob
fs.put(local_join(source, "subdir"), fs_target, recursive=True)
assert fs.isfile(fs_join(fs_target, "subfile.txt"))
assert not fs.isfile(fs_join(fs_target, "subdir.txt"))
fs.rm([fs_join(target, "subfile.txt")])
if supports_empty_directories:
assert fs.ls(target) == []
else:
assert not fs.exists(target)
# Test with glob
fs.put(local_join(source, "subdir*"), fs_target, recursive=True)
assert fs.isdir(fs_join(fs_target, "subdir"))
assert fs.isfile(fs_join(fs_target, "subdir", "subfile.txt"))
assert fs.isfile(fs_join(fs_target, "subdir.txt"))
def test_copy_with_source_and_destination_as_list(
self, fs, fs_target, fs_join, local_join, local_10_files_with_hashed_names
):
# Create the test dir
source = local_10_files_with_hashed_names
target = fs_target
# Create list of files for source and destination
source_files = []
destination_files = []
for i in range(10):
hashed_i = md5(str(i).encode("utf-8")).hexdigest()
source_files.append(local_join(source, f"{hashed_i}.txt"))
destination_files.append(fs_join(target, f"{hashed_i}.txt"))
# Copy and assert order was kept
fs.put(lpath=source_files, rpath=destination_files)
for i in range(10):
file_content = fs.cat(destination_files[i]).decode("utf-8")
assert file_content == str(i)
filesystem_spec-2025.3.2/fsspec/tests/conftest.py 0000664 0000000 0000000 00000015072 14772531734 0022011 0 ustar 00root root 0000000 0000000 import contextlib
import gzip
import json
import os
import threading
from collections import ChainMap
from http.server import BaseHTTPRequestHandler, HTTPServer
from types import SimpleNamespace
import pytest
requests = pytest.importorskip("requests")
data = b"\n".join([b"some test data"] * 1000)
listing = open(
os.path.join(os.path.dirname(__file__), "data", "listing.html"), "rb"
).read()
win = os.name == "nt"
def _make_realfile(baseurl):
return f"{baseurl}/index/realfile"
def _make_index_listing(baseurl):
realfile = _make_realfile(baseurl)
return b'Link' % realfile.encode()
def _make_listing(*paths):
def _make_listing_port(baseurl):
return "\n".join(
f'Link_{i}' for i, f in enumerate(paths)
).encode()
return _make_listing_port
@pytest.fixture
def reset_files():
yield
# Reset the newly added files after the
# test is completed.
HTTPTestHandler.dynamic_files.clear()
class HTTPTestHandler(BaseHTTPRequestHandler):
static_files = {
"/index/realfile": data,
"/index/otherfile": data,
"/index": _make_index_listing,
"/data/20020401": listing,
"/simple/": _make_listing("/simple/file", "/simple/dir/"),
"/simple/file": data,
"/simple/dir/": _make_listing("/simple/dir/file"),
"/simple/dir/file": data,
}
dynamic_files = {}
files = ChainMap(dynamic_files, static_files)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _respond(self, code=200, headers=None, data=b""):
headers = headers or {}
headers.update({"User-Agent": "test"})
self.send_response(code)
for k, v in headers.items():
self.send_header(k, str(v))
self.end_headers()
if data:
self.wfile.write(data)
def do_GET(self):
baseurl = f"http://127.0.0.1:{self.server.server_port}"
file_path = self.path
if file_path.endswith("/") and file_path.rstrip("/") in self.files:
file_path = file_path.rstrip("/")
file_data = self.files.get(file_path)
if callable(file_data):
file_data = file_data(baseurl)
if "give_path" in self.headers:
return self._respond(200, data=json.dumps({"path": self.path}).encode())
if "redirect" in self.headers and file_path != "/index/realfile":
new_url = _make_realfile(baseurl)
return self._respond(301, {"Location": new_url})
if file_data is None:
return self._respond(404)
status = 200
content_range = f"bytes 0-{len(file_data) - 1}/{len(file_data)}"
if ("Range" in self.headers) and ("ignore_range" not in self.headers):
ran = self.headers["Range"]
b, ran = ran.split("=")
start, end = ran.split("-")
if start:
content_range = f"bytes {start}-{end}/{len(file_data)}"
file_data = file_data[int(start) : (int(end) + 1) if end else None]
else:
# suffix only
l = len(file_data)
content_range = f"bytes {l - int(end)}-{l - 1}/{l}"
file_data = file_data[-int(end) :]
if "use_206" in self.headers:
status = 206
if "give_length" in self.headers:
if "gzip_encoding" in self.headers:
file_data = gzip.compress(file_data)
response_headers = {
"Content-Length": len(file_data),
"Content-Encoding": "gzip",
}
else:
response_headers = {"Content-Length": len(file_data)}
self._respond(status, response_headers, file_data)
elif "give_range" in self.headers:
self._respond(status, {"Content-Range": content_range}, file_data)
elif "give_mimetype" in self.headers:
self._respond(
status, {"Content-Type": "text/html; charset=utf-8"}, file_data
)
else:
self._respond(status, data=file_data)
def do_POST(self):
length = self.headers.get("Content-Length")
file_path = self.path.rstrip("/")
if length is None:
assert self.headers.get("Transfer-Encoding") == "chunked"
self.files[file_path] = b"".join(self.read_chunks())
else:
self.files[file_path] = self.rfile.read(int(length))
self._respond(200)
do_PUT = do_POST
def read_chunks(self):
length = -1
while length != 0:
line = self.rfile.readline().strip()
if len(line) == 0:
length = 0
else:
length = int(line, 16)
yield self.rfile.read(length)
self.rfile.readline()
def do_HEAD(self):
r_headers = {}
if "head_not_auth" in self.headers:
r_headers["Content-Length"] = 123
return self._respond(403, r_headers, b"not authorized for HEAD request")
elif "head_ok" not in self.headers:
return self._respond(405)
file_path = self.path.rstrip("/")
file_data = self.files.get(file_path)
if file_data is None:
return self._respond(404)
if ("give_length" in self.headers) or ("head_give_length" in self.headers):
if "zero_length" in self.headers:
r_headers["Content-Length"] = 0
elif "gzip_encoding" in self.headers:
file_data = gzip.compress(file_data)
r_headers["Content-Encoding"] = "gzip"
r_headers["Content-Length"] = len(file_data)
else:
r_headers["Content-Length"] = len(file_data)
elif "give_range" in self.headers:
r_headers["Content-Range"] = f"0-{len(file_data) - 1}/{len(file_data)}"
elif "give_etag" in self.headers:
r_headers["ETag"] = "xxx"
if self.headers.get("accept_range") == "none":
r_headers["Accept-Ranges"] = "none"
self._respond(200, r_headers)
@contextlib.contextmanager
def serve():
server_address = ("", 0)
httpd = HTTPServer(server_address, HTTPTestHandler)
th = threading.Thread(target=httpd.serve_forever)
th.daemon = True
th.start()
try:
yield f"http://127.0.0.1:{httpd.server_port}"
finally:
httpd.socket.close()
httpd.shutdown()
th.join()
@pytest.fixture(scope="module")
def server():
with serve() as s:
server = SimpleNamespace(address=s, realfile=_make_realfile(s))
yield server
filesystem_spec-2025.3.2/fsspec/tests/data/ 0000775 0000000 0000000 00000000000 14772531734 0020516 5 ustar 00root root 0000000 0000000 filesystem_spec-2025.3.2/fsspec/tests/data/listing.html 0000664 0000000 0000000 00000003070 14772531734 0023055 0 ustar 00root root 0000000 0000000 \nnasagrace.unl.edu - /data/20020401/nasagrace.unl.edu - /data/20020401/
\n\n[To Parent Directory]
1/27/2020 9:54 AM 1194073 GRACE_GWS_20020401.pdf
1/27/2020 9:54 AM 380043 GRACE_GWS_20020401.png
1/27/2020 9:54 AM 1192987 GRACE_RTZSM_20020401.pdf
1/27/2020 9:54 AM 384342 GRACE_RTZSM_20020401.png
1/27/2020 9:55 AM 1202046 GRACE_SFSM_20020401.pdf
1/27/2020 9:55 AM 387932 GRACE_SFSM_20020401.png
1/27/2020 9:54 AM 4975980 GRACEDADM_CLSM0125US_7D.A20020401.030.nc4
1/27/2020 9:54 AM 345640 gws_perc_0125deg_US_20020401.img
1/27/2020 9:54 AM 2272 gws_perc_0125deg_US_20020401.img.aux.xml
1/27/2020 9:54 AM 5678 gws_perc_0125deg_US_20020401.img.xml
1/27/2020 9:54 AM 136081 gws_perc_0125deg_US_20020401.rrd
1/27/2020 9:54 AM 83
filesystem_spec-2025.3.2/fsspec/tests/test_api.py 0000664 0000000 0000000 00000034421 14772531734 0021773 0 ustar 00root root 0000000 0000000 """Tests the spec, using memoryfs"""
import contextlib
import os
import pickle
import tempfile
from unittest.mock import Mock
import pytest
import fsspec
from fsspec.implementations.memory import MemoryFile, MemoryFileSystem
def test_idempotent():
MemoryFileSystem.clear_instance_cache()
fs = MemoryFileSystem()
fs2 = MemoryFileSystem()
assert fs is fs2
assert MemoryFileSystem.current() is fs2
MemoryFileSystem.clear_instance_cache()
assert not MemoryFileSystem._cache
fs2 = MemoryFileSystem().current()
assert fs == fs2
def test_pickle():
fs = MemoryFileSystem()
fs2 = pickle.loads(pickle.dumps(fs))
assert fs == fs2
def test_class_methods():
assert MemoryFileSystem._strip_protocol("memory://stuff") == "/stuff"
assert MemoryFileSystem._strip_protocol("stuff") == "/stuff"
assert MemoryFileSystem._strip_protocol("other://stuff") == "other://stuff"
assert MemoryFileSystem._get_kwargs_from_urls("memory://user@thing") == {}
def test_multi(m):
m.pipe("/afile", b"data")
fs, token, paths = fsspec.core.get_fs_token_paths(["/afile", "/afile"])
assert len(paths) == 2
def test_get_put(tmpdir, m):
tmpdir = str(tmpdir)
fn = os.path.join(tmpdir, "one")
open(fn, "wb").write(b"one")
os.mkdir(os.path.join(tmpdir, "dir"))
fn2 = os.path.join(tmpdir, "dir", "two")
open(fn2, "wb").write(b"two")
fs = MemoryFileSystem()
fs.put(fn, "/afile")
assert fs.cat("/afile") == b"one"
fs.store["/bfile"] = MemoryFile(fs, "/bfile", b"data")
fn3 = os.path.join(tmpdir, "three")
fs.get("/bfile", fn3)
assert open(fn3, "rb").read() == b"data"
fs.put(tmpdir, "/more", recursive=True)
assert fs.find("/more") == ["/more/dir/two", "/more/one", "/more/three"]
@contextlib.contextmanager
def tmp_chdir(path):
curdir = os.getcwd()
os.chdir(path)
try:
yield
finally:
os.chdir(curdir)
with tmp_chdir(os.path.join(tmpdir, os.path.pardir)):
fs.put(os.path.basename(tmpdir), "/moretwo", recursive=True)
assert fs.find("/moretwo") == [
"/moretwo/dir/two",
"/moretwo/one",
"/moretwo/three",
]
with tmp_chdir(tmpdir):
fs.put(os.path.curdir, "/morethree", recursive=True)
assert fs.find("/morethree") == [
"/morethree/dir/two",
"/morethree/one",
"/morethree/three",
]
for f in [fn, fn2, fn3]:
os.remove(f)
os.rmdir(os.path.join(tmpdir, "dir"))
fs.get("/more/", tmpdir + "/", recursive=True)
assert open(fn3, "rb").read() == b"data"
assert open(fn, "rb").read() == b"one"
def test_du(m):
fs = MemoryFileSystem()
fs.store.update(
{
"/dir/afile": MemoryFile(fs, "/afile", b"a"),
"/dir/dirb/afile": MemoryFile(fs, "/afile", b"bb"),
"/dir/dirb/bfile": MemoryFile(fs, "/afile", b"ccc"),
}
)
assert fs.du("/dir") == 6
assert fs.du("/dir", total=False) == {
"/dir/afile": 1,
"/dir/dirb/afile": 2,
"/dir/dirb/bfile": 3,
}
assert fs.du("/dir", withdirs=True) == 6
assert fs.du("/dir", total=False, withdirs=True) == {
"/dir": 0,
"/dir/afile": 1,
"/dir/dirb": 0,
"/dir/dirb/afile": 2,
"/dir/dirb/bfile": 3,
}
with pytest.raises(ValueError):
assert fs.du("/dir", maxdepth=0) == 1
assert fs.du("/dir", total=False, withdirs=True, maxdepth=1) == {
"/dir": 0,
"/dir/afile": 1,
"/dir/dirb": 0,
}
# Size of file only.
assert fs.du("/dir/afile") == 1
assert fs.du("/dir/afile", withdirs=True) == 1
def test_head_tail(m):
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"I had a nice big cabbage")
assert fs.head("/myfile", 5) == b"I had"
assert fs.tail("/myfile", 7) == b"cabbage"
def test_move(m):
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"I had a nice big cabbage")
fs.move("/myfile", "/otherfile")
assert not fs.exists("/myfile")
assert fs.info("/otherfile")
assert isinstance(fs.ukey("/otherfile"), str)
def test_recursive_get_put(tmpdir, m):
fs = MemoryFileSystem()
os.makedirs(f"{tmpdir}/nest")
for file in ["one", "two", "nest/other"]:
with open(f"{tmpdir}/{file}", "wb") as f:
f.write(b"data")
fs.put(str(tmpdir), "test", recursive=True)
# get to directory with slash
d = tempfile.mkdtemp()
fs.get("test/", d, recursive=True)
for file in ["one", "two", "nest/other"]:
with open(f"{d}/{file}", "rb") as f:
assert f.read() == b"data"
# get to directory without slash
d = tempfile.mkdtemp()
fs.get("test", d, recursive=True)
for file in ["test/one", "test/two", "test/nest/other"]:
with open(f"{d}/{file}", "rb") as f:
assert f.read() == b"data"
def test_pipe_cat(m):
fs = MemoryFileSystem()
fs.pipe("afile", b"contents")
assert fs.cat("afile") == b"contents"
data = {"/bfile": b"more", "/cfile": b"stuff"}
fs.pipe(data)
assert fs.cat(list(data)) == data
def test_read_block_delimiter(m):
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"some\nlines\nof\ntext")
assert fs.read_block("/myfile", 0, 2, b"\n") == b"some\n"
assert fs.read_block("/myfile", 2, 6, b"\n") == b"lines\n"
assert fs.read_block("/myfile", 6, 2, b"\n") == b""
assert fs.read_block("/myfile", 2, 9, b"\n") == b"lines\nof\n"
assert fs.read_block("/myfile", 12, 6, b"\n") == b"text"
assert fs.read_block("/myfile", 0, None) == fs.cat("/myfile")
def test_open_text(m):
fs = MemoryFileSystem()
with fs.open("/myfile", "wb") as f:
f.write(b"some\nlines\nof\ntext")
f = fs.open("/myfile", "r", encoding="latin1")
assert f.encoding == "latin1"
def test_read_text(m):
with m.open("/myfile", "w", encoding="utf-8") as f:
f.write("some\nlines\nof\ntext")
assert m.read_text("/myfile", encoding="utf-8") == "some\nlines\nof\ntext"
def test_write_text(m):
m.write_text("/myfile", "some\nlines\nof\ntext", encoding="utf-8")
assert m.read_text("/myfile", encoding="utf-8") == "some\nlines\nof\ntext"
def test_chained_fs():
d1 = tempfile.mkdtemp()
d2 = tempfile.mkdtemp()
f1 = os.path.join(d1, "f1")
with open(f1, "wb") as f:
f.write(b"test")
of = fsspec.open(
f"simplecache::file://{f1}",
simplecache={"cache_storage": d2, "same_names": True},
)
with of as f:
assert f.read() == b"test"
assert os.listdir(d2) == ["f1"]
@pytest.mark.xfail(reason="see issue #334", strict=True)
def test_multilevel_chained_fs():
"""This test reproduces fsspec/filesystem_spec#334"""
import zipfile
d1 = tempfile.mkdtemp()
f1 = os.path.join(d1, "f1.zip")
with zipfile.ZipFile(f1, mode="w") as z:
# filename, content
z.writestr("foo.txt", "foo.txt")
z.writestr("bar.txt", "bar.txt")
# We expected this to be the correct syntax
with pytest.raises(IsADirectoryError):
of = fsspec.open_files(f"zip://*.txt::simplecache::file://{f1}")
assert len(of) == 2
# But this is what is actually valid...
of = fsspec.open_files(f"zip://*.txt::simplecache://{f1}::file://")
assert len(of) == 2
for open_file in of:
with open_file as f:
assert f.read().decode("utf-8") == f.name
def test_multilevel_chained_fs_zip_zip_file():
"""This test reproduces fsspec/filesystem_spec#334"""
import zipfile
d1 = tempfile.mkdtemp()
f1 = os.path.join(d1, "f1.zip")
f2 = os.path.join(d1, "f2.zip")
with zipfile.ZipFile(f1, mode="w") as z:
# filename, content
z.writestr("foo.txt", "foo.txt")
z.writestr("bar.txt", "bar.txt")
with zipfile.ZipFile(f2, mode="w") as z:
with open(f1, "rb") as f:
z.writestr("f1.zip", f.read())
# We expected this to be the correct syntax
of = fsspec.open_files(f"zip://*.txt::zip://f1.zip::file://{f2}")
assert len(of) == 2
for open_file in of:
with open_file as f:
assert f.read().decode("utf-8") == f.name
def test_chained_equivalent():
d1 = tempfile.mkdtemp()
d2 = tempfile.mkdtemp()
f1 = os.path.join(d1, "f1")
with open(f1, "wb") as f:
f.write(b"test1")
of = fsspec.open(
f"simplecache::file://{f1}",
simplecache={"cache_storage": d2, "same_names": True},
)
of2 = fsspec.open(
f"simplecache://{f1}",
cache_storage=d2,
same_names=True,
target_protocol="file",
target_options={},
)
# the following line passes by fluke - they are not quite the same instance,
# since the parameters don't quite match. Also, the url understood by the two
# of s are not the same (path gets munged a bit differently)
assert of.fs == of2.fs
assert hash(of.fs) == hash(of2.fs)
assert of.open().read() == of2.open().read()
def test_chained_fs_multi():
d1 = tempfile.mkdtemp()
d2 = tempfile.mkdtemp()
f1 = os.path.join(d1, "f1")
f2 = os.path.join(d1, "f2")
with open(f1, "wb") as f:
f.write(b"test1")
with open(f2, "wb") as f:
f.write(b"test2")
of = fsspec.open_files(
f"simplecache::file://{d1}/*",
simplecache={"cache_storage": d2, "same_names": True},
)
with of[0] as f:
assert f.read() == b"test1"
with of[1] as f:
assert f.read() == b"test2"
assert sorted(os.listdir(d2)) == ["f1", "f2"]
d2 = tempfile.mkdtemp()
of = fsspec.open_files(
[f"simplecache::file://{f1}", f"simplecache::file://{f2}"],
simplecache={"cache_storage": d2, "same_names": True},
)
with of[0] as f:
assert f.read() == b"test1"
with of[1] as f:
assert f.read() == b"test2"
assert sorted(os.listdir(d2)) == ["f1", "f2"]
def test_chained_fo():
import zipfile
d1 = tempfile.mkdtemp()
f1 = os.path.join(d1, "temp.zip")
d3 = tempfile.mkdtemp()
with zipfile.ZipFile(f1, mode="w") as z:
z.writestr("afile", b"test")
of = fsspec.open(f"zip://afile::file://{f1}")
with of as f:
assert f.read() == b"test"
of = fsspec.open_files(f"zip://*::file://{f1}")
with of[0] as f:
assert f.read() == b"test"
of = fsspec.open_files(
f"simplecache::zip://*::file://{f1}",
simplecache={"cache_storage": d3, "same_names": True},
)
with of[0] as f:
assert f.read() == b"test"
assert "afile" in os.listdir(d3)
def test_url_to_fs():
url = "memory://a.txt"
fs, url2 = fsspec.core.url_to_fs(url)
assert isinstance(fs, MemoryFileSystem)
assert url2 == "/a.txt"
def test_walk(m):
# depth = 0
dir1 = "/dir1"
# depth = 1 (2 dirs, 1 file)
dir11 = dir1 + "/dir11"
dir12 = dir1 + "/dir12"
file11 = dir1 + "/file11"
# depth = 2
dir111 = dir11 + "/dir111"
file111 = dir11 + "/file111"
file121 = dir12 + "/file121"
# depth = 3
file1111 = dir111 + "/file1111"
m.mkdir(dir111) # Creates parents too
m.mkdir(dir12) # Creates parents too
m.touch(file11)
m.touch(file111)
m.touch(file121)
m.touch(file1111)
# No maxdepth
assert list(m.walk(dir1, topdown=True)) == [
(dir1, ["dir11", "dir12"], ["file11"]),
(dir11, ["dir111"], ["file111"]),
(dir111, [], ["file1111"]),
(dir12, [], ["file121"]),
]
assert list(m.walk(dir1, topdown=False)) == [
(dir111, [], ["file1111"]),
(dir11, ["dir111"], ["file111"]),
(dir12, [], ["file121"]),
(dir1, ["dir11", "dir12"], ["file11"]),
]
# maxdepth=2
assert list(m.walk(dir1, maxdepth=2, topdown=True)) == [
(dir1, ["dir11", "dir12"], ["file11"]),
(dir11, ["dir111"], ["file111"]),
(dir12, [], ["file121"]),
]
assert list(m.walk(dir1, maxdepth=2, topdown=False)) == [
(dir11, ["dir111"], ["file111"]),
(dir12, [], ["file121"]),
(dir1, ["dir11", "dir12"], ["file11"]),
]
# maxdepth=1
assert list(m.walk(dir1, maxdepth=1, topdown=True)) == [
(dir1, ["dir11", "dir12"], ["file11"]),
]
assert list(m.walk(dir1, maxdepth=1, topdown=False)) == [
(dir1, ["dir11", "dir12"], ["file11"]),
]
# maxdepth=0
with pytest.raises(ValueError):
list(m.walk(dir1, maxdepth=0, topdown=True))
with pytest.raises(ValueError):
list(m.walk(dir1, maxdepth=0, topdown=False))
# prune dir111
def _walk(*args, **kwargs):
for path, dirs, files in m.walk(*args, **kwargs):
yield (path, dirs.copy(), files)
if "dir111" in dirs:
dirs.remove("dir111")
assert list(_walk(dir1, topdown=True)) == [
(dir1, ["dir11", "dir12"], ["file11"]),
(dir11, ["dir111"], ["file111"]),
(dir12, [], ["file121"]),
]
assert list(_walk(dir1, topdown=False)) == [
(dir111, [], ["file1111"]),
(dir11, ["dir111"], ["file111"]),
(dir12, [], ["file121"]),
(dir1, ["dir11", "dir12"], ["file11"]),
]
# reverse dirs order
def _walk(*args, **kwargs):
for path, dirs, files in m.walk(*args, **kwargs):
yield (path, dirs.copy(), files)
dirs.reverse()
assert list(_walk(dir1, topdown=True)) == [
(dir1, ["dir11", "dir12"], ["file11"]),
# Here dir12 comes before dir11
(dir12, [], ["file121"]),
(dir11, ["dir111"], ["file111"]),
(dir111, [], ["file1111"]),
]
assert list(_walk(dir1, topdown=False)) == [
(dir111, [], ["file1111"]),
(dir11, ["dir111"], ["file111"]),
(dir12, [], ["file121"]),
(dir1, ["dir11", "dir12"], ["file11"]),
]
# on_error omit by default
assert list(m.walk("do_not_exist")) == []
# on_error omit
assert list(m.walk("do_not_exist", on_error="omit")) == []
# on_error raise
with pytest.raises(FileNotFoundError):
list(m.walk("do_not_exist", on_error="raise"))
# on_error callable function
mock = Mock()
assert list(m.walk("do_not_exist", on_error=mock.onerror)) == []
mock.onerror.assert_called()
assert mock.onerror.call_args.kwargs == {}
assert len(mock.onerror.call_args.args) == 1
assert isinstance(mock.onerror.call_args.args[0], FileNotFoundError)
filesystem_spec-2025.3.2/fsspec/tests/test_async.py 0000664 0000000 0000000 00000015042 14772531734 0022335 0 ustar 00root root 0000000 0000000 import asyncio
import inspect
import io
import os
import time
import pytest
import fsspec
import fsspec.asyn
from fsspec.asyn import _run_coros_in_chunks
def test_sync_methods():
inst = fsspec.asyn.AsyncFileSystem()
assert inspect.iscoroutinefunction(inst._info)
assert hasattr(inst, "info")
assert inst.info.__qualname__ == "AsyncFileSystem._info"
assert not inspect.iscoroutinefunction(inst.info)
def test_when_sync_methods_are_disabled():
class TestFS(fsspec.asyn.AsyncFileSystem):
mirror_sync_methods = False
inst = TestFS()
assert inspect.iscoroutinefunction(inst._info)
assert not inspect.iscoroutinefunction(inst.info)
assert inst.info.__qualname__ == "AbstractFileSystem.info"
def test_interrupt():
loop = fsspec.asyn.get_loop()
async def f():
await asyncio.sleep(1000000)
return True
fut = asyncio.run_coroutine_threadsafe(f(), loop)
time.sleep(0.01) # task launches
out = fsspec.asyn._dump_running_tasks(with_task=True)
task = out[0]["task"]
assert task.done() and fut.done()
assert isinstance(fut.exception(), fsspec.asyn.FSSpecCoroutineCancel)
class _DummyAsyncKlass:
def __init__(self):
self.loop = fsspec.asyn.get_loop()
async def _dummy_async_func(self):
# Sleep 1 second function to test timeout
await asyncio.sleep(1)
return True
async def _bad_multiple_sync(self):
fsspec.asyn.sync_wrapper(_DummyAsyncKlass._dummy_async_func)(self)
return True
dummy_func = fsspec.asyn.sync_wrapper(_dummy_async_func)
bad_multiple_sync_func = fsspec.asyn.sync_wrapper(_bad_multiple_sync)
def test_sync_wrapper_timeout_on_less_than_expected_wait_time_not_finish_function():
test_obj = _DummyAsyncKlass()
with pytest.raises(fsspec.FSTimeoutError):
test_obj.dummy_func(timeout=0.1)
def test_sync_wrapper_timeout_on_more_than_expected_wait_time_will_finish_function():
test_obj = _DummyAsyncKlass()
assert test_obj.dummy_func(timeout=5)
def test_sync_wrapper_timeout_none_will_wait_func_finished():
test_obj = _DummyAsyncKlass()
assert test_obj.dummy_func(timeout=None)
def test_sync_wrapper_treat_timeout_0_as_none():
test_obj = _DummyAsyncKlass()
assert test_obj.dummy_func(timeout=0)
def test_sync_wrapper_bad_multiple_sync():
test_obj = _DummyAsyncKlass()
with pytest.raises(NotImplementedError):
test_obj.bad_multiple_sync_func(timeout=5)
def test_run_coros_in_chunks(monkeypatch):
total_running = 0
async def runner():
nonlocal total_running
total_running += 1
await asyncio.sleep(0)
if total_running > 4:
raise ValueError("More than 4 coroutines are running together")
total_running -= 1
return 1
async def main(**kwargs):
nonlocal total_running
total_running = 0
coros = [runner() for _ in range(32)]
results = await _run_coros_in_chunks(coros, **kwargs)
for result in results:
if isinstance(result, Exception):
raise result
return results
assert sum(asyncio.run(main(batch_size=4))) == 32
with pytest.raises(ValueError):
asyncio.run(main(batch_size=5))
with pytest.raises(ValueError):
asyncio.run(main(batch_size=-1))
assert sum(asyncio.run(main(batch_size=4))) == 32
monkeypatch.setitem(fsspec.config.conf, "gather_batch_size", 5)
with pytest.raises(ValueError):
asyncio.run(main())
assert sum(asyncio.run(main(batch_size=4))) == 32 # override
monkeypatch.setitem(fsspec.config.conf, "gather_batch_size", 4)
assert sum(asyncio.run(main())) == 32 # override
@pytest.mark.skipif(os.name != "nt", reason="only for windows")
def test_windows_policy():
from asyncio.windows_events import SelectorEventLoop
loop = fsspec.asyn.get_loop()
policy = asyncio.get_event_loop_policy()
# Ensure that the created loop always uses selector policy
assert isinstance(loop, SelectorEventLoop)
# Ensure that the global policy is not changed and it is
# set to the default one. This is important since the
# get_loop() method will temporarily override the policy
# with the one which uses selectors on windows, so this
# check ensures that we are restoring the old policy back
# after our change.
assert isinstance(policy, asyncio.DefaultEventLoopPolicy)
def test_running_async():
assert not fsspec.asyn.running_async()
async def go():
assert fsspec.asyn.running_async()
asyncio.run(go())
class DummyAsyncFS(fsspec.asyn.AsyncFileSystem):
_file_class = fsspec.asyn.AbstractAsyncStreamedFile
async def _info(self, path, **kwargs):
return {"name": "misc/foo.txt", "type": "file", "size": 100}
async def open_async(
self,
path,
mode="rb",
block_size=None,
autocommit=True,
cache_options=None,
**kwargs,
):
return DummyAsyncStreamedFile(
self,
path,
mode,
block_size,
autocommit,
cache_options=cache_options,
**kwargs,
)
class DummyAsyncStreamedFile(fsspec.asyn.AbstractAsyncStreamedFile):
def __init__(self, fs, path, mode, block_size, autocommit, **kwargs):
super().__init__(fs, path, mode, block_size, autocommit, **kwargs)
self.temp_buffer = io.BytesIO(b"foo-bar" * 20)
async def _fetch_range(self, start, end):
return self.temp_buffer.read(end - start)
async def _initiate_upload(self):
# Reinitialize for new uploads.
self.temp_buffer = io.BytesIO()
async def _upload_chunk(self, final=False):
self.temp_buffer.write(self.buffer.getbuffer())
async def get_data(self):
return self.temp_buffer.getbuffer().tobytes()
@pytest.mark.asyncio
async def test_async_streamed_file_write():
test_fs = DummyAsyncFS()
streamed_file = await test_fs.open_async("misc/foo.txt", mode="wb")
inp_data = b"foo-bar" * streamed_file.blocksize * 2
await streamed_file.write(inp_data)
assert streamed_file.loc == len(inp_data)
await streamed_file.close()
out_data = await streamed_file.get_data()
assert out_data.count(b"foo-bar") == streamed_file.blocksize * 2
@pytest.mark.asyncio
async def test_async_streamed_file_read():
test_fs = DummyAsyncFS()
streamed_file = await test_fs.open_async("misc/foo.txt", mode="rb")
assert (
await streamed_file.read(7 * 3) + await streamed_file.read(7 * 18)
== b"foo-bar" * 20
)
await streamed_file.close()
filesystem_spec-2025.3.2/fsspec/tests/test_caches.py 0000664 0000000 0000000 00000022675 14772531734 0022460 0 ustar 00root root 0000000 0000000 import pickle
import string
import pytest
from fsspec.caching import (
BlockCache,
FirstChunkCache,
MMapCache,
ReadAheadCache,
caches,
register_cache,
)
from fsspec.implementations.cached import WholeFileCacheFileSystem
def test_cache_getitem(Cache_imp):
cacher = Cache_imp(4, letters_fetcher, len(string.ascii_letters))
assert cacher._fetch(0, 4) == b"abcd"
assert cacher._fetch(None, 4) == b"abcd"
assert cacher._fetch(2, 4) == b"cd"
def test_block_cache_lru():
# BlockCache is a cache that stores blocks of data and uses LRU to evict
block_size = 4
cache = BlockCache(
block_size, letters_fetcher, len(string.ascii_letters), maxblocks=2
)
# miss
cache._fetch(0, 2)
assert cache.cache_info().misses == 1
assert cache.cache_info().currsize == 1
assert cache.total_requested_bytes == block_size * cache.miss_count
assert cache.size == 52
# hit
cache._fetch(0, 2)
assert cache.cache_info().misses == 1
assert cache.cache_info().currsize == 1
assert cache.total_requested_bytes == block_size * cache.miss_count
# hit
cache._fetch(0, 2)
assert cache.cache_info().misses == 1
assert cache.cache_info().currsize == 1
# this works as a counter since all the reads are from the cache
assert cache.hit_count == 3
assert cache.miss_count == 1
# so far only 4 bytes have been read using range requests
assert cache.total_requested_bytes == block_size * cache.miss_count
# miss
cache._fetch(4, 6)
assert cache.cache_info().misses == 2
assert cache.cache_info().currsize == 2
assert cache.total_requested_bytes == block_size * cache.miss_count
# miss & evict
cache._fetch(12, 13)
assert cache.cache_info().misses == 3
assert cache.cache_info().currsize == 2
assert cache.hit_count == 5
assert cache.miss_count == 3
assert cache.total_requested_bytes == block_size * cache.miss_count
def test_first_cache():
"""
FirstChunkCache is a cache that only caches the first chunk of data
when some of that first block is requested.
"""
block_size = 5
cache = FirstChunkCache(block_size, letters_fetcher, len(string.ascii_letters))
assert cache.cache is None
assert cache._fetch(12, 15) == letters_fetcher(12, 15)
assert cache.miss_count == 1
assert cache.hit_count == 0
assert cache.cache is None
total_requested_bytes = 15 - 12
assert cache.total_requested_bytes == total_requested_bytes
# because we overlap with the cache range, it will be cached
assert cache._fetch(3, 10) == letters_fetcher(3, 10)
assert cache.miss_count == 2
assert cache.hit_count == 0
# we'll read the first 5 and then the rest
total_requested_bytes += block_size + 5
assert cache.total_requested_bytes == total_requested_bytes
# partial hit again
assert cache._fetch(3, 10) == letters_fetcher(3, 10)
assert cache.miss_count == 2
assert cache.hit_count == 1
# we have the first 5 bytes cached
total_requested_bytes += 10 - 5
assert cache.total_requested_bytes == total_requested_bytes
assert cache.cache == letters_fetcher(0, 5)
assert cache._fetch(0, 4) == letters_fetcher(0, 4)
assert cache.hit_count == 2
assert cache.miss_count == 2
assert cache.total_requested_bytes == 18
def test_readahead_cache():
"""
ReadAheadCache is a cache that reads ahead of the requested range.
If the access pattern is not sequential it will be very inefficient.
"""
block_size = 5
cache = ReadAheadCache(block_size, letters_fetcher, len(string.ascii_letters))
assert cache._fetch(12, 15) == letters_fetcher(12, 15)
assert cache.miss_count == 1
assert cache.hit_count == 0
total_requested_bytes = 15 - 12 + block_size
assert cache.total_requested_bytes == total_requested_bytes
assert cache._fetch(3, 10) == letters_fetcher(3, 10)
assert cache.miss_count == 2
assert cache.hit_count == 0
assert len(cache.cache) == 12
total_requested_bytes += (10 - 3) + block_size
assert cache.total_requested_bytes == total_requested_bytes
# caache hit again
assert cache._fetch(3, 10) == letters_fetcher(3, 10)
assert cache.miss_count == 2
assert cache.hit_count == 1
assert len(cache.cache) == 12
assert cache.total_requested_bytes == total_requested_bytes
assert cache.cache == letters_fetcher(3, 15)
# cache miss
assert cache._fetch(0, 4) == letters_fetcher(0, 4)
assert cache.hit_count == 1
assert cache.miss_count == 3
assert len(cache.cache) == 9
total_requested_bytes += (4 - 0) + block_size
assert cache.total_requested_bytes == total_requested_bytes
def _fetcher(start, end):
return b"0" * (end - start)
def letters_fetcher(start, end):
return string.ascii_letters[start:end].encode()
def multi_letters_fetcher(ranges):
return [string.ascii_letters[start:end].encode() for start, end in ranges]
not_parts_caches = {k: v for k, v in caches.items() if k != "parts"}
@pytest.fixture(params=not_parts_caches.values(), ids=list(not_parts_caches))
def Cache_imp(request):
return request.param
def test_cache_empty_file(Cache_imp):
blocksize = 5
size = 0
cache = Cache_imp(blocksize, _fetcher, size)
assert cache._fetch(0, 0) == b""
def test_cache_pickleable(Cache_imp):
blocksize = 5
size = 100
cache = Cache_imp(blocksize, _fetcher, size)
cache._fetch(0, 5) # fill in cache
unpickled = pickle.loads(pickle.dumps(cache))
assert isinstance(unpickled, Cache_imp)
assert unpickled.blocksize == blocksize
assert unpickled.size == size
assert unpickled._fetch(0, 10) == b"0" * 10
def test_first_cache():
c = FirstChunkCache(5, letters_fetcher, 52)
assert c.cache is None
assert c._fetch(12, 15) == letters_fetcher(12, 15)
assert c.cache is None
assert c._fetch(3, 10) == letters_fetcher(3, 10)
assert c.cache == letters_fetcher(0, 5)
c.fetcher = None
assert c._fetch(1, 4) == letters_fetcher(1, 4)
def test_mmap_cache(mocker):
fetcher = mocker.Mock(wraps=letters_fetcher)
c = MMapCache(5, fetcher, 52)
assert c._fetch(6, 8) == letters_fetcher(6, 8)
assert fetcher.call_count == 1
assert c._fetch(17, 22) == letters_fetcher(17, 22)
assert fetcher.call_count == 2
assert c._fetch(1, 38) == letters_fetcher(1, 38)
assert fetcher.call_count == 5
multi_fetcher = mocker.Mock(wraps=multi_letters_fetcher)
m = MMapCache(5, fetcher, size=52, multi_fetcher=multi_fetcher)
assert m._fetch(6, 8) == letters_fetcher(6, 8)
assert multi_fetcher.call_count == 1
assert m._fetch(17, 22) == letters_fetcher(17, 22)
assert multi_fetcher.call_count == 2
assert m._fetch(1, 38) == letters_fetcher(1, 38)
assert multi_fetcher.call_count == 3
assert fetcher.call_count == 5
@pytest.mark.parametrize(
"size_requests",
[[(0, 30), (0, 35), (51, 52)], [(0, 1), (1, 11), (1, 52)], [(0, 52), (11, 15)]],
)
@pytest.mark.parametrize("blocksize", [1, 10, 52, 100])
def test_cache_basic(Cache_imp, blocksize, size_requests):
cache = Cache_imp(blocksize, letters_fetcher, len(string.ascii_letters))
for start, end in size_requests:
result = cache._fetch(start, end)
expected = string.ascii_letters[start:end].encode()
assert result == expected
@pytest.mark.parametrize("strict", [True, False])
@pytest.mark.parametrize("sort", [True, False])
def test_known(sort, strict):
parts = {(10, 20): b"1" * 10, (20, 30): b"2" * 10, (0, 10): b"0" * 10}
if sort:
parts = dict(sorted(parts.items()))
c = caches["parts"](None, None, 100, parts, strict=strict)
assert (0, 30) in c.data # got consolidated
assert c._fetch(5, 15) == b"0" * 5 + b"1" * 5
assert c._fetch(15, 25) == b"1" * 5 + b"2" * 5
if strict:
# Over-read will raise error
with pytest.raises(ValueError):
# tries to call None fetcher
c._fetch(25, 35)
else:
# Over-read will be zero-padded
assert c._fetch(25, 35) == b"2" * 5 + b"\x00" * 5
def test_background(server, monkeypatch):
import threading
import time
import fsspec
head = {"head_ok": "true", "head_give_length": "true"}
urla = server.realfile
h = fsspec.filesystem("http", headers=head)
thread_ids = {threading.current_thread().ident}
f = h.open(urla, block_size=5, cache_type="background")
orig = f.cache._fetch_block
def wrapped(*a, **kw):
thread_ids.add(threading.current_thread().ident)
return orig(*a, **kw)
f.cache._fetch_block = wrapped
assert len(thread_ids) == 1
f.read(1)
time.sleep(0.1) # second block is loading
assert len(thread_ids) == 2
def test_register_cache():
# just test that we have them populated and fail to re-add again unless overload
with pytest.raises(ValueError):
register_cache(BlockCache)
register_cache(BlockCache, clobber=True)
def test_cache_kwargs(mocker):
# test that kwargs are passed to the underlying filesystem after cache commit
fs = WholeFileCacheFileSystem(target_protocol="memory")
fs.touch("test")
fs.fs.put = mocker.MagicMock()
with fs.open("test", "wb", overwrite=True) as file_handle:
file_handle.write(b"foo")
# We don't care about the first parameter, just retrieve its expected value.
# It is a random location that cannot be predicted.
# The important thing is the 'overwrite' kwarg
fs.fs.put.assert_called_with(fs.fs.put.call_args[0][0], "/test", overwrite=True)
filesystem_spec-2025.3.2/fsspec/tests/test_callbacks.py 0000664 0000000 0000000 00000005050 14772531734 0023135 0 ustar 00root root 0000000 0000000 import pytest
from fsspec.callbacks import Callback, TqdmCallback
def test_callbacks():
empty_callback = Callback()
assert empty_callback.call("something", somearg=None) is None
hooks = {"something": lambda *_, arg=None: arg + 2}
simple_callback = Callback(hooks=hooks)
assert simple_callback.call("something", arg=2) == 4
hooks = {"something": lambda *_, arg1=None, arg2=None: arg1 + arg2}
multi_arg_callback = Callback(hooks=hooks)
assert multi_arg_callback.call("something", arg1=2, arg2=2) == 4
def test_callbacks_as_callback():
empty_callback = Callback.as_callback(None)
assert empty_callback.call("something", arg="somearg") is None
assert Callback.as_callback(None) is Callback.as_callback(None)
hooks = {"something": lambda *_, arg=None: arg + 2}
real_callback = Callback.as_callback(Callback(hooks=hooks))
assert real_callback.call("something", arg=2) == 4
def test_callbacks_as_context_manager(mocker):
spy_close = mocker.spy(Callback, "close")
with Callback() as cb:
assert isinstance(cb, Callback)
spy_close.assert_called_once()
def test_callbacks_branched():
callback = Callback()
branch = callback.branched("path_1", "path_2")
assert branch is not callback
assert isinstance(branch, Callback)
@pytest.mark.asyncio
async def test_callbacks_branch_coro(mocker):
async_fn = mocker.AsyncMock(return_value=10)
callback = Callback()
wrapped_fn = callback.branch_coro(async_fn)
spy = mocker.spy(callback, "branched")
assert await wrapped_fn("path_1", "path_2", key="value") == 10
spy.assert_called_once_with("path_1", "path_2", key="value")
async_fn.assert_called_once_with(
"path_1", "path_2", callback=spy.spy_return, key="value"
)
def test_callbacks_wrap():
events = []
class TestCallback(Callback):
def relative_update(self, inc=1):
events.append(inc)
callback = TestCallback()
for _ in callback.wrap(range(10)):
...
assert events == [1] * 10
@pytest.mark.parametrize("tqdm_kwargs", [{}, {"desc": "A custom desc"}])
def test_tqdm_callback(tqdm_kwargs, mocker):
pytest.importorskip("tqdm")
callback = TqdmCallback(tqdm_kwargs=tqdm_kwargs)
mocker.patch.object(callback, "_tqdm_cls")
callback.set_size(10)
for _ in callback.wrap(range(10)):
...
assert callback.tqdm.update.call_count == 11
if not tqdm_kwargs:
callback._tqdm_cls.assert_called_with(total=10)
else:
callback._tqdm_cls.assert_called_with(total=10, **tqdm_kwargs)
filesystem_spec-2025.3.2/fsspec/tests/test_compression.py 0000664 0000000 0000000 00000012142 14772531734 0023557 0 ustar 00root root 0000000 0000000 import pathlib
import pytest
import fsspec.core
from fsspec.compression import compr, register_compression
from fsspec.utils import compressions, infer_compression
def test_infer_custom_compression():
"""Inferred compression gets values from fsspec.compression.compr."""
assert infer_compression("fn.zip") == "zip"
assert infer_compression("fn.gz") == "gzip"
assert infer_compression("fn.unknown") is None
assert infer_compression("fn.test_custom") is None
assert infer_compression("fn.tst") is None
register_compression("test_custom", lambda f, **kwargs: f, "tst")
try:
assert infer_compression("fn.zip") == "zip"
assert infer_compression("fn.gz") == "gzip"
assert infer_compression("fn.unknown") is None
assert infer_compression("fn.test_custom") is None
assert infer_compression("fn.tst") == "test_custom"
# Duplicate registration in name or extension raises a value error.
with pytest.raises(ValueError):
register_compression("test_custom", lambda f, **kwargs: f, "tst")
with pytest.raises(ValueError):
register_compression("test_conflicting", lambda f, **kwargs: f, "tst")
assert "test_conflicting" not in compr
# ...but can be forced.
register_compression(
"test_conflicting", lambda f, **kwargs: f, "tst", force=True
)
assert infer_compression("fn.zip") == "zip"
assert infer_compression("fn.gz") == "gzip"
assert infer_compression("fn.unknown") is None
assert infer_compression("fn.test_custom") is None
assert infer_compression("fn.tst") == "test_conflicting"
finally:
del compr["test_custom"]
del compr["test_conflicting"]
del compressions["tst"]
def test_infer_uppercase_compression():
assert infer_compression("fn.ZIP") == "zip"
assert infer_compression("fn.GZ") == "gzip"
assert infer_compression("fn.UNKNOWN") is None
assert infer_compression("fn.TEST_UPPERCASE") is None
assert infer_compression("fn.TEST") is None
def test_lzma_compression_name():
pytest.importorskip("lzma")
assert infer_compression("fn.xz") == "xz"
assert infer_compression("fn.lzma") == "lzma"
def test_lz4_compression(tmpdir):
"""Infer lz4 compression for .lz4 files if lz4 is available."""
tmp_path = pathlib.Path(str(tmpdir))
lz4 = pytest.importorskip("lz4")
tmp_path.mkdir(exist_ok=True)
tdat = "foobar" * 100
with fsspec.core.open(
str(tmp_path / "out.lz4"), mode="wt", compression="infer"
) as outfile:
outfile.write(tdat)
compressed = (tmp_path / "out.lz4").open("rb").read()
assert lz4.frame.decompress(compressed).decode() == tdat
with fsspec.core.open(
str(tmp_path / "out.lz4"), mode="rt", compression="infer"
) as infile:
assert infile.read() == tdat
with fsspec.core.open(
str(tmp_path / "out.lz4"), mode="rt", compression="lz4"
) as infile:
assert infile.read() == tdat
def test_zstd_compression(tmpdir):
"""Infer zstd compression for .zst files if zstandard is available."""
tmp_path = pathlib.Path(str(tmpdir))
zstd = pytest.importorskip("zstandard")
tmp_path.mkdir(exist_ok=True)
tdat = "foobar" * 100
with fsspec.core.open(
str(tmp_path / "out.zst"), mode="wt", compression="infer"
) as outfile:
outfile.write(tdat)
compressed = (tmp_path / "out.zst").open("rb").read()
assert zstd.ZstdDecompressor().decompress(compressed, len(tdat)).decode() == tdat
with fsspec.core.open(
str(tmp_path / "out.zst"), mode="rt", compression="infer"
) as infile:
assert infile.read() == tdat
with fsspec.core.open(
str(tmp_path / "out.zst"), mode="rt", compression="zstd"
) as infile:
assert infile.read() == tdat
# fails in https://github.com/fsspec/filesystem_spec/issues/725
infile = fsspec.core.open(
str(tmp_path / "out.zst"), mode="rb", compression="infer"
).open()
infile.close()
def test_snappy_compression(tmpdir):
"""No registered compression for snappy, but can be specified."""
tmp_path = pathlib.Path(str(tmpdir))
snappy = pytest.importorskip("snappy")
tmp_path.mkdir(exist_ok=True)
tdat = "foobar" * 100
# Snappy isn't inferred.
with fsspec.core.open(
str(tmp_path / "out.snappy"), mode="wt", compression="infer"
) as outfile:
outfile.write(tdat)
assert (tmp_path / "out.snappy").open("rb").read().decode() == tdat
# but can be specified.
with fsspec.core.open(
str(tmp_path / "out.snappy"), mode="wt", compression="snappy"
) as outfile:
outfile.write(tdat)
compressed = (tmp_path / "out.snappy").open("rb").read()
assert snappy.StreamDecompressor().decompress(compressed).decode() == tdat
with fsspec.core.open(
str(tmp_path / "out.snappy"), mode="rb", compression="infer"
) as infile:
assert infile.read() == compressed
with fsspec.core.open(
str(tmp_path / "out.snappy"), mode="rt", compression="snappy"
) as infile:
assert infile.read() == tdat
filesystem_spec-2025.3.2/fsspec/tests/test_config.py 0000664 0000000 0000000 00000006624 14772531734 0022473 0 ustar 00root root 0000000 0000000 import os
from warnings import catch_warnings
import pytest
import fsspec
from fsspec.config import conf, set_conf_env, set_conf_files
@pytest.fixture
def clean_conf():
"""Tests should start and end with clean config dict"""
conf.clear()
yield
conf.clear()
def test_from_env_ignored(clean_conf):
env = {
"FSSPEC": "missing_protocol",
"FSSPEC_": "missing_protocol",
"FSSPEC__INVALID_KEY": "invalid_protocol",
"FSSPEC_INVALID1": "not_json_dict",
"FSSPEC_INVALID2": '["not_json_dict"]',
}
cd = {}
with catch_warnings(record=True) as w:
set_conf_env(conf_dict=cd, envdict=env)
assert len(w) == 5
assert "unexpected name" in str(w[0].message)
assert "unexpected name" in str(w[1].message)
assert "unexpected name" in str(w[2].message)
assert "parse failure" in str(w[3].message)
assert "not being a dict" in str(w[4].message)
assert cd == {}
def test_from_env_kwargs(clean_conf):
env = {
"FSSPEC_PROTO_KEY": "value",
"FSSPEC_PROTO_LONG_KEY": "othervalue",
"FSSPEC_MALFORMED": "novalue",
}
cd = {}
with catch_warnings(record=True) as w:
set_conf_env(conf_dict=cd, envdict=env)
assert len(w) == 1
assert "parse failure" in str(w[0].message)
assert cd == {"proto": {"key": "value", "long_key": "othervalue"}}
def test_from_env_protocol_dict(clean_conf):
env = {
"FSSPEC_PROTO": '{"int": 1, "float": 2.3, "bool": true, "dict": {"key": "val"}}'
}
cd = {}
set_conf_env(conf_dict=cd, envdict=env)
assert cd == {
"proto": {"int": 1, "float": 2.3, "bool": True, "dict": {"key": "val"}}
}
def test_from_env_kwargs_override_protocol_dict(clean_conf):
env = {
"FSSPEC_PROTO_LONG_KEY": "override1",
"FSSPEC_PROTO": '{"key": "value1", "long_key": "value2", "otherkey": "value3"}',
"FSSPEC_PROTO_KEY": "override2",
}
cd = {}
set_conf_env(conf_dict=cd, envdict=env)
assert cd == {
"proto": {"key": "override2", "long_key": "override1", "otherkey": "value3"}
}
def test_from_file_ini(clean_conf, tmpdir):
file1 = os.path.join(tmpdir, "1.ini")
file2 = os.path.join(tmpdir, "2.ini")
with open(file1, "w") as f:
f.write(
"""[proto]
key=value
other_key:othervalue
overwritten=dont_see
"""
)
with open(file2, "w") as f:
f.write(
"""[proto]
overwritten=see
"""
)
cd = {}
set_conf_files(tmpdir, cd)
assert cd == {
"proto": {"key": "value", "other_key": "othervalue", "overwritten": "see"}
}
def test_from_file_json(clean_conf, tmpdir):
file1 = os.path.join(tmpdir, "1.json")
file2 = os.path.join(tmpdir, "2.json")
with open(file1, "w") as f:
f.write(
"""{"proto":
{"key": "value",
"other_key": "othervalue",
"overwritten": false}}
"""
)
with open(file2, "w") as f:
f.write(
"""{"proto":
{"overwritten": true}}
"""
)
cd = {}
set_conf_files(tmpdir, cd)
assert cd == {
"proto": {"key": "value", "other_key": "othervalue", "overwritten": True}
}
def test_apply(clean_conf):
conf["file"] = {"auto_mkdir": "test"}
fs = fsspec.filesystem("file")
assert fs.auto_mkdir == "test"
fs = fsspec.filesystem("file", auto_mkdir=True)
assert fs.auto_mkdir is True
filesystem_spec-2025.3.2/fsspec/tests/test_core.py 0000664 0000000 0000000 00000033657 14772531734 0022164 0 ustar 00root root 0000000 0000000 import os
import pickle
import tempfile
import zipfile
from contextlib import contextmanager
from pathlib import Path
import pytest
import fsspec
from fsspec.core import (
OpenFile,
OpenFiles,
_expand_paths,
expand_paths_if_needed,
get_compression,
get_fs_token_paths,
open_files,
open_local,
)
@contextmanager
def tempzip(data=None):
data = data or {}
f = tempfile.mkstemp(suffix="zip")[1]
with zipfile.ZipFile(f, mode="w") as z:
for k, v in data.items():
z.writestr(k, v)
try:
yield f
finally:
try:
os.remove(f)
except OSError:
pass
@pytest.mark.parametrize(
"path, name_function, num, out",
[
[["apath"], None, 1, ["apath"]],
["apath.*.csv", None, 1, ["apath.0.csv"]],
["apath.*.csv", None, 2, ["apath.0.csv", "apath.1.csv"]],
["a*", lambda x: "abc"[x], 2, ["aa", "ab"]],
],
)
def test_expand_paths(path, name_function, num, out):
assert _expand_paths(path, name_function, num) == out
@pytest.mark.parametrize(
"create_files, path, out",
[
[["apath"], "apath", ["apath"]],
[["apath1"], "apath*", ["apath1"]],
[["apath1", "apath2"], "apath*", ["apath1", "apath2"]],
[["apath1", "apath2"], "apath[1]", ["apath1"]],
[["apath1", "apath11"], "apath?", ["apath1"]],
],
)
def test_expand_paths_if_needed_in_read_mode(create_files, path, out):
d = str(tempfile.mkdtemp())
for f in create_files:
f = os.path.join(d, f)
open(f, "w").write("test")
path = os.path.join(d, path)
fs = fsspec.filesystem("file")
res = expand_paths_if_needed([path], "r", 0, fs, None)
assert [os.path.basename(p) for p in res] == out
def test_expand_error():
with pytest.raises(ValueError):
_expand_paths("*.*", None, 1)
@pytest.mark.parametrize("mode", ["w", "w+", "x", "x+"])
def test_expand_fs_token_paths(mode):
assert len(get_fs_token_paths("path", mode, num=2, expand=True)[-1]) == 2
def test_openfile_api(m):
m.open("somepath", "wb").write(b"data")
of = OpenFile(m, "somepath")
assert str(of) == ""
f = of.open()
assert f.read() == b"data"
f.close()
with OpenFile(m, "somepath", mode="rt") as f:
assert f.read() == "data"
def test_openfile_open(m):
of = OpenFile(m, "somepath", mode="wt")
f = of.open()
f.write("hello")
assert m.size("somepath") == 0 # no flush yet
of.close()
assert m.size("somepath") == 5
def test_open_local_w_cache():
d1 = str(tempfile.mkdtemp())
f1 = os.path.join(d1, "f1")
open(f1, "w").write("test1")
d2 = str(tempfile.mkdtemp())
fn = open_local(f"simplecache://{f1}", cache_storage=d2, target_protocol="file")
assert isinstance(fn, str)
assert open(fn).read() == "test1"
assert d2 in fn
def test_open_local_w_magic():
d1 = str(tempfile.mkdtemp())
f1 = os.path.join(d1, "f1")
open(f1, "w").write("test1")
fn = open_local(os.path.join(d1, "f*"))
assert len(fn) == 1
assert isinstance(fn, list)
def test_open_local_w_list_of_str():
d1 = str(tempfile.mkdtemp())
f1 = os.path.join(d1, "f1")
open(f1, "w").write("test1")
fn = open_local([f1, f1])
assert len(fn) == 2
assert isinstance(fn, list)
assert all(isinstance(elem, str) for elem in fn)
def test_open_local_w_path():
d1 = str(tempfile.mkdtemp())
f1 = os.path.join(d1, "f1")
open(f1, "w").write("test1")
p = Path(f1)
fn = open_local(p)
assert isinstance(fn, str)
def test_open_local_w_list_of_path():
d1 = str(tempfile.mkdtemp())
f1 = os.path.join(d1, "f1")
open(f1, "w").write("test1")
p = Path(f1)
fn = open_local([p, p])
assert len(fn) == 2
assert isinstance(fn, list)
assert all(isinstance(elem, str) for elem in fn)
def test_xz_lzma_compressions():
pytest.importorskip("lzma")
# Ensure that both 'xz' and 'lzma' compression names can be parsed
assert get_compression("some_file.xz", "infer") == "xz"
assert get_compression("some_file.xz", "xz") == "xz"
assert get_compression("some_file.xz", "lzma") == "lzma"
def test_list():
here = os.path.abspath(os.path.dirname(__file__))
flist = os.listdir(here)
plist = [os.path.join(here, p).replace("\\", "/") for p in flist]
of = open_files(plist)
assert len(of) == len(flist)
assert [f.path for f in of] == plist
def test_open_expand(m, monkeypatch):
m.pipe("/myfile", b"hello")
with pytest.raises(FileNotFoundError, match="expand=True"):
with fsspec.open("memory://my*", expand=False):
pass
with fsspec.open("memory://my*", expand=True) as f:
assert f.path == "/myfile"
monkeypatch.setattr(fsspec.core, "DEFAULT_EXPAND", True)
with fsspec.open("memory://my*") as f:
assert f.path == "/myfile"
def test_pathobject(tmpdir):
import pathlib
tmpdir = str(tmpdir)
plist_str = [os.path.join(str(tmpdir), f).replace("\\", "/") for f in ["a", "b"]]
open(plist_str[0], "w").write("first file")
open(plist_str[1], "w").write("second file")
plist = [pathlib.Path(p) for p in plist_str]
of = open_files(plist)
assert len(of) == 2
assert [f.path for f in of] == plist_str
of = open_files(plist[0])
assert len(of) == 1
assert of[0].path == plist_str[0]
with of[0] as f:
assert f.read() == open(plist_str[0], "rb").read()
def test_automkdir(tmpdir):
dir = os.path.join(str(tmpdir), "a")
of = fsspec.open(os.path.join(dir, "afile"), "w", auto_mkdir=False)
with pytest.raises(IOError):
with of:
pass
dir = os.path.join(str(tmpdir), "b")
of = fsspec.open(os.path.join(dir, "bfile"), "w", auto_mkdir=True)
with of:
pass
assert "bfile" in os.listdir(dir)
dir = os.path.join(str(tmpdir), "c")
with pytest.raises(FileNotFoundError):
of = fsspec.open(os.path.join(dir, "bfile"), "w", auto_mkdir=False)
with of:
pass
def test_automkdir_readonly(tmpdir):
dir = os.path.join(str(tmpdir), "d")
with pytest.raises(FileNotFoundError):
of = fsspec.open(os.path.join(dir, "dfile"), "r")
with of:
pass
def test_openfile_pickle_newline():
# GH#318
test = fsspec.open(__file__, newline=b"")
pickled = pickle.dumps(test)
restored = pickle.loads(pickled)
assert test.newline == restored.newline
def test_pickle_after_open_open():
of = fsspec.open(__file__, mode="rt")
test = of.open()
of2 = pickle.loads(pickle.dumps(of))
test2 = of2.open()
test.close()
assert not test2.closed
of.close()
of2.close()
# Define a list of special glob characters.
# Note that we need to escape some characters and also consider file system limitations.
# '*' and '?' are excluded because they are not valid for many file systems.
# Similarly, we're careful with '{', '}', and '@' as their special meaning is
# context-specific and might not be considered special for filenames.
# Add tests for more file systems and for more glob magic later
glob_magic_characters = ["[", "]", "!"]
if os.name != "nt":
glob_magic_characters.extend(("*", "?")) # not valid on Windows
@pytest.mark.parametrize("char", glob_magic_characters)
def test_open_file_read_with_special_characters(tmp_path, char):
# Create a filename incorporating the special character
file_name = f"test{char}.txt"
file_path = tmp_path / file_name
expected_content = "Hello, world!"
with open(file_path, "w") as f:
f.write(expected_content)
with fsspec.open(file_path, "r") as f:
actual_content = f.read()
assert actual_content == expected_content
@pytest.mark.parametrize("char", glob_magic_characters)
def test_open_files_read_with_special_characters(tmp_path, char):
# Create a filename incorporating the special character
file_name = f"test{char}.txt"
file_path = tmp_path / file_name
expected_content = "Hello, world!"
with open(file_path, "w") as f:
f.write(expected_content)
with fsspec.open_files(file_path, "r")[0] as f:
actual_content = f.read()
assert actual_content == expected_content
@pytest.mark.parametrize("char", glob_magic_characters)
def test_open_file_write_with_special_characters(tmp_path, char, monkeypatch):
# Create a filename incorporating the special character
file_name = f"test{char}.txt"
file_path = tmp_path / file_name
expected_content = "Hello, world!"
with fsspec.open(file_path, "w", expand=False) as f:
f.write(expected_content)
with open(file_path, "r") as f:
actual_content = f.read()
monkeypatch.setattr(fsspec.core, "DEFAULT_EXPAND", False)
with fsspec.open(file_path, "w") as f:
f.write(expected_content * 2)
with open(file_path, "r") as f:
assert f.read() == actual_content * 2
assert actual_content == expected_content
@pytest.mark.parametrize("char", glob_magic_characters)
def test_open_files_read_with_special_characters(tmp_path, char):
# Create a filename incorporating the special character
file_name = f"test{char}.txt"
file_path = tmp_path / file_name
expected_content = "Hello, world!"
with open(file_path, "w") as f:
f.write(expected_content)
with fsspec.open_files(
urlpath=[os.fspath(file_path)], mode="r", auto_mkdir=False, expand=False
)[0] as f:
actual_content = f.read()
assert actual_content == expected_content
@pytest.mark.parametrize("char", glob_magic_characters)
def test_open_files_write_with_special_characters(tmp_path, char):
# Create a filename incorporating the special character
file_name = f"test{char}.txt"
file_path = tmp_path / file_name
expected_content = "Hello, world!"
with fsspec.open_files(
urlpath=[os.fspath(file_path)], mode="w", auto_mkdir=False, expand=False
)[0] as f:
f.write(expected_content)
with open(file_path, "r") as f:
actual_content = f.read()
assert actual_content == expected_content
def test_mismatch():
pytest.importorskip("s3fs")
with pytest.raises(ValueError):
open_files(["s3://test/path.csv", "/other/path.csv"])
def test_url_kwargs_chain(ftp_writable):
host, port, username, password = ftp_writable
data = b"hello"
with fsspec.open(
"ftp:///afile", "wb", host=host, port=port, username=username, password=password
) as f:
f.write(data)
with fsspec.open(
f"simplecache::ftp://{username}:{password}@{host}:{port}//afile", "rb"
) as f:
assert f.read() == data
def test_multi_context(tmpdir):
fns = [os.path.join(tmpdir, fn) for fn in ["a", "b"]]
files = open_files(fns, "wb")
assert isinstance(files, OpenFiles)
assert isinstance(files[0], OpenFile)
assert len(files) == 2
assert isinstance(files[:1], OpenFiles)
assert len(files[:1]) == 1
with files as of:
assert len(of) == 2
assert not of[0].closed
assert of[0].name.endswith("a")
assert of[0].closed
assert repr(files) == ""
def test_not_local():
with pytest.raises(ValueError, match="attribute local_file=True"):
open_local("memory://afile")
def test_url_to_fs(ftp_writable):
host, port, username, password = ftp_writable
data = b"hello"
with fsspec.open(f"ftp://{username}:{password}@{host}:{port}/afile", "wb") as f:
f.write(data)
fs, url = fsspec.core.url_to_fs(
f"simplecache::ftp://{username}:{password}@{host}:{port}/afile"
)
assert url == "/afile"
fs, url = fsspec.core.url_to_fs(f"ftp://{username}:{password}@{host}:{port}/afile")
assert url == "/afile"
with fsspec.open(f"ftp://{username}:{password}@{host}:{port}/afile.zip", "wb") as f:
import zipfile
with zipfile.ZipFile(f, "w") as z:
with z.open("inner", "w") as f2:
f2.write(b"hello")
f.write(data)
fs, url = fsspec.core.url_to_fs(
f"zip://inner::ftp://{username}:{password}@{host}:{port}/afile.zip"
)
assert url == "inner"
fs, url = fsspec.core.url_to_fs(
f"simplecache::zip::ftp://{username}:{password}@{host}:{port}/afile.zip"
)
assert url == ""
def test_target_protocol_options(ftp_writable):
host, port, username, password = ftp_writable
data = {"afile": b"hello"}
options = {"host": host, "port": port, "username": username, "password": password}
with tempzip(data) as lfile, fsspec.open(
"ftp:///archive.zip", "wb", **options
) as f:
f.write(open(lfile, "rb").read())
with fsspec.open(
"zip://afile",
"rb",
target_protocol="ftp",
target_options=options,
fo="archive.zip",
) as f:
assert f.read() == data["afile"]
def test_chained_url(ftp_writable):
host, port, username, password = ftp_writable
data = {"afile": b"hello"}
cls = fsspec.get_filesystem_class("ftp")
fs = cls(host=host, port=port, username=username, password=password)
with tempzip(data) as lfile:
fs.put_file(lfile, "archive.zip")
urls = [
"zip://afile",
"zip://afile::simplecache",
"simplecache::zip://afile",
"simplecache::zip://afile::simplecache",
]
for url in urls:
url += f"::ftp://{username}:{password}@{host}:{port}/archive.zip"
with fsspec.open(url, "rb") as f:
assert f.read() == data["afile"]
def test_automkdir_local():
fs, _ = fsspec.core.url_to_fs("file://", auto_mkdir=True)
assert fs.auto_mkdir is True
def test_repeated_argument():
pytest.importorskip("adlfs")
from fsspec.core import url_to_fs
fs, url = url_to_fs(
"az://DIR@ACCOUNT.blob.core.windows.net/DATA",
anon=False,
account_name="ACCOUNT",
)
assert fs.storage_options == {"account_name": "ACCOUNT", "anon": False}
with pytest.raises(TypeError):
url_to_fs(
"az://DIR@ACCOUNT.blob.core.windows.net/DATA",
anon=False,
account_name="OTHER",
)
filesystem_spec-2025.3.2/fsspec/tests/test_downstream.py 0000664 0000000 0000000 00000001775 14772531734 0023413 0 ustar 00root root 0000000 0000000 import pytest
pytest.importorskip("s3fs")
pytest.importorskip("moto")
try:
from s3fs.tests.test_s3fs import ( # noqa: F401
endpoint_uri,
s3,
s3_base,
test_bucket_name,
)
except ImportError:
pytest.skip("s3 tests not available.")
so = {"anon": False, "client_kwargs": {"endpoint_url": endpoint_uri}}
def test_pandas(s3):
pd = pytest.importorskip("pandas")
df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
df.to_csv(f"s3://{test_bucket_name}/a.csv", storage_options=so)
df2 = pd.read_csv(f"s3://{test_bucket_name}/a.csv", storage_options=so)
assert df.a.equals(df2.a)
def test_xarray_zarr(s3):
xr = pytest.importorskip("xarray")
pytest.importorskip("zarr")
import numpy as np
x = np.arange(5)
xarr = xr.DataArray(x)
ds = xr.Dataset({"x": xarr})
ds.to_zarr(f"s3://{test_bucket_name}/a.zarr", storage_options=so)
ds2 = xr.open_zarr(f"s3://{test_bucket_name}/a.zarr", storage_options=so)
assert (ds.x == ds2.x).all()
filesystem_spec-2025.3.2/fsspec/tests/test_file.py 0000664 0000000 0000000 00000012462 14772531734 0022142 0 ustar 00root root 0000000 0000000 """Tests abstract buffered file API, using FTP implementation"""
import pickle
import pytest
from fsspec.implementations.tests.test_ftp import FTPFileSystem
data = b"hello" * 10000
def test_pickle(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
f = ftp.open("/out", "rb")
f2 = pickle.loads(pickle.dumps(f))
assert f == f2
def test_file_read_attributes(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
f = ftp.open("/out", "rb")
assert f.info()["size"] == len(data)
assert f.tell() == 0
assert f.seekable()
assert f.readable()
assert not f.writable()
out = bytearray(len(data))
assert f.read() == data
assert f.read() == b""
f.seek(0)
assert f.readuntil(b"l") == b"hel"
assert f.tell() == 3
f.readinto1(out)
assert out[:-3] == data[3:]
with pytest.raises(ValueError):
f.write(b"")
f.close()
with pytest.raises(ValueError):
f.read()(b"")
def test_seek(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
f = ftp.open("/out", "rb")
assert f.seek(-10, 2) == len(data) - 10
assert f.tell() == len(data) - 10
assert f.seek(-1, 1) == len(data) - 11
with pytest.raises(ValueError):
f.seek(-1)
with pytest.raises(ValueError):
f.seek(0, 7)
def test_file_idempotent(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
f = ftp.open("/out", "rb")
f2 = ftp.open("/out", "rb")
assert hash(f) == hash(f2)
assert f == f2
ftp.touch("/out2")
f2 = ftp.open("/out2", "rb")
assert hash(f2) != hash(f)
assert f != f2
f2 = ftp.open("/out", "wb")
assert hash(f2) != hash(f)
def test_file_text_attributes(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
data = b"hello\n" * 1000
with ftp.open("/out2", "wb") as f:
f.write(data)
f = ftp.open("/out2", "rb")
assert f.readline() == b"hello\n"
f.seek(0)
assert list(f) == [d + b"\n" for d in data.split()]
f.seek(0)
assert f.readlines() == [d + b"\n" for d in data.split()]
f = ftp.open("/out2", "rt")
assert f.readline() == "hello\n"
assert f.encoding
def test_file_write_attributes(ftp_writable):
host, port, user, pw = ftp_writable
ftp = FTPFileSystem(host=host, port=port, username=user, password=pw)
f = ftp.open("/out2", "wb")
with pytest.raises(ValueError):
f.info()
with pytest.raises(OSError):
f.seek(0)
with pytest.raises(ValueError):
f.read(0)
assert not f.readable()
assert f.writable()
f.flush() # no-op
assert f.write(b"hello") == 5
assert f.write(b"hello") == 5
assert not f.closed
f.close()
assert f.closed
with pytest.raises(ValueError):
f.write(b"")
with pytest.raises(ValueError):
f.flush()
def test_midread_cache(ftp_writable):
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host=host, port=port, username=user, password=pw)
fn = "/myfile"
with fs.open(fn, "wb") as f:
f.write(b"a" * 175627146)
with fs.open(fn, "rb") as f:
f.seek(175561610)
d1 = f.read(65536)
assert len(d1) == 65536
f.seek(4)
size = 17562198
d2 = f.read(size)
assert len(d2) == size
f.seek(17562288)
size = 17562187
d3 = f.read(size)
assert len(d3) == size
def test_read_block(ftp_writable):
# not the same as test_read_block in test_utils, this depends on the
# behaviour of the bytest caching
from fsspec.utils import read_block
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host=host, port=port, username=user, password=pw)
fn = "/myfile"
with fs.open(fn, "wb") as f:
f.write(b"a,b\n1,2")
f = fs.open(fn, "rb", cache_type="bytes")
assert read_block(f, 0, 6400, b"\n") == b"a,b\n1,2"
def test_with_gzip(ftp_writable):
import gzip
data = b"some compressible stuff"
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host=host, port=port, username=user, password=pw)
fn = "/myfile"
with fs.open(fn, "wb") as f:
gf = gzip.GzipFile(fileobj=f, mode="w")
gf.write(data)
gf.close()
with fs.open(fn, "rb") as f:
gf = gzip.GzipFile(fileobj=f, mode="r")
assert gf.read() == data
def test_auto_compression(m):
fs = m
with fs.open("myfile.gz", mode="wt", compression="infer") as f:
f.write("text")
with fs.open("myfile.gz", mode="rt", compression="infer") as f:
assert f.read() == "text"
def test_with_zip(ftp_writable):
import zipfile
data = b"hello zip"
host, port, user, pw = ftp_writable
fs = FTPFileSystem(host=host, port=port, username=user, password=pw)
fn = "/myfile.zip"
inner_file = "test.txt"
with fs.open(fn, "wb") as f:
zf = zipfile.ZipFile(f, mode="w")
zf.writestr(inner_file, data)
zf.close()
with fs.open(fn, "rb") as f:
zf = zipfile.ZipFile(f, mode="r")
assert zf.read(inner_file) == data
filesystem_spec-2025.3.2/fsspec/tests/test_fuse.py 0000664 0000000 0000000 00000007221 14772531734 0022162 0 ustar 00root root 0000000 0000000 import os
import subprocess
import time
from multiprocessing import Process
import pytest
try:
pytest.importorskip("fuse")
except OSError:
# can succeed in importing fuse, but fail to load so
pytest.importorskip("nonexistent")
from fsspec.fuse import main, run
from fsspec.implementations.memory import MemoryFileSystem
def host_fuse(mountdir):
fs = MemoryFileSystem()
fs.touch("/mounted/testfile")
run(fs, "/mounted/", mountdir)
def test_basic(tmpdir, capfd):
mountdir = str(tmpdir.mkdir("mount"))
fuse_process = Process(target=host_fuse, args=(str(mountdir),))
fuse_process.start()
try:
timeout = 10
while True:
try:
# can fail with device not ready while waiting for fuse
if "testfile" in os.listdir(mountdir):
break
except Exception:
pass
timeout -= 1
time.sleep(1)
if not timeout > 0:
import pdb
pdb.set_trace()
pytest.skip(msg="fuse didn't come live")
fn = os.path.join(mountdir, "test")
with open(fn, "wb") as f:
f.write(b"data")
with open(fn) as f:
assert f.read() == "data"
os.remove(fn)
os.mkdir(fn)
assert os.listdir(fn) == []
os.mkdir(fn + "/inner")
with pytest.raises(OSError):
os.rmdir(fn)
captured = capfd.readouterr()
assert "Traceback" not in captured.out
assert "Traceback" not in captured.err
os.rmdir(fn + "/inner")
os.rmdir(fn)
finally:
fuse_process.terminate()
fuse_process.join(timeout=10)
if fuse_process.is_alive():
fuse_process.kill()
fuse_process.join()
def host_mount_local(source_dir, mount_dir, debug_log):
main(["local", source_dir, mount_dir, "-l", debug_log, "--ready-file"])
@pytest.fixture()
def mount_local(tmpdir):
source_dir = tmpdir.mkdir("source")
mount_dir = tmpdir.mkdir("local")
debug_log = tmpdir / "debug.log"
fuse_process = Process(
target=host_mount_local, args=(str(source_dir), str(mount_dir), str(debug_log))
)
fuse_process.start()
ready_file = mount_dir / ".fuse_ready"
for _ in range(20):
if ready_file.exists() and open(ready_file).read() == b"ready":
break
time.sleep(0.1)
try:
yield (source_dir, mount_dir)
finally:
fuse_process.terminate()
fuse_process.join(timeout=10)
if fuse_process.is_alive():
fuse_process.kill()
fuse_process.join()
def test_mount(mount_local):
source_dir, mount_dir = mount_local
assert os.listdir(mount_dir) == []
assert os.listdir(source_dir) == []
mount_dir.mkdir("a")
assert os.listdir(mount_dir) == ["a"]
assert os.listdir(source_dir) == ["a"]
def test_chmod(mount_local):
source_dir, mount_dir = mount_local
open(mount_dir / "text", "w").write("test")
assert os.listdir(source_dir) == ["text"]
cp = subprocess.run(
["cp", str(mount_dir / "text"), str(mount_dir / "new")],
capture_output=True,
check=False,
)
assert cp.stderr == b""
assert cp.stdout == b""
assert set(os.listdir(source_dir)) == {"text", "new"}
assert open(mount_dir / "new").read() == "test"
def test_seek_rw(mount_local):
source_dir, mount_dir = mount_local
fh = open(mount_dir / "text", "w")
fh.write("teST")
fh.seek(2)
fh.write("st")
fh.close()
fh = open(mount_dir / "text", "r")
assert fh.read() == "test"
fh.seek(2)
assert fh.read() == "st"
fh.close()
filesystem_spec-2025.3.2/fsspec/tests/test_generic.py 0000664 0000000 0000000 00000005334 14772531734 0022637 0 ustar 00root root 0000000 0000000 import pytest
import fsspec
from fsspec.tests.conftest import data, server # noqa: F401
def test_remote_async_ops(server):
fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
fs = fsspec.filesystem("generic", default_method="current")
out = fs.info(server.realfile)
assert out["size"] == len(data)
assert out["type"] == "file"
assert fs.isfile(server.realfile) # this method from superclass
def test_touch_rm(m):
m.touch("afile")
m.touch("dir/afile")
fs = fsspec.filesystem("generic", default_method="current")
fs.rm("memory://afile")
assert not m.exists("afile")
fs.rm("memory://dir", recursive=True)
assert not m.exists("dir/afile")
assert not m.exists("dir")
def test_cp_async_to_sync(server, m):
fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
fs = fsspec.filesystem("generic", default_method="current")
fs.cp([server.realfile], ["memory://realfile"])
assert m.cat("realfile") == data
fs.rm("memory://realfile")
assert not m.exists("realfile")
def test_pipe_cat_sync(m):
fs = fsspec.filesystem("generic", default_method="current")
fs.pipe("memory://afile", b"data")
assert fs.cat("memory://afile") == b"data"
def test_cat_async(server):
fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
fs = fsspec.filesystem("generic", default_method="current")
assert fs.cat(server.realfile) == data
def test_rsync(tmpdir, m):
from fsspec.generic import GenericFileSystem, rsync
fs = GenericFileSystem()
fs.pipe("memory:///deep/path/afile", b"data1")
fs.pipe("memory:///deep/afile", b"data2")
with pytest.raises(ValueError):
rsync("memory:///deep/afile", f"file://{tmpdir}")
rsync("memory://", f"file://{tmpdir}")
allfiles = fs.find(f"file://{tmpdir}", withdirs=True, detail=True)
pos_tmpdir = fsspec.implementations.local.make_path_posix(str(tmpdir)) # for WIN
assert set(allfiles) == {
f"file://{pos_tmpdir}{_}"
for _ in [
"",
"/deep",
"/deep/path",
"/deep/path/afile",
"/deep/afile",
]
}
fs.rm("memory:///deep/afile")
rsync("memory://", f"file://{tmpdir}", delete_missing=True)
allfiles2 = fs.find(f"file://{tmpdir}", withdirs=True, detail=True)
assert set(allfiles2) == {
f"file://{pos_tmpdir}{_}"
for _ in [
"",
"/deep",
"/deep/path",
"/deep/path/afile",
]
}
# the file was not updated, since size was correct
assert (
allfiles[f"file://{pos_tmpdir}/deep/path/afile"]
== allfiles2[f"file://{pos_tmpdir}/deep/path/afile"]
)
filesystem_spec-2025.3.2/fsspec/tests/test_gui.py 0000664 0000000 0000000 00000000774 14772531734 0022012 0 ustar 00root root 0000000 0000000 import pytest
panel = pytest.importorskip("panel")
def test_basic():
import fsspec.gui
gui = fsspec.gui.FileSelector()
assert "url" in str(gui.panel)
def test_kwargs(tmpdir):
"""confirm kwargs are passed to the filesystem instance"""
import fsspec.gui
gui = fsspec.gui.FileSelector(f"file://{tmpdir}", kwargs="{'auto_mkdir': True}")
assert gui.fs.auto_mkdir
gui = fsspec.gui.FileSelector(f"file://{tmpdir}", kwargs={"auto_mkdir": True})
assert gui.fs.auto_mkdir
filesystem_spec-2025.3.2/fsspec/tests/test_mapping.py 0000664 0000000 0000000 00000014556 14772531734 0022664 0 ustar 00root root 0000000 0000000 import os
import pickle
import platform
import sys
import uuid
import pytest
import fsspec
from fsspec.implementations.local import LocalFileSystem
from fsspec.implementations.memory import MemoryFileSystem
def test_mapping_prefix(tmpdir):
tmpdir = str(tmpdir)
os.makedirs(os.path.join(tmpdir, "afolder"))
open(os.path.join(tmpdir, "afile"), "w").write("test")
open(os.path.join(tmpdir, "afolder", "anotherfile"), "w").write("test2")
m = fsspec.get_mapper(f"file://{tmpdir}")
assert "afile" in m
assert m["afolder/anotherfile"] == b"test2"
fs = fsspec.filesystem("file")
m2 = fs.get_mapper(tmpdir)
m3 = fs.get_mapper(f"file://{tmpdir}")
assert m == m2 == m3
def test_getitems_errors(tmpdir):
tmpdir = str(tmpdir)
os.makedirs(os.path.join(tmpdir, "afolder"))
open(os.path.join(tmpdir, "afile"), "w").write("test")
open(os.path.join(tmpdir, "afolder", "anotherfile"), "w").write("test2")
m = fsspec.get_mapper(f"file://{tmpdir}")
assert m.getitems(["afile", "bfile"], on_error="omit") == {"afile": b"test"}
with pytest.raises(KeyError):
m.getitems(["afile", "bfile"])
out = m.getitems(["afile", "bfile"], on_error="return")
assert isinstance(out["bfile"], KeyError)
m = fsspec.get_mapper(f"file://{tmpdir}", missing_exceptions=())
assert m.getitems(["afile", "bfile"], on_error="omit") == {"afile": b"test"}
with pytest.raises(FileNotFoundError):
m.getitems(["afile", "bfile"])
def test_ops():
MemoryFileSystem.store.clear()
m = fsspec.get_mapper("memory://")
assert not m
assert list(m) == []
with pytest.raises(KeyError):
m["hi"]
assert m.pop("key", 0) == 0
m["key0"] = b"data"
assert list(m) == ["key0"]
assert m["key0"] == b"data"
m.clear()
assert list(m) == []
def test_pickle():
m = fsspec.get_mapper("memory://")
assert isinstance(m.fs, MemoryFileSystem)
m["key"] = b"data"
m2 = pickle.loads(pickle.dumps(m))
assert list(m) == list(m2)
assert m.missing_exceptions == m2.missing_exceptions
def test_keys_view():
# https://github.com/fsspec/filesystem_spec/issues/186
m = fsspec.get_mapper("memory://")
m["key"] = b"data"
keys = m.keys()
assert len(keys) == 1
# check that we don't consume the keys
assert len(keys) == 1
m.clear()
def test_multi():
m = fsspec.get_mapper("memory:///")
data = {"a": b"data1", "b": b"data2"}
m.setitems(data)
assert m.getitems(list(data)) == data
m.delitems(list(data))
assert not list(m)
def test_setitem_types():
import array
m = fsspec.get_mapper("memory://")
m["a"] = array.array("i", [1])
if sys.byteorder == "little":
assert m["a"] == b"\x01\x00\x00\x00"
else:
assert m["a"] == b"\x00\x00\x00\x01"
m["b"] = bytearray(b"123")
assert m["b"] == b"123"
m.setitems({"c": array.array("i", [1]), "d": bytearray(b"123")})
if sys.byteorder == "little":
assert m["c"] == b"\x01\x00\x00\x00"
else:
assert m["c"] == b"\x00\x00\x00\x01"
assert m["d"] == b"123"
def test_setitem_numpy():
m = fsspec.get_mapper("memory://")
np = pytest.importorskip("numpy")
m["c"] = np.array(1, dtype="foo{i}\nFOOBAR{i}\n" for i in range(100000))).encode()
# Read single record at beginning.
# All reads include beginning of file and read through termination of
# delimited record.
assert read_block(io.BytesIO(d), 0, 10, delimiter=b"\n") == b"#header>foo0\n"
assert (
read_block(io.BytesIO(d), 0, 10, delimiter=b"\n", split_before=True)
== b"#header>foo0"
)
assert (
read_block(io.BytesIO(d), 0, 10, delimiter=b">") == b"#header>foo0\nFOOBAR0\n>"
)
assert (
read_block(io.BytesIO(d), 0, 10, delimiter=b">", split_before=True)
== b"#header>foo0\nFOOBAR0\n"
)
# Read multiple records at beginning.
# All reads include beginning of file and read through termination of
# delimited record.
assert (
read_block(io.BytesIO(d), 0, 27, delimiter=b"\n")
== b"#header>foo0\nFOOBAR0\n>foo1\nFOOBAR1\n"
)
assert (
read_block(io.BytesIO(d), 0, 27, delimiter=b"\n", split_before=True)
== b"#header>foo0\nFOOBAR0\n>foo1\nFOOBAR1"
)
assert (
read_block(io.BytesIO(d), 0, 27, delimiter=b">")
== b"#header>foo0\nFOOBAR0\n>foo1\nFOOBAR1\n>"
)
assert (
read_block(io.BytesIO(d), 0, 27, delimiter=b">", split_before=True)
== b"#header>foo0\nFOOBAR0\n>foo1\nFOOBAR1\n"
)
# Read with offset spanning into next record, splits on either side of delimiter.
# Read not spanning the full record returns nothing.
assert read_block(io.BytesIO(d), 10, 3, delimiter=b"\n") == b"FOOBAR0\n"
assert (
read_block(io.BytesIO(d), 10, 3, delimiter=b"\n", split_before=True)
== b"\nFOOBAR0"
)
assert read_block(io.BytesIO(d), 10, 3, delimiter=b">") == b""
assert read_block(io.BytesIO(d), 10, 3, delimiter=b">", split_before=True) == b""
# Read with offset spanning multiple records, splits on either side of delimiter
assert (
read_block(io.BytesIO(d), 10, 20, delimiter=b"\n")
== b"FOOBAR0\n>foo1\nFOOBAR1\n"
)
assert (
read_block(io.BytesIO(d), 10, 20, delimiter=b"\n", split_before=True)
== b"\nFOOBAR0\n>foo1\nFOOBAR1"
)
assert read_block(io.BytesIO(d), 10, 20, delimiter=b">") == b"foo1\nFOOBAR1\n>"
assert (
read_block(io.BytesIO(d), 10, 20, delimiter=b">", split_before=True)
== b">foo1\nFOOBAR1\n"
)
# Read record at end, all records read to end
tlen = len(d)
assert (
read_block(io.BytesIO(d), tlen - 30, 35, delimiter=b"\n")
== b">foo99999\nFOOBAR99999\n"
)
assert (
read_block(io.BytesIO(d), tlen - 30, 35, delimiter=b"\n", split_before=True)
== b"\n>foo99999\nFOOBAR99999\n"
)
assert (
read_block(io.BytesIO(d), tlen - 30, 35, delimiter=b">")
== b"foo99999\nFOOBAR99999\n"
)
assert (
read_block(io.BytesIO(d), tlen - 30, 35, delimiter=b">", split_before=True)
== b">foo99999\nFOOBAR99999\n"
)
def test_seek_delimiter_endline():
f = io.BytesIO(b"123\n456\n789")
# if at zero, stay at zero
seek_delimiter(f, b"\n", 5)
assert f.tell() == 0
# choose the first block
for bs in [1, 5, 100]:
f.seek(1)
seek_delimiter(f, b"\n", blocksize=bs)
assert f.tell() == 4
# handle long delimiters well, even with short blocksizes
f = io.BytesIO(b"123abc456abc789")
for bs in [1, 2, 3, 4, 5, 6, 10]:
f.seek(1)
seek_delimiter(f, b"abc", blocksize=bs)
assert f.tell() == 6
# End at the end
f = io.BytesIO(b"123\n456")
f.seek(5)
seek_delimiter(f, b"\n", 5)
assert f.tell() == 7
def test_infer_options():
so = infer_storage_options("/mnt/datasets/test.csv")
assert so.pop("protocol") == "file"
assert so.pop("path") == "/mnt/datasets/test.csv"
assert not so
assert infer_storage_options("./test.csv")["path"] == "./test.csv"
assert infer_storage_options("../test.csv")["path"] == "../test.csv"
so = infer_storage_options("C:\\test.csv")
assert so.pop("protocol") == "file"
assert so.pop("path") == "C:\\test.csv"
assert not so
assert infer_storage_options("d:\\test.csv")["path"] == "d:\\test.csv"
assert infer_storage_options("\\test.csv")["path"] == "\\test.csv"
assert infer_storage_options(".\\test.csv")["path"] == ".\\test.csv"
assert infer_storage_options("test.csv")["path"] == "test.csv"
so = infer_storage_options(
"hdfs://username:pwd@Node:123/mnt/datasets/test.csv?q=1#fragm",
inherit_storage_options={"extra": "value"},
)
assert so.pop("protocol") == "hdfs"
assert so.pop("username") == "username"
assert so.pop("password") == "pwd"
assert so.pop("host") == "Node"
assert so.pop("port") == 123
assert so.pop("path") == "/mnt/datasets/test.csv#fragm"
assert so.pop("url_query") == "q=1"
assert so.pop("url_fragment") == "fragm"
assert so.pop("extra") == "value"
assert not so
so = infer_storage_options("hdfs://User-name@Node-name.com/mnt/datasets/test.csv")
assert so.pop("username") == "User-name"
assert so.pop("host") == "Node-name.com"
u = "http://127.0.0.1:8080/test.csv"
assert infer_storage_options(u) == {"protocol": "http", "path": u}
# For s3 and gcs the netloc is actually the bucket name, so we want to
# include it in the path. Test that:
# - Parsing doesn't lowercase the bucket
# - The bucket is included in path
for protocol in ["s3", "s3a", "gcs", "gs"]:
options = infer_storage_options(f"{protocol}://Bucket-name.com/test.csv")
assert options["path"] == "Bucket-name.com/test.csv"
with pytest.raises(KeyError):
infer_storage_options("file:///bucket/file.csv", {"path": "collide"})
with pytest.raises(KeyError):
infer_storage_options("hdfs:///bucket/file.csv", {"protocol": "collide"})
def test_infer_simple():
out = infer_storage_options("//mnt/datasets/test.csv")
assert out["protocol"] == "file"
assert out["path"] == "//mnt/datasets/test.csv"
assert out.get("host", None) is None
@pytest.mark.parametrize(
"urlpath, expected_path",
(
(r"c:\foo\bar", r"c:\foo\bar"),
(r"C:\\foo\bar", r"C:\\foo\bar"),
(r"c:/foo/bar", r"c:/foo/bar"),
(r"file:///c|\foo\bar", r"c:\foo\bar"),
(r"file:///C|/foo/bar", r"C:/foo/bar"),
(r"file:///C:/foo/bar", r"C:/foo/bar"),
),
)
def test_infer_storage_options_c(urlpath, expected_path):
so = infer_storage_options(urlpath)
assert so["protocol"] == "file"
assert so["path"] == expected_path
@pytest.mark.parametrize(
"paths, out",
(
(["/more/dir/", "/more/dir/two", "/more/one", "/more/three"], "/more"),
(["/", "", "/"], ""),
(["/", "/"], "/"),
(["/more/", "/"], ""),
(["/more/", "/more"], "/more"),
(["more/dir/", "more/dir/two", "more/one", "more/three"], "more"),
),
)
def test_common_prefix(paths, out):
assert common_prefix(paths) == out
@pytest.mark.parametrize(
"paths, other, exists, expected",
(
(["/path1"], "/path2", False, ["/path2"]),
(["/path1"], "/path2", True, ["/path2/path1"]),
(["/path1"], "/path2/", True, ["/path2/path1"]),
(["/path1"], ["/path2"], False, ["/path2"]),
(["/path1"], ["/path2"], True, ["/path2"]),
(["/path1", "/path2"], "/path2", False, ["/path2/path1", "/path2/path2"]),
(["/path1", "/path2"], "/path2", True, ["/path2/path1", "/path2/path2"]),
(
["/more/path1", "/more/path2"],
"/path2",
False,
["/path2/path1", "/path2/path2"],
),
(
["/more/path1", "/more/path2"],
"/path2",
True,
["/path2/more/path1", "/path2/more/path2"],
),
(
["/more/path1", "/more/path2"],
"/path2/",
False,
["/path2/path1", "/path2/path2"],
),
(
["/more/path1", "/more/path2"],
"/path2/",
True,
["/path2/more/path1", "/path2/more/path2"],
),
(
["/more/path1", "/diff/path2"],
"/path2/",
False,
["/path2/more/path1", "/path2/diff/path2"],
),
(
["/more/path1", "/diff/path2"],
"/path2/",
True,
["/path2/more/path1", "/path2/diff/path2"],
),
(["a", "b/", "b/c"], "dest/", False, ["dest/a", "dest/b/", "dest/b/c"]),
(
["/a", "/b/", "/b/c"],
"dest/",
False,
["dest/a", "dest/b/", "dest/b/c"],
),
),
)
def test_other_paths(paths, other, exists, expected):
assert other_paths(paths, other, exists) == expected
def test_log():
import logging
logger = setup_logging(logger_name="fsspec.test")
assert logger.level == logging.DEBUG
@pytest.mark.parametrize(
"par",
[
("afile", "file"),
("file://afile", "file"),
("noproto://afile", "noproto"),
("noproto::stuff", "noproto"),
("simplecache::stuff", "simplecache"),
("simplecache://stuff", "simplecache"),
("s3://afile", "s3"),
(Path("afile"), "file"),
],
)
def test_get_protocol(par):
url, outcome = par
assert get_protocol(url) == outcome
@pytest.mark.parametrize(
"par",
[
("afile", True),
("file://afile", True),
("noproto://afile", False),
("noproto::stuff", False),
("simplecache::stuff", True),
("simplecache://stuff", True),
(Path("afile"), True),
],
)
def test_can_local(par):
url, outcome = par
assert can_be_local(url) == outcome
def test_mirror_from():
mock = Mock()
mock.attr = 1
@mirror_from("client", ["attr", "func_1", "func_2"])
class Real:
@property
def client(self):
return mock
def func_2(self):
raise AssertionError("have to overwrite this")
def func_3(self):
return "should succeed"
obj = Real()
assert obj.attr == mock.attr
obj.func_1()
mock.func_1.assert_called()
obj.func_2(1, 2)
mock.func_2.assert_called_with(1, 2)
assert obj.func_3() == "should succeed"
mock.func_3.assert_not_called()
@pytest.mark.parametrize("max_gap", [0, 32])
@pytest.mark.parametrize("max_block", [None, 128])
def test_merge_offset_ranges(max_gap, max_block):
# Input ranges
# (Using out-of-order ranges for full coverage)
paths = ["foo", "bar", "bar", "bar", "foo"]
starts = [0, 0, 512, 64, 32]
ends = [32, 32, 1024, 256, 64]
# Call merge_offset_ranges
(
result_paths,
result_starts,
result_ends,
) = merge_offset_ranges(
paths,
starts,
ends,
max_gap=max_gap,
max_block=max_block,
)
# Check result
if max_block is None and max_gap == 32:
expect_paths = ["bar", "bar", "foo"]
expect_starts = [0, 512, 0]
expect_ends = [256, 1024, 64]
else:
expect_paths = ["bar", "bar", "bar", "foo"]
expect_starts = [0, 64, 512, 0]
expect_ends = [32, 256, 1024, 64]
assert expect_paths == result_paths
assert expect_starts == result_starts
assert expect_ends == result_ends
def test_size():
f = io.BytesIO(b"hello")
assert fsspec.utils.file_size(f) == 5
assert f.tell() == 0
class _HasFspath:
def __fspath__(self):
return "foo"
class _HasPathAttr:
def __init__(self):
self.path = "foo"
@pytest.mark.parametrize(
"path,expected",
[
# coerce to string
("foo", "foo"),
(Path("foo"), "foo"),
(PurePath("foo"), "foo"),
(_HasFspath(), "foo"),
(_HasPathAttr(), "foo"),
# passthrough
(b"bytes", b"bytes"),
(None, None),
(1, 1),
(True, True),
(o := object(), o),
([], []),
((), ()),
(set(), set()),
],
)
def test_stringify_path(path, expected):
path = fsspec.utils.stringify_path(path)
assert path == expected
filesystem_spec-2025.3.2/fsspec/transaction.py 0000664 0000000 0000000 00000004536 14772531734 0021352 0 ustar 00root root 0000000 0000000 from collections import deque
class Transaction:
"""Filesystem transaction write context
Gathers files for deferred commit or discard, so that several write
operations can be finalized semi-atomically. This works by having this
instance as the ``.transaction`` attribute of the given filesystem
"""
def __init__(self, fs, **kwargs):
"""
Parameters
----------
fs: FileSystem instance
"""
self.fs = fs
self.files = deque()
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""End transaction and commit, if exit is not due to exception"""
# only commit if there was no exception
self.complete(commit=exc_type is None)
if self.fs:
self.fs._intrans = False
self.fs._transaction = None
self.fs = None
def start(self):
"""Start a transaction on this FileSystem"""
self.files = deque() # clean up after previous failed completions
self.fs._intrans = True
def complete(self, commit=True):
"""Finish transaction: commit or discard all deferred files"""
while self.files:
f = self.files.popleft()
if commit:
f.commit()
else:
f.discard()
self.fs._intrans = False
self.fs._transaction = None
self.fs = None
class FileActor:
def __init__(self):
self.files = []
def commit(self):
for f in self.files:
f.commit()
self.files.clear()
def discard(self):
for f in self.files:
f.discard()
self.files.clear()
def append(self, f):
self.files.append(f)
class DaskTransaction(Transaction):
def __init__(self, fs):
"""
Parameters
----------
fs: FileSystem instance
"""
import distributed
super().__init__(fs)
client = distributed.default_client()
self.files = client.submit(FileActor, actor=True).result()
def complete(self, commit=True):
"""Finish transaction: commit or discard all deferred files"""
if commit:
self.files.commit().result()
else:
self.files.discard().result()
self.fs._intrans = False
self.fs = None
filesystem_spec-2025.3.2/fsspec/utils.py 0000664 0000000 0000000 00000054704 14772531734 0020167 0 ustar 00root root 0000000 0000000 from __future__ import annotations
import contextlib
import logging
import math
import os
import re
import sys
import tempfile
from functools import partial
from hashlib import md5
from importlib.metadata import version
from typing import (
IO,
TYPE_CHECKING,
Any,
Callable,
Iterable,
Iterator,
Sequence,
TypeVar,
)
from urllib.parse import urlsplit
if TYPE_CHECKING:
import pathlib
from typing_extensions import TypeGuard
from fsspec.spec import AbstractFileSystem
DEFAULT_BLOCK_SIZE = 5 * 2**20
T = TypeVar("T")
def infer_storage_options(
urlpath: str, inherit_storage_options: dict[str, Any] | None = None
) -> dict[str, Any]:
"""Infer storage options from URL path and merge it with existing storage
options.
Parameters
----------
urlpath: str or unicode
Either local absolute file path or URL (hdfs://namenode:8020/file.csv)
inherit_storage_options: dict (optional)
Its contents will get merged with the inferred information from the
given path
Returns
-------
Storage options dict.
Examples
--------
>>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP
{"protocol": "file", "path", "/mnt/datasets/test.csv"}
>>> infer_storage_options(
... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1',
... inherit_storage_options={'extra': 'value'},
... ) # doctest: +SKIP
{"protocol": "hdfs", "username": "username", "password": "pwd",
"host": "node", "port": 123, "path": "/mnt/datasets/test.csv",
"url_query": "q=1", "extra": "value"}
"""
# Handle Windows paths including disk name in this special case
if (
re.match(r"^[a-zA-Z]:[\\/]", urlpath)
or re.match(r"^[a-zA-Z0-9]+://", urlpath) is None
):
return {"protocol": "file", "path": urlpath}
parsed_path = urlsplit(urlpath)
protocol = parsed_path.scheme or "file"
if parsed_path.fragment:
path = "#".join([parsed_path.path, parsed_path.fragment])
else:
path = parsed_path.path
if protocol == "file":
# Special case parsing file protocol URL on Windows according to:
# https://msdn.microsoft.com/en-us/library/jj710207.aspx
windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path)
if windows_path:
drive, path = windows_path.groups()
path = f"{drive}:{path}"
if protocol in ["http", "https"]:
# for HTTP, we don't want to parse, as requests will anyway
return {"protocol": protocol, "path": urlpath}
options: dict[str, Any] = {"protocol": protocol, "path": path}
if parsed_path.netloc:
# Parse `hostname` from netloc manually because `parsed_path.hostname`
# lowercases the hostname which is not always desirable (e.g. in S3):
# https://github.com/dask/dask/issues/1417
options["host"] = parsed_path.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0]
if protocol in ("s3", "s3a", "gcs", "gs"):
options["path"] = options["host"] + options["path"]
else:
options["host"] = options["host"]
if parsed_path.port:
options["port"] = parsed_path.port
if parsed_path.username:
options["username"] = parsed_path.username
if parsed_path.password:
options["password"] = parsed_path.password
if parsed_path.query:
options["url_query"] = parsed_path.query
if parsed_path.fragment:
options["url_fragment"] = parsed_path.fragment
if inherit_storage_options:
update_storage_options(options, inherit_storage_options)
return options
def update_storage_options(
options: dict[str, Any], inherited: dict[str, Any] | None = None
) -> None:
if not inherited:
inherited = {}
collisions = set(options) & set(inherited)
if collisions:
for collision in collisions:
if options.get(collision) != inherited.get(collision):
raise KeyError(
f"Collision between inferred and specified storage "
f"option:\n{collision}"
)
options.update(inherited)
# Compression extensions registered via fsspec.compression.register_compression
compressions: dict[str, str] = {}
def infer_compression(filename: str) -> str | None:
"""Infer compression, if available, from filename.
Infer a named compression type, if registered and available, from filename
extension. This includes builtin (gz, bz2, zip) compressions, as well as
optional compressions. See fsspec.compression.register_compression.
"""
extension = os.path.splitext(filename)[-1].strip(".").lower()
if extension in compressions:
return compressions[extension]
return None
def build_name_function(max_int: float) -> Callable[[int], str]:
"""Returns a function that receives a single integer
and returns it as a string padded by enough zero characters
to align with maximum possible integer
>>> name_f = build_name_function(57)
>>> name_f(7)
'07'
>>> name_f(31)
'31'
>>> build_name_function(1000)(42)
'0042'
>>> build_name_function(999)(42)
'042'
>>> build_name_function(0)(0)
'0'
"""
# handle corner cases max_int is 0 or exact power of 10
max_int += 1e-8
pad_length = int(math.ceil(math.log10(max_int)))
def name_function(i: int) -> str:
return str(i).zfill(pad_length)
return name_function
def seek_delimiter(file: IO[bytes], delimiter: bytes, blocksize: int) -> bool:
r"""Seek current file to file start, file end, or byte after delimiter seq.
Seeks file to next chunk delimiter, where chunks are defined on file start,
a delimiting sequence, and file end. Use file.tell() to see location afterwards.
Note that file start is a valid split, so must be at offset > 0 to seek for
delimiter.
Parameters
----------
file: a file
delimiter: bytes
a delimiter like ``b'\n'`` or message sentinel, matching file .read() type
blocksize: int
Number of bytes to read from the file at once.
Returns
-------
Returns True if a delimiter was found, False if at file start or end.
"""
if file.tell() == 0:
# beginning-of-file, return without seek
return False
# Interface is for binary IO, with delimiter as bytes, but initialize last
# with result of file.read to preserve compatibility with text IO.
last: bytes | None = None
while True:
current = file.read(blocksize)
if not current:
# end-of-file without delimiter
return False
full = last + current if last else current
try:
if delimiter in full:
i = full.index(delimiter)
file.seek(file.tell() - (len(full) - i) + len(delimiter))
return True
elif len(current) < blocksize:
# end-of-file without delimiter
return False
except (OSError, ValueError):
pass
last = full[-len(delimiter) :]
def read_block(
f: IO[bytes],
offset: int,
length: int | None,
delimiter: bytes | None = None,
split_before: bool = False,
) -> bytes:
"""Read a block of bytes from a file
Parameters
----------
f: File
Open file
offset: int
Byte offset to start read
length: int
Number of bytes to read, read through end of file if None
delimiter: bytes (optional)
Ensure reading starts and stops at delimiter bytestring
split_before: bool (optional)
Start/stop read *before* delimiter bytestring.
If using the ``delimiter=`` keyword argument we ensure that the read
starts and stops at delimiter boundaries that follow the locations
``offset`` and ``offset + length``. If ``offset`` is zero then we
start at zero, regardless of delimiter. The bytestring returned WILL
include the terminating delimiter string.
Examples
--------
>>> from io import BytesIO # doctest: +SKIP
>>> f = BytesIO(b'Alice, 100\\nBob, 200\\nCharlie, 300') # doctest: +SKIP
>>> read_block(f, 0, 13) # doctest: +SKIP
b'Alice, 100\\nBo'
>>> read_block(f, 0, 13, delimiter=b'\\n') # doctest: +SKIP
b'Alice, 100\\nBob, 200\\n'
>>> read_block(f, 10, 10, delimiter=b'\\n') # doctest: +SKIP
b'Bob, 200\\nCharlie, 300'
"""
if delimiter:
f.seek(offset)
found_start_delim = seek_delimiter(f, delimiter, 2**16)
if length is None:
return f.read()
start = f.tell()
length -= start - offset
f.seek(start + length)
found_end_delim = seek_delimiter(f, delimiter, 2**16)
end = f.tell()
# Adjust split location to before delimiter if seek found the
# delimiter sequence, not start or end of file.
if found_start_delim and split_before:
start -= len(delimiter)
if found_end_delim and split_before:
end -= len(delimiter)
offset = start
length = end - start
f.seek(offset)
# TODO: allow length to be None and read to the end of the file?
assert length is not None
b = f.read(length)
return b
def tokenize(*args: Any, **kwargs: Any) -> str:
"""Deterministic token
(modified from dask.base)
>>> tokenize([1, 2, '3'])
'9d71491b50023b06fc76928e6eddb952'
>>> tokenize('Hello') == tokenize('Hello')
True
"""
if kwargs:
args += (kwargs,)
try:
h = md5(str(args).encode())
except ValueError:
# FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380
h = md5(str(args).encode(), usedforsecurity=False)
return h.hexdigest()
def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str:
"""Attempt to convert a path-like object to a string.
Parameters
----------
filepath: object to be converted
Returns
-------
filepath_str: maybe a string version of the object
Notes
-----
Objects supporting the fspath protocol are coerced according to its
__fspath__ method.
For backwards compatibility with older Python version, pathlib.Path
objects are specially coerced.
Any other object is passed through unchanged, which includes bytes,
strings, buffers, or anything else that's not even path-like.
"""
if isinstance(filepath, str):
return filepath
elif hasattr(filepath, "__fspath__"):
return filepath.__fspath__()
elif hasattr(filepath, "path"):
return filepath.path
else:
return filepath # type: ignore[return-value]
def make_instance(
cls: Callable[..., T], args: Sequence[Any], kwargs: dict[str, Any]
) -> T:
inst = cls(*args, **kwargs)
inst._determine_worker() # type: ignore[attr-defined]
return inst
def common_prefix(paths: Iterable[str]) -> str:
"""For a list of paths, find the shortest prefix common to all"""
parts = [p.split("/") for p in paths]
lmax = min(len(p) for p in parts)
end = 0
for i in range(lmax):
end = all(p[i] == parts[0][i] for p in parts)
if not end:
break
i += end
return "/".join(parts[0][:i])
def other_paths(
paths: list[str],
path2: str | list[str],
exists: bool = False,
flatten: bool = False,
) -> list[str]:
"""In bulk file operations, construct a new file tree from a list of files
Parameters
----------
paths: list of str
The input file tree
path2: str or list of str
Root to construct the new list in. If this is already a list of str, we just
assert it has the right number of elements.
exists: bool (optional)
For a str destination, it is already exists (and is a dir), files should
end up inside.
flatten: bool (optional)
Whether to flatten the input directory tree structure so that the output files
are in the same directory.
Returns
-------
list of str
"""
if isinstance(path2, str):
path2 = path2.rstrip("/")
if flatten:
path2 = ["/".join((path2, p.split("/")[-1])) for p in paths]
else:
cp = common_prefix(paths)
if exists:
cp = cp.rsplit("/", 1)[0]
if not cp and all(not s.startswith("/") for s in paths):
path2 = ["/".join([path2, p]) for p in paths]
else:
path2 = [p.replace(cp, path2, 1) for p in paths]
else:
assert len(paths) == len(path2)
return path2
def is_exception(obj: Any) -> bool:
return isinstance(obj, BaseException)
def isfilelike(f: Any) -> TypeGuard[IO[bytes]]:
return all(hasattr(f, attr) for attr in ["read", "close", "tell"])
def get_protocol(url: str) -> str:
url = stringify_path(url)
parts = re.split(r"(\:\:|\://)", url, maxsplit=1)
if len(parts) > 1:
return parts[0]
return "file"
def can_be_local(path: str) -> bool:
"""Can the given URL be used with open_local?"""
from fsspec import get_filesystem_class
try:
return getattr(get_filesystem_class(get_protocol(path)), "local_file", False)
except (ValueError, ImportError):
# not in registry or import failed
return False
def get_package_version_without_import(name: str) -> str | None:
"""For given package name, try to find the version without importing it
Import and package.__version__ is still the backup here, so an import
*might* happen.
Returns either the version string, or None if the package
or the version was not readily found.
"""
if name in sys.modules:
mod = sys.modules[name]
if hasattr(mod, "__version__"):
return mod.__version__
try:
return version(name)
except: # noqa: E722
pass
try:
import importlib
mod = importlib.import_module(name)
return mod.__version__
except (ImportError, AttributeError):
return None
def setup_logging(
logger: logging.Logger | None = None,
logger_name: str | None = None,
level: str = "DEBUG",
clear: bool = True,
) -> logging.Logger:
if logger is None and logger_name is None:
raise ValueError("Provide either logger object or logger name")
logger = logger or logging.getLogger(logger_name)
handle = logging.StreamHandler()
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s"
)
handle.setFormatter(formatter)
if clear:
logger.handlers.clear()
logger.addHandler(handle)
logger.setLevel(level)
return logger
def _unstrip_protocol(name: str, fs: AbstractFileSystem) -> str:
return fs.unstrip_protocol(name)
def mirror_from(
origin_name: str, methods: Iterable[str]
) -> Callable[[type[T]], type[T]]:
"""Mirror attributes and methods from the given
origin_name attribute of the instance to the
decorated class"""
def origin_getter(method: str, self: Any) -> Any:
origin = getattr(self, origin_name)
return getattr(origin, method)
def wrapper(cls: type[T]) -> type[T]:
for method in methods:
wrapped_method = partial(origin_getter, method)
setattr(cls, method, property(wrapped_method))
return cls
return wrapper
@contextlib.contextmanager
def nullcontext(obj: T) -> Iterator[T]:
yield obj
def merge_offset_ranges(
paths: list[str],
starts: list[int] | int,
ends: list[int] | int,
max_gap: int = 0,
max_block: int | None = None,
sort: bool = True,
) -> tuple[list[str], list[int], list[int]]:
"""Merge adjacent byte-offset ranges when the inter-range
gap is <= `max_gap`, and when the merged byte range does not
exceed `max_block` (if specified). By default, this function
will re-order the input paths and byte ranges to ensure sorted
order. If the user can guarantee that the inputs are already
sorted, passing `sort=False` will skip the re-ordering.
"""
# Check input
if not isinstance(paths, list):
raise TypeError
if not isinstance(starts, list):
starts = [starts] * len(paths)
if not isinstance(ends, list):
ends = [ends] * len(paths)
if len(starts) != len(paths) or len(ends) != len(paths):
raise ValueError
# Early Return
if len(starts) <= 1:
return paths, starts, ends
starts = [s or 0 for s in starts]
# Sort by paths and then ranges if `sort=True`
if sort:
paths, starts, ends = (
list(v)
for v in zip(
*sorted(
zip(paths, starts, ends),
)
)
)
if paths:
# Loop through the coupled `paths`, `starts`, and
# `ends`, and merge adjacent blocks when appropriate
new_paths = paths[:1]
new_starts = starts[:1]
new_ends = ends[:1]
for i in range(1, len(paths)):
if paths[i] == paths[i - 1] and new_ends[-1] is None:
continue
elif (
paths[i] != paths[i - 1]
or ((starts[i] - new_ends[-1]) > max_gap)
or (max_block is not None and (ends[i] - new_starts[-1]) > max_block)
):
# Cannot merge with previous block.
# Add new `paths`, `starts`, and `ends` elements
new_paths.append(paths[i])
new_starts.append(starts[i])
new_ends.append(ends[i])
else:
# Merge with previous block by updating the
# last element of `ends`
new_ends[-1] = ends[i]
return new_paths, new_starts, new_ends
# `paths` is empty. Just return input lists
return paths, starts, ends
def file_size(filelike: IO[bytes]) -> int:
"""Find length of any open read-mode file-like"""
pos = filelike.tell()
try:
return filelike.seek(0, 2)
finally:
filelike.seek(pos)
@contextlib.contextmanager
def atomic_write(path: str, mode: str = "wb"):
"""
A context manager that opens a temporary file next to `path` and, on exit,
replaces `path` with the temporary file, thereby updating `path`
atomically.
"""
fd, fn = tempfile.mkstemp(
dir=os.path.dirname(path), prefix=os.path.basename(path) + "-"
)
try:
with open(fd, mode) as fp:
yield fp
except BaseException:
with contextlib.suppress(FileNotFoundError):
os.unlink(fn)
raise
else:
os.replace(fn, path)
def _translate(pat, STAR, QUESTION_MARK):
# Copied from: https://github.com/python/cpython/pull/106703.
res: list[str] = []
add = res.append
i, n = 0, len(pat)
while i < n:
c = pat[i]
i = i + 1
if c == "*":
# compress consecutive `*` into one
if (not res) or res[-1] is not STAR:
add(STAR)
elif c == "?":
add(QUESTION_MARK)
elif c == "[":
j = i
if j < n and pat[j] == "!":
j = j + 1
if j < n and pat[j] == "]":
j = j + 1
while j < n and pat[j] != "]":
j = j + 1
if j >= n:
add("\\[")
else:
stuff = pat[i:j]
if "-" not in stuff:
stuff = stuff.replace("\\", r"\\")
else:
chunks = []
k = i + 2 if pat[i] == "!" else i + 1
while True:
k = pat.find("-", k, j)
if k < 0:
break
chunks.append(pat[i:k])
i = k + 1
k = k + 3
chunk = pat[i:j]
if chunk:
chunks.append(chunk)
else:
chunks[-1] += "-"
# Remove empty ranges -- invalid in RE.
for k in range(len(chunks) - 1, 0, -1):
if chunks[k - 1][-1] > chunks[k][0]:
chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:]
del chunks[k]
# Escape backslashes and hyphens for set difference (--).
# Hyphens that create ranges shouldn't be escaped.
stuff = "-".join(
s.replace("\\", r"\\").replace("-", r"\-") for s in chunks
)
# Escape set operations (&&, ~~ and ||).
stuff = re.sub(r"([&~|])", r"\\\1", stuff)
i = j + 1
if not stuff:
# Empty range: never match.
add("(?!)")
elif stuff == "!":
# Negated empty range: match any character.
add(".")
else:
if stuff[0] == "!":
stuff = "^" + stuff[1:]
elif stuff[0] in ("^", "["):
stuff = "\\" + stuff
add(f"[{stuff}]")
else:
add(re.escape(c))
assert i == n
return res
def glob_translate(pat):
# Copied from: https://github.com/python/cpython/pull/106703.
# The keyword parameters' values are fixed to:
# recursive=True, include_hidden=True, seps=None
"""Translate a pathname with shell wildcards to a regular expression."""
if os.path.altsep:
seps = os.path.sep + os.path.altsep
else:
seps = os.path.sep
escaped_seps = "".join(map(re.escape, seps))
any_sep = f"[{escaped_seps}]" if len(seps) > 1 else escaped_seps
not_sep = f"[^{escaped_seps}]"
one_last_segment = f"{not_sep}+"
one_segment = f"{one_last_segment}{any_sep}"
any_segments = f"(?:.+{any_sep})?"
any_last_segments = ".*"
results = []
parts = re.split(any_sep, pat)
last_part_idx = len(parts) - 1
for idx, part in enumerate(parts):
if part == "*":
results.append(one_segment if idx < last_part_idx else one_last_segment)
continue
if part == "**":
results.append(any_segments if idx < last_part_idx else any_last_segments)
continue
elif "**" in part:
raise ValueError(
"Invalid pattern: '**' can only be an entire path component"
)
if part:
results.extend(_translate(part, f"{not_sep}*", not_sep))
if idx < last_part_idx:
results.append(any_sep)
res = "".join(results)
return rf"(?s:{res})\Z"
filesystem_spec-2025.3.2/install_s3fs.sh 0000664 0000000 0000000 00000000745 14772531734 0020126 0 ustar 00root root 0000000 0000000 # s3fs tests are not currently accessible to import via a normal install.
# We need to instead do an editable install. Further, s3fs is very specific
# about the version of fsspec that it installs.
rm -r -f downstream
mkdir ./downstream
# Download source to get tests
git clone https://github.com/fsspec/s3fs
mv s3fs ./downstream/s3fs
# s3fs is pinned to a specific version of fsspec
pip install --no-input -e ./downstream/s3fs # installs all deps, including latest released fsspec
filesystem_spec-2025.3.2/pyproject.toml 0000664 0000000 0000000 00000011570 14772531734 0020100 0 ustar 00root root 0000000 0000000 [build-system]
requires = ["hatchling", "hatch-vcs"]
build-backend = "hatchling.build"
[project]
name = "fsspec"
dynamic = ["version"]
description = "File-system specification"
readme = "README.md"
license = { file = "LICENSE" }
requires-python = ">=3.9"
maintainers = [{ name = "Martin Durant", email = "mdurant@anaconda.com" }]
keywords = ["file"]
classifiers = [
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
]
[project.optional-dependencies]
abfs = ["adlfs"]
adl = ["adlfs"]
arrow = ["pyarrow >= 1"]
dask = ["dask", "distributed"]
dev = ["ruff", "pre-commit"]
doc = ["sphinx", "numpydoc", "sphinx-design", "sphinx-rtd-theme", "yarl"]
dropbox = ["dropbox", "dropboxdrivefs", "requests"]
entrypoints = []
full = [
'adlfs',
'aiohttp !=4.0.0a0, !=4.0.0a1',
'dask',
'distributed',
'dropbox',
'dropboxdrivefs',
'fusepy',
'gcsfs',
'libarchive-c',
'ocifs',
'panel',
'paramiko',
'pyarrow >= 1',
'pygit2',
'requests',
's3fs',
'smbprotocol',
'tqdm',
]
fuse = ["fusepy"]
gcs = ["gcsfs"]
git = ["pygit2"]
github = ["requests"]
gs = ["gcsfs"]
gui = ["panel"]
hdfs = ["pyarrow >= 1"]
http = ["aiohttp !=4.0.0a0, !=4.0.0a1"]
libarchive = ["libarchive-c"]
oci = ["ocifs"]
s3 = ["s3fs"]
sftp = ["paramiko"]
smb = ["smbprotocol"]
ssh = ["paramiko"]
test = [
"aiohttp!=4.0.0a0, !=4.0.0a1",
"numpy",
"pytest",
"requests",
"pytest-benchmark",
"pytest-cov",
"pytest-mock",
"pytest-recording",
"pytest-rerunfailures",
"pytest-asyncio !=0.22.0",
]
test_full = [
'Jinja2',
'adlfs',
'aiohttp !=4.0.0a0, !=4.0.0a1',
'aiohttp!=4.0.0a0, !=4.0.0a1',
'cloudpickle',
'dask',
'distributed',
'dropbox',
'dropboxdrivefs',
'fastparquet',
'fusepy',
'gcsfs',
"kerchunk",
'libarchive-c',
'lz4',
'notebook',
'numpy',
'ocifs',
'pandas',
'panel',
'paramiko',
'pyarrow',
'pyarrow >= 1',
'pyftpdlib',
'pygit2',
'pytest',
'pytest-asyncio !=0.22.0',
'pytest-benchmark',
'pytest-cov',
'pytest-mock',
'pytest-rerunfailures',
'pytest-recording',
'requests',
'smbprotocol',
'python-snappy',
'tqdm',
'urllib3',
'zarr',
'zstandard',
]
test_downstream = [
"dask[dataframe,test]",
"moto[server]>4,<5",
"pytest-timeout",
"xarray",
"aiobotocore>=2.5.4,<3.0.0",
]
tqdm = ["tqdm"]
[project.urls]
Changelog = "https://filesystem-spec.readthedocs.io/en/latest/changelog.html"
Documentation = "https://filesystem-spec.readthedocs.io/en/latest/"
Homepage = "https://github.com/fsspec/filesystem_spec"
[tool.hatch.version]
source = "vcs"
raw-options = {'version_scheme'='post-release'}
[tool.hatch.build.hooks.vcs]
version-file = "fsspec/_version.py"
[tool.hatch.build]
exclude = ["**/tests/*", "!**/tests/abstract/"]
[tool.ruff]
target-version = "py38"
exclude = [".tox", "build", "docs/source/conf.py", "fsspec/_version"]
line-length = 88
[tool.ruff.lint]
select = [
# fix noqas in fsspec/implementations/http.py
"ASYNC",
"B",
"B904",
"C4",
"G",
"E4",
"E7",
"E9",
"F",
"FURB",
"I",
"LOG",
"PERF",
"PLC",
"PLE",
"PLR1722",
"PLR1736",
"PLW1501",
"PLW1510",
"PLW3301",
"PIE800",
"PIE804",
"PIE807",
"PIE810",
# "PT", enable in later PR
"PYI",
"RUF006",
"RUF015",
"RUF024",
"SIM",
"SLOT",
"SIM101",
"TCH",
"UP",
]
ignore = [
# Loop control variable `loop` not used within loop body
"B007",
# Use of `functools.lru_cache` or `functools.cache` on methods can lead to memory leaks
"B019",
# Star-arg unpacking after a keyword argument is strongly discouraged
"B026",
# No explicit `stacklevel` keyword argument found
"B028",
# Assigning lambda expression
"E731",
# Ambiguous variable names
"E741",
# line break before binary operator
# uncomment when implemented in ruff
# "W503",
# whitespace before :
"E203",
# redefs
"F811",
# Fix these codes later
"G004",
"PERF203",
"UP007",
"UP011",
"UP015",
"UP018",
"SIM102",
"SIM105",
"SIM108",
"SIM114",
"SIM115",
"SIM117",
# https://github.com/astral-sh/ruff/issues/7871
"UP038",
# https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
"W191",
"E111",
"E114",
"E117",
"D206",
"D300",
"Q000",
"Q001",
"Q002",
"Q003",
"COM812",
"COM819",
]
[tool.pytest.ini_options]
# custom markers, need to be defined to avoid warnings
markers = ["asyncio", "vcr"]
filesystem_spec-2025.3.2/readthedocs.yml 0000664 0000000 0000000 00000000430 14772531734 0020165 0 ustar 00root root 0000000 0000000 version: 2
build:
os: ubuntu-22.04
tools:
python: miniconda3-4.7
conda:
environment: docs/environment.yml
python:
install:
- method: pip
path: .
extra_requirements:
- doc
sphinx:
configuration: docs/source/conf.py
fail_on_warning: true
filesystem_spec-2025.3.2/setup.cfg 0000664 0000000 0000000 00000002253 14772531734 0017003 0 ustar 00root root 0000000 0000000 [metadata]
long_description: file: README.rst
[flake8]
exclude = .tox,build,docs/source/conf.py,fsspec/_version
max-line-length = 88
ignore =
# Assigning lambda expression
E731
# Ambiguous variable names
E741
# line break before binary operator
W503
# whitespace before :
E203
# redefs
F811
[isort]
known_first_party=fsspec
known_third_party=aiohttp,dask,distributed,fuse,libarchive,numpy,panel,paramiko,pygit2,pytest,requests,smbclient,yarl
multi_line_output=3
include_trailing_comma=True
force_grid_wrap=0
combine_as_imports=True
line_length=88
skip=
.tox
build
docs/source/conf.py
fsspec/_version
[mypy]
follow_imports = normal
ignore_missing_imports = True
enable_error_code = ignore-without-code,truthy-bool,truthy-iterable,unused-awaitable
disallow_untyped_decorators = False
strict_equality = False
warn_redundant_casts = True
warn_unused_configs = True
warn_unused_ignores = True
# don't bother type-checking test_*.py or conftest.py files
exclude = (test.*|conftest)\.py$
# TODO: turn on globally once we type enough of fsspec
[mypy-fsspec.caching]
check_untyped_defs = True
[mypy-fsspec.utils]
check_untyped_defs = True
