pax_global_header00006660000000000000000000000064147067542350014527gustar00rootroot0000000000000052 comment=561ce1e20600bab42133b5e51511483800d3ef76 s3fs-fuse-1.95/000077500000000000000000000000001470675423500133035ustar00rootroot00000000000000s3fs-fuse-1.95/.clang-tidy000066400000000000000000000056271470675423500153510ustar00rootroot00000000000000WarningsAsErrors: '*' Checks: ' *, -abseil-*, -altera-*, -android-*, -boost-*, -bugprone-assignment-in-if-condition, -bugprone-branch-clone, -bugprone-easily-swappable-parameters, -bugprone-implicit-widening-of-multiplication-result, -bugprone-macro-parentheses, -bugprone-narrowing-conversions, -bugprone-unhandled-self-assignment, -cert-dcl50-cpp, -cert-env33-c, -cert-err33-c, -cert-err58-cpp, -clang-analyzer-*, -concurrency-mt-unsafe, -cppcoreguidelines-avoid-c-arrays, -cppcoreguidelines-avoid-do-while, -cppcoreguidelines-avoid-magic-numbers, -cppcoreguidelines-avoid-non-const-global-variables, -cppcoreguidelines-init-variables, -cppcoreguidelines-macro-usage, -cppcoreguidelines-no-malloc, -cppcoreguidelines-owning-memory, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, -cppcoreguidelines-pro-bounds-constant-array-index, -cppcoreguidelines-pro-bounds-pointer-arithmetic, -cppcoreguidelines-pro-type-member-init, -cppcoreguidelines-pro-type-reinterpret-cast, -cppcoreguidelines-pro-type-union-access, -cppcoreguidelines-pro-type-vararg, -cppcoreguidelines-special-member-functions, -fuchsia-*, -google-build-using-namespace, -google-readability-casting, -google-readability-function-size, -google-readability-todo, -google-runtime-int, -google-runtime-references, -hicpp-*, -llvm-*, -llvmlibc-*, -misc-const-correctness, -misc-include-cleaner, -misc-no-recursion, -misc-non-private-member-variables-in-classes, -misc-redundant-expression, -misc-unused-parameters, -misc-use-anonymous-namespace, -misc-use-internal-linkage, -modernize-avoid-c-arrays, -modernize-loop-convert, -modernize-make-unique, -modernize-use-nodiscard, -modernize-raw-string-literal, -modernize-return-braced-init-list, -modernize-use-default-member-init, -modernize-use-trailing-return-type, -modernize-use-using, -performance-avoid-endl, -performance-no-int-to-ptr, -readability-avoid-nested-conditional-operator, -readability-braces-around-statements, -readability-else-after-return, -readability-function-cognitive-complexity, -readability-function-size, -readability-identifier-length, -readability-inconsistent-declaration-parameter-name, -readability-isolate-declaration, -readability-magic-numbers, -readability-math-missing-parentheses, -readability-named-parameter, -readability-redundant-access-specifiers, -readability-redundant-declaration, -readability-simplify-boolean-expr, -readability-suspicious-call-argument' CheckOptions: cppcoreguidelines-narrowing-conversions.WarnOnEquivalentBitWidth: 'false' readability-implicit-bool-conversion.AllowIntegerConditions: 'true' readability-implicit-bool-conversion.AllowPointerConditions: 'true' s3fs-fuse-1.95/.gitattributes000066400000000000000000000017201470675423500161760ustar00rootroot00000000000000# # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright(C) 2007 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # * text eol=lf *.png binary # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: noet sw=4 ts=4 fdm=marker # vim<600: noet sw=4 ts=4 # s3fs-fuse-1.95/.github/000077500000000000000000000000001470675423500146435ustar00rootroot00000000000000s3fs-fuse-1.95/.github/ISSUE_TEMPLATE.md000066400000000000000000000026231470675423500173530ustar00rootroot00000000000000 ### Additional Information #### Version of s3fs being used (`s3fs --version`) #### Version of fuse being used (`pkg-config --modversion fuse`, `rpm -qi fuse` or `dpkg -s fuse`) #### Kernel information (`uname -r`) #### GNU/Linux Distribution, if applicable (`cat /etc/os-release`) #### How to run s3fs, if applicable [] command line [] /etc/fstab ``` ``` #### s3fs syslog messages (`grep s3fs /var/log/syslog`, `journalctl | grep s3fs`, or `s3fs outputs`) ``` ``` ### Details about issue s3fs-fuse-1.95/.github/PULL_REQUEST_TEMPLATE.md000066400000000000000000000006741470675423500204530ustar00rootroot00000000000000 ### Relevant Issue (if applicable) ### Details s3fs-fuse-1.95/.github/workflows/000077500000000000000000000000001470675423500167005ustar00rootroot00000000000000s3fs-fuse-1.95/.github/workflows/ci.yml000066400000000000000000000250731470675423500200250ustar00rootroot00000000000000# # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright(C) 2007 Takeshi Nakatani # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # name: s3fs-fuse CI on: push: pull_request: # # CRON event is fired on every sunday (UTC). # schedule: - cron: '0 0 * * 0' # # Jobs # jobs: Linux: runs-on: ubuntu-latest # # build matrix for containers # strategy: # # do not stop jobs automatically if any of the jobs fail # fail-fast: false # # matrix for containers # matrix: container: - ubuntu:24.04 - ubuntu:22.04 - ubuntu:20.04 - debian:bookworm - debian:bullseye - rockylinux:9 - rockylinux:8 - fedora:40 - fedora:39 - opensuse/leap:15 - alpine:3.20 - centos:centos7 container: image: ${{ matrix.container }} options: "--privileged --cap-add SYS_ADMIN --device /dev/fuse" env: # [NOTE] # Installation special environment variables for debian and ubuntu. # DEBIAN_FRONTEND: noninteractive # [NOTE] # actions/checkout uses node20, but can only run up to node16 on centos7. # (glibc 2.27 or later is required to use node20) # This is a temporary solution and will be removed when centos7 support is discontinued. # ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: ${{ matrix.container == 'centos:centos7' && true || false }} steps: # [NOTE] # On openSUSE, tar and gzip must be installed before action/checkout. # - name: Install packages before checkout if: matrix.container == 'opensuse/leap:15' run: | zypper install -y tar gzip # [NOTE] # Use actions/checkout@v4 except for centos7. # On centos7, we need to run actions/checkout@v3 and configure it to use node16. # - name: Checkout source code(other than centos7) if: matrix.container != 'centos:centos7' uses: actions/checkout@v4 - name: Checkout source code(only centos7) if: matrix.container == 'centos:centos7' uses: actions/checkout@v3 # [NOTE] # Matters that depend on OS:VERSION are determined and executed in the following script. # Please note that the option to configure (CONFIGURE_OPTIONS) is set in the environment variable. # - name: Install packages run: | .github/workflows/linux-ci-helper.sh ${{ matrix.container }} - name: Build run: | ./autogen.sh /bin/sh -c "./configure ${CONFIGURE_OPTIONS}" make --jobs=$(nproc) - name: Cppcheck run: | # specify the version range to run cppcheck (cppcheck version number is x.y or x.y.z) if cppcheck --version | sed -e 's/\./ /g' | awk '{if (($2 * 1000 + $3) <= 2004) { exit(1) } }'; then make cppcheck fi - name: Shellcheck run: | if shellcheck --version | awk -F '[ .]' '/version:/ && ($2 * 1000 + $3 <= 7) { exit(1) }'; then make shellcheck fi - name: Test suite run: | make check -C src make ALL_TESTS=1 check -C test || (test/filter-suite-log.sh test/test-suite.log; exit 1) # [NOTE] # Using macos-fuse-t # This product(package) is a workaround for osxfuse which required an OS reboot(macos 11 and later). # see. https://github.com/macos-fuse-t/fuse-t # About osxfuse # This job doesn't work with GitHub Actions using macOS 11+ because "load_osxfuse" returns # "exit code = 1".(requires OS reboot) # macos12: runs-on: macos-12 steps: - name: Checkout source code uses: actions/checkout@v4 - name: Brew tap run: | TAPS="$(brew --repository)/Library/Taps"; if [ -e "$TAPS/caskroom/homebrew-cask" ]; then rm -rf "$TAPS/caskroom/homebrew-cask"; fi; HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask HOMEBREW_NO_AUTO_UPDATE=1 brew tap macos-fuse-t/homebrew-cask - name: Install fuse-t run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install fuse-t - name: Install brew other packages run: | S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed shellcheck jq'; for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi done - name: Install awscli2 run: | cd /tmp curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" sudo installer -pkg AWSCLIV2.pkg -target / - name: Build run: | ./autogen.sh PKG_CONFIG_PATH=/usr/local/opt/curl/lib/pkgconfig:/usr/local/opt/openssl/lib/pkgconfig ./configure CXXFLAGS='-std=c++11 -DS3FS_PTHREAD_ERRORCHECK=1' make --jobs=$(sysctl -n hw.ncpu) - name: Cppcheck run: | # specify the version range to run cppcheck (cppcheck version number is x.y or x.y.z) if cppcheck --version | sed -e 's/\./ /g' | awk '{if (($2 * 1000 + $3) <= 2004) { exit(1) } }'; then make cppcheck fi - name: Shellcheck run: | if shellcheck --version | awk -F '[ .]' '/version:/ && ($2 * 1000 + $3 <= 7) { exit(1) }'; then make shellcheck fi - name: Test suite run: | make check -C src make ALL_TESTS=1 check -C test || (test/filter-suite-log.sh test/test-suite.log; exit 1) MemoryTest: runs-on: ubuntu-latest # # build matrix for containers # strategy: # # do not stop jobs automatically if any of the jobs fail # fail-fast: false # # matrix for type of checking # # [NOTE] # Currently following test is not supported: # - sanitize_memory : Future support planned # matrix: checktype: - glibc_debug - sanitize_address - sanitize_others - sanitize_thread - thread_safety - valgrind container: image: fedora:40 options: "--privileged --cap-add SYS_ADMIN --device /dev/fuse" steps: - name: Checkout source code uses: actions/checkout@v4 - name: Install packages run: | .github/workflows/linux-ci-helper.sh fedora:40 - name: Install clang run: | dnf install -y clang if [ "${{ matrix.checktype }}" = "valgrind" ]; then dnf install -y valgrind fi # # Set CXX/CXXFLAGS and Variables for test # - name: Set variables run: | COMMON_CXXFLAGS='-g -Wno-cpp -DS3FS_PTHREAD_ERRORCHECK=1' { if [ "${{ matrix.checktype }}" = "glibc_debug" ]; then echo "CXXFLAGS=${COMMON_CXXFLAGS} -O0 -D_GLIBCXX_DEBUG" elif [ "${{ matrix.checktype }}" = "sanitize_address" ]; then echo 'CXX=clang++' echo "CXXFLAGS=${COMMON_CXXFLAGS} -O0 -fsanitize=address -fsanitize-address-use-after-scope" echo 'ASAN_OPTIONS=detect_leaks=1,detect_stack_use_after_return=1' elif [ "${{ matrix.checktype }}" = "sanitize_memory" ]; then echo 'CXX=clang++' echo "CXXFLAGS=${COMMON_CXXFLAGS} -O0 -fsanitize=memory" elif [ "${{ matrix.checktype }}" = "sanitize_thread" ]; then echo 'CXX=clang++' echo "CXXFLAGS=${COMMON_CXXFLAGS} -O0 -fsanitize=thread" echo 'TSAN_OPTIONS=halt_on_error=1,suppressions=threadsanitizer_suppressions.txt' # [NOTE] # Set this to avoid following error when running configure. # "FATAL: ThreadSanitizer: unexpected memory mapping" sysctl vm.mmap_rnd_bits=28 elif [ "${{ matrix.checktype }}" = "sanitize_others" ]; then echo 'CXX=clang++' echo "CXXFLAGS=${COMMON_CXXFLAGS} -O1 -fsanitize=undefined,implicit-conversion,local-bounds,unsigned-integer-overflow" elif [ "${{ matrix.checktype }}" = "thread_safety" ]; then echo 'CXX=clang++' echo "CXXFLAGS=${COMMON_CXXFLAGS} -O1 -Wthread-safety -stdlib=libc++ -D_LIBCPP_ENABLE_THREAD_SAFETY_ANNOTATIONS -Werror" echo 'LDFLAGS=-DCLANG_DEFAULT_LINKER=lld' elif [ "${{ matrix.checktype }}" = "valgrind" ]; then echo "CXXFLAGS=${COMMON_CXXFLAGS} -O1" echo 'VALGRIND="--leak-check=full --error-exitcode=1"' echo 'RETRIES=100' echo 'S3_URL=http://127.0.0.1:8081' fi } >> "$GITHUB_ENV" - name: Build run: | ./autogen.sh /bin/sh -c "CXX=${CXX} CXXFLAGS=\"${CXXFLAGS}\" LDFLAGS=\"${LDFLAGS}\" TSAN_OPTIONS=\"\" ./configure --prefix=/usr --with-openssl" make - name: Test suite run: | /bin/sh -c "ALL_TESTS=1 ASAN_OPTIONS=${ASAN_OPTIONS} TSAN_OPTIONS=${TSAN_OPTIONS} VALGRIND=${VALGRIND} RETRIES=${RETRIES} make check -C test || (test/filter-suite-log.sh test/test-suite.log; exit 1)" clang-tidy: runs-on: ubuntu-latest container: image: fedora:40 steps: - name: Checkout source code uses: actions/checkout@v4 - name: Install packages run: | .github/workflows/linux-ci-helper.sh fedora:40 - name: Build run: | ./autogen.sh /bin/sh -c "./configure ${CONFIGURE_OPTIONS}" make --jobs=$(nproc) - name: clang-tidy run: | make -C src/ clang-tidy make -C test/ clang-tidy # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/.github/workflows/linux-ci-helper.sh000077500000000000000000000340751470675423500222550ustar00rootroot00000000000000#!/bin/sh # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright(C) 2007 Takeshi Nakatani # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # [NOTE] # Since bash is not present in some Runner containers, this script # runs in sh. # pipefail etc. are not native variables of sh. It exists in bash's # sh compatibility mode, but doesn't work in sh compatibility mode # of ash such as alpine. # However, it's not fatal that pipefail doesn't work for this script. # set -o errexit set -o nounset #set -o pipefail #----------------------------------------------------------- # Common variables #----------------------------------------------------------- PRGNAME=$(basename "$0") echo "${PRGNAME} [INFO] Start Linux helper for installing packages." #----------------------------------------------------------- # Parameter check #----------------------------------------------------------- # # Usage: ${PRGNAME} "OS:VERSION" # if [ $# -ne 1 ]; then echo "${PRGNAME} [ERROR] No container name options specified." fi #----------------------------------------------------------- # Container OS variables #----------------------------------------------------------- CONTAINER_FULLNAME=$1 # shellcheck disable=SC2034 CONTAINER_OSNAME=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $1}') # shellcheck disable=SC2034 CONTAINER_OSVERSION=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $2}') #----------------------------------------------------------- # Common variables for awscli2 #----------------------------------------------------------- AWSCLI_URI="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" AWSCLI_ZIP_FILE="awscliv2.zip" #----------------------------------------------------------- # Parameters for configure(set environments) #----------------------------------------------------------- CXX="g++" CXXFLAGS="-O -DS3FS_PTHREAD_ERRORCHECK=1" LDFLAGS="" CONFIGURE_OPTIONS="--prefix=/usr --with-openssl" #----------------------------------------------------------- # OS dependent variables #----------------------------------------------------------- # # Default values # PACKAGE_ENABLE_REPO_OPTIONS="" PACKAGE_INSTALL_ADDITIONAL_OPTIONS="" SHELLCHECK_DIRECT_INSTALL=0 AWSCLI_DIRECT_INSTALL=1 if [ "${CONTAINER_FULLNAME}" = "ubuntu:24.04" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="autoconf autotools-dev clang-tidy openjdk-21-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mailcap libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "ubuntu:22.04" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="autoconf autotools-dev clang-tidy openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "debian:bookworm" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="autoconf autotools-dev clang-tidy openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "rockylinux:9" ]; then PACKAGE_MANAGER_BIN="dnf" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" PACKAGE_ENABLE_REPO_OPTIONS="--enablerepo=crb" # [NOTE] # Rocky Linux 9 (or CentOS Stream 9) images may have curl installation issues that # conflict with the curl-minimal package. # PACKAGE_INSTALL_ADDITIONAL_OPTIONS="--allowerasing" INSTALL_PACKAGES="clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-17-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel attr diffutils curl python3 procps unzip xz https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm" INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel" # [NOTE] # For RockyLinux, ShellCheck is downloaded from the github archive and installed. # SHELLCHECK_DIRECT_INSTALL=1 elif [ "${CONTAINER_FULLNAME}" = "rockylinux:8" ]; then PACKAGE_MANAGER_BIN="dnf" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-17-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel attr diffutils curl python3 unzip" INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools" # [NOTE] # For RockyLinux, ShellCheck is downloaded from the github archive and installed. # SHELLCHECK_DIRECT_INSTALL=1 elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then # [NOTE] # CentOS 7 will reach EOL on June 30, 2024. # After EOL, we will prepare the environment for building and testing using the Vault repository. # The following process switches from the EOL repository to Vault. # grep -v '^enabled=' /etc/yum.repos.d/CentOS-Base.repo | sed -e 's|gpgcheck=1|gpgcheck=1\nenabled=0|g' > /tmp/CentOS-Base.repo cat /tmp/CentOS-Base.repo > /etc/yum.repos.d/CentOS-Base.repo { # # CentOS-Vault.repo does not have C7.9.2009 entry, so add it. # echo '' echo '# C7.9.2009' echo '[C7.9.2009-base]' echo 'name=CentOS-7.9.2009 - Base' echo "baseurl=http://vault.centos.org/7.9.2009/os/\$basearch/" echo 'gpgcheck=1' echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7' echo 'enabled=1' echo '' echo '[C7.9.2009-updates]' echo 'name=CentOS-7.9.2009 - Updates' echo "baseurl=http://vault.centos.org/7.9.2009/updates/\$basearch/" echo 'gpgcheck=1' echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7' echo 'enabled=1' echo '' echo '[C7.9.2009-extras]' echo 'name=CentOS-7.9.2009 - Extras' echo "baseurl=http://vault.centos.org/7.9.2009/extras/\$basearch/" echo 'gpgcheck=1' echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7' echo 'enabled=1' echo '' echo '[C7.9.2009-centosplus]' echo 'name=CentOS-7.9.2009 - CentOSPlus' echo "baseurl=http://vault.centos.org/7.9.2009/centosplus/\$basearch/" echo 'gpgcheck=1' echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7' echo 'enabled=1' echo '' echo '[C7.9.2009-fasttrack]' echo 'name=CentOS-7.9.2009 - Fasttrack' echo "baseurl=http://vault.centos.org/7.9.2009/fasttrack/\$basearch/" echo 'gpgcheck=1' echo 'gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7' echo 'enabled=1' echo '' } >> /etc/yum.repos.d/CentOS-Base.repo yum clean all yum update -q -y # # Finish change package repo to Vault. # PACKAGE_MANAGER_BIN="yum" PACKAGE_UPDATE_OPTIONS="update -y" PACKAGE_INSTALL_OPTIONS="install -y" # [NOTE] # ShellCheck version(0.3.8) is too low to check. # And in this version, it cannot be passed due to following error. # "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)" # INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel llvm-toolset-7-clang-tools-extra gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel attr curl python3 epel-release unzip" INSTALL_CHECKER_PKGS="cppcheck jq" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel" elif [ "${CONTAINER_FULLNAME}" = "fedora:40" ]; then PACKAGE_MANAGER_BIN="dnf" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="clang clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip libcxx libcxx-devel" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "fedora:39" ]; then PACKAGE_MANAGER_BIN="dnf" PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then PACKAGE_MANAGER_BIN="zypper" PACKAGE_UPDATE_OPTIONS="refresh" PACKAGE_INSTALL_OPTIONS="install -y" INSTALL_PACKAGES="automake clang-tools curl-devel fuse fuse-devel gcc-c++ java-17-openjdk-headless jq libxml2-devel make openssl openssl-devel python3-pip curl attr ShellCheck procps unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" elif [ "${CONTAINER_FULLNAME}" = "alpine:3.20" ]; then PACKAGE_MANAGER_BIN="apk" PACKAGE_UPDATE_OPTIONS="update --no-progress" PACKAGE_INSTALL_OPTIONS="add --no-progress --no-cache" INSTALL_PACKAGES="bash clang-extra-tools curl g++ make automake autoconf libtool git curl-dev fuse-dev jq libxml2-dev openssl coreutils procps attr sed mailcap openjdk17 aws-cli" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" AWSCLI_DIRECT_INSTALL=0 else echo "No container configured for: ${CONTAINER_FULLNAME}" exit 1 fi #----------------------------------------------------------- # Install #----------------------------------------------------------- # # Update packages (ex. apt-get update -y -qq) # echo "${PRGNAME} [INFO] Updates." /bin/sh -c "${PACKAGE_MANAGER_BIN} ${PACKAGE_UPDATE_OPTIONS}" # # Install packages ( with cppcheck ) # echo "${PRGNAME} [INFO] Install packages." /bin/sh -c "${PACKAGE_MANAGER_BIN} ${PACKAGE_ENABLE_REPO_OPTIONS} ${PACKAGE_INSTALL_OPTIONS} ${PACKAGE_INSTALL_ADDITIONAL_OPTIONS} ${INSTALL_PACKAGES}" echo "${PRGNAME} [INFO] Install cppcheck package." /bin/sh -c "${PACKAGE_MANAGER_BIN} ${INSTALL_CHECKER_PKG_OPTIONS} ${PACKAGE_INSTALL_OPTIONS} ${INSTALL_CHECKER_PKGS}" # # Install ShellCheck manually # if [ "${SHELLCHECK_DIRECT_INSTALL}" -eq 1 ]; then echo "${PRGNAME} [INFO] Install shellcheck package from github archive." if ! LATEST_SHELLCHECK_DOWNLOAD_URL=$(curl --silent --show-error https://api.github.com/repos/koalaman/shellcheck/releases/latest | jq -r '.assets[].browser_download_url | select(contains("linux.x86_64"))'); then echo "Could not get shellcheck package url" exit 1 fi if ! curl -s -S -L -o /tmp/shellcheck.tar.xz "${LATEST_SHELLCHECK_DOWNLOAD_URL}"; then echo "Failed to download shellcheck package from ${LATEST_SHELLCHECK_DOWNLOAD_URL}" exit 1 fi if ! tar -C /usr/bin/ -xf /tmp/shellcheck.tar.xz --no-anchored 'shellcheck' --strip=1; then echo "Failed to extract and install shellcheck." rm -f /tmp/shellcheck.tar.xz exit 1 fi rm -f /tmp/shellcheck.tar.xz fi # Check Java version java -version # # Install awscli # if [ "${AWSCLI_DIRECT_INSTALL}" -eq 1 ]; then echo "${PRGNAME} [INFO] Install awscli2 package." CURRENT_DIR=$(pwd) cd /tmp || exit 1 curl "${AWSCLI_URI}" -o "${AWSCLI_ZIP_FILE}" unzip "${AWSCLI_ZIP_FILE}" ./aws/install cd "${CURRENT_DIR}" || exit 1 fi #----------------------------------------------------------- # Set environment for configure #----------------------------------------------------------- echo "${PRGNAME} [INFO] Set environment for configure options" cat << EOF >> "${GITHUB_ENV}" CXX=${CXX} CXXFLAGS=${CXXFLAGS} LDFLAGS=${LDFLAGS} CONFIGURE_OPTIONS=${CONFIGURE_OPTIONS} EOF echo "${PRGNAME} [INFO] Finish Linux helper for installing packages." exit 0 # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/.gitignore000066400000000000000000000032621470675423500152760ustar00rootroot00000000000000# # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright(C) 2007 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # Compiled Object files # *.slo *.lo *.o *.Po *.Plo # # autotools/automake # aclocal.m4 autom4te.cache autoscan.log config.guess config.h config.h.in config.h.in~ config.log config.status config.sub configure configure.scan configure.ac~ depcomp install-sh libtool ltmain.sh m4 m4/* missing stamp-h1 Makefile Makefile.in test-driver compile missing # # man page # doc/man/s3fs.1 # # object directories # .deps .libs */.deps */.deps/* */.libs */.libs/* # # each directories # *.log *.trs default_commit_hash src/s3fs src/test_curl_util src/test_page_list src/test_string_util test/chaos-http-proxy-* test/junk_data test/s3proxy-* test/write_multiblock test/mknod_test test/truncate_read_file test/cr_filename # # Windows ports # *.dll *.exe fuse.pc WinFsp/ bin/ # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: noet sw=4 ts=4 fdm=marker # vim<600: noet sw=4 ts=4 # s3fs-fuse-1.95/.mailmap000066400000000000000000000007231470675423500147260ustar00rootroot00000000000000Adrian Petrescu Adrian Petrescu Ben Lemasurier Dan Moore Randy Rizun Randy Rizun Takeshi Nakatani s3fs-fuse-1.95/AUTHORS000066400000000000000000000007551470675423500143620ustar00rootroot000000000000001. Randy Rizun Wrote from scratch the initial version of S3FS. 2. Dan Moore Patches and improvements. 3. Adrian Petrescu Converted the project to be autotools-based. 4. Ben LeMasurier Bugfixes, performance and other improvements. 5. Takeshi Nakatani Bugfixes, performance and other improvements. 6. Andrew Gaul Bugfixes, performance and other improvements. s3fs-fuse-1.95/COMPILATION.md000066400000000000000000000063571470675423500154160ustar00rootroot00000000000000# Compilation from source code These are generic instructions should work on almost any GNU/Linux, macOS, BSD, or similar. If you want specific instructions for some distributions, check the [wiki](https://github.com/s3fs-fuse/s3fs-fuse/wiki/Installation-Notes). Keep in mind using the pre-built packages when available. ## Compilation on Linux ### Ensure your system satisfies build and runtime dependencies for: * fuse >= 2.8.4 * automake * gcc-c++ * make * libcurl * libxml2 * openssl/gnutls/nss * Please prepare the library according to the OS on which you will compile. * It is necessary to match the library used by libcurl. * Install the OpenSSL, GnuTLS or NSS devel package. * mime.types (the package providing depends on the OS) * s3fs tries to detect `/etc/mime.types` as default regardless of the OS * Else s3fs tries to detect `/etc/apache2/mime.types` if OS is macOS * s3fs exits with an error if these files are not exist * Alternatively, you can set mime.types file path with `mime` option without detecting these default files * pkg-config (or your OS equivalent) * NOTE If you have any trouble about details on required packages, see `INSTALL_PACKAGES` in [linux-ci-helper.sh](https://github.com/s3fs-fuse/s3fs-fuse/blob/master/.github/workflows/linux-ci-helper.sh). ### Then compile from master via the following commands: 1. Clone the source code: ```sh git clone https://github.com/s3fs-fuse/s3fs-fuse.git ``` 2. Configuration: ```sh cd s3fs-fuse ./autogen.sh ./configure ``` Depending on the TLS library (OpenSSL/GnuTLS/NSS), add `--with-openssl`, `--with-gnutls` or `--with-nss` when executing `configure`. (If omitted, it is equivalent to `--with-openssl`.) 3. Building: ```sh make ``` 4. Installing: ```sh sudo make install ``` ### NOTE - The required libraries/components required to run s3fs are: * fuse >= 2.8.4 * libcurl * libxml2 * openssl/gnutls/nss * mime.types (the package providing depends on the OS) ## Compilation on Windows (using MSYS2) On Windows, use [MSYS2](https://www.msys2.org/) to compile for itself. 1. Install [WinFsp](https://github.com/billziss-gh/winfsp) to your machine. 2. Install dependencies onto MSYS2: ```sh pacman -S git autoconf automake gcc make pkg-config openssl-devel libcurl-devel libxml2-devel libzstd-devel ``` 3. Clone this repository, then change directory into the cloned one. 4. Copy WinFsp files to the directory: ```sh cp -r "/c/Program Files (x86)/WinFsp" "./WinFsp" ``` 5. Write `fuse.pc` to resolve the package correctly: ```sh cat > ./fuse.pc << 'EOS' arch=x64 prefix=${pcfiledir}/WinFsp incdir=${prefix}/inc/fuse implib=${prefix}/bin/winfsp-${arch}.dll Name: fuse Description: WinFsp FUSE compatible API Version: 2.8.4 URL: http://www.secfs.net/winfsp/ Libs: "${implib}" Cflags: -I"${incdir}" EOS ``` 6. Compile using the command line: ```sh ./autogen.sh PKG_CONFIG_PATH="$PKG_CONFIG_PATH:$(pwd)" ./configure make CXXFLAGS="-I/usr/include" ``` 7. Copy binary files to distribute at one place: ```sh mkdir ./bin cp ./src/s3fs.exe ./bin/ cp ./WinFsp/bin/winfsp-x64.dll ./bin/ cp /usr/bin/msys-*.dll ./bin/ ``` 8. Distribute these files. s3fs-fuse-1.95/COPYING000066400000000000000000000431031470675423500143370ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. s3fs-fuse-1.95/ChangeLog000066400000000000000000000733301470675423500150630ustar00rootroot00000000000000ChangeLog for S3FS ------------------ Version 1.95 -- 25 Oct, 2024 (major changes only) #2424 - Add ipresolve option to select IPv4- or IPv6-only #2443 - Retry request on HTTP 429 error #2448 - Changed s3fs logo #2455 - Fix deadlock in FdManager::ChangeEntityToTempPath #2487 - #2492 - #2493 - Enable static lock checking and fix locking errors #2506 - #2517 - Fix Windows compilation #2515 - Fix FreeBSD support #2532 - Fix use-after-free in FdManager::ChangeEntityToTempPath Version 1.94 -- 23 Feb, 2024 (major changes only) #2409 - Fixed a bug that mounting with ksmid specified to fail #2404 - Fixed ordering problem between fdatasync and flush #2399 - Fixed ListBucket/IAM edge cases #2376 - Corrected list_bucket to search in stat cache during creating new file #2369 - Make dir size 4096 not 0 #2351 - Added option free_space_ratio to control cache size #2325 - Fixed a bug upload boundary calculation in StreamUpload #2298 - Abort MPU when MPU fails to avoid litter #2261 - Use explicit ownership for memory #2179 - Require C++11 Version 1.93 -- 19 Jul, 2023 (major changes only) #2212 - Allow listing implicit directories #2194 - #2209 - #2211 - #2214 - #2215 - Fix thread safety issues #2191 - #2201 - Add support for FUSE-T on macOS Version 1.92 -- 21 May, 2023 (major changes only) #1802 - #2104 - New option: streamupload #1922 - Enable noobj_cache by default #1927 - #2101 - New option: credlib and credlib_ops #1957 - Fixed a bug that regular files could not be created by mknod #1964 - Added stat information to the mount point #1970 - #1986 - Enable notsup_compat_dir by default #2000 - #2001 - Set mtime/ctime/atime of all objects as nanosecond #2065 - Compatible with OpenSSL 3.0 #2075 - Added proxy and proxy_cred_file option #2135 - Changed to rename cache files when renaming large files #2148 - New option: bucket_size Version 1.91 -- 07 Mar, 2022 (major changes only) #1753 - Fix RowFlush can not upload last part smaller than 5MB using NoCacheMultipartPost #1760 - Fix IAM role retrieval from IMDSv2 #1801 - Add option to allow unsigned payloads #1809 - Fix mixupload return EntityTooSmall while a copypart is less than 5MB after split #1855 - Allow compilation on Windows via MSYS2 #1868 - Handle utimensat UTIME_NOW and UTIME_OMIT special values #1871 - #1880 - Preserve sub-second precision in more situations #1879 - Always flush open files with O_CREAT flag #1887 - Fixed not to call Flush even if the file size is increased #1888 - Include climits to support musl libc Version 1.90 -- 07 Aug, 2021 (major changes only) #1599 - Don't ignore nomultipart when storage is low #1600 - #1602 - #1604 - #1617 - #1619 - #1620 - #1623 - #1624 - Fix POSIX compatibility issues found by pjdfstest #1630 - Fail CheckBucket when S3 returns PermanentRedirect #1640 - #1655 - Do not create zero-byte object when creating file #1648 - Allow arbitrary size AWS secret keys #1668 - #1678 - Fix race conditions #1696 - Set explicit Content-Length: 0 when initiating MPU #1681 - Set CURLOPT_UNRESTRICTED_AUTH when authenticating #1723 - Add jitter to avoid thundering herd #1728 - Loosen CheckBucket to check only the bucket #1729 - Add support for AWS-style environment variables Version 1.89 -- 22 Feb, 2021 (major changes only) #1520 - #1525 - #1534 - #1549 - Propagate S3 errors to errno more accurately #1546 - #1559 - Allow writing > 5 GB single-part objects supported by some non-AWS S3 #1553 - #1555 - Allow configuration of multipart copy size and limit to 5 GB #1562 - Allow configuration of multipart upload threshold and reduce default to 25 MB #1565 - Set default stat timeout to 900 seconds correctly #1579 - #1582 - Fix data corruption while updating metadata with use_cache Version 1.88 -- 4 Jan, 2021 (major changes only) #1349 - Fixed a bug about move file over limit of ensure space #1363 - #1366 - #1439 - Fix multiple race conditions #1365 - Dynamically determine whether lseek extended options are supported #1374 - Add support for deep archive storage class #1385 - Plug FdEntity leaks #1388 - Fix use_session_token option parsing #1392 - Allow 32-bit platforms to upload single-part objects > 2 GB #1404 - Fix dead lock in disk insufficient and optimize code #1408 - Ensure environment variable is set when using ECS #1413 - not call put headers if not exist pending meta #1425 - Do not send SSE headers during bucket creation #1432 - Add sigv4 only option #1437 - Add atime and correct atime/mtime/ctime operations #1447 - Fixed a bug that symlink could not be read after restarting s3fs #1448 - #1467 - Periodically flush written data to reduce temporary local storage #1449 - Added logfile option for non-syslog logging #1462 - Add AWS IMDSv2 support #1502 - #1503 - #1505 - Fix multiple issues when retrying requests Version 1.87 -- 10 Aug, 2020 (major changes only) #1244 - use correct content-type when complete multipart upload #1265 - Fixed a bug of stats cache compression #1271 - Fixed the truncation bug of stat file for cache file #1274 - Improved strictness of cache file stats(file) #1277 - Fixed insufficient upload size for mix multipart upload #1282 - Warn about missing MIME types instead of exiting #1285 - Not abort process by exception threw from s3fs_strtoofft #1286 - Support Google Cloud Storage headers #1295 - Added a parameter to output body to curldbg option #1302 - Fix renames of open files with nocopyapi option #1303 - Relink cache stats file atomically via rename #1305 - Ignore case when comparing ETags #1306 - Retry with exponential backoff during 500 error #1312 - Fixed a bug about serializing from cache file #1313 - Fixed about ParallelMixMultipartUpload #1316 - Add support for glacier storage class #1319 - Fixed upload error about mixuploading sparse file and truncating file #1334 - Added SIGUSR1 option for cache file integrity test #1341 - Change default stat_cache_expire Version 1.86 -- 04 Feb, 2020 (major changes only) #965 - enable various optimizations when using modern curl #1002 - allow SSE-C keys to have NUL bytes #1008 - add session token support #1039 - allow large files on 32-bit systems like Raspberry Pi #1049 - fix data corruption when external modification changes a cached object #1063 - fix data corruption when opening a second fd to an unflushed file #1066 - fix clock skew errors when writing large files #1081 - allow concurrent metadata queries during data operations #1098 - use server-side copy for partially modified files #1107 - #1108 - fix multiple concurrency issues #1199 - add requester_pays support #1209 - add symlink cache #1224 - add intelligent_ia storage tier Version 1.85 -- 11 Mar, 2019 #804 - add Backblaze B2 #812 - Fix typo s/mutliple/multiple/ #819 - #691: Made instructions for creating password file more obvious. #820 - Enable big writes if capable #826 - For RPM distributions fuse-libs is enough #831 - Add support for storage class ONEZONE_IA. #832 - Simplify hex conversion #833 - New installation instructions for Fedora >= 27 and CentOS7 #834 - Improve template for issues #835 - Make the compilation instructions generic #840 - Replace all mentions to MacOS X to macOS #849 - Correct typo #851 - Correctly compare list_object_max_keys #852 - Allow credentials from ${HOME}/.aws/credentials #853 - Replace ~ with ${HOME} in examples #855 - Include StackOverflow in FAQs #856 - Add icon for s3fs #859 - Upload S3 parts without batching #861 - Add 'profile' option to command line help. #865 - fix multihead warning check #866 - Multi-arch support for ppc64le #870 - Correct typos in command-line parsing #874 - Address cppcheck 1.86 errors #877 - Check arguments and environment before .aws/creds #882 - [curl] Assume long encryption keys are base64 encoded #885 - Update s3fs_util.cpp for correspondence of Nextcloud contype #888 - Add Server Fault to FAQs #892 - Repair xattr tests #893 - Store and retrieve file change time #894 - Default uid/gid/mode when object lacks permissions #895 - Emit more friendly error for buckets with dots #898 - Flush file before renaming #899 - Tighten up HTTP response code check #900 - Plug memory leak #901 - Plug memory leaks #902 - Avoid pass-by-value when not necessary #903 - Prefer find(char) over find(const char *) #904 - Remove unnecessary calls to std::string::c_str #905 - Fix comparison in s3fs_strtoofft #906 - Prefer HTTPS links where possible #908 - Added an error message when HTTP 301 status #909 - Ignore after period character of floating point in x-amz-meta-mtime #910 - Added a missing extension to .gitignore, and formatted dot files #911 - Added detail error message when HTTP 301/307 status #912 - Automatic region change made possible other than us-east-1(default) #913 - Prefer abort over assert(false) #914 - Issue readdir HEAD requests without batching #917 - Reference better-known AWS CLI for compatibility #918 - Load tail range during overwrite #919 - Add test for mv non-empty directory #920 - Remove unnecessary string copies #921 - Remove redundant string initializations #923 - Reverted automatic region change and changed messages #924 - Prefer empty over size checks #925 - Remove redundant null checks before delete #926 - Accept paths with : in them #930 - Correct enable_content_md5 docs #931 - Correct sigv2 typo #932 - Prefer AutoLock for synchronization #933 - Remove mirror path when deleting cache #934 - Checked and corrected all typo #937 - Disable malloc_trim #938 - Remove unneeded void parameter #939 - Prefer specific [io]stringstream where possible #940 - Copy parts in parallel #942 - Ensure s3fs compiles with C++03 #943 - Return not supported when hard linking #944 - Repair utility mode #946 - Simplify async request completion code #948 - Add logging for too many parts #949 - Implement exponential backoff for 503 #950 - Added S3FS_MALLOC_TRIM build switch #951 - Added a non-interactive option to utility mode #952 - Automatically abort failed multipart requests #953 - Update s3ql link #954 - Clear containers instead of individual erases #955 - Address miscellaneous clang-tidy warnings #957 - Upgrade to S3Proxy 1.6.1 #958 - Document lack of inotify support #959 - Fixed code for latest cppcheck error on OSX #960 - Wtf8 #961 - Work around cppcheck warnings #965 - Improvement of curl session pool for multipart #967 - Increase FdEntity reference count when returning #969 - Fix lazy typo #970 - Remove from file from stat cache during rename #972 - Add instructions for Amazon Linux #974 - Changed the description order of man page options #975 - Fixed ref-count when error occurred. #977 - Make macOS instructions consistent with others Version 1.84 -- Jul 8, 2018 #704 - Update README.md with details about .passwd-s3fs #710 - add disk space reservation #712 - Added Cygwin build options #714 - reduce lock contention on file open #724 - don't fail multirequest on single thread error #726 - add an instance_name option for logging #727 - Fixed Travis CI error about cppcheck - #713 #729 - FreeBSD build fixes #733 - More useful error message for dupe entries in passwd file #739 - cleanup curl handle state on retries #745 - don't fail mkdir when directory exists #753 - fix xpath selector in bucket listing #754 - Validate the URL format for http/https #755 - Added reset curl handle when returning to handle pool #756 - Optimize defaults #761 - Simplify installation for Ubuntu 16.04 #762 - Upgrade to S3Proxy 1.6.0 #763 - cleanup curl handles before curl share #764 - Remove false multihead warnings #765 - Add Debian installation instructions #766 - Remove s3fs-python #768 - Fixed memory leak #769 - Revert "enable FUSE read_sync by default" #774 - Option for IAM authentication endpoint #780 - gnutls_auth: initialize libgcrypt #781 - Fixed an error by cppcheck on OSX #786 - Log messages for 5xx and 4xx HTTP response code #789 - Instructions for SUSE and openSUSE prebuilt packages #793 - Added list_object_max_keys option based on #783 PR Version 1.83 -- Dec 17, 2017 #606 - Add Homebrew instructions #608 - Fix chown_nocopy losing existing uid/gid if unspecified #609 - Group permission checks sometimes fail with large number of groups #611 - Fixed clock_gettime build failure on macOS 10.12 Sierra - #600 #621 - Upgrade to S3Proxy 1.5.3 #627 - Update README.md #630 - Added travis test on osx for #601 #631 - Merged macosx branch into master branch #601 #636 - Fix intermittent upload failures on macOS #637 - Add blurb about non-Amazon S3 implementations #638 - Minor fixes to README #639 - Update Homebrew instructions #642 - Fixed potential atomic violation in S3fsCurl::AddUserAgent - #633 #644 - Fixed with unnecessary equal in POST uploads url argument - #643 #645 - Configure S3Proxy for SSL #646 - Simplify S3Proxy PID handling #652 - Fix s3fs_init message #659 - Do not fail updating directory when removing old-style object(ref #658) #660 - Refixed s3fs_init message(ref #652) #663 - Lock FdEntity when mutating orgmeta #664 - auth headers insertion refactoring #668 - Changed .travis.yml for fixing not found gpg2 on osx #669 - add IBM IAM authentication support #670 - Fixed a bug in S3fsCurl::LocateBundle #671 - Add support for ECS metadata endpoint #675 - Reduce use of preprocessor #676 - Move str definition from header to implementation #677 - Add s3proxy to .gitignore #679 - README.md Addition #681 - Changed functions about reading passwd file #684 - Correct signedness warning #686 - remove use of jsoncpp #688 - Improved use of temporary files - #678 #690 - Added option ecs description to man page #692 - Updated template md files for issue and pr #695 - fix condition for parallel download #697 - Fixing race condition in FdEntity::GetStats #699 - Fix dbglevel usage Version 1.82 -- May 13, 2017 #597 - Not fallback to HTTP - #596 #598 - Updated ChangeLog and configure.ac for release 1.82 Version 1.81 -- May 13, 2017 #426 - Updated to correct ChangeLog #431 - fix typo s/controll/control/ #432 - Include location constraint when creating bucket #433 - Correct search and replace typo #440 - Handled all curl error without exiting process - #437 #443 - Fix for leaks during stat cache entry expiry / truncation (#340) #444 - Add mirror file logic for removing cache file #447 - added fuse package for mounting via /etc/fstab, fixes #417 #449 - Accept mount options compatible with mtab #451 - Correct path in README #454 - Changed for accepting mount options compatible with mtab - #449 #466 - Fixed a bug about could not copy file mode from org file #471 - Added use_xattr option for #467 and #460 #477 - OS-specific correspondence of the extended attribute header #483 - Trim symbolic link original path in file #487 - Split header debugging onto multiple lines for easier reading #488 - Fixed searching Content-Length without case sensitive - #480 #489 - Changed headers_t map using nocase compare function - #488 #494 - Fix typo s/destroied/destroyed/ #495 - Fix invalid V4 signature on multipart copy requests #498 - Upgrade to S3Proxy 1.5.1 #502 - Fixed issue#435 branch codes for remaining bugs(2) #503 - Add missing call to mtime test #504 - Use describe helper function #505 - Correct typos #509 - Use server-provided ETag during complete upload #511 - Fixed a bug about uploading NULL to some part of the file contents #512 - Changed clock_gettime func to s3fs_clock_gettime for homebrew - #468 #513 - Added issue and PR templates. #517 - Update s3fs.1 - removed duplicated word #520 - Added links for eventual consistency in README.md - #515 #539 - Upgrade to S3Proxy 1.5.2 #540 - Address cppcheck 1.77 warnings #545 - Changed base cached time of stat_cache_expire option - #523 #546 - Fixed double initialization of SSL library at foreground #550 - Add umount instruction for unprivileged user #551 - Updated stat_cache_expire option description - #545 #552 - switch S3fsMultiCurl to use foreground threads #553 - add TLS cipher suites customization #554 - cleanup cache directory when running out of disk space #555 - don't sign empty headers (as they are discarded #556 - fix multipart upload handling without cache #557 - Added check_cache_dir_exist option(refixed #347) - #538 #558 - Fixed a bug in logic about truncating stat cache #560 - Fixed about multipart uploading at no free space related to #509 #567 - Do not send ACL unless overridden #576 - Added option for complementing lack of stat mode #578 - Refactored the get_object_attribute function #579 - Added notsup_compat_dir option #580 - Enhanced bucket/path parameter check #582 - Check errors returned in 200 OK responses for put header request #583 - Updated limit object size in s3fs man page #585 - Fixed failure to upload/copy with SSE_C and SSE_KMS #587 - Changed copyright year format for debian pkg #588 - Default transport to HTTPS #590 - Updated man page for default_acl option - #567 #593 - Backward compatible for changing default transport to HTTPS #594 - Check bucket at public bucket and add nocopyapi option automatically #595 - Updated ChangeLog and configure.ac for release 1.81 Version 1.80 -- May 29, 2016 #213 - Parse ETag from copy multipart correctly #215 - Fix mem leak in openssl_auth.cpp:s3fs_sha256hexsum #217 - Override install, so that the make install does not install rename_before_close under /test #219 - Address Coverity errors #220 - Test removing a non-empty directory #221 - Compare idiomatically #222 - Annotate constructors as explicit #224 - Configure cppcheck #229 - Convert rename_before_close to a shell script #231 - Rewrite AutoLock #232 - Always hold stat_cache_lock when using stat_cache #233 - Remove IntToStr #234 - Update README #235 - Plug leak during complete multipart upload #237 - Refactor tests into individual functions #238 - Enable all cppcheck rules #239 - Update stale Google Code reference in --help #240 - Enable Content-MD5 during multipart upload part #243 - Run cppcheck during Travis builds #245 - Elide duplicate lookups of std::map via iterators #246 - Unlock during early return in TruncateCache #247 - Base64 cleanup #248 - Enable integration tests for Travis #249 - Silence wget #250 - s3fs can print version with short commit hash - #228 #251 - Skip xattr tests if utilities are missing #252 - This fixes an issue with caching when the creation of a subdirectory … #253 - Added checking cache dir perms at starting. #256 - Add no atomic rename to limitations #257 - Update README.md: Bugfix password file permissions errors #258 - Update README.md to better explain mount upon boot #260 - Wrap help text at 80 characters #261 - Correct help timeouts #263 - Allow integration testing against Amazon S3 #265 - Fix integration tests #266 - Cleanup from PR #265 #267 - Added the _netdev option to the fstab example. #268 - Use 127.0.0.1 not localhost in s3proxy wait loop #271 - Add support for standard_ia storage class #274 - Modified man page for storage_class option(#271) #275 - Changed and cleaned the logic for debug message. #278 - Supported for SSE KMS(#270) #280 - Supported a object which is larger than free disk space #285 - Add test for symlink #288 - Fixed a bug about head request(copy) for SSE - issue#286 #289 - Print source file in log messages #291 - File opened with O_TRUNC is not flushed - Issue #290 #293 - Fix a small spelling issue. #295 - File opened with O_TRUNC is not flushed - changed #291 #300 - Update integration-test-main.sh #302 - Fix syslog level used by S3FS_PRN_EXIT() #304 - Fixed a bug about mtime - #299 #306 - Fix read concurrency to work in parallel count #307 - Fix pthread portability problem #308 - Changed ensure free disk space as additional change for #306 #309 - Check pthread portability in configure as additional change for #307 #310 - Update integration-test-main.sh as additional change for #300 #311 - Change error log to debug log in s3fs_read() #313 - fix gitignore #319 - Clean up mount point on errors in s3fs_init() #321 - delete stat cache entry in s3fs_fsync so st_size is refreshed - #320 #323 - Add goofys to references #328 - Fix v4 signature with use_path_request_style #329 - Correct multiple issues with GET and v4 signing #330 - Pass by const reference where possible #331 - Address various clang warnings #334 - Bucket host should include port and not path #336 - update README.md for fstab #338 - Fixed a bug about IAMCRED type could not be retried. #339 - Updated README.md for fstab example. #341 - Fix the memory leak issue in fdcache. #346 - Fix empty directory check against AWS S3 #348 - Integration test summary, continue on error #350 - Changed cache out logic for stat - #340 #351 - Check cache directory path and attributes - #347 #352 - Remove stat file cache dir if specified del_cache - #337 #354 - Supported regex type for additional header format - #343 #355 - Fixed codes about clock_gettime for osx #356 - Fixed codes about clock_gettime for osx(2) #357 - Fixed codes about clock_gettime for osx(3) #359 - Remove optional parameter from Content-Type header - #358 #360 - Fix clock_gettime autotools detection on Linux #364 - Checked content-type by no case-sensitivity - #363 #371 - Always set stats cache for opened file #372 - Fixed a bug about etag comparison in stats cache, etc. #376 - Test for writing after an lseek past end of file #379 - Fixed a bug about writing sparsed file - #375 #385 - fix typo in curl.cpp: s/returing/returning/ #391 - Update s3fs.1 #394 - Revert "Fixed a bug about writing sparsed file - #375" #395 - Fixed writing sparsed file - #375,#379,#394 #397 - Supported User-Agent header - #383 #403 - Fix a bug of truncating empty file #404 - Add curl handler pool to reuse connections #409 - Fixed 'load_sse_c' option not working - #388 #410 - Allow duplicate key in ahbe_conf - #386 #411 - loading IAM role name automatically(iam_role option) - #387 #415 - Fixed a bug about stat_cache_expire - #382 #420 - Skip early credential checks when iam_role=auto #422 - Fixes for iam_role=auto #424 - Added travis CI badge in README.md #425 - Updated ChangeLog and configure.ac for release 1.80 Version 1.79 -- Jul 19, 2015 issue #60 - Emit user-friendly log messages on failed CheckBucket requests issue #62 - Remove stray chars from source files issue #63 - Fix spelling errors issue #68 - FreeBSD issue issue #69 - Address clang always true warnings issue #73 - Small gitignore fixes issue #74 - url: handle scheme omission issue #83 - Changed option processing to use strtol() to get a umask issue #93 - Add simple unit tests for trim functions issue #100 - CURL handles not properly initialized to use DNS or SSL session caching issue #101 - Optimized function "bool directory_empty()" issue #103 - Remove prefix option in s3fs man page - issue#87 issue #104 - fix rename before close issue #116 - Supported signature version 4 issue #119 - Added new mp_umask option about issue#107, pr#110 issue #124 - Fallback to v2 signatures correctly. issue #130 - refactor integration tests create/cleanup file issue #131 - Test ls issue #132 - Use S3Proxy to run integration tests issue #134 - Include Content-Type in complete MPU V2 signature issue #135 - Correct V4 signature for initiate multipart upload issue #136 - Small fixes to integration tests issue #137 - Add test for multi-part upload issue #138 - Fixed bugs, not turn use_cache off and ty to load to end - issue#97 issue #143 - Fixed a bug no use_cache case about fixed #138 - issue#141 issue #144 - Add Travis configuration issue #146 - add exit handler to cleanup on failures issue #147 - Use S3Proxy 1.4.0-SNAPSHOT issue #150 - Fixed a bug not handling fsync - #145 issue #154 - Fixed url-encoding for ampersand etc on sigv4 - Improvement/#149 issue #155 - Fixed a bug: unable to mount bucket subdirectory issue #156 - Fixed a bug about ssl session sharing with libcurl older 7.23.0 - issue#126 issue #159 - Upgrade to S3Proxy 1.4.0 issue #164 - send the correct Host header when using -o url issue #165 - Auth v4 refactor issue #167 - Increased default connecting/reading/writing timeout value issue #168 - switch to use region specific endpoints to compute correct v4 signature issue #170 - Reviewed and fixed response codes print in curl.cpp - #157 issue #171 - Support buckets with mixed-case names issue #173 - Run integration tests via Travis issue #176 - configure.ac: detect target, if target is darwin (OSX), then #176 issue #177 - Add .mailmap issue #178 - Update .gitignore issue #184 - Add usage information for multipart_size issue #185 - Correct obvious typos in usage and README issue #190 - Add a no_check_certificate option. issue #194 - Tilda in a file-name breaks things (EPERM) issue #198 - Disable integration tests for Travis issue #199 - Supported extended attributes(retry) issue #200 - fixed fallback to sigv2 for bucket create and GCS issue #202 - Specialize {set,get}xattr for OS X issue #204 - Add integration test for xattr issue #207 - Fixed a few small spelling issues. Version 1.78 -- Sep 15, 2014 issue #29 - Possible to create Debian/Ubuntu packages?(googlecode issue 109) issue 417(googlecode) - Password file with DOS format is not handled properly issue #41 - Failed making signature issue #40 - Moving a directory containing more than 1000 files truncates the directory issue #49 - use_sse is ignored when creating new files issue #39 - Support for SSE-C issue #50 - Cannot find pkg-config when configured with any SSL backend except openssl Version 1.77 -- Apr 19, 2014 issue 405(googlecode) - enable_content_md5 Input/output error issue #14 - s3fs -u should return 0 if there are no lost multiparts issue #16 - empty file is written to s3 issue #18 - s3fs crashes with segfault issue #22 - Fix typos in docs for max_stat_cache_size issue #23 - curl ssl problems issue #28 - Address signedness warning in FdCache::Init Version 1.76 -- Jan 21, 2014 issue #5 - du shows incorrect usage stats issue #8 - version in configure.ac is 1.74 for release 1.75 Version 1.75 -- Jan 6, 2014 issue #1 - Using %20 instead of the plus (+) sign for encoding spaces issue #3 - Fixed local timezone was incorrectly being applied to IAM and Last-Modified dates. issue #4 - Fix compilation error on MacOSX with missing const Version 1.74 -- Nov 24, 2013 This version is initial version on GitHub, same as on GoogleCodes(s3fs). https://github.com/s3fs-fuse/s3fs-fuse/releases/tag/v1.74 see more detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.74.tar.gz Version 1.73 -- Aug 23, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.73.tar.gz Version 1.72 -- Aug 10, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.72.tar.gz Version 1.71 -- Jun 15, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.71.tar.gz Version 1.70 -- Jun 01, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.70.tar.gz Version 1.69 -- May 15, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.69.tar.gz Version 1.68 -- Apr 30, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.68.tar.gz Version 1.67 -- Apr 13, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.67.tar.gz Version 1.66 -- Apr 06, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.66.tar.gz Version 1.65 -- Mar 30, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.65.tar.gz Version 1.64 -- Mar 23, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.64.tar.gz Version 1.63 -- Feb 24, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.63.tar.gz Version 1.62 -- Jan 27, 2013 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.62.tar.gz Version 1.61 -- Aug 30, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.61.tar.gz Version 1.60 -- Aug 29, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.60.tar.gz Version 1.59 -- Jul 28, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.59.tar.gz Version 1.58 -- Jul 19, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.58.tar.gz Version 1.57 -- Jul 07, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.57.tar.gz Version 1.56 -- Jul 07, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.56.tar.gz Version 1.55 -- Jul 02, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.55.tar.gz Version 1.54 -- Jun 25, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.54.tar.gz Version 1.53 -- Jun 22, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.53.tar.gz Version 1.40 -- Feb 11, 2011 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.40.tar.gz Version 1.33 -- Dec 30, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.33.tar.gz Version 1.25 -- Dec 16, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.25.tar.gz Version 1.19 -- Dec 2, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.19.tar.gz Version 1.16 -- Nov 22, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.16.tar.gz Version 1.10 -- Nov 6, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.10.tar.gz Version 1.02 -- Oct 29, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.02.tar.gz Version 1.01 -- Oct 28, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.01.tar.gz Version 1.0 -- Oct 24, 2010 see detail on googlecodes: https://code.google.com/p/s3fs/downloads/detail?name=s3fs-1.0.tar.gz ------ Version 1.1 -- Mon Oct 18 2010 Dan Moore reopens the project and fixes various issues that had accumulated in the tracker. Adrian Petrescu converts the project to autotools and posts it to GitHub. Version 1.0 -- 2008 Randy Rizun releases a basic version of S3FS on Google Code. s3fs-fuse-1.95/INSTALL000066400000000000000000000363301470675423500143410ustar00rootroot00000000000000Installation Instructions ************************* Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc. Copying and distribution of this file, with or without modification, are permitted in any medium without royalty provided the copyright notice and this notice are preserved. This file is offered as-is, without warranty of any kind. Basic Installation ================== Briefly, the shell commands `./configure; make; make install' should configure, build, and install this package. The following more-detailed instructions are generic; see the `README' file for instructions specific to this package. Some packages provide this `INSTALL' file but do not implement all of the features documented below. The lack of an optional feature in a given package is not necessarily a bug. More recommendations for GNU packages can be found in *note Makefile Conventions: (standards)Makefile Conventions. The `configure' shell script attempts to guess correct values for various system-dependent variables used during compilation. It uses those values to create a `Makefile' in each directory of the package. It may also create one or more `.h' files containing system-dependent definitions. Finally, it creates a shell script `config.status' that you can run in the future to recreate the current configuration, and a file `config.log' containing compiler output (useful mainly for debugging `configure'). It can also use an optional file (typically called `config.cache' and enabled with `--cache-file=config.cache' or simply `-C') that saves the results of its tests to speed up reconfiguring. Caching is disabled by default to prevent problems with accidental use of stale cache files. If you need to do unusual things to compile the package, please try to figure out how `configure' could check whether to do them, and mail diffs or instructions to the address given in the `README' so they can be considered for the next release. If you are using the cache, and at some point `config.cache' contains results you don't want to keep, you may remove or edit it. The file `configure.ac' (or `configure.in') is used to create `configure' by a program called `autoconf'. You need `configure.ac' if you want to change it or regenerate `configure' using a newer version of `autoconf'. The simplest way to compile this package is: 1. `cd' to the directory containing the package's source code and type `./configure' to configure the package for your system. Running `configure' might take a while. While running, it prints some messages telling which features it is checking for. 2. Type `make' to compile the package. 3. Optionally, type `make check' to run any self-tests that come with the package, generally using the just-built uninstalled binaries. 4. Type `make install' to install the programs and any data files and documentation. When installing into a prefix owned by root, it is recommended that the package be configured and built as a regular user, and only the `make install' phase executed with root privileges. 5. Optionally, type `make installcheck' to repeat any self-tests, but this time using the binaries in their final installed location. This target does not install anything. Running this target as a regular user, particularly if the prior `make install' required root privileges, verifies that the installation completed correctly. 6. You can remove the program binaries and object files from the source code directory by typing `make clean'. To also remove the files that `configure' created (so you can compile the package for a different kind of computer), type `make distclean'. There is also a `make maintainer-clean' target, but that is intended mainly for the package's developers. If you use it, you may have to get all sorts of other programs in order to regenerate files that came with the distribution. 7. Often, you can also type `make uninstall' to remove the installed files again. In practice, not all packages have tested that uninstallation works correctly, even though it is required by the GNU Coding Standards. 8. Some packages, particularly those that use Automake, provide `make distcheck', which can by used by developers to test that all other targets like `make install' and `make uninstall' work correctly. This target is generally not run by end users. Compilers and Options ===================== Some systems require unusual options for compilation or linking that the `configure' script does not know about. Run `./configure --help' for details on some of the pertinent environment variables. You can give `configure' initial values for configuration parameters by setting variables in the command line or in the environment. Here is an example: ./configure CC=c99 CFLAGS=-g LIBS=-lposix *Note Defining Variables::, for more details. Compiling For Multiple Architectures ==================================== You can compile the package for more than one kind of computer at the same time, by placing the object files for each architecture in their own directory. To do this, you can use GNU `make'. `cd' to the directory where you want the object files and executables to go and run the `configure' script. `configure' automatically checks for the source code in the directory that `configure' is in and in `..'. This is known as a "VPATH" build. With a non-GNU `make', it is safer to compile the package for one architecture at a time in the source code directory. After you have installed the package for one architecture, use `make distclean' before reconfiguring for another architecture. On macOS 10.5 and later systems, you can create libraries and executables that work on multiple system types--known as "fat" or "universal" binaries--by specifying multiple `-arch' options to the compiler but only a single `-arch' option to the preprocessor. Like this: ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ CPP="gcc -E" CXXCPP="g++ -E" This is not guaranteed to produce working output in all cases, you may have to build one architecture at a time and combine the results using the `lipo' tool if you have problems. Installation Names ================== By default, `make install' installs the package's commands under `/usr/local/bin', include files under `/usr/local/include', etc. You can specify an installation prefix other than `/usr/local' by giving `configure' the option `--prefix=PREFIX', where PREFIX must be an absolute file name. You can specify separate installation prefixes for architecture-specific files and architecture-independent files. If you pass the option `--exec-prefix=PREFIX' to `configure', the package uses PREFIX as the prefix for installing programs and libraries. Documentation and other data files still use the regular prefix. In addition, if you use an unusual directory layout you can give options like `--bindir=DIR' to specify different values for particular kinds of files. Run `configure --help' for a list of the directories you can set and what kinds of files go in them. In general, the default for these options is expressed in terms of `${prefix}', so that specifying just `--prefix' will affect all of the other directory specifications that were not explicitly provided. The most portable way to affect installation locations is to pass the correct locations to `configure'; however, many packages provide one or both of the following shortcuts of passing variable assignments to the `make install' command line to change installation locations without having to reconfigure or recompile. The first method involves providing an override variable for each affected directory. For example, `make install prefix=/alternate/directory' will choose an alternate location for all directory configuration variables that were expressed in terms of `${prefix}'. Any directories that were specified during `configure', but not in terms of `${prefix}', must each be overridden at install time for the entire installation to be relocated. The approach of makefile variable overrides for each directory variable is required by the GNU Coding Standards, and ideally causes no recompilation. However, some platforms have known limitations with the semantics of shared libraries that end up requiring recompilation when using this method, particularly noticeable in packages that use GNU Libtool. The second method involves providing the `DESTDIR' variable. For example, `make install DESTDIR=/alternate/directory' will prepend `/alternate/directory' before all installation names. The approach of `DESTDIR' overrides is not required by the GNU Coding Standards, and does not work on platforms that have drive letters. On the other hand, it does better at avoiding recompilation issues, and works well even when some directory options were not specified in terms of `${prefix}' at `configure' time. Optional Features ================= If the package supports it, you can cause programs to be installed with an extra prefix or suffix on their names by giving `configure' the option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. Some packages pay attention to `--enable-FEATURE' options to `configure', where FEATURE indicates an optional part of the package. They may also pay attention to `--with-PACKAGE' options, where PACKAGE is something like `gnu-as' or `x' (for the X Window System). The `README' should mention any `--enable-' and `--with-' options that the package recognizes. For packages that use the X Window System, `configure' can usually find the X include and library files automatically, but if it doesn't, you can use the `configure' options `--x-includes=DIR' and `--x-libraries=DIR' to specify their locations. Some packages offer the ability to configure how verbose the execution of `make' will be. For these packages, running `./configure --enable-silent-rules' sets the default to minimal output, which can be overridden with `make V=1'; while running `./configure --disable-silent-rules' sets the default to verbose, which can be overridden with `make V=0'. Particular systems ================== On HP-UX, the default C compiler is not ANSI C compatible. If GNU CC is not installed, it is recommended to use the following options in order to use an ANSI C compiler: ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" and if that doesn't work, install pre-built binaries of GCC for HP-UX. On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot parse its `' header file. The option `-nodtk' can be used as a workaround. If GNU CC is not installed, it is therefore recommended to try ./configure CC="cc" and if that doesn't work, try ./configure CC="cc -nodtk" On Solaris, don't put `/usr/ucb' early in your `PATH'. This directory contains several dysfunctional programs; working variants of these programs are available in `/usr/bin'. So, if you need `/usr/ucb' in your `PATH', put it _after_ `/usr/bin'. On Haiku, software installed for all users goes in `/boot/common', not `/usr/local'. It is recommended to use the following options: ./configure --prefix=/boot/common Specifying the System Type ========================== There may be some features `configure' cannot figure out automatically, but needs to determine by the type of machine the package will run on. Usually, assuming the package is built to be run on the _same_ architectures, `configure' can figure that out, but if it prints a message saying it cannot guess the machine type, give it the `--build=TYPE' option. TYPE can either be a short name for the system type, such as `sun4', or a canonical name which has the form: CPU-COMPANY-SYSTEM where SYSTEM can have one of these forms: OS KERNEL-OS See the file `config.sub' for the possible values of each field. If `config.sub' isn't included in this package, then this package doesn't need to know the machine type. If you are _building_ compiler tools for cross-compiling, you should use the option `--target=TYPE' to select the type of system they will produce code for. If you want to _use_ a cross compiler, that generates code for a platform different from the build platform, you should specify the "host" platform (i.e., that on which the generated programs will eventually be run) with `--host=TYPE'. Sharing Defaults ================ If you want to set default values for `configure' scripts to share, you can create a site shell script called `config.site' that gives default values for variables like `CC', `cache_file', and `prefix'. `configure' looks for `PREFIX/share/config.site' if it exists, then `PREFIX/etc/config.site' if it exists. Or, you can set the `CONFIG_SITE' environment variable to the location of the site script. A warning: not all `configure' scripts look for a site script. Defining Variables ================== Variables not defined in a site shell script can be set in the environment passed to `configure'. However, some packages may run configure again during the build, and the customized values of these variables may be lost. In order to avoid this problem, you should set them in the `configure' command line, using `VAR=value'. For example: ./configure CC=/usr/local2/bin/gcc causes the specified `gcc' to be used as the C compiler (unless it is overridden in the site shell script). Unfortunately, this technique does not work for `CONFIG_SHELL' due to an Autoconf bug. Until the bug is fixed you can use this workaround: CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash `configure' Invocation ====================== `configure' recognizes the following options to control how it operates. `--help' `-h' Print a summary of all of the options to `configure', and exit. `--help=short' `--help=recursive' Print a summary of the options unique to this package's `configure', and exit. The `short' variant lists options used only in the top level, while the `recursive' variant lists options also present in any nested packages. `--version' `-V' Print the version of Autoconf used to generate the `configure' script, and exit. `--cache-file=FILE' Enable the cache: use and save the results of the tests in FILE, traditionally `config.cache'. FILE defaults to `/dev/null' to disable caching. `--config-cache' `-C' Alias for `--cache-file=config.cache'. `--quiet' `--silent' `-q' Do not print messages saying which checks are being made. To suppress all normal output, redirect it to `/dev/null' (any error messages will still be shown). `--srcdir=DIR' Look for the package's source code in directory DIR. Usually `configure' can determine that directory automatically. `--prefix=DIR' Use DIR as the installation prefix. *note Installation Names:: for more details, including other options available for fine-tuning the installation locations. `--no-create' `-n' Run the configure checks, but stop before creating any output files. `configure' also accepts some other, not widely useful, options. Run `configure --help' for more details. s3fs-fuse-1.95/Makefile.am000066400000000000000000000060001470675423500153330ustar00rootroot00000000000000###################################################################### # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ###################################################################### SUBDIRS=src test doc EXTRA_DIST=doc default_commit_hash dist-hook: rm -rf `find $(distdir)/doc -type d -name .svn` rm -f `find $(distdir)/doc -type f -name Makefile` release : dist ../utils/release.sh ../utils/release.sh $(DIST_ARCHIVES) .PHONY: cppcheck shellcheck cppcheck: cppcheck --quiet --error-exitcode=1 \ --inline-suppr \ --std=c++11 \ --xml \ -D HAVE_ATTR_XATTR_H \ -D HAVE_SYS_EXTATTR_H \ -D HAVE_MALLOC_TRIM \ -U CURLE_PEER_FAILED_VERIFICATION \ -U ENOATTR \ --enable=warning,style,information,missingInclude \ --suppress=missingIncludeSystem \ --suppress=unmatchedSuppression \ --suppress=useStlAlgorithm \ --suppress=checkLevelNormal \ --suppress=normalCheckLevelMaxBranches \ src/ test/ # # ShellCheck # SHELLCHECK_CMD = shellcheck SHELLCHECK_SH_OPT = --shell=sh SHELLCHECK_BASH_OPT = --shell=bash # [NOTE] # To control error warnings as a whole, specify the "SC" with the following variables. # SHELLCHECK_COMMON_IGN = --exclude=SC1091 SHELLCHECK_CUSTOM_IGN = --exclude=SC1091 shellcheck: @if type shellcheck > /dev/null 2>&1; then \ echo "* ShellCheck version"; \ $(SHELLCHECK_CMD) --version; \ echo ""; \ echo "* Check all sh files with ShellCheck"; \ LC_ALL=C.UTF-8 $(SHELLCHECK_CMD) $(SHELLCHECK_SH_OPT) $(SHELLCHECK_COMMON_IGN) $$(grep '#![[:space:]]*/bin/sh' $$(find . -type f -name \*.sh) | sed -e 's|^\(.*\):#\!.*$$|\1|g') || exit 1; \ echo "-> No error was detected."; \ echo ""; \ echo "* Check all bash files with ShellCheck"; \ LC_ALL=C.UTF-8 $(SHELLCHECK_CMD) $(SHELLCHECK_BASH_OPT) $(SHELLCHECK_COMMON_IGN) $$(grep '#![[:space:]]*/bin/bash' $$(find . -type f -name \*.sh) | sed -e 's|^\(.*\):#\!.*$$|\1|g') || exit 1; \ echo "-> No error was detected."; \ else \ echo "* ShellCheck is not installed, so skip this."; \ fi # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/README.md000066400000000000000000000136351470675423500145720ustar00rootroot00000000000000# s3fs s3fs allows Linux, macOS, and FreeBSD to mount an S3 bucket via [FUSE(Filesystem in Userspace)](https://github.com/libfuse/libfuse). s3fs makes you operate files and directories in S3 bucket like a local file system. s3fs preserves the native object format for files, allowing use of other tools like [AWS CLI](https://github.com/aws/aws-cli). [![s3fs-fuse CI](https://github.com/s3fs-fuse/s3fs-fuse/actions/workflows/ci.yml/badge.svg)](https://github.com/s3fs-fuse/s3fs-fuse/actions/workflows/ci.yml) [![Twitter Follow](https://img.shields.io/twitter/follow/s3fsfuse.svg?style=social&label=Follow)](https://twitter.com/s3fsfuse) ![s3fs-fuse](https://github.com/ggtakec/s3fs-fuse-images/blob/master/images/s3fslogo.png) ## Features * large subset of POSIX including reading/writing files, directories, symlinks, mode, uid/gid, and extended attributes * compatible with Amazon S3, and other [S3-based object stores](https://github.com/s3fs-fuse/s3fs-fuse/wiki/Non-Amazon-S3) * allows random writes and appends * large files via multi-part upload * renames via server-side copy * optional server-side encryption * data integrity via MD5 hashes * in-memory metadata caching * local disk data caching * user-specified regions, including Amazon GovCloud * authenticate via v2 or v4 signatures ## Installation Many systems provide pre-built packages: * Amazon Linux via EPEL: ``` sudo amazon-linux-extras install epel sudo yum install s3fs-fuse ``` * Arch Linux: ``` sudo pacman -S s3fs-fuse ``` * Debian 9 and Ubuntu 16.04 or newer: ``` sudo apt install s3fs ``` * Fedora 27 or newer: ``` sudo dnf install s3fs-fuse ``` * Gentoo: ``` sudo emerge net-fs/s3fs ``` * RHEL and CentOS 7 or newer via EPEL: ``` sudo yum install epel-release sudo yum install s3fs-fuse ``` * SUSE 12 and openSUSE 42.1 or newer: ``` sudo zypper install s3fs ``` * macOS 10.12 and newer via [Homebrew](https://brew.sh/): ``` brew install --cask macfuse brew install gromgit/fuse/s3fs-mac ``` * FreeBSD: ``` pkg install fusefs-s3fs ``` * Windows: Windows has its own install, seening in [this link](COMPILATION.md) Otherwise consult the [compilation instructions](COMPILATION.md). ## Examples s3fs supports the standard [AWS credentials file](https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) stored in `${HOME}/.aws/credentials`. Alternatively, s3fs supports a custom passwd file. Finally s3fs recognizes the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables. The default location for the s3fs password file can be created: * using a `.passwd-s3fs` file in the users home directory (i.e. `${HOME}/.passwd-s3fs`) * using the system-wide `/etc/passwd-s3fs` file Enter your credentials in a file `${HOME}/.passwd-s3fs` and set owner-only permissions: ``` echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs chmod 600 ${HOME}/.passwd-s3fs ``` Run s3fs with an existing bucket `mybucket` and directory `/path/to/mountpoint`: ``` s3fs mybucket /path/to/mountpoint -o passwd_file=${HOME}/.passwd-s3fs ``` If you encounter any errors, enable debug output: ``` s3fs mybucket /path/to/mountpoint -o passwd_file=${HOME}/.passwd-s3fs -o dbglevel=info -f -o curldbg ``` You can also mount on boot by entering the following line to `/etc/fstab`: ``` mybucket /path/to/mountpoint fuse.s3fs _netdev,allow_other 0 0 ``` If you use s3fs with a non-Amazon S3 implementation, specify the URL and path-style requests: ``` s3fs mybucket /path/to/mountpoint -o passwd_file=${HOME}/.passwd-s3fs -o url=https://url.to.s3/ -o use_path_request_style ``` or(fstab) ``` mybucket /path/to/mountpoint fuse.s3fs _netdev,allow_other,use_path_request_style,url=https://url.to.s3/ 0 0 ``` Note: You may also want to create the global credential file first ``` echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > /etc/passwd-s3fs chmod 600 /etc/passwd-s3fs ``` Note2: You may also need to make sure `netfs` service is start on boot ## Limitations Generally S3 cannot offer the same performance or semantics as a local file system. More specifically: * random writes or appends to files require rewriting the entire object, optimized with multi-part upload copy * metadata operations such as listing directories have poor performance due to network latency * non-AWS providers may have [eventual consistency](https://en.wikipedia.org/wiki/Eventual_consistency) so reads can temporarily yield stale data (AWS offers read-after-write consistency [since Dec 2020](https://aws.amazon.com/about-aws/whats-new/2020/12/amazon-s3-now-delivers-strong-read-after-write-consistency-automatically-for-all-applications/)) * no atomic renames of files or directories * no coordination between multiple clients mounting the same bucket * no hard links * inotify detects only local modifications, not external ones by other clients or tools ## References * [CSI for S3](https://github.com/ctrox/csi-s3) - Kubernetes CSI driver * [docker-s3fs-client](https://github.com/efrecon/docker-s3fs-client) - Docker image containing s3fs * [goofys](https://github.com/kahing/goofys) - similar to s3fs but has better performance and less POSIX compatibility * [s3backer](https://github.com/archiecobbs/s3backer) - mount an S3 bucket as a single file * [S3Proxy](https://github.com/gaul/s3proxy) - combine with s3fs to mount Backblaze B2, EMC Atmos, Microsoft Azure, and OpenStack Swift buckets * [s3ql](https://github.com/s3ql/s3ql/) - similar to s3fs but uses its own object format * [YAS3FS](https://github.com/danilop/yas3fs) - similar to s3fs but uses SNS to allow multiple clients to mount a bucket ## Frequently Asked Questions * [FAQ wiki page](https://github.com/s3fs-fuse/s3fs-fuse/wiki/FAQ) * [s3fs on Stack Overflow](https://stackoverflow.com/questions/tagged/s3fs) * [s3fs on Server Fault](https://serverfault.com/questions/tagged/s3fs) ## License Copyright (C) 2010 Randy Rizun Licensed under the GNU GPL version 2 s3fs-fuse-1.95/autogen.sh000077500000000000000000000027431470675423500153120ustar00rootroot00000000000000#!/bin/sh # # This file is part of S3FS. # # Copyright 2009, 2010 Free Software Foundation, Inc. # # S3FS is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or (at # your option) any later version. # # S3FS is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. # # See the file ChangeLog for a revision history. echo "--- Make commit hash file -------" SHORTHASH="" if command -v git > /dev/null 2>&1 && test -d .git; then if SHORTHASH=$(git rev-parse --short HEAD); then echo " -> Git commit hash : ${SHORTHASH}" else echo " -> Not get git commit hash" fi else echo " -> Not found git command or .git directory" fi echo "${SHORTHASH}" > default_commit_hash echo "--- Finished commit hash file ---" echo "--- Start autotools -------------" autoupdate \ && aclocal \ && autoheader \ && automake --add-missing \ && autoconf echo "--- Finished autotools ----------" exit 0 # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/configure.ac000066400000000000000000000311521470675423500155730ustar00rootroot00000000000000###################################################################### # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ###################################################################### dnl Process this file with autoconf to produce a configure script. AC_PREREQ([2.69]) AC_INIT([s3fs],[1.95]) AC_CONFIG_HEADER([config.h]) AC_CANONICAL_TARGET AM_INIT_AUTOMAKE([foreign]) AC_PROG_CXX AC_PROG_CC AC_CHECK_HEADERS([sys/xattr.h]) AC_CHECK_HEADERS([attr/xattr.h]) AC_CHECK_HEADERS([sys/extattr.h]) AC_CHECK_FUNCS([fallocate]) CXXFLAGS="-Wall -fno-exceptions -D_FILE_OFFSET_BITS=64 -D_FORTIFY_SOURCE=3 -std=c++11 $CXXFLAGS" dnl ---------------------------------------------- dnl For macOS dnl ---------------------------------------------- case "$target" in *-cygwin* ) # Do something specific for windows using winfsp CXXFLAGS="$CXXFLAGS -D_GNU_SOURCE=1" min_fuse_version=2.8 ;; *-darwin* ) # Do something specific for mac min_fuse_version=2.7.3 min_fuse_t_version=1.0.20 ;; *) # Default Case # assume other supported linux system min_fuse_version=2.8.4 ;; esac dnl ---------------------------------------------- dnl Checking the FUSE library dnl ---------------------------------------------- dnl Distinguish between Linux (libfuse) and macOS (FUSE-T). dnl found_fuse_t=no PKG_CHECK_MODULES([FUSE_T], [fuse-t >= ${min_fuse_t_version}], [found_fuse_t=yes], [found_fuse_t=no]) AS_IF([test "$found_fuse_t" = "yes"], [PKG_CHECK_MODULES([fuse_library_checking], [fuse-t >= ${min_fuse_t_version}])], [PKG_CHECK_MODULES([fuse_library_checking], [fuse >= ${min_fuse_version}])]) dnl ---------------------------------------------- dnl Choice SSL library dnl ---------------------------------------------- auth_lib=na nettle_lib=no use_openssl_30=no dnl dnl nettle library dnl AC_MSG_CHECKING([s3fs build with nettle(GnuTLS)]) AC_ARG_WITH( nettle, [AS_HELP_STRING([--with-nettle], [s3fs build with nettle in GnuTLS(default no)])], [ case "${withval}" in yes) AC_MSG_RESULT(yes) nettle_lib=yes ;; *) AC_MSG_RESULT(no) ;; esac ], [ AC_MSG_RESULT(no) ]) dnl dnl use openssl library for ssl dnl AC_MSG_CHECKING([s3fs build with OpenSSL]) AC_ARG_WITH( openssl, [AS_HELP_STRING([--with-openssl], [s3fs build with OpenSSL(default is no)])], [ case "${withval}" in yes) AC_MSG_RESULT(yes) AS_IF( [test $nettle_lib = no], [auth_lib=openssl], [AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])]) ;; *) AC_MSG_RESULT(no) ;; esac ], [ AC_MSG_RESULT(no) ]) dnl dnl use GnuTLS library for ssl dnl AC_MSG_CHECKING([s3fs build with GnuTLS]) AC_ARG_WITH( gnutls, [AS_HELP_STRING([--with-gnutls], [s3fs build with GnuTLS(default is no)])], [ case "${withval}" in yes) AC_MSG_RESULT(yes) AS_IF( [test $auth_lib = na], [ AS_IF( [test $nettle_lib = no], [auth_lib=gnutls], [auth_lib=nettle]) ], [AC_MSG_ERROR([could not set gnutls because already set another ssl library])]) ;; *) AC_MSG_RESULT(no) ;; esac ], [ AC_MSG_RESULT(no) ]) dnl dnl use nss library for ssl dnl AC_MSG_CHECKING([s3fs build with NSS]) AC_ARG_WITH( nss, [AS_HELP_STRING([--with-nss], [s3fs build with NSS(default is no)])], [ case "${withval}" in yes) AC_MSG_RESULT(yes) AS_IF( [test $auth_lib = na], [ AS_IF( [test $nettle_lib = no], [auth_lib=nss], [AC_MSG_ERROR([could not set openssl with nettle, nettle is only for gnutls library])]) ], [AC_MSG_ERROR([could not set nss because already set another ssl library])]) ;; *) AC_MSG_RESULT(no) ;; esac ], [ AC_MSG_RESULT(no) ]) AS_IF( [test $auth_lib = na], AS_IF( [test $nettle_lib = no], [auth_lib=openssl], [AC_MSG_ERROR([could not set nettle without GnuTLS library])] ) ) dnl dnl For PKG_CONFIG before checking nss/gnutls. dnl AC_MSG_CHECKING([compile s3fs with]) case "${auth_lib}" in openssl) AC_MSG_RESULT(OpenSSL) AS_IF([test "$found_fuse_t" = "yes"], [PKG_CHECK_MODULES([DEPS], [fuse-t >= ${min_fuse_t_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 ])], [PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 ])]) AC_MSG_CHECKING([openssl 3.0 or later]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[#include #if !defined(LIBRESSL_VERSION_NUMBER) && OPENSSL_VERSION_NUMBER >= 0x30000000L #error "found openssl is 3.0 or later(so compiling is stopped with error)" #endif]], [[]])], [AC_MSG_RESULT(no)], [AC_MSG_RESULT(yes); use_openssl_30=yes]) ;; gnutls) AC_MSG_RESULT(GnuTLS-gcrypt) gnutls_nettle="" AC_CHECK_LIB(gnutls, gcry_control, [gnutls_nettle=0]) AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])]) AS_IF([test $gnutls_nettle = 0], [ AS_IF([test "$found_fuse_t" = "yes"], [PKG_CHECK_MODULES([DEPS], [fuse-t >= ${min_fuse_t_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])], [PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])]) LIBS="-lgnutls -lgcrypt $LIBS" AC_MSG_CHECKING([gnutls is build with]) AC_MSG_RESULT(gcrypt) ], [AC_MSG_ERROR([GnuTLS found, but gcrypt not found])]) ;; nettle) AC_MSG_RESULT(GnuTLS-nettle) gnutls_nettle="" AC_CHECK_LIB(gnutls, nettle_MD5Init, [gnutls_nettle=1]) AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])]) AS_IF([test $gnutls_nettle = 1], [ AS_IF([test "$found_fuse_t" = "yes"], [PKG_CHECK_MODULES([DEPS], [fuse-t >= ${min_fuse_t_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])], [PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])]) LIBS="-lgnutls -lnettle $LIBS" AC_MSG_CHECKING([gnutls is build with]) AC_MSG_RESULT(nettle) ], [AC_MSG_ERROR([GnuTLS found, but nettle not found])]) ;; nss) AC_MSG_RESULT(NSS) AS_IF([test "$found_fuse_t" = "yes"], [PKG_CHECK_MODULES([DEPS], [fuse-t >= ${min_fuse_t_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])], [PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])]) ;; *) AC_MSG_ERROR([unknown ssl library type.]) ;; esac AM_CONDITIONAL([USE_SSL_OPENSSL], [test "$auth_lib" = openssl]) AM_CONDITIONAL([USE_SSL_OPENSSL_30], [test "$use_openssl_30" = yes]) AM_CONDITIONAL([USE_SSL_GNUTLS], [test "$auth_lib" = gnutls -o "$auth_lib" = nettle]) AM_CONDITIONAL([USE_GNUTLS_NETTLE], [test "$auth_lib" = nettle]) AM_CONDITIONAL([USE_SSL_NSS], [test "$auth_lib" = nss]) dnl ---------------------------------------------- dnl check functions dnl ---------------------------------------------- dnl malloc_trim function AC_CHECK_FUNCS([malloc_trim]) dnl clock_gettime function(macos) AC_SEARCH_LIBS([clock_gettime],[rt posix4]) AC_CHECK_FUNCS([clock_gettime]) dnl ---------------------------------------------- dnl check symbols/macros/enums dnl ---------------------------------------------- dnl PTHREAD_MUTEX_RECURSIVE AC_MSG_CHECKING([pthread mutex recursive]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[#include ]], [[int i = PTHREAD_MUTEX_RECURSIVE;]]) ], [AC_DEFINE(S3FS_MUTEX_RECURSIVE, PTHREAD_MUTEX_RECURSIVE, [Define if you have PTHREAD_MUTEX_RECURSIVE]) AC_MSG_RESULT(PTHREAD_MUTEX_RECURSIVE) ], [AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[#include ]], [[int i = PTHREAD_MUTEX_RECURSIVE_NP;]]) ], [AC_DEFINE(S3FS_MUTEX_RECURSIVE, PTHREAD_MUTEX_RECURSIVE_NP, [Define if you have PTHREAD_MUTEX_RECURSIVE_NP]) AC_MSG_RESULT(PTHREAD_MUTEX_RECURSIVE_NP) ], [AC_MSG_ERROR([do not have PTHREAD_MUTEX_RECURSIVE symbol])]) ] ) dnl ---------------------------------------------- dnl check CURLoption dnl ---------------------------------------------- dnl CURLOPT_TCP_KEEPALIVE (is supported by 7.25.0 and later) AC_MSG_CHECKING([checking CURLOPT_TCP_KEEPALIVE]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[#include ]], [[CURLoption opt = CURLOPT_TCP_KEEPALIVE;]]) ], [AC_DEFINE(HAVE_CURLOPT_TCP_KEEPALIVE, 1, [Define to 1 if libcurl has CURLOPT_TCP_KEEPALIVE CURLoption]) AC_MSG_RESULT(yes) ], [AC_DEFINE(HAVE_CURLOPT_TCP_KEEPALIVE, 0, [Define to 1 if libcurl has CURLOPT_TCP_KEEPALIVE CURLoption]) AC_MSG_RESULT(no) ] ) dnl CURLOPT_SSL_ENABLE_ALPN (is supported by 7.36.0 and later) AC_MSG_CHECKING([checking CURLOPT_SSL_ENABLE_ALPN]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[#include ]], [[CURLoption opt = CURLOPT_SSL_ENABLE_ALPN;]]) ], [AC_DEFINE(HAVE_CURLOPT_SSL_ENABLE_ALPN, 1, [Define to 1 if libcurl has CURLOPT_SSL_ENABLE_ALPN CURLoption]) AC_MSG_RESULT(yes) ], [AC_DEFINE(HAVE_CURLOPT_SSL_ENABLE_ALPN, 0, [Define to 1 if libcurl has CURLOPT_SSL_ENABLE_ALPN CURLoption]) AC_MSG_RESULT(no) ] ) dnl CURLOPT_KEEP_SENDING_ON_ERROR (is supported by 7.51.0 and later) AC_MSG_CHECKING([checking CURLOPT_KEEP_SENDING_ON_ERROR]) AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[#include ]], [[CURLoption opt = CURLOPT_KEEP_SENDING_ON_ERROR;]]) ], [AC_DEFINE(HAVE_CURLOPT_KEEP_SENDING_ON_ERROR, 1, [Define to 1 if libcurl has CURLOPT_KEEP_SENDING_ON_ERROR CURLoption]) AC_MSG_RESULT(yes) ], [AC_DEFINE(HAVE_CURLOPT_KEEP_SENDING_ON_ERROR, 0, [Define to 1 if libcurl has CURLOPT_KEEP_SENDING_ON_ERROR CURLoption]) AC_MSG_RESULT(no) ] ) dnl ---------------------------------------------- dnl dl library dnl ---------------------------------------------- AC_CHECK_LIB([dl], [dlopen, dlclose, dlerror, dlsym], [], [AC_MSG_ERROR([Could not found dlopen, dlclose, dlerror and dlsym])]) dnl ---------------------------------------------- dnl build date dnl ---------------------------------------------- AC_SUBST([MAN_PAGE_DATE], [$(date -r doc/man/s3fs.1.in +"%B %Y")]) dnl ---------------------------------------------- dnl output files dnl ---------------------------------------------- AC_CONFIG_FILES(Makefile src/Makefile test/Makefile doc/Makefile doc/man/s3fs.1) dnl ---------------------------------------------- dnl short commit hash dnl ---------------------------------------------- AC_CHECK_PROG([GITCMD], [git --version], [yes], [no]) AS_IF([test -d .git], [DOTGITDIR=yes], [DOTGITDIR=no]) AC_MSG_CHECKING([github short commit hash]) if test "x${GITCMD}" = "xyes" -a "x${DOTGITDIR}" = "xyes"; then TMP_GITCOMMITHASH=`git rev-parse --short HEAD` UNTRACKED_FILES=`git status -s --untracked-files=no` if test -n "${UNTRACKED_FILES}"; then GITCOMMITHASH="(commit:${TMP_GITCOMMITHASH} +untracked files)" else GITCOMMITHASH="(commit:${TMP_GITCOMMITHASH})" fi elif test -f default_commit_hash; then TMP_GITCOMMITHASH=`cat default_commit_hash` if test -n "${TMP_GITCOMMITHASH}"; then GITCOMMITHASH="(base commit:${TMP_GITCOMMITHASH})" else GITCOMMITHASH="" fi else GITCOMMITHASH="" fi AC_MSG_RESULT([${GITCOMMITHASH}]) AC_DEFINE_UNQUOTED([COMMIT_HASH_VAL], ["${GITCOMMITHASH}"], [short commit hash value on github]) dnl ---------------------------------------------- dnl put dnl ---------------------------------------------- AC_OUTPUT dnl ---------------------------------------------- dnl end configuration dnl ---------------------------------------------- # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/doc/000077500000000000000000000000001470675423500140505ustar00rootroot00000000000000s3fs-fuse-1.95/doc/Makefile.am000066400000000000000000000017321470675423500161070ustar00rootroot00000000000000###################################################################### # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ###################################################################### dist_man1_MANS = man/s3fs.1 s3fs-fuse-1.95/doc/man/000077500000000000000000000000001470675423500146235ustar00rootroot00000000000000s3fs-fuse-1.95/doc/man/s3fs.1.in000066400000000000000000000773311470675423500162030ustar00rootroot00000000000000.TH S3FS "1" "@MAN_PAGE_DATE@" "S3FS" "User Commands" .SH NAME S3FS \- FUSE-based file system backed by Amazon S3 .SH SYNOPSIS .SS mounting .TP \fBs3fs bucket[:/path] mountpoint \fP [options] .TP \fBs3fs mountpoint \fP [options (must specify bucket= option)] .SS unmounting .TP \fBumount mountpoint For root. .TP \fBfusermount -u mountpoint For unprivileged user. .SS utility mode (remove interrupted multipart uploading objects) .TP \fBs3fs --incomplete-mpu-list (-u) bucket .TP \fBs3fs --incomplete-mpu-abort[=all | =] bucket .SH DESCRIPTION s3fs is a FUSE filesystem that allows you to mount an Amazon S3 bucket as a local filesystem. It stores files natively and transparently in S3 (i.e., you can use other programs to access the same files). .SH AUTHENTICATION s3fs supports the standard AWS credentials file (https://docs.aws.amazon.com/cli/latest/userguide/cli-config-files.html) stored in `${HOME}/.aws/credentials`. Alternatively, s3fs supports a custom passwd file. Only AWS credentials file format can be used when AWS session token is required. The s3fs password file has this format (use this format if you have only one set of credentials): .RS 4 \fBaccessKeyId\fP:\fBsecretAccessKey\fP .RE If you have more than one set of credentials, this syntax is also recognized: .RS 4 \fBbucketName\fP:\fBaccessKeyId\fP:\fBsecretAccessKey\fP .RE .PP Password files can be stored in two locations: .RS 4 \fB/etc/passwd-s3fs\fP [0640] \fB$HOME/.passwd-s3fs\fP [0600] .RE .PP s3fs also recognizes the \fBAWS_ACCESS_KEY_ID\fP and \fBAWS_SECRET_ACCESS_KEY\fP environment variables. .SH OPTIONS .SS "general options" .TP \fB\-h\fR \fB\-\-help\fR print help .TP \fB\ \fR \fB\-\-version\fR print version .TP \fB\-f\fR FUSE foreground option - do not run as daemon. .TP \fB\-s\fR FUSE single-threaded option (disables multi-threaded operation) .SS "mount options" .TP All s3fs options must given in the form where "opt" is: = .TP \fB\-o\fR bucket if it is not specified bucket name (and path) in command line, must specify this option after \-o option for bucket name. .TP \fB\-o\fR default_acl (default="private") the default canned acl to apply to all written s3 objects, e.g., "private", "public-read". see https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl for the full list of canned ACLs. .TP \fB\-o\fR retries (default="5") number of times to retry a failed S3 transaction. .TP \fB\-o\fR tmpdir (default="/tmp") local folder for temporary files. .TP \fB\-o\fR use_cache (default="" which means disabled) local folder to use for local file cache. .TP \fB\-o\fR check_cache_dir_exist (default is disable) If use_cache is set, check if the cache directory exists. If this option is not specified, it will be created at runtime when the cache directory does not exist. .TP \fB\-o\fR del_cache - delete local file cache delete local file cache when s3fs starts and exits. .TP \fB\-o\fR storage_class (default="standard") store object with specified storage class. Possible values: standard, standard_ia, onezone_ia, reduced_redundancy, intelligent_tiering, glacier, glacier_ir, and deep_archive. .TP \fB\-o\fR use_rrs (default is disable) use Amazon's Reduced Redundancy Storage. this option can not be specified with use_sse. (can specify use_rrs=1 for old version) this option has been replaced by new storage_class option. .TP \fB\-o\fR use_sse (default is disable) Specify three type Amazon's Server-Site Encryption: SSE-S3, SSE-C or SSE-KMS. SSE-S3 uses Amazon S3-managed encryption keys, SSE-C uses customer-provided encryption keys, and SSE-KMS uses the master key which you manage in AWS KMS. You can specify "use_sse" or "use_sse=1" enables SSE-S3 type (use_sse=1 is old type parameter). Case of setting SSE-C, you can specify "use_sse=custom", "use_sse=custom:" or "use_sse=" (only specified is old type parameter). You can use "c" for short "custom". The custom key file must be 600 permission. The file can have some lines, each line is one SSE-C key. The first line in file is used as Customer-Provided Encryption Keys for uploading and changing headers etc. If there are some keys after first line, those are used downloading object which are encrypted by not first key. So that, you can keep all SSE-C keys in file, that is SSE-C key history. If you specify "custom" ("c") without file path, you need to set custom key by load_sse_c option or AWSSSECKEYS environment. (AWSSSECKEYS environment has some SSE-C keys with ":" separator.) This option is used to decide the SSE type. So that if you do not want to encrypt a object at uploading, but you need to decrypt encrypted object at downloading, you can use load_sse_c option instead of this option. For setting SSE-KMS, specify "use_sse=kmsid" or "use_sse=kmsid:". You can use "k" for short "kmsid". If you san specify SSE-KMS type with your in AWS KMS, you can set it after "kmsid:" (or "k:"). If you specify only "kmsid" ("k"), you need to set AWSSSEKMSID environment which value is . You must be careful about that you can not use the KMS id which is not same EC2 region. Additionally, if you specify SSE-KMS, your endpoints must use Secure Sockets Layer(SSL) or Transport Layer Security(TLS). .TP \fB\-o\fR load_sse_c - specify SSE-C keys Specify the custom-provided encryption keys file path for decrypting at downloading. If you use the custom-provided encryption key at uploading, you specify with "use_sse=custom". The file has many lines, one line means one custom key. So that you can keep all SSE-C keys in file, that is SSE-C key history. AWSSSECKEYS environment is as same as this file contents. .TP \fB\-o\fR passwd_file (default="") specify the path to the password file, which which takes precedence over the password in $HOME/.passwd-s3fs and /etc/passwd-s3fs .TP \fB\-o\fR ahbe_conf (default="" which means disabled) This option specifies the configuration file path which file is the additional HTTP header by file (object) extension. The configuration file format is below: ----------- line = [file suffix or regex] HTTP-header [HTTP-values] file suffix = file (object) suffix, if this field is empty, it means "reg:(.*)".(=all object). regex = regular expression to match the file (object) path. this type starts with "reg:" prefix. HTTP-header = additional HTTP header name HTTP-values = additional HTTP header value ----------- Sample: ----------- .gz Content-Encoding gzip .Z Content-Encoding compress reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2 ----------- A sample configuration file is uploaded in "test" directory. If you specify this option for set "Content-Encoding" HTTP header, please take care for RFC 2616. .TP \fB\-o\fR profile (default="default") Choose a profile from ${HOME}/.aws/credentials to authenticate against S3. Note that this format matches the AWS CLI format and differs from the s3fs passwd format. .TP \fB\-o\fR public_bucket (default="" which means disabled) anonymously mount a public bucket when set to 1, ignores the $HOME/.passwd-s3fs and /etc/passwd-s3fs files. S3 does not allow copy object api for anonymous users, then s3fs sets nocopyapi option automatically when public_bucket=1 option is specified. .TP \fB\-o\fR connect_timeout (default="300" seconds) time to wait for connection before giving up. .TP \fB\-o\fR readwrite_timeout (default="120" seconds) time to wait between read/write activity before giving up. .TP \fB\-o\fR list_object_max_keys (default="1000") specify the maximum number of keys returned by S3 list object API. The default is 1000. you can set this value to 1000 or more. .TP \fB\-o\fR max_stat_cache_size (default="100,000" entries (about 40MB)) maximum number of entries in the stat cache and symbolic link cache. .TP \fB\-o\fR stat_cache_expire (default is 900) specify expire time (seconds) for entries in the stat cache and symbolic link cache. This expire time indicates the time since cached. .TP \fB\-o\fR stat_cache_interval_expire (default is 900) specify expire time (seconds) for entries in the stat cache and symbolic link cache. This expire time is based on the time from the last access time of those cache. This option is exclusive with stat_cache_expire, and is left for compatibility with older versions. .TP \fB\-o\fR disable_noobj_cache (default is enable) By default s3fs memorizes when an object does not exist up until the stat cache timeout. This caching can cause staleness for applications. If disabled, s3fs will not memorize objects and may cause extra HeadObject requests and reduce performance. .TP \fB\-o\fR no_check_certificate (by default this option is disabled) server certificate won't be checked against the available certificate authorities. .TP \fB\-o\fR ssl_verify_hostname (default="2") When 0, do not verify the SSL certificate against the hostname. .TP \fB\-o\fR ssl_client_cert (default="") Specify an SSL client certificate. Specify this optional parameter in the following format: "[:[:[: [:]]]]" : Client certificate. Specify the file path or NickName(for NSS, etc.). : Type of certificate, default is "PEM"(optional). : Certificate's private key file(optional). : Type of private key, default is "PEM"(optional). : Passphrase of the private key(optional). It is also possible to omit this value and specify it using the environment variable "S3FS_SSL_PRIVKEY_PASSWORD". .TP \fB\-o\fR nodnscache - disable DNS cache. s3fs is always using DNS cache, this option make DNS cache disable. .TP \fB\-o\fR nosscache - disable SSL session cache. s3fs is always using SSL session cache, this option make SSL session cache disable. .TP \fB\-o\fR multireq_max (default="20") maximum number of parallel request for listing objects. .TP \fB\-o\fR parallel_count (default="5") number of parallel request for uploading big objects. s3fs uploads large object (over 25MB by default) by multipart post request, and sends parallel requests. This option limits parallel request count which s3fs requests at once. It is necessary to set this value depending on a CPU and a network band. .TP \fB\-o\fR multipart_size (default="10") part size, in MB, for each multipart request. The minimum value is 5 MB and the maximum value is 5 GB. .TP \fB\-o\fR multipart_copy_size (default="512") part size, in MB, for each multipart copy request, used for renames and mixupload. The minimum value is 5 MB and the maximum value is 5 GB. Must be at least 512 MB to copy the maximum 5 TB object size but lower values may improve performance. .TP \fB\-o\fR max_dirty_data (default="5120") Flush dirty data to S3 after a certain number of MB written. The minimum value is 50 MB. -1 value means disable. Cannot be used with nomixupload. .TP \fB\-o\fR bucket_size (default=maximum long unsigned integer value) The size of the bucket with which the corresponding elements of the statvfs structure will be filled. The option argument is an integer optionally followed by a multiplicative suffix (GB, GiB, TB, TiB, PB, PiB, EB, EiB) (no spaces in between). If no suffix is supplied, bytes are assumed; eg: 20000000, 30GB, 45TiB. Note that s3fs does not compute the actual volume size (too expensive): by default it will assume the maximum possible size; however, since this may confuse other software which uses s3fs, the advertised bucket size can be set with this option. .TP \fB\-o\fR ensure_diskfree (default 0) sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs. s3fs makes file for downloading, uploading and caching files. If the disk free space is smaller than this value, s3fs do not use disk space as possible in exchange for the performance. .TP \fB\-o\fR free_space_ratio (default="10") sets min free space ratio of the disk. The value of this option can be between 0 and 100. It will control the size of the cache according to this ratio to ensure that the idle ratio of the disk is greater than this value. For example, when the disk space is 50GB, the default value will ensure that the disk will reserve at least 50GB * 10%% = 5GB of remaining space. .TP \fB\-o\fR multipart_threshold (default="25") threshold, in MB, to use multipart upload instead of single-part. Must be at least 5 MB. .TP \fB\-o\fR singlepart_copy_limit (default="512") maximum size, in MB, of a single-part copy before trying multipart copy. .TP \fB\-o\fR host (default="https://s3.amazonaws.com") Set a non-Amazon host, e.g., https://example.com. .TP \fB\-o\fR servicepath (default="/") Set a service path when the non-Amazon host requires a prefix. .TP \fB\-o\fR url (default="https://s3.amazonaws.com") sets the url to use to access Amazon S3. If you want to use HTTP, then you can set "url=http://s3.amazonaws.com". If you do not use https, please specify the URL with the url option. .TP \fB\-o\fR endpoint (default="us-east-1") sets the endpoint to use on signature version 4. If this option is not specified, s3fs uses "us-east-1" region as the default. If the s3fs could not connect to the region specified by this option, s3fs could not run. But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region. So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server. .TP \fB\-o\fR sigv2 (default is signature version 4 falling back to version 2) sets signing AWS requests by using only signature version 2. .TP \fB\-o\fR sigv4 (default is signature version 4 falling back to version 2) sets signing AWS requests by using only signature version 4. .TP \fB\-o\fR mp_umask (default is "0000") sets umask for the mount point directory. If allow_other option is not set, s3fs allows access to the mount point only to the owner. In the opposite case s3fs allows access to all users as the default. But if you set the allow_other with this option, you can control the permissions of the mount point by this option like umask. .TP \fB\-o\fR umask (default is "0000") sets umask for files under the mountpoint. This can allow users other than the mounting user to read and write to files that they did not create. .TP \fB\-o\fR nomultipart - disable multipart uploads .TP \fB\-o\fR streamupload (default is disable) Enable stream upload. If this option is enabled, a sequential upload will be performed in parallel with the write from the part that has been written during a multipart upload. This is expected to give better performance than other upload functions. Note that this option is still experimental and may change in the future. .TP \fB\-o\fR max_thread_count (default is "5") Specifies the number of threads waiting for stream uploads. Note that this option and Stream Upload are still experimental and subject to change in the future. This option will be merged with "parallel_count" in the future. .TP \fB\-o\fR enable_content_md5 (default is disable) Allow S3 server to check data integrity of uploads via the Content-MD5 header. This can add CPU overhead to transfers. .TP \fB\-o\fR enable_unsigned_payload (default is disable) Do not calculate Content-SHA256 for PutObject and UploadPart payloads. This can reduce CPU overhead to transfers. .TP \fB\-o\fR ecs (default is disable) This option instructs s3fs to query the ECS container credential metadata address instead of the instance metadata address. .TP \fB\-o\fR iam_role (default is no IAM role) This option requires the IAM role name or "auto". If you specify "auto", s3fs will automatically use the IAM role names that are set to an instance. If you specify this option without any argument, it is the same as that you have specified the "auto". .TP \fB\-o\fR imdsv1only (default is to use IMDSv2 with fallback to v1) AWS instance metadata service, used with IAM role authentication, supports the use of an API token. If you're using an IAM role in an environment that does not support IMDSv2, setting this flag will skip retrieval and usage of the API token when retrieving IAM credentials. .TP \fB\-o\fR ibm_iam_auth (default is not using IBM IAM authentication) This option instructs s3fs to use IBM IAM authentication. In this mode, the AWSAccessKey and AWSSecretKey will be used as IBM's Service-Instance-ID and APIKey, respectively. .TP \fB\-o\fR ibm_iam_endpoint (default is https://iam.cloud.ibm.com) Sets the URL to use for IBM IAM authentication. .TP \fB\-o\fR credlib (default=\"\" which means disabled) Specifies the shared library that handles the credentials containing the authentication token. If this option is specified, the specified credential and token processing provided by the shared library ant will be performed instead of the built-in credential processing. This option cannot be specified with passwd_file, profile, use_session_token, ecs, ibm_iam_auth, ibm_iam_endpoint, imdsv1only and iam_role option. .TP \fB\-o\fR credlib_opts (default=\"\" which means disabled) Specifies the options to pass when the shared library specified in credlib is loaded and then initialized. For the string specified in this option, specify the string defined by the shared library. .TP \fB\-o\fR use_xattr (default is not handling the extended attribute) Enable to handle the extended attribute (xattrs). If you set this option, you can use the extended attribute. For example, encfs and ecryptfs need to support the extended attribute. Notice: if s3fs handles the extended attribute, s3fs can not work to copy command with preserve=mode. .TP \fB\-o\fR noxmlns - disable registering xml name space. disable registering xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01". This option should not be specified now, because s3fs looks up xmlns automatically after v1.66. .TP \fB\-o\fR nomixupload - disable copy in multipart uploads. Disable to use PUT (copy api) when multipart uploading large size objects. By default, when doing multipart upload, the range of unchanged data will use PUT (copy api) whenever possible. When nocopyapi or norenameapi is specified, use of PUT (copy api) is invalidated even if this option is not specified. .TP \fB\-o\fR nocopyapi - for other incomplete compatibility object storage. For a distributed object storage which is compatibility S3 API without PUT (copy api). If you set this option, s3fs do not use PUT with "x-amz-copy-source" (copy api). Because traffic is increased 2-3 times by this option, we do not recommend this. .TP \fB\-o\fR norenameapi - for other incomplete compatibility object storage. For a distributed object storage which is compatibility S3 API without PUT (copy api). This option is a subset of nocopyapi option. The nocopyapi option does not use copy-api for all command (ex. chmod, chown, touch, mv, etc), but this option does not use copy-api for only rename command (ex. mv). If this option is specified with nocopyapi, then s3fs ignores it. .TP \fB\-o\fR use_path_request_style (use legacy API calling style) Enable compatibility with S3-like APIs which do not support the virtual-host request style, by using the older path request style. .TP \fB\-o\fR listobjectsv2 (use ListObjectsV2) Issue ListObjectsV2 instead of ListObjects, useful on object stores without ListObjects support. .TP \fB\-o\fR noua (suppress User-Agent header) Usually s3fs outputs of the User-Agent in "s3fs/ (commit hash ; )" format. If this option is specified, s3fs suppresses the output of the User-Agent. .TP \fB\-o\fR cipher_suites Customize the list of TLS cipher suites. Expects a colon separated list of cipher suite names. A list of available cipher suites, depending on your TLS engine, can be found on the CURL library documentation: https://curl.haxx.se/docs/ssl-ciphers.html .TP \fB\-o\fR instance_name The instance name of the current s3fs mountpoint. This name will be added to logging messages and user agent headers sent by s3fs. .TP \fB\-o\fR complement_stat (complement lack of file/directory mode) s3fs complements lack of information about file/directory mode if a file or a directory object does not have x-amz-meta-mode header. As default, s3fs does not complements stat information for a object, then the object will not be able to be allowed to list/modify. .TP \fB\-o\fR compat_dir (enable support of alternative directory names) .RS s3fs supports two different naming schemas "dir/" and "dir" to map directory names to S3 objects and vice versa by default. As a third variant, directories can be determined indirectly if there is a file object with a path (e.g. "/dir/file") but without the parent directory. This option enables a fourth variant, "dir_$folder$", created by older applications. .TP S3fs uses only the first schema "dir/" to create S3 objects for directories. .TP The support for these different naming schemas causes an increased communication effort. .TP If you do not have access permissions to the bucket and specify a directory path created by a client other than s3fs for the mount point, you cannot start because the mount point directory cannot be found by s3fs. But by specifying this option, you can avoid this error. .RE .TP \fB\-o\fR use_wtf8 - support arbitrary file system encoding. S3 requires all object names to be valid UTF-8. But some clients, notably Windows NFS clients, use their own encoding. This option re-encodes invalid UTF-8 object names into valid UTF-8 by mapping offending codes into a 'private' codepage of the Unicode set. Useful on clients not using UTF-8 as their file system encoding. .TP \fB\-o\fR use_session_token - indicate that session token should be provided. If credentials are provided by environment variables this switch forces presence check of AWS_SESSION_TOKEN variable. Otherwise an error is returned. .TP \fB\-o\fR requester_pays (default is disable) This option instructs s3fs to enable requests involving Requester Pays buckets (It includes the 'x-amz-request-payer=requester' entry in the request header). .TP \fB\-o\fR mime (default is "/etc/mime.types") Specify the path of the mime.types file. If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information. If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well. .TP \fB\-o\fR proxy (default="") This option specifies a proxy to S3 server. Specify the proxy with '[]' formatted. '://' can be omitted, and 'http://' is used when omitted. Also, ':' can also be omitted. If omitted, port 443 is used for HTTPS schema, and port 1080 is used otherwise. This option is the same as the curl command's '--proxy(-x)' option and libcurl's 'CURLOPT_PROXY' flag. This option is equivalent to and takes precedence over the environment variables 'http_proxy', 'all_proxy', etc. .TP \fB\-o\fR proxy_cred_file (default="") This option specifies the file that describes the username and passphrase for authentication of the proxy when the HTTP schema proxy is specified by the 'proxy' option. Username and passphrase are valid only for HTTP schema. If the HTTP proxy does not require authentication, this option is not required. Separate the username and passphrase with a ':' character and specify each as a URL-encoded string. .TP \fB\-o\fR ipresolve (default="whatever") Select what type of IP addresses to use when establishing a connection. Default('whatever') can use addresses of all IP versions(IPv4 and IPv6) that your system allows. If you specify 'IPv4', only IPv4 addresses are used. And when 'IPv6' is specified, only IPv6 addresses will be used. .TP \fB\-o\fR logfile - specify the log output file. s3fs outputs the log file to syslog. Alternatively, if s3fs is started with the "-f" option specified, the log will be output to the stdout/stderr. You can use this option to specify the log file that s3fs outputs. If you specify a log file with this option, it will reopen the log file when s3fs receives a SIGHUP signal. You can use the SIGHUP signal for log rotation. .TP \fB\-o\fR dbglevel (default="crit") Set the debug message level. set value as crit (critical), err (error), warn (warning), info (information) to debug level. default debug level is critical. If s3fs run with "-d" option, the debug level is set information. When s3fs catch the signal SIGUSR2, the debug level is bump up. .TP \fB\-o\fR curldbg - put curl debug message Put the debug message from libcurl when this option is specified. Specify "normal" or "body" for the parameter. If the parameter is omitted, it is the same as "normal". If "body" is specified, some API communication body data will be output in addition to the debug message output as "normal". .TP \fB\-o\fR no_time_stamp_msg - no time stamp in debug message The time stamp is output to the debug message by default. If this option is specified, the time stamp will not be output in the debug message. It is the same even if the environment variable "S3FS_MSGTIMESTAMP" is set to "no". .TP \fB\-o\fR set_check_cache_sigusr1 (default is stdout) If the cache is enabled, you can check the integrity of the cache file and the cache file's stats info file. This option is specified and when sending the SIGUSR1 signal to the s3fs process checks the cache status at that time. This option can take a file path as parameter to output the check result to that file. The file path parameter can be omitted. If omitted, the result will be output to stdout or syslog. .TP \fB\-o\fR update_parent_dir_stat (default is disable) The parent directory's mtime and ctime are updated when a file or directory is created or deleted (when the parent directory's inode is updated). By default, parent directory statistics are not updated. .SS "utility mode options" .TP \fB\-u\fR or \fB\-\-incomplete\-mpu\-list\fR Lists multipart incomplete objects uploaded to the specified bucket. .TP \fB\-\-incomplete\-mpu\-abort\fR all or date format (default="24H") Delete the multipart incomplete object uploaded to the specified bucket. If "all" is specified for this option, all multipart incomplete objects will be deleted. If you specify no argument as an option, objects older than 24 hours (24H) will be deleted (This is the default value). You can specify an optional date format. It can be specified as year, month, day, hour, minute, second, and it is expressed as "Y", "M", "D", "h", "m", "s" respectively. For example, "1Y6M10D12h30m30s". .SH FUSE/MOUNT OPTIONS .TP Most of the generic mount options described in 'man mount' are supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime, noatime, sync, async, dirsync). Filesystems are mounted with '\-onodev,nosuid' by default, which can only be overridden by a privileged user. .TP There are many FUSE specific mount options that can be specified. e.g. allow_other. See the FUSE README for the full set. .SH SERVER URL/REQUEST STYLE Be careful when specifying the server endpoint(URL). .TP If your bucket name contains dots("."), you should use the path request style(using "use_path_request_style" option). .TP Also, if you are using a server other than Amazon S3, you need to specify the endpoint with the "url" option. At that time, depending on the server you are using, you may have to specify the path request style("use_path_request_style" option). .SH LOCAL STORAGE CONSUMPTION .TP s3fs requires local caching for operation. You can enable a local cache with "\-o use_cache" or s3fs uses temporary files to cache pending requests to s3. .TP Apart from the requirements discussed below, it is recommended to keep enough cache resp. temporary storage to allow one copy each of all files open for reading and writing at any one time. .TP .SS Local cache with \[dq]\-o use_cache\[dq] .TP s3fs automatically maintains a local cache of files. The cache folder is specified by the parameter of "\-o use_cache". It is only a local cache that can be deleted at any time. s3fs rebuilds it if necessary. .TP Whenever s3fs needs to read or write a file on S3, it first creates the file in the cache directory and operates on it. .TP The amount of local cache storage used can be indirectly controlled with "\-o ensure_diskfree". .TP .SS Without local cache .TP Since s3fs always requires some storage space for operation, it creates temporary files to store incoming write requests until the required s3 request size is reached and the segment has been uploaded. After that, this data is truncated in the temporary file to free up storage space. .TP Per file you need at least twice the part size (default 5MB or "-o multipart_size") for writing multipart requests or space for the whole file if single requests are enabled ("\-o nomultipart"). .SH PERFORMANCE CONSIDERATIONS .TP This section discusses settings to improve s3fs performance. .TP In most cases, backend performance cannot be controlled and is therefore not part of this discussion. .TP Details of the local storage usage is discussed in "LOCAL STORAGE CONSUMPTION". .TP .SS CPU and Memory Consumption .TP s3fs is a multi-threaded application. Depending on the workload it may use multiple CPUs and a certain amount of memory. You can monitor the CPU and memory consumption with the "top" utility. .TP .SS Performance of S3 requests .TP s3fs provides several options (e.g. "\-o multipart_size", "\-o parallel_count") to control behaviour and thus indirectly the performance. The possible combinations of these options in conjunction with the various S3 backends are so varied that there is no individual recommendation other than the default values. Improved individual settings can be found by testing and measuring. .TP The two options "Enable no object cache" ("\-o enable_noobj_cache") and "Disable support of alternative directory names" ("\-o notsup_compat_dir") can be used to control shared access to the same bucket by different applications: .TP .IP \[bu] Enable no object cache ("\-o enable_noobj_cache") .RS .TP If a bucket is used exclusively by an s3fs instance, you can enable the cache for non-existent files and directories with "\-o enable_noobj_cache". This eliminates repeated requests to check the existence of an object, saving time and possibly money. .RE .IP \[bu] Enable support of alternative directory names ("\-o compat_dir") .RS .TP s3fs recognizes "dir/" objects as directories. Clients other than s3fs may use "dir", "dir_$folder$" objects as directories, or directory objects may not exist. In order for s3fs to recognize these as directories, you can specify the "compat_dir" option. .RE .IP \[bu] Completion of file and directory information ("\-o complement_stat") .RS .TP s3fs uses the "x-amz-meta-mode header" to determine if an object is a file or a directory. For this reason, objects that do not have the "x-amz-meta-mode header" may not produce the expected results(The directory cannot be displayed, etc.). By specifying the "complement_stat" option, s3fs can automatically complete this missing attribute information, and you can get the expected results. .RE .SH NOTES .TP The maximum size of objects that s3fs can handle depends on Amazon S3. For example, up to 5 GB when using single PUT API. And up to 5 TB is supported when Multipart Upload API is used. .TP s3fs leverages /etc/mime.types to "guess" the "correct" content-type based on file name extension. This means that you can copy a website to S3 and serve it up directly from S3 with correct content-types! .SH SEE ALSO fuse(8), mount(8), fusermount(1), fstab(5) .SH BUGS Due to S3's "eventual consistency" limitations, file creation can and will occasionally fail. Even after a successful create, subsequent reads can fail for an indeterminate time, even after one or more successful reads. Create and read enough files and you will eventually encounter this failure. This is not a flaw in s3fs and it is not something a FUSE wrapper like s3fs can work around. The retries option does not address this issue. Your application must either tolerate or compensate for these failures, for example by retrying creates or reads. .SH AUTHOR s3fs has been written by Randy Rizun . s3fs-fuse-1.95/src/000077500000000000000000000000001470675423500140725ustar00rootroot00000000000000s3fs-fuse-1.95/src/Makefile.am000066400000000000000000000056741470675423500161420ustar00rootroot00000000000000###################################################################### # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ###################################################################### bin_PROGRAMS=s3fs AM_CPPFLAGS = $(DEPS_CFLAGS) if USE_GNUTLS_NETTLE AM_CPPFLAGS += -DUSE_GNUTLS_NETTLE endif if USE_SSL_OPENSSL_30 AM_CPPFLAGS += -DUSE_OPENSSL_30 endif s3fs_SOURCES = \ s3fs.cpp \ s3fs_global.cpp \ s3fs_help.cpp \ s3fs_logger.cpp \ s3fs_xml.cpp \ metaheader.cpp \ mpu_util.cpp \ curl.cpp \ curl_handlerpool.cpp \ curl_multi.cpp \ curl_util.cpp \ s3objlist.cpp \ cache.cpp \ string_util.cpp \ s3fs_cred.cpp \ s3fs_util.cpp \ fdcache.cpp \ fdcache_entity.cpp \ fdcache_page.cpp \ fdcache_stat.cpp \ fdcache_auto.cpp \ fdcache_fdinfo.cpp \ fdcache_pseudofd.cpp \ fdcache_untreated.cpp \ addhead.cpp \ sighandlers.cpp \ threadpoolman.cpp \ common_auth.cpp if USE_SSL_OPENSSL s3fs_SOURCES += openssl_auth.cpp endif if USE_SSL_GNUTLS s3fs_SOURCES += gnutls_auth.cpp endif if USE_SSL_NSS s3fs_SOURCES += nss_auth.cpp endif s3fs_LDADD = $(DEPS_LIBS) noinst_PROGRAMS = \ test_curl_util \ test_page_list \ test_string_util test_curl_util_SOURCES = common_auth.cpp curl_util.cpp string_util.cpp test_curl_util.cpp s3fs_global.cpp s3fs_logger.cpp if USE_SSL_OPENSSL test_curl_util_SOURCES += openssl_auth.cpp endif if USE_SSL_GNUTLS test_curl_util_SOURCES += gnutls_auth.cpp endif if USE_SSL_NSS test_curl_util_SOURCES += nss_auth.cpp endif test_curl_util_LDADD = $(DEPS_LIBS) test_page_list_SOURCES = \ fdcache_page.cpp \ s3fs_global.cpp \ s3fs_logger.cpp \ string_util.cpp \ test_page_list.cpp test_string_util_SOURCES = string_util.cpp test_string_util.cpp s3fs_logger.cpp TESTS = \ test_curl_util \ test_page_list \ test_string_util clang-tidy: clang-tidy -extra-arg-before=-xc++ \ *.h $(s3fs_SOURCES) test_curl_util.cpp test_page_list.cpp test_string_util.cpp \ -- $(DEPS_CFLAGS) $(CPPFLAGS) # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: noexpandtab sw=4 ts=4 fdm=marker # vim<600: noexpandtab sw=4 ts=4 # s3fs-fuse-1.95/src/addhead.cpp000066400000000000000000000160251470675423500161540ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include "s3fs.h" #include "addhead.h" #include "curl_util.h" #include "s3fs_logger.h" //------------------------------------------------------------------- // Symbols //------------------------------------------------------------------- static constexpr char ADD_HEAD_REGEX[] = "reg:"; //------------------------------------------------------------------- // Class AdditionalHeader //------------------------------------------------------------------- AdditionalHeader AdditionalHeader::singleton; //------------------------------------------------------------------- // Class AdditionalHeader method //------------------------------------------------------------------- AdditionalHeader::AdditionalHeader() { if(this == AdditionalHeader::get()){ is_enable = false; }else{ abort(); } } AdditionalHeader::~AdditionalHeader() { if(this == AdditionalHeader::get()){ Unload(); }else{ abort(); } } bool AdditionalHeader::Load(const char* file) { if(!file){ S3FS_PRN_WARN("file is nullptr."); return false; } Unload(); std::ifstream AH(file); if(!AH.good()){ S3FS_PRN_WARN("Could not open file(%s).", file); return false; } // read file std::string line; while(getline(AH, line)){ if(line.empty()){ continue; } if('#' == line[0]){ continue; } // load a line std::istringstream ss(line); std::string key; // suffix(key) std::string head; // additional HTTP header std::string value; // header value if(0 == isblank(line[0])){ ss >> key; } if(ss){ ss >> head; if(ss && static_cast(ss.tellg()) < line.size()){ value = line.substr(static_cast(ss.tellg()) + 1); } } // check it if(head.empty()){ if(key.empty()){ continue; } S3FS_PRN_ERR("file format error: %s key(suffix) is no HTTP header value.", key.c_str()); Unload(); return false; } if(0 == strncasecmp(key.c_str(), ADD_HEAD_REGEX, strlen(ADD_HEAD_REGEX))){ // regex if(key.size() <= strlen(ADD_HEAD_REGEX)){ S3FS_PRN_ERR("file format error: %s key(suffix) does not have key std::string.", key.c_str()); continue; } key.erase(0, strlen(ADD_HEAD_REGEX)); // compile RegexPtr preg(new regex_t, regfree); int result; if(0 != (result = regcomp(preg.get(), key.c_str(), REG_EXTENDED | REG_NOSUB))){ // we do not need matching info char errbuf[256]; regerror(result, preg.get(), errbuf, sizeof(errbuf)); S3FS_PRN_ERR("failed to compile regex from %s key by %s.", key.c_str(), errbuf); continue; } addheadlist.emplace_back(std::move(preg), key, head, value); }else{ // not regex, directly comparing addheadlist.emplace_back(RegexPtr(nullptr, regfree), key, head, value); } // set flag is_enable = true; } return true; } void AdditionalHeader::Unload() { is_enable = false; addheadlist.clear(); } bool AdditionalHeader::AddHeader(headers_t& meta, const char* path) const { if(!is_enable){ return true; } if(!path){ S3FS_PRN_WARN("path is nullptr."); return false; } size_t pathlength = strlen(path); // loop // // [NOTE] // Because to allow duplicate key, and then scanning the entire table. // for(auto iter = addheadlist.cbegin(); iter != addheadlist.cend(); ++iter){ const add_header *paddhead = &*iter; if(paddhead->pregex){ // regex regmatch_t match; // not use if(0 == regexec(paddhead->pregex.get(), path, 1, &match, 0)){ // match -> adding header meta[paddhead->headkey] = paddhead->headvalue; } }else{ // directly comparing if(paddhead->basestring.length() < pathlength){ if(paddhead->basestring.empty() || paddhead->basestring == &path[pathlength - paddhead->basestring.length()]){ // match -> adding header meta[paddhead->headkey] = paddhead->headvalue; } } } } return true; } struct curl_slist* AdditionalHeader::AddHeader(struct curl_slist* list, const char* path) const { headers_t meta; if(!AddHeader(meta, path)){ return list; } for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ // Adding header list = curl_slist_sort_insert(list, iter->first.c_str(), iter->second.c_str()); } meta.clear(); S3FS_MALLOCTRIM(0); return list; } bool AdditionalHeader::Dump() const { if(!S3fsLog::IsS3fsLogDbg()){ return true; } std::ostringstream ssdbg; int cnt = 1; ssdbg << "Additional Header list[" << addheadlist.size() << "] = {" << std::endl; for(auto iter = addheadlist.cbegin(); iter != addheadlist.cend(); ++iter, ++cnt){ const add_header *paddhead = &*iter; ssdbg << " [" << cnt << "] = {" << std::endl; if(paddhead->pregex){ ssdbg << " type\t\t--->\tregex" << std::endl; }else{ ssdbg << " type\t\t--->\tsuffix matching" << std::endl; } ssdbg << " base std::string\t--->\t" << paddhead->basestring << std::endl; ssdbg << " add header\t--->\t" << paddhead->headkey << ": " << paddhead->headvalue << std::endl; ssdbg << " }" << std::endl; } ssdbg << "}" << std::endl; // print all S3FS_PRN_DBG("%s", ssdbg.str().c_str()); return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/addhead.h000066400000000000000000000057561470675423500156320ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_ADDHEAD_H_ #define S3FS_ADDHEAD_H_ #include #include #include #include #include #include "metaheader.h" //---------------------------------------------- // Structure / Typedef //---------------------------------------------- typedef std::unique_ptr RegexPtr; struct add_header{ add_header(RegexPtr pregex, std::string basestring, std::string headkey, std::string headvalue) : pregex(std::move(pregex)) , basestring(std::move(basestring)) , headkey(std::move(headkey)) , headvalue(std::move(headvalue)) {} add_header(const add_header&) = delete; add_header(add_header&& val) = default; add_header& operator=(const add_header&) = delete; add_header& operator=(add_header&&) = delete; RegexPtr pregex; // not nullptr means using regex, nullptr means comparing suffix directly. std::string basestring; std::string headkey; std::string headvalue; }; typedef std::vector addheadlist_t; //---------------------------------------------- // Class AdditionalHeader //---------------------------------------------- class AdditionalHeader { private: static AdditionalHeader singleton; bool is_enable; addheadlist_t addheadlist; protected: AdditionalHeader(); ~AdditionalHeader(); public: AdditionalHeader(const AdditionalHeader&) = delete; AdditionalHeader(AdditionalHeader&&) = delete; AdditionalHeader& operator=(const AdditionalHeader&) = delete; AdditionalHeader& operator=(AdditionalHeader&&) = delete; // Reference singleton static AdditionalHeader* get() { return &singleton; } bool Load(const char* file); void Unload(); bool AddHeader(headers_t& meta, const char* path) const; struct curl_slist* AddHeader(struct curl_slist* list, const char* path) const; bool Dump() const; }; #endif // S3FS_ADDHEAD_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/cache.cpp000066400000000000000000000634011470675423500156450ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include "s3fs.h" #include "s3fs_logger.h" #include "s3fs_util.h" #include "cache.h" #include "string_util.h" //------------------------------------------------------------------- // Utility //------------------------------------------------------------------- static void SetStatCacheTime(struct timespec& ts) { if(-1 == clock_gettime(static_cast(CLOCK_MONOTONIC_COARSE), &ts)){ S3FS_PRN_CRIT("clock_gettime failed: %d", errno); abort(); } } static int CompareStatCacheTime(const struct timespec& ts1, const struct timespec& ts2) { // return -1: ts1 < ts2 // 0: ts1 == ts2 // 1: ts1 > ts2 if(ts1.tv_sec < ts2.tv_sec){ return -1; }else if(ts1.tv_sec > ts2.tv_sec){ return 1; }else{ if(ts1.tv_nsec < ts2.tv_nsec){ return -1; }else if(ts1.tv_nsec > ts2.tv_nsec){ return 1; } } return 0; } static bool IsExpireStatCacheTime(const struct timespec& ts, time_t expire) { struct timespec nowts; SetStatCacheTime(nowts); nowts.tv_sec -= expire; return (0 < CompareStatCacheTime(nowts, ts)); } // // For stats cache out // typedef std::vector statiterlist_t; struct sort_statiterlist{ // ascending order bool operator()(const stat_cache_t::iterator& src1, const stat_cache_t::iterator& src2) const { int result = CompareStatCacheTime(src1->second.cache_date, src2->second.cache_date); if(0 == result){ if(src1->second.hit_count < src2->second.hit_count){ result = -1; } } return (result < 0); } }; // // For symbolic link cache out // typedef std::vector symlinkiterlist_t; struct sort_symlinkiterlist{ // ascending order bool operator()(const symlink_cache_t::iterator& src1, const symlink_cache_t::iterator& src2) const { int result = CompareStatCacheTime(src1->second.cache_date, src2->second.cache_date); // use the same as Stats if(0 == result){ if(src1->second.hit_count < src2->second.hit_count){ result = -1; } } return (result < 0); } }; //------------------------------------------------------------------- // Static //------------------------------------------------------------------- StatCache StatCache::singleton; std::mutex StatCache::stat_cache_lock; //------------------------------------------------------------------- // Constructor/Destructor //------------------------------------------------------------------- StatCache::StatCache() : IsExpireTime(true), IsExpireIntervalType(false), ExpireTime(15 * 60), CacheSize(100000), IsCacheNoObject(true) { if(this == StatCache::getStatCacheData()){ stat_cache.clear(); }else{ abort(); } } StatCache::~StatCache() { if(this == StatCache::getStatCacheData()){ Clear(); }else{ abort(); } } //------------------------------------------------------------------- // Methods //------------------------------------------------------------------- unsigned long StatCache::GetCacheSize() const { return CacheSize; } unsigned long StatCache::SetCacheSize(unsigned long size) { unsigned long old = CacheSize; CacheSize = size; return old; } time_t StatCache::GetExpireTime() const { return (IsExpireTime ? ExpireTime : (-1)); } time_t StatCache::SetExpireTime(time_t expire, bool is_interval) { time_t old = ExpireTime; ExpireTime = expire; IsExpireTime = true; IsExpireIntervalType = is_interval; return old; } time_t StatCache::UnsetExpireTime() { time_t old = IsExpireTime ? ExpireTime : (-1); ExpireTime = 0; IsExpireTime = false; IsExpireIntervalType = false; return old; } bool StatCache::SetCacheNoObject(bool flag) { bool old = IsCacheNoObject; IsCacheNoObject = flag; return old; } void StatCache::Clear() { const std::lock_guard lock(StatCache::stat_cache_lock); stat_cache.clear(); S3FS_MALLOCTRIM(0); } bool StatCache::GetStat(const std::string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce) { bool is_delete_cache = false; std::string strpath = key; const std::lock_guard lock(StatCache::stat_cache_lock); auto iter = stat_cache.end(); if(overcheck && '/' != *strpath.rbegin()){ strpath += "/"; iter = stat_cache.find(strpath); } if(iter == stat_cache.end()){ strpath = key; iter = stat_cache.find(strpath); } if(iter != stat_cache.end()){ stat_cache_entry* ent = &iter->second; if(0 < ent->notruncate || !IsExpireTime || !IsExpireStatCacheTime(ent->cache_date, ExpireTime)){ if(ent->noobjcache){ if(!IsCacheNoObject){ // need to delete this cache. DelStatHasLock(strpath); }else{ // noobjcache = true means no object. } return false; } // hit without checking etag std::string stretag; if(petag){ // find & check ETag for(auto hiter = ent->meta.cbegin(); hiter != ent->meta.cend(); ++hiter){ std::string tag = lower(hiter->first); if(tag == "etag"){ stretag = hiter->second; if('\0' != petag[0] && petag != stretag){ is_delete_cache = true; } break; } } } if(is_delete_cache){ // not hit by different ETag S3FS_PRN_DBG("stat cache not hit by ETag[path=%s][time=%lld.%09ld][hit count=%lu][ETag(%s)!=(%s)]", strpath.c_str(), static_cast(ent->cache_date.tv_sec), ent->cache_date.tv_nsec, ent->hit_count, petag ? petag : "null", stretag.c_str()); }else{ // hit S3FS_PRN_DBG("stat cache hit [path=%s][time=%lld.%09ld][hit count=%lu]", strpath.c_str(), static_cast(ent->cache_date.tv_sec), ent->cache_date.tv_nsec, ent->hit_count); if(pst!= nullptr){ *pst= ent->stbuf; } if(meta != nullptr){ *meta = ent->meta; } if(pisforce != nullptr){ (*pisforce) = ent->isforce; } ent->hit_count++; if(IsExpireIntervalType){ SetStatCacheTime(ent->cache_date); } return true; } }else{ // timeout is_delete_cache = true; } } if(is_delete_cache){ DelStatHasLock(strpath); } return false; } bool StatCache::IsNoObjectCache(const std::string& key, bool overcheck) { bool is_delete_cache = false; std::string strpath = key; if(!IsCacheNoObject){ return false; } const std::lock_guard lock(StatCache::stat_cache_lock); auto iter = stat_cache.end(); if(overcheck && '/' != *strpath.rbegin()){ strpath += "/"; iter = stat_cache.find(strpath); } if(iter == stat_cache.end()){ strpath = key; iter = stat_cache.find(strpath); } if(iter != stat_cache.end()) { const stat_cache_entry* ent = &iter->second; if(0 < ent->notruncate || !IsExpireTime || !IsExpireStatCacheTime(iter->second.cache_date, ExpireTime)){ if(iter->second.noobjcache){ // noobjcache = true means no object. SetStatCacheTime((*iter).second.cache_date); return true; } }else{ // timeout is_delete_cache = true; } } if(is_delete_cache){ DelStatHasLock(strpath); } return false; } bool StatCache::AddStat(const std::string& key, const headers_t& meta, bool forcedir, bool no_truncate) { if(!no_truncate && CacheSize< 1){ return true; } S3FS_PRN_INFO3("add stat cache entry[path=%s]", key.c_str()); const std::lock_guard lock(StatCache::stat_cache_lock); if(stat_cache.cend() != stat_cache.find(key)){ // found cache DelStatHasLock(key); }else{ // check: need to truncate cache if(stat_cache.size() > CacheSize){ // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!TruncateCache()){ return false; } } } // make new stat_cache_entry ent; if(!convert_header_to_stat(key.c_str(), meta, &ent.stbuf, forcedir)){ return false; } ent.hit_count = 0; ent.isforce = forcedir; ent.noobjcache = false; ent.notruncate = (no_truncate ? 1L : 0L); ent.meta.clear(); SetStatCacheTime(ent.cache_date); // Set time. //copy only some keys for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ std::string tag = lower(iter->first); std::string value = iter->second; if(tag == "content-type"){ ent.meta[iter->first] = value; }else if(tag == "content-length"){ ent.meta[iter->first] = value; }else if(tag == "etag"){ ent.meta[iter->first] = value; }else if(tag == "last-modified"){ ent.meta[iter->first] = value; }else if(is_prefix(tag.c_str(), "x-amz")){ ent.meta[tag] = value; // key is lower case for "x-amz" } } const auto& value = stat_cache[key] = std::move(ent); // check symbolic link cache if(!S_ISLNK(value.stbuf.st_mode)){ if(symlink_cache.cend() != symlink_cache.find(key)){ // if symbolic link cache has key, thus remove it. DelSymlinkHasLock(key); } } // If no_truncate flag is set, set file name to notruncate_file_cache // if(no_truncate){ AddNotruncateCache(key); } return true; } // [NOTE] // Updates only meta data if cached data exists. // And when these are updated, it also updates the cache time. // // Since the file mode may change while the file is open, it is // updated as well. // bool StatCache::UpdateMetaStats(const std::string& key, const headers_t& meta) { if(CacheSize < 1){ return true; } S3FS_PRN_INFO3("update stat cache entry[path=%s]", key.c_str()); const std::lock_guard lock(StatCache::stat_cache_lock); auto iter = stat_cache.find(key); if(stat_cache.cend() == iter){ return true; } stat_cache_entry* ent = &iter->second; // update only meta keys for(auto metaiter = meta.cbegin(); metaiter != meta.cend(); ++metaiter){ std::string tag = lower(metaiter->first); std::string value = metaiter->second; if(tag == "content-type"){ ent->meta[metaiter->first] = value; }else if(tag == "content-length"){ ent->meta[metaiter->first] = value; }else if(tag == "etag"){ ent->meta[metaiter->first] = value; }else if(tag == "last-modified"){ ent->meta[metaiter->first] = value; }else if(is_prefix(tag.c_str(), "x-amz")){ ent->meta[tag] = value; // key is lower case for "x-amz" } } // Update time. SetStatCacheTime(ent->cache_date); // Update only mode ent->stbuf.st_mode = get_mode(meta, key); return true; } bool StatCache::AddNoObjectCache(const std::string& key) { if(!IsCacheNoObject){ return true; // pretend successful } if(CacheSize < 1){ return true; } S3FS_PRN_INFO3("add no object cache entry[path=%s]", key.c_str()); const std::lock_guard lock(StatCache::stat_cache_lock); if(stat_cache.cend() != stat_cache.find(key)){ // found DelStatHasLock(key); }else{ // check: need to truncate cache if(stat_cache.size() > CacheSize){ // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!TruncateCache()){ return false; } } } // make new stat_cache_entry ent{}; ent.hit_count = 0; ent.isforce = false; ent.noobjcache = true; ent.notruncate = 0L; ent.meta.clear(); SetStatCacheTime(ent.cache_date); // Set time. stat_cache[key] = std::move(ent); // check symbolic link cache if(symlink_cache.cend() != symlink_cache.find(key)){ // if symbolic link cache has key, thus remove it. DelSymlinkHasLock(key); } return true; } void StatCache::ChangeNoTruncateFlag(const std::string& key, bool no_truncate) { const std::lock_guard lock(StatCache::stat_cache_lock); auto iter = stat_cache.find(key); if(stat_cache.cend() != iter){ stat_cache_entry* ent = &iter->second; if(no_truncate){ if(0L == ent->notruncate){ // need to add no truncate cache. AddNotruncateCache(key); } ++(ent->notruncate); }else{ if(0L < ent->notruncate){ --(ent->notruncate); if(0L == ent->notruncate){ // need to delete from no truncate cache. DelNotruncateCache(key); } } } } } bool StatCache::TruncateCache() { if(stat_cache.empty()){ return true; } // 1) erase over expire time if(IsExpireTime){ for(auto iter = stat_cache.cbegin(); iter != stat_cache.cend(); ){ const stat_cache_entry* entry = &iter->second; if(0L == entry->notruncate && IsExpireStatCacheTime(entry->cache_date, ExpireTime)){ iter = stat_cache.erase(iter); }else{ ++iter; } } } // 2) check stat cache count if(stat_cache.size() < CacheSize){ return true; } // 3) erase from the old cache in order size_t erase_count= stat_cache.size() - CacheSize + 1; statiterlist_t erase_iters; for(auto iter = stat_cache.begin(); iter != stat_cache.end() && 0 < erase_count; ++iter){ // check no truncate const stat_cache_entry* ent = &iter->second; if(0L < ent->notruncate){ // skip for no truncate entry and keep extra counts for this entity. if(0 < erase_count){ --erase_count; // decrement } }else{ // iter is not have notruncate flag erase_iters.push_back(iter); } if(erase_count < erase_iters.size()){ std::sort(erase_iters.begin(), erase_iters.end(), sort_statiterlist()); while(erase_count < erase_iters.size()){ erase_iters.pop_back(); } } } for(auto iiter = erase_iters.cbegin(); iiter != erase_iters.cend(); ++iiter){ auto siter = *iiter; S3FS_PRN_DBG("truncate stat cache[path=%s]", siter->first.c_str()); stat_cache.erase(siter); } S3FS_MALLOCTRIM(0); return true; } bool StatCache::DelStatHasLock(const std::string& key) { S3FS_PRN_INFO3("delete stat cache entry[path=%s]", key.c_str()); stat_cache_t::iterator iter; if(stat_cache.cend() != (iter = stat_cache.find(key))){ stat_cache.erase(iter); DelNotruncateCache(key); } if(!key.empty() && key != "/"){ std::string strpath = key; if('/' == *strpath.rbegin()){ // If there is "path" cache, delete it. strpath.erase(strpath.length() - 1); }else{ // If there is "path/" cache, delete it. strpath += "/"; } if(stat_cache.cend() != (iter = stat_cache.find(strpath))){ stat_cache.erase(iter); DelNotruncateCache(strpath); } } S3FS_MALLOCTRIM(0); return true; } bool StatCache::GetSymlink(const std::string& key, std::string& value) { bool is_delete_cache = false; const std::string& strpath = key; const std::lock_guard lock(StatCache::stat_cache_lock); auto iter = symlink_cache.find(strpath); if(iter != symlink_cache.cend()){ symlink_cache_entry* ent = &iter->second; if(!IsExpireTime || !IsExpireStatCacheTime(ent->cache_date, ExpireTime)){ // use the same as Stats // found S3FS_PRN_DBG("symbolic link cache hit [path=%s][time=%lld.%09ld][hit count=%lu]", strpath.c_str(), static_cast(ent->cache_date.tv_sec), ent->cache_date.tv_nsec, ent->hit_count); value = ent->link; ent->hit_count++; if(IsExpireIntervalType){ SetStatCacheTime(ent->cache_date); } return true; }else{ // timeout is_delete_cache = true; } } if(is_delete_cache){ DelSymlinkHasLock(strpath); } return false; } bool StatCache::AddSymlink(const std::string& key, const std::string& value) { if(CacheSize< 1){ return true; } S3FS_PRN_INFO3("add symbolic link cache entry[path=%s, value=%s]", key.c_str(), value.c_str()); const std::lock_guard lock(StatCache::stat_cache_lock); if(symlink_cache.cend() != symlink_cache.find(key)){ // found DelSymlinkHasLock(key); }else{ // check: need to truncate cache if(symlink_cache.size() > CacheSize){ // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!TruncateSymlink()){ return false; } } } // make new symlink_cache_entry ent; ent.link = value; ent.hit_count = 0; SetStatCacheTime(ent.cache_date); // Set time(use the same as Stats). symlink_cache[key] = std::move(ent); return true; } bool StatCache::TruncateSymlink() { if(symlink_cache.empty()){ return true; } // 1) erase over expire time if(IsExpireTime){ for(auto iter = symlink_cache.cbegin(); iter != symlink_cache.cend(); ){ const symlink_cache_entry* entry = &iter->second; if(IsExpireStatCacheTime(entry->cache_date, ExpireTime)){ // use the same as Stats iter = symlink_cache.erase(iter); }else{ ++iter; } } } // 2) check stat cache count if(symlink_cache.size() < CacheSize){ return true; } // 3) erase from the old cache in order size_t erase_count= symlink_cache.size() - CacheSize + 1; symlinkiterlist_t erase_iters; for(auto iter = symlink_cache.begin(); iter != symlink_cache.end(); ++iter){ erase_iters.push_back(iter); sort(erase_iters.begin(), erase_iters.end(), sort_symlinkiterlist()); if(erase_count < erase_iters.size()){ erase_iters.pop_back(); } } for(auto iiter = erase_iters.cbegin(); iiter != erase_iters.cend(); ++iiter){ auto siter = *iiter; S3FS_PRN_DBG("truncate symbolic link cache[path=%s]", siter->first.c_str()); symlink_cache.erase(siter); } S3FS_MALLOCTRIM(0); return true; } bool StatCache::DelSymlinkHasLock(const std::string& key) { S3FS_PRN_INFO3("delete symbolic link cache entry[path=%s]", key.c_str()); symlink_cache_t::iterator iter; if(symlink_cache.cend() != (iter = symlink_cache.find(key))){ symlink_cache.erase(iter); } S3FS_MALLOCTRIM(0); return true; } bool StatCache::AddNotruncateCache(const std::string& key) { if(key.empty() || '/' == *key.rbegin()){ return false; } std::string parentdir = mydirname(key); std::string filename = mybasename(key); if(parentdir.empty() || filename.empty()){ return false; } parentdir += '/'; // directory path must be '/' termination. auto iter = notruncate_file_cache.find(parentdir); if(iter == notruncate_file_cache.cend()){ // add new list notruncate_filelist_t list; list.push_back(filename); notruncate_file_cache[parentdir] = list; }else{ // add filename to existed list notruncate_filelist_t& filelist = iter->second; auto fiter = std::find(filelist.cbegin(), filelist.cend(), filename); if(fiter == filelist.cend()){ filelist.push_back(filename); } } return true; } bool StatCache::DelNotruncateCache(const std::string& key) { if(key.empty() || '/' == *key.rbegin()){ return false; } std::string parentdir = mydirname(key); std::string filename = mybasename(key); if(parentdir.empty() || filename.empty()){ return false; } parentdir += '/'; // directory path must be '/' termination. auto iter = notruncate_file_cache.find(parentdir); if(iter != notruncate_file_cache.cend()){ // found directory in map notruncate_filelist_t& filelist = iter->second; auto fiter = std::find(filelist.begin(), filelist.end(), filename); if(fiter != filelist.cend()){ // found filename in directory file list filelist.erase(fiter); if(filelist.empty()){ notruncate_file_cache.erase(parentdir); } } } return true; } // [Background] // When s3fs creates a new file, the file does not exist until the file contents // are uploaded.(because it doesn't create a 0 byte file) // From the time this file is created(opened) until it is uploaded(flush), it // will have a Stat cache with the No truncate flag added. // This avoids file not existing errors in operations such as chmod and utimens // that occur in the short period before file upload. // Besides this, we also need to support readdir(list_bucket), this method is // called to maintain the cache for readdir and return its value. // // [NOTE] // Add the file names under parentdir to the list. // However, if the same file name exists in the list, it will not be added. // parentdir must be terminated with a '/'. // bool StatCache::GetNotruncateCache(const std::string& parentdir, notruncate_filelist_t& list) { if(parentdir.empty()){ return false; } std::string dirpath = parentdir; if('/' != *dirpath.rbegin()){ dirpath += '/'; } const std::lock_guard lock(StatCache::stat_cache_lock); auto iter = notruncate_file_cache.find(dirpath); if(iter == notruncate_file_cache.cend()){ // not found directory map return true; } // found directory in map const notruncate_filelist_t& filelist = iter->second; for(auto fiter = filelist.cbegin(); fiter != filelist.cend(); ++fiter){ if(list.cend() == std::find(list.cbegin(), list.cend(), *fiter)){ // found notuncate file that does not exist in the list, so add it. list.push_back(*fiter); } } return true; } //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- bool convert_header_to_stat(const char* path, const headers_t& meta, struct stat* pst, bool forcedir) { if(!path || !pst){ return false; } *pst = {}; pst->st_nlink = 1; // see fuse FAQ // mode pst->st_mode = get_mode(meta, path, true, forcedir); // blocks if(S_ISREG(pst->st_mode)){ pst->st_blocks = get_blocks(pst->st_size); } pst->st_blksize = 4096; // mtime struct timespec mtime = get_mtime(meta); if(pst->st_mtime < 0){ pst->st_mtime = 0L; }else{ if(mtime.tv_sec < 0){ mtime.tv_sec = 0; mtime.tv_nsec = 0; } set_timespec_to_stat(*pst, stat_time_type::MTIME, mtime); } // ctime struct timespec ctime = get_ctime(meta); if(pst->st_ctime < 0){ pst->st_ctime = 0L; }else{ if(ctime.tv_sec < 0){ ctime.tv_sec = 0; ctime.tv_nsec = 0; } set_timespec_to_stat(*pst, stat_time_type::CTIME, ctime); } // atime struct timespec atime = get_atime(meta); if(pst->st_atime < 0){ pst->st_atime = 0L; }else{ if(atime.tv_sec < 0){ atime.tv_sec = 0; atime.tv_nsec = 0; } set_timespec_to_stat(*pst, stat_time_type::ATIME, atime); } // size if(S_ISDIR(pst->st_mode)){ pst->st_size = 4096; }else{ pst->st_size = get_size(meta); } // uid/gid pst->st_uid = get_uid(meta); pst->st_gid = get_gid(meta); return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/cache.h000066400000000000000000000175661470675423500153250ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_CACHE_H_ #define S3FS_CACHE_H_ #include #include #include #include #include #include #include "common.h" #include "metaheader.h" //------------------------------------------------------------------- // Structure //------------------------------------------------------------------- // // Struct for stats cache // struct stat_cache_entry { struct stat stbuf = {}; unsigned long hit_count = 0; struct timespec cache_date = {0, 0}; headers_t meta; bool isforce = false; bool noobjcache = false; // Flag: cache is no object for no listing. unsigned long notruncate = 0L; // 0<: not remove automatically at checking truncate }; typedef std::map stat_cache_t; // key=path // // Struct for symbolic link cache // struct symlink_cache_entry { std::string link; unsigned long hit_count = 0; struct timespec cache_date = {0, 0}; // The function that operates timespec uses the same as Stats }; typedef std::map symlink_cache_t; // // Typedefs for No truncate file name cache // typedef std::vector notruncate_filelist_t; // untruncated file name list in dir typedef std::map notruncate_dir_map_t; // key is parent dir path //------------------------------------------------------------------- // Class StatCache //------------------------------------------------------------------- // [NOTE] About Symbolic link cache // The Stats cache class now also has a symbolic link cache. // It is possible to take out the Symbolic link cache in another class, // but the cache out etc. should be synchronized with the Stats cache // and implemented in this class. // Symbolic link cache size and timeout use the same settings as Stats // cache. This simplifies user configuration, and from a user perspective, // the symbolic link cache appears to be included in the Stats cache. // class StatCache { private: static StatCache singleton; static std::mutex stat_cache_lock; stat_cache_t stat_cache GUARDED_BY(stat_cache_lock); bool IsExpireTime; bool IsExpireIntervalType; // if this flag is true, cache data is updated at last access time. time_t ExpireTime; unsigned long CacheSize; bool IsCacheNoObject; symlink_cache_t symlink_cache GUARDED_BY(stat_cache_lock); notruncate_dir_map_t notruncate_file_cache GUARDED_BY(stat_cache_lock); StatCache(); ~StatCache(); void Clear(); bool GetStat(const std::string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce); // Truncate stat cache bool TruncateCache() REQUIRES(StatCache::stat_cache_lock); // Truncate symbolic link cache bool TruncateSymlink() REQUIRES(StatCache::stat_cache_lock); bool AddNotruncateCache(const std::string& key) REQUIRES(stat_cache_lock); bool DelNotruncateCache(const std::string& key) REQUIRES(stat_cache_lock); public: StatCache(const StatCache&) = delete; StatCache(StatCache&&) = delete; StatCache& operator=(const StatCache&) = delete; StatCache& operator=(StatCache&&) = delete; // Reference singleton static StatCache* getStatCacheData() { return &singleton; } // Attribute unsigned long GetCacheSize() const; unsigned long SetCacheSize(unsigned long size); time_t GetExpireTime() const; time_t SetExpireTime(time_t expire, bool is_interval = false); time_t UnsetExpireTime(); bool SetCacheNoObject(bool flag); bool EnableCacheNoObject() { return SetCacheNoObject(true); } bool DisableCacheNoObject() { return SetCacheNoObject(false); } bool GetCacheNoObject() const { return IsCacheNoObject; } // Get stat cache bool GetStat(const std::string& key, struct stat* pst, headers_t* meta, bool overcheck = true, bool* pisforce = nullptr) { return GetStat(key, pst, meta, overcheck, nullptr, pisforce); } bool GetStat(const std::string& key, struct stat* pst, bool overcheck = true) { return GetStat(key, pst, nullptr, overcheck, nullptr, nullptr); } bool GetStat(const std::string& key, headers_t* meta, bool overcheck = true) { return GetStat(key, nullptr, meta, overcheck, nullptr, nullptr); } bool HasStat(const std::string& key, bool overcheck = true) { return GetStat(key, nullptr, nullptr, overcheck, nullptr, nullptr); } bool HasStat(const std::string& key, const char* etag, bool overcheck = true) { return GetStat(key, nullptr, nullptr, overcheck, etag, nullptr); } bool HasStat(const std::string& key, struct stat* pst, const char* etag) { return GetStat(key, pst, nullptr, true, etag, nullptr); } // Cache For no object bool IsNoObjectCache(const std::string& key, bool overcheck = true); bool AddNoObjectCache(const std::string& key); // Add stat cache bool AddStat(const std::string& key, const headers_t& meta, bool forcedir = false, bool no_truncate = false); // Update meta stats bool UpdateMetaStats(const std::string& key, const headers_t& meta); // Change no truncate flag void ChangeNoTruncateFlag(const std::string& key, bool no_truncate); // Delete stat cache bool DelStat(const std::string& key) { const std::lock_guard lock(StatCache::stat_cache_lock); return DelStatHasLock(key); } bool DelStatHasLock(const std::string& key) REQUIRES(StatCache::stat_cache_lock); // Cache for symbolic link bool GetSymlink(const std::string& key, std::string& value); bool AddSymlink(const std::string& key, const std::string& value); bool DelSymlink(const std::string& key) { const std::lock_guard lock(StatCache::stat_cache_lock); return DelSymlinkHasLock(key); } bool DelSymlinkHasLock(const std::string& key) REQUIRES(stat_cache_lock); // Cache for Notruncate file bool GetNotruncateCache(const std::string& parentdir, notruncate_filelist_t& list); }; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- bool convert_header_to_stat(const char* path, const headers_t& meta, struct stat* pst, bool forcedir = false); #endif // S3FS_CACHE_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/common.h000066400000000000000000000054241470675423500155400ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_COMMON_H_ #define S3FS_COMMON_H_ #include #include #include "../config.h" //------------------------------------------------------------------- // Global variables //------------------------------------------------------------------- // TODO: namespace these static constexpr int64_t FIVE_GB = 5LL * 1024LL * 1024LL * 1024LL; static constexpr off_t MIN_MULTIPART_SIZE = 5 * 1024 * 1024; extern bool foreground; extern bool nomultipart; extern bool pathrequeststyle; extern bool complement_stat; extern bool noxmlns; extern std::string program_name; extern std::string service_path; extern std::string s3host; extern std::string mount_prefix; extern std::string endpoint; extern std::string cipher_suites; extern std::string instance_name; //------------------------------------------------------------------- // For weak attribute //------------------------------------------------------------------- #define S3FS_FUNCATTR_WEAK __attribute__ ((weak,unused)) //------------------------------------------------------------------- // For clang -Wthread-safety //------------------------------------------------------------------- #if defined(__clang__) #define THREAD_ANNOTATION_ATTRIBUTE(x) __attribute__((x)) #else #define THREAD_ANNOTATION_ATTRIBUTE(x) // no-op #endif #define GUARDED_BY(x) \ THREAD_ANNOTATION_ATTRIBUTE(guarded_by(x)) #define PT_GUARDED_BY(x) \ THREAD_ANNOTATION_ATTRIBUTE(pt_guarded_by(x)) #define REQUIRES(...) \ THREAD_ANNOTATION_ATTRIBUTE(requires_capability(__VA_ARGS__)) #define RETURN_CAPABILITY(...) \ THREAD_ANNOTATION_ATTRIBUTE(lock_returned(__VA_ARGS__)) #define NO_THREAD_SAFETY_ANALYSIS \ THREAD_ANNOTATION_ATTRIBUTE(no_thread_safety_analysis) #endif // S3FS_COMMON_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/common_auth.cpp000066400000000000000000000033111470675423500171050ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include "s3fs_auth.h" #include "string_util.h" //------------------------------------------------------------------- // Utility Function //------------------------------------------------------------------- std::string s3fs_get_content_md5(int fd) { md5_t md5; if(!s3fs_md5_fd(fd, 0, -1, &md5)){ // TODO: better return value? return ""; } return s3fs_base64(md5.data(), md5.size()); } std::string s3fs_sha256_hex_fd(int fd, off_t start, off_t size) { sha256_t sha256; if(!s3fs_sha256_fd(fd, start, size, &sha256)){ // TODO: better return value? return ""; } std::string sha256hex = s3fs_hex_lower(sha256.data(), sha256.size()); return sha256hex; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl.cpp000066400000000000000000004734431470675423500155620ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #include "common.h" #include "s3fs.h" #include "s3fs_logger.h" #include "curl.h" #include "curl_multi.h" #include "curl_util.h" #include "s3fs_auth.h" #include "curl_handlerpool.h" #include "s3fs_cred.h" #include "s3fs_util.h" #include "string_util.h" #include "addhead.h" //------------------------------------------------------------------- // Symbols //------------------------------------------------------------------- static constexpr char EMPTY_PAYLOAD_HASH[] = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; static constexpr char EMPTY_MD5_BASE64_HASH[] = "1B2M2Y8AsgTpgAmY7PhCfg=="; //------------------------------------------------------------------- // Class S3fsCurl //------------------------------------------------------------------- static constexpr int MULTIPART_SIZE = 10 * 1024 * 1024; static constexpr int GET_OBJECT_RESPONSE_LIMIT = 1024; // [NOTE] about default mime.types file // If no mime.types file is specified in the mime option, s3fs // will look for /etc/mime.types on all operating systems and // load mime information. // However, in the case of macOS, when this file does not exist, // it tries to detect the /etc/apache2/mime.types file. // The reason for this is that apache2 is preinstalled on macOS, // and the mime.types file is expected to exist in this path. // If the mime.types file is not found, s3fs will exit with an // error. // static constexpr char DEFAULT_MIME_FILE[] = "/etc/mime.types"; static constexpr char SPECIAL_DARWIN_MIME_FILE[] = "/etc/apache2/mime.types"; // [NOTICE] // This symbol is for libcurl under 7.23.0 #ifndef CURLSHE_NOT_BUILT_IN #define CURLSHE_NOT_BUILT_IN 5 #endif #if LIBCURL_VERSION_NUM >= 0x073100 #define S3FS_CURLOPT_XFERINFOFUNCTION CURLOPT_XFERINFOFUNCTION #else #define S3FS_CURLOPT_XFERINFOFUNCTION CURLOPT_PROGRESSFUNCTION #endif // Wrappers to pass std::unique_ptr to raw pointer functions. Undefine curl_easy_setopt to work around curl variadic argument macro. #undef curl_easy_setopt template CURLcode curl_easy_setopt(const CurlUniquePtr& handle, CURLoption option, Arg arg) { return curl_easy_setopt(handle.get(), option, arg); } template CURLcode curl_easy_getinfo(const CurlUniquePtr& handle, CURLINFO info, Arg arg) { return curl_easy_getinfo(handle.get(), info, arg); } //------------------------------------------------------------------- // Class S3fsCurl //------------------------------------------------------------------- constexpr char S3fsCurl::S3FS_SSL_PRIVKEY_PASSWORD[]; std::mutex S3fsCurl::curl_handles_lock; S3fsCurl::callback_locks_t S3fsCurl::callback_locks; bool S3fsCurl::is_initglobal_done = false; std::unique_ptr S3fsCurl::sCurlPool; int S3fsCurl::sCurlPoolSize = 32; CURLSH* S3fsCurl::hCurlShare = nullptr; bool S3fsCurl::is_cert_check = true; // default bool S3fsCurl::is_dns_cache = true; // default bool S3fsCurl::is_ssl_session_cache= true; // default long S3fsCurl::connect_timeout = 300; // default time_t S3fsCurl::readwrite_timeout = 120; // default int S3fsCurl::retries = 5; // default bool S3fsCurl::is_public_bucket = false; acl_t S3fsCurl::default_acl = acl_t::PRIVATE; std::string S3fsCurl::storage_class = "STANDARD"; sseckeylist_t S3fsCurl::sseckeys; std::string S3fsCurl::ssekmsid; sse_type_t S3fsCurl::ssetype = sse_type_t::SSE_DISABLE; bool S3fsCurl::is_content_md5 = false; bool S3fsCurl::is_verbose = false; bool S3fsCurl::is_dump_body = false; S3fsCred* S3fsCurl::ps3fscred = nullptr; long S3fsCurl::ssl_verify_hostname = 1; // default(original code...) // SSL client cert options std::string S3fsCurl::client_cert; std::string S3fsCurl::client_cert_type; std::string S3fsCurl::client_priv_key; std::string S3fsCurl::client_priv_key_type; std::string S3fsCurl::client_key_password; std::atomic S3fsCurl::curl_warnings_once(false); // protected by curl_handles_lock std::map S3fsCurl::curl_progress; std::string S3fsCurl::curl_ca_bundle; mimes_t S3fsCurl::mimeTypes; std::string S3fsCurl::userAgent; int S3fsCurl::max_parallel_cnt = 5; // default int S3fsCurl::max_multireq = 20; // default off_t S3fsCurl::multipart_size = MULTIPART_SIZE; // default off_t S3fsCurl::multipart_copy_size = 512 * 1024 * 1024; // default signature_type_t S3fsCurl::signature_type = signature_type_t::V2_OR_V4; // default bool S3fsCurl::is_unsigned_payload = false; // default bool S3fsCurl::is_ua = true; // default bool S3fsCurl::listobjectsv2 = false; // default bool S3fsCurl::requester_pays = false; // default std::string S3fsCurl::proxy_url; bool S3fsCurl::proxy_http = false; std::string S3fsCurl::proxy_userpwd; long S3fsCurl::ipresolve_type = CURL_IPRESOLVE_WHATEVER; //------------------------------------------------------------------- // Class methods for S3fsCurl //------------------------------------------------------------------- bool S3fsCurl::InitS3fsCurl() { if(!S3fsCurl::InitGlobalCurl()){ return false; } if(!S3fsCurl::InitShareCurl()){ return false; } if(!S3fsCurl::InitCryptMutex()){ return false; } // [NOTE] // sCurlPoolSize must be over parallel(or multireq) count. // sCurlPoolSize = std::max({sCurlPoolSize, GetMaxParallelCount(), GetMaxMultiRequest()}); sCurlPool.reset(new CurlHandlerPool(sCurlPoolSize)); if (!sCurlPool->Init()) { return false; } return true; } bool S3fsCurl::DestroyS3fsCurl() { bool result = true; if(!S3fsCurl::DestroyCryptMutex()){ result = false; } if(!sCurlPool->Destroy()){ result = false; } sCurlPool.reset(); if(!S3fsCurl::DestroyShareCurl()){ result = false; } if(!S3fsCurl::DestroyGlobalCurl()){ result = false; } return result; } bool S3fsCurl::InitGlobalCurl() { if(S3fsCurl::is_initglobal_done){ return false; } if(CURLE_OK != curl_global_init(CURL_GLOBAL_ALL)){ S3FS_PRN_ERR("init_curl_global_all returns error."); return false; } S3fsCurl::is_initglobal_done = true; return true; } bool S3fsCurl::DestroyGlobalCurl() { if(!S3fsCurl::is_initglobal_done){ return false; } curl_global_cleanup(); S3fsCurl::is_initglobal_done = false; return true; } bool S3fsCurl::InitShareCurl() { CURLSHcode nSHCode; if(!S3fsCurl::is_dns_cache && !S3fsCurl::is_ssl_session_cache){ S3FS_PRN_INFO("Curl does not share DNS data."); return true; } if(S3fsCurl::hCurlShare){ S3FS_PRN_WARN("already initiated."); return false; } if(nullptr == (S3fsCurl::hCurlShare = curl_share_init())){ S3FS_PRN_ERR("curl_share_init failed"); return false; } if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_LOCKFUNC, S3fsCurl::LockCurlShare))){ S3FS_PRN_ERR("curl_share_setopt(LOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode)); return false; } if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_UNLOCKFUNC, S3fsCurl::UnlockCurlShare))){ S3FS_PRN_ERR("curl_share_setopt(UNLOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode)); return false; } if(S3fsCurl::is_dns_cache){ nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){ S3FS_PRN_ERR("curl_share_setopt(DNS) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode)); return false; }else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){ S3FS_PRN_WARN("curl_share_setopt(DNS) returns %d(%s), but continue without shared dns data.", nSHCode, curl_share_strerror(nSHCode)); } } if(S3fsCurl::is_ssl_session_cache){ nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){ S3FS_PRN_ERR("curl_share_setopt(SSL SESSION) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode)); return false; }else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){ S3FS_PRN_WARN("curl_share_setopt(SSL SESSION) returns %d(%s), but continue without shared ssl session data.", nSHCode, curl_share_strerror(nSHCode)); } } if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_USERDATA, &S3fsCurl::callback_locks))){ S3FS_PRN_ERR("curl_share_setopt(USERDATA) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode)); return false; } return true; } bool S3fsCurl::DestroyShareCurl() { if(!S3fsCurl::hCurlShare){ if(!S3fsCurl::is_dns_cache && !S3fsCurl::is_ssl_session_cache){ return true; } S3FS_PRN_WARN("already destroy share curl."); return false; } if(CURLSHE_OK != curl_share_cleanup(S3fsCurl::hCurlShare)){ return false; } S3fsCurl::hCurlShare = nullptr; return true; } void S3fsCurl::LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr) { if(!hCurlShare){ return; } auto* locks = static_cast(useptr); if(CURL_LOCK_DATA_DNS == nLockData){ locks->dns.lock(); }else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){ locks->ssl_session.lock(); } } void S3fsCurl::UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr) { if(!hCurlShare){ return; } auto* locks = static_cast(useptr); if(CURL_LOCK_DATA_DNS == nLockData){ locks->dns.unlock(); }else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){ locks->ssl_session.unlock(); } } bool S3fsCurl::InitCryptMutex() { return s3fs_init_crypt_mutex(); } bool S3fsCurl::DestroyCryptMutex() { return s3fs_destroy_crypt_mutex(); } // homegrown timeout mechanism int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) { CURL* curl = static_cast(clientp); time_t now = time(nullptr); const std::lock_guard lock(S3fsCurl::curl_handles_lock); // any progress? auto& value = S3fsCurl::curl_progress[curl]; if(value.dl_progress != dlnow || value.ul_progress != ulnow){ // yes! value = {now, dlnow, ulnow}; }else{ // timeout? if(now - value.time > readwrite_timeout){ S3FS_PRN_ERR("timeout now: %lld, curl_times[curl]: %lld, readwrite_timeout: %lld", static_cast(now), static_cast((value.time)), static_cast(readwrite_timeout)); return CURLE_ABORTED_BY_CALLBACK; } } return 0; } bool S3fsCurl::InitCredentialObject(S3fsCred* pcredobj) { // Set the only Credential object if(!pcredobj || S3fsCurl::ps3fscred){ S3FS_PRN_ERR("Unable to set the only Credential object."); return false; } S3fsCurl::ps3fscred = pcredobj; return true; } bool S3fsCurl::InitMimeType(const std::string& strFile) { std::string MimeFile; if(!strFile.empty()){ MimeFile = strFile; }else{ // search default mime.types std::string errPaths = DEFAULT_MIME_FILE; struct stat st; if(0 == stat(DEFAULT_MIME_FILE, &st)){ MimeFile = DEFAULT_MIME_FILE; }else if(compare_sysname("Darwin")){ // for macOS, search another default file. if(0 == stat(SPECIAL_DARWIN_MIME_FILE, &st)){ MimeFile = SPECIAL_DARWIN_MIME_FILE; }else{ errPaths += " and "; errPaths += SPECIAL_DARWIN_MIME_FILE; } } if(MimeFile.empty()){ S3FS_PRN_WARN("Could not find mime.types files, you have to create file(%s) or specify mime option for existing mime.types file.", errPaths.c_str()); return false; } } S3FS_PRN_DBG("Try to load mime types from %s file.", MimeFile.c_str()); std::ifstream MT(MimeFile.c_str()); if(MT.good()){ S3FS_PRN_DBG("The old mime types are cleared to load new mime types."); S3fsCurl::mimeTypes.clear(); std::string line; while(getline(MT, line)){ if(line.empty()){ continue; } if(line[0]=='#'){ continue; } std::istringstream tmp(line); std::string mimeType; tmp >> mimeType; std::string ext; while(tmp >> ext){ S3fsCurl::mimeTypes[ext] = mimeType; } } S3FS_PRN_INIT_INFO("Loaded mime information from %s", MimeFile.c_str()); }else{ S3FS_PRN_WARN("Could not load mime types from %s, please check the existence and permissions of this file.", MimeFile.c_str()); return false; } return true; } void S3fsCurl::InitUserAgent() { if(S3fsCurl::userAgent.empty()){ S3fsCurl::userAgent = "s3fs/"; S3fsCurl::userAgent += VERSION; S3fsCurl::userAgent += " (commit hash "; S3fsCurl::userAgent += COMMIT_HASH_VAL; S3fsCurl::userAgent += "; "; S3fsCurl::userAgent += s3fs_crypt_lib_name(); S3fsCurl::userAgent += ")"; S3fsCurl::userAgent += instance_name; } } // // @param s e.g., "index.html" // @return e.g., "text/html" // std::string S3fsCurl::LookupMimeType(const std::string& name) { if(!name.empty() && name[name.size() - 1] == '/'){ return "application/x-directory"; } std::string result("application/octet-stream"); std::string::size_type last_pos = name.find_last_of('.'); std::string::size_type first_pos = name.find_first_of('.'); std::string prefix, ext, ext2; // No dots in name, just return if(last_pos == std::string::npos){ return result; } // extract the last extension ext = name.substr(1+last_pos, std::string::npos); if (last_pos != std::string::npos) { // one dot was found, now look for another if (first_pos != std::string::npos && first_pos < last_pos) { prefix = name.substr(0, last_pos); // Now get the second to last file extension std::string::size_type next_pos = prefix.find_last_of('.'); if (next_pos != std::string::npos) { ext2 = prefix.substr(1+next_pos, std::string::npos); } } } // if we get here, then we have an extension (ext) auto iter = S3fsCurl::mimeTypes.find(ext); // if the last extension matches a mimeType, then return // that mime type if (iter != S3fsCurl::mimeTypes.cend()) { result = (*iter).second; return result; } // return with the default result if there isn't a second extension if(first_pos == last_pos){ return result; } // Didn't find a mime-type for the first extension // Look for second extension in mimeTypes, return if found iter = S3fsCurl::mimeTypes.find(ext2); if (iter != S3fsCurl::mimeTypes.cend()) { result = (*iter).second; return result; } // neither the last extension nor the second-to-last extension // matched a mimeType, return the default mime type return result; } bool S3fsCurl::LocateBundle() { // See if environment variable CURL_CA_BUNDLE is set // if so, check it, if it is a good path, then set the // curl_ca_bundle variable to it if(S3fsCurl::curl_ca_bundle.empty()){ char* CURL_CA_BUNDLE = getenv("CURL_CA_BUNDLE"); if(CURL_CA_BUNDLE != nullptr) { // check for existence and readability of the file std::ifstream BF(CURL_CA_BUNDLE); if(!BF.good()){ S3FS_PRN_ERR("%s: file specified by CURL_CA_BUNDLE environment variable is not readable", program_name.c_str()); return false; } BF.close(); S3fsCurl::curl_ca_bundle = CURL_CA_BUNDLE; return true; } }else{ // Already set ca bundle variable return true; } // not set via environment variable, look in likely locations /////////////////////////////////////////// // following comment from curl's (7.21.2) acinclude.m4 file /////////////////////////////////////////// // dnl CURL_CHECK_CA_BUNDLE // dnl ------------------------------------------------- // dnl Check if a default ca-bundle should be used // dnl // dnl regarding the paths this will scan: // dnl /etc/ssl/certs/ca-certificates.crt Debian systems // dnl /etc/pki/tls/certs/ca-bundle.crt Redhat and Mandriva // dnl /usr/share/ssl/certs/ca-bundle.crt old(er) Redhat // dnl /usr/local/share/certs/ca-root.crt FreeBSD // dnl /etc/ssl/cert.pem OpenBSD // dnl /etc/ssl/certs/ (ca path) SUSE /////////////////////////////////////////// // Within CURL the above path should have been checked // according to the OS. Thus, although we do not need // to check files here, we will only examine some files. // std::ifstream BF("/etc/pki/tls/certs/ca-bundle.crt"); if(BF.good()){ BF.close(); S3fsCurl::curl_ca_bundle = "/etc/pki/tls/certs/ca-bundle.crt"; }else{ BF.open("/etc/ssl/certs/ca-certificates.crt"); if(BF.good()){ BF.close(); S3fsCurl::curl_ca_bundle = "/etc/ssl/certs/ca-certificates.crt"; }else{ BF.open("/usr/share/ssl/certs/ca-bundle.crt"); if(BF.good()){ BF.close(); S3fsCurl::curl_ca_bundle = "/usr/share/ssl/certs/ca-bundle.crt"; }else{ BF.open("/usr/local/share/certs/ca-root.crt"); if(BF.good()){ BF.close(); S3fsCurl::curl_ca_bundle = "/usr/share/ssl/certs/ca-bundle.crt"; }else{ S3FS_PRN_ERR("%s: /.../ca-bundle.crt is not readable", program_name.c_str()); return false; } } } } return true; } size_t S3fsCurl::WriteMemoryCallback(void* ptr, size_t blockSize, size_t numBlocks, void* data) { auto* body = static_cast(data); body->append(static_cast(ptr), blockSize * numBlocks); return (blockSize * numBlocks); } size_t S3fsCurl::ReadCallback(void* ptr, size_t size, size_t nmemb, void* userp) { auto* pCurl = static_cast(userp); if(1 > (size * nmemb)){ return 0; } if(0 >= pCurl->postdata_remaining){ return 0; } size_t copysize = std::min(static_cast(size * nmemb), pCurl->postdata_remaining); memcpy(ptr, pCurl->postdata, copysize); pCurl->postdata_remaining = (pCurl->postdata_remaining > static_cast(copysize) ? (pCurl->postdata_remaining - copysize) : 0); pCurl->postdata += copysize; return copysize; } size_t S3fsCurl::HeaderCallback(void* data, size_t blockSize, size_t numBlocks, void* userPtr) { auto* headers = static_cast(userPtr); std::string header(static_cast(data), blockSize * numBlocks); std::string key; std::istringstream ss(header); if(getline(ss, key, ':')){ // Force to lower, only "x-amz" std::string lkey = key; transform(lkey.cbegin(), lkey.cend(), lkey.begin(), static_cast(std::tolower)); if(is_prefix(lkey.c_str(), "x-amz")){ key = lkey; } std::string value; getline(ss, value); (*headers)[key] = trim(value); } return blockSize * numBlocks; } size_t S3fsCurl::UploadReadCallback(void* ptr, size_t size, size_t nmemb, void* userp) { auto* pCurl = static_cast(userp); if(1 > (size * nmemb)){ return 0; } if(-1 == pCurl->partdata.fd || 0 >= pCurl->partdata.size){ return 0; } // read size ssize_t copysize = (size * nmemb) < (size_t)pCurl->partdata.size ? (size * nmemb) : (size_t)pCurl->partdata.size; ssize_t readbytes; ssize_t totalread; // read and set for(totalread = 0, readbytes = 0; totalread < copysize; totalread += readbytes){ readbytes = pread(pCurl->partdata.fd, &(static_cast(ptr))[totalread], (copysize - totalread), pCurl->partdata.startpos + totalread); if(0 == readbytes){ // eof break; }else if(-1 == readbytes){ // error S3FS_PRN_ERR("read file error(%d).", errno); return 0; } } pCurl->partdata.startpos += totalread; pCurl->partdata.size -= totalread; return totalread; } size_t S3fsCurl::DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, void* userp) { auto* pCurl = static_cast(userp); if(1 > (size * nmemb)){ return 0; } if(-1 == pCurl->partdata.fd || 0 >= pCurl->partdata.size){ return 0; } // Buffer initial bytes in case it is an XML error response. if(pCurl->bodydata.size() < GET_OBJECT_RESPONSE_LIMIT){ pCurl->bodydata.append(static_cast(ptr), std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size())); } // write size ssize_t copysize = (size * nmemb) < (size_t)pCurl->partdata.size ? (size * nmemb) : (size_t)pCurl->partdata.size; ssize_t writebytes; ssize_t totalwrite; // write for(totalwrite = 0, writebytes = 0; totalwrite < copysize; totalwrite += writebytes){ writebytes = pwrite(pCurl->partdata.fd, &(static_cast(ptr))[totalwrite], (copysize - totalwrite), pCurl->partdata.startpos + totalwrite); if(0 == writebytes){ // eof? break; }else if(-1 == writebytes){ // error S3FS_PRN_ERR("write file error(%d).", errno); return 0; } } pCurl->partdata.startpos += totalwrite; pCurl->partdata.size -= totalwrite; return totalwrite; } bool S3fsCurl::SetCheckCertificate(bool isCertCheck) { bool old = S3fsCurl::is_cert_check; S3fsCurl::is_cert_check = isCertCheck; return old; } bool S3fsCurl::SetDnsCache(bool isCache) { bool old = S3fsCurl::is_dns_cache; S3fsCurl::is_dns_cache = isCache; return old; } void S3fsCurl::ResetOffset(S3fsCurl* pCurl) { pCurl->partdata.startpos = pCurl->b_partdata_startpos; pCurl->partdata.size = pCurl->b_partdata_size; } bool S3fsCurl::SetSslSessionCache(bool isCache) { bool old = S3fsCurl::is_ssl_session_cache; S3fsCurl::is_ssl_session_cache = isCache; return old; } long S3fsCurl::SetConnectTimeout(long timeout) { long old = S3fsCurl::connect_timeout; S3fsCurl::connect_timeout = timeout; return old; } time_t S3fsCurl::SetReadwriteTimeout(time_t timeout) { time_t old = S3fsCurl::readwrite_timeout; S3fsCurl::readwrite_timeout = timeout; return old; } int S3fsCurl::SetRetries(int count) { int old = S3fsCurl::retries; S3fsCurl::retries = count; return old; } bool S3fsCurl::SetPublicBucket(bool flag) { bool old = S3fsCurl::is_public_bucket; S3fsCurl::is_public_bucket = flag; return old; } acl_t S3fsCurl::SetDefaultAcl(acl_t acl) { acl_t old = S3fsCurl::default_acl; S3fsCurl::default_acl = acl; return old; } acl_t S3fsCurl::GetDefaultAcl() { return S3fsCurl::default_acl; } std::string S3fsCurl::SetStorageClass(const std::string& storage_class) { std::string old = S3fsCurl::storage_class; S3fsCurl::storage_class = storage_class; // AWS requires uppercase storage class values transform(S3fsCurl::storage_class.cbegin(), S3fsCurl::storage_class.cend(), S3fsCurl::storage_class.begin(), ::toupper); return old; } bool S3fsCurl::PushbackSseKeys(const std::string& input) { std::string onekey = trim(input); if(onekey.empty()){ return false; } if('#' == onekey[0]){ return false; } // make base64 if the key is short enough, otherwise assume it is already so std::string base64_key; std::string raw_key; if(onekey.length() > 256 / 8){ std::string p_key(s3fs_decode64(onekey.c_str(), onekey.size())); raw_key = p_key; base64_key = onekey; } else { base64_key = s3fs_base64(reinterpret_cast(onekey.c_str()), onekey.length()); raw_key = onekey; } // make MD5 std::string strMd5; if(!make_md5_from_binary(raw_key.c_str(), raw_key.length(), strMd5)){ S3FS_PRN_ERR("Could not make MD5 from SSE-C keys(%s).", raw_key.c_str()); return false; } // mapped MD5 = SSE Key sseckeymap_t md5map; md5map.clear(); md5map[strMd5] = base64_key; S3fsCurl::sseckeys.push_back(md5map); return true; } sse_type_t S3fsCurl::SetSseType(sse_type_t type) { sse_type_t old = S3fsCurl::ssetype; S3fsCurl::ssetype = type; return old; } bool S3fsCurl::SetSseCKeys(const char* filepath) { if(!filepath){ S3FS_PRN_ERR("SSE-C keys filepath is empty."); return false; } struct stat st; if(0 != stat(filepath, &st)){ S3FS_PRN_ERR("could not open use_sse keys file(%s).", filepath); return false; } if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){ S3FS_PRN_ERR("use_sse keys file %s should be 0600 permissions.", filepath); return false; } S3fsCurl::sseckeys.clear(); std::ifstream ssefs(filepath); if(!ssefs.good()){ S3FS_PRN_ERR("Could not open SSE-C keys file(%s).", filepath); return false; } std::string line; while(getline(ssefs, line)){ S3fsCurl::PushbackSseKeys(line); } if(S3fsCurl::sseckeys.empty()){ S3FS_PRN_ERR("There is no SSE Key in file(%s).", filepath); return false; } return true; } bool S3fsCurl::SetSseKmsid(const char* kmsid) { if(!kmsid || '\0' == kmsid[0]){ S3FS_PRN_ERR("SSE-KMS kms id is empty."); return false; } S3fsCurl::ssekmsid = kmsid; return true; } // [NOTE] // Because SSE is set by some options and environment, // this function check the integrity of the SSE data finally. bool S3fsCurl::FinalCheckSse() { switch(S3fsCurl::ssetype){ case sse_type_t::SSE_DISABLE: S3fsCurl::ssekmsid.clear(); return true; case sse_type_t::SSE_S3: S3fsCurl::ssekmsid.clear(); return true; case sse_type_t::SSE_C: if(S3fsCurl::sseckeys.empty()){ S3FS_PRN_ERR("sse type is SSE-C, but there is no custom key."); return false; } S3fsCurl::ssekmsid.clear(); return true; case sse_type_t::SSE_KMS: if(S3fsCurl::ssekmsid.empty()){ S3FS_PRN_ERR("sse type is SSE-KMS, but there is no specified kms id."); return false; } if(S3fsCurl::GetSignatureType() == signature_type_t::V2_ONLY){ S3FS_PRN_ERR("sse type is SSE-KMS, but signature type is not v4. SSE-KMS require signature v4."); return false; } // SSL/TLS is required for KMS // if(!is_prefix(s3host.c_str(), "https://")){ S3FS_PRN_ERR("The sse type is SSE-KMS, but it is not configured to use SSL/TLS. SSE-KMS requires SSL/TLS communication."); return false; } return true; } S3FS_PRN_ERR("sse type is unknown(%d).", static_cast(S3fsCurl::ssetype)); return false; } bool S3fsCurl::LoadEnvSseCKeys() { char* envkeys = getenv("AWSSSECKEYS"); if(nullptr == envkeys){ // nothing to do return true; } S3fsCurl::sseckeys.clear(); std::istringstream fullkeys(envkeys); std::string onekey; while(getline(fullkeys, onekey, ':')){ S3fsCurl::PushbackSseKeys(onekey); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(S3fsCurl::sseckeys.empty()){ S3FS_PRN_ERR("There is no SSE Key in environment(AWSSSECKEYS=%s).", envkeys); return false; } return true; } bool S3fsCurl::LoadEnvSseKmsid() { const char* envkmsid = getenv("AWSSSEKMSID"); if(nullptr == envkmsid){ // nothing to do return true; } return S3fsCurl::SetSseKmsid(envkmsid); } // // If md5 is empty, returns first(current) sse key. // bool S3fsCurl::GetSseKey(std::string& md5, std::string& ssekey) { for(auto iter = S3fsCurl::sseckeys.cbegin(); iter != S3fsCurl::sseckeys.cend(); ++iter){ if(md5.empty() || md5 == (*iter).cbegin()->first){ md5 = iter->begin()->first; ssekey = iter->begin()->second; return true; } } return false; } bool S3fsCurl::GetSseKeyMd5(size_t pos, std::string& md5) { if(S3fsCurl::sseckeys.size() <= pos){ return false; } size_t cnt = 0; for(auto iter = S3fsCurl::sseckeys.cbegin(); iter != S3fsCurl::sseckeys.cend(); ++iter, ++cnt){ if(pos == cnt){ md5 = iter->begin()->first; return true; } } return false; } size_t S3fsCurl::GetSseKeyCount() { return S3fsCurl::sseckeys.size(); } bool S3fsCurl::SetContentMd5(bool flag) { bool old = S3fsCurl::is_content_md5; S3fsCurl::is_content_md5 = flag; return old; } bool S3fsCurl::SetVerbose(bool flag) { bool old = S3fsCurl::is_verbose; S3fsCurl::is_verbose = flag; return old; } bool S3fsCurl::SetDumpBody(bool flag) { bool old = S3fsCurl::is_dump_body; S3fsCurl::is_dump_body = flag; return old; } long S3fsCurl::SetSslVerifyHostname(long value) { if(0 != value && 1 != value){ return -1; } long old = S3fsCurl::ssl_verify_hostname; S3fsCurl::ssl_verify_hostname = value; return old; } bool S3fsCurl::SetSSLClientCertOptions(const std::string& values) { // Parse values: // = "::::" // if(values.empty()){ return false; } std::list valarr; std::string::size_type start_pos = 0; std::string::size_type pos; do{ if(std::string::npos == (pos = values.find(':', start_pos))){ valarr.push_back(values.substr(start_pos)); start_pos = pos; }else{ if(0 < (pos - start_pos)){ valarr.push_back(values.substr(start_pos, (pos - start_pos))); }else{ valarr.emplace_back(""); } start_pos = ++pos; } }while(std::string::npos != start_pos); // set client cert if(!valarr.empty() && !valarr.front().empty()){ S3fsCurl::client_cert = valarr.front(); valarr.pop_front(); // set client cert type if(!valarr.empty()){ S3fsCurl::client_cert_type = valarr.front(); // allow empty(default: PEM) valarr.pop_front(); // set client private key if(!valarr.empty()){ S3fsCurl::client_priv_key = valarr.front(); // allow empty valarr.pop_front(); // set client private key type if(!valarr.empty()){ S3fsCurl::client_priv_key_type = valarr.front(); // allow empty(default: PEM) valarr.pop_front(); // set key password if(!valarr.empty()){ S3fsCurl::client_key_password = valarr.front(); // allow empty } } } } } // [NOTE] // If the private key is set but the password is not set, // check the environment variables. // if(!S3fsCurl::client_priv_key.empty() && S3fsCurl::client_key_password.empty()){ const char* pass = std::getenv(S3fsCurl::S3FS_SSL_PRIVKEY_PASSWORD); if(pass != nullptr){ S3fsCurl::client_key_password = pass; } } return true; } bool S3fsCurl::SetMultipartSize(off_t size) { size = size * 1024 * 1024; if(size < MIN_MULTIPART_SIZE){ return false; } S3fsCurl::multipart_size = size; return true; } bool S3fsCurl::SetMultipartCopySize(off_t size) { size = size * 1024 * 1024; if(size < MIN_MULTIPART_SIZE){ return false; } S3fsCurl::multipart_copy_size = size; return true; } int S3fsCurl::SetMaxParallelCount(int value) { int old = S3fsCurl::max_parallel_cnt; S3fsCurl::max_parallel_cnt = value; return old; } int S3fsCurl::SetMaxMultiRequest(int max) { int old = S3fsCurl::max_multireq; S3fsCurl::max_multireq = max; return old; } // [NOTE] // This proxy setting is as same as the "--proxy" option of the curl command, // and equivalent to the "CURLOPT_PROXY" option of the curl_easy_setopt() // function. // However, currently s3fs does not provide another option to set the schema // and port, so you need to specify these it in this function. (Other than // this function, there is no means of specifying the schema and port.) // Therefore, it should be specified "url" as "[://][:]". // s3fs passes this string to curl_easy_setopt() function with "CURLOPT_PROXY". // If no "schema" is specified, "http" will be used as default, and if no port // is specified, "443" will be used for "HTTPS" and "1080" otherwise. // (See the description of "CURLOPT_PROXY" in libcurl document.) // bool S3fsCurl::SetProxy(const char* url) { if(!url || '\0' == url[0]){ return false; } std::string tmpurl = url; // check schema bool is_http = true; size_t pos = 0; if(std::string::npos != (pos = tmpurl.find("://", pos))){ if(0 == pos){ // no schema string before "://" return false; } pos += strlen("://"); // Check if it is other than "http://" if(0 != tmpurl.find("http://", 0)){ is_http = false; } }else{ // not have schema string pos = 0; } // check fqdn and port number string if(std::string::npos != (pos = tmpurl.find(':', pos))){ // specify port if(0 == pos){ // no fqdn(hostname) string before ":" return false; } pos += strlen(":"); if(std::string::npos != tmpurl.find(':', pos)){ // found wrong separator return false; } } S3fsCurl::proxy_url = tmpurl; S3fsCurl::proxy_http = is_http; return true; } // [NOTE] // This function loads proxy credentials(username and passphrase) // from a file. // The loaded values is set to "CURLOPT_PROXYUSERPWD" in the // curl_easy_setopt() function. (But only used if the proxy is HTTP // schema.) // // The file is expected to contain only one valid line: // ------------------------ // # comment line // : // ------------------------ // Lines starting with a '#' character are treated as comments. // Lines with only space characters and blank lines are ignored. // If the user name contains spaces, it must be url encoded(ex. %20). // bool S3fsCurl::SetProxyUserPwd(const char* file) { if(!file || '\0' == file[0]){ return false; } if(!S3fsCurl::proxy_userpwd.empty()){ S3FS_PRN_WARN("Already set username and passphrase for proxy."); return false; } std::ifstream credFileStream(file); if(!credFileStream.good()){ S3FS_PRN_WARN("Could not load username and passphrase for proxy from %s.", file); return false; } std::string userpwd; std::string line; while(getline(credFileStream, line)){ line = trim(line); if(line.empty()){ continue; } if(line[0]=='#'){ continue; } if(!userpwd.empty()){ S3FS_PRN_WARN("Multiple valid username and passphrase found in %s file. Should specify only one pair.", file); return false; } // check separator for username and passphrase size_t pos = 0; if(std::string::npos == (pos = line.find(':', pos))){ S3FS_PRN_WARN("Found string for username and passphrase in %s file does not have separator ':'.", file); return false; } if(0 == pos || (pos + 1) == line.length()){ S3FS_PRN_WARN("Found string for username or passphrase in %s file is empty.", file); return false; } if(std::string::npos != line.find(':', ++pos)){ S3FS_PRN_WARN("Found string for username and passphrase in %s file has multiple separator ':'.", file); return false; } userpwd = line; } if(userpwd.empty()){ S3FS_PRN_WARN("No valid username and passphrase found in %s.", file); return false; } S3fsCurl::proxy_userpwd = userpwd; return true; } bool S3fsCurl::SetIPResolveType(const char* value) { if(!value){ return false; } if(0 == strcasecmp(value, "ipv4")){ S3fsCurl::ipresolve_type = CURL_IPRESOLVE_V4; }else if(0 == strcasecmp(value, "ipv6")){ S3fsCurl::ipresolve_type = CURL_IPRESOLVE_V6; }else if(0 == strcasecmp(value, "whatever")){ // = default type S3fsCurl::ipresolve_type = CURL_IPRESOLVE_WHATEVER; }else{ return false; } return true; } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress constParameter // cppcheck-suppress constParameterCallback bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl, void* param) { if(!s3fscurl || param){ // this callback does not need a parameter return false; } return s3fscurl->UploadMultipartPostComplete(); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress constParameter // cppcheck-suppress constParameterCallback bool S3fsCurl::MixMultipartPostCallback(S3fsCurl* s3fscurl, void* param) { if(!s3fscurl || param){ // this callback does not need a parameter return false; } return s3fscurl->MixMultipartPostComplete(); } std::unique_ptr S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl) { if(!s3fscurl){ return nullptr; } // parse and get part_num, upload_id. std::string upload_id; std::string part_num_str; int part_num; off_t tmp_part_num = 0; if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){ return nullptr; } upload_id = urlDecode(upload_id); // decode if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){ return nullptr; } if(!s3fs_strtoofft(&tmp_part_num, part_num_str.c_str(), /*base=*/ 10)){ return nullptr; } part_num = static_cast(tmp_part_num); if(s3fscurl->retry_count >= S3fsCurl::retries){ S3FS_PRN_ERR("Over retry count(%d) limit(%s:%d).", s3fscurl->retry_count, s3fscurl->path.c_str(), part_num); return nullptr; } // duplicate request std::unique_ptr newcurl(new S3fsCurl(s3fscurl->IsUseAhbe())); newcurl->partdata.petag = s3fscurl->partdata.petag; newcurl->partdata.fd = s3fscurl->partdata.fd; newcurl->partdata.startpos = s3fscurl->b_partdata_startpos; newcurl->partdata.size = s3fscurl->b_partdata_size; newcurl->b_partdata_startpos = s3fscurl->b_partdata_startpos; newcurl->b_partdata_size = s3fscurl->b_partdata_size; newcurl->retry_count = s3fscurl->retry_count + 1; newcurl->op = s3fscurl->op; newcurl->type = s3fscurl->type; // setup new curl object if(0 != newcurl->UploadMultipartPostSetup(s3fscurl->path.c_str(), part_num, upload_id)){ S3FS_PRN_ERR("Could not duplicate curl object(%s:%d).", s3fscurl->path.c_str(), part_num); return nullptr; } return newcurl; } std::unique_ptr S3fsCurl::CopyMultipartPostRetryCallback(S3fsCurl* s3fscurl) { if(!s3fscurl){ return nullptr; } // parse and get part_num, upload_id. std::string upload_id; std::string part_num_str; int part_num; off_t tmp_part_num = 0; if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){ return nullptr; } upload_id = urlDecode(upload_id); // decode if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){ return nullptr; } if(!s3fs_strtoofft(&tmp_part_num, part_num_str.c_str(), /*base=*/ 10)){ return nullptr; } part_num = static_cast(tmp_part_num); if(s3fscurl->retry_count >= S3fsCurl::retries){ S3FS_PRN_ERR("Over retry count(%d) limit(%s:%d).", s3fscurl->retry_count, s3fscurl->path.c_str(), part_num); return nullptr; } // duplicate request std::unique_ptr newcurl(new S3fsCurl(s3fscurl->IsUseAhbe())); newcurl->partdata.petag = s3fscurl->partdata.petag; newcurl->b_from = s3fscurl->b_from; newcurl->b_meta = s3fscurl->b_meta; newcurl->retry_count = s3fscurl->retry_count + 1; newcurl->op = s3fscurl->op; newcurl->type = s3fscurl->type; // setup new curl object if(0 != newcurl->CopyMultipartPostSetup(s3fscurl->b_from.c_str(), s3fscurl->path.c_str(), part_num, upload_id, s3fscurl->b_meta)){ S3FS_PRN_ERR("Could not duplicate curl object(%s:%d).", s3fscurl->path.c_str(), part_num); return nullptr; } return newcurl; } std::unique_ptr S3fsCurl::MixMultipartPostRetryCallback(S3fsCurl* s3fscurl) { if(!s3fscurl){ return nullptr; } if(-1 == s3fscurl->partdata.fd){ return S3fsCurl::CopyMultipartPostRetryCallback(s3fscurl); }else{ return S3fsCurl::UploadMultipartPostRetryCallback(s3fscurl); } } int S3fsCurl::MapPutErrorResponse(int result) { if(result != 0){ return result; } // PUT returns 200 status code with something error, thus // we need to check body. // // example error body: // // // AccessDenied // Access Denied // E4CA6F6767D6685C // BHzLOATeDuvN8Es1wI8IcERq4kl4dc2A9tOB8Yqr39Ys6fl7N4EJ8sjGiVvu6wLP // // const char* pstrbody = bodydata.c_str(); std::string code; if(simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){ S3FS_PRN_ERR("Put request get 200 status response, but it included error body(or nullptr). The request failed during copying the object in S3. Code: %s", code.c_str()); // TODO: parse more specific error from result = -EIO; } return result; } // [NOTE] // It is a factory method as utility because it requires an S3fsCurl object // initialized for multipart upload from outside this class. // std::unique_ptr S3fsCurl::CreateParallelS3fsCurl(const char* tpath, int fd, off_t start, off_t size, int part_num, bool is_copy, etagpair* petag, const std::string& upload_id, int& result) { // duplicate fd if(!tpath || -1 == fd || start < 0 || size <= 0 || !petag){ S3FS_PRN_ERR("Parameters are wrong: tpath(%s), fd(%d), start(%lld), size(%lld), petag(%s)", SAFESTRPTR(tpath), fd, static_cast(start), static_cast(size), (petag ? "not null" : "null")); result = -EIO; return nullptr; } result = 0; std::unique_ptr s3fscurl(new S3fsCurl(true)); if(!is_copy){ s3fscurl->partdata.fd = fd; s3fscurl->partdata.startpos = start; s3fscurl->partdata.size = size; s3fscurl->partdata.is_copy = is_copy; s3fscurl->partdata.petag = petag; // [NOTE] be careful, the value is set directly s3fscurl->b_partdata_startpos = s3fscurl->partdata.startpos; s3fscurl->b_partdata_size = s3fscurl->partdata.size; S3FS_PRN_INFO3("Upload Part [tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast(start), static_cast(size), part_num); if(0 != (result = s3fscurl->UploadMultipartPostSetup(tpath, part_num, upload_id))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return nullptr; } }else{ headers_t meta; std::string srcresource; std::string srcurl; MakeUrlResource(get_realpath(tpath).c_str(), srcresource, srcurl); meta["x-amz-copy-source"] = srcresource; std::ostringstream strrange; strrange << "bytes=" << start << "-" << (start + size - 1); meta["x-amz-copy-source-range"] = strrange.str(); s3fscurl->b_from = SAFESTRPTR(tpath); s3fscurl->b_meta = meta; s3fscurl->partdata.petag = petag; // [NOTE] be careful, the value is set directly S3FS_PRN_INFO3("Copy Part [tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast(start), static_cast(size), part_num); if(0 != (result = s3fscurl->CopyMultipartPostSetup(tpath, tpath, part_num, upload_id, meta))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return nullptr; } } // Call lazy function if(!s3fscurl->fpLazySetup || !s3fscurl->fpLazySetup(s3fscurl.get())){ S3FS_PRN_ERR("failed lazy function setup for uploading part"); result = -EIO; return nullptr; } return s3fscurl; } int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd) { int result; std::string upload_id; struct stat st; etaglist_t list; off_t remaining_bytes; S3fsCurl s3fscurl(true); S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd); if(-1 == fstat(fd, &st)){ S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno); return -errno; } if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id))){ return result; } s3fscurl.DestroyCurlHandle(); // Initialize S3fsMultiCurl S3fsMultiCurl curlmulti(GetMaxParallelCount()); curlmulti.SetSuccessCallback(S3fsCurl::UploadMultipartPostCallback); curlmulti.SetRetryCallback(S3fsCurl::UploadMultipartPostRetryCallback); // cycle through open fd, pulling off 10MB chunks at a time for(remaining_bytes = st.st_size; 0 < remaining_bytes; ){ off_t chunk = remaining_bytes > S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes; // s3fscurl sub object std::unique_ptr s3fscurl_para(new S3fsCurl(true)); s3fscurl_para->partdata.fd = fd; s3fscurl_para->partdata.startpos = st.st_size - remaining_bytes; s3fscurl_para->partdata.size = chunk; s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos; s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size; s3fscurl_para->partdata.add_etag_list(list); // initiate upload part for parallel if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, s3fscurl_para->partdata.get_part_number(), upload_id))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return result; } // set into parallel object if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl_para))){ S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath); return -EIO; } remaining_bytes -= chunk; } // Multi request if(0 != (result = curlmulti.Request())){ S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result); S3fsCurl s3fscurl_abort(true); int result2 = s3fscurl_abort.AbortMultipartUpload(tpath, upload_id); s3fscurl_abort.DestroyCurlHandle(); if(result2 != 0){ S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2); } return result; } if(0 != (result = s3fscurl.CompleteMultipartPostRequest(tpath, upload_id, list))){ return result; } return 0; } int S3fsCurl::ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const fdpage_list_t& mixuppages) { int result; std::string upload_id; struct stat st; etaglist_t list; S3fsCurl s3fscurl(true); S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd); if(-1 == fstat(fd, &st)){ S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno); return -errno; } if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id))){ return result; } s3fscurl.DestroyCurlHandle(); // for copy multipart std::string srcresource; std::string srcurl; MakeUrlResource(get_realpath(tpath).c_str(), srcresource, srcurl); meta["Content-Type"] = S3fsCurl::LookupMimeType(tpath); meta["x-amz-copy-source"] = srcresource; // Initialize S3fsMultiCurl S3fsMultiCurl curlmulti(GetMaxParallelCount()); curlmulti.SetSuccessCallback(S3fsCurl::MixMultipartPostCallback); curlmulti.SetRetryCallback(S3fsCurl::MixMultipartPostRetryCallback); for(auto iter = mixuppages.cbegin(); iter != mixuppages.cend(); ++iter){ if(iter->modified){ // Multipart upload std::unique_ptr s3fscurl_para(new S3fsCurl(true)); s3fscurl_para->partdata.fd = fd; s3fscurl_para->partdata.startpos = iter->offset; s3fscurl_para->partdata.size = iter->bytes; s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos; s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size; s3fscurl_para->partdata.add_etag_list(list); S3FS_PRN_INFO3("Upload Part [tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast(iter->offset), static_cast(iter->bytes), s3fscurl_para->partdata.get_part_number()); // initiate upload part for parallel if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, s3fscurl_para->partdata.get_part_number(), upload_id))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return result; } // set into parallel object if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl_para))){ S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath); return -EIO; } }else{ // Multipart copy for(off_t i = 0, bytes = 0; i < iter->bytes; i += bytes){ std::unique_ptr s3fscurl_para(new S3fsCurl(true)); bytes = std::min(GetMultipartCopySize(), iter->bytes - i); /* every part should be larger than MIN_MULTIPART_SIZE and smaller than FIVE_GB */ off_t remain_bytes = iter->bytes - i - bytes; if ((MIN_MULTIPART_SIZE > remain_bytes) && (0 < remain_bytes)){ if(FIVE_GB < (bytes + remain_bytes)){ bytes = (bytes + remain_bytes)/2; } else{ bytes += remain_bytes; } } std::ostringstream strrange; strrange << "bytes=" << (iter->offset + i) << "-" << (iter->offset + i + bytes - 1); meta["x-amz-copy-source-range"] = strrange.str(); s3fscurl_para->b_from = SAFESTRPTR(tpath); s3fscurl_para->b_meta = meta; s3fscurl_para->partdata.add_etag_list(list); S3FS_PRN_INFO3("Copy Part [tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast(iter->offset + i), static_cast(bytes), s3fscurl_para->partdata.get_part_number()); // initiate upload part for parallel if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, s3fscurl_para->partdata.get_part_number(), upload_id, meta))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return result; } // set into parallel object if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl_para))){ S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath); return -EIO; } } } } // Multi request if(0 != (result = curlmulti.Request())){ S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result); S3fsCurl s3fscurl_abort(true); int result2 = s3fscurl_abort.AbortMultipartUpload(tpath, upload_id); s3fscurl_abort.DestroyCurlHandle(); if(result2 != 0){ S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2); } return result; } if(0 != (result = s3fscurl.CompleteMultipartPostRequest(tpath, upload_id, list))){ return result; } return 0; } std::unique_ptr S3fsCurl::ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl) { int result; if(!s3fscurl){ return nullptr; } if(s3fscurl->retry_count >= S3fsCurl::retries){ S3FS_PRN_ERR("Over retry count(%d) limit(%s).", s3fscurl->retry_count, s3fscurl->path.c_str()); return nullptr; } // duplicate request(setup new curl object) std::unique_ptr newcurl(new S3fsCurl(s3fscurl->IsUseAhbe())); if(0 != (result = newcurl->PreGetObjectRequest(s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size, s3fscurl->b_ssetype, s3fscurl->b_ssevalue))){ S3FS_PRN_ERR("failed downloading part setup(%d)", result); return nullptr; } newcurl->retry_count = s3fscurl->retry_count + 1; return newcurl; } int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, off_t size) { S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd); sse_type_t ssetype = sse_type_t::SSE_DISABLE; std::string ssevalue; if(!get_object_sse_type(tpath, ssetype, ssevalue)){ S3FS_PRN_WARN("Failed to get SSE type for file(%s).", SAFESTRPTR(tpath)); } int result = 0; off_t remaining_bytes; // cycle through open fd, pulling off 10MB chunks at a time for(remaining_bytes = size; 0 < remaining_bytes; ){ S3fsMultiCurl curlmulti(GetMaxParallelCount()); int para_cnt; off_t chunk; // Initialize S3fsMultiCurl //curlmulti.SetSuccessCallback(nullptr); // not need to set success callback curlmulti.SetRetryCallback(S3fsCurl::ParallelGetObjectRetryCallback); // Loop for setup parallel upload(multipart) request. for(para_cnt = 0; para_cnt < S3fsCurl::max_parallel_cnt && 0 < remaining_bytes; para_cnt++, remaining_bytes -= chunk){ // chunk size chunk = remaining_bytes > S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes; // s3fscurl sub object std::unique_ptr s3fscurl_para(new S3fsCurl(true)); if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk, ssetype, ssevalue))){ S3FS_PRN_ERR("failed downloading part setup(%d)", result); return result; } // set into parallel object if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl_para))){ S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath); return -EIO; } } // Multi request if(0 != (result = curlmulti.Request())){ S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result); break; } // reinit for loop. curlmulti.Clear(); } return result; } bool S3fsCurl::UploadMultipartPostSetCurlOpts(S3fsCurl* s3fscurl) { if(!s3fscurl){ return false; } if(!s3fscurl->CreateCurlHandle()){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_UPLOAD, true)){ // HTTP PUT return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&s3fscurl->bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, reinterpret_cast(&s3fscurl->responseHeaders))){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE_LARGE, static_cast(s3fscurl->partdata.size))){ // Content-Length return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READFUNCTION, UploadReadCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READDATA, reinterpret_cast(s3fscurl))){ return false; } if(!S3fsCurl::AddUserAgent(s3fscurl->hCurl)){ // put User-Agent return false; } return true; } bool S3fsCurl::CopyMultipartPostSetCurlOpts(S3fsCurl* s3fscurl) { if(!s3fscurl){ return false; } if(!s3fscurl->CreateCurlHandle()){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_UPLOAD, true)){ // HTTP PUT return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&s3fscurl->bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, reinterpret_cast(&s3fscurl->headdata))){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE, 0)){ // Content-Length return false; } if(!S3fsCurl::AddUserAgent(s3fscurl->hCurl)){ // put User-Agent return false; } return true; } bool S3fsCurl::PreGetObjectRequestSetCurlOpts(S3fsCurl* s3fscurl) { if(!s3fscurl){ return false; } if(!s3fscurl->CreateCurlHandle()){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, DownloadWriteCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, reinterpret_cast(s3fscurl))){ return false; } if(!S3fsCurl::AddUserAgent(s3fscurl->hCurl)){ // put User-Agent return false; } return true; } bool S3fsCurl::PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl) { if(!s3fscurl){ return false; } if(!s3fscurl->CreateCurlHandle()){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_NOBODY, true)){ // HEAD return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_FILETIME, true)){ // Last-Modified return false; } // responseHeaders if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, reinterpret_cast(&s3fscurl->responseHeaders))){ return false; } if(CURLE_OK != curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback)){ return false; } if(!S3fsCurl::AddUserAgent(s3fscurl->hCurl)){ // put User-Agent return false; } return true; } bool S3fsCurl::AddUserAgent(const CurlUniquePtr& hCurl) { if(!hCurl){ return false; } if(S3fsCurl::IsUserAgentFlag()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_USERAGENT, S3fsCurl::userAgent.c_str())){ return false; } } return true; } int S3fsCurl::CurlDebugFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr) { return S3fsCurl::RawCurlDebugFunc(hcurl, type, data, size, userptr, CURLINFO_END); } int S3fsCurl::CurlDebugBodyInFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr) { return S3fsCurl::RawCurlDebugFunc(hcurl, type, data, size, userptr, CURLINFO_DATA_IN); } int S3fsCurl::CurlDebugBodyOutFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr) { return S3fsCurl::RawCurlDebugFunc(hcurl, type, data, size, userptr, CURLINFO_DATA_OUT); } int S3fsCurl::RawCurlDebugFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr, curl_infotype datatype) { if(!hcurl){ // something wrong... return 0; } switch(type){ case CURLINFO_TEXT: // Swap tab indentation with spaces so it stays pretty in syslog int indent; indent = 0; while (*data == '\t' && size > 0) { indent += 4; size--; data++; } if(foreground && 0 < size && '\n' == data[size - 1]){ size--; } S3FS_PRN_CURL("* %*s%.*s", indent, "", (int)size, data); break; case CURLINFO_DATA_IN: case CURLINFO_DATA_OUT: if(type != datatype || !S3fsCurl::is_dump_body){ // not put break; } case CURLINFO_HEADER_IN: case CURLINFO_HEADER_OUT: size_t remaining; char* p; // Print each line individually for tidy output remaining = size; p = data; do { char* eol = reinterpret_cast(memchr(p, '\n', remaining)); int newline = 0; if (eol == nullptr) { eol = reinterpret_cast(memchr(p, '\r', remaining)); if (eol == nullptr) { // No newlines, just emit entire line. eol = p + remaining; } } else { if (eol > p && *(eol - 1) == '\r') { newline++; } newline++; eol++; } size_t length = eol - p; S3FS_PRN_CURL("%s %.*s", getCurlDebugHead(type), (int)length - newline, p); remaining -= length; p = eol; } while (p != nullptr && remaining > 0); break; case CURLINFO_SSL_DATA_IN: case CURLINFO_SSL_DATA_OUT: // not put break; default: // why break; } return 0; } //------------------------------------------------------------------- // Methods for S3fsCurl //------------------------------------------------------------------- S3fsCurl::S3fsCurl(bool ahbe) : type(REQTYPE::UNSET), requestHeaders(nullptr), LastResponseCode(S3FSCURL_RESPONSECODE_NOTSET), postdata(nullptr), postdata_remaining(0), is_use_ahbe(ahbe), retry_count(0), b_postdata(nullptr), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0), b_ssekey_pos(-1), b_ssetype(sse_type_t::SSE_DISABLE), sem(nullptr), completed_tids_lock(nullptr), completed_tids(nullptr), fpLazySetup(nullptr), curlCode(CURLE_OK) { if(!S3fsCurl::ps3fscred){ S3FS_PRN_CRIT("The object of S3fs Credential class is not initialized."); abort(); } } S3fsCurl::~S3fsCurl() { DestroyCurlHandle(); } bool S3fsCurl::ResetHandle() { bool run_once = curl_warnings_once.exchange(true); sCurlPool->ResetHandler(hCurl.get()); if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_NOSIGNAL, 1)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_FOLLOWLOCATION, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_CONNECTTIMEOUT, S3fsCurl::connect_timeout)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_NOPROGRESS, 0)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_XFERINFOFUNCTION, S3fsCurl::CurlProgress)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl.get())){ return false; } // curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1); if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_TCP_KEEPALIVE, 1) && !run_once){ S3FS_PRN_WARN("The CURLOPT_TCP_KEEPALIVE option could not be set. For maximize performance you need to enable this option and you should use libcurl 7.25.0 or later."); } if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_SSL_ENABLE_ALPN, 0) && !run_once){ S3FS_PRN_WARN("The CURLOPT_SSL_ENABLE_ALPN option could not be unset. S3 server does not support ALPN, then this option should be disabled to maximize performance. you need to use libcurl 7.36.0 or later."); } if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_KEEP_SENDING_ON_ERROR, 1) && !run_once){ S3FS_PRN_WARN("The S3FS_CURLOPT_KEEP_SENDING_ON_ERROR option could not be set. For maximize performance you need to enable this option and you should use libcurl 7.51.0 or later."); } if(CURL_IPRESOLVE_WHATEVER != S3fsCurl::ipresolve_type){ // CURL_IPRESOLVE_WHATEVER is default, so not need to set. if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_IPRESOLVE, S3fsCurl::ipresolve_type)){ return false; } } if(type != REQTYPE::IAMCRED && type != REQTYPE::IAMROLE){ // REQTYPE::IAMCRED and REQTYPE::IAMROLE are always HTTP if(0 == S3fsCurl::ssl_verify_hostname){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0)){ return false; } } if(!S3fsCurl::curl_ca_bundle.empty()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_CAINFO, S3fsCurl::curl_ca_bundle.c_str())){ return false; } } } // SSL Client Cert if(!S3fsCurl::client_cert.empty()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLCERT, S3fsCurl::client_cert.c_str())){ return false; } if(!S3fsCurl::client_cert_type.empty() && 0 != strcasecmp(S3fsCurl::client_cert_type.c_str(), "PEM")){ // "PEM" is default if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLCERTTYPE, S3fsCurl::client_cert_type.c_str())){ return false; } } // Private key if(!S3fsCurl::client_priv_key.empty()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLKEY, S3fsCurl::client_priv_key.c_str())){ return false; } if(!S3fsCurl::client_priv_key_type.empty() && 0 != strcasecmp(S3fsCurl::client_priv_key_type.c_str(), "PEM")){ // "PEM" is default if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLKEYTYPE, S3fsCurl::client_priv_key_type.c_str())){ return false; } } // Password if(!S3fsCurl::client_key_password.empty()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_KEYPASSWD, S3fsCurl::client_key_password.c_str())){ return false; } } } } if((S3fsCurl::is_dns_cache || S3fsCurl::is_ssl_session_cache) && S3fsCurl::hCurlShare){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SHARE, S3fsCurl::hCurlShare)){ return false; } } if(!S3fsCurl::is_cert_check) { S3FS_PRN_DBG("'no_check_certificate' option in effect."); S3FS_PRN_DBG("The server certificate won't be checked against the available certificate authorities."); if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYPEER, false)){ return false; } } if(S3fsCurl::is_verbose){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_VERBOSE, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_DEBUGFUNCTION, S3fsCurl::CurlDebugFunc)){ return false; } } if(!cipher_suites.empty()) { if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSL_CIPHER_LIST, cipher_suites.c_str())){ return false; } } if(!S3fsCurl::proxy_url.empty()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXY, S3fsCurl::proxy_url.c_str())){ return false; } if(S3fsCurl::proxy_http){ if(!S3fsCurl::proxy_userpwd.empty()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXYUSERPWD, S3fsCurl::proxy_userpwd.c_str())){ return false; } } }else if(!S3fsCurl::proxy_userpwd.empty()){ S3FS_PRN_DBG("Username and passphrase are specified even though proxy is not 'http' scheme, so skip to set those."); } } S3fsCurl::curl_progress[hCurl.get()] = {time(nullptr), -1, -1}; return true; } bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake) { const std::lock_guard lock(S3fsCurl::curl_handles_lock); if(hCurl && remake){ if(!DestroyCurlHandleHasLock(false, true)){ S3FS_PRN_ERR("could not destroy handle."); return false; } S3FS_PRN_INFO3("already has handle, so destroyed it or restored it to pool."); } if(!hCurl){ if(nullptr == (hCurl = sCurlPool->GetHandler(only_pool))){ if(!only_pool){ S3FS_PRN_ERR("Failed to create handle."); return false; }else{ // [NOTE] // Further initialization processing is left to lazy processing to be executed later. // (Currently we do not use only_pool=true, but this code is remained for the future) return true; } } } ResetHandle(); return true; } bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data) { const std::lock_guard lock(S3fsCurl::curl_handles_lock); return DestroyCurlHandleHasLock(restore_pool, clear_internal_data); } bool S3fsCurl::DestroyCurlHandleHasLock(bool restore_pool, bool clear_internal_data) { // [NOTE] // If type is REQTYPE::IAMCRED or REQTYPE::IAMROLE, do not clear type. // Because that type only uses HTTP protocol, then the special // logic in ResetHandle function. // if(type != REQTYPE::IAMCRED && type != REQTYPE::IAMROLE){ type = REQTYPE::UNSET; } if(clear_internal_data){ ClearInternalData(); } if(hCurl){ S3fsCurl::curl_progress.erase(hCurl.get()); sCurlPool->ReturnHandler(std::move(hCurl), restore_pool); }else{ return false; } return true; } bool S3fsCurl::ClearInternalData() { // Always clear internal data // type = REQTYPE::UNSET; path = ""; base_path = ""; saved_path = ""; url = ""; op = ""; query_string= ""; if(requestHeaders){ curl_slist_free_all(requestHeaders); requestHeaders = nullptr; } responseHeaders.clear(); bodydata.clear(); headdata.clear(); LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; postdata = nullptr; postdata_remaining = 0; retry_count = 0; b_infile.reset(); b_postdata = nullptr; b_postdata_remaining = 0; b_partdata_startpos = 0; b_partdata_size = 0; partdata.clear(); fpLazySetup = nullptr; S3FS_MALLOCTRIM(0); return true; } bool S3fsCurl::SetUseAhbe(bool ahbe) { bool old = is_use_ahbe; is_use_ahbe = ahbe; return old; } bool S3fsCurl::GetResponseCode(long& responseCode, bool from_curl_handle) const { responseCode = -1; if(!from_curl_handle){ responseCode = LastResponseCode; }else{ if(!hCurl){ return false; } if(CURLE_OK != curl_easy_getinfo(hCurl, CURLINFO_RESPONSE_CODE, &LastResponseCode)){ return false; } responseCode = LastResponseCode; } return true; } // // Reset all options for retrying // bool S3fsCurl::RemakeHandle() { S3FS_PRN_INFO3("Retry request. [type=%d][url=%s][path=%s]", static_cast(type), url.c_str(), path.c_str()); if(REQTYPE::UNSET == type){ return false; } // rewind file struct stat st; if(b_infile){ if(-1 == fseek(b_infile.get(), 0, SEEK_SET)){ S3FS_PRN_WARN("Could not reset position(fd=%d)", fileno(b_infile.get())); return false; } if(-1 == fstat(fileno(b_infile.get()), &st)){ S3FS_PRN_WARN("Could not get file stat(fd=%d)", fileno(b_infile.get())); return false; } } // reinitialize internal data requestHeaders = curl_slist_remove(requestHeaders, "Authorization"); responseHeaders.clear(); bodydata.clear(); headdata.clear(); LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; // count up(only use for multipart) retry_count++; // set from backup postdata = b_postdata; postdata_remaining = b_postdata_remaining; partdata.startpos = b_partdata_startpos; partdata.size = b_partdata_size; // reset handle { const std::lock_guard lock(S3fsCurl::curl_handles_lock); ResetHandle(); } // set options switch(type){ case REQTYPE::DELETE: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE")){ return false; } break; case REQTYPE::HEAD: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_NOBODY, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_FILETIME, true)){ return false; } // responseHeaders if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, reinterpret_cast(&responseHeaders))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback)){ return false; } break; case REQTYPE::PUTHEAD: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ return false; } break; case REQTYPE::PUT: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(b_infile){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast(st.st_size))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILE, b_infile.get())){ return false; } }else{ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ return false; } } break; case REQTYPE::GET: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, S3fsCurl::DownloadWriteCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(this))){ return false; } break; case REQTYPE::CHKBUCKET: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } break; case REQTYPE::LISTBUCKET: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } break; case REQTYPE::PREMULTIPOST: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0)){ return false; } break; case REQTYPE::COMPLETEMULTIPOST: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast(postdata_remaining))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, reinterpret_cast(this))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback)){ return false; } break; case REQTYPE::UPLOADMULTIPOST: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, reinterpret_cast(&responseHeaders))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast(partdata.size))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::UploadReadCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, reinterpret_cast(this))){ return false; } break; case REQTYPE::COPYMULTIPOST: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, reinterpret_cast(&headdata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ return false; } break; case REQTYPE::MULTILIST: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } break; case REQTYPE::IAMCRED: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast(postdata_remaining))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, reinterpret_cast(this))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback)){ return false; } } break; case REQTYPE::ABORTMULTIUPLOAD: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE")){ return false; } break; case REQTYPE::IAMROLE: if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } break; default: S3FS_PRN_ERR("request type is unknown(%d)", static_cast(type)); return false; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return false; } return true; } // // returns curl return code // int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/) { if(S3fsLog::IsS3fsLogDbg()){ char* ptr_url = nullptr; curl_easy_getinfo(hCurl, CURLINFO_EFFECTIVE_URL, &ptr_url); S3FS_PRN_DBG("connecting to URL %s", SAFESTRPTR(ptr_url)); } LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; long responseCode = S3FSCURL_RESPONSECODE_NOTSET; int result = S3FSCURL_PERFORM_RESULT_NOTSET; // 1 attempt + retries... for(int retrycnt = 0; S3FSCURL_PERFORM_RESULT_NOTSET == result && retrycnt < S3fsCurl::retries; ++retrycnt){ // Reset response code responseCode = S3FSCURL_RESPONSECODE_NOTSET; // Insert headers if(!dontAddAuthHeaders) { if(!insertAuthHeaders()){ return -EIO; } } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders)){ S3FS_PRN_ERR("Failed to call curl_easy_setopt, returned NOT CURLE_OK."); return -EIO; } // Requests curlCode = curl_easy_perform(hCurl.get()); // Check result switch(curlCode){ case CURLE_OK: // Need to look at the HTTP response code if(0 != curl_easy_getinfo(hCurl, CURLINFO_RESPONSE_CODE, &responseCode)){ S3FS_PRN_ERR("curl_easy_getinfo failed while trying to retrieve HTTP response code"); responseCode = S3FSCURL_RESPONSECODE_FATAL_ERROR; result = -EIO; break; } if(responseCode >= 200 && responseCode < 300){ S3FS_PRN_INFO3("HTTP response code %ld", responseCode); result = 0; break; } { // Try to parse more specific AWS error code otherwise fall back to HTTP error code. std::string value; if(simple_parse_xml(bodydata.c_str(), bodydata.size(), "Code", value)){ // TODO: other error codes if(value == "EntityTooLarge"){ result = -EFBIG; break; }else if(value == "InvalidObjectState"){ result = -EREMOTE; break; }else if(value == "KeyTooLongError"){ result = -ENAMETOOLONG; break; } } } // Service response codes which are >= 300 && < 500 switch(responseCode){ case 301: case 307: S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.c_str()); S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options."); result = -EIO; break; case 400: if(op == "HEAD"){ if(path.size() > 1024){ S3FS_PRN_ERR("HEAD HTTP response code %ld with path longer than 1024, returning ENAMETOOLONG.", responseCode); return -ENAMETOOLONG; } S3FS_PRN_ERR("HEAD HTTP response code %ld, returning EPERM.", responseCode); result = -EPERM; }else{ S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str()); result = -EIO; } break; case 403: S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.c_str()); result = -EPERM; break; case 404: S3FS_PRN_INFO3("HTTP response code 404 was returned, returning ENOENT"); S3FS_PRN_DBG("Body Text: %s", bodydata.c_str()); result = -ENOENT; break; case 416: S3FS_PRN_INFO3("HTTP response code 416 was returned, returning EIO"); result = -EIO; break; case 501: S3FS_PRN_INFO3("HTTP response code 501 was returned, returning ENOTSUP"); S3FS_PRN_DBG("Body Text: %s", bodydata.c_str()); result = -ENOTSUP; break; case 429: case 500: case 503: { S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode); S3FS_PRN_DBG("Body Text: %s", bodydata.c_str()); // Add jitter to avoid thundering herd. unsigned int sleep_time = 2 << retry_count; sleep(sleep_time + static_cast(random()) % sleep_time); break; } default: S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str()); result = -EIO; break; } break; case CURLE_WRITE_ERROR: S3FS_PRN_ERR("### CURLE_WRITE_ERROR"); sleep(2); break; case CURLE_OPERATION_TIMEDOUT: S3FS_PRN_ERR("### CURLE_OPERATION_TIMEDOUT"); sleep(2); break; case CURLE_COULDNT_RESOLVE_HOST: S3FS_PRN_ERR("### CURLE_COULDNT_RESOLVE_HOST"); sleep(2); break; case CURLE_COULDNT_CONNECT: S3FS_PRN_ERR("### CURLE_COULDNT_CONNECT"); sleep(4); break; case CURLE_GOT_NOTHING: S3FS_PRN_ERR("### CURLE_GOT_NOTHING"); sleep(4); break; case CURLE_ABORTED_BY_CALLBACK: S3FS_PRN_ERR("### CURLE_ABORTED_BY_CALLBACK"); sleep(4); { const std::lock_guard lock(S3fsCurl::curl_handles_lock); S3fsCurl::curl_progress[hCurl.get()] = {time(nullptr), -1, -1}; } break; case CURLE_PARTIAL_FILE: S3FS_PRN_ERR("### CURLE_PARTIAL_FILE"); sleep(4); break; case CURLE_SEND_ERROR: S3FS_PRN_ERR("### CURLE_SEND_ERROR"); sleep(2); break; case CURLE_RECV_ERROR: S3FS_PRN_ERR("### CURLE_RECV_ERROR"); sleep(2); break; case CURLE_SSL_CONNECT_ERROR: S3FS_PRN_ERR("### CURLE_SSL_CONNECT_ERROR"); sleep(2); break; case CURLE_SSL_CACERT: S3FS_PRN_ERR("### CURLE_SSL_CACERT"); // try to locate cert, if successful, then set the // option and continue if(S3fsCurl::curl_ca_bundle.empty()){ if(!S3fsCurl::LocateBundle()){ S3FS_PRN_ERR("could not get CURL_CA_BUNDLE."); result = -EIO; } // retry with CAINFO }else{ S3FS_PRN_ERR("curlCode: %d msg: %s", curlCode, curl_easy_strerror(curlCode)); result = -EIO; } break; #ifdef CURLE_PEER_FAILED_VERIFICATION case CURLE_PEER_FAILED_VERIFICATION: S3FS_PRN_ERR("### CURLE_PEER_FAILED_VERIFICATION"); first_pos = S3fsCred::GetBucket().find_first_of('.'); if(first_pos != std::string::npos){ S3FS_PRN_INFO("curl returned a CURL_PEER_FAILED_VERIFICATION error"); S3FS_PRN_INFO("security issue found: buckets with periods in their name are incompatible with http"); S3FS_PRN_INFO("This check can be over-ridden by using the -o ssl_verify_hostname=0"); S3FS_PRN_INFO("The certificate will still be checked but the hostname will not be verified."); S3FS_PRN_INFO("A more secure method would be to use a bucket name without periods."); }else{ S3FS_PRN_INFO("my_curl_easy_perform: curlCode: %d -- %s", curlCode, curl_easy_strerror(curlCode)); } result = -EIO; break; #endif // This should be invalid since curl option HTTP FAILONERROR is now off case CURLE_HTTP_RETURNED_ERROR: S3FS_PRN_ERR("### CURLE_HTTP_RETURNED_ERROR"); if(0 != curl_easy_getinfo(hCurl, CURLINFO_RESPONSE_CODE, &responseCode)){ result = -EIO; }else{ S3FS_PRN_INFO3("HTTP response code =%ld", responseCode); // Let's try to retrieve the if(404 == responseCode){ result = -ENOENT; }else if(500 > responseCode){ result = -EIO; } } break; // Unknown CURL return code default: S3FS_PRN_ERR("###curlCode: %d msg: %s", curlCode, curl_easy_strerror(curlCode)); result = -EIO; break; } // switch if(S3FSCURL_PERFORM_RESULT_NOTSET == result){ S3FS_PRN_INFO("### retrying..."); if(!RemakeHandle()){ S3FS_PRN_INFO("Failed to reset handle and internal data for retrying."); result = -EIO; break; } } } // for // set last response code if(S3FSCURL_RESPONSECODE_NOTSET == responseCode){ LastResponseCode = S3FSCURL_RESPONSECODE_FATAL_ERROR; }else{ LastResponseCode = responseCode; } if(S3FSCURL_PERFORM_RESULT_NOTSET == result){ S3FS_PRN_ERR("### giving up"); result = -EIO; } return result; } // // Returns the Amazon AWS signature for the given parameters. // // @param method e.g., "GET" // @param content_type e.g., "application/x-directory" // @param date e.g., get_date_rfc850() // @param resource e.g., "/pub" // std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token) { std::string Signature; std::string StringToSign; if(!access_token.empty()){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", access_token.c_str()); } StringToSign += method + "\n"; StringToSign += strMD5 + "\n"; // md5 StringToSign += content_type + "\n"; StringToSign += date + "\n"; StringToSign += get_canonical_headers(requestHeaders, true); StringToSign += resource; const void* key = secret_access_key.data(); size_t key_len = secret_access_key.size(); const auto* sdata = reinterpret_cast(StringToSign.data()); size_t sdata_len = StringToSign.size(); unsigned int md_len = 0; std::unique_ptr md = s3fs_HMAC(key, key_len, sdata, sdata_len, &md_len); Signature = s3fs_base64(md.get(), md_len); return Signature; } std::string S3fsCurl::CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token) { std::string StringCQ, StringToSign; std::string uriencode; if(!access_token.empty()){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", access_token.c_str()); } uriencode = urlEncodePath(canonical_uri); StringCQ = method + "\n"; if(method == "HEAD" || method == "PUT" || method == "DELETE"){ StringCQ += uriencode + "\n"; }else if(method == "GET" && uriencode.empty()){ StringCQ +="/\n"; }else if(method == "GET" && is_prefix(uriencode.c_str(), "/")){ StringCQ += uriencode +"\n"; }else if(method == "GET" && !is_prefix(uriencode.c_str(), "/")){ StringCQ += "/\n" + urlEncodeQuery(canonical_uri) +"\n"; }else if(method == "POST"){ StringCQ += uriencode + "\n"; } StringCQ += urlEncodeQuery(query_string) + "\n"; StringCQ += get_canonical_headers(requestHeaders) + "\n"; StringCQ += get_sorted_header_keys(requestHeaders) + "\n"; StringCQ += payload_hash; std::string kSecret = "AWS4" + secret_access_key; unsigned int kDate_len,kRegion_len, kService_len, kSigning_len = 0; std::unique_ptr kDate = s3fs_HMAC256(kSecret.c_str(), kSecret.size(), reinterpret_cast(strdate.data()), strdate.size(), &kDate_len); std::unique_ptr kRegion = s3fs_HMAC256(kDate.get(), kDate_len, reinterpret_cast(endpoint.c_str()), endpoint.size(), &kRegion_len); std::unique_ptr kService = s3fs_HMAC256(kRegion.get(), kRegion_len, reinterpret_cast("s3"), sizeof("s3") - 1, &kService_len); std::unique_ptr kSigning = s3fs_HMAC256(kService.get(), kService_len, reinterpret_cast("aws4_request"), sizeof("aws4_request") - 1, &kSigning_len); const auto* cRequest = reinterpret_cast(StringCQ.c_str()); size_t cRequest_len = StringCQ.size(); sha256_t sRequest; s3fs_sha256(cRequest, cRequest_len, &sRequest); StringToSign = "AWS4-HMAC-SHA256\n"; StringToSign += date8601 + "\n"; StringToSign += strdate + "/" + endpoint + "/s3/aws4_request\n"; StringToSign += s3fs_hex_lower(sRequest.data(), sRequest.size()); const auto* cscope = reinterpret_cast(StringToSign.c_str()); size_t cscope_len = StringToSign.size(); unsigned int md_len = 0; std::unique_ptr md = s3fs_HMAC256(kSigning.get(), kSigning_len, cscope, cscope_len, &md_len); return s3fs_hex_lower(md.get(), md_len); } std::string S3fsCurl::extractURI(const std::string& url) { // If the URL is empty, return "/" if (url.empty()) { return "/"; } // Find the position of "://" std::size_t protocol_pos = url.find("://"); if (protocol_pos == std::string::npos) { // If "://" is not found, return "/" return "/"; } // Find the position of the first "/" after "://" std::size_t uri_pos = url.find('/', protocol_pos + 3); if (uri_pos == std::string::npos) { // If no "/" is found after the domain, return "/" return "/"; } // Extract the URI std::string uri = url.substr(uri_pos); // Ensure the URI ends with "/" if (uri.back() != '/') { uri += '/'; } return uri; } bool S3fsCurl::insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token) { std::string server_path = type == REQTYPE::LISTBUCKET ? "/" : path; std::string payload_hash; switch (type) { case REQTYPE::PUT: if(GetUnsignedPayload()){ payload_hash = "UNSIGNED-PAYLOAD"; }else{ payload_hash = s3fs_sha256_hex_fd(b_infile == nullptr ? -1 : fileno(b_infile.get()), 0, -1); } break; case REQTYPE::COMPLETEMULTIPOST: { size_t cRequest_len = strlen(reinterpret_cast(b_postdata)); sha256_t sRequest; s3fs_sha256(b_postdata, cRequest_len, &sRequest); payload_hash = s3fs_hex_lower(sRequest.data(), sRequest.size()); break; } case REQTYPE::UPLOADMULTIPOST: if(GetUnsignedPayload()){ payload_hash = "UNSIGNED-PAYLOAD"; }else{ payload_hash = s3fs_sha256_hex_fd(partdata.fd, partdata.startpos, partdata.size); } break; default: break; } if(b_infile != nullptr && payload_hash.empty()){ S3FS_PRN_ERR("Failed to make SHA256."); return false; } S3FS_PRN_INFO3("computing signature [%s] [%s] [%s] [%s]", op.c_str(), server_path.c_str(), query_string.c_str(), payload_hash.c_str()); std::string strdate; std::string date8601; get_date_sigv3(strdate, date8601); std::string contentSHA256 = payload_hash.empty() ? EMPTY_PAYLOAD_HASH : payload_hash; const std::string realpath = pathrequeststyle ? S3fsCurl::extractURI(s3host) + S3fsCred::GetBucket() + server_path : server_path; //string canonical_headers, signed_headers; requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", contentSHA256.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); if (S3fsCurl::IsRequesterPays()) { requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-request-payer", "requester"); } if(!S3fsCurl::IsPublicBucket()){ std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE::PREMULTIPOST || type == REQTYPE::MULTILIST ? "=" : ""), strdate, contentSHA256, date8601, secret_access_key, access_token); std::string auth = "AWS4-HMAC-SHA256 Credential=" + access_key_id + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature; requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str()); } return true; } void S3fsCurl::insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token) { std::string resource; std::string turl; std::string server_path = type == REQTYPE::LISTBUCKET ? "/" : path; MakeUrlResource(server_path.c_str(), resource, turl); if(!query_string.empty() && type != REQTYPE::CHKBUCKET && type != REQTYPE::LISTBUCKET){ resource += "?" + query_string; } std::string date = get_date_rfc850(); requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); if(op != "PUT" && op != "POST"){ requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", nullptr); } if(!S3fsCurl::IsPublicBucket()){ std::string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource, secret_access_key, access_token); requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("AWS " + access_key_id + ":" + Signature).c_str()); } } void S3fsCurl::insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token) { requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + access_token).c_str()); if(op == "PUT" && path == mount_prefix + "/"){ // ibm-service-instance-id header is required for bucket creation requests requestHeaders = curl_slist_sort_insert(requestHeaders, "ibm-service-instance-id", access_key_id.c_str()); } } bool S3fsCurl::insertAuthHeaders() { std::string access_key_id; std::string secret_access_key; std::string access_token; // check and get credential variables if(!S3fsCurl::ps3fscred->CheckIAMCredentialUpdate(&access_key_id, &secret_access_key, &access_token)){ S3FS_PRN_ERR("An error occurred in checking IAM credential."); return false; // do not insert auth headers on error } if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){ insertIBMIAMHeaders(access_key_id, access_token); }else if(S3fsCurl::signature_type == signature_type_t::V2_ONLY){ insertV2Headers(access_key_id, secret_access_key, access_token); }else{ if(!insertV4Headers(access_key_id, secret_access_key, access_token)){ return false; } } return true; } int S3fsCurl::DeleteRequest(const char* tpath) { S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(!tpath){ return -EINVAL; } if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); op = "DELETE"; type = REQTYPE::DELETE; if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE")){ return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } return RequestPerform(); } int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char* token_ttl_hdr, std::string& response) { if(!token_url || !token_ttl_hdr){ S3FS_PRN_ERR("IAMv2 token url(%s) or ttl_hdr(%s) parameter are wrong.", token_url ? token_url : "null", token_ttl_hdr ? token_ttl_hdr : "null"); return -EIO; } response.clear(); url = token_url; if(!CreateCurlHandle()){ return -EIO; } requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); std::string ttlstr = std::to_string(token_ttl); requestHeaders = curl_slist_sort_insert(requestHeaders, token_ttl_hdr, ttlstr.c_str()); // Curl appends an "Expect: 100-continue" header to the token request, // and aws responds with a 417 Expectation Failed. This ensures the // Expect header is empty before the request is sent. requestHeaders = curl_slist_sort_insert(requestHeaders, "Expect", ""); if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } // [NOTE] // Be sure to give "dontAddAuthHeaders=true". // If set to false(default), it will deadlock in S3fsCred. // int result = RequestPerform(true); if(0 == result){ response.swap(bodydata); }else{ S3FS_PRN_ERR("Error(%d) occurred, could not get IAMv2 api token.", result); } bodydata.clear(); return result; } // // Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role, // and Set these value to class variable. // bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response) { if(!cred_url){ S3FS_PRN_ERR("url is null."); return false; } url = cred_url; response.clear(); // at first set type for handle type = REQTYPE::IAMCRED; if(!CreateCurlHandle()){ return false; } requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); std::string postContent; if(ibm_secret_access_key){ // make contents postContent += "grant_type=urn:ibm:params:oauth:grant-type:apikey"; postContent += "&response_type=cloud_iam"; postContent += "&apikey="; postContent += ibm_secret_access_key; // set postdata postdata = reinterpret_cast(postContent.c_str()); b_postdata = postdata; postdata_remaining = postContent.size(); // without null b_postdata_remaining = postdata_remaining; requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", "Basic Yng6Yng="); if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ // POST return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast(postdata_remaining))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, reinterpret_cast(this))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback)){ return false; } } if(iam_v2_token){ requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token); } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return false; } // [NOTE] // Be sure to give "dontAddAuthHeaders=true". // If set to false(default), it will deadlock in S3fsCred. // int result = RequestPerform(true); // analyzing response if(0 == result){ response.swap(bodydata); }else{ S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name.", result); } bodydata.clear(); return (0 == result); } // // Get IAM role name automatically. // bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token) { if(!cred_url){ S3FS_PRN_ERR("url is null."); return false; } url = cred_url; token.clear(); S3FS_PRN_INFO3("Get IAM Role name"); // at first set type for handle type = REQTYPE::IAMROLE; if(!CreateCurlHandle()){ return false; } requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); if(iam_v2_token){ requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token); } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return false; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return false; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return false; } // [NOTE] // Be sure to give "dontAddAuthHeaders=true". // If set to false(default), it will deadlock in S3fsCred. // int result = RequestPerform(true); // analyzing response if(0 == result){ token.swap(bodydata); }else{ S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name from meta data.", result); } bodydata.clear(); return (0 == result); } bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, const std::string& input, bool is_copy) { std::string ssevalue = input; switch(ssetype){ case sse_type_t::SSE_DISABLE: return true; case sse_type_t::SSE_S3: if(!is_copy){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256"); } return true; case sse_type_t::SSE_C: { std::string sseckey; if(S3fsCurl::GetSseKey(ssevalue, sseckey)){ if(is_copy){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm", "AES256"); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key", sseckey.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key-md5", ssevalue.c_str()); }else{ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm", "AES256"); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key", sseckey.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key-md5", ssevalue.c_str()); } }else{ S3FS_PRN_WARN("Failed to insert SSE-C header."); } return true; } case sse_type_t::SSE_KMS: if(!is_copy){ if(ssevalue.empty()){ ssevalue = S3fsCurl::GetSseKmsId(); } requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "aws:kms"); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-aws-kms-key-id", ssevalue.c_str()); } return true; } S3FS_PRN_ERR("sse type is unknown(%d).", static_cast(S3fsCurl::ssetype)); return false; } // // tpath : target path for head request // bpath : saved into base_path // savedpath : saved into saved_path // ssekey_pos : -1 means "not" SSE-C type // 0 - X means SSE-C type and position for SSE-C key(0 is latest key) // bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath, size_t ssekey_pos) { S3FS_PRN_INFO3("[tpath=%s][bpath=%s][save=%s][sseckeypos=%zu]", SAFESTRPTR(tpath), SAFESTRPTR(bpath), SAFESTRPTR(savedpath), ssekey_pos); if(!tpath){ return false; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); // libcurl 7.17 does deep copy of url, deep copy "stable" url url = prepare_url(turl.c_str()); path = get_realpath(tpath); base_path = SAFESTRPTR(bpath); saved_path = SAFESTRPTR(savedpath); requestHeaders = nullptr; responseHeaders.clear(); // requestHeaders(SSE-C) if(0 <= static_cast(ssekey_pos) && ssekey_pos < S3fsCurl::sseckeys.size()){ std::string md5; if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseRequestHead(sse_type_t::SSE_C, md5, false)){ S3FS_PRN_ERR("Failed to set SSE-C headers for sse-c key pos(%zu)(=md5(%s)).", ssekey_pos, md5.c_str()); return false; } } b_ssekey_pos = ssekey_pos; op = "HEAD"; type = REQTYPE::HEAD; // set lazy function fpLazySetup = PreHeadRequestSetCurlOpts; return true; } int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta) { int result = -1; S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); // At first, try to get without SSE-C headers if(!PreHeadRequest(tpath) || !fpLazySetup || !fpLazySetup(this) || 0 != (result = RequestPerform())){ // If has SSE-C keys, try to get with all SSE-C keys. for(size_t pos = 0; pos < S3fsCurl::sseckeys.size(); pos++){ if(!DestroyCurlHandle()){ break; } if(!PreHeadRequest(tpath, nullptr, nullptr, pos)){ break; } if(!fpLazySetup || !fpLazySetup(this)){ S3FS_PRN_ERR("Failed to lazy setup in single head request."); break; } if(0 == (result = RequestPerform())){ break; } } if(0 != result){ DestroyCurlHandle(); // not check result. return result; } } // file exists in s3 // fixme: clean this up. meta.clear(); for(auto iter = responseHeaders.cbegin(); iter != responseHeaders.cend(); ++iter){ std::string key = lower(iter->first); std::string value = iter->second; if(key == "content-type"){ meta[iter->first] = value; }else if(key == "content-length"){ meta[iter->first] = value; }else if(key == "etag"){ meta[iter->first] = value; }else if(key == "last-modified"){ meta[iter->first] = value; }else if(is_prefix(key.c_str(), "x-amz")){ meta[key] = value; // key is lower case for "x-amz" } } return 0; } int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) { S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(!tpath){ return -EINVAL; } if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); std::string contype = S3fsCurl::LookupMimeType(tpath); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); // Make request headers for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ std::string key = lower(iter->first); std::string value = iter->second; if(is_prefix(key.c_str(), "x-amz-acl")){ // not set value, but after set it. }else if(is_prefix(key.c_str(), "x-amz-meta")){ requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); }else if(key == "x-amz-copy-source"){ requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){ // skip this header, because this header is specified after logic. }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){ // skip this header, because this header is specified after logic. }else if(key == "x-amz-server-side-encryption-customer-key-md5"){ // Only copy mode. if(is_copy){ if(!AddSseRequestHead(sse_type_t::SSE_C, value, true)){ S3FS_PRN_WARN("Failed to insert SSE-C header."); } } } } // "x-amz-acl", storage class, sse if(S3fsCurl::default_acl != acl_t::PRIVATE){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", str(S3fsCurl::default_acl)); } if(strcasecmp(GetStorageClass().c_str(), "STANDARD") != 0){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str()); } // SSE if(S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } if(is_use_ahbe){ // set additional header by ahbe conf requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); } op = "PUT"; type = REQTYPE::PUTHEAD; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ // HTTP PUT return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ // Content-Length return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } S3FS_PRN_INFO3("copying... [path=%s]", tpath); int result = RequestPerform(); result = MapPutErrorResponse(result); bodydata.clear(); return result; } int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) { struct stat st; S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(!tpath){ return -EINVAL; } if(-1 != fd){ // duplicate fd // // [NOTE] // This process requires FILE*, then it is linked to fd with fdopen. // After processing, the FILE* is closed with fclose, and fd is closed together. // The fd should not be closed here, so call dup here to duplicate it. // std::unique_ptr file(nullptr, &s3fs_fclose); int fd2; if(-1 == (fd2 = dup(fd)) || -1 == fstat(fd2, &st) || 0 != lseek(fd2, 0, SEEK_SET) || nullptr == (file = {fdopen(fd2, "rb"), &s3fs_fclose})){ S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno); if(-1 != fd2){ close(fd2); } return -errno; } b_infile = std::move(file); }else{ // This case is creating zero byte object.(calling by create_file_object()) S3FS_PRN_INFO3("create zero byte file object."); } if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); // Make request headers if(S3fsCurl::is_content_md5){ std::string strMD5; if(-1 != fd){ strMD5 = s3fs_get_content_md5(fd); if(strMD5.empty()){ S3FS_PRN_ERR("Failed to make MD5."); return -EIO; } }else{ strMD5 = EMPTY_MD5_BASE64_HASH; } requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str()); } std::string contype = S3fsCurl::LookupMimeType(tpath); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ std::string key = lower(iter->first); std::string value = iter->second; if(is_prefix(key.c_str(), "x-amz-acl")){ // not set value, but after set it. }else if(is_prefix(key.c_str(), "x-amz-meta")){ requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){ // skip this header, because this header is specified after logic. }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){ // skip this header, because this header is specified after logic. }else if(key == "x-amz-server-side-encryption-customer-key-md5"){ // skip this header, because this header is specified after logic. } } // "x-amz-acl", storage class, sse if(S3fsCurl::default_acl != acl_t::PRIVATE){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", str(S3fsCurl::default_acl)); } if(strcasecmp(GetStorageClass().c_str(), "STANDARD") != 0){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str()); } // SSE // do not add SSE for create bucket if(0 != strcmp(tpath, "/")){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } if(is_use_ahbe){ // set additional header by ahbe conf requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); } op = "PUT"; type = REQTYPE::PUT; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true)){ // HTTP PUT return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(b_infile){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast(st.st_size))){ // Content-Length return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILE, b_infile.get())){ return -EIO; } }else{ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ // Content-Length: 0 return -EIO; } } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } S3FS_PRN_INFO3("uploading... [path=%s][fd=%d][size=%lld]", tpath, fd, static_cast(-1 != fd ? st.st_size : 0)); int result = RequestPerform(); result = MapPutErrorResponse(result); bodydata.clear(); return result; } int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, off_t size, sse_type_t ssetype, const std::string& ssevalue) { S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld]", SAFESTRPTR(tpath), static_cast(start), static_cast(size)); if(!tpath || -1 == fd || 0 > start || 0 > size){ return -EINVAL; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); if(0 < size){ std::string range = "bytes="; range += std::to_string(start); range += "-"; range += std::to_string(start + size - 1); requestHeaders = curl_slist_sort_insert(requestHeaders, "Range", range.c_str()); } // SSE-C if(sse_type_t::SSE_C == ssetype){ if(!AddSseRequestHead(ssetype, ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } op = "GET"; type = REQTYPE::GET; // set lazy function fpLazySetup = PreGetObjectRequestSetCurlOpts; // set info for callback func. // (use only fd, startpos and size, other member is not used.) partdata.clear(); partdata.fd = fd; partdata.startpos = start; partdata.size = size; b_partdata_startpos = start; b_partdata_size = size; b_ssetype = ssetype; b_ssevalue = ssevalue; b_ssekey_pos = -1; // not use this value for get object. return 0; } int S3fsCurl::GetObjectRequest(const char* tpath, int fd, off_t start, off_t size) { int result; S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld]", SAFESTRPTR(tpath), static_cast(start), static_cast(size)); if(!tpath){ return -EINVAL; } sse_type_t local_ssetype = sse_type_t::SSE_DISABLE; std::string ssevalue; if(!get_object_sse_type(tpath, local_ssetype, ssevalue)){ S3FS_PRN_WARN("Failed to get SSE type for file(%s).", SAFESTRPTR(tpath)); } if(0 != (result = PreGetObjectRequest(tpath, fd, start, size, local_ssetype, ssevalue))){ return result; } if(!fpLazySetup || !fpLazySetup(this)){ S3FS_PRN_ERR("Failed to lazy setup in single get object request."); return -EIO; } S3FS_PRN_INFO3("downloading... [path=%s][fd=%d]", tpath, fd); result = RequestPerform(); partdata.clear(); return result; } int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir, bool force_no_sse) { S3FS_PRN_INFO3("check a bucket path(%s)%s.", (check_path && 0 < strlen(check_path)) ? check_path : "", compat_dir ? " containing compatible directory paths" : ""); if(!check_path || 0 == strlen(check_path)){ return -EIO; } if(!CreateCurlHandle()){ return -EIO; } std::string strCheckPath; std::string urlargs; if(S3fsCurl::IsListObjectsV2()){ query_string = "list-type=2"; }else{ query_string.clear(); } if(!compat_dir){ // do not check compatibility directories strCheckPath = check_path; }else{ // check path including compatibility directory strCheckPath = "/"; if(1 < strlen(check_path)){ // for directory path ("/...") not root path("/") if(!query_string.empty()){ query_string += '&'; } query_string += "prefix="; query_string += &check_path[1]; // skip first '/' character } } if(!query_string.empty()){ urlargs = "?" + query_string; } std::string resource; std::string turl; MakeUrlResource(strCheckPath.c_str(), resource, turl); turl += urlargs; url = prepare_url(turl.c_str()); path = strCheckPath; requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); // SSE if(!force_no_sse && S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } op = "GET"; type = REQTYPE::CHKBUCKET; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_UNRESTRICTED_AUTH, 1L)){ return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } int result = RequestPerform(); if (result != 0) { S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.c_str()); } return result; } int S3fsCurl::ListBucketRequest(const char* tpath, const char* query) { S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(!tpath){ return -EINVAL; } if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; MakeUrlResource("", resource, turl); // NOTICE: path is "". if(query){ turl += "?"; turl += query; query_string = query; } url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); op = "GET"; type = REQTYPE::LISTBUCKET; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(S3fsCurl::is_verbose){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_DEBUGFUNCTION, S3fsCurl::CurlDebugBodyInFunc)){ // replace debug function return -EIO; } } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } return RequestPerform(); } // // Initialize multipart upload // // Example : // POST /example-object?uploads HTTP/1.1 // Host: example-bucket.s3.amazonaws.com // Date: Mon, 1 Nov 2010 20:34:56 GMT // Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZCBieSBlbHZpbmc= // int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id) { S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(!tpath){ return -EINVAL; } if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); query_string = "uploads"; turl += "?" + query_string; url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; bodydata.clear(); responseHeaders.clear(); std::string contype = S3fsCurl::LookupMimeType(tpath); for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ std::string key = lower(iter->first); std::string value = iter->second; if(is_prefix(key.c_str(), "x-amz-acl")){ // not set value, but after set it. }else if(is_prefix(key.c_str(), "x-amz-meta")){ requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){ // skip this header, because this header is specified after logic. }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){ // skip this header, because this header is specified after logic. }else if(key == "x-amz-server-side-encryption-customer-key-md5"){ // skip this header, because this header is specified after logic. } } // "x-amz-acl", storage class, sse if(S3fsCurl::default_acl != acl_t::PRIVATE){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", str(S3fsCurl::default_acl)); } if(strcasecmp(GetStorageClass().c_str(), "STANDARD") != 0){ requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str()); } // SSE if(S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } if(is_use_ahbe){ // set additional header by ahbe conf requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); } requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); op = "POST"; type = REQTYPE::PREMULTIPOST; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ // POST return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0)){ // Content-Length return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } // request int result; if(0 != (result = RequestPerform())){ bodydata.clear(); return result; } if(!simple_parse_xml(bodydata.c_str(), bodydata.size(), "UploadId", upload_id)){ bodydata.clear(); return -EIO; } bodydata.clear(); return 0; } int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string& upload_id, const etaglist_t& parts) { S3FS_PRN_INFO3("[tpath=%s][parts=%zu]", SAFESTRPTR(tpath), parts.size()); if(!tpath){ return -EINVAL; } // make contents std::string postContent; postContent += "\n"; for(auto it = parts.cbegin(); it != parts.cend(); ++it){ if(it->etag.empty()){ S3FS_PRN_ERR("%d file part is not finished uploading.", it->part_num); return -EIO; } postContent += "\n"; postContent += " " + std::to_string(it->part_num) + "\n"; postContent += " " + it->etag + "\n"; postContent += "\n"; } postContent += "\n"; // set postdata postdata = reinterpret_cast(postContent.c_str()); b_postdata = postdata; postdata_remaining = postContent.size(); // without null b_postdata_remaining = postdata_remaining; if(!CreateCurlHandle()){ postdata = nullptr; b_postdata = nullptr; return -EIO; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); // [NOTE] // Encode the upload_id here. // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. // query_string = "uploadId=" + urlEncodeGeneral(upload_id); turl += "?" + query_string; url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; bodydata.clear(); responseHeaders.clear(); std::string contype = "application/xml"; requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } op = "POST"; type = REQTYPE::COMPLETEMULTIPOST; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ // POST return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast(postdata_remaining))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, reinterpret_cast(this))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback)){ return -EIO; } if(S3fsCurl::is_verbose){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_DEBUGFUNCTION, S3fsCurl::CurlDebugBodyOutFunc)){ // replace debug function return -EIO; } } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } // request int result = RequestPerform(); bodydata.clear(); postdata = nullptr; b_postdata = nullptr; return result; } int S3fsCurl::MultipartListRequest(std::string& body) { S3FS_PRN_INFO3("list request(multipart)"); if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; path = get_realpath("/"); MakeUrlResource(path.c_str(), resource, turl); query_string = "uploads"; turl += "?" + query_string; url = prepare_url(turl.c_str()); requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr); op = "GET"; type = REQTYPE::MULTILIST; // setopt if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, reinterpret_cast(&bodydata))){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } int result; if(0 == (result = RequestPerform()) && !bodydata.empty()){ body.swap(bodydata); }else{ body = ""; } bodydata.clear(); return result; } int S3fsCurl::AbortMultipartUpload(const char* tpath, const std::string& upload_id) { S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(!tpath){ return -EINVAL; } if(!CreateCurlHandle()){ return -EIO; } std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); // [NOTE] // Encode the upload_id here. // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. // query_string = "uploadId=" + urlEncodeGeneral(upload_id); turl += "?" + query_string; url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); op = "DELETE"; type = REQTYPE::ABORTMULTIUPLOAD; if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ return -EIO; } if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE")){ return -EIO; } if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent return -EIO; } return RequestPerform(); } // // PUT /ObjectName?partNumber=PartNumber&uploadId=UploadId HTTP/1.1 // Host: BucketName.s3.amazonaws.com // Date: date // Content-Length: Size // Authorization: Signature // // PUT /my-movie.m2ts?partNumber=1&uploadId=VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR HTTP/1.1 // Host: example-bucket.s3.amazonaws.com // Date: Mon, 1 Nov 2010 20:34:56 GMT // Content-Length: 10485760 // Content-MD5: pUNXr/BjKK5G2UKvaRRrOA== // Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZGGieSRlbHZpbmc= // int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id) { S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast(partdata.startpos), static_cast(partdata.size), part_num); if(-1 == partdata.fd || -1 == partdata.startpos || -1 == partdata.size){ return -EINVAL; } requestHeaders = nullptr; // make md5 and file pointer if(S3fsCurl::is_content_md5){ md5_t md5raw; if(!s3fs_md5_fd(partdata.fd, partdata.startpos, partdata.size, &md5raw)){ S3FS_PRN_ERR("Could not make md5 for file(part %d)", part_num); return -EIO; } partdata.etag = s3fs_hex_lower(md5raw.data(), md5raw.size()); std::string md5base64 = s3fs_base64(md5raw.data(), md5raw.size()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", md5base64.c_str()); } // make request // // [NOTE] // Encode the upload_id here. // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. // query_string = "partNumber=" + std::to_string(part_num) + "&uploadId=" + urlEncodeGeneral(upload_id); std::string urlargs = "?" + query_string; std::string resource; std::string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); turl += urlargs; url = prepare_url(turl.c_str()); path = get_realpath(tpath); bodydata.clear(); headdata.clear(); responseHeaders.clear(); // SSE-C if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr); op = "PUT"; type = REQTYPE::UPLOADMULTIPOST; // set lazy function fpLazySetup = UploadMultipartPostSetCurlOpts; return 0; } int S3fsCurl::UploadMultipartPostRequest(const char* tpath, int part_num, const std::string& upload_id) { int result; S3FS_PRN_INFO3("[tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast(partdata.startpos), static_cast(partdata.size), part_num); // setup if(0 != (result = S3fsCurl::UploadMultipartPostSetup(tpath, part_num, upload_id))){ return result; } if(!fpLazySetup || !fpLazySetup(this)){ S3FS_PRN_ERR("Failed to lazy setup in multipart upload post request."); return -EIO; } // request if(0 == (result = RequestPerform())){ if(!UploadMultipartPostComplete()){ result = -EIO; } } // closing bodydata.clear(); headdata.clear(); return result; } int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta) { S3FS_PRN_INFO3("[from=%s][to=%s][part=%d]", SAFESTRPTR(from), SAFESTRPTR(to), part_num); if(!from || !to){ return -EINVAL; } // [NOTE] // Encode the upload_id here. // In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included. // query_string = "partNumber=" + std::to_string(part_num) + "&uploadId=" + urlEncodeGeneral(upload_id); std::string urlargs = "?" + query_string; std::string resource; std::string turl; MakeUrlResource(get_realpath(to).c_str(), resource, turl); turl += urlargs; url = prepare_url(turl.c_str()); path = get_realpath(to); requestHeaders = nullptr; responseHeaders.clear(); bodydata.clear(); headdata.clear(); std::string contype = S3fsCurl::LookupMimeType(to); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); // Make request headers for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ std::string key = lower(iter->first); std::string value = iter->second; if(key == "x-amz-copy-source"){ requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); }else if(key == "x-amz-copy-source-range"){ requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); }else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){ // skip this header }else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){ // skip this header }else if(key == "x-amz-server-side-encryption-customer-key-md5"){ if(!AddSseRequestHead(sse_type_t::SSE_C, value, true)){ S3FS_PRN_WARN("Failed to insert SSE-C header."); } } } // SSE-C if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){ std::string ssevalue; if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){ S3FS_PRN_WARN("Failed to set SSE header, but continue..."); } } op = "PUT"; type = REQTYPE::COPYMULTIPOST; // set lazy function fpLazySetup = CopyMultipartPostSetCurlOpts; // request S3FS_PRN_INFO3("copying... [from=%s][to=%s][part=%d]", from, to, part_num); return 0; } bool S3fsCurl::UploadMultipartPostComplete() { auto it = responseHeaders.find("ETag"); if (it == responseHeaders.cend()) { return false; } std::string etag = peeloff(it->second); // check etag(md5); // // The ETAG when using SSE_C and SSE_KMS does not reflect the MD5 we sent // SSE_C: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html // SSE_KMS is ignored in the above, but in the following it states the same in the highlights: // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html // if(S3fsCurl::is_content_md5 && sse_type_t::SSE_C != S3fsCurl::GetSseType() && sse_type_t::SSE_KMS != S3fsCurl::GetSseType()){ if(!etag_equals(etag, partdata.etag)){ return false; } } partdata.petag->etag = etag; partdata.uploaded = true; return true; } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress constParameter // cppcheck-suppress constParameterCallback bool S3fsCurl::CopyMultipartPostCallback(S3fsCurl* s3fscurl, void* param) { if(!s3fscurl || param){ // this callback does not need a parameter return false; } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse return s3fscurl->CopyMultipartPostComplete(); } bool S3fsCurl::CopyMultipartPostComplete() { std::string etag; partdata.uploaded = simple_parse_xml(bodydata.c_str(), bodydata.size(), "ETag", etag); partdata.petag->etag = peeloff(etag); bodydata.clear(); headdata.clear(); return true; } bool S3fsCurl::MixMultipartPostComplete() { bool result; if(-1 == partdata.fd){ result = CopyMultipartPostComplete(); }else{ result = UploadMultipartPostComplete(); } return result; } int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta) { int result; std::string upload_id; off_t chunk; off_t bytes_remaining; etaglist_t list; S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id))){ return result; } DestroyCurlHandle(); // Initialize S3fsMultiCurl S3fsMultiCurl curlmulti(GetMaxParallelCount()); curlmulti.SetSuccessCallback(S3fsCurl::CopyMultipartPostCallback); curlmulti.SetRetryCallback(S3fsCurl::CopyMultipartPostRetryCallback); for(bytes_remaining = size, chunk = 0; 0 < bytes_remaining; bytes_remaining -= chunk){ chunk = bytes_remaining > GetMultipartCopySize() ? GetMultipartCopySize() : bytes_remaining; std::ostringstream strrange; strrange << "bytes=" << (size - bytes_remaining) << "-" << (size - bytes_remaining + chunk - 1); meta["x-amz-copy-source-range"] = strrange.str(); // s3fscurl sub object std::unique_ptr s3fscurl_para(new S3fsCurl(true)); s3fscurl_para->b_from = SAFESTRPTR(tpath); s3fscurl_para->b_meta = meta; s3fscurl_para->partdata.add_etag_list(list); // initiate upload part for parallel if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, s3fscurl_para->partdata.get_part_number(), upload_id, meta))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return result; } // set into parallel object if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl_para))){ S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", tpath); return -EIO; } } // Multi request if(0 != (result = curlmulti.Request())){ S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result); S3fsCurl s3fscurl_abort(true); int result2 = s3fscurl_abort.AbortMultipartUpload(tpath, upload_id); s3fscurl_abort.DestroyCurlHandle(); if(result2 != 0){ S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2); } return result; } if(0 != (result = CompleteMultipartPostRequest(tpath, upload_id, list))){ return result; } return 0; } int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etagpair* petagpair) { S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast(offset), static_cast(size)); // set partdata.fd = fd; partdata.startpos = offset; partdata.size = size; b_partdata_startpos = partdata.startpos; b_partdata_size = partdata.size; partdata.set_etag(petagpair); // upload part int result; if(0 != (result = UploadMultipartPostRequest(tpath, petagpair->part_num, upload_id))){ S3FS_PRN_ERR("failed uploading %d part by error(%d)", petagpair->part_num, result); return result; } DestroyCurlHandle(); return 0; } int S3fsCurl::MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size) { int result; std::string upload_id; off_t chunk; off_t bytes_remaining; etaglist_t list; S3FS_PRN_INFO3("[from=%s][to=%s]", SAFESTRPTR(from), SAFESTRPTR(to)); std::string srcresource; std::string srcurl; MakeUrlResource(get_realpath(from).c_str(), srcresource, srcurl); meta["Content-Type"] = S3fsCurl::LookupMimeType(to); meta["x-amz-copy-source"] = srcresource; if(0 != (result = PreMultipartPostRequest(to, meta, upload_id))){ return result; } DestroyCurlHandle(); // Initialize S3fsMultiCurl S3fsMultiCurl curlmulti(GetMaxParallelCount()); curlmulti.SetSuccessCallback(S3fsCurl::CopyMultipartPostCallback); curlmulti.SetRetryCallback(S3fsCurl::CopyMultipartPostRetryCallback); for(bytes_remaining = size, chunk = 0; 0 < bytes_remaining; bytes_remaining -= chunk){ chunk = bytes_remaining > GetMultipartCopySize() ? GetMultipartCopySize() : bytes_remaining; std::ostringstream strrange; strrange << "bytes=" << (size - bytes_remaining) << "-" << (size - bytes_remaining + chunk - 1); meta["x-amz-copy-source-range"] = strrange.str(); // s3fscurl sub object std::unique_ptr s3fscurl_para(new S3fsCurl(true)); s3fscurl_para->b_from = SAFESTRPTR(from); s3fscurl_para->b_meta = meta; s3fscurl_para->partdata.add_etag_list(list); // initiate upload part for parallel if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(from, to, s3fscurl_para->partdata.get_part_number(), upload_id, meta))){ S3FS_PRN_ERR("failed uploading part setup(%d)", result); return result; } // set into parallel object if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl_para))){ S3FS_PRN_ERR("Could not make curl object into multi curl(%s).", to); return -EIO; } } // Multi request if(0 != (result = curlmulti.Request())){ S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result); S3fsCurl s3fscurl_abort(true); int result2 = s3fscurl_abort.AbortMultipartUpload(to, upload_id); s3fscurl_abort.DestroyCurlHandle(); if(result2 != 0){ S3FS_PRN_ERR("error aborting multipart upload(errno=%d).", result2); } return result; } if(0 != (result = CompleteMultipartPostRequest(to, upload_id, list))){ return result; } return 0; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl.h000066400000000000000000000553751470675423500152270ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_CURL_H_ #define S3FS_CURL_H_ #include #include #include #include #include #include #include #include #include #include "common.h" #include "fdcache_page.h" #include "metaheader.h" #include "s3fs_util.h" #include "types.h" //---------------------------------------------- // Avoid dependency on libcurl version //---------------------------------------------- // [NOTE] // The following symbols (enum) depend on the version of libcurl. // CURLOPT_TCP_KEEPALIVE 7.25.0 and later // CURLOPT_SSL_ENABLE_ALPN 7.36.0 and later // CURLOPT_KEEP_SENDING_ON_ERROR 7.51.0 and later // // s3fs uses these, if you build s3fs with the old libcurl, // substitute the following symbols to avoid errors. // If the version of libcurl linked at runtime is old, // curl_easy_setopt results in an error(CURLE_UNKNOWN_OPTION) and // a message is output. // #if defined(HAVE_CURLOPT_TCP_KEEPALIVE) && (HAVE_CURLOPT_TCP_KEEPALIVE == 1) #define S3FS_CURLOPT_TCP_KEEPALIVE CURLOPT_TCP_KEEPALIVE #else #define S3FS_CURLOPT_TCP_KEEPALIVE static_cast(213) #endif #if defined(HAVE_CURLOPT_SSL_ENABLE_ALPN) && (HAVE_CURLOPT_SSL_ENABLE_ALPN == 1) #define S3FS_CURLOPT_SSL_ENABLE_ALPN CURLOPT_SSL_ENABLE_ALPN #else #define S3FS_CURLOPT_SSL_ENABLE_ALPN static_cast(226) #endif #if defined(HAVE_CURLOPT_KEEP_SENDING_ON_ERROR) && (HAVE_CURLOPT_KEEP_SENDING_ON_ERROR == 1) #define S3FS_CURLOPT_KEEP_SENDING_ON_ERROR CURLOPT_KEEP_SENDING_ON_ERROR #else #define S3FS_CURLOPT_KEEP_SENDING_ON_ERROR static_cast(245) #endif //---------------------------------------------- // Structure / Typedefs //---------------------------------------------- struct curlprogress { time_t time; double dl_progress; double ul_progress; }; typedef std::unique_ptr CurlUniquePtr; //---------------------------------------------- // class S3fsCurl //---------------------------------------------- class CurlHandlerPool; class S3fsCred; class S3fsCurl; class Semaphore; // Prototype function for lazy setup options for curl handle typedef bool (*s3fscurl_lazy_setup)(S3fsCurl* s3fscurl); typedef std::map sseckeymap_t; typedef std::vector sseckeylist_t; // Class for lapping curl // class S3fsCurl { friend class S3fsMultiCurl; private: enum class REQTYPE : int8_t { UNSET = -1, DELETE, HEAD, PUTHEAD, PUT, GET, CHKBUCKET, LISTBUCKET, PREMULTIPOST, COMPLETEMULTIPOST, UPLOADMULTIPOST, COPYMULTIPOST, MULTILIST, IAMCRED, ABORTMULTIUPLOAD, IAMROLE }; // Environment name static constexpr char S3FS_SSL_PRIVKEY_PASSWORD[] = "S3FS_SSL_PRIVKEY_PASSWORD"; // class variables static std::atomic curl_warnings_once; // emit older curl warnings only once static std::mutex curl_handles_lock; static struct callback_locks_t { std::mutex dns; std::mutex ssl_session; } callback_locks; static bool is_initglobal_done; static std::unique_ptr sCurlPool; static int sCurlPoolSize; static CURLSH* hCurlShare; static bool is_cert_check; static bool is_dns_cache; static bool is_ssl_session_cache; static long connect_timeout; static time_t readwrite_timeout; static int retries; static bool is_public_bucket; static acl_t default_acl; static std::string storage_class; static sseckeylist_t sseckeys; static std::string ssekmsid; static sse_type_t ssetype; static bool is_content_md5; static bool is_verbose; static bool is_dump_body; static S3fsCred* ps3fscred; static long ssl_verify_hostname; static std::string client_cert; static std::string client_cert_type; static std::string client_priv_key; static std::string client_priv_key_type; static std::string client_key_password; static std::map curl_progress; static std::string curl_ca_bundle; static mimes_t mimeTypes; static std::string userAgent; static int max_parallel_cnt; static int max_multireq; static off_t multipart_size; static off_t multipart_copy_size; static signature_type_t signature_type; static bool is_unsigned_payload; static bool is_ua; // User-Agent static bool listobjectsv2; static bool requester_pays; static std::string proxy_url; static bool proxy_http; static std::string proxy_userpwd; // load from file(:) static long ipresolve_type; // this value is a libcurl symbol. // variables CurlUniquePtr hCurl PT_GUARDED_BY(curl_handles_lock) = {nullptr, curl_easy_cleanup}; REQTYPE type; // type of request std::string path; // target object path std::string base_path; // base path (for multi curl head request) std::string saved_path; // saved path = cache key (for multi curl head request) std::string url; // target object path(url) struct curl_slist* requestHeaders; headers_t responseHeaders; // header data by HeaderCallback std::string bodydata; // body data by WriteMemoryCallback std::string headdata; // header data by WriteMemoryCallback long LastResponseCode; const unsigned char* postdata; // use by post method and read callback function. off_t postdata_remaining; // use by post method and read callback function. filepart partdata; // use by multipart upload/get object callback bool is_use_ahbe; // additional header by extension int retry_count; // retry count for multipart std::unique_ptr b_infile = {nullptr, &s3fs_fclose}; // backup for retrying const unsigned char* b_postdata; // backup for retrying off_t b_postdata_remaining; // backup for retrying off_t b_partdata_startpos; // backup for retrying off_t b_partdata_size; // backup for retrying size_t b_ssekey_pos; // backup for retrying std::string b_ssevalue; // backup for retrying sse_type_t b_ssetype; // backup for retrying std::string b_from; // backup for retrying(for copy request) headers_t b_meta; // backup for retrying(for copy request) std::string op; // the HTTP verb of the request ("PUT", "GET", etc.) std::string query_string; // request query string Semaphore *sem; std::mutex *completed_tids_lock; std::vector *completed_tids PT_GUARDED_BY(*completed_tids_lock); s3fscurl_lazy_setup fpLazySetup; // curl options for lazy setting function CURLcode curlCode; // handle curl return public: static constexpr long S3FSCURL_RESPONSECODE_NOTSET = -1; static constexpr long S3FSCURL_RESPONSECODE_FATAL_ERROR = -2; static constexpr int S3FSCURL_PERFORM_RESULT_NOTSET = 1; public: // constructor/destructor explicit S3fsCurl(bool ahbe = false); ~S3fsCurl(); S3fsCurl(const S3fsCurl&) = delete; S3fsCurl(S3fsCurl&&) = delete; S3fsCurl& operator=(const S3fsCurl&) = delete; S3fsCurl& operator=(S3fsCurl&&) = delete; private: // class methods static bool InitGlobalCurl(); static bool DestroyGlobalCurl(); static bool InitShareCurl(); static bool DestroyShareCurl(); static void LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr) NO_THREAD_SAFETY_ANALYSIS; static void UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr) NO_THREAD_SAFETY_ANALYSIS; static bool InitCryptMutex(); static bool DestroyCryptMutex(); static int CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow); static std::string extractURI(const std::string& url); static bool LocateBundle(); static size_t HeaderCallback(void *data, size_t blockSize, size_t numBlocks, void *userPtr); static size_t WriteMemoryCallback(void *ptr, size_t blockSize, size_t numBlocks, void *data); static size_t ReadCallback(void *ptr, size_t size, size_t nmemb, void *userp); static size_t UploadReadCallback(void *ptr, size_t size, size_t nmemb, void *userp); static size_t DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, void* userp); static bool UploadMultipartPostCallback(S3fsCurl* s3fscurl, void* param); static bool CopyMultipartPostCallback(S3fsCurl* s3fscurl, void* param); static bool MixMultipartPostCallback(S3fsCurl* s3fscurl, void* param); static std::unique_ptr UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl); static std::unique_ptr CopyMultipartPostRetryCallback(S3fsCurl* s3fscurl); static std::unique_ptr MixMultipartPostRetryCallback(S3fsCurl* s3fscurl); static std::unique_ptr ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl); // lazy functions for set curl options static bool CopyMultipartPostSetCurlOpts(S3fsCurl* s3fscurl); static bool PreGetObjectRequestSetCurlOpts(S3fsCurl* s3fscurl); static bool PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl); static bool LoadEnvSseCKeys(); static bool LoadEnvSseKmsid(); static bool PushbackSseKeys(const std::string& onekey); static bool AddUserAgent(const CurlUniquePtr& hCurl); static int CurlDebugFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr); static int CurlDebugBodyInFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr); static int CurlDebugBodyOutFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr); static int RawCurlDebugFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr, curl_infotype datatype); // methods bool ResetHandle() REQUIRES(S3fsCurl::curl_handles_lock); bool RemakeHandle(); bool ClearInternalData(); bool insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token); void insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token); void insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token); bool insertAuthHeaders(); bool AddSseRequestHead(sse_type_t ssetype, const std::string& ssevalue, bool is_copy); std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token); std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token); int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id); int CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta); bool UploadMultipartPostComplete(); bool CopyMultipartPostComplete(); int MapPutErrorResponse(int result); public: // class methods static bool InitS3fsCurl(); static bool InitCredentialObject(S3fsCred* pcredobj); static bool InitMimeType(const std::string& strFile); static bool DestroyS3fsCurl(); static std::unique_ptr CreateParallelS3fsCurl(const char* tpath, int fd, off_t start, off_t size, int part_num, bool is_copy, etagpair* petag, const std::string& upload_id, int& result); static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd); static int ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const fdpage_list_t& mixuppages); static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, off_t size); // lazy functions for set curl options(public) static bool UploadMultipartPostSetCurlOpts(S3fsCurl* s3fscurl); // class methods(variables) static std::string LookupMimeType(const std::string& name); static bool SetCheckCertificate(bool isCertCheck); static bool SetDnsCache(bool isCache); static bool SetSslSessionCache(bool isCache); static long SetConnectTimeout(long timeout); static time_t SetReadwriteTimeout(time_t timeout); static time_t GetReadwriteTimeout() { return S3fsCurl::readwrite_timeout; } static int SetRetries(int count); static bool SetPublicBucket(bool flag); static bool IsPublicBucket() { return S3fsCurl::is_public_bucket; } static acl_t SetDefaultAcl(acl_t acl); static acl_t GetDefaultAcl(); static std::string SetStorageClass(const std::string& storage_class); static std::string GetStorageClass() { return S3fsCurl::storage_class; } static bool LoadEnvSse() { return (S3fsCurl::LoadEnvSseCKeys() && S3fsCurl::LoadEnvSseKmsid()); } static sse_type_t SetSseType(sse_type_t type); static sse_type_t GetSseType() { return S3fsCurl::ssetype; } static bool IsSseDisable() { return (sse_type_t::SSE_DISABLE == S3fsCurl::ssetype); } static bool IsSseS3Type() { return (sse_type_t::SSE_S3 == S3fsCurl::ssetype); } static bool IsSseCType() { return (sse_type_t::SSE_C == S3fsCurl::ssetype); } static bool IsSseKmsType() { return (sse_type_t::SSE_KMS == S3fsCurl::ssetype); } static bool FinalCheckSse(); static bool SetSseCKeys(const char* filepath); static bool SetSseKmsid(const char* kmsid); static bool IsSetSseKmsId() { return !S3fsCurl::ssekmsid.empty(); } static const char* GetSseKmsId() { return S3fsCurl::ssekmsid.c_str(); } static bool GetSseKey(std::string& md5, std::string& ssekey); static bool GetSseKeyMd5(size_t pos, std::string& md5); static size_t GetSseKeyCount(); static bool SetContentMd5(bool flag); static bool SetVerbose(bool flag); static bool GetVerbose() { return S3fsCurl::is_verbose; } static bool SetDumpBody(bool flag); static bool IsDumpBody() { return S3fsCurl::is_dump_body; } static long SetSslVerifyHostname(long value); static long GetSslVerifyHostname() { return S3fsCurl::ssl_verify_hostname; } static bool SetSSLClientCertOptions(const std::string& values); static void ResetOffset(S3fsCurl* pCurl); // maximum parallel GET and PUT requests static int SetMaxParallelCount(int value); static int GetMaxParallelCount() { return S3fsCurl::max_parallel_cnt; } // maximum parallel HEAD requests static int SetMaxMultiRequest(int max); static int GetMaxMultiRequest() { return S3fsCurl::max_multireq; } static bool SetMultipartSize(off_t size); static off_t GetMultipartSize() { return S3fsCurl::multipart_size; } static bool SetMultipartCopySize(off_t size); static off_t GetMultipartCopySize() { return S3fsCurl::multipart_copy_size; } static signature_type_t SetSignatureType(signature_type_t signature_type) { signature_type_t bresult = S3fsCurl::signature_type; S3fsCurl::signature_type = signature_type; return bresult; } static signature_type_t GetSignatureType() { return S3fsCurl::signature_type; } static bool SetUnsignedPayload(bool issset) { bool bresult = S3fsCurl::is_unsigned_payload; S3fsCurl::is_unsigned_payload = issset; return bresult; } static bool GetUnsignedPayload() { return S3fsCurl::is_unsigned_payload; } static bool SetUserAgentFlag(bool isset) { bool bresult = S3fsCurl::is_ua; S3fsCurl::is_ua = isset; return bresult; } static bool IsUserAgentFlag() { return S3fsCurl::is_ua; } static void InitUserAgent(); static bool SetListObjectsV2(bool isset) { bool bresult = S3fsCurl::listobjectsv2; S3fsCurl::listobjectsv2 = isset; return bresult; } static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; } static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; } static bool IsRequesterPays() { return S3fsCurl::requester_pays; } static bool SetProxy(const char* url); static bool SetProxyUserPwd(const char* userpwd); static bool SetIPResolveType(const char* value); // methods bool CreateCurlHandle(bool only_pool = false, bool remake = false); bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true); bool DestroyCurlHandleHasLock(bool restore_pool = true, bool clear_internal_data = true) REQUIRES(S3fsCurl::curl_handles_lock); bool GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response); bool GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token); bool GetResponseCode(long& responseCode, bool from_curl_handle = true) const; int RequestPerform(bool dontAddAuthHeaders=false); int DeleteRequest(const char* tpath); int GetIAMv2ApiToken(const char* token_url, int token_ttl, const char* token_ttl_hdr, std::string& response); bool PreHeadRequest(const char* tpath, const char* bpath = nullptr, const char* savedpath = nullptr, size_t ssekey_pos = -1); bool PreHeadRequest(const std::string& tpath, const std::string& bpath, const std::string& savedpath, size_t ssekey_pos = -1) { return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str(), ssekey_pos); } int HeadRequest(const char* tpath, headers_t& meta); int PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy); int PutRequest(const char* tpath, headers_t& meta, int fd); int PreGetObjectRequest(const char* tpath, int fd, off_t start, off_t size, sse_type_t ssetype, const std::string& ssevalue); int GetObjectRequest(const char* tpath, int fd, off_t start = -1, off_t size = -1); int CheckBucket(const char* check_path, bool compat_dir, bool force_no_sse); int ListBucketRequest(const char* tpath, const char* query); int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id); int CompleteMultipartPostRequest(const char* tpath, const std::string& upload_id, const etaglist_t& parts); int UploadMultipartPostRequest(const char* tpath, int part_num, const std::string& upload_id); bool MixMultipartPostComplete(); int MultipartListRequest(std::string& body); int AbortMultipartUpload(const char* tpath, const std::string& upload_id); int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta); int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etagpair* petagpair); int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size); // methods(variables) const std::string& GetPath() const { return path; } const std::string& GetBasePath() const { return base_path; } const std::string& GetSpecialSavedPath() const { return saved_path; } const std::string& GetUrl() const { return url; } const std::string& GetOp() const { return op; } const headers_t* GetResponseHeaders() const { return &responseHeaders; } const std::string& GetBodyData() const { return bodydata; } const std::string& GetHeadData() const { return headdata; } CURLcode GetCurlCode() const { return curlCode; } long GetLastResponseCode() const { return LastResponseCode; } bool SetUseAhbe(bool ahbe); bool EnableUseAhbe() { return SetUseAhbe(true); } bool DisableUseAhbe() { return SetUseAhbe(false); } bool IsUseAhbe() const { return is_use_ahbe; } int GetMultipartRetryCount() const { return retry_count; } void SetMultipartRetryCount(int retrycnt) { retry_count = retrycnt; } bool IsOverMultipartRetryCount() const { return (retry_count >= S3fsCurl::retries); } size_t GetLastPreHeadSeecKeyPos() const { return b_ssekey_pos; } }; #endif // S3FS_CURL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl_handlerpool.cpp000066400000000000000000000057621470675423500201440ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include "s3fs_logger.h" #include "curl_handlerpool.h" //------------------------------------------------------------------- // Class CurlHandlerPool //------------------------------------------------------------------- bool CurlHandlerPool::Init() { for(int cnt = 0; cnt < mMaxHandlers; ++cnt){ CurlUniquePtr hCurl(curl_easy_init(), &curl_easy_cleanup); if(!hCurl){ S3FS_PRN_ERR("Init curl handlers pool failed"); Destroy(); return false; } const std::lock_guard lock(mLock); mPool.push_back(std::move(hCurl)); } return true; } bool CurlHandlerPool::Destroy() { const std::lock_guard lock(mLock); while(!mPool.empty()){ mPool.pop_back(); } return true; } CurlUniquePtr CurlHandlerPool::GetHandler(bool only_pool) { const std::lock_guard lock(mLock); CurlUniquePtr hCurl(nullptr, curl_easy_cleanup); if(!mPool.empty()){ hCurl = std::move(mPool.back()); mPool.pop_back(); S3FS_PRN_DBG("Get handler from pool: rest = %d", static_cast(mPool.size())); } if(only_pool){ return hCurl; } if(!hCurl){ S3FS_PRN_INFO("Pool empty: force to create new handler"); hCurl.reset(curl_easy_init()); } return hCurl; } void CurlHandlerPool::ReturnHandler(CurlUniquePtr&& hCurl, bool restore_pool) { if(!hCurl){ return; } const std::lock_guard lock(mLock); if(restore_pool){ S3FS_PRN_DBG("Return handler to pool"); mPool.push_back(std::move(hCurl)); while(mMaxHandlers < static_cast(mPool.size())){ S3FS_PRN_INFO("Pool full: destroy the oldest handler"); mPool.pop_front(); } }else{ S3FS_PRN_INFO("Pool full: destroy the handler"); } } void CurlHandlerPool::ResetHandler(CURL* hCurl) { if(!hCurl){ return; } const std::lock_guard lock(mLock); curl_easy_reset(hCurl); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl_handlerpool.h000066400000000000000000000043371470675423500176060ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_CURL_HANDLERPOOL_H_ #define S3FS_CURL_HANDLERPOOL_H_ #include #include #include #include #include #include "common.h" //---------------------------------------------- // Typedefs //---------------------------------------------- typedef std::unique_ptr CurlUniquePtr; //---------------------------------------------- // class CurlHandlerPool //---------------------------------------------- class CurlHandlerPool { public: explicit CurlHandlerPool(int maxHandlers) : mMaxHandlers(maxHandlers) { assert(maxHandlers > 0); } CurlHandlerPool(const CurlHandlerPool&) = delete; CurlHandlerPool(CurlHandlerPool&&) = delete; CurlHandlerPool& operator=(const CurlHandlerPool&) = delete; CurlHandlerPool& operator=(CurlHandlerPool&&) = delete; bool Init(); bool Destroy(); CurlUniquePtr GetHandler(bool only_pool); void ReturnHandler(CurlUniquePtr&& hCurl, bool restore_pool); void ResetHandler(CURL* hCurl); private: int mMaxHandlers; std::mutex mLock; std::list mPool GUARDED_BY(mLock); }; #endif // S3FS_CURL_HANDLERPOOL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl_multi.cpp000066400000000000000000000301251470675423500167560ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include "s3fs.h" #include "s3fs_logger.h" #include "curl_multi.h" #include "curl.h" #include "psemaphore.h" //------------------------------------------------------------------- // Class S3fsMultiCurl //------------------------------------------------------------------- S3fsMultiCurl::S3fsMultiCurl(int maxParallelism, bool not_abort) : maxParallelism(maxParallelism), not_abort(not_abort), SuccessCallback(nullptr), NotFoundCallback(nullptr), RetryCallback(nullptr), pSuccessCallbackParam(nullptr), pNotFoundCallbackParam(nullptr) { } S3fsMultiCurl::~S3fsMultiCurl() { Clear(); } bool S3fsMultiCurl::ClearEx(bool is_all) { s3fscurllist_t::const_iterator iter; for(iter = clist_req.cbegin(); iter != clist_req.cend(); ++iter){ S3fsCurl* s3fscurl = iter->get(); if(s3fscurl){ s3fscurl->DestroyCurlHandle(); } } clist_req.clear(); if(is_all){ for(iter = clist_all.cbegin(); iter != clist_all.cend(); ++iter){ S3fsCurl* s3fscurl = iter->get(); s3fscurl->DestroyCurlHandle(); } clist_all.clear(); } S3FS_MALLOCTRIM(0); return true; } S3fsMultiSuccessCallback S3fsMultiCurl::SetSuccessCallback(S3fsMultiSuccessCallback function) { S3fsMultiSuccessCallback old = SuccessCallback; SuccessCallback = function; return old; } S3fsMultiNotFoundCallback S3fsMultiCurl::SetNotFoundCallback(S3fsMultiNotFoundCallback function) { S3fsMultiNotFoundCallback old = NotFoundCallback; NotFoundCallback = function; return old; } S3fsMultiRetryCallback S3fsMultiCurl::SetRetryCallback(S3fsMultiRetryCallback function) { S3fsMultiRetryCallback old = RetryCallback; RetryCallback = function; return old; } void* S3fsMultiCurl::SetSuccessCallbackParam(void* param) { void* old = pSuccessCallbackParam; pSuccessCallbackParam = param; return old; } void* S3fsMultiCurl::SetNotFoundCallbackParam(void* param) { void* old = pNotFoundCallbackParam; pNotFoundCallbackParam = param; return old; } bool S3fsMultiCurl::SetS3fsCurlObject(std::unique_ptr s3fscurl) { if(!s3fscurl){ return false; } clist_all.push_back(std::move(s3fscurl)); return true; } int S3fsMultiCurl::MultiPerform() { std::map>> threads; int result = 0; bool isMultiHead = false; Semaphore sem(GetMaxParallelism()); for(auto iter = clist_req.cbegin(); iter != clist_req.cend(); ++iter) { S3fsCurl* s3fscurl = iter->get(); if(!s3fscurl){ continue; } sem.wait(); { const std::lock_guard lock(completed_tids_lock); for(const auto &thread_id : completed_tids){ auto it = threads.find(thread_id); it->second.first.join(); long int int_retval = it->second.second.get(); if (int_retval && !(int_retval == -ENOENT && isMultiHead)) { S3FS_PRN_WARN("thread terminated with non-zero return code: %ld", int_retval); } threads.erase(it); } completed_tids.clear(); } s3fscurl->sem = &sem; s3fscurl->completed_tids_lock = &completed_tids_lock; s3fscurl->completed_tids = &completed_tids; isMultiHead |= s3fscurl->GetOp() == "HEAD"; std::promise promise; std::future future = promise.get_future(); std::thread thread(S3fsMultiCurl::RequestPerformWrapper, s3fscurl, std::move(promise)); auto thread_id = thread.get_id(); threads.emplace(std::piecewise_construct, std::forward_as_tuple(thread_id), std::forward_as_tuple(std::move(thread), std::move(future))); } for(int i = 0; i < sem.get_value(); ++i){ sem.wait(); } const std::lock_guard lock(completed_tids_lock); for(const auto &thread_id : completed_tids){ auto it = threads.find(thread_id); it->second.first.join(); auto int_retval = it->second.second.get(); if (int_retval && !(int_retval == -ENOENT && isMultiHead)) { S3FS_PRN_WARN("thread terminated with non-zero return code: %d", int_retval); result = int_retval; } threads.erase(it); } completed_tids.clear(); return result; } int S3fsMultiCurl::MultiRead() { int result = 0; for(auto iter = clist_req.begin(); iter != clist_req.end(); ){ std::unique_ptr s3fscurl(std::move(*iter)); bool isRetry = false; bool isPostpone = false; bool isNeedResetOffset = true; long responseCode = S3fsCurl::S3FSCURL_RESPONSECODE_NOTSET; CURLcode curlCode = s3fscurl->GetCurlCode(); if(s3fscurl->GetResponseCode(responseCode, false) && curlCode == CURLE_OK){ if(S3fsCurl::S3FSCURL_RESPONSECODE_NOTSET == responseCode){ // This is a case where the processing result has not yet been updated (should be very rare). isPostpone = true; }else if(400 > responseCode){ // add into stat cache // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownPointerToBool if(SuccessCallback && !SuccessCallback(s3fscurl.get(), pSuccessCallbackParam)){ S3FS_PRN_WARN("error from success callback function(%s).", s3fscurl->url.c_str()); } }else if(400 == responseCode){ // as possibly in multipart S3FS_PRN_WARN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str()); isRetry = true; }else if(404 == responseCode){ // not found // HEAD requests on readdir_multi_head can return 404 if(s3fscurl->GetOp() != "HEAD"){ S3FS_PRN_WARN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str()); } // Call callback function // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownPointerToBool if(NotFoundCallback && !NotFoundCallback(s3fscurl.get(), pNotFoundCallbackParam)){ S3FS_PRN_WARN("error from not found callback function(%s).", s3fscurl->url.c_str()); } }else if(500 == responseCode){ // case of all other result, do retry.(11/13/2013) // because it was found that s3fs got 500 error from S3, but could success // to retry it. S3FS_PRN_WARN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str()); isRetry = true; }else{ // Retry in other case. S3FS_PRN_WARN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str()); isRetry = true; } }else{ S3FS_PRN_ERR("failed a request(Unknown response code: %s)", s3fscurl->url.c_str()); // Reuse particular file switch(curlCode){ case CURLE_OPERATION_TIMEDOUT: isRetry = true; isNeedResetOffset = false; break; case CURLE_PARTIAL_FILE: isRetry = true; isNeedResetOffset = false; break; default: S3FS_PRN_ERR("###curlCode: %d msg: %s", curlCode, curl_easy_strerror(curlCode)); isRetry = true; break; } } if(isPostpone){ clist_req.erase(iter); clist_req.push_back(std::move(s3fscurl)); // Re-evaluate at the end iter = clist_req.begin(); }else{ if(!isRetry || (!not_abort && 0 != result)){ // If an EIO error has already occurred, it will be terminated // immediately even if retry processing is required. s3fscurl->DestroyCurlHandle(); }else{ // Reset offset if(isNeedResetOffset){ S3fsCurl::ResetOffset(s3fscurl.get()); } // For retry std::unique_ptr retrycurl; const S3fsCurl* retrycurl_ptr = retrycurl.get(); // save this due to std::move below if(RetryCallback){ retrycurl = RetryCallback(s3fscurl.get()); if(nullptr != retrycurl){ clist_all.push_back(std::move(retrycurl)); }else{ // set EIO and wait for other parts. result = -EIO; } } // cppcheck-suppress mismatchingContainers if(s3fscurl.get() != retrycurl_ptr){ s3fscurl->DestroyCurlHandle(); } } iter = clist_req.erase(iter); } } clist_req.clear(); if(!not_abort && 0 != result){ // If an EIO error has already occurred, clear all retry objects. for(auto iter = clist_all.cbegin(); iter != clist_all.cend(); ++iter){ S3fsCurl* s3fscurl = iter->get(); s3fscurl->DestroyCurlHandle(); } clist_all.clear(); } return result; } int S3fsMultiCurl::Request() { S3FS_PRN_INFO3("[count=%zu]", clist_all.size()); // Make request list. // // Send multi request loop( with retry ) // (When many request is sends, sometimes gets "Couldn't connect to server") // while(!clist_all.empty()){ // set curl handle to multi handle int result; for(auto iter = clist_all.begin(); iter != clist_all.end(); ++iter){ clist_req.push_back(std::move(*iter)); } clist_all.clear(); // Send multi request. if(0 != (result = MultiPerform())){ Clear(); return result; } // Read the result if(0 != (result = MultiRead())){ Clear(); return result; } // Cleanup curl handle in multi handle ClearEx(false); } return 0; } // // thread function for performing an S3fsCurl request // void S3fsMultiCurl::RequestPerformWrapper(S3fsCurl* s3fscurl, std::promise promise) { int result = 0; if(!s3fscurl){ // this doesn't signal completion but also never happens promise.set_value(-EIO); return; } if(s3fscurl->fpLazySetup){ if(!s3fscurl->fpLazySetup(s3fscurl)){ S3FS_PRN_ERR("Failed to lazy setup, then respond EIO."); result = -EIO; } } if(!result){ result = s3fscurl->RequestPerform(); s3fscurl->DestroyCurlHandle(true, false); } const std::lock_guard lock(*s3fscurl->completed_tids_lock); s3fscurl->completed_tids->push_back(std::this_thread::get_id()); s3fscurl->sem->post(); promise.set_value(result); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl_multi.h000066400000000000000000000070421470675423500164250ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_CURL_MULTI_H_ #define S3FS_CURL_MULTI_H_ #include #include #include #include #include #include "common.h" //---------------------------------------------- // Typedef //---------------------------------------------- class S3fsCurl; typedef std::vector> s3fscurllist_t; typedef bool (*S3fsMultiSuccessCallback)(S3fsCurl* s3fscurl, void* param); // callback for succeed multi request typedef bool (*S3fsMultiNotFoundCallback)(S3fsCurl* s3fscurl, void* param); // callback for succeed multi request typedef std::unique_ptr (*S3fsMultiRetryCallback)(S3fsCurl* s3fscurl); // callback for failure and retrying //---------------------------------------------- // class S3fsMultiCurl //---------------------------------------------- class S3fsMultiCurl { private: const int maxParallelism; s3fscurllist_t clist_all; // all of curl requests s3fscurllist_t clist_req; // curl requests are sent bool not_abort; // complete all requests without aborting on errors S3fsMultiSuccessCallback SuccessCallback; S3fsMultiNotFoundCallback NotFoundCallback; S3fsMultiRetryCallback RetryCallback; void* pSuccessCallbackParam; void* pNotFoundCallbackParam; std::mutex completed_tids_lock; std::vector completed_tids GUARDED_BY(completed_tids_lock); private: bool ClearEx(bool is_all); int MultiPerform(); int MultiRead(); static void RequestPerformWrapper(S3fsCurl* s3fscurl, std::promise promise); public: explicit S3fsMultiCurl(int maxParallelism, bool not_abort = false); ~S3fsMultiCurl(); S3fsMultiCurl(const S3fsMultiCurl&) = delete; S3fsMultiCurl(S3fsMultiCurl&&) = delete; S3fsMultiCurl& operator=(const S3fsMultiCurl&) = delete; S3fsMultiCurl& operator=(S3fsMultiCurl&&) = delete; int GetMaxParallelism() const { return maxParallelism; } S3fsMultiSuccessCallback SetSuccessCallback(S3fsMultiSuccessCallback function); S3fsMultiNotFoundCallback SetNotFoundCallback(S3fsMultiNotFoundCallback function); S3fsMultiRetryCallback SetRetryCallback(S3fsMultiRetryCallback function); void* SetSuccessCallbackParam(void* param); void* SetNotFoundCallbackParam(void* param); bool Clear() { return ClearEx(true); } bool SetS3fsCurlObject(std::unique_ptr s3fscurl); int Request(); }; #endif // S3FS_CURL_MULTI_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl_util.cpp000066400000000000000000000216111470675423500166010ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include "common.h" #include "s3fs_logger.h" #include "curl_util.h" #include "string_util.h" #include "s3fs_auth.h" #include "s3fs_cred.h" //------------------------------------------------------------------- // Utility Functions //------------------------------------------------------------------- // // curl_slist_sort_insert // This function is like curl_slist_append function, but this adds data by a-sorting. // Because AWS signature needs sorted header. // struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value) { if(!key){ return list; } // key & value are trimmed and lower (only key) std::string strkey = trim(key); std::string strval = value ? trim(value) : ""; std::string strnew = key + std::string(": ") + strval; char* data; if(nullptr == (data = strdup(strnew.c_str()))){ return list; } struct curl_slist **p = &list; for(;*p; p = &(*p)->next){ std::string strcur = (*p)->data; size_t pos; if(std::string::npos != (pos = strcur.find(':', 0))){ strcur.erase(pos); } int result = strcasecmp(strkey.c_str(), strcur.c_str()); if(0 == result){ free((*p)->data); (*p)->data = data; return list; }else if(result < 0){ break; } } struct curl_slist* new_item; // Must use malloc since curl_slist_free_all calls free. if(nullptr == (new_item = static_cast(malloc(sizeof(*new_item))))){ free(data); return list; } struct curl_slist* before = *p; *p = new_item; new_item->data = data; new_item->next = before; return list; } struct curl_slist* curl_slist_remove(struct curl_slist* list, const char* key) { if(!key){ return list; } std::string strkey = trim(key); struct curl_slist **p = &list; while(*p){ std::string strcur = (*p)->data; size_t pos; if(std::string::npos != (pos = strcur.find(':', 0))){ strcur.erase(pos); } int result = strcasecmp(strkey.c_str(), strcur.c_str()); if(0 == result){ free((*p)->data); struct curl_slist *tmp = *p; *p = (*p)->next; free(tmp); }else{ p = &(*p)->next; } } return list; } std::string get_sorted_header_keys(const struct curl_slist* list) { std::string sorted_headers; if(!list){ return sorted_headers; } for( ; list; list = list->next){ std::string strkey = list->data; size_t pos; if(std::string::npos != (pos = strkey.find(':', 0))){ if (trim(strkey.substr(pos + 1)).empty()) { // skip empty-value headers (as they are discarded by libcurl) continue; } strkey.erase(pos); } if(!sorted_headers.empty()){ sorted_headers += ";"; } sorted_headers += lower(strkey); } return sorted_headers; } std::string get_header_value(const struct curl_slist* list, const std::string &key) { if(!list){ return ""; } for( ; list; list = list->next){ std::string strkey = list->data; size_t pos; if(std::string::npos != (pos = strkey.find(':', 0))){ if(0 == strcasecmp(trim(strkey.substr(0, pos)).c_str(), key.c_str())){ return trim(strkey.substr(pos+1)); } } } return ""; } std::string get_canonical_headers(const struct curl_slist* list, bool only_amz) { std::string canonical_headers; if(!list){ canonical_headers = "\n"; return canonical_headers; } for( ; list; list = list->next){ std::string strhead = list->data; size_t pos; if(std::string::npos != (pos = strhead.find(':', 0))){ std::string strkey = trim(lower(strhead.substr(0, pos))); std::string strval = trim(strhead.substr(pos + 1)); if (strval.empty()) { // skip empty-value headers (as they are discarded by libcurl) continue; } strhead = strkey; strhead += ":"; strhead += strval; }else{ strhead = trim(lower(strhead)); } if(only_amz && strhead.substr(0, 5) != "x-amz"){ continue; } canonical_headers += strhead; canonical_headers += "\n"; } return canonical_headers; } // function for using global values bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url) { if(!realpath){ return false; } resourcepath = urlEncodePath(service_path + S3fsCred::GetBucket() + realpath); url = s3host + resourcepath; return true; } std::string prepare_url(const char* url) { S3FS_PRN_INFO3("URL is %s", url); std::string uri; std::string hostname; std::string path; std::string url_str = url; std::string token = "/" + S3fsCred::GetBucket(); size_t bucket_pos; size_t bucket_length = token.size(); size_t uri_length = 0; if(!strncasecmp(url_str.c_str(), "https://", 8)){ uri_length = 8; } else if(!strncasecmp(url_str.c_str(), "http://", 7)) { uri_length = 7; } uri = url_str.substr(0, uri_length); bucket_pos = url_str.find(token, uri_length); if(!pathrequeststyle){ hostname = S3fsCred::GetBucket() + "." + url_str.substr(uri_length, bucket_pos - uri_length); path = url_str.substr((bucket_pos + bucket_length)); }else{ hostname = url_str.substr(uri_length, bucket_pos - uri_length); std::string part = url_str.substr((bucket_pos + bucket_length)); if('/' != part[0]){ part = "/" + part; } path = "/" + S3fsCred::GetBucket() + part; } url_str = uri + hostname + path; S3FS_PRN_INFO3("URL changed is %s", url_str.c_str()); return url_str; } bool make_md5_from_binary(const char* pstr, size_t length, std::string& md5) { if(!pstr || '\0' == pstr[0]){ S3FS_PRN_ERR("Parameter is wrong."); return false; } md5_t binary; if(!s3fs_md5(reinterpret_cast(pstr), length, &binary)){ return false; } md5 = s3fs_base64(binary.data(), binary.size()); return true; } std::string url_to_host(const std::string &url) { S3FS_PRN_INFO3("url is %s", url.c_str()); static constexpr char HTTP[] = "http://"; static constexpr char HTTPS[] = "https://"; std::string hostname; if (is_prefix(url.c_str(), HTTP)) { hostname = url.substr(sizeof(HTTP) - 1); } else if (is_prefix(url.c_str(), HTTPS)) { hostname = url.substr(sizeof(HTTPS) - 1); } else { S3FS_PRN_EXIT("url does not begin with http:// or https://"); abort(); } size_t idx; if ((idx = hostname.find('/')) != std::string::npos) { return hostname.substr(0, idx); } else { return hostname; } } std::string get_bucket_host() { if(!pathrequeststyle){ return S3fsCred::GetBucket() + "." + url_to_host(s3host); } return url_to_host(s3host); } const char* getCurlDebugHead(curl_infotype type) { const char* unknown = ""; const char* dataIn = "BODY <"; const char* dataOut = "BODY >"; const char* headIn = "<"; const char* headOut = ">"; switch(type){ case CURLINFO_DATA_IN: return dataIn; case CURLINFO_DATA_OUT: return dataOut; case CURLINFO_HEADER_IN: return headIn; case CURLINFO_HEADER_OUT: return headOut; default: break; } return unknown; } // // compare ETag ignoring quotes and case // bool etag_equals(const std::string& s1, const std::string& s2) { return 0 == strcasecmp(peeloff(s1).c_str(), peeloff(s2).c_str()); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/curl_util.h000066400000000000000000000042431470675423500162500ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_CURL_UTIL_H_ #define S3FS_CURL_UTIL_H_ #include #include #include enum class sse_type_t : uint8_t; //---------------------------------------------- // Functions //---------------------------------------------- struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value); struct curl_slist* curl_slist_remove(struct curl_slist* list, const char* key); std::string get_sorted_header_keys(const struct curl_slist* list); std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false); std::string get_header_value(const struct curl_slist* list, const std::string &key); bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url); std::string prepare_url(const char* url); bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue); // implement in s3fs.cpp bool make_md5_from_binary(const char* pstr, size_t length, std::string& md5); std::string url_to_host(const std::string &url); std::string get_bucket_host(); const char* getCurlDebugHead(curl_infotype type); bool etag_equals(const std::string& s1, const std::string& s2); #endif // S3FS_CURL_UTIL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache.cpp000066400000000000000000001101161470675423500161530ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include "fdcache.h" #include "fdcache_stat.h" #include "s3fs_util.h" #include "s3fs_logger.h" #include "s3fs_cred.h" #include "string_util.h" // // The following symbols are used by FdManager::RawCheckAllCache(). // // These must be #defines due to string literal concatenation. #define CACHEDBG_FMT_HEAD "---------------------------------------------------------------------------\n" \ "Check cache file and its stats file consistency at %s\n" \ "---------------------------------------------------------------------------" #define CACHEDBG_FMT_FOOT "---------------------------------------------------------------------------\n" \ "Summary - Total files: %d\n" \ " Detected error files: %d\n" \ " Detected error directories: %d\n" \ "---------------------------------------------------------------------------" #define CACHEDBG_FMT_FILE_OK "File: %s%s -> [OK] no problem" #define CACHEDBG_FMT_FILE_PROB "File: %s%s" #define CACHEDBG_FMT_DIR_PROB "Directory: %s" #define CACHEDBG_FMT_ERR_HEAD " -> [E] there is a mark that data exists in stats, but there is no data in the cache file." #define CACHEDBG_FMT_WARN_HEAD " -> [W] These show no data in stats, but there is evidence of data in the cache file(no problem)." #define CACHEDBG_FMT_WARN_OPEN "\n -> [W] This file is currently open and may not provide accurate analysis results." #define CACHEDBG_FMT_CRIT_HEAD " -> [C] %s" #define CACHEDBG_FMT_CRIT_HEAD2 " -> [C] " #define CACHEDBG_FMT_PROB_BLOCK " 0x%016zx(0x%016zx bytes)" // [NOTE] // NOCACHE_PATH_PREFIX symbol needs for not using cache mode. // Now s3fs I/F functions in s3fs.cpp has left the processing // to FdManager and FdEntity class. FdManager class manages // the list of local file stat and file descriptor in conjunction // with the FdEntity class. // When s3fs is not using local cache, it means FdManager must // return new temporary file descriptor at each opening it. // Then FdManager caches fd by key which is dummy file path // instead of real file path. // This process may not be complete, but it is easy way can // be realized. // static constexpr char NOCACHE_PATH_PREFIX_FORM[] = " __S3FS_UNEXISTED_PATH_%lx__ / "; // important space words for simply //------------------------------------------------ // FdManager class variable //------------------------------------------------ FdManager FdManager::singleton; std::mutex FdManager::fd_manager_lock; std::mutex FdManager::cache_cleanup_lock; std::mutex FdManager::reserved_diskspace_lock; std::mutex FdManager::except_entmap_lock; std::string FdManager::cache_dir; bool FdManager::check_cache_dir_exist(false); off_t FdManager::free_disk_space = 0; off_t FdManager::fake_used_disk_space = 0; std::string FdManager::check_cache_output; bool FdManager::checked_lseek(false); bool FdManager::have_lseek_hole(false); std::string FdManager::tmp_dir = "/tmp"; //------------------------------------------------ // FdManager class methods //------------------------------------------------ bool FdManager::SetCacheDir(const char* dir) { if(!dir || '\0' == dir[0]){ cache_dir = ""; }else{ cache_dir = dir; } return true; } bool FdManager::SetCacheCheckOutput(const char* path) { if(!path || '\0' == path[0]){ check_cache_output.clear(); }else{ check_cache_output = path; } return true; } bool FdManager::DeleteCacheDirectory() { if(FdManager::cache_dir.empty()){ return true; } std::string cache_path; if(!FdManager::MakeCachePath(nullptr, cache_path, false)){ return false; } if(!delete_files_in_dir(cache_path.c_str(), true)){ return false; } std::string mirror_path = FdManager::cache_dir + "/." + S3fsCred::GetBucket() + ".mirror"; if(!delete_files_in_dir(mirror_path.c_str(), true)){ return false; } return true; } int FdManager::DeleteCacheFile(const char* path) { S3FS_PRN_INFO3("[path=%s]", SAFESTRPTR(path)); if(!path){ return -EIO; } if(FdManager::cache_dir.empty()){ return 0; } std::string cache_path; if(!FdManager::MakeCachePath(path, cache_path, false)){ return 0; } int result = 0; if(0 != unlink(cache_path.c_str())){ if(ENOENT == errno){ S3FS_PRN_DBG("failed to delete file(%s): errno=%d", path, errno); }else{ S3FS_PRN_ERR("failed to delete file(%s): errno=%d", path, errno); } return -errno; } if(0 != (result = CacheFileStat::DeleteCacheFileStat(path))){ if(-ENOENT == result){ S3FS_PRN_DBG("failed to delete stat file(%s): errno=%d", path, result); }else{ S3FS_PRN_ERR("failed to delete stat file(%s): errno=%d", path, result); } } return result; } bool FdManager::MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir, bool is_mirror_path) { if(FdManager::cache_dir.empty()){ cache_path = ""; return true; } std::string resolved_path(FdManager::cache_dir); if(!is_mirror_path){ resolved_path += "/"; resolved_path += S3fsCred::GetBucket(); }else{ resolved_path += "/."; resolved_path += S3fsCred::GetBucket(); resolved_path += ".mirror"; } if(is_create_dir){ int result; if(0 != (result = mkdirp(resolved_path + mydirname(path), 0777))){ S3FS_PRN_ERR("failed to create dir(%s) by errno(%d).", path, result); return false; } } if(!path || '\0' == path[0]){ cache_path = resolved_path; }else{ cache_path = resolved_path + SAFESTRPTR(path); } return true; } bool FdManager::CheckCacheTopDir() { if(FdManager::cache_dir.empty()){ return true; } std::string toppath(FdManager::cache_dir + "/" + S3fsCred::GetBucket()); return check_exist_dir_permission(toppath.c_str()); } bool FdManager::MakeRandomTempPath(const char* path, std::string& tmppath) { char szBuff[64]; snprintf(szBuff, sizeof(szBuff), NOCACHE_PATH_PREFIX_FORM, random()); // worry for performance, but maybe don't worry. szBuff[sizeof(szBuff) - 1] = '\0'; // for safety tmppath = szBuff; tmppath += path ? path : ""; return true; } bool FdManager::SetCheckCacheDirExist(bool is_check) { bool old = FdManager::check_cache_dir_exist; FdManager::check_cache_dir_exist = is_check; return old; } bool FdManager::CheckCacheDirExist() { if(!FdManager::check_cache_dir_exist){ return true; } if(FdManager::cache_dir.empty()){ return true; } return IsDir(cache_dir); } off_t FdManager::GetEnsureFreeDiskSpaceHasLock() { return FdManager::free_disk_space; } off_t FdManager::SetEnsureFreeDiskSpace(off_t size) { const std::lock_guard lock(FdManager::reserved_diskspace_lock); off_t old = FdManager::free_disk_space; FdManager::free_disk_space = size; return old; } bool FdManager::InitFakeUsedDiskSize(off_t fake_freesize) { const std::lock_guard lock(FdManager::reserved_diskspace_lock); FdManager::fake_used_disk_space = 0; // At first, clear this value because this value is used in GetFreeDiskSpaceHasLock. off_t actual_freesize = FdManager::GetFreeDiskSpaceHasLock(nullptr); if(fake_freesize < actual_freesize){ FdManager::fake_used_disk_space = actual_freesize - fake_freesize; }else{ FdManager::fake_used_disk_space = 0; } return true; } off_t FdManager::GetTotalDiskSpaceByRatio(int ratio) { return FdManager::GetTotalDiskSpace(nullptr) * ratio / 100; } off_t FdManager::GetTotalDiskSpace(const char* path) { struct statvfs vfsbuf; int result = FdManager::GetVfsStat(path, &vfsbuf); if(result == -1){ return 0; } off_t actual_totalsize = vfsbuf.f_blocks * vfsbuf.f_frsize; return actual_totalsize; } off_t FdManager::GetFreeDiskSpaceHasLock(const char* path) { struct statvfs vfsbuf; int result = FdManager::GetVfsStat(path, &vfsbuf); if(result == -1){ return 0; } off_t actual_freesize = vfsbuf.f_bavail * vfsbuf.f_frsize; return (FdManager::fake_used_disk_space < actual_freesize ? (actual_freesize - FdManager::fake_used_disk_space) : 0); } int FdManager::GetVfsStat(const char* path, struct statvfs* vfsbuf){ std::string ctoppath; if(!FdManager::cache_dir.empty()){ ctoppath = FdManager::cache_dir + "/"; ctoppath = get_exist_directory_path(ctoppath); // existed directory if(ctoppath != "/"){ ctoppath += "/"; } }else{ ctoppath = tmp_dir + "/"; } if(path && '\0' != *path){ ctoppath += path; }else{ ctoppath += "."; } if(-1 == statvfs(ctoppath.c_str(), vfsbuf)){ S3FS_PRN_ERR("could not get vfs stat by errno(%d)", errno); return -1; } return 0; } bool FdManager::IsSafeDiskSpace(const char* path, off_t size, bool withmsg) { const std::lock_guard lock(FdManager::reserved_diskspace_lock); off_t fsize = FdManager::GetFreeDiskSpaceHasLock(path); off_t needsize = size + FdManager::GetEnsureFreeDiskSpaceHasLock(); if(fsize < needsize){ if(withmsg){ S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by s3fs. Requires %.3f MB, already has %.3f MB.", static_cast(needsize) / 1024 / 1024, static_cast(fsize) / 1024 / 1024); } return false; } return true; } bool FdManager::HaveLseekHole() { if(FdManager::checked_lseek){ return FdManager::have_lseek_hole; } // create temporary file int fd; auto ptmpfp = MakeTempFile(); if(nullptr == ptmpfp || -1 == (fd = fileno(ptmpfp.get()))){ S3FS_PRN_ERR("failed to open temporary file by errno(%d)", errno); FdManager::checked_lseek = true; FdManager::have_lseek_hole = false; return false; } // check SEEK_DATA/SEEK_HOLE options bool result = true; if(-1 == lseek(fd, 0, SEEK_DATA)){ if(EINVAL == errno){ S3FS_PRN_ERR("lseek does not support SEEK_DATA"); result = false; } } if(result && -1 == lseek(fd, 0, SEEK_HOLE)){ if(EINVAL == errno){ S3FS_PRN_ERR("lseek does not support SEEK_HOLE"); result = false; } } FdManager::checked_lseek = true; FdManager::have_lseek_hole = result; return FdManager::have_lseek_hole; } bool FdManager::SetTmpDir(const char *dir) { if(!dir || '\0' == dir[0]){ tmp_dir = "/tmp"; }else{ tmp_dir = dir; } return true; } bool FdManager::IsDir(const std::string& dir) { // check the directory struct stat st; if(0 != stat(dir.c_str(), &st)){ S3FS_PRN_ERR("could not stat() directory %s by errno(%d).", dir.c_str(), errno); return false; } if(!S_ISDIR(st.st_mode)){ S3FS_PRN_ERR("the directory %s is not a directory.", dir.c_str()); return false; } return true; } bool FdManager::CheckTmpDirExist() { if(FdManager::tmp_dir.empty()){ return true; } return IsDir(tmp_dir); } std::unique_ptr FdManager::MakeTempFile() { int fd; char cfn[PATH_MAX]; std::string fn = tmp_dir + "/s3fstmp.XXXXXX"; strncpy(cfn, fn.c_str(), sizeof(cfn) - 1); cfn[sizeof(cfn) - 1] = '\0'; fd = mkstemp(cfn); if (-1 == fd) { S3FS_PRN_ERR("failed to create tmp file. errno(%d)", errno); return {nullptr, &s3fs_fclose}; } if (-1 == unlink(cfn)) { S3FS_PRN_ERR("failed to delete tmp file. errno(%d)", errno); return {nullptr, &s3fs_fclose}; } return {fdopen(fd, "rb+"), &s3fs_fclose}; } bool FdManager::HasOpenEntityFd(const char* path) { const std::lock_guard lock(FdManager::fd_manager_lock); const FdEntity* ent; int fd = -1; if(nullptr == (ent = FdManager::singleton.GetFdEntityHasLock(path, fd, false))){ return false; } return (0 < ent->GetOpenCount()); } // [NOTE] // Returns the number of open pseudo fd. // int FdManager::GetOpenFdCount(const char* path) { const std::lock_guard lock(FdManager::fd_manager_lock); return FdManager::singleton.GetPseudoFdCount(path); } //------------------------------------------------ // FdManager methods //------------------------------------------------ FdManager::FdManager() { if(this != FdManager::get()){ abort(); } } FdManager::~FdManager() { if(this == FdManager::get()){ for(auto iter = fent.cbegin(); fent.cend() != iter; ++iter){ FdEntity* ent = (*iter).second.get(); S3FS_PRN_WARN("To exit with the cache file opened: path=%s, refcnt=%d", ent->GetPath().c_str(), ent->GetOpenCount()); } fent.clear(); except_fent.clear(); }else{ abort(); } } FdEntity* FdManager::GetFdEntityHasLock(const char* path, int& existfd, bool newfd) { S3FS_PRN_INFO3("[path=%s][pseudo_fd=%d]", SAFESTRPTR(path), existfd); if(!path || '\0' == path[0]){ return nullptr; } UpdateEntityToTempPath(); auto fiter = fent.find(path); if(fent.cend() != fiter && fiter->second){ if(-1 == existfd){ if(newfd){ existfd = fiter->second->OpenPseudoFd(O_RDWR); // [NOTE] O_RDWR flags } return fiter->second.get(); }else{ if(fiter->second->FindPseudoFd(existfd)){ if(newfd){ existfd = fiter->second->Dup(existfd); } return fiter->second.get(); } } } if(-1 != existfd){ for(auto iter = fent.cbegin(); iter != fent.cend(); ++iter){ if(iter->second && iter->second->FindPseudoFd(existfd)){ // found opened fd in map if(iter->second->GetPath() == path){ if(newfd){ existfd = iter->second->Dup(existfd); } return iter->second.get(); } // found fd, but it is used another file(file descriptor is recycled) // so returns nullptr. break; } } } // If the cache directory is not specified, s3fs opens a temporary file // when the file is opened. if(!FdManager::IsCacheDir()){ for(auto iter = fent.cbegin(); iter != fent.cend(); ++iter){ if(iter->second && iter->second->IsOpen() && iter->second->GetPath() == path){ return iter->second.get(); } } } return nullptr; } FdEntity* FdManager::Open(int& fd, const char* path, const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags, bool force_tmpfile, bool is_create, bool ignore_modify) { S3FS_PRN_DBG("[path=%s][size=%lld][ts_mctime=%s][flags=0x%x][force_tmpfile=%s][create=%s][ignore_modify=%s]", SAFESTRPTR(path), static_cast(size), str(ts_mctime).c_str(), flags, (force_tmpfile ? "yes" : "no"), (is_create ? "yes" : "no"), (ignore_modify ? "yes" : "no")); if(!path || '\0' == path[0]){ return nullptr; } const std::lock_guard lock(FdManager::fd_manager_lock); UpdateEntityToTempPath(); // search in mapping by key(path) auto iter = fent.find(path); if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){ // If the cache directory is not specified, s3fs opens a temporary file // when the file is opened. // Then if it could not find a entity in map for the file, s3fs should // search a entity in all which opened the temporary file. // for(iter = fent.begin(); iter != fent.end(); ++iter){ if(iter->second && iter->second->IsOpen() && iter->second->GetPath() == path){ break; // found opened fd in mapping } } } if(fent.end() != iter){ // found FdEntity* ent = iter->second.get(); // [NOTE] // If the file is being modified and ignore_modify flag is false, // the file size will not be changed even if there is a request // to reduce the size of the modified file. // If you do, the "test_open_second_fd" test will fail. // if(!ignore_modify && ent->IsModified()){ // If the file is being modified and it's size is larger than size parameter, it will not be resized. off_t cur_size = 0; if(ent->GetSize(cur_size) && size <= cur_size){ size = -1; } } // (re)open if(0 > (fd = ent->Open(pmeta, size, ts_mctime, flags))){ S3FS_PRN_ERR("failed to (re)open and create new pseudo fd for path(%s).", path); return nullptr; } return ent; }else if(is_create){ // not found std::string cache_path; if(!force_tmpfile && !FdManager::MakeCachePath(path, cache_path, true)){ S3FS_PRN_ERR("failed to make cache path for object(%s).", path); return nullptr; } // make new obj auto ent = std::make_shared(path, cache_path.c_str()); // open if(0 > (fd = ent->Open(pmeta, size, ts_mctime, flags))){ S3FS_PRN_ERR("failed to open and create new pseudo fd for path(%s) errno:%d.", path, fd); return nullptr; } if(!cache_path.empty()){ // using cache return (fent[path] = std::move(ent)).get(); }else{ // not using cache, so the key of fdentity is set not really existing path. // (but not strictly unexisting path.) // // [NOTE] // The reason why this process here, please look at the definition of the // comments of NOCACHE_PATH_PREFIX_FORM symbol. // std::string tmppath; FdManager::MakeRandomTempPath(path, tmppath); return (fent[tmppath] = std::move(ent)).get(); } }else{ return nullptr; } } // [NOTE] // This method does not create a new pseudo fd. // It just finds existfd and returns the corresponding entity. // FdEntity* FdManager::GetExistFdEntity(const char* path, int existfd) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d]", SAFESTRPTR(path), existfd); const std::lock_guard lock(FdManager::fd_manager_lock); UpdateEntityToTempPath(); // search from all entity. for(auto iter = fent.cbegin(); iter != fent.cend(); ++iter){ if(iter->second && iter->second->FindPseudoFd(existfd)){ // found existfd in entity return iter->second.get(); } } // not found entity return nullptr; } FdEntity* FdManager::OpenExistFdEntity(const char* path, int& fd, int flags) { S3FS_PRN_DBG("[path=%s][flags=0x%x]", SAFESTRPTR(path), flags); // search entity by path, and create pseudo fd FdEntity* ent = Open(fd, path, nullptr, -1, S3FS_OMIT_TS, flags, false, false, false); if(!ent){ // Not found entity return nullptr; } return ent; } int FdManager::GetPseudoFdCount(const char* path) { S3FS_PRN_DBG("[path=%s]", SAFESTRPTR(path)); if(!path || '\0' == path[0]){ return 0; } UpdateEntityToTempPath(); // search from all entity. for(auto iter = fent.cbegin(); iter != fent.cend(); ++iter){ if(iter->second && iter->second->GetPath() == path){ // found the entity for the path return iter->second->GetOpenCount(); } } // not found entity return 0; } void FdManager::Rename(const std::string &from, const std::string &to) { const std::lock_guard lock(FdManager::fd_manager_lock); UpdateEntityToTempPath(); auto iter = fent.find(from); if(fent.end() == iter && !FdManager::IsCacheDir()){ // If the cache directory is not specified, s3fs opens a temporary file // when the file is opened. // Then if it could not find a entity in map for the file, s3fs should // search a entity in all which opened the temporary file. // for(iter = fent.begin(); iter != fent.end(); ++iter){ if(iter->second && iter->second->IsOpen() && iter->second->GetPath() == from){ break; // found opened fd in mapping } } } if(fent.end() != iter){ // found S3FS_PRN_DBG("[from=%s][to=%s]", from.c_str(), to.c_str()); auto ent(std::move(iter->second)); // retrieve old fd entity from map fent.erase(iter); // rename path and caches in fd entity std::string fentmapkey; if(!ent->RenamePath(to, fentmapkey)){ S3FS_PRN_ERR("Failed to rename FdEntity object for %s to %s", from.c_str(), to.c_str()); return; } // set new fd entity to map fent[fentmapkey] = std::move(ent); } } bool FdManager::Close(FdEntity* ent, int fd) { S3FS_PRN_DBG("[ent->file=%s][pseudo_fd=%d]", ent ? ent->GetPath().c_str() : "", fd); if(!ent || -1 == fd){ return true; // returns success } const std::lock_guard lock(FdManager::fd_manager_lock); UpdateEntityToTempPath(); for(auto iter = fent.cbegin(); iter != fent.cend(); ++iter){ if(iter->second.get() == ent){ ent->Close(fd); if(!ent->IsOpen()){ // remove found entity from map. iter = fent.erase(iter); // check another key name for entity value to be on the safe side for(; iter != fent.cend(); ){ if(iter->second.get() == ent){ iter = fent.erase(iter); }else{ ++iter; } } } return true; } } return false; } bool FdManager::ChangeEntityToTempPath(std::shared_ptr ent, const char* path) { const std::lock_guard lock(FdManager::except_entmap_lock); except_fent[path] = std::move(ent); return true; } bool FdManager::UpdateEntityToTempPath() { const std::lock_guard lock(FdManager::except_entmap_lock); for(auto except_iter = except_fent.cbegin(); except_iter != except_fent.cend(); ){ std::string tmppath; FdManager::MakeRandomTempPath(except_iter->first.c_str(), tmppath); auto iter = fent.find(except_iter->first); if(fent.cend() != iter && iter->second.get() == except_iter->second.get()){ // Move the entry to the new key fent[tmppath] = std::move(iter->second); fent.erase(iter); except_iter = except_fent.erase(except_iter); }else{ // [NOTE] // ChangeEntityToTempPath method is called and the FdEntity pointer // set into except_fent is mapped into fent. // And since this method is always called before manipulating fent, // it will not enter here. // Thus, if it enters here, a warning is output. // S3FS_PRN_WARN("For some reason the FdEntity pointer(for %s) is not found in the fent map. Recovery procedures are being performed, but the cause needs to be identified.", except_iter->first.c_str()); // Add the entry for recovery procedures fent[tmppath] = except_iter->second; except_iter = except_fent.erase(except_iter); } } return true; } void FdManager::CleanupCacheDir() { //S3FS_PRN_DBG("cache cleanup requested"); if(!FdManager::IsCacheDir()){ return; } if(FdManager::cache_cleanup_lock.try_lock()){ //S3FS_PRN_DBG("cache cleanup started"); CleanupCacheDirInternal(""); //S3FS_PRN_DBG("cache cleanup ended"); }else{ // wait for other thread to finish cache cleanup FdManager::cache_cleanup_lock.lock(); } FdManager::cache_cleanup_lock.unlock(); } void FdManager::CleanupCacheDirInternal(const std::string &path) { DIR* dp; struct dirent* dent; std::string abs_path = cache_dir + "/" + S3fsCred::GetBucket() + path; if(nullptr == (dp = opendir(abs_path.c_str()))){ S3FS_PRN_ERR("could not open cache dir(%s) - errno(%d)", abs_path.c_str(), errno); return; } for(dent = readdir(dp); dent; dent = readdir(dp)){ if(0 == strcmp(dent->d_name, "..") || 0 == strcmp(dent->d_name, ".")){ continue; } std::string fullpath = abs_path; fullpath += "/"; fullpath += dent->d_name; struct stat st; if(0 != lstat(fullpath.c_str(), &st)){ S3FS_PRN_ERR("could not get stats of file(%s) - errno(%d)", fullpath.c_str(), errno); closedir(dp); return; } std::string next_path = path + "/" + dent->d_name; if(S_ISDIR(st.st_mode)){ CleanupCacheDirInternal(next_path); }else{ if(!FdManager::fd_manager_lock.try_lock()){ S3FS_PRN_INFO("could not get fd_manager_lock when clean up file(%s), then skip it.", next_path.c_str()); continue; } UpdateEntityToTempPath(); auto iter = fent.find(next_path); if(fent.cend() == iter) { S3FS_PRN_DBG("cleaned up: %s", next_path.c_str()); FdManager::DeleteCacheFile(next_path.c_str()); } FdManager::fd_manager_lock.unlock(); } } closedir(dp); } bool FdManager::ReserveDiskSpace(off_t size) { if(IsSafeDiskSpace(nullptr, size)){ const std::lock_guard lock(FdManager::reserved_diskspace_lock); FdManager::free_disk_space += size; return true; } return false; } void FdManager::FreeReservedDiskSpace(off_t size) { const std::lock_guard lock(FdManager::reserved_diskspace_lock); FdManager::free_disk_space -= size; } // // Inspect all files for stats file for cache file // // [NOTE] // The minimum sub_path parameter is "/". // The sub_path is a directory path starting from "/" and ending with "/". // // This method produces the following output. // // * Header // ------------------------------------------------------------ // Check cache file and its stats file consistency // ------------------------------------------------------------ // * When the cache file and its stats information match // File path: -> [OK] no problem // // * If there is a problem with the cache file and its stats information // File path: // -> [P] // -> [E] there is a mark that data exists in stats, but there is no data in the cache file. // (bytes) // ... // ... // -> [W] These show no data in stats, but there is evidence of data in the cache file.(no problem.) // (bytes) // ... // ... // bool FdManager::RawCheckAllCache(FILE* fp, const char* cache_stat_top_dir, const char* sub_path, int& total_file_cnt, int& err_file_cnt, int& err_dir_cnt) { if(!cache_stat_top_dir || '\0' == cache_stat_top_dir[0] || !sub_path || '\0' == sub_path[0]){ S3FS_PRN_ERR("Parameter cache_stat_top_dir is empty."); return false; } // open directory of cache file's stats DIR* statsdir; std::string target_dir = cache_stat_top_dir; target_dir += sub_path; if(nullptr == (statsdir = opendir(target_dir.c_str()))){ S3FS_PRN_ERR("Could not open directory(%s) by errno(%d)", target_dir.c_str(), errno); return false; } // loop in directory of cache file's stats const struct dirent* pdirent = nullptr; while(nullptr != (pdirent = readdir(statsdir))){ if(DT_DIR == pdirent->d_type){ // found directory if(0 == strcmp(pdirent->d_name, ".") || 0 == strcmp(pdirent->d_name, "..")){ continue; } // reentrant for sub directory std::string subdir_path = sub_path; subdir_path += pdirent->d_name; subdir_path += '/'; if(!RawCheckAllCache(fp, cache_stat_top_dir, subdir_path.c_str(), total_file_cnt, err_file_cnt, err_dir_cnt)){ // put error message for this dir. ++err_dir_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_DIR_PROB, subdir_path.c_str()); S3FS_PRN_CACHE(fp, CACHEDBG_FMT_CRIT_HEAD, "Something error is occurred in checking this directory"); } }else{ ++total_file_cnt; // make cache file path std::string strOpenedWarn; std::string cache_path; std::string object_file_path = sub_path; object_file_path += pdirent->d_name; if(!FdManager::MakeCachePath(object_file_path.c_str(), cache_path, false, false) || cache_path.empty()){ ++err_file_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_PROB, object_file_path.c_str(), strOpenedWarn.c_str()); S3FS_PRN_CACHE(fp, CACHEDBG_FMT_CRIT_HEAD, "Could not make cache file path"); continue; } // check if the target file is currently in operation. { const std::lock_guard lock(FdManager::fd_manager_lock); UpdateEntityToTempPath(); auto iter = fent.find(object_file_path); if(fent.cend() != iter){ // This file is opened now, then we need to put warning message. strOpenedWarn = CACHEDBG_FMT_WARN_OPEN; } } // open cache file int cache_file_fd; if(-1 == (cache_file_fd = open(cache_path.c_str(), O_RDONLY))){ ++err_file_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_PROB, object_file_path.c_str(), strOpenedWarn.c_str()); S3FS_PRN_CACHE(fp, CACHEDBG_FMT_CRIT_HEAD, "Could not open cache file"); continue; } scope_guard guard([&]() { close(cache_file_fd); }); // get inode number for cache file struct stat st; if(0 != fstat(cache_file_fd, &st)){ ++err_file_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_PROB, object_file_path.c_str(), strOpenedWarn.c_str()); S3FS_PRN_CACHE(fp, CACHEDBG_FMT_CRIT_HEAD, "Could not get file inode number for cache file"); continue; } ino_t cache_file_inode = st.st_ino; // open cache stat file and load page info. PageList pagelist; CacheFileStat cfstat(object_file_path.c_str()); if(!cfstat.ReadOnlyOpen() || !pagelist.Deserialize(cfstat, cache_file_inode)){ ++err_file_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_PROB, object_file_path.c_str(), strOpenedWarn.c_str()); S3FS_PRN_CACHE(fp, CACHEDBG_FMT_CRIT_HEAD, "Could not load cache file stats information"); continue; } cfstat.Release(); // compare cache file size and stats information if(st.st_size != pagelist.Size()){ ++err_file_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_PROB, object_file_path.c_str(), strOpenedWarn.c_str()); S3FS_PRN_CACHE(fp, CACHEDBG_FMT_CRIT_HEAD2 "The cache file size(%lld) and the value(%lld) from cache file stats are different", static_cast(st.st_size), static_cast(pagelist.Size())); continue; } // compare cache file stats and cache file blocks fdpage_list_t err_area_list; fdpage_list_t warn_area_list; if(!pagelist.CompareSparseFile(cache_file_fd, st.st_size, err_area_list, warn_area_list)){ // Found some error or warning S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_PROB, object_file_path.c_str(), strOpenedWarn.c_str()); if(!warn_area_list.empty()){ S3FS_PRN_CACHE(fp, CACHEDBG_FMT_WARN_HEAD); for(auto witer = warn_area_list.cbegin(); witer != warn_area_list.cend(); ++witer){ S3FS_PRN_CACHE(fp, CACHEDBG_FMT_PROB_BLOCK, static_cast(witer->offset), static_cast(witer->bytes)); } } if(!err_area_list.empty()){ ++err_file_cnt; S3FS_PRN_CACHE(fp, CACHEDBG_FMT_ERR_HEAD); for(auto eiter = err_area_list.cbegin(); eiter != err_area_list.cend(); ++eiter){ S3FS_PRN_CACHE(fp, CACHEDBG_FMT_PROB_BLOCK, static_cast(eiter->offset), static_cast(eiter->bytes)); } } }else{ // There is no problem! if(!strOpenedWarn.empty()){ strOpenedWarn += "\n "; } S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FILE_OK, object_file_path.c_str(), strOpenedWarn.c_str()); } err_area_list.clear(); warn_area_list.clear(); } } closedir(statsdir); return true; } bool FdManager::CheckAllCache() { if(!FdManager::HaveLseekHole()){ S3FS_PRN_ERR("lseek does not support SEEK_DATA/SEEK_HOLE, then could not check cache."); return false; } std::unique_ptr pfp(nullptr, &s3fs_fclose); FILE* fp; if(FdManager::check_cache_output.empty()){ fp = stdout; }else{ pfp.reset(fp = fopen(FdManager::check_cache_output.c_str(), "a+")); if(nullptr == pfp){ S3FS_PRN_ERR("Could not open(create) output file(%s) for checking all cache by errno(%d)", FdManager::check_cache_output.c_str(), errno); return false; } } // print head message S3FS_PRN_CACHE(fp, CACHEDBG_FMT_HEAD, S3fsLog::GetCurrentTime().c_str()); // Loop in directory of cache file's stats std::string top_path = CacheFileStat::GetCacheFileStatTopDir(); int total_file_cnt = 0; int err_file_cnt = 0; int err_dir_cnt = 0; bool result = RawCheckAllCache(fp, top_path.c_str(), "/", total_file_cnt, err_file_cnt, err_dir_cnt); if(!result){ S3FS_PRN_ERR("Processing failed due to some problem."); } // print foot message S3FS_PRN_CACHE(fp, CACHEDBG_FMT_FOOT, total_file_cnt, err_file_cnt, err_dir_cnt); return result; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache.h000066400000000000000000000140511470675423500156210ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_H_ #define S3FS_FDCACHE_H_ #include #include #include "common.h" #include "fdcache_entity.h" #include "s3fs_util.h" //------------------------------------------------ // class FdManager //------------------------------------------------ class FdManager { private: static FdManager singleton; static std::mutex fd_manager_lock; static std::mutex cache_cleanup_lock; static std::mutex reserved_diskspace_lock; static std::mutex except_entmap_lock; static std::string cache_dir; static bool check_cache_dir_exist; static off_t free_disk_space GUARDED_BY(reserved_diskspace_lock); // limit free disk space static off_t fake_used_disk_space GUARDED_BY(reserved_diskspace_lock); // difference between fake free disk space and actual at startup(for test/debug) static std::string check_cache_output; static bool checked_lseek; static bool have_lseek_hole; static std::string tmp_dir; fdent_map_t fent GUARDED_BY(fd_manager_lock); fdent_map_t except_fent GUARDED_BY(except_entmap_lock); // A map of delayed deletion fdentity private: static off_t GetFreeDiskSpaceHasLock(const char* path) REQUIRES(FdManager::reserved_diskspace_lock); static off_t GetTotalDiskSpace(const char* path); static bool IsDir(const std::string& dir); static int GetVfsStat(const char* path, struct statvfs* vfsbuf); static off_t GetEnsureFreeDiskSpaceHasLock() REQUIRES(FdManager::reserved_diskspace_lock); // Returns the number of open pseudo fd. int GetPseudoFdCount(const char* path) REQUIRES(fd_manager_lock); bool UpdateEntityToTempPath() REQUIRES(fd_manager_lock); void CleanupCacheDirInternal(const std::string &path = "") REQUIRES(cache_cleanup_lock); bool RawCheckAllCache(FILE* fp, const char* cache_stat_top_dir, const char* sub_path, int& total_file_cnt, int& err_file_cnt, int& err_dir_cnt); public: FdManager(); ~FdManager(); FdManager(const FdManager&) = delete; FdManager(FdManager&&) = delete; FdManager& operator=(const FdManager&) = delete; FdManager& operator=(FdManager&&) = delete; // Reference singleton static FdManager* get() { return &singleton; } static bool DeleteCacheDirectory(); static int DeleteCacheFile(const char* path); static bool SetCacheDir(const char* dir); static bool IsCacheDir() { return !FdManager::cache_dir.empty(); } static const char* GetCacheDir() { return FdManager::cache_dir.c_str(); } static bool SetCacheCheckOutput(const char* path); static const char* GetCacheCheckOutput() { return FdManager::check_cache_output.c_str(); } static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true, bool is_mirror_path = false); static bool CheckCacheTopDir(); static bool MakeRandomTempPath(const char* path, std::string& tmppath); static bool SetCheckCacheDirExist(bool is_check); static bool CheckCacheDirExist(); static bool HasOpenEntityFd(const char* path); static int GetOpenFdCount(const char* path); static off_t GetEnsureFreeDiskSpace() { const std::lock_guard lock(FdManager::reserved_diskspace_lock); return FdManager::GetEnsureFreeDiskSpaceHasLock(); } static off_t SetEnsureFreeDiskSpace(off_t size); static bool InitFakeUsedDiskSize(off_t fake_freesize); static bool IsSafeDiskSpace(const char* path, off_t size, bool withmsg = false); static void FreeReservedDiskSpace(off_t size); static bool ReserveDiskSpace(off_t size); static bool HaveLseekHole(); static bool SetTmpDir(const char* dir); static bool CheckTmpDirExist(); static std::unique_ptr MakeTempFile(); static off_t GetTotalDiskSpaceByRatio(int ratio); // Return FdEntity associated with path, returning nullptr on error. This operation increments the reference count; callers must decrement via Close after use. FdEntity* GetFdEntity(const char* path, int& existfd, bool newfd = true) { const std::lock_guard lock(FdManager::fd_manager_lock); return GetFdEntityHasLock(path, existfd, newfd); } FdEntity* GetFdEntityHasLock(const char* path, int& existfd, bool newfd = true) REQUIRES(FdManager::fd_manager_lock); FdEntity* Open(int& fd, const char* path, const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags, bool force_tmpfile, bool is_create, bool ignore_modify); FdEntity* GetExistFdEntity(const char* path, int existfd = -1); FdEntity* OpenExistFdEntity(const char* path, int& fd, int flags = O_RDONLY); void Rename(const std::string &from, const std::string &to); bool Close(FdEntity* ent, int fd); bool ChangeEntityToTempPath(std::shared_ptr ent, const char* path); void CleanupCacheDir(); bool CheckAllCache(); }; #endif // S3FS_FDCACHE_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_auto.cpp000066400000000000000000000063551470675423500172140ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include "s3fs_logger.h" #include "fdcache_auto.h" #include "fdcache.h" //------------------------------------------------ // AutoFdEntity methods //------------------------------------------------ AutoFdEntity::AutoFdEntity() : pFdEntity(nullptr), pseudo_fd(-1) { } AutoFdEntity::~AutoFdEntity() { Close(); } bool AutoFdEntity::Close() { if(pFdEntity){ if(!FdManager::get()->Close(pFdEntity, pseudo_fd)){ S3FS_PRN_ERR("Failed to close fdentity."); return false; } pFdEntity = nullptr; pseudo_fd = -1; } return true; } // [NOTE] // This method touches the internal fdentity with. // This is used to keep the file open. // int AutoFdEntity::Detach() { if(!pFdEntity){ S3FS_PRN_ERR("Does not have a associated FdEntity."); return -1; } int fd = pseudo_fd; pseudo_fd = -1; pFdEntity = nullptr; return fd; } FdEntity* AutoFdEntity::Attach(const char* path, int existfd) { Close(); if(nullptr == (pFdEntity = FdManager::get()->GetFdEntity(path, existfd, false))){ S3FS_PRN_DBG("Could not find fd entity object(file=%s, pseudo_fd=%d)", path, existfd); return nullptr; } pseudo_fd = existfd; return pFdEntity; } FdEntity* AutoFdEntity::Open(const char* path, const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, int* error) { Close(); if(nullptr == (pFdEntity = FdManager::get()->Open(pseudo_fd, path, pmeta, size, ts_mctime, flags, force_tmpfile, is_create, ignore_modify))){ if(error){ *error = pseudo_fd; } pseudo_fd = -1; return nullptr; } return pFdEntity; } // [NOTE] // the fd obtained by this method is not a newly created pseudo fd. // FdEntity* AutoFdEntity::GetExistFdEntity(const char* path, int existfd) { Close(); FdEntity* ent; if(nullptr == (ent = FdManager::get()->GetExistFdEntity(path, existfd))){ return nullptr; } return ent; } FdEntity* AutoFdEntity::OpenExistFdEntity(const char* path, int flags) { Close(); if(nullptr == (pFdEntity = FdManager::get()->OpenExistFdEntity(path, pseudo_fd, flags))){ return nullptr; } return pFdEntity; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_auto.h000066400000000000000000000044601470675423500166540ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_AUTO_H_ #define S3FS_FDCACHE_AUTO_H_ #include #include "common.h" #include "metaheader.h" class FdEntity; //------------------------------------------------ // class AutoFdEntity //------------------------------------------------ // A class that opens fdentry and closes it automatically. // This class object is used to prevent inconsistencies in // the number of references in fdentry. // The methods are wrappers to the method of the FdManager class. // class AutoFdEntity { private: FdEntity* pFdEntity; int pseudo_fd; public: AutoFdEntity(); ~AutoFdEntity(); AutoFdEntity(const AutoFdEntity&) = delete; AutoFdEntity(AutoFdEntity&&) = delete; AutoFdEntity& operator=(const AutoFdEntity&) = delete; AutoFdEntity& operator=(AutoFdEntity&&) = delete; bool Close(); int Detach(); FdEntity* Attach(const char* path, int existfd); int GetPseudoFd() const { return pseudo_fd; } FdEntity* Open(const char* path, const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, int* error = nullptr); FdEntity* GetExistFdEntity(const char* path, int existfd = -1); FdEntity* OpenExistFdEntity(const char* path, int flags = O_RDONLY); }; #endif // S3FS_FDCACHE_AUTO_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_entity.cpp000066400000000000000000002623341470675423500175610ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include "common.h" #include "fdcache_entity.h" #include "fdcache_fdinfo.h" #include "fdcache_stat.h" #include "fdcache_untreated.h" #include "fdcache.h" #include "string_util.h" #include "s3fs_logger.h" #include "s3fs_util.h" #include "curl.h" #include "s3fs_cred.h" //------------------------------------------------ // Symbols //------------------------------------------------ static constexpr int MAX_MULTIPART_CNT = 10 * 1000; // S3 multipart max count //------------------------------------------------ // FdEntity class variables //------------------------------------------------ bool FdEntity::mixmultipart = true; bool FdEntity::streamupload = false; //------------------------------------------------ // FdEntity class methods //------------------------------------------------ bool FdEntity::SetNoMixMultipart() { bool old = mixmultipart; mixmultipart = false; return old; } bool FdEntity::SetStreamUpload(bool isstream) { bool old = streamupload; streamupload = isstream; return old; } int FdEntity::FillFile(int fd, unsigned char byte, off_t size, off_t start) { unsigned char bytes[1024 * 32]; // 32kb memset(bytes, byte, std::min(static_cast(sizeof(bytes)), size)); for(off_t total = 0, onewrote = 0; total < size; total += onewrote){ if(-1 == (onewrote = pwrite(fd, bytes, std::min(static_cast(sizeof(bytes)), size - total), start + total))){ S3FS_PRN_ERR("pwrite failed. errno(%d)", errno); return -errno; } } return 0; } // [NOTE] // If fd is wrong or something error is occurred, return 0. // The ino_t is allowed zero, but inode 0 is not realistic. // So this method returns 0 on error assuming the correct // inode is never 0. // The caller must have exclusive control. // ino_t FdEntity::GetInode(int fd) { if(-1 == fd){ S3FS_PRN_ERR("file descriptor is wrong."); return 0; } struct stat st; if(0 != fstat(fd, &st)){ S3FS_PRN_ERR("could not get stat for physical file descriptor(%d) by errno(%d).", fd, errno); return 0; } return st.st_ino; } //------------------------------------------------ // FdEntity methods //------------------------------------------------ FdEntity::FdEntity(const char* tpath, const char* cpath) : path(SAFESTRPTR(tpath)), physical_fd(-1), inode(0), size_orgmeta(0), cachepath(SAFESTRPTR(cpath)), pending_status(pending_status_t::NO_UPDATE_PENDING) { holding_mtime.tv_sec = -1; holding_mtime.tv_nsec = 0; } FdEntity::~FdEntity() { Clear(); } void FdEntity::Clear() { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); pseudo_fd_map.clear(); if(-1 != physical_fd){ if(!cachepath.empty()){ // [NOTE] // Compare the inode of the existing cache file with the inode of // the cache file output by this object, and if they are the same, // serialize the pagelist. // ino_t cur_inode = GetInode(); if(0 != cur_inode && cur_inode == inode){ CacheFileStat cfstat(path.c_str()); if(!pagelist.Serialize(cfstat, inode)){ S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str()); } } } pfile.reset(); physical_fd = -1; inode = 0; if(!mirrorpath.empty()){ if(-1 == unlink(mirrorpath.c_str())){ S3FS_PRN_WARN("failed to remove mirror cache file(%s) by errno(%d).", mirrorpath.c_str(), errno); } mirrorpath.clear(); } } pagelist.Init(0, false, false); path = ""; cachepath = ""; } // [NOTE] // This method returns the inode of the file in cachepath. // The return value is the same as the class method GetInode(). // The caller must have exclusive control. // ino_t FdEntity::GetInode() const { if(cachepath.empty()){ S3FS_PRN_INFO("cache file path is empty, then return inode as 0."); return 0; } struct stat st; if(0 != stat(cachepath.c_str(), &st)){ S3FS_PRN_INFO("could not get stat for file(%s) by errno(%d).", cachepath.c_str(), errno); return 0; } return st.st_ino; } void FdEntity::Close(int fd) { const std::lock_guard lock(fdent_lock); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d]", path.c_str(), fd, physical_fd); // search pseudo fd and close it. auto iter = pseudo_fd_map.find(fd); if(pseudo_fd_map.cend() != iter){ pseudo_fd_map.erase(iter); }else{ S3FS_PRN_WARN("Not found pseudo_fd(%d) in entity object(%s)", fd, path.c_str()); } // check pseudo fd count if(-1 != physical_fd && 0 == GetOpenCountHasLock()){ const std::lock_guard data_lock(fdent_data_lock); if(!cachepath.empty()){ // [NOTE] // Compare the inode of the existing cache file with the inode of // the cache file output by this object, and if they are the same, // serialize the pagelist. // ino_t cur_inode = GetInode(); if(0 != cur_inode && cur_inode == inode){ CacheFileStat cfstat(path.c_str()); if(!pagelist.Serialize(cfstat, inode)){ S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str()); } } } pfile.reset(); physical_fd = -1; inode = 0; if(!mirrorpath.empty()){ if(-1 == unlink(mirrorpath.c_str())){ S3FS_PRN_WARN("failed to remove mirror cache file(%s) by errno(%d).", mirrorpath.c_str(), errno); } mirrorpath.clear(); } } } int FdEntity::DupWithLock(int fd) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][pseudo fd count=%zu]", path.c_str(), fd, physical_fd, pseudo_fd_map.size()); if(-1 == physical_fd){ return -1; } auto iter = pseudo_fd_map.find(fd); if(pseudo_fd_map.cend() == iter){ S3FS_PRN_ERR("Not found pseudo_fd(%d) in entity object(%s) for physical_fd(%d)", fd, path.c_str(), physical_fd); return -1; } const PseudoFdInfo* org_pseudoinfo = iter->second.get(); std::unique_ptr ppseudoinfo(new PseudoFdInfo(physical_fd, (org_pseudoinfo ? org_pseudoinfo->GetFlags() : 0))); int pseudo_fd = ppseudoinfo->GetPseudoFd(); pseudo_fd_map[pseudo_fd] = std::move(ppseudoinfo); return pseudo_fd; } int FdEntity::OpenPseudoFd(int flags) { const std::lock_guard lock(fdent_lock); S3FS_PRN_DBG("[path=%s][physical_fd=%d][pseudo fd count=%zu]", path.c_str(), physical_fd, pseudo_fd_map.size()); if(-1 == physical_fd){ return -1; } std::unique_ptr ppseudoinfo(new PseudoFdInfo(physical_fd, flags)); int pseudo_fd = ppseudoinfo->GetPseudoFd(); pseudo_fd_map[pseudo_fd] = std::move(ppseudoinfo); return pseudo_fd; } int FdEntity::GetOpenCountHasLock() const { return static_cast(pseudo_fd_map.size()); } // // Open mirror file which is linked cache file. // int FdEntity::OpenMirrorFile() { if(cachepath.empty()){ S3FS_PRN_ERR("cache path is empty, why come here"); return -EIO; } // make temporary directory std::string bupdir; if(!FdManager::MakeCachePath(nullptr, bupdir, true, true)){ S3FS_PRN_ERR("could not make bup cache directory path or create it."); return -EIO; } // create seed generating mirror file name auto seed = static_cast(time(nullptr)); int urandom_fd; if(-1 != (urandom_fd = open("/dev/urandom", O_RDONLY))){ unsigned int rand_data; if(sizeof(rand_data) == read(urandom_fd, &rand_data, sizeof(rand_data))){ seed ^= rand_data; } close(urandom_fd); } // try to link mirror file while(true){ // make random(temp) file path // (do not care for threading, because allowed any value returned.) // char szfile[NAME_MAX + 1]; snprintf(szfile, sizeof(szfile), "%x.tmp", rand_r(&seed)); szfile[NAME_MAX] = '\0'; // for safety mirrorpath = bupdir + "/" + szfile; // link mirror file to cache file if(0 == link(cachepath.c_str(), mirrorpath.c_str())){ break; } if(EEXIST != errno){ S3FS_PRN_ERR("could not link mirror file(%s) to cache file(%s) by errno(%d).", mirrorpath.c_str(), cachepath.c_str(), errno); return -errno; } ++seed; } // open mirror file int mirrorfd; if(-1 == (mirrorfd = open(mirrorpath.c_str(), O_RDWR))){ S3FS_PRN_ERR("could not open mirror file(%s) by errno(%d).", mirrorpath.c_str(), errno); return -errno; } return mirrorfd; } bool FdEntity::FindPseudoFdWithLock(int fd) const { if(-1 == fd){ return false; } if(pseudo_fd_map.cend() == pseudo_fd_map.find(fd)){ return false; } return true; } PseudoFdInfo* FdEntity::CheckPseudoFdFlags(int fd, bool writable) { if(-1 == fd){ return nullptr; } auto iter = pseudo_fd_map.find(fd); if(pseudo_fd_map.cend() == iter || nullptr == iter->second){ return nullptr; } if(writable){ if(!iter->second->Writable()){ return nullptr; } }else{ if(!iter->second->Readable()){ return nullptr; } } return iter->second.get(); } bool FdEntity::IsUploading() { for(auto iter = pseudo_fd_map.cbegin(); iter != pseudo_fd_map.cend(); ++iter){ const PseudoFdInfo* ppseudoinfo = iter->second.get(); if(ppseudoinfo && ppseudoinfo->IsUploading()){ return true; } } return false; } // [NOTE] // If the open is successful, returns pseudo fd. // If it fails, it returns an error code with a negative value. // // ts_mctime argument is a variable for mtime/ctime. // If you want to disable this variable, specify UTIME_OMIT for // tv_nsec in timespec member(in this case tv_sec member is ignored). // This is similar to utimens operation. // You can use "S3FS_OMIT_TS" global variable for UTIME_OMIT. // int FdEntity::Open(const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags) { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); S3FS_PRN_DBG("[path=%s][physical_fd=%d][size=%lld][ts_mctime=%s][flags=0x%x]", path.c_str(), physical_fd, static_cast(size), str(ts_mctime).c_str(), flags); // [NOTE] // When the file size is incremental by truncating, it must be keeped // as an untreated area, and this area is set to these variables. // off_t truncated_start = 0; off_t truncated_size = 0; if(-1 != physical_fd){ // // already open file // // check only file size(do not need to save cfs and time. if(0 <= size && pagelist.Size() != size){ // truncate temporary file size if(-1 == ftruncate(physical_fd, size) || -1 == fsync(physical_fd)){ S3FS_PRN_ERR("failed to truncate temporary file(physical_fd=%d) by errno(%d).", physical_fd, errno); return -errno; } // resize page list if(!pagelist.Resize(size, false, true)){ // Areas with increased size are modified S3FS_PRN_ERR("failed to truncate temporary file information(physical_fd=%d).", physical_fd); return -EIO; } } // set untreated area if(0 <= size && size_orgmeta < size){ // set untreated area truncated_start = size_orgmeta; truncated_size = size - size_orgmeta; } // set original headers and set size. off_t new_size = (0 <= size ? size : size_orgmeta); if(pmeta){ orgmeta = *pmeta; size_orgmeta = get_size(orgmeta); } size_orgmeta = std::min(new_size, size_orgmeta); }else{ // // file is not opened yet // bool need_save_csf = false; // need to save(reset) cache stat file bool is_truncate = false; // need to truncate std::unique_ptr pcfstat; if(!cachepath.empty()){ // using cache struct stat st; if(stat(cachepath.c_str(), &st) == 0){ if(0 > compare_timespec(st, stat_time_type::MTIME, ts_mctime)){ S3FS_PRN_DBG("cache file stale, removing: %s", cachepath.c_str()); if(unlink(cachepath.c_str()) != 0){ return (0 == errno ? -EIO : -errno); } } } // open cache and cache stat file, load page info. pcfstat.reset(new CacheFileStat(path.c_str())); // try to open cache file if( -1 != (physical_fd = open(cachepath.c_str(), O_RDWR)) && 0 != (inode = FdEntity::GetInode(physical_fd)) && pagelist.Deserialize(*pcfstat, inode)) { // succeed to open cache file and to load stats data st = {}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat is failed. errno(%d)", errno); physical_fd = -1; inode = 0; return (0 == errno ? -EIO : -errno); } // check size, st_size, loading stat file if(-1 == size){ if(st.st_size != pagelist.Size()){ pagelist.Resize(st.st_size, false, true); // Areas with increased size are modified need_save_csf = true; // need to update page info } size = st.st_size; }else{ // First if the current cache file size and pagelist do not match, fix pagelist. if(st.st_size != pagelist.Size()){ pagelist.Resize(st.st_size, false, true); // Areas with increased size are modified need_save_csf = true; // need to update page info } if(size != pagelist.Size()){ pagelist.Resize(size, false, true); // Areas with increased size are modified need_save_csf = true; // need to update page info } if(size != st.st_size){ is_truncate = true; } } }else{ if(-1 != physical_fd){ close(physical_fd); } inode = 0; // could not open cache file or could not load stats data, so initialize it. if(-1 == (physical_fd = open(cachepath.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0600))){ S3FS_PRN_ERR("failed to open file(%s). errno(%d)", cachepath.c_str(), errno); // remove cache stat file if it is existed int result; if(0 != (result = CacheFileStat::DeleteCacheFileStat(path.c_str()))){ if(-ENOENT != result){ S3FS_PRN_WARN("failed to delete current cache stat file(%s) by errno(%d), but continue...", path.c_str(), result); } } return result; } need_save_csf = true; // need to update page info inode = FdEntity::GetInode(physical_fd); if(-1 == size){ size = 0; pagelist.Init(0, false, false); }else{ // [NOTE] // The modify flag must not be set when opening a file, // if the ts_mctime parameter(mtime) is specified(tv_nsec != UTIME_OMIT) // and the cache file does not exist. // If mtime is specified for the file and the cache file // mtime is older than it, the cache file is removed and // the processing comes here. // pagelist.Resize(size, false, (UTIME_OMIT == ts_mctime.tv_nsec ? true : false)); is_truncate = true; } } // open mirror file int mirrorfd; if(0 >= (mirrorfd = OpenMirrorFile())){ S3FS_PRN_ERR("failed to open mirror file linked cache file(%s).", cachepath.c_str()); return (0 == mirrorfd ? -EIO : mirrorfd); } // switch fd close(physical_fd); physical_fd = mirrorfd; // make file pointer(for being same tmpfile) if(nullptr == (pfile = {fdopen(physical_fd, "wb"), &s3fs_fclose})){ S3FS_PRN_ERR("failed to get fileno(%s). errno(%d)", cachepath.c_str(), errno); close(physical_fd); physical_fd = -1; inode = 0; return (0 == errno ? -EIO : -errno); } }else{ // not using cache inode = 0; // open temporary file auto tmpfile = FdManager::MakeTempFile(); if(nullptr == tmpfile || -1 ==(physical_fd = fileno(tmpfile.get()))){ S3FS_PRN_ERR("failed to open temporary file by errno(%d)", errno); return (0 == errno ? -EIO : -errno); } pfile = std::move(tmpfile); if(-1 == size){ size = 0; pagelist.Init(0, false, false); }else{ // [NOTE] // The modify flag must not be set when opening a file, // if the ts_mctime parameter(mtime) is specified(tv_nsec != UTIME_OMIT) // and the cache file does not exist. // If mtime is specified for the file and the cache file // mtime is older than it, the cache file is removed and // the processing comes here. // pagelist.Resize(size, false, (UTIME_OMIT == ts_mctime.tv_nsec ? true : false)); is_truncate = true; } } // truncate cache(tmp) file if(is_truncate){ if(0 != ftruncate(physical_fd, size) || 0 != fsync(physical_fd)){ S3FS_PRN_ERR("ftruncate(%s) or fsync returned err(%d)", cachepath.c_str(), errno); pfile.reset(); physical_fd = -1; inode = 0; return (0 == errno ? -EIO : -errno); } } // reset cache stat file if(need_save_csf && pcfstat.get()){ if(!pagelist.Serialize(*pcfstat, inode)){ S3FS_PRN_WARN("failed to save cache stat file(%s), but continue...", path.c_str()); } } // set original headers and size in it. if(pmeta){ orgmeta = *pmeta; size_orgmeta = get_size(orgmeta); }else{ orgmeta.clear(); size_orgmeta = 0; } // set untreated area if(0 <= size && size_orgmeta < size){ truncated_start = size_orgmeta; truncated_size = size - size_orgmeta; } // set mtime and ctime(set "x-amz-meta-mtime" and "x-amz-meta-ctime" in orgmeta) if(UTIME_OMIT != ts_mctime.tv_nsec){ if(0 != SetMCtimeHasLock(ts_mctime, ts_mctime)){ S3FS_PRN_ERR("failed to set mtime/ctime. errno(%d)", errno); pfile.reset(); physical_fd = -1; inode = 0; return (0 == errno ? -EIO : -errno); } } } // create new pseudo fd, and set it to map std::unique_ptr ppseudoinfo(new PseudoFdInfo(physical_fd, flags)); int pseudo_fd = ppseudoinfo->GetPseudoFd(); pseudo_fd_map[pseudo_fd] = std::move(ppseudoinfo); // if there is untreated area, set it to pseudo object. if(0 < truncated_size){ if(!AddUntreated(truncated_start, truncated_size)){ pseudo_fd_map.erase(pseudo_fd); pfile.reset(); } } return pseudo_fd; } // [NOTE] // This method is called for only nocopyapi functions. // So we do not check disk space for this option mode, if there is no enough // disk space this method will be failed. // bool FdEntity::LoadAll(int fd, off_t* size, bool force_load) { const std::lock_guard lock(fdent_lock); S3FS_PRN_INFO3("[path=%s][pseudo_fd=%d][physical_fd=%d]", path.c_str(), fd, physical_fd); if(-1 == physical_fd || !FindPseudoFdWithLock(fd)){ S3FS_PRN_ERR("pseudo_fd(%d) and physical_fd(%d) for path(%s) is not opened yet", fd, physical_fd, path.c_str()); return false; } const std::lock_guard data_lock(fdent_data_lock); if(force_load){ SetAllStatusUnloaded(); } // // TODO: possibly do background for delay loading // int result; if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0))){ S3FS_PRN_ERR("could not download, result(%d)", result); return false; } if(size){ *size = pagelist.Size(); } return true; } // // Rename file path. // // This method sets the FdManager::fent map registration key to fentmapkey. // // [NOTE] // This method changes the file path of FdEntity. // Old file is deleted after linking to the new file path, and this works // without problem because the file descriptor is not affected even if the // cache file is open. // The mirror file descriptor is also the same. The mirror file path does // not need to be changed and will remain as it is. // bool FdEntity::RenamePath(const std::string& newpath, std::string& fentmapkey) { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); if(!cachepath.empty()){ // has cache path // make new cache path std::string newcachepath; if(!FdManager::MakeCachePath(newpath.c_str(), newcachepath, true)){ S3FS_PRN_ERR("failed to make cache path for object(%s).", newpath.c_str()); return false; } // rename cache file if(-1 == rename(cachepath.c_str(), newcachepath.c_str())){ S3FS_PRN_ERR("failed to rename old cache path(%s) to new cache path(%s) by errno(%d).", cachepath.c_str(), newcachepath.c_str(), errno); return false; } // link and unlink cache file stat if(!CacheFileStat::RenameCacheFileStat(path.c_str(), newpath.c_str())){ S3FS_PRN_ERR("failed to rename cache file stat(%s to %s).", path.c_str(), newpath.c_str()); return false; } fentmapkey = newpath; cachepath = newcachepath; }else{ // does not have cache path fentmapkey.clear(); FdManager::MakeRandomTempPath(newpath.c_str(), fentmapkey); } // set new path path = newpath; return true; } bool FdEntity::IsModified() const { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); return pagelist.IsModified(); } bool FdEntity::GetStatsHasLock(struct stat& st) const { if(-1 == physical_fd){ return false; } st = {}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat failed. errno(%d)", errno); return false; } return true; } int FdEntity::SetCtimeHasLock(struct timespec time) { S3FS_PRN_INFO3("[path=%s][physical_fd=%d][time=%s]", path.c_str(), physical_fd, str(time).c_str()); if(-1 == time.tv_sec){ return 0; } orgmeta["x-amz-meta-ctime"] = str(time); return 0; } int FdEntity::SetAtimeHasLock(struct timespec time) { S3FS_PRN_INFO3("[path=%s][physical_fd=%d][time=%s]", path.c_str(), physical_fd, str(time).c_str()); if(-1 == time.tv_sec){ return 0; } orgmeta["x-amz-meta-atime"] = str(time); return 0; } // [NOTE] // This method updates mtime as well as ctime. // int FdEntity::SetMCtimeHasLock(struct timespec mtime, struct timespec ctime) { S3FS_PRN_INFO3("[path=%s][physical_fd=%d][mtime=%s][ctime=%s]", path.c_str(), physical_fd, str(mtime).c_str(), str(ctime).c_str()); if(mtime.tv_sec < 0 || ctime.tv_sec < 0){ return 0; } if(-1 != physical_fd){ struct timespec ts[2]; ts[0].tv_sec = mtime.tv_sec; ts[0].tv_nsec = mtime.tv_nsec; ts[1].tv_sec = ctime.tv_sec; ts[1].tv_nsec = ctime.tv_nsec; if(-1 == futimens(physical_fd, ts)){ S3FS_PRN_ERR("futimens failed. errno(%d)", errno); return -errno; } }else if(!cachepath.empty()){ // not opened file yet. struct timespec ts[2]; ts[0].tv_sec = ctime.tv_sec; ts[0].tv_nsec = ctime.tv_nsec; ts[1].tv_sec = mtime.tv_sec; ts[1].tv_nsec = mtime.tv_nsec; if(-1 == utimensat(AT_FDCWD, cachepath.c_str(), ts, 0)){ S3FS_PRN_ERR("utimensat failed. errno(%d)", errno); return -errno; } } orgmeta["x-amz-meta-mtime"] = str(mtime); orgmeta["x-amz-meta-ctime"] = str(ctime); return 0; } bool FdEntity::UpdateCtime() { const std::lock_guard lock(fdent_lock); struct stat st; if(!GetStatsHasLock(st)){ return false; } orgmeta["x-amz-meta-ctime"] = str_stat_time(st, stat_time_type::CTIME); return true; } bool FdEntity::UpdateAtime() { const std::lock_guard lock(fdent_lock); struct stat st; if(!GetStatsHasLock(st)){ return false; } orgmeta["x-amz-meta-atime"] = str_stat_time(st, stat_time_type::ATIME); return true; } bool FdEntity::UpdateMtime(bool clear_holding_mtime) { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); if(0 <= holding_mtime.tv_sec){ // [NOTE] // This conditional statement is very special. // If you copy a file with "cp -p" etc., utimens or chown will be // called after opening the file, after that call to write, flush. // If normally utimens are not called(cases like "cp" only), mtime // should be updated at the file flush. // Here, check the holding_mtime value to prevent mtime from being // overwritten. // if(clear_holding_mtime){ if(!ClearHoldingMtime()){ return false; } // [NOTE] // If come here after fdatasync has been processed, the file // content update has already taken place. However, the metadata // update is necessary and needs to be flagged in order to // perform it with flush, // pending_status = pending_status_t::UPDATE_META_PENDING; } }else{ struct stat st; if(!GetStatsHasLock(st)){ return false; } orgmeta["x-amz-meta-mtime"] = str_stat_time(st, stat_time_type::MTIME); } return true; } bool FdEntity::SetHoldingMtime(struct timespec mtime) { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); S3FS_PRN_INFO3("[path=%s][physical_fd=%d][mtime=%s]", path.c_str(), physical_fd, str(mtime).c_str()); if(mtime.tv_sec < 0){ return false; } holding_mtime = mtime; return true; } bool FdEntity::ClearHoldingMtime() { if(holding_mtime.tv_sec < 0){ return false; } struct stat st; if(!GetStatsHasLock(st)){ return false; } if(-1 != physical_fd){ struct timespec ts[2]; struct timespec ts_ctime; ts[0].tv_sec = holding_mtime.tv_sec; ts[0].tv_nsec = holding_mtime.tv_nsec; set_stat_to_timespec(st, stat_time_type::CTIME, ts_ctime); ts[1].tv_sec = ts_ctime.tv_sec; ts[1].tv_nsec = ts_ctime.tv_nsec; if(-1 == futimens(physical_fd, ts)){ S3FS_PRN_ERR("futimens failed. errno(%d)", errno); return false; } }else if(!cachepath.empty()){ // not opened file yet. struct timespec ts[2]; struct timespec ts_ctime; set_stat_to_timespec(st, stat_time_type::CTIME, ts_ctime); ts[0].tv_sec = ts_ctime.tv_sec; ts[0].tv_nsec = ts_ctime.tv_nsec; ts[1].tv_sec = holding_mtime.tv_sec; ts[1].tv_nsec = holding_mtime.tv_nsec; if(-1 == utimensat(AT_FDCWD, cachepath.c_str(), ts, 0)){ S3FS_PRN_ERR("utimensat failed. errno(%d)", errno); return false; } } holding_mtime.tv_sec = -1; holding_mtime.tv_nsec = 0; return true; } bool FdEntity::GetSize(off_t& size) const { const std::lock_guard lock(fdent_lock); if(-1 == physical_fd){ return false; } const std::lock_guard data_lock(fdent_data_lock); size = pagelist.Size(); return true; } bool FdEntity::GetXattr(std::string& xattr) const { const std::lock_guard lock(fdent_lock); auto iter = orgmeta.find("x-amz-meta-xattr"); if(iter == orgmeta.cend()){ return false; } xattr = iter->second; return true; } bool FdEntity::SetXattr(const std::string& xattr) { const std::lock_guard lock(fdent_lock); orgmeta["x-amz-meta-xattr"] = xattr; return true; } bool FdEntity::SetModeHasLock(mode_t mode) { orgmeta["x-amz-meta-mode"] = std::to_string(mode); return true; } bool FdEntity::SetUIdHasLock(uid_t uid) { orgmeta["x-amz-meta-uid"] = std::to_string(uid); return true; } bool FdEntity::SetGIdHasLock(gid_t gid) { orgmeta["x-amz-meta-gid"] = std::to_string(gid); return true; } bool FdEntity::SetContentType(const char* path) { if(!path){ return false; } const std::lock_guard lock(fdent_lock); orgmeta["Content-Type"] = S3fsCurl::LookupMimeType(path); return true; } bool FdEntity::SetAllStatus(bool is_loaded) { S3FS_PRN_INFO3("[path=%s][physical_fd=%d][%s]", path.c_str(), physical_fd, is_loaded ? "loaded" : "unloaded"); if(-1 == physical_fd){ return false; } // get file size struct stat st{}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat is failed. errno(%d)", errno); return false; } // Reinit pagelist.Init(st.st_size, is_loaded, false); return true; } int FdEntity::Load(off_t start, off_t size, bool is_modified_flag) { S3FS_PRN_DBG("[path=%s][physical_fd=%d][offset=%lld][size=%lld]", path.c_str(), physical_fd, static_cast(start), static_cast(size)); if(-1 == physical_fd){ return -EBADF; } int result = 0; // check loaded area & load fdpage_list_t unloaded_list; if(0 < pagelist.GetUnloadedPages(unloaded_list, start, size)){ for(auto iter = unloaded_list.cbegin(); iter != unloaded_list.cend(); ++iter){ if(0 != size && start + size <= iter->offset){ // reached end break; } // check loading size off_t need_load_size = 0; if(iter->offset < size_orgmeta){ // original file size(on S3) is smaller than request. need_load_size = (iter->next() <= size_orgmeta ? iter->bytes : (size_orgmeta - iter->offset)); } // download if(S3fsCurl::GetMultipartSize() <= need_load_size && !nomultipart){ // parallel request result = S3fsCurl::ParallelGetObjectRequest(path.c_str(), physical_fd, iter->offset, need_load_size); }else{ // single request if(0 < need_load_size){ S3fsCurl s3fscurl; result = s3fscurl.GetObjectRequest(path.c_str(), physical_fd, iter->offset, need_load_size); }else{ result = 0; } } if(0 != result){ break; } // Set loaded flag pagelist.SetPageLoadedStatus(iter->offset, iter->bytes, (is_modified_flag ? PageList::page_status::LOAD_MODIFIED : PageList::page_status::LOADED)); } PageList::FreeList(unloaded_list); } return result; } // [NOTE] // At no disk space for caching object. // This method is downloading by dividing an object of the specified range // and uploading by multipart after finishing downloading it. // // [NOTICE] // Need to lock before calling this method. // int FdEntity::NoCacheLoadAndPost(PseudoFdInfo* pseudo_obj, off_t start, off_t size) { int result = 0; S3FS_PRN_INFO3("[path=%s][physical_fd=%d][offset=%lld][size=%lld]", path.c_str(), physical_fd, static_cast(start), static_cast(size)); if(!pseudo_obj){ S3FS_PRN_ERR("Pseudo object is nullptr."); return -EIO; } if(-1 == physical_fd){ return -EBADF; } // [NOTE] // This method calling means that the cache file is never used no more. // if(!cachepath.empty()){ // remove cache files(and cache stat file) FdManager::DeleteCacheFile(path.c_str()); // cache file path does not use no more. cachepath.clear(); mirrorpath.clear(); } // Change entity key in manager mapping FdManager::get()->ChangeEntityToTempPath(get_shared_ptr(), path.c_str()); // open temporary file int tmpfd; auto ptmpfp = FdManager::MakeTempFile(); if(nullptr == ptmpfp || -1 == (tmpfd = fileno(ptmpfp.get()))){ S3FS_PRN_ERR("failed to open temporary file by errno(%d)", errno); return (0 == errno ? -EIO : -errno); } // loop uploading by multipart for(auto iter = pagelist.pages.begin(); iter != pagelist.pages.end(); ++iter){ if(iter->end() < start){ continue; } if(0 != size && start + size <= iter->offset){ break; } // download each multipart size(default 10MB) in unit for(off_t oneread = 0, totalread = (iter->offset < start ? start : 0); totalread < iter->bytes; totalread += oneread){ int upload_fd = physical_fd; off_t offset = iter->offset + totalread; oneread = std::min(iter->bytes - totalread, S3fsCurl::GetMultipartSize()); // check rest size is over minimum part size // // [NOTE] // If the final part size is smaller than 5MB, it is not allowed by S3 API. // For this case, if the previous part of the final part is not over 5GB, // we incorporate the final part to the previous part. If the previous part // is over 5GB, we want to even out the last part and the previous part. // if((iter->bytes - totalread - oneread) < MIN_MULTIPART_SIZE){ if(FIVE_GB < iter->bytes - totalread){ oneread = (iter->bytes - totalread) / 2; }else{ oneread = iter->bytes - totalread; } } if(!iter->loaded){ // // loading or initializing // upload_fd = tmpfd; // load offset & size size_t need_load_size = 0; if(size_orgmeta <= offset){ // all area is over of original size need_load_size = 0; }else{ if(size_orgmeta < (offset + oneread)){ // original file size(on S3) is smaller than request. need_load_size = size_orgmeta - offset; }else{ need_load_size = oneread; } } size_t over_size = oneread - need_load_size; // [NOTE] // truncate file to zero and set length to part offset + size // after this, file length is (offset + size), but file does not use any disk space. // if(-1 == ftruncate(tmpfd, 0) || -1 == ftruncate(tmpfd, (offset + oneread))){ S3FS_PRN_ERR("failed to truncate temporary file(physical_fd=%d).", tmpfd); result = -EIO; break; } // single area get request if(0 < need_load_size){ S3fsCurl s3fscurl; if(0 != (result = s3fscurl.GetObjectRequest(path.c_str(), tmpfd, offset, oneread))){ S3FS_PRN_ERR("failed to get object(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast(offset), static_cast(oneread), tmpfd); break; } } // initialize fd without loading if(0 < over_size){ if(0 != (result = FdEntity::FillFile(tmpfd, 0, over_size, offset + need_load_size))){ S3FS_PRN_ERR("failed to fill rest bytes for physical_fd(%d). errno(%d)", tmpfd, result); break; } } }else{ // already loaded area } // single area upload by multipart post if(0 != (result = NoCacheMultipartPost(pseudo_obj, upload_fd, offset, oneread))){ S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast(offset), static_cast(oneread), upload_fd); break; } } if(0 != result){ break; } // set loaded flag if(!iter->loaded){ if(iter->offset < start){ fdpage page(iter->offset, start - iter->offset, iter->loaded, false); iter->bytes -= (start - iter->offset); iter->offset = start; iter = pagelist.pages.insert(iter, page); } if(0 != size && start + size < iter->next()){ fdpage page(iter->offset, start + size - iter->offset, true, false); iter->bytes -= (start + size - iter->offset); iter->offset = start + size; iter = pagelist.pages.insert(iter, page); }else{ iter->loaded = true; iter->modified = false; } } } if(0 == result){ // compress pagelist pagelist.Compress(); // fd data do empty if(-1 == ftruncate(physical_fd, 0)){ S3FS_PRN_ERR("failed to truncate file(physical_fd=%d), but continue...", physical_fd); } } return result; } // [NOTE] // At no disk space for caching object. // This method is starting multipart uploading. // int FdEntity::NoCachePreMultipartPost(PseudoFdInfo* pseudo_obj) { if(!pseudo_obj){ S3FS_PRN_ERR("Internal error, pseudo fd object pointer is null."); return -EIO; } // initialize multipart upload values pseudo_obj->ClearUploadInfo(true); S3fsCurl s3fscurl(true); std::string upload_id; int result; if(0 != (result = s3fscurl.PreMultipartPostRequest(path.c_str(), orgmeta, upload_id))){ return result; } s3fscurl.DestroyCurlHandle(); // Clear the dirty flag, because the meta data is updated. pending_status = pending_status_t::NO_UPDATE_PENDING; // reset upload_id if(!pseudo_obj->InitialUploadInfo(upload_id)){ return -EIO; } return 0; } // [NOTE] // At no disk space for caching object. // This method is uploading one part of multipart. // int FdEntity::NoCacheMultipartPost(PseudoFdInfo* pseudo_obj, int tgfd, off_t start, off_t size) { if(-1 == tgfd || !pseudo_obj || !pseudo_obj->IsUploading()){ S3FS_PRN_ERR("Need to initialize for multipart post."); return -EIO; } // get upload id std::string upload_id; if(!pseudo_obj->GetUploadId(upload_id)){ return -EIO; } // append new part and get it's etag string pointer etagpair* petagpair = nullptr; if(!pseudo_obj->AppendUploadPart(start, size, false, &petagpair)){ return -EIO; } S3fsCurl s3fscurl(true); return s3fscurl.MultipartUploadRequest(upload_id, path.c_str(), tgfd, start, size, petagpair); } // [NOTE] // At no disk space for caching object. // This method is finishing multipart uploading. // int FdEntity::NoCacheCompleteMultipartPost(PseudoFdInfo* pseudo_obj) { etaglist_t etaglist; if(!pseudo_obj || !pseudo_obj->IsUploading() || !pseudo_obj->GetEtaglist(etaglist)){ S3FS_PRN_ERR("There is no upload id or etag list."); return -EIO; } // get upload id std::string upload_id; if(!pseudo_obj->GetUploadId(upload_id)){ return -EIO; } S3fsCurl s3fscurl(true); int result = s3fscurl.CompleteMultipartPostRequest(path.c_str(), upload_id, etaglist); s3fscurl.DestroyCurlHandle(); if(0 != result){ S3fsCurl s3fscurl_abort(true); int result2 = s3fscurl.AbortMultipartUpload(path.c_str(), upload_id); s3fscurl_abort.DestroyCurlHandle(); if(0 != result2){ S3FS_PRN_ERR("failed to abort multipart upload by errno(%d)", result2); } return result; } // clear multipart upload info untreated_list.ClearAll(); pseudo_obj->ClearUploadInfo(); return 0; } off_t FdEntity::BytesModified() { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); return pagelist.BytesModified(); } // [NOTE] // There are conditions that allow you to perform multipart uploads. // // According to the AWS spec: // - 1 to 10,000 parts are allowed // - minimum size of parts is 5MB (except for the last part) // // For example, if you set the minimum part size to 5MB, you can upload // a maximum (5 * 10,000)MB file. // The part size can be changed in MB units, then the maximum file size // that can be handled can be further increased. // Files smaller than the minimum part size will not be multipart uploaded, // but will be uploaded as single part(normally). // int FdEntity::RowFlushHasLock(int fd, const char* tpath, bool force_sync) { S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(tpath), path.c_str(), fd, physical_fd); if(-1 == physical_fd){ return -EBADF; } // check pseudo fd and its flag const auto miter = pseudo_fd_map.find(fd); if(pseudo_fd_map.cend() == miter || nullptr == miter->second){ return -EBADF; } if(!miter->second->Writable() && !(miter->second->GetFlags() & O_CREAT)){ // If the entity is opened read-only, it will end normally without updating. return 0; } PseudoFdInfo* pseudo_obj = miter->second.get(); int result; if(!force_sync && !pagelist.IsModified() && !IsDirtyMetadata()){ // nothing to update. return 0; } if(S3fsLog::IsS3fsLogDbg()){ pagelist.Dump(); } if(nomultipart){ // No multipart upload if(!force_sync && !pagelist.IsModified()){ // for only push pending headers result = UploadPendingHasLock(-1); }else{ result = RowFlushNoMultipart(pseudo_obj, tpath); } }else if(FdEntity::streamupload){ // Stream multipart upload result = RowFlushStreamMultipart(pseudo_obj, tpath); }else if(FdEntity::mixmultipart){ // Mix multipart upload result = RowFlushMixMultipart(pseudo_obj, tpath); }else{ // Normal multipart upload result = RowFlushMultipart(pseudo_obj, tpath); } // [NOTE] // if something went wrong, so if you are using a cache file, // the cache file may not be correct. So delete cache files. // if(0 != result && !cachepath.empty()){ FdManager::DeleteCacheFile(tpath); } return result; } int FdEntity::RowFlushNoMultipart(const PseudoFdInfo* pseudo_obj, const char* tpath) { S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(tpath), path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd); if(-1 == physical_fd || !pseudo_obj){ return -EBADF; } if(pseudo_obj->IsUploading()){ S3FS_PRN_ERR("Why uploading now, even though s3fs is No Multipart uploading mode."); return -EBADF; } int result; std::string tmppath = path; headers_t tmporgmeta = orgmeta; // If there is no loading all of the area, loading all area. off_t restsize = pagelist.GetTotalUnloadedPageSize(); if(0 < restsize){ // check disk space if(!ReserveDiskSpace(restsize)){ // no enough disk space S3FS_PRN_WARN("Not enough local storage to flush: [path=%s][pseudo_fd=%d][physical_fd=%d]", path.c_str(), pseudo_obj->GetPseudoFd(), physical_fd); return -ENOSPC; // No space left on device } } FdManager::FreeReservedDiskSpace(restsize); // Always load all uninitialized area if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0))){ S3FS_PRN_ERR("failed to upload all area(errno=%d)", result); return result; } // check size if(pagelist.Size() > MAX_MULTIPART_CNT * S3fsCurl::GetMultipartSize()){ S3FS_PRN_ERR("Part count exceeds %d. Increase multipart size and try again.", MAX_MULTIPART_CNT); return -EFBIG; } // backup upload file size struct stat st{}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno); } S3fsCurl s3fscurl(true); result = s3fscurl.PutRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd); // reset uploaded file size size_orgmeta = st.st_size; untreated_list.ClearAll(); if(0 == result){ pagelist.ClearAllModified(); } return result; } int FdEntity::RowFlushMultipart(PseudoFdInfo* pseudo_obj, const char* tpath) { S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(tpath), path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd); if(-1 == physical_fd || !pseudo_obj){ return -EBADF; } int result = 0; if(!pseudo_obj->IsUploading()){ // Start uploading // If there is no loading all of the area, loading all area. off_t restsize = pagelist.GetTotalUnloadedPageSize(); // Check rest size and free disk space if(0 < restsize && !ReserveDiskSpace(restsize)){ // no enough disk space if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){ S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); return result; } // upload all by multipart uploading if(0 != (result = NoCacheLoadAndPost(pseudo_obj))){ S3FS_PRN_ERR("failed to upload all area by multipart uploading(errno=%d)", result); return result; } }else{ // enough disk space or no rest size std::string tmppath = path; headers_t tmporgmeta = orgmeta; FdManager::FreeReservedDiskSpace(restsize); // Load all uninitialized area(no mix multipart uploading) if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0))){ S3FS_PRN_ERR("failed to upload all area(errno=%d)", result); return result; } // backup upload file size struct stat st{}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno); } if(pagelist.Size() > MAX_MULTIPART_CNT * S3fsCurl::GetMultipartSize()){ S3FS_PRN_ERR("Part count exceeds %d. Increase multipart size and try again.", MAX_MULTIPART_CNT); return -EFBIG; }else if(pagelist.Size() >= S3fsCurl::GetMultipartSize()){ // multipart uploading result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd); }else{ // normal uploading (too small part size) S3fsCurl s3fscurl(true); result = s3fscurl.PutRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd); } // reset uploaded file size size_orgmeta = st.st_size; } untreated_list.ClearAll(); }else{ // Already start uploading // upload rest data off_t untreated_start = 0; off_t untreated_size = 0; if(untreated_list.GetLastUpdatedPart(untreated_start, untreated_size, S3fsCurl::GetMultipartSize(), 0) && 0 < untreated_size){ if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){ S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast(untreated_start), static_cast(untreated_size), physical_fd); return result; } untreated_list.ClearParts(untreated_start, untreated_size); } // complete multipart uploading. if(0 != (result = NoCacheCompleteMultipartPost(pseudo_obj))){ S3FS_PRN_ERR("failed to complete(finish) multipart post for file(physical_fd=%d).", physical_fd); return result; } // truncate file to zero if(-1 == ftruncate(physical_fd, 0)){ // So the file has already been removed, skip error. S3FS_PRN_ERR("failed to truncate file(physical_fd=%d) to zero, but continue...", physical_fd); } // put pending headers or create new file if(0 != (result = UploadPendingHasLock(-1))){ return result; } } if(0 == result){ pagelist.ClearAllModified(); pending_status = pending_status_t::NO_UPDATE_PENDING; } return result; } int FdEntity::RowFlushMixMultipart(PseudoFdInfo* pseudo_obj, const char* tpath) { S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(tpath), path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd); if(-1 == physical_fd || !pseudo_obj){ return -EBADF; } int result = 0; if(!pseudo_obj->IsUploading()){ // Start uploading // If there is no loading all of the area, loading all area. off_t restsize = pagelist.GetTotalUnloadedPageSize(/* start */ 0, /* size = all */ 0, MIN_MULTIPART_SIZE); // Check rest size and free disk space if(0 < restsize && !ReserveDiskSpace(restsize)){ // no enough disk space if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){ S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); return result; } // upload all by multipart uploading if(0 != (result = NoCacheLoadAndPost(pseudo_obj))){ S3FS_PRN_ERR("failed to upload all area by multipart uploading(errno=%d)", result); return result; } }else{ // enough disk space or no rest size std::string tmppath = path; headers_t tmporgmeta = orgmeta; FdManager::FreeReservedDiskSpace(restsize); // backup upload file size struct stat st{}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno); } if(pagelist.Size() > MAX_MULTIPART_CNT * S3fsCurl::GetMultipartSize()){ S3FS_PRN_ERR("Part count exceeds %d. Increase multipart size and try again.", MAX_MULTIPART_CNT); return -EFBIG; }else if(pagelist.Size() >= S3fsCurl::GetMultipartSize()){ // mix multipart uploading // This is to ensure that each part is 5MB or more. // If the part is less than 5MB, download it. fdpage_list_t dlpages; fdpage_list_t mixuppages; if(!pagelist.GetPageListsForMultipartUpload(dlpages, mixuppages, S3fsCurl::GetMultipartSize())){ S3FS_PRN_ERR("something error occurred during getting download pagelist."); return -1; } // [TODO] should use parallel downloading // for(auto iter = dlpages.cbegin(); iter != dlpages.cend(); ++iter){ if(0 != (result = Load(iter->offset, iter->bytes, /*is_modified_flag=*/ true))){ // set loaded and modified flag S3FS_PRN_ERR("failed to get parts(start=%lld, size=%lld) before uploading.", static_cast(iter->offset), static_cast(iter->bytes)); return result; } } // multipart uploading with copy api result = S3fsCurl::ParallelMixMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd, mixuppages); }else{ // normal uploading (too small part size) // If there are unloaded pages, they are loaded at here. if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0))){ S3FS_PRN_ERR("failed to load parts before uploading object(%d)", result); return result; } S3fsCurl s3fscurl(true); result = s3fscurl.PutRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd); } // reset uploaded file size size_orgmeta = st.st_size; } untreated_list.ClearAll(); }else{ // Already start uploading // upload rest data off_t untreated_start = 0; off_t untreated_size = 0; if(untreated_list.GetLastUpdatedPart(untreated_start, untreated_size, S3fsCurl::GetMultipartSize(), 0) && 0 < untreated_size){ if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){ S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast(untreated_start), static_cast(untreated_size), physical_fd); return result; } untreated_list.ClearParts(untreated_start, untreated_size); } // complete multipart uploading. if(0 != (result = NoCacheCompleteMultipartPost(pseudo_obj))){ S3FS_PRN_ERR("failed to complete(finish) multipart post for file(physical_fd=%d).", physical_fd); return result; } // truncate file to zero if(-1 == ftruncate(physical_fd, 0)){ // So the file has already been removed, skip error. S3FS_PRN_ERR("failed to truncate file(physical_fd=%d) to zero, but continue...", physical_fd); } // put pending headers or create new file if(0 != (result = UploadPendingHasLock(-1))){ return result; } } if(0 == result){ pagelist.ClearAllModified(); pending_status = pending_status_t::NO_UPDATE_PENDING; } return result; } int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpath) { S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d][mix_upload=%s]", SAFESTRPTR(tpath), path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, (FdEntity::mixmultipart ? "true" : "false")); if(-1 == physical_fd || !pseudo_obj){ return -EBADF; } int result = 0; if(pagelist.Size() <= S3fsCurl::GetMultipartSize()){ // // Use normal upload instead of multipart upload(too small part size) // // backup upload file size struct stat st{}; if(-1 == fstat(physical_fd, &st)){ S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno); } // If there are unloaded pages, they are loaded at here. if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0))){ S3FS_PRN_ERR("failed to load parts before uploading object(%d)", result); return result; } headers_t tmporgmeta = orgmeta; S3fsCurl s3fscurl(true); result = s3fscurl.PutRequest(path.c_str(), tmporgmeta, physical_fd); // reset uploaded file size size_orgmeta = st.st_size; untreated_list.ClearAll(); if(0 == result){ pagelist.ClearAllModified(); } }else{ // // Make upload/download/copy/cancel lists from file // mp_part_list_t to_upload_list; mp_part_list_t to_copy_list; mp_part_list_t to_download_list; filepart_list_t cancel_uploaded_list; bool wait_upload_complete = false; if(!pseudo_obj->ExtractUploadPartsFromAllArea(untreated_list, to_upload_list, to_copy_list, to_download_list, cancel_uploaded_list, wait_upload_complete, S3fsCurl::GetMultipartSize(), pagelist.Size(), FdEntity::mixmultipart)){ S3FS_PRN_ERR("Failed to extract various upload parts list from all area: errno(EIO)"); return -EIO; } // // Check total size for downloading and Download // off_t total_download_size = total_mp_part_list(to_download_list); if(0 < total_download_size){ // // Check if there is enough free disk space for the total download size // if(!ReserveDiskSpace(total_download_size)){ // no enough disk space // // [NOTE] // Because there is no left space size to download, we can't solve this anymore // in this case which is uploading in sequence. // S3FS_PRN_WARN("Not enough local storage(%lld byte) to cache write request for whole of the file: [path=%s][physical_fd=%d]", static_cast(total_download_size), path.c_str(), physical_fd); return -ENOSPC; // No space left on device } // enough disk space // // Download all parts // // [TODO] // Execute in parallel downloading with multiple thread. // for(auto download_iter = to_download_list.cbegin(); download_iter != to_download_list.cend(); ++download_iter){ if(0 != (result = Load(download_iter->start, download_iter->size))){ break; } } FdManager::FreeReservedDiskSpace(total_download_size); if(0 != result){ S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); return result; } } // // Has multipart uploading already started? // if(!pseudo_obj->IsUploading()){ // // Multipart uploading hasn't started yet, so start it. // S3fsCurl s3fscurl(true); std::string upload_id; if(0 != (result = s3fscurl.PreMultipartPostRequest(path.c_str(), orgmeta, upload_id))){ S3FS_PRN_ERR("failed to setup multipart upload(create upload id) by errno(%d)", result); return result; } if(!pseudo_obj->InitialUploadInfo(upload_id)){ S3FS_PRN_ERR("failed to setup multipart upload(set upload id to object)"); return -EIO; } // Clear the dirty flag, because the meta data is updated. pending_status = pending_status_t::NO_UPDATE_PENDING; } // // Output debug level information // // When canceling(overwriting) a part that has already been uploaded, output it. // if(S3fsLog::IsS3fsLogDbg()){ for(auto cancel_iter = cancel_uploaded_list.cbegin(); cancel_iter != cancel_uploaded_list.cend(); ++cancel_iter){ S3FS_PRN_DBG("Cancel uploaded: start(%lld), size(%lld), part number(%d)", static_cast(cancel_iter->startpos), static_cast(cancel_iter->size), (cancel_iter->petag ? cancel_iter->petag->part_num : -1)); } } // [NOTE] // If there is a part where has already been uploading, that part // is re-updated after finishing uploading, so the part of the last // uploaded must be canceled. // (These are cancel_uploaded_list, cancellation processing means // re-uploading the same area.) // // In rare cases, the completion of the previous upload and the // re-upload may be reversed, causing the ETag to be reversed, // in which case the upload will fail. // To prevent this, if the upload of the same area as the re-upload // is incomplete, we must wait for it to complete here. // if(wait_upload_complete){ if(0 != (result = pseudo_obj->WaitAllThreadsExit())){ S3FS_PRN_ERR("Some cancel area uploads that were waiting to complete failed with %d.", result); return result; } } // // Upload multipart and copy parts and wait exiting them // if(!pseudo_obj->ParallelMultipartUploadAll(path.c_str(), to_upload_list, to_copy_list, result)){ S3FS_PRN_ERR("Failed to upload multipart parts."); untreated_list.ClearAll(); pseudo_obj->ClearUploadInfo(); // clear multipart upload info return -EIO; } if(0 != result){ S3FS_PRN_ERR("An error(%d) occurred in some threads that were uploading parallel multiparts, but continue to clean up..", result); untreated_list.ClearAll(); pseudo_obj->ClearUploadInfo(); // clear multipart upload info return result; } // // Complete uploading // std::string upload_id; etaglist_t etaglist; if(!pseudo_obj->GetUploadId(upload_id) || !pseudo_obj->GetEtaglist(etaglist)){ S3FS_PRN_ERR("There is no upload id or etag list."); untreated_list.ClearAll(); pseudo_obj->ClearUploadInfo(); // clear multipart upload info return -EIO; }else{ S3fsCurl s3fscurl(true); result = s3fscurl.CompleteMultipartPostRequest(path.c_str(), upload_id, etaglist); s3fscurl.DestroyCurlHandle(); if(0 != result){ S3FS_PRN_ERR("failed to complete multipart upload by errno(%d)", result); untreated_list.ClearAll(); pseudo_obj->ClearUploadInfo(); // clear multipart upload info S3fsCurl s3fscurl_abort(true); int result2 = s3fscurl.AbortMultipartUpload(path.c_str(), upload_id); s3fscurl_abort.DestroyCurlHandle(); if(0 != result2){ S3FS_PRN_ERR("failed to abort multipart upload by errno(%d)", result2); } return result; } } untreated_list.ClearAll(); pseudo_obj->ClearUploadInfo(); // clear multipart upload info // put pending headers or create new file if(0 != (result = UploadPendingHasLock(-1))){ return result; } } untreated_list.ClearAll(); if(0 == result){ pagelist.ClearAllModified(); } return result; } // [NOTICE] // Need to lock before calling this method. bool FdEntity::ReserveDiskSpace(off_t size) { if(FdManager::ReserveDiskSpace(size)){ return true; } if(!pagelist.IsModified()){ // try to clear all cache for this fd. pagelist.Init(pagelist.Size(), false, false); if(-1 == ftruncate(physical_fd, 0) || -1 == ftruncate(physical_fd, pagelist.Size())){ S3FS_PRN_ERR("failed to truncate temporary file(physical_fd=%d).", physical_fd); return false; } if(FdManager::ReserveDiskSpace(size)){ return true; } } FdManager::get()->CleanupCacheDir(); return FdManager::ReserveDiskSpace(size); } ssize_t FdEntity::Read(int fd, char* bytes, off_t start, size_t size, bool force_load) { const std::lock_guard lock(fdent_lock); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, physical_fd, static_cast(start), size); if(-1 == physical_fd || nullptr == CheckPseudoFdFlags(fd, false)){ S3FS_PRN_DBG("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not readable", fd, physical_fd, path.c_str()); return -EBADF; } const std::lock_guard data_lock(fdent_data_lock); if(force_load){ pagelist.SetPageLoadedStatus(start, size, PageList::page_status::NOT_LOAD_MODIFIED); } ssize_t rsize; // check disk space if(0 < pagelist.GetTotalUnloadedPageSize(start, size)){ // load size(for prefetch) size_t load_size = size; if(start + static_cast(size) < pagelist.Size()){ ssize_t prefetch_max_size = std::max(static_cast(size), S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()); if(start + prefetch_max_size < pagelist.Size()){ load_size = prefetch_max_size; }else{ load_size = pagelist.Size() - start; } } if(!ReserveDiskSpace(load_size)){ S3FS_PRN_WARN("could not reserve disk space for pre-fetch download"); load_size = size; if(!ReserveDiskSpace(load_size)){ S3FS_PRN_ERR("could not reserve disk space for pre-fetch download"); return -ENOSPC; } } // Loading int result = 0; if(0 < size){ result = Load(start, load_size); } FdManager::FreeReservedDiskSpace(load_size); if(0 != result){ S3FS_PRN_ERR("could not download. start(%lld), size(%zu), errno(%d)", static_cast(start), size, result); return result; } } // Reading if(-1 == (rsize = pread(physical_fd, bytes, size, start))){ S3FS_PRN_ERR("pread failed. errno(%d)", errno); return -errno; } return rsize; } ssize_t FdEntity::Write(int fd, const char* bytes, off_t start, size_t size) { const std::lock_guard lock(fdent_lock); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, physical_fd, static_cast(start), size); PseudoFdInfo* pseudo_obj = nullptr; if(-1 == physical_fd || nullptr == (pseudo_obj = CheckPseudoFdFlags(fd, false))){ S3FS_PRN_ERR("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable", fd, physical_fd, path.c_str()); return -EBADF; } // check if not enough disk space left BEFORE locking fd if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(nullptr, size)){ FdManager::get()->CleanupCacheDir(); } const std::lock_guard data_lock(fdent_data_lock); // check file size if(pagelist.Size() < start){ // grow file size if(-1 == ftruncate(physical_fd, start)){ S3FS_PRN_ERR("failed to truncate temporary file(physical_fd=%d).", physical_fd); return -errno; } // set untreated area if(!AddUntreated(pagelist.Size(), (start - pagelist.Size()))){ S3FS_PRN_ERR("failed to set untreated area by incremental."); return -EIO; } // add new area pagelist.SetPageLoadedStatus(pagelist.Size(), start - pagelist.Size(), PageList::page_status::MODIFIED); } ssize_t wsize; if(nomultipart){ // No multipart upload wsize = WriteNoMultipart(pseudo_obj, bytes, start, size); }else if(FdEntity::streamupload){ // Stream upload wsize = WriteStreamUpload(pseudo_obj, bytes, start, size); }else if(FdEntity::mixmultipart){ // Mix multipart upload wsize = WriteMixMultipart(pseudo_obj, bytes, start, size); }else{ // Normal multipart upload wsize = WriteMultipart(pseudo_obj, bytes, start, size); } return wsize; } ssize_t FdEntity::WriteNoMultipart(const PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, static_cast(start), size); if(-1 == physical_fd || !pseudo_obj){ S3FS_PRN_ERR("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable", (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, path.c_str()); return -EBADF; } int result = 0; if(pseudo_obj->IsUploading()){ S3FS_PRN_ERR("Why uploading now, even though s3fs is No Multipart uploading mode."); return -EBADF; } // check disk space off_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; if(!ReserveDiskSpace(restsize)){ // no enough disk space S3FS_PRN_WARN("Not enough local storage to cache write request: [path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast(start), size); return -ENOSPC; // No space left on device } // Load uninitialized area which starts from 0 to (start + size) before writing. if(0 < start){ result = Load(0, start); } FdManager::FreeReservedDiskSpace(restsize); if(0 != result){ S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); return result; } // Writing ssize_t wsize; if(-1 == (wsize = pwrite(physical_fd, bytes, size, start))){ S3FS_PRN_ERR("pwrite failed. errno(%d)", errno); return -errno; } if(0 < wsize){ pagelist.SetPageLoadedStatus(start, wsize, PageList::page_status::LOAD_MODIFIED); AddUntreated(start, wsize); } // Load uninitialized area which starts from (start + size) to EOF after writing. if(pagelist.Size() > start + static_cast(size)){ result = Load(start + size, pagelist.Size()); if(0 != result){ S3FS_PRN_ERR("failed to load uninitialized area after writing(errno=%d)", result); return result; } } return wsize; } ssize_t FdEntity::WriteMultipart(PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, static_cast(start), size); if(-1 == physical_fd || !pseudo_obj){ S3FS_PRN_ERR("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable", (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, path.c_str()); return -EBADF; } int result = 0; if(!pseudo_obj->IsUploading()){ // check disk space off_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; if(ReserveDiskSpace(restsize)){ // enough disk space // Load uninitialized area which starts from 0 to (start + size) before writing. if(0 < start){ result = Load(0, start); } FdManager::FreeReservedDiskSpace(restsize); if(0 != result){ S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); return result; } }else{ // no enough disk space if((start + static_cast(size)) <= S3fsCurl::GetMultipartSize()){ S3FS_PRN_WARN("Not enough local storage to cache write request till multipart upload can start: [path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast(start), size); return -ENOSPC; // No space left on device } if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){ S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); return result; } // start multipart uploading if(0 != (result = NoCacheLoadAndPost(pseudo_obj, 0, start))){ S3FS_PRN_ERR("failed to load uninitialized area and multipart uploading it(errno=%d)", result); return result; } untreated_list.ClearAll(); } }else{ // already start multipart uploading } // Writing ssize_t wsize; if(-1 == (wsize = pwrite(physical_fd, bytes, size, start))){ S3FS_PRN_ERR("pwrite failed. errno(%d)", errno); return -errno; } if(0 < wsize){ pagelist.SetPageLoadedStatus(start, wsize, PageList::page_status::LOAD_MODIFIED); AddUntreated(start, wsize); } // Load uninitialized area which starts from (start + size) to EOF after writing. if(pagelist.Size() > start + static_cast(size)){ result = Load(start + size, pagelist.Size()); if(0 != result){ S3FS_PRN_ERR("failed to load uninitialized area after writing(errno=%d)", result); return result; } } // check multipart uploading if(pseudo_obj->IsUploading()){ // get last untreated part(maximum size is multipart size) off_t untreated_start = 0; off_t untreated_size = 0; if(untreated_list.GetLastUpdatedPart(untreated_start, untreated_size, S3fsCurl::GetMultipartSize())){ // when multipart max size is reached if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){ S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast(untreated_start), static_cast(untreated_size), physical_fd); return result; } // [NOTE] // truncate file to zero and set length to part offset + size // after this, file length is (offset + size), but file does not use any disk space. // if(-1 == ftruncate(physical_fd, 0) || -1 == ftruncate(physical_fd, (untreated_start + untreated_size))){ S3FS_PRN_ERR("failed to truncate file(physical_fd=%d).", physical_fd); return -errno; } untreated_list.ClearParts(untreated_start, untreated_size); } } return wsize; } ssize_t FdEntity::WriteMixMultipart(PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, static_cast(start), size); if(-1 == physical_fd || !pseudo_obj){ S3FS_PRN_ERR("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable", (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, path.c_str()); return -EBADF; } int result; if(!pseudo_obj->IsUploading()){ // check disk space off_t restsize = pagelist.GetTotalUnloadedPageSize(0, start, MIN_MULTIPART_SIZE) + size; if(ReserveDiskSpace(restsize)){ // enough disk space FdManager::FreeReservedDiskSpace(restsize); }else{ // no enough disk space if((start + static_cast(size)) <= S3fsCurl::GetMultipartSize()){ S3FS_PRN_WARN("Not enough local storage to cache write request till multipart upload can start: [path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast(start), size); return -ENOSPC; // No space left on device } if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){ S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); return result; } // start multipart uploading if(0 != (result = NoCacheLoadAndPost(pseudo_obj, 0, start))){ S3FS_PRN_ERR("failed to load uninitialized area and multipart uploading it(errno=%d)", result); return result; } untreated_list.ClearAll(); } }else{ // already start multipart uploading } // Writing ssize_t wsize; if(-1 == (wsize = pwrite(physical_fd, bytes, size, start))){ S3FS_PRN_ERR("pwrite failed. errno(%d)", errno); return -errno; } if(0 < wsize){ pagelist.SetPageLoadedStatus(start, wsize, PageList::page_status::LOAD_MODIFIED); AddUntreated(start, wsize); } // check multipart uploading if(pseudo_obj->IsUploading()){ // get last untreated part(maximum size is multipart size) off_t untreated_start = 0; off_t untreated_size = 0; if(untreated_list.GetLastUpdatedPart(untreated_start, untreated_size, S3fsCurl::GetMultipartSize())){ // when multipart max size is reached if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){ S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(physical_fd=%d).", static_cast(untreated_start), static_cast(untreated_size), physical_fd); return result; } // [NOTE] // truncate file to zero and set length to part offset + size // after this, file length is (offset + size), but file does not use any disk space. // if(-1 == ftruncate(physical_fd, 0) || -1 == ftruncate(physical_fd, (untreated_start + untreated_size))){ S3FS_PRN_ERR("failed to truncate file(physical_fd=%d).", physical_fd); return -errno; } untreated_list.ClearParts(untreated_start, untreated_size); } } return wsize; } // // On Stream upload, the uploading is executed in another thread when the // written area exceeds the maximum size of multipart upload. // ssize_t FdEntity::WriteStreamUpload(PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, static_cast(start), size); if(-1 == physical_fd || !pseudo_obj){ S3FS_PRN_ERR("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable", (pseudo_obj ? pseudo_obj->GetPseudoFd() : -1), physical_fd, path.c_str()); return -EBADF; } // Writing ssize_t wsize; if(-1 == (wsize = pwrite(physical_fd, bytes, size, start))){ S3FS_PRN_ERR("pwrite failed. errno(%d)", errno); return -errno; } if(0 < wsize){ pagelist.SetPageLoadedStatus(start, wsize, PageList::page_status::LOAD_MODIFIED); AddUntreated(start, wsize); } // Check and Upload // // If the last updated Untreated area exceeds the maximum upload size, // upload processing is performed. // headers_t tmporgmeta = orgmeta; bool isuploading = pseudo_obj->IsUploading(); ssize_t result; if(0 != (result = pseudo_obj->UploadBoundaryLastUntreatedArea(path.c_str(), tmporgmeta, this))){ S3FS_PRN_ERR("Failed to upload the last untreated parts(area) : result=%zd", result); return result; } if(!isuploading && pseudo_obj->IsUploading()){ // Clear the dirty flag, because the meta data is updated. pending_status = pending_status_t::NO_UPDATE_PENDING; } return wsize; } // [NOTE] // Returns true if merged to orgmeta. // If true is returned, the caller can update the header. // If it is false, do not update the header because multipart upload is in progress. // In this case, the header is pending internally and is updated after the upload // is complete(flush file). // bool FdEntity::MergeOrgMeta(headers_t& updatemeta) { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); merge_headers(orgmeta, updatemeta, true); // overwrite all keys // [NOTE] // this is special cases, we remove the key which has empty values. for(auto hiter = orgmeta.cbegin(); hiter != orgmeta.cend(); ){ if(hiter->second.empty()){ hiter = orgmeta.erase(hiter); }else{ ++hiter; } } updatemeta = orgmeta; orgmeta.erase("x-amz-copy-source"); // update ctime/mtime/atime struct timespec mtime = get_mtime(updatemeta, false); // not overcheck struct timespec ctime = get_ctime(updatemeta, false); // not overcheck struct timespec atime = get_atime(updatemeta, false); // not overcheck if(0 <= mtime.tv_sec){ SetMCtimeHasLock(mtime, (ctime.tv_sec < 0 ? mtime : ctime)); } if(0 <= atime.tv_sec){ SetAtimeHasLock(atime); } if(pending_status_t::NO_UPDATE_PENDING == pending_status && (IsUploading() || pagelist.IsModified())){ pending_status = pending_status_t::UPDATE_META_PENDING; } return (pending_status_t::NO_UPDATE_PENDING != pending_status); } // global function in s3fs.cpp int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size = true); int FdEntity::UploadPendingHasLock(int fd) { int result; if(pending_status_t::NO_UPDATE_PENDING == pending_status){ // nothing to do result = 0; }else if(pending_status_t::UPDATE_META_PENDING == pending_status){ headers_t updatemeta = orgmeta; updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(path.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // put headers, no need to update mtime to avoid dead lock result = put_headers(path.c_str(), updatemeta, true); if(0 != result){ S3FS_PRN_ERR("failed to put header after flushing file(%s) by(%d).", path.c_str(), result); }else{ pending_status = pending_status_t::NO_UPDATE_PENDING; } }else{ // CREATE_FILE_PENDING == pending_status if(-1 == fd){ S3FS_PRN_ERR("could not create a new file(%s), because fd is not specified.", path.c_str()); result = -EBADF; }else{ result = RowFlushHasLock(fd, nullptr, true); if(0 != result){ S3FS_PRN_ERR("failed to flush for file(%s) by(%d).", path.c_str(), result); }else{ pending_status = pending_status_t::NO_UPDATE_PENDING; } } } return result; } // [NOTE] // For systems where the fallocate function cannot be detected, use a dummy function. // ex. OSX // #if !defined(HAVE_FALLOCATE) || defined(__MSYS__) static int fallocate(int /*fd*/, int /*mode*/, off_t /*offset*/, off_t /*len*/) { errno = ENOSYS; // This is a bad idea, but the caller can handle it simply. return -1; } #endif // HAVE_FALLOCATE // [NOTE] // If HAVE_FALLOCATE is undefined, or versions prior to 2.6.38(fallocate function exists), // following flags are undefined. Then we need these symbols defined in fallocate, so we // define them here. // The definitions are copied from linux/falloc.h, but if HAVE_FALLOCATE is undefined, // these values can be anything. // #ifndef FALLOC_FL_PUNCH_HOLE #define FALLOC_FL_PUNCH_HOLE 0x02 /* de-allocates range */ #endif #ifndef FALLOC_FL_KEEP_SIZE #define FALLOC_FL_KEEP_SIZE 0x01 #endif // [NOTE] // This method punches an area(on cache file) that has no data at the time it is called. // This is called to prevent the cache file from growing. // However, this method uses the non-portable(Linux specific) system call fallocate(). // Also, depending on the file system, FALLOC_FL_PUNCH_HOLE mode may not work and HOLE // will not open.(Filesystems for which this method works are ext4, btrfs, xfs, etc.) // bool FdEntity::PunchHole(off_t start, size_t size) { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); S3FS_PRN_DBG("[path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast(start), size); if(-1 == physical_fd){ return false; } // get page list that have no data fdpage_list_t nodata_pages; if(!pagelist.GetNoDataPageLists(nodata_pages)){ S3FS_PRN_ERR("failed to get page list that have no data."); return false; } if(nodata_pages.empty()){ S3FS_PRN_DBG("there is no page list that have no data, so nothing to do."); return true; } // try to punch hole to file for(auto iter = nodata_pages.cbegin(); iter != nodata_pages.cend(); ++iter){ if(0 != fallocate(physical_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, iter->offset, iter->bytes)){ if(ENOSYS == errno || EOPNOTSUPP == errno){ S3FS_PRN_ERR("failed to fallocate for punching hole to file with errno(%d), it maybe the fallocate function is not implemented in this kernel, or the file system does not support FALLOC_FL_PUNCH_HOLE.", errno); }else{ S3FS_PRN_ERR("failed to fallocate for punching hole to file with errno(%d)", errno); } return false; } if(!pagelist.SetPageLoadedStatus(iter->offset, iter->bytes, PageList::page_status::NOT_LOAD_MODIFIED)){ S3FS_PRN_ERR("succeed to punch HOLEs in the cache file, but failed to update the cache stat."); return false; } S3FS_PRN_DBG("made a hole at [%lld - %lld bytes](into a boundary) of the cache file.", static_cast(iter->offset), static_cast(iter->bytes)); } return true; } // [NOTE] // Indicate that a new file's is dirty. // This ensures that both metadata and data are synced during flush. // void FdEntity::MarkDirtyNewFile() { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); pagelist.Init(0, false, true); pending_status = pending_status_t::CREATE_FILE_PENDING; } bool FdEntity::IsDirtyNewFile() const { const std::lock_guard lock(fdent_data_lock); return (pending_status_t::CREATE_FILE_PENDING == pending_status); } // [NOTE] // The fdatasync call only uploads the content but does not update // the meta data. In the flush call, if there is no update contents, // need to upload only metadata, so use these functions. // void FdEntity::MarkDirtyMetadata() { const std::lock_guard lock(fdent_lock); const std::lock_guard data_lock(fdent_data_lock); if(pending_status_t::NO_UPDATE_PENDING == pending_status){ pending_status = pending_status_t::UPDATE_META_PENDING; } } bool FdEntity::IsDirtyMetadata() const { return (pending_status_t::UPDATE_META_PENDING == pending_status); } bool FdEntity::AddUntreated(off_t start, off_t size) { bool result = untreated_list.AddPart(start, size); if(!result){ S3FS_PRN_DBG("Failed adding untreated area part."); }else if(S3fsLog::IsS3fsLogDbg()){ untreated_list.Dump(); } return result; } bool FdEntity::GetLastUpdateUntreatedPart(off_t& start, off_t& size) const { // Get last untreated area if(!untreated_list.GetLastUpdatePart(start, size)){ return false; } return true; } bool FdEntity::ReplaceLastUpdateUntreatedPart(off_t front_start, off_t front_size, off_t behind_start, off_t behind_size) { if(0 < front_size){ if(!untreated_list.ReplaceLastUpdatePart(front_start, front_size)){ return false; } }else{ if(!untreated_list.RemoveLastUpdatePart()){ return false; } } if(0 < behind_size){ if(!untreated_list.AddPart(behind_start, behind_size)){ return false; } } return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_entity.h000066400000000000000000000312641470675423500172220ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_ENTITY_H_ #define S3FS_FDCACHE_ENTITY_H_ #include #include #include #include #include #include "common.h" #include "fdcache_page.h" #include "fdcache_untreated.h" #include "metaheader.h" #include "s3fs_util.h" //---------------------------------------------- // Typedef //---------------------------------------------- class PseudoFdInfo; typedef std::map> fdinfo_map_t; //------------------------------------------------ // class FdEntity //------------------------------------------------ class FdEntity : public std::enable_shared_from_this { private: // [NOTE] // Distinguish between meta pending and new file creation pending, // because the processing(request) at these updates is different. // Therefore, the pending state is expressed by this enum type. // enum class pending_status_t : uint8_t { NO_UPDATE_PENDING = 0, UPDATE_META_PENDING, // pending meta header CREATE_FILE_PENDING // pending file creation and meta header }; static bool mixmultipart; // whether multipart uploading can use copy api. static bool streamupload; // whether stream uploading. mutable std::mutex fdent_lock; std::string path GUARDED_BY(fdent_lock); // object path int physical_fd GUARDED_BY(fdent_lock); // physical file(cache or temporary file) descriptor UntreatedParts untreated_list GUARDED_BY(fdent_lock); // list of untreated parts that have been written and not yet uploaded(for streamupload) fdinfo_map_t pseudo_fd_map GUARDED_BY(fdent_lock); // pseudo file descriptor information map std::unique_ptr pfile GUARDED_BY(fdent_lock) = {nullptr, &s3fs_fclose}; // file pointer(tmp file or cache file) ino_t inode GUARDED_BY(fdent_lock); // inode number for cache file headers_t orgmeta GUARDED_BY(fdent_lock); // original headers at opening off_t size_orgmeta GUARDED_BY(fdent_lock); // original file size in original headers mutable std::mutex fdent_data_lock; // protects the following members PageList pagelist GUARDED_BY(fdent_data_lock); std::string cachepath GUARDED_BY(fdent_data_lock); // local cache file path // (if this is empty, does not load/save pagelist.) std::string mirrorpath GUARDED_BY(fdent_data_lock); // mirror file path to local cache file path pending_status_t pending_status GUARDED_BY(fdent_data_lock); // status for new file creation and meta update struct timespec holding_mtime GUARDED_BY(fdent_data_lock); // if mtime is updated while the file is open, it is set time_t value private: static int FillFile(int fd, unsigned char byte, off_t size, off_t start); static ino_t GetInode(int fd); void Clear(); ino_t GetInode() const REQUIRES(FdEntity::fdent_data_lock); int OpenMirrorFile() REQUIRES(FdEntity::fdent_data_lock); int NoCacheLoadAndPost(PseudoFdInfo* pseudo_obj, off_t start = 0, off_t size = 0) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); // size=0 means loading to end PseudoFdInfo* CheckPseudoFdFlags(int fd, bool writable) REQUIRES(FdEntity::fdent_lock); bool IsUploading() REQUIRES(FdEntity::fdent_lock); bool SetAllStatus(bool is_loaded) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); bool SetAllStatusUnloaded() REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock) { return SetAllStatus(false); } int NoCachePreMultipartPost(PseudoFdInfo* pseudo_obj) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); int NoCacheMultipartPost(PseudoFdInfo* pseudo_obj, int tgfd, off_t start, off_t size) REQUIRES(FdEntity::fdent_lock); int NoCacheCompleteMultipartPost(PseudoFdInfo* pseudo_obj) REQUIRES(FdEntity::fdent_lock); int RowFlushHasLock(int fd, const char* tpath, bool force_sync) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); int RowFlushNoMultipart(const PseudoFdInfo* pseudo_obj, const char* tpath) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); int RowFlushMultipart(PseudoFdInfo* pseudo_obj, const char* tpath) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); int RowFlushMixMultipart(PseudoFdInfo* pseudo_obj, const char* tpath) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); int RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpath) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); ssize_t WriteNoMultipart(const PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); ssize_t WriteMultipart(PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); ssize_t WriteMixMultipart(PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); ssize_t WriteStreamUpload(PseudoFdInfo* pseudo_obj, const char* bytes, off_t start, size_t size) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); int UploadPendingHasLock(int fd) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); bool ReserveDiskSpace(off_t size) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); bool AddUntreated(off_t start, off_t size) REQUIRES(FdEntity::fdent_lock); bool IsDirtyMetadata() const REQUIRES(FdEntity::fdent_data_lock); std::shared_ptr get_shared_ptr() { return shared_from_this(); } public: static bool GetNoMixMultipart() { return mixmultipart; } static bool SetNoMixMultipart(); static bool GetStreamUpload() { return streamupload; } static bool SetStreamUpload(bool isstream); explicit FdEntity(const char* tpath = nullptr, const char* cpath = nullptr); ~FdEntity(); FdEntity(const FdEntity&) = delete; FdEntity(FdEntity&&) = delete; FdEntity& operator=(const FdEntity&) = delete; FdEntity& operator=(FdEntity&&) = delete; void Close(int fd); bool IsOpen() const { const std::lock_guard lock(fdent_lock); return (-1 != physical_fd); } bool FindPseudoFd(int fd) const { const std::lock_guard lock(fdent_lock); return FindPseudoFdWithLock(fd); } bool FindPseudoFdWithLock(int fd) const REQUIRES(FdEntity::fdent_lock); int Open(const headers_t* pmeta, off_t size, const struct timespec& ts_mctime, int flags); bool LoadAll(int fd, off_t* size = nullptr, bool force_load = false); int Dup(int fd) { const std::lock_guard lock(fdent_lock); return DupWithLock(fd); } int DupWithLock(int fd) REQUIRES(FdEntity::fdent_lock); int OpenPseudoFd(int flags = O_RDONLY); int GetOpenCount() const { const std::lock_guard lock(fdent_lock); return GetOpenCountHasLock(); } int GetOpenCountHasLock() const REQUIRES(FdEntity::fdent_lock); std::string GetPath() const { const std::lock_guard lock(fdent_lock); return path; } bool RenamePath(const std::string& newpath, std::string& fentmapkey); int GetPhysicalFd() const REQUIRES(FdEntity::fdent_lock) { return physical_fd; } bool IsModified() const; bool MergeOrgMeta(headers_t& updatemeta); int UploadPending(int fd) { const std::lock_guard lock(fdent_lock); const std::lock_guard lock_data(fdent_data_lock); return UploadPendingHasLock(fd); } bool GetStats(struct stat& st) const { const std::lock_guard lock(fdent_lock); return GetStatsHasLock(st); } bool GetStatsHasLock(struct stat& st) const REQUIRES(FdEntity::fdent_lock); int SetCtime(struct timespec time) { const std::lock_guard lock(fdent_lock); return SetCtimeHasLock(time); } int SetCtimeHasLock(struct timespec time) REQUIRES(FdEntity::fdent_lock); int SetAtime(struct timespec time) { const std::lock_guard lock(fdent_lock); return SetAtimeHasLock(time); } int SetAtimeHasLock(struct timespec time) REQUIRES(FdEntity::fdent_lock); int SetMCtime(struct timespec mtime, struct timespec ctime) { const std::lock_guard lock(fdent_lock); const std::lock_guard lock2(fdent_data_lock); return SetMCtimeHasLock(mtime, ctime); } int SetMCtimeHasLock(struct timespec mtime, struct timespec ctime) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); bool UpdateCtime(); bool UpdateAtime(); bool UpdateMtime(bool clear_holding_mtime = false); bool UpdateMCtime(); bool SetHoldingMtime(struct timespec mtime); bool ClearHoldingMtime() REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); bool GetSize(off_t& size) const; bool GetXattr(std::string& xattr) const; bool SetXattr(const std::string& xattr); bool SetMode(mode_t mode) { const std::lock_guard lock(fdent_lock); return SetModeHasLock(mode); } bool SetModeHasLock(mode_t mode) REQUIRES(FdEntity::fdent_lock); bool SetUId(uid_t uid) { const std::lock_guard lock(fdent_lock); return SetUIdHasLock(uid); } bool SetUIdHasLock(uid_t uid) REQUIRES(FdEntity::fdent_lock); bool SetGId(gid_t gid) { const std::lock_guard lock(fdent_lock); return SetGIdHasLock(gid); } bool SetGIdHasLock(gid_t gid) REQUIRES(FdEntity::fdent_lock); bool SetContentType(const char* path); int Load(off_t start, off_t size, bool is_modified_flag = false) REQUIRES(FdEntity::fdent_lock, FdEntity::fdent_data_lock); // size=0 means loading to end off_t BytesModified(); int RowFlush(int fd, const char* tpath, bool force_sync = false) { const std::lock_guard lock(fdent_lock); const std::lock_guard lock_data(fdent_data_lock); return RowFlushHasLock(fd, tpath, force_sync); } int Flush(int fd, bool force_sync = false) { return RowFlush(fd, nullptr, force_sync); } ssize_t Read(int fd, char* bytes, off_t start, size_t size, bool force_load = false); ssize_t Write(int fd, const char* bytes, off_t start, size_t size); bool PunchHole(off_t start = 0, size_t size = 0); void MarkDirtyNewFile(); bool IsDirtyNewFile() const; void MarkDirtyMetadata(); bool GetLastUpdateUntreatedPart(off_t& start, off_t& size) const REQUIRES(FdEntity::fdent_lock); bool ReplaceLastUpdateUntreatedPart(off_t front_start, off_t front_size, off_t behind_start, off_t behind_size) REQUIRES(FdEntity::fdent_lock); // Intentionally unimplemented -- for lock checking only. std::mutex* GetMutex() RETURN_CAPABILITY(fdent_lock); }; typedef std::map> fdent_map_t; // key=path, value=FdEntity #endif // S3FS_FDCACHE_ENTITY_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_fdinfo.cpp000066400000000000000000001222171470675423500175050ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include "common.h" #include "s3fs_logger.h" #include "s3fs_util.h" #include "fdcache_fdinfo.h" #include "fdcache_pseudofd.h" #include "fdcache_entity.h" #include "curl.h" #include "string_util.h" #include "threadpoolman.h" //------------------------------------------------ // PseudoFdInfo class variables //------------------------------------------------ int PseudoFdInfo::max_threads = -1; int PseudoFdInfo::opt_max_threads = -1; //------------------------------------------------ // PseudoFdInfo class methods //------------------------------------------------ // // Worker function for uploading // void* PseudoFdInfo::MultipartUploadThreadWorker(void* arg) { std::unique_ptr pthparam(static_cast(arg)); if(!pthparam || !(pthparam->ppseudofdinfo)){ return reinterpret_cast(-EIO); } S3FS_PRN_INFO3("Upload Part Thread [tpath=%s][start=%lld][size=%lld][part=%d]", pthparam->path.c_str(), static_cast(pthparam->start), static_cast(pthparam->size), pthparam->part_num); int result; { const std::lock_guard lock(pthparam->ppseudofdinfo->upload_list_lock); if(0 != (result = pthparam->ppseudofdinfo->last_result)){ S3FS_PRN_DBG("Already occurred error, thus this thread worker is exiting."); if(!pthparam->ppseudofdinfo->CompleteInstruction(result)){ // result will be overwritten with the same value. result = -EIO; } return reinterpret_cast(result); } } // setup and make curl object std::unique_ptr s3fscurl(S3fsCurl::CreateParallelS3fsCurl(pthparam->path.c_str(), pthparam->upload_fd, pthparam->start, pthparam->size, pthparam->part_num, pthparam->is_copy, pthparam->petag, pthparam->upload_id, result)); if(nullptr == s3fscurl){ S3FS_PRN_ERR("failed creating s3fs curl object for uploading [path=%s][start=%lld][size=%lld][part=%d]", pthparam->path.c_str(), static_cast(pthparam->start), static_cast(pthparam->size), pthparam->part_num); // set result for exiting const std::lock_guard lock(pthparam->ppseudofdinfo->upload_list_lock); if(!pthparam->ppseudofdinfo->CompleteInstruction(result)){ result = -EIO; } return reinterpret_cast(result); } // Send request and get result if(0 == (result = s3fscurl->RequestPerform())){ S3FS_PRN_DBG("succeed uploading [path=%s][start=%lld][size=%lld][part=%d]", pthparam->path.c_str(), static_cast(pthparam->start), static_cast(pthparam->size), pthparam->part_num); if(!s3fscurl->MixMultipartPostComplete()){ S3FS_PRN_ERR("failed completion uploading [path=%s][start=%lld][size=%lld][part=%d]", pthparam->path.c_str(), static_cast(pthparam->start), static_cast(pthparam->size), pthparam->part_num); result = -EIO; } }else{ S3FS_PRN_ERR("failed uploading with error(%d) [path=%s][start=%lld][size=%lld][part=%d]", result, pthparam->path.c_str(), static_cast(pthparam->start), static_cast(pthparam->size), pthparam->part_num); } s3fscurl->DestroyCurlHandle(true, false); // set result const std::lock_guard lock(pthparam->ppseudofdinfo->upload_list_lock); if(!pthparam->ppseudofdinfo->CompleteInstruction(result)){ S3FS_PRN_WARN("This thread worker is about to end, so it doesn't return an EIO here and runs to the end."); } return reinterpret_cast(result); } //------------------------------------------------ // PseudoFdInfo methods //------------------------------------------------ PseudoFdInfo::PseudoFdInfo(int fd, int open_flags) : pseudo_fd(-1), physical_fd(fd), flags(0), upload_fd(-1), instruct_count(0), completed_count(0), last_result(0), uploaded_sem(0) { if(-1 != physical_fd){ pseudo_fd = PseudoFdManager::Get(); flags = open_flags; } } PseudoFdInfo::~PseudoFdInfo() { Clear(); // call before destroying the mutex } bool PseudoFdInfo::Clear() { // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!CancelAllThreads()){ return false; } { const std::lock_guard lock(upload_list_lock); // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!ResetUploadInfo()){ return false; } } CloseUploadFd(); if(-1 != pseudo_fd){ PseudoFdManager::Release(pseudo_fd); } pseudo_fd = -1; physical_fd = -1; return true; } bool PseudoFdInfo::IsUploadingHasLock() const { return !upload_id.empty(); } bool PseudoFdInfo::IsUploading() const { const std::lock_guard lock(upload_list_lock); return IsUploadingHasLock(); } void PseudoFdInfo::CloseUploadFd() { const std::lock_guard lock(upload_list_lock); if(-1 != upload_fd){ close(upload_fd); } } bool PseudoFdInfo::OpenUploadFd() { const std::lock_guard lock(upload_list_lock); if(-1 != upload_fd){ // already initialized return true; } if(-1 == physical_fd){ S3FS_PRN_ERR("physical_fd is not initialized yet."); return false; } // duplicate fd int fd; if(-1 == (fd = dup(physical_fd))){ S3FS_PRN_ERR("Could not duplicate physical file descriptor(errno=%d)", errno); return false; } scope_guard guard([&]() { close(fd); }); if(0 != lseek(fd, 0, SEEK_SET)){ S3FS_PRN_ERR("Could not seek physical file descriptor(errno=%d)", errno); return false; } struct stat st; if(-1 == fstat(fd, &st)){ S3FS_PRN_ERR("Invalid file descriptor for uploading(errno=%d)", errno); return false; } guard.dismiss(); upload_fd = fd; return true; } bool PseudoFdInfo::Set(int fd, int open_flags) { if(-1 == fd){ return false; } Clear(); physical_fd = fd; pseudo_fd = PseudoFdManager::Get(); flags = open_flags; return true; } bool PseudoFdInfo::Writable() const { if(-1 == pseudo_fd){ return false; } if(0 == (flags & (O_WRONLY | O_RDWR))){ return false; } return true; } bool PseudoFdInfo::Readable() const { if(-1 == pseudo_fd){ return false; } // O_RDONLY is 0x00, it means any pattern is readable. return true; } bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp) { if(is_cancel_mp){ // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!CancelAllThreads()){ return false; } } const std::lock_guard lock(upload_list_lock); return ResetUploadInfo(); } bool PseudoFdInfo::ResetUploadInfo() { upload_id.clear(); upload_list.clear(); instruct_count = 0; completed_count = 0; last_result = 0; return true; } bool PseudoFdInfo::RowInitialUploadInfo(const std::string& id, bool is_cancel_mp) { if(is_cancel_mp){ // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!ClearUploadInfo(is_cancel_mp)){ return false; } }else{ const std::lock_guard lock(upload_list_lock); // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!ResetUploadInfo()){ return false; } } const std::lock_guard lock(upload_list_lock); upload_id = id; return true; } void PseudoFdInfo::IncreaseInstructionCount() { const std::lock_guard lock(upload_list_lock); ++instruct_count; } bool PseudoFdInfo::CompleteInstruction(int result) { if(0 != result){ last_result = result; } if(0 >= instruct_count){ S3FS_PRN_ERR("Internal error: instruct_count caused an underflow."); return false; } --instruct_count; ++completed_count; return true; } bool PseudoFdInfo::GetUploadInfo(std::string& id, int& fd) const { const std::lock_guard lock(upload_list_lock); if(!IsUploadingHasLock()){ S3FS_PRN_ERR("Multipart Upload has not started yet."); return false; } id = upload_id; fd = upload_fd; return true; } bool PseudoFdInfo::GetUploadId(std::string& id) const { int fd = -1; return GetUploadInfo(id, fd); } bool PseudoFdInfo::GetEtaglist(etaglist_t& list) const { const std::lock_guard lock(upload_list_lock); if(!IsUploadingHasLock()){ S3FS_PRN_ERR("Multipart Upload has not started yet."); return false; } list.clear(); for(auto iter = upload_list.cbegin(); iter != upload_list.cend(); ++iter){ if(iter->petag){ list.push_back(*(iter->petag)); }else{ S3FS_PRN_ERR("The pointer to the etag string is null(internal error)."); return false; } } return !list.empty(); } // [NOTE] // This method adds a part for a multipart upload. // The added new part must be an area that is exactly continuous with the // immediately preceding part. // An error will occur if it is discontinuous or if it overlaps with an // existing area. // bool PseudoFdInfo::AppendUploadPart(off_t start, off_t size, bool is_copy, etagpair** ppetag) { const std::lock_guard lock(upload_list_lock); if(!IsUploadingHasLock()){ S3FS_PRN_ERR("Multipart Upload has not started yet."); return false; } off_t next_start_pos = 0; if(!upload_list.empty()){ next_start_pos = upload_list.back().startpos + upload_list.back().size; } if(start != next_start_pos){ S3FS_PRN_ERR("The expected starting position for the next part is %lld, but %lld was specified.", static_cast(next_start_pos), static_cast(start)); return false; } // make part number int partnumber = static_cast(upload_list.size()) + 1; // add new part etagpair* petag_entity = etag_entities.add(etagpair(nullptr, partnumber)); // [NOTE] Create the etag entity and register it in the list. upload_list.emplace_back(false, physical_fd, start, size, is_copy, petag_entity); // set etag pointer if(ppetag){ *ppetag = petag_entity; } return true; } // // Utility for sorting upload list // static bool filepart_partnum_compare(const filepart& src1, const filepart& src2) { return src1.get_part_number() < src2.get_part_number(); } bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag) { const std::lock_guard lock(upload_list_lock); //S3FS_PRN_DBG("[start=%lld][size=%lld][part_num=%d][is_copy=%s]", static_cast(start), static_cast(size), part_num, (is_copy ? "true" : "false")); if(!IsUploadingHasLock()){ S3FS_PRN_ERR("Multipart Upload has not started yet."); return false; } if(start < 0 || size <= 0 || part_num < 0 || !ppetag){ S3FS_PRN_ERR("Parameters are wrong."); return false; } // insert new part etagpair* petag_entity = etag_entities.add(etagpair(nullptr, part_num)); upload_list.emplace_back(false, physical_fd, start, size, is_copy, petag_entity); // sort by part number std::sort(upload_list.begin(), upload_list.end(), filepart_partnum_compare); // set etag pointer *ppetag = petag_entity; return true; } // [NOTE] // This method only launches the upload thread. // Check the maximum number of threads before calling. // bool PseudoFdInfo::ParallelMultipartUpload(const char* path, const mp_part_list_t& mplist, bool is_copy) { //S3FS_PRN_DBG("[path=%s][mplist(%zu)]", SAFESTRPTR(path), mplist.size()); if(mplist.empty()){ // nothing to do return true; } if(!OpenUploadFd()){ return false; } // Get upload id/fd before loop std::string tmp_upload_id; int tmp_upload_fd = -1; if(!GetUploadInfo(tmp_upload_id, tmp_upload_fd)){ return false; } for(auto iter = mplist.cbegin(); iter != mplist.cend(); ++iter){ // Insert upload part etagpair* petag = nullptr; if(!InsertUploadPart(iter->start, iter->size, iter->part_num, is_copy, &petag)){ S3FS_PRN_ERR("Failed to insert insert upload part(path=%s, start=%lld, size=%lld, part=%d, copy=%s) to mplist", SAFESTRPTR(path), static_cast(iter->start), static_cast(iter->size), iter->part_num, (is_copy ? "true" : "false")); return false; } // make parameter for my thread auto* thargs = new pseudofdinfo_thparam; thargs->ppseudofdinfo = this; thargs->path = SAFESTRPTR(path); thargs->upload_id = tmp_upload_id; thargs->upload_fd = tmp_upload_fd; thargs->start = iter->start; thargs->size = iter->size; thargs->is_copy = is_copy; thargs->part_num = iter->part_num; thargs->petag = petag; // make parameter for thread pool thpoolman_param ppoolparam; ppoolparam.args = thargs; ppoolparam.psem = &uploaded_sem; ppoolparam.pfunc = PseudoFdInfo::MultipartUploadThreadWorker; // setup instruction if(!ThreadPoolMan::Instruct(ppoolparam)){ S3FS_PRN_ERR("failed setup instruction for uploading."); delete thargs; return false; } IncreaseInstructionCount(); } return true; } bool PseudoFdInfo::ParallelMultipartUploadAll(const char* path, const mp_part_list_t& to_upload_list, const mp_part_list_t& copy_list, int& result) { S3FS_PRN_DBG("[path=%s][to_upload_list(%zu)][copy_list(%zu)]", SAFESTRPTR(path), to_upload_list.size(), copy_list.size()); result = 0; if(!OpenUploadFd()){ return false; } if(!ParallelMultipartUpload(path, to_upload_list, false) || !ParallelMultipartUpload(path, copy_list, true)){ S3FS_PRN_ERR("Failed setup instruction for uploading(path=%s, to_upload_list=%zu, copy_list=%zu).", SAFESTRPTR(path), to_upload_list.size(), copy_list.size()); return false; } // Wait for all thread exiting result = WaitAllThreadsExit(); return true; } // // Upload the last updated Untreated area // // [Overview] // Uploads untreated areas with the maximum multipart upload size as the // boundary. // // * The starting position of the untreated area is aligned with the maximum // multipart upload size as the boundary. // * If there is an uploaded area that overlaps with the aligned untreated // area, that uploaded area is canceled and absorbed by the untreated area. // * Upload only when the aligned untreated area exceeds the maximum multipart // upload size. // * When the start position of the untreated area is changed to boundary // alignment(to backward), and if that gap area is remained, that area is // rest to untreated area. // ssize_t PseudoFdInfo::UploadBoundaryLastUntreatedArea(const char* path, headers_t& meta, FdEntity* pfdent) { S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(path), pseudo_fd, physical_fd); if(!path || -1 == physical_fd || -1 == pseudo_fd || !pfdent){ S3FS_PRN_ERR("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable, or pfdent is nullptr.", pseudo_fd, physical_fd, path); return -EBADF; } // // Get last update untreated area // off_t last_untreated_start = 0; off_t last_untreated_size = 0; if(!pfdent->GetLastUpdateUntreatedPart(last_untreated_start, last_untreated_size) || last_untreated_start < 0 || last_untreated_size <= 0){ S3FS_PRN_WARN("Not found last update untreated area or it is empty, thus return without any error."); return 0; } // // Aligns the start position of the last updated raw area with the boundary // // * Align the last updated raw space with the maximum upload size boundary. // * The remaining size of the part before the boundary is will not be uploaded. // off_t max_mp_size = S3fsCurl::GetMultipartSize(); off_t aligned_start = ((last_untreated_start / max_mp_size) + (0 < (last_untreated_start % max_mp_size) ? 1 : 0)) * max_mp_size; if((last_untreated_start + last_untreated_size) <= aligned_start){ S3FS_PRN_INFO("After the untreated area(start=%lld, size=%lld) is aligned with the boundary, the aligned start(%lld) exceeds the untreated area, so there is nothing to do.", static_cast(last_untreated_start), static_cast(last_untreated_size), static_cast(aligned_start)); return 0; } off_t aligned_size = (((last_untreated_start + last_untreated_size) - aligned_start) / max_mp_size) * max_mp_size; if(0 == aligned_size){ S3FS_PRN_DBG("After the untreated area(start=%lld, size=%lld) is aligned with the boundary(start is %lld), the aligned size is empty, so nothing to do.", static_cast(last_untreated_start), static_cast(last_untreated_size), static_cast(aligned_start)); return 0; } off_t front_rem_start = last_untreated_start; // start of the remainder untreated area in front of the boundary off_t front_rem_size = aligned_start - last_untreated_start; // size of the remainder untreated area in front of the boundary // // Get the area for uploading, if last update treated area can be uploaded. // // [NOTE] // * Create the updoad area list, if the untreated area aligned with the boundary // exceeds the maximum upload size. // * If it overlaps with an area that has already been uploaded(unloaded list), // that area is added to the cancellation list and included in the untreated area. // mp_part_list_t to_upload_list; filepart_list_t cancel_uploaded_list; if(!ExtractUploadPartsFromUntreatedArea(aligned_start, aligned_size, to_upload_list, cancel_uploaded_list, S3fsCurl::GetMultipartSize())){ S3FS_PRN_ERR("Failed to extract upload parts from last untreated area."); return -EIO; } if(to_upload_list.empty()){ S3FS_PRN_INFO("There is nothing to upload. In most cases, the untreated area does not meet the upload size."); return 0; } // // Has multipart uploading already started? // if(!IsUploading()){ // Multipart uploading hasn't started yet, so start it. // S3fsCurl s3fscurl(true); std::string tmp_upload_id; int result; if(0 != (result = s3fscurl.PreMultipartPostRequest(path, meta, tmp_upload_id))){ S3FS_PRN_ERR("failed to setup multipart upload(create upload id) by errno(%d)", result); return result; } if(!RowInitialUploadInfo(tmp_upload_id, false/* not need to cancel */)){ S3FS_PRN_ERR("failed to setup multipart upload(set upload id to object)"); return result; } } // // Output debug level information // // When canceling(overwriting) a part that has already been uploaded, output it. // if(S3fsLog::IsS3fsLogDbg()){ for(auto cancel_iter = cancel_uploaded_list.cbegin(); cancel_iter != cancel_uploaded_list.cend(); ++cancel_iter){ S3FS_PRN_DBG("Cancel uploaded: start(%lld), size(%lld), part number(%d)", static_cast(cancel_iter->startpos), static_cast(cancel_iter->size), (cancel_iter->petag ? cancel_iter->petag->part_num : -1)); } } // // Upload Multipart parts // if(!ParallelMultipartUpload(path, to_upload_list, false)){ S3FS_PRN_ERR("Failed to upload multipart parts."); return -EIO; } // // Exclude the uploaded Untreated area and update the last Untreated area. // off_t behind_rem_start = aligned_start + aligned_size; off_t behind_rem_size = (last_untreated_start + last_untreated_size) - behind_rem_start; if(!pfdent->ReplaceLastUpdateUntreatedPart(front_rem_start, front_rem_size, behind_rem_start, behind_rem_size)){ S3FS_PRN_WARN("The last untreated area could not be detected and the uploaded area could not be excluded from it, but continue because it does not affect the overall processing."); } return 0; } int PseudoFdInfo::WaitAllThreadsExit() { int result; bool is_loop = true; { const std::lock_guard lock(upload_list_lock); if(0 == instruct_count && 0 == completed_count){ result = last_result; is_loop = false; } } while(is_loop){ // need to wait the worker exiting uploaded_sem.wait(); { const std::lock_guard lock(upload_list_lock); if(0 < completed_count){ --completed_count; } if(0 == instruct_count && 0 == completed_count){ // break loop result = last_result; is_loop = false; } } } return result; } bool PseudoFdInfo::CancelAllThreads() { bool need_cancel = false; { const std::lock_guard lock(upload_list_lock); if(0 < instruct_count && 0 < completed_count){ S3FS_PRN_INFO("The upload thread is running, so cancel them and wait for the end."); need_cancel = true; last_result = -ECANCELED; // to stop thread running } } if(need_cancel){ WaitAllThreadsExit(); } return true; } // // Extract the list for multipart upload from the Unteated Area // // The untreated_start parameter must be set aligning it with the boundaries // of the maximum multipart upload size. This method expects it to be bounded. // // This method creates the upload area aligned from the untreated area by // maximum size and creates the required list. // If it overlaps with an area that has already been uploaded, the overlapped // upload area will be canceled and absorbed by the untreated area. // If the list creation process is complete and areas smaller than the maximum // size remain, those area will be reset to untreated_start and untreated_size // and returned to the caller. // If the called untreated area is smaller than the maximum size of the // multipart upload, no list will be created. // // [NOTE] // Maximum multipart upload size must be uploading boundary. // bool PseudoFdInfo::ExtractUploadPartsFromUntreatedArea(off_t untreated_start, off_t untreated_size, mp_part_list_t& to_upload_list, filepart_list_t& cancel_upload_list, off_t max_mp_size) { if(untreated_start < 0 || untreated_size <= 0){ S3FS_PRN_ERR("Parameters are wrong(untreated_start=%lld, untreated_size=%lld).", static_cast(untreated_start), static_cast(untreated_size)); return false; } // Initialize lists to_upload_list.clear(); cancel_upload_list.clear(); // // Align start position with maximum multipart upload boundaries // off_t aligned_start = (untreated_start / max_mp_size) * max_mp_size; off_t aligned_size = untreated_size + (untreated_start - aligned_start); // // Check aligned untreated size // if(aligned_size < max_mp_size){ S3FS_PRN_INFO("untreated area(start=%lld, size=%lld) to aligned boundary(start=%lld, size=%lld) is smaller than max mp size(%lld), so nothing to do.", static_cast(untreated_start), static_cast(untreated_size), static_cast(aligned_start), static_cast(aligned_size), static_cast(max_mp_size)); return true; // successful termination } // // Check each unloaded area in list // // [NOTE] // The uploaded area must be to be aligned by boundary. // Also, it is assumed that it must not be a copy area. // So if the areas overlap, include uploaded area as an untreated area. // { const std::lock_guard lock(upload_list_lock); for(auto cur_iter = upload_list.begin(); cur_iter != upload_list.end(); /* ++cur_iter */){ // Check overlap if((cur_iter->startpos + cur_iter->size - 1) < aligned_start || (aligned_start + aligned_size - 1) < cur_iter->startpos){ // Areas do not overlap ++cur_iter; }else{ // The areas overlap // // Since the start position of the uploaded area is aligned with the boundary, // it is not necessary to check the start position. // If the uploaded area exceeds the untreated area, expand the untreated area. // if((aligned_start + aligned_size - 1) < (cur_iter->startpos + cur_iter->size - 1)){ aligned_size += (cur_iter->startpos + cur_iter->size) - (aligned_start + aligned_size); } // // Add this to cancel list // cancel_upload_list.push_back(*cur_iter); // Copy and Push to cancel list cur_iter = upload_list.erase(cur_iter); } } } // // Add upload area to the list // while(max_mp_size <= aligned_size){ int part_num = static_cast((aligned_start / max_mp_size) + 1); to_upload_list.emplace_back(aligned_start, max_mp_size, part_num); aligned_start += max_mp_size; aligned_size -= max_mp_size; } return true; } // // Extract the area lists to be uploaded/downloaded for the entire file. // // [Parameters] // to_upload_list : A list of areas to upload in multipart upload. // to_copy_list : A list of areas for copy upload in multipart upload. // to_download_list : A list of areas that must be downloaded before multipart upload. // cancel_upload_list : A list of areas that have already been uploaded and will be canceled(overwritten). // wait_upload_complete : If cancellation areas exist, this flag is set to true when it is necessary to wait until the upload of those cancellation areas is complete. // file_size : The size of the upload file. // use_copy : Specify true if copy multipart upload is available. // // [NOTE] // The untreated_list in fdentity does not change, but upload_list is changed. // (If you want to restore it, you can use cancel_upload_list.) // bool PseudoFdInfo::ExtractUploadPartsFromAllArea(UntreatedParts& untreated_list, mp_part_list_t& to_upload_list, mp_part_list_t& to_copy_list, mp_part_list_t& to_download_list, filepart_list_t& cancel_upload_list, bool& wait_upload_complete, off_t max_mp_size, off_t file_size, bool use_copy) { const std::lock_guard lock(upload_list_lock); // Initialize lists to_upload_list.clear(); to_copy_list.clear(); to_download_list.clear(); cancel_upload_list.clear(); wait_upload_complete = false; // Duplicate untreated list untreated_list_t dup_untreated_list; untreated_list.Duplicate(dup_untreated_list); // Initialize the iterator of each list first auto dup_untreated_iter = dup_untreated_list.begin(); auto uploaded_iter = upload_list.begin(); // // Loop to extract areas to upload and download // // Check at the boundary of the maximum upload size from the beginning of the file // for(off_t cur_start = 0, cur_size = 0; cur_start < file_size; cur_start += cur_size){ // // Set part size // (To avoid confusion, the area to be checked is called the "current area".) // cur_size = ((cur_start + max_mp_size) <= file_size ? max_mp_size : (file_size - cur_start)); // // Extract the untreated erea that overlaps this current area. // (The extracted area is deleted from dup_untreated_list.) // untreated_list_t cur_untreated_list; for(cur_untreated_list.clear(); dup_untreated_iter != dup_untreated_list.end(); ){ if((dup_untreated_iter->start < (cur_start + cur_size)) && (cur_start < (dup_untreated_iter->start + dup_untreated_iter->size))){ // this untreated area is overlap off_t tmp_untreated_start; off_t tmp_untreated_size; if(dup_untreated_iter->start < cur_start){ // [NOTE] // This untreated area overlaps with the current area, but starts // in front of the target area. // This state should not be possible, but if this state is detected, // the part before the target area will be deleted. // tmp_untreated_start = cur_start; tmp_untreated_size = dup_untreated_iter->size - (cur_start - dup_untreated_iter->start); }else{ tmp_untreated_start = dup_untreated_iter->start; tmp_untreated_size = dup_untreated_iter->size; } // // Check the end of the overlapping untreated area. // if((tmp_untreated_start + tmp_untreated_size) <= (cur_start + cur_size)){ // // All of untreated areas are within the current area // // - Add this untreated area to cur_untreated_list // - Delete this from dup_untreated_list // cur_untreated_list.emplace_back(tmp_untreated_start, tmp_untreated_size); dup_untreated_iter = dup_untreated_list.erase(dup_untreated_iter); }else{ // // The untreated area exceeds the end of the current area // // Adjust untreated area tmp_untreated_size = (cur_start + cur_size) - tmp_untreated_start; // Add adjusted untreated area to cur_untreated_list cur_untreated_list.emplace_back(tmp_untreated_start, tmp_untreated_size); // Remove this adjusted untreated area from the area pointed // to by dup_untreated_iter. dup_untreated_iter->size = (dup_untreated_iter->start + dup_untreated_iter->size) - (cur_start + cur_size); dup_untreated_iter->start = tmp_untreated_start + tmp_untreated_size; } }else if((cur_start + cur_size - 1) < dup_untreated_iter->start){ // this untreated area is over the current area, thus break loop. break; }else{ ++dup_untreated_iter; } } // // Check uploaded area // // [NOTE] // The uploaded area should be aligned with the maximum upload size boundary. // It also assumes that each size of uploaded area must be a maximum upload // size. // auto overlap_uploaded_iter = upload_list.end(); for(; uploaded_iter != upload_list.end(); ++uploaded_iter){ if((cur_start < (uploaded_iter->startpos + uploaded_iter->size)) && (uploaded_iter->startpos < (cur_start + cur_size))){ if(overlap_uploaded_iter != upload_list.end()){ // // Something wrong in this unloaded area. // // This area is not aligned with the boundary, then this condition // is unrecoverable and return failure. // S3FS_PRN_ERR("The uploaded list may not be the boundary for the maximum multipart upload size. No further processing is possible."); return false; } // Set this iterator to overlap iter overlap_uploaded_iter = uploaded_iter; }else if((cur_start + cur_size - 1) < uploaded_iter->startpos){ break; } } // // Create upload/download/cancel/copy list for this current area // int part_num = static_cast((cur_start / max_mp_size) + 1); if(cur_untreated_list.empty()){ // // No untreated area was detected in this current area // if(overlap_uploaded_iter != upload_list.end()){ // // This current area already uploaded, then nothing to add to lists. // S3FS_PRN_DBG("Already uploaded: start=%lld, size=%lld", static_cast(cur_start), static_cast(cur_size)); }else{ // // This current area has not been uploaded // (neither an uploaded area nor an untreated area.) // if(use_copy){ // // Copy multipart upload available // S3FS_PRN_DBG("To copy: start=%lld, size=%lld", static_cast(cur_start), static_cast(cur_size)); to_copy_list.emplace_back(cur_start, cur_size, part_num); }else{ // // This current area needs to be downloaded and uploaded // S3FS_PRN_DBG("To download and upload: start=%lld, size=%lld", static_cast(cur_start), static_cast(cur_size)); to_download_list.emplace_back(cur_start, cur_size); to_upload_list.emplace_back(cur_start, cur_size, part_num); } } }else{ // // Found untreated area in this current area // if(overlap_uploaded_iter != upload_list.end()){ // // This current area is also the uploaded area // // [NOTE] // The uploaded area is aligned with boundary, there are all data in // this current area locally(which includes all data of untreated area). // So this current area only needs to be uploaded again. // S3FS_PRN_DBG("Cancel upload: start=%lld, size=%lld", static_cast(overlap_uploaded_iter->startpos), static_cast(overlap_uploaded_iter->size)); if(!overlap_uploaded_iter->uploaded){ S3FS_PRN_DBG("This cancel upload area is still uploading, so you must wait for it to complete before starting any Stream uploads."); wait_upload_complete = true; } cancel_upload_list.push_back(*overlap_uploaded_iter); // add this uploaded area to cancel_upload_list uploaded_iter = upload_list.erase(overlap_uploaded_iter); // remove it from upload_list S3FS_PRN_DBG("To upload: start=%lld, size=%lld", static_cast(cur_start), static_cast(cur_size)); to_upload_list.emplace_back(cur_start, cur_size, part_num); // add new uploading area to list }else{ // // No uploaded area overlap this current area // (Areas other than the untreated area must be downloaded.) // // [NOTE] // Need to consider the case where there is a gap between the start // of the current area and the untreated area. // This gap is the area that should normally be downloaded. // But it is the area that can be copied if we can use copy multipart // upload. Then If we can use copy multipart upload and the previous // area is used copy multipart upload, this gap will be absorbed by // the previous area. // Unifying the copy multipart upload area can reduce the number of // upload requests. // off_t tmp_cur_start = cur_start; off_t tmp_cur_size = cur_size; off_t changed_start = cur_start; off_t changed_size = cur_size; bool first_area = true; for(auto tmp_cur_untreated_iter = cur_untreated_list.cbegin(); tmp_cur_untreated_iter != cur_untreated_list.cend(); ++tmp_cur_untreated_iter, first_area = false){ if(tmp_cur_start < tmp_cur_untreated_iter->start){ // // Detected a gap at the start of area // bool include_prev_copy_part = false; if(first_area && use_copy && !to_copy_list.empty()){ // // Make sure that the area of the last item in to_copy_list // is contiguous with this current area. // // [NOTE] // Areas can be unified if the total size of the areas is // within 5GB and the remaining area after unification is // larger than the minimum multipart upload size. // auto copy_riter = to_copy_list.rbegin(); if( (copy_riter->start + copy_riter->size) == tmp_cur_start && (copy_riter->size + (tmp_cur_untreated_iter->start - tmp_cur_start)) <= FIVE_GB && ((tmp_cur_start + tmp_cur_size) - tmp_cur_untreated_iter->start) >= MIN_MULTIPART_SIZE ) { // // Unify to this area to previous copy area. // copy_riter->size += tmp_cur_untreated_iter->start - tmp_cur_start; S3FS_PRN_DBG("Resize to copy: start=%lld, size=%lld", static_cast(copy_riter->start), static_cast(copy_riter->size)); changed_size -= (tmp_cur_untreated_iter->start - changed_start); changed_start = tmp_cur_untreated_iter->start; include_prev_copy_part = true; } } if(!include_prev_copy_part){ // // If this area is not unified, need to download this area // S3FS_PRN_DBG("To download: start=%lld, size=%lld", static_cast(tmp_cur_start), static_cast(tmp_cur_untreated_iter->start - tmp_cur_start)); to_download_list.emplace_back(tmp_cur_start, tmp_cur_untreated_iter->start - tmp_cur_start); } } // // Set next start position // tmp_cur_size = (tmp_cur_start + tmp_cur_size) - (tmp_cur_untreated_iter->start + tmp_cur_untreated_iter->size); tmp_cur_start = tmp_cur_untreated_iter->start + tmp_cur_untreated_iter->size; } // // Add download area to list, if remaining size // if(0 < tmp_cur_size){ S3FS_PRN_DBG("To download: start=%lld, size=%lld", static_cast(tmp_cur_start), static_cast(tmp_cur_size)); to_download_list.emplace_back(tmp_cur_start, tmp_cur_size); } // // Set upload area(whole of area) to list // S3FS_PRN_DBG("To upload: start=%lld, size=%lld", static_cast(changed_start), static_cast(changed_size)); to_upload_list.emplace_back(changed_start, changed_size, part_num); } } } return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_fdinfo.h000066400000000000000000000133611470675423500171510ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_FDINFO_H_ #define S3FS_FDCACHE_FDINFO_H_ #include #include #include #include "common.h" #include "fdcache_entity.h" #include "psemaphore.h" #include "metaheader.h" #include "types.h" class UntreatedParts; //------------------------------------------------ // Structure of parameters to pass to thread //------------------------------------------------ class PseudoFdInfo; struct pseudofdinfo_thparam { PseudoFdInfo* ppseudofdinfo = nullptr; std::string path; std::string upload_id; int upload_fd = -1; off_t start = 0; off_t size = 0; bool is_copy = false; int part_num = -1; etagpair* petag = nullptr; }; //------------------------------------------------ // Class PseudoFdInfo //------------------------------------------------ class PseudoFdInfo { private: static int max_threads; static int opt_max_threads; // for option value int pseudo_fd; int physical_fd; int flags; // flags at open mutable std::mutex upload_list_lock; // protects upload_id/fd, upload_list, etc. std::string upload_id GUARDED_BY(upload_list_lock); // int upload_fd GUARDED_BY(upload_list_lock); // duplicated fd for uploading filepart_list_t upload_list GUARDED_BY(upload_list_lock); petagpool etag_entities GUARDED_BY(upload_list_lock); // list of etag string and part number entities(to maintain the etag entity even if MPPART_INFO is destroyed) int instruct_count GUARDED_BY(upload_list_lock); // number of instructions for processing by threads int completed_count GUARDED_BY(upload_list_lock); // number of completed processes by thread int last_result GUARDED_BY(upload_list_lock); // the result of thread processing Semaphore uploaded_sem; // use a semaphore to trigger an upload completion like event flag private: static void* MultipartUploadThreadWorker(void* arg); bool Clear(); void CloseUploadFd(); bool OpenUploadFd(); bool ResetUploadInfo() REQUIRES(upload_list_lock); bool RowInitialUploadInfo(const std::string& id, bool is_cancel_mp); void IncreaseInstructionCount(); bool CompleteInstruction(int result) REQUIRES(upload_list_lock); bool GetUploadInfo(std::string& id, int& fd) const; bool ParallelMultipartUpload(const char* path, const mp_part_list_t& mplist, bool is_copy); bool InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag); bool CancelAllThreads(); bool ExtractUploadPartsFromUntreatedArea(off_t untreated_start, off_t untreated_size, mp_part_list_t& to_upload_list, filepart_list_t& cancel_upload_list, off_t max_mp_size); bool IsUploadingHasLock() const REQUIRES(upload_list_lock); public: explicit PseudoFdInfo(int fd = -1, int open_flags = 0); ~PseudoFdInfo(); PseudoFdInfo(const PseudoFdInfo&) = delete; PseudoFdInfo(PseudoFdInfo&&) = delete; PseudoFdInfo& operator=(const PseudoFdInfo&) = delete; PseudoFdInfo& operator=(PseudoFdInfo&&) = delete; int GetPhysicalFd() const { return physical_fd; } int GetPseudoFd() const { return pseudo_fd; } int GetFlags() const { return flags; } bool Writable() const; bool Readable() const; bool Set(int fd, int open_flags); bool ClearUploadInfo(bool is_cancel_mp = false); bool InitialUploadInfo(const std::string& id){ return RowInitialUploadInfo(id, true); } bool IsUploading() const; bool GetUploadId(std::string& id) const; bool GetEtaglist(etaglist_t& list) const; bool AppendUploadPart(off_t start, off_t size, bool is_copy = false, etagpair** ppetag = nullptr); bool ParallelMultipartUploadAll(const char* path, const mp_part_list_t& to_upload_list, const mp_part_list_t& copy_list, int& result); int WaitAllThreadsExit(); ssize_t UploadBoundaryLastUntreatedArea(const char* path, headers_t& meta, FdEntity* pfdent) REQUIRES(pfdent->GetMutex()); bool ExtractUploadPartsFromAllArea(UntreatedParts& untreated_list, mp_part_list_t& to_upload_list, mp_part_list_t& to_copy_list, mp_part_list_t& to_download_list, filepart_list_t& cancel_upload_list, bool& wait_upload_complete, off_t max_mp_size, off_t file_size, bool use_copy); }; typedef std::map> fdinfo_map_t; #endif // S3FS_FDCACHE_FDINFO_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_page.cpp000066400000000000000000001061231470675423500171520ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include "common.h" #include "s3fs_logger.h" #include "fdcache_page.h" #include "fdcache_stat.h" #include "string_util.h" //------------------------------------------------ // Symbols //------------------------------------------------ static constexpr int CHECK_CACHEFILE_PART_SIZE = 1024 * 16; // Buffer size in PageList::CheckZeroAreaInFile() //------------------------------------------------ // fdpage_list_t utility //------------------------------------------------ // Inline function for repeated processing static inline void raw_add_compress_fdpage_list(fdpage_list_t& pagelist, const fdpage& orgpage, bool ignore_load, bool ignore_modify, bool default_load, bool default_modify) { if(0 < orgpage.bytes){ // [NOTE] // The page variable is subject to change here. // fdpage page = orgpage; if(ignore_load){ page.loaded = default_load; } if(ignore_modify){ page.modified = default_modify; } pagelist.push_back(page); } } // Compress the page list // // ignore_load: Ignore the flag of loaded member and compress // ignore_modify: Ignore the flag of modified member and compress // default_load: loaded flag value in the list after compression when ignore_load=true // default_modify: modified flag value in the list after compression when default_modify=true // // NOTE: ignore_modify and ignore_load cannot both be true. // Zero size pages will be deleted. However, if the page information is the only one, // it will be left behind. This is what you need to do to create a new empty file. // static void raw_compress_fdpage_list(const fdpage_list_t& pages, fdpage_list_t& compressed_pages, bool ignore_load, bool ignore_modify, bool default_load, bool default_modify) { compressed_pages.clear(); fdpage* lastpage = nullptr; fdpage_list_t::iterator add_iter; for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(0 == iter->bytes){ continue; } if(!lastpage){ // First item raw_add_compress_fdpage_list(compressed_pages, (*iter), ignore_load, ignore_modify, default_load, default_modify); lastpage = &(compressed_pages.back()); }else{ // check page continuity if(lastpage->next() != iter->offset){ // Non-consecutive with last page, so add a page filled with default values if( (!ignore_load && (lastpage->loaded != false)) || (!ignore_modify && (lastpage->modified != false)) ) { // add new page fdpage tmppage(lastpage->next(), (iter->offset - lastpage->next()), false, false); raw_add_compress_fdpage_list(compressed_pages, tmppage, ignore_load, ignore_modify, default_load, default_modify); add_iter = compressed_pages.end(); --add_iter; lastpage = &(*add_iter); }else{ // Expand last area lastpage->bytes = iter->offset - lastpage->offset; } } // add current page if( (!ignore_load && (lastpage->loaded != iter->loaded )) || (!ignore_modify && (lastpage->modified != iter->modified)) ) { // Add new page raw_add_compress_fdpage_list(compressed_pages, (*iter), ignore_load, ignore_modify, default_load, default_modify); add_iter = compressed_pages.end(); --add_iter; lastpage = &(*add_iter); }else{ // Expand last area lastpage->bytes += iter->bytes; } } } } static void compress_fdpage_list_ignore_modify(const fdpage_list_t& pages, fdpage_list_t& compressed_pages, bool default_modify) { raw_compress_fdpage_list(pages, compressed_pages, /* ignore_load= */ false, /* ignore_modify= */ true, /* default_load= */false, /* default_modify= */default_modify); } static void compress_fdpage_list_ignore_load(const fdpage_list_t& pages, fdpage_list_t& compressed_pages, bool default_load) { raw_compress_fdpage_list(pages, compressed_pages, /* ignore_load= */ true, /* ignore_modify= */ false, /* default_load= */default_load, /* default_modify= */false); } static void compress_fdpage_list(const fdpage_list_t& pages, fdpage_list_t& compressed_pages) { raw_compress_fdpage_list(pages, compressed_pages, /* ignore_load= */ false, /* ignore_modify= */ false, /* default_load= */false, /* default_modify= */false); } static fdpage_list_t parse_partsize_fdpage_list(const fdpage_list_t& pages, off_t max_partsize) { fdpage_list_t parsed_pages; for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(iter->modified){ // modified page fdpage tmppage = *iter; for(off_t start = iter->offset, rest_bytes = iter->bytes; 0 < rest_bytes; ){ if((max_partsize * 2) < rest_bytes){ // do parse tmppage.offset = start; tmppage.bytes = max_partsize; parsed_pages.push_back(tmppage); start += max_partsize; rest_bytes -= max_partsize; }else{ // Since the number of remaining bytes is less than twice max_partsize, // one of the divided areas will be smaller than max_partsize. // Therefore, this area at the end should not be divided. tmppage.offset = start; tmppage.bytes = rest_bytes; parsed_pages.push_back(tmppage); start += rest_bytes; rest_bytes = 0; } } }else{ // not modified page is not parsed parsed_pages.push_back(*iter); } } return parsed_pages; } //------------------------------------------------ // PageList class methods //------------------------------------------------ // // Examine and return the status of each block in the file. // // Assuming the file is a sparse file, check the HOLE and DATA areas // and return it in fdpage_list_t. The loaded flag of each fdpage is // set to false for HOLE blocks and true for DATA blocks. // bool PageList::GetSparseFilePages(int fd, size_t file_size, fdpage_list_t& sparse_list) { // [NOTE] // Express the status of the cache file using fdpage_list_t. // There is a hole in the cache file(sparse file), and the // state of this hole is expressed by the "loaded" member of // struct fdpage. (the "modified" member is not used) // if(0 == file_size){ // file is empty return true; } bool is_hole = false; off_t hole_pos = lseek(fd, 0, SEEK_HOLE); off_t data_pos = lseek(fd, 0, SEEK_DATA); if(-1 == hole_pos && -1 == data_pos){ S3FS_PRN_ERR("Could not find the first position both HOLE and DATA in the file(physical_fd=%d).", fd); return false; }else if(-1 == hole_pos){ is_hole = false; }else if(-1 == data_pos){ is_hole = true; }else if(hole_pos < data_pos){ is_hole = true; }else{ is_hole = false; } for(off_t cur_pos = 0, next_pos = 0; 0 <= cur_pos; cur_pos = next_pos, is_hole = !is_hole){ fdpage page; page.offset = cur_pos; page.loaded = !is_hole; page.modified = false; next_pos = lseek(fd, cur_pos, (is_hole ? SEEK_DATA : SEEK_HOLE)); if(-1 == next_pos){ page.bytes = static_cast(file_size - cur_pos); }else{ page.bytes = next_pos - cur_pos; } sparse_list.push_back(page); } return true; } // // Confirm that the specified area is ZERO // bool PageList::CheckZeroAreaInFile(int fd, off_t start, size_t bytes) { std::unique_ptr readbuff(new char[CHECK_CACHEFILE_PART_SIZE]); for(size_t comp_bytes = 0, check_bytes = 0; comp_bytes < bytes; comp_bytes += check_bytes){ if(CHECK_CACHEFILE_PART_SIZE < (bytes - comp_bytes)){ check_bytes = CHECK_CACHEFILE_PART_SIZE; }else{ check_bytes = bytes - comp_bytes; } bool found_bad_data = false; ssize_t read_bytes; if(-1 == (read_bytes = pread(fd, readbuff.get(), check_bytes, (start + comp_bytes)))){ S3FS_PRN_ERR("Something error is occurred in reading %zu bytes at %lld from file(physical_fd=%d).", check_bytes, static_cast(start + comp_bytes), fd); found_bad_data = true; }else{ check_bytes = static_cast(read_bytes); for(size_t tmppos = 0; tmppos < check_bytes; ++tmppos){ if('\0' != readbuff[tmppos]){ // found not ZERO data. found_bad_data = true; break; } } } if(found_bad_data){ return false; } } return true; } // // Checks that the specified area matches the state of the sparse file. // // [Parameters] // checkpage: This is one state of the cache file, it is loaded from the stats file. // sparse_list: This is a list of the results of directly checking the cache file status(HOLE/DATA). // In the HOLE area, the "loaded" flag of fdpage is false. The DATA area has it set to true. // fd: opened file descriptor to target cache file. // bool PageList::CheckAreaInSparseFile(const struct fdpage& checkpage, const fdpage_list_t& sparse_list, int fd, fdpage_list_t& err_area_list, fdpage_list_t& warn_area_list) { // Check the block status of a part(Check Area: checkpage) of the target file. // The elements of sparse_list have 5 patterns that overlap this block area. // // File |<---...--------------------------------------...--->| // Check Area (offset)<-------------------->(offset + bytes - 1) // Area case(0) <-------> // Area case(1) <-------> // Area case(2) <--------> // Area case(3) <----------> // Area case(4) <-----------> // Area case(5) <-----------------------------> // bool result = true; for(auto iter = sparse_list.cbegin(); iter != sparse_list.cend(); ++iter){ off_t check_start = 0; off_t check_bytes = 0; if((iter->offset + iter->bytes) <= checkpage.offset){ // case 0 continue; // next }else if((checkpage.offset + checkpage.bytes) <= iter->offset){ // case 1 break; // finish }else if(iter->offset < checkpage.offset && (iter->offset + iter->bytes) < (checkpage.offset + checkpage.bytes)){ // case 2 check_start = checkpage.offset; check_bytes = iter->bytes - (checkpage.offset - iter->offset); }else if((checkpage.offset + checkpage.bytes) < (iter->offset + iter->bytes)){ // here, already "iter->offset < (checkpage.offset + checkpage.bytes)" is true. // case 3 check_start = iter->offset; check_bytes = checkpage.bytes - (iter->offset - checkpage.offset); }else if(checkpage.offset < iter->offset && (iter->offset + iter->bytes) < (checkpage.offset + checkpage.bytes)){ // case 4 check_start = iter->offset; check_bytes = iter->bytes; }else{ // (iter->offset <= checkpage.offset && (checkpage.offset + checkpage.bytes) <= (iter->offset + iter->bytes)) // case 5 check_start = checkpage.offset; check_bytes = checkpage.bytes; } // check target area type if(checkpage.loaded || checkpage.modified){ // target area must be not HOLE(DATA) area. if(!iter->loaded){ // Found bad area, it is HOLE area. fdpage page(check_start, check_bytes, false, false); err_area_list.push_back(page); result = false; } }else{ // target area should be HOLE area.(If it is not a block boundary, it may be a DATA area.) if(iter->loaded){ // need to check this area's each data, it should be ZERO. if(!PageList::CheckZeroAreaInFile(fd, check_start, static_cast(check_bytes))){ // Discovered an area that has un-initial status data but it probably does not effect bad. fdpage page(check_start, check_bytes, true, false); warn_area_list.push_back(page); result = false; } } } } return result; } //------------------------------------------------ // PageList methods //------------------------------------------------ void PageList::FreeList(fdpage_list_t& list) { list.clear(); } PageList::PageList(off_t size, bool is_loaded, bool is_modified, bool shrunk) : is_shrink(shrunk) { Init(size, is_loaded, is_modified); } PageList::~PageList() { Clear(); } void PageList::Clear() { PageList::FreeList(pages); is_shrink = false; } bool PageList::Init(off_t size, bool is_loaded, bool is_modified) { Clear(); if(0 <= size){ fdpage page(0, size, is_loaded, is_modified); pages.push_back(page); } return true; } off_t PageList::Size() const { if(pages.empty()){ return 0; } auto riter = pages.rbegin(); return riter->next(); } bool PageList::Compress() { fdpage* lastpage = nullptr; for(auto iter = pages.begin(); iter != pages.end(); ){ if(!lastpage){ // First item lastpage = &(*iter); ++iter; }else{ // check page continuity if(lastpage->next() != iter->offset){ // Non-consecutive with last page, so add a page filled with default values if(lastpage->loaded || lastpage->modified){ // insert new page before current pos fdpage tmppage(lastpage->next(), (iter->offset - lastpage->next()), false, false); iter = pages.insert(iter, tmppage); lastpage = &(*iter); ++iter; }else{ // Expand last area lastpage->bytes = iter->offset - lastpage->offset; } } // check current page if(lastpage->loaded == iter->loaded && lastpage->modified == iter->modified){ // Expand last area and remove current pos lastpage->bytes += iter->bytes; iter = pages.erase(iter); }else{ lastpage = &(*iter); ++iter; } } } return true; } bool PageList::Parse(off_t new_pos) { for(auto iter = pages.begin(); iter != pages.end(); ++iter){ if(new_pos == iter->offset){ // nothing to do return true; }else if(iter->offset < new_pos && new_pos < iter->next()){ fdpage page(iter->offset, new_pos - iter->offset, iter->loaded, iter->modified); iter->bytes -= (new_pos - iter->offset); iter->offset = new_pos; pages.insert(iter, page); return true; } } return false; } bool PageList::Resize(off_t size, bool is_loaded, bool is_modified) { off_t total = Size(); if(0 == total){ // [NOTE] // The is_shrink flag remains unchanged in this function. // bool backup_is_shrink = is_shrink; Init(size, is_loaded, is_modified); is_shrink = backup_is_shrink; }else if(total < size){ // add new area fdpage page(total, (size - total), is_loaded, is_modified); pages.push_back(page); }else if(size < total){ // cut area for(auto iter = pages.begin(); iter != pages.end(); ){ if(iter->next() <= size){ ++iter; }else{ if(size <= iter->offset){ iter = pages.erase(iter); }else{ iter->bytes = size - iter->offset; } } } if(is_modified){ is_shrink = true; } }else{ // total == size // nothing to do } // compress area return Compress(); } bool PageList::IsPageLoaded(off_t start, off_t size) const { for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(iter->end() < start){ continue; } if(!iter->loaded){ return false; } if(0 != size && start + size <= iter->next()){ break; } } return true; } bool PageList::SetPageLoadedStatus(off_t start, off_t size, PageList::page_status pstatus, bool is_compress) { off_t now_size = Size(); bool is_loaded = (page_status::LOAD_MODIFIED == pstatus || page_status::LOADED == pstatus); bool is_modified = (page_status::LOAD_MODIFIED == pstatus || page_status::MODIFIED == pstatus); if(now_size <= start){ if(now_size < start){ // add Resize(start, false, is_modified); // set modified flag from now end pos to specified start pos. } Resize(start + size, is_loaded, is_modified); }else if(now_size <= start + size){ // cut Resize(start, false, false); // not changed loaded/modified flags in existing area. // add Resize(start + size, is_loaded, is_modified); }else{ // start-size are inner pages area // parse "start", and "start + size" position Parse(start); Parse(start + size); // set loaded flag for(auto iter = pages.begin(); iter != pages.end(); ++iter){ if(iter->end() < start){ continue; }else if(start + size <= iter->offset){ break; }else{ iter->loaded = is_loaded; iter->modified = is_modified; } } } // compress area return (is_compress ? Compress() : true); } bool PageList::FindUnloadedPage(off_t start, off_t& resstart, off_t& ressize) const { for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(start <= iter->end()){ if(!iter->loaded && !iter->modified){ // Do not load unloaded and modified areas resstart = iter->offset; ressize = iter->bytes; return true; } } } return false; } // [NOTE] // Accumulates the range of unload that is smaller than the Limit size. // If you want to integrate all unload ranges, set the limit size to 0. // off_t PageList::GetTotalUnloadedPageSize(off_t start, off_t size, off_t limit_size) const { // If size is 0, it means loading to end. if(0 == size){ if(start < Size()){ size = Size() - start; } } off_t next = start + size; off_t restsize = 0; for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(iter->next() <= start){ continue; } if(next <= iter->offset){ break; } if(iter->loaded || iter->modified){ continue; } off_t tmpsize; if(iter->offset <= start){ if(iter->next() <= next){ tmpsize = (iter->next() - start); }else{ tmpsize = next - start; // = size } }else{ if(iter->next() <= next){ tmpsize = iter->next() - iter->offset; // = iter->bytes }else{ tmpsize = next - iter->offset; } } if(0 == limit_size || tmpsize < limit_size){ restsize += tmpsize; } } return restsize; } size_t PageList::GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start, off_t size) const { // If size is 0, it means loading to end. if(0 == size){ if(start < Size()){ size = Size() - start; } } off_t next = start + size; for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(iter->next() <= start){ continue; } if(next <= iter->offset){ break; } if(iter->loaded || iter->modified){ continue; // already loaded or modified } // page area off_t page_start = std::max(iter->offset, start); off_t page_next = std::min(iter->next(), next); off_t page_size = page_next - page_start; // add list auto riter = unloaded_list.rbegin(); if(riter != unloaded_list.rend() && riter->next() == page_start){ // merge to before page riter->bytes += page_size; }else{ fdpage page(page_start, page_size, false, false); unloaded_list.push_back(page); } } return unloaded_list.size(); } // [NOTE] // This method is called in advance when mixing POST and COPY in multi-part upload. // The minimum size of each part must be 5 MB, and the data area below this must be // downloaded from S3. // This method checks the current PageList status and returns the area that needs // to be downloaded so that each part is at least 5 MB. // bool PageList::GetPageListsForMultipartUpload(fdpage_list_t& dlpages, fdpage_list_t& mixuppages, off_t max_partsize) { // compress before this processing Compress(); // always true // make a list by modified flag fdpage_list_t modified_pages; fdpage_list_t download_pages; // A non-contiguous page list showing the areas that need to be downloaded fdpage_list_t mixupload_pages; // A continuous page list showing only modified flags for mixupload compress_fdpage_list_ignore_load(pages, modified_pages, false); fdpage prev_page; for(auto iter = modified_pages.cbegin(); iter != modified_pages.cend(); ++iter){ if(iter->modified){ // current is modified area if(!prev_page.modified){ // previous is not modified area if(prev_page.bytes < MIN_MULTIPART_SIZE){ // previous(not modified) area is too small for one multipart size, // then all of previous area is needed to download. download_pages.push_back(prev_page); // previous(not modified) area is set upload area. prev_page.modified = true; mixupload_pages.push_back(prev_page); }else{ // previous(not modified) area is set copy area. prev_page.modified = false; mixupload_pages.push_back(prev_page); } // set current to previous prev_page = *iter; }else{ // previous is modified area, too prev_page.bytes += iter->bytes; } }else{ // current is not modified area if(!prev_page.modified){ // previous is not modified area, too prev_page.bytes += iter->bytes; }else{ // previous is modified area if(prev_page.bytes < MIN_MULTIPART_SIZE){ // previous(modified) area is too small for one multipart size, // then part or all of current area is needed to download. off_t missing_bytes = MIN_MULTIPART_SIZE - prev_page.bytes; if((missing_bytes + MIN_MULTIPART_SIZE) < iter-> bytes){ // The current size is larger than the missing size, and the remainder // after deducting the missing size is larger than the minimum size. fdpage missing_page(iter->offset, missing_bytes, false, false); download_pages.push_back(missing_page); // previous(not modified) area is set upload area. prev_page.bytes = MIN_MULTIPART_SIZE; mixupload_pages.push_back(prev_page); // set current to previous prev_page = *iter; prev_page.offset += missing_bytes; prev_page.bytes -= missing_bytes; }else{ // The current size is less than the missing size, or the remaining // size less the missing size is less than the minimum size. download_pages.push_back(*iter); // add current to previous prev_page.bytes += iter->bytes; } }else{ // previous(modified) area is enough size for one multipart size. mixupload_pages.push_back(prev_page); // set current to previous prev_page = *iter; } } } } // last area if(0 < prev_page.bytes){ mixupload_pages.push_back(prev_page); } // compress compress_fdpage_list_ignore_modify(download_pages, dlpages, false); compress_fdpage_list_ignore_load(mixupload_pages, mixuppages, false); // parse by max pagesize dlpages = parse_partsize_fdpage_list(dlpages, max_partsize); mixuppages = parse_partsize_fdpage_list(mixuppages, max_partsize); return true; } bool PageList::GetNoDataPageLists(fdpage_list_t& nodata_pages, off_t start, size_t size) { // compress before this processing Compress(); // always true // extract areas without data fdpage_list_t tmp_pagelist; off_t stop_pos = (0L == size ? -1 : (start + size)); for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if((iter->offset + iter->bytes) < start){ continue; } if(-1 != stop_pos && stop_pos <= iter->offset){ break; } if(iter->modified){ continue; } fdpage tmppage; tmppage.offset = std::max(iter->offset, start); tmppage.bytes = (-1 == stop_pos ? iter->bytes : std::min(iter->bytes, (stop_pos - tmppage.offset))); tmppage.loaded = iter->loaded; tmppage.modified = iter->modified; tmp_pagelist.push_back(tmppage); } if(tmp_pagelist.empty()){ nodata_pages.clear(); }else{ // compress compress_fdpage_list(tmp_pagelist, nodata_pages); } return true; } off_t PageList::BytesModified() const { off_t total = 0; for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(iter->modified){ total += iter->bytes; } } return total; } bool PageList::IsModified() const { if(is_shrink){ return true; } for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(iter->modified){ return true; } } return false; } bool PageList::ClearAllModified() { is_shrink = false; for(auto iter = pages.begin(); iter != pages.end(); ++iter){ if(iter->modified){ iter->modified = false; } } return Compress(); } bool PageList::Serialize(CacheFileStat& file, ino_t inode) { if(!file.Open()){ return false; } // // put to file // std::ostringstream ssall; ssall << inode << ":" << Size(); for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ ssall << "\n" << iter->offset << ":" << iter->bytes << ":" << (iter->loaded ? "1" : "0") << ":" << (iter->modified ? "1" : "0"); } if(-1 == ftruncate(file.GetFd(), 0)){ S3FS_PRN_ERR("failed to truncate file(to 0) for stats(%d)", errno); return false; } std::string strall = ssall.str(); if(0 >= pwrite(file.GetFd(), strall.c_str(), strall.length(), 0)){ S3FS_PRN_ERR("failed to write stats(%d)", errno); return false; } return true; } bool PageList::Deserialize(CacheFileStat& file, ino_t inode) { if(!file.Open()){ return false; } // // loading from file // struct stat st{}; if(-1 == fstat(file.GetFd(), &st)){ S3FS_PRN_ERR("fstat is failed. errno(%d)", errno); return false; } if(0 >= st.st_size){ // nothing Init(0, false, false); return true; } std::unique_ptr ptmp(new char[st.st_size + 1]); ssize_t result; // read from file if(0 >= (result = pread(file.GetFd(), ptmp.get(), st.st_size, 0))){ S3FS_PRN_ERR("failed to read stats(%d)", errno); return false; } ptmp[result] = '\0'; std::string oneline; std::istringstream ssall(ptmp.get()); // loaded Clear(); // load head line(for size and inode) off_t total; ino_t cache_inode; // if this value is 0, it means old format. if(!getline(ssall, oneline, '\n')){ S3FS_PRN_ERR("failed to parse stats."); return false; }else{ std::istringstream sshead(oneline); std::string strhead1; std::string strhead2; // get first part in head line. if(!getline(sshead, strhead1, ':')){ S3FS_PRN_ERR("failed to parse stats."); return false; } // get second part in head line. if(!getline(sshead, strhead2, ':')){ // old head format is "\n" total = cvt_strtoofft(strhead1.c_str(), /* base= */10); cache_inode = 0; }else{ // current head format is ":\n" total = cvt_strtoofft(strhead2.c_str(), /* base= */10); cache_inode = static_cast(cvt_strtoofft(strhead1.c_str(), /* base= */10)); if(0 == cache_inode){ S3FS_PRN_ERR("wrong inode number in parsed cache stats."); return false; } } } // check inode number if(0 != cache_inode && cache_inode != inode){ S3FS_PRN_ERR("differ inode and inode number in parsed cache stats."); return false; } // load each part bool is_err = false; while(getline(ssall, oneline, '\n')){ std::string part; std::istringstream ssparts(oneline); // offset if(!getline(ssparts, part, ':')){ is_err = true; break; } off_t offset = cvt_strtoofft(part.c_str(), /* base= */10); // size if(!getline(ssparts, part, ':')){ is_err = true; break; } off_t size = cvt_strtoofft(part.c_str(), /* base= */10); // loaded if(!getline(ssparts, part, ':')){ is_err = true; break; } bool is_loaded = (1 == cvt_strtoofft(part.c_str(), /* base= */10) ? true : false); bool is_modified; if(!getline(ssparts, part, ':')){ is_modified = false; // old version does not have this part. }else{ is_modified = (1 == cvt_strtoofft(part.c_str(), /* base= */10) ? true : false); } // add new area PageList::page_status pstatus = PageList::page_status::NOT_LOAD_MODIFIED; if(is_loaded){ if(is_modified){ pstatus = PageList::page_status::LOAD_MODIFIED; }else{ pstatus = PageList::page_status::LOADED; } }else{ if(is_modified){ pstatus = PageList::page_status::MODIFIED; } } SetPageLoadedStatus(offset, size, pstatus); } if(is_err){ S3FS_PRN_ERR("failed to parse stats."); Clear(); return false; } // check size if(total != Size()){ S3FS_PRN_ERR("different size(%lld - %lld).", static_cast(total), static_cast(Size())); Clear(); return false; } return true; } void PageList::Dump() const { int cnt = 0; S3FS_PRN_DBG("pages (shrunk=%s) = {", (is_shrink ? "yes" : "no")); for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter, ++cnt){ S3FS_PRN_DBG(" [%08d] -> {%014lld - %014lld : %s / %s}", cnt, static_cast(iter->offset), static_cast(iter->bytes), iter->loaded ? "loaded" : "unloaded", iter->modified ? "modified" : "not modified"); } S3FS_PRN_DBG("}"); } // // Compare the fdpage_list_t pages of the object with the state of the file. // // The loaded=true or modified=true area of pages must be a DATA block // (not a HOLE block) in the file. // The other area is a HOLE block in the file or is a DATA block(but the // data of the target area in that block should be ZERO). // If it is a bad area in the previous case, it will be reported as an error. // If the latter case does not match, it will be reported as a warning. // bool PageList::CompareSparseFile(int fd, size_t file_size, fdpage_list_t& err_area_list, fdpage_list_t& warn_area_list) { err_area_list.clear(); warn_area_list.clear(); // First, list the block disk allocation area of the cache file. // The cache file has holes(sparse file) and no disk block areas // are assigned to any holes. fdpage_list_t sparse_list; if(!PageList::GetSparseFilePages(fd, file_size, sparse_list)){ S3FS_PRN_ERR("Something error is occurred in parsing hole/data of the cache file(physical_fd=%d).", fd); fdpage page(0, static_cast(file_size), false, false); err_area_list.push_back(page); return false; } if(sparse_list.empty() && pages.empty()){ // both file and stats information are empty, it means cache file size is ZERO. return true; } // Compare each pages and sparse_list bool result = true; for(auto iter = pages.cbegin(); iter != pages.cend(); ++iter){ if(!PageList::CheckAreaInSparseFile(*iter, sparse_list, fd, err_area_list, warn_area_list)){ result = false; } } return result; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_page.h000066400000000000000000000115301470675423500166140ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_PAGE_H_ #define S3FS_FDCACHE_PAGE_H_ #include #include #include //------------------------------------------------ // Symbols //------------------------------------------------ // [NOTE] // If the following symbols in lseek whence are undefined, define them. // If it is not supported by lseek, s3fs judges by the processing result of lseek. // #ifndef SEEK_DATA #define SEEK_DATA 3 #endif #ifndef SEEK_HOLE #define SEEK_HOLE 4 #endif //------------------------------------------------ // Structure fdpage //------------------------------------------------ // page block information struct fdpage { off_t offset; off_t bytes; bool loaded; bool modified; explicit fdpage(off_t start = 0, off_t size = 0, bool is_loaded = false, bool is_modified = false) : offset(start), bytes(size), loaded(is_loaded), modified(is_modified) {} off_t next() const { return (offset + bytes); } off_t end() const { return (0 < bytes ? offset + bytes - 1 : 0); } }; typedef std::vector fdpage_list_t; //------------------------------------------------ // Class PageList //------------------------------------------------ class CacheFileStat; class FdEntity; // cppcheck-suppress copyCtorAndEqOperator class PageList { friend class FdEntity; // only one method access directly pages. private: fdpage_list_t pages; bool is_shrink; // [NOTE] true if it has been shrunk even once public: enum class page_status : int8_t { NOT_LOAD_MODIFIED = 0, LOADED, MODIFIED, LOAD_MODIFIED }; private: static bool GetSparseFilePages(int fd, size_t file_size, fdpage_list_t& sparse_list); static bool CheckZeroAreaInFile(int fd, off_t start, size_t bytes); static bool CheckAreaInSparseFile(const struct fdpage& checkpage, const fdpage_list_t& sparse_list, int fd, fdpage_list_t& err_area_list, fdpage_list_t& warn_area_list); void Clear(); bool Parse(off_t new_pos); bool Serialize(CacheFileStat& file, ino_t inode); public: static void FreeList(fdpage_list_t& list); explicit PageList(off_t size = 0, bool is_loaded = false, bool is_modified = false, bool shrunk = false); PageList(const PageList&) = delete; PageList(PageList&&) = delete; PageList& operator=(const PageList&) = delete; PageList& operator=(PageList&&) = delete; ~PageList(); bool Init(off_t size, bool is_loaded, bool is_modified); off_t Size() const; bool Resize(off_t size, bool is_loaded, bool is_modified); bool IsPageLoaded(off_t start = 0, off_t size = 0) const; // size=0 is checking to end of list bool SetPageLoadedStatus(off_t start, off_t size, PageList::page_status pstatus = page_status::LOADED, bool is_compress = true); bool FindUnloadedPage(off_t start, off_t& resstart, off_t& ressize) const; off_t GetTotalUnloadedPageSize(off_t start = 0, off_t size = 0, off_t limit_size = 0) const; // size=0 is checking to end of list size_t GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start = 0, off_t size = 0) const; // size=0 is checking to end of list bool GetPageListsForMultipartUpload(fdpage_list_t& dlpages, fdpage_list_t& mixuppages, off_t max_partsize); bool GetNoDataPageLists(fdpage_list_t& nodata_pages, off_t start = 0, size_t size = 0); off_t BytesModified() const; bool IsModified() const; bool ClearAllModified(); bool Compress(); bool Deserialize(CacheFileStat& file, ino_t inode); void Dump() const; bool CompareSparseFile(int fd, size_t file_size, fdpage_list_t& err_area_list, fdpage_list_t& warn_area_list); }; #endif // S3FS_FDCACHE_PAGE_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_pseudofd.cpp000066400000000000000000000055361470675423500200550ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include "fdcache_pseudofd.h" //------------------------------------------------ // Symbols //------------------------------------------------ // [NOTE] // The minimum pseudo fd value starts 2. // This is to avoid mistakes for 0(stdout) and 1(stderr), which are usually used. // static constexpr int MIN_PSEUDOFD_NUMBER = 2; //------------------------------------------------ // PseudoFdManager class methods //------------------------------------------------ PseudoFdManager& PseudoFdManager::GetManager() { static PseudoFdManager singleton; return singleton; } int PseudoFdManager::Get() { return (PseudoFdManager::GetManager()).CreatePseudoFd(); } bool PseudoFdManager::Release(int fd) { return (PseudoFdManager::GetManager()).ReleasePseudoFd(fd); } //------------------------------------------------ // PseudoFdManager methods //------------------------------------------------ int PseudoFdManager::GetUnusedMinPseudoFd() const { int min_fd = MIN_PSEUDOFD_NUMBER; // Look for the first discontinuous value. for(auto iter = pseudofd_list.cbegin(); iter != pseudofd_list.cend(); ++iter){ if(min_fd == (*iter)){ ++min_fd; }else if(min_fd < (*iter)){ break; } } return min_fd; } int PseudoFdManager::CreatePseudoFd() { const std::lock_guard lock(pseudofd_list_lock); int new_fd = PseudoFdManager::GetUnusedMinPseudoFd(); pseudofd_list.push_back(new_fd); std::sort(pseudofd_list.begin(), pseudofd_list.end()); return new_fd; } bool PseudoFdManager::ReleasePseudoFd(int fd) { const std::lock_guard lock(pseudofd_list_lock); for(auto iter = pseudofd_list.begin(); iter != pseudofd_list.end(); ++iter){ if(fd == (*iter)){ pseudofd_list.erase(iter); return true; } } return false; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_pseudofd.h000066400000000000000000000042561470675423500175200ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_PSEUDOFD_H_ #define S3FS_FDCACHE_PSEUDOFD_H_ #include #include #include "common.h" //------------------------------------------------ // Typdefs //------------------------------------------------ // List of pseudo fd in use // typedef std::vector pseudofd_list_t; //------------------------------------------------ // Class PseudoFdManager //------------------------------------------------ class PseudoFdManager { private: pseudofd_list_t pseudofd_list GUARDED_BY(pseudofd_list_lock); std::mutex pseudofd_list_lock; // protects pseudofd_list static PseudoFdManager& GetManager(); PseudoFdManager() = default; ~PseudoFdManager() = default; int GetUnusedMinPseudoFd() const REQUIRES(pseudofd_list_lock); int CreatePseudoFd(); bool ReleasePseudoFd(int fd); public: PseudoFdManager(const PseudoFdManager&) = delete; PseudoFdManager(PseudoFdManager&&) = delete; PseudoFdManager& operator=(const PseudoFdManager&) = delete; PseudoFdManager& operator=(PseudoFdManager&&) = delete; static int Get(); static bool Release(int fd); }; #endif // S3FS_FDCACHE_PSEUDOFD_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_stat.cpp000066400000000000000000000175301470675423500172140ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include "s3fs_logger.h" #include "fdcache_stat.h" #include "fdcache.h" #include "s3fs_util.h" #include "s3fs_cred.h" #include "string_util.h" //------------------------------------------------ // CacheFileStat class methods //------------------------------------------------ std::string CacheFileStat::GetCacheFileStatTopDir() { std::string top_path; if(!FdManager::IsCacheDir() || S3fsCred::GetBucket().empty()){ return top_path; } // stat top dir( "//..stat" ) top_path += FdManager::GetCacheDir(); top_path += "/."; top_path += S3fsCred::GetBucket(); top_path += ".stat"; return top_path; } int CacheFileStat::MakeCacheFileStatPath(const char* path, std::string& sfile_path, bool is_create_dir) { std::string top_path = CacheFileStat::GetCacheFileStatTopDir(); if(top_path.empty()){ S3FS_PRN_ERR("The path to cache top dir is empty."); return -EIO; } if(is_create_dir){ int result; if(0 != (result = mkdirp(top_path + mydirname(path), 0777))){ S3FS_PRN_ERR("failed to create dir(%s) by errno(%d).", path, result); return result; } } if(!path || '\0' == path[0]){ sfile_path = top_path; }else{ sfile_path = top_path + SAFESTRPTR(path); } return 0; } bool CacheFileStat::CheckCacheFileStatTopDir() { std::string top_path = CacheFileStat::GetCacheFileStatTopDir(); if(top_path.empty()){ S3FS_PRN_INFO("The path to cache top dir is empty, thus not need to check permission."); return true; } return check_exist_dir_permission(top_path.c_str()); } int CacheFileStat::DeleteCacheFileStat(const char* path) { if(!path || '\0' == path[0]){ return -EINVAL; } // stat path std::string sfile_path; int result; if(0 != (result = CacheFileStat::MakeCacheFileStatPath(path, sfile_path, false))){ S3FS_PRN_ERR("failed to create cache stat file path(%s)", path); return result; } if(0 != unlink(sfile_path.c_str())){ result = -errno; if(-ENOENT == result){ S3FS_PRN_DBG("failed to delete file(%s): errno=%d", path, result); }else{ S3FS_PRN_ERR("failed to delete file(%s): errno=%d", path, result); } return result; } return 0; } // [NOTE] // If remove stat file directory, it should do before removing // file cache directory. // bool CacheFileStat::DeleteCacheFileStatDirectory() { std::string top_path = CacheFileStat::GetCacheFileStatTopDir(); if(top_path.empty()){ S3FS_PRN_INFO("The path to cache top dir is empty, thus not need to remove it."); return true; } return delete_files_in_dir(top_path.c_str(), true); } bool CacheFileStat::RenameCacheFileStat(const char* oldpath, const char* newpath) { if(!oldpath || '\0' == oldpath[0] || !newpath || '\0' == newpath[0]){ return false; } // stat path std::string old_filestat; std::string new_filestat; if(0 != CacheFileStat::MakeCacheFileStatPath(oldpath, old_filestat, false) || 0 != CacheFileStat::MakeCacheFileStatPath(newpath, new_filestat, false)){ return false; } // check new stat path struct stat st; if(0 == stat(new_filestat.c_str(), &st)){ // new stat path is existed, then unlink it. if(-1 == unlink(new_filestat.c_str())){ S3FS_PRN_ERR("failed to unlink new cache file stat path(%s) by errno(%d).", new_filestat.c_str(), errno); return false; } } // check old stat path if(0 != stat(old_filestat.c_str(), &st)){ // old stat path is not existed, then nothing to do any more. return true; } // link and unlink if(-1 == link(old_filestat.c_str(), new_filestat.c_str())){ S3FS_PRN_ERR("failed to link old cache file stat path(%s) to new cache file stat path(%s) by errno(%d).", old_filestat.c_str(), new_filestat.c_str(), errno); return false; } if(-1 == unlink(old_filestat.c_str())){ S3FS_PRN_ERR("failed to unlink old cache file stat path(%s) by errno(%d).", old_filestat.c_str(), errno); return false; } return true; } //------------------------------------------------ // CacheFileStat methods //------------------------------------------------ CacheFileStat::CacheFileStat(const char* tpath) : fd(-1) { if(tpath && '\0' != tpath[0]){ SetPath(tpath, true); } } CacheFileStat::~CacheFileStat() { Release(); } bool CacheFileStat::SetPath(const char* tpath, bool is_open) { if(!tpath || '\0' == tpath[0]){ return false; } if(!Release()){ // could not close old stat file. return false; } path = tpath; if(!is_open){ return true; } return Open(); } bool CacheFileStat::RawOpen(bool readonly) { if(path.empty()){ return false; } if(-1 != fd){ // already opened return true; } // stat path std::string sfile_path; if(0 != CacheFileStat::MakeCacheFileStatPath(path.c_str(), sfile_path, true)){ S3FS_PRN_ERR("failed to create cache stat file path(%s)", path.c_str()); return false; } // open int tmpfd; if(readonly){ if(-1 == (tmpfd = open(sfile_path.c_str(), O_RDONLY))){ S3FS_PRN_ERR("failed to read only open cache stat file path(%s) - errno(%d)", path.c_str(), errno); return false; } }else{ if(-1 == (tmpfd = open(sfile_path.c_str(), O_CREAT|O_RDWR, 0600))){ S3FS_PRN_ERR("failed to open cache stat file path(%s) - errno(%d)", path.c_str(), errno); return false; } } scope_guard guard([&]() { close(tmpfd); }); // lock if(-1 == flock(tmpfd, LOCK_EX)){ S3FS_PRN_ERR("failed to lock cache stat file(%s) - errno(%d)", path.c_str(), errno); return false; } // seek top if(0 != lseek(tmpfd, 0, SEEK_SET)){ S3FS_PRN_ERR("failed to lseek cache stat file(%s) - errno(%d)", path.c_str(), errno); flock(tmpfd, LOCK_UN); return false; } S3FS_PRN_DBG("file locked(%s - %s)", path.c_str(), sfile_path.c_str()); guard.dismiss(); fd = tmpfd; return true; } bool CacheFileStat::Open() { return RawOpen(false); } bool CacheFileStat::ReadOnlyOpen() { return RawOpen(true); } bool CacheFileStat::Release() { if(-1 == fd){ // already release return true; } // unlock if(-1 == flock(fd, LOCK_UN)){ S3FS_PRN_ERR("failed to unlock cache stat file(%s) - errno(%d)", path.c_str(), errno); return false; } S3FS_PRN_DBG("file unlocked(%s)", path.c_str()); if(-1 == close(fd)){ S3FS_PRN_ERR("failed to close cache stat file(%s) - errno(%d)", path.c_str(), errno); return false; } fd = -1; return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_stat.h000066400000000000000000000043241470675423500166560ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_STAT_H_ #define S3FS_FDCACHE_STAT_H_ #include //------------------------------------------------ // CacheFileStat //------------------------------------------------ class CacheFileStat { private: std::string path; int fd; private: static int MakeCacheFileStatPath(const char* path, std::string& sfile_path, bool is_create_dir = true); bool RawOpen(bool readonly); public: static std::string GetCacheFileStatTopDir(); static int DeleteCacheFileStat(const char* path); static bool CheckCacheFileStatTopDir(); static bool DeleteCacheFileStatDirectory(); static bool RenameCacheFileStat(const char* oldpath, const char* newpath); explicit CacheFileStat(const char* tpath = nullptr); ~CacheFileStat(); CacheFileStat(const CacheFileStat&) = delete; CacheFileStat(CacheFileStat&&) = delete; CacheFileStat& operator=(const CacheFileStat&) = delete; CacheFileStat& operator=(CacheFileStat&&) = delete; bool Open(); bool ReadOnlyOpen(); bool Release(); bool SetPath(const char* tpath, bool is_open = true); int GetFd() const { return fd; } }; #endif // S3FS_FDCACHE_STAT_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_untreated.cpp000066400000000000000000000203761470675423500202360ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include "s3fs_logger.h" #include "fdcache_untreated.h" //------------------------------------------------ // UntreatedParts methods //------------------------------------------------ bool UntreatedParts::empty() { const std::lock_guard lock(untreated_list_lock); return untreated_list.empty(); } bool UntreatedParts::AddPart(off_t start, off_t size) { if(start < 0 || size <= 0){ S3FS_PRN_ERR("Parameter are wrong(start=%lld, size=%lld).", static_cast(start), static_cast(size)); return false; } const std::lock_guard lock(untreated_list_lock); ++last_tag; // Check the overlap with the existing part and add the part. for(auto iter = untreated_list.begin(); iter != untreated_list.end(); ++iter){ if(iter->stretch(start, size, last_tag)){ // the part was stretched, thus check if it overlaps with next parts auto niter = iter; for(++niter; niter != untreated_list.cend(); ){ if(!iter->stretch(niter->start, niter->size, last_tag)){ // This next part does not overlap with the current part break; } // Since the parts overlap and the current part is stretched, delete this next part. niter = untreated_list.erase(niter); } // success to stretch and compress existed parts return true; }else if((start + size) < iter->start){ // The part to add should be inserted before the current part. untreated_list.insert(iter, untreatedpart(start, size, last_tag)); // success to stretch and compress existed parts return true; } } // There are no overlapping parts in the untreated_list, then add the part at end of list untreated_list.emplace_back(start, size, last_tag); return true; } bool UntreatedParts::RowGetPart(off_t& start, off_t& size, off_t max_size, off_t min_size, bool lastpart) const { if(max_size <= 0 || min_size < 0 || max_size < min_size){ S3FS_PRN_ERR("Parameter are wrong(max_size=%lld, min_size=%lld).", static_cast(max_size), static_cast(min_size)); return false; } const std::lock_guard lock(untreated_list_lock); // Check the overlap with the existing part and add the part. for(auto iter = untreated_list.cbegin(); iter != untreated_list.cend(); ++iter){ if(!lastpart || iter->untreated_tag == last_tag){ if(min_size <= iter->size){ if(iter->size <= max_size){ // whole part( min <= part size <= max ) start = iter->start; size = iter->size; }else{ // Partially take out part( max < part size ) start = iter->start; size = max_size; } return true; }else{ if(lastpart){ return false; } } } } return false; } // [NOTE] // If size is specified as 0, all areas(parts) after start will be deleted. // bool UntreatedParts::ClearParts(off_t start, off_t size) { if(start < 0 || size < 0){ S3FS_PRN_ERR("Parameter are wrong(start=%lld, size=%lld).", static_cast(start), static_cast(size)); return false; } const std::lock_guard lock(untreated_list_lock); if(untreated_list.empty()){ return true; } // Check the overlap with the existing part. for(auto iter = untreated_list.begin(); iter != untreated_list.end(); ){ if(0 != size && (start + size) <= iter->start){ // clear area is in front of iter area, no more to do. break; }else if(start <= iter->start){ if(0 != size && (start + size) <= (iter->start + iter->size)){ // clear area overlaps with iter area(on the start side) iter->size = (iter->start + iter->size) - (start + size); iter->start = start + size; if(0 == iter->size){ iter = untreated_list.erase(iter); } }else{ // clear area overlaps with all of iter area iter = untreated_list.erase(iter); } }else if(start < (iter->start + iter->size)){ // clear area overlaps with iter area(on the end side) if(0 == size || (iter->start + iter->size) <= (start + size)){ // start to iter->end is clear iter->size = start - iter->start; }else{ // parse current part iter->size = start - iter->start; // add new part off_t next_start = start + size; off_t next_size = (iter->start + iter->size) - (start + size); long next_tag = iter->untreated_tag; ++iter; iter = untreated_list.insert(iter, untreatedpart(next_start, next_size, next_tag)); ++iter; } }else{ // clear area is in behind of iter area ++iter; } } return true; } // // Update the last updated Untreated part // bool UntreatedParts::GetLastUpdatePart(off_t& start, off_t& size) const { const std::lock_guard lock(untreated_list_lock); for(auto iter = untreated_list.cbegin(); iter != untreated_list.cend(); ++iter){ if(iter->untreated_tag == last_tag){ start = iter->start; size = iter->size; return true; } } return false; } // // Replaces the last updated Untreated part. // // [NOTE] // If size <= 0, delete that part // bool UntreatedParts::ReplaceLastUpdatePart(off_t start, off_t size) { const std::lock_guard lock(untreated_list_lock); for(auto iter = untreated_list.begin(); iter != untreated_list.end(); ++iter){ if(iter->untreated_tag == last_tag){ if(0 < size){ iter->start = start; iter->size = size; }else{ untreated_list.erase(iter); } return true; } } return false; } // // Remove the last updated Untreated part. // bool UntreatedParts::RemoveLastUpdatePart() { const std::lock_guard lock(untreated_list_lock); for(auto iter = untreated_list.begin(); iter != untreated_list.end(); ++iter){ if(iter->untreated_tag == last_tag){ untreated_list.erase(iter); return true; } } return false; } // // Duplicate the internally untreated_list. // bool UntreatedParts::Duplicate(untreated_list_t& list) { const std::lock_guard lock(untreated_list_lock); list = untreated_list; return true; } void UntreatedParts::Dump() { const std::lock_guard lock(untreated_list_lock); S3FS_PRN_DBG("untreated list = ["); for(auto iter = untreated_list.cbegin(); iter != untreated_list.cend(); ++iter){ S3FS_PRN_DBG(" {%014lld - %014lld : tag=%ld}", static_cast(iter->start), static_cast(iter->size), iter->untreated_tag); } S3FS_PRN_DBG("]"); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/fdcache_untreated.h000066400000000000000000000050711470675423500176760ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_FDCACHE_UNTREATED_H_ #define S3FS_FDCACHE_UNTREATED_H_ #include #include "common.h" #include "types.h" //------------------------------------------------ // Class UntreatedParts //------------------------------------------------ class UntreatedParts { private: mutable std::mutex untreated_list_lock; // protects untreated_list untreated_list_t untreated_list GUARDED_BY(untreated_list_lock); long last_tag GUARDED_BY(untreated_list_lock) = 0; // [NOTE] Use this to identify the latest updated part. private: bool RowGetPart(off_t& start, off_t& size, off_t max_size, off_t min_size, bool lastpart) const; public: UntreatedParts() = default; ~UntreatedParts() = default; UntreatedParts(const UntreatedParts&) = delete; UntreatedParts(UntreatedParts&&) = delete; UntreatedParts& operator=(const UntreatedParts&) = delete; UntreatedParts& operator=(UntreatedParts&&) = delete; bool empty(); bool AddPart(off_t start, off_t size); bool GetLastUpdatedPart(off_t& start, off_t& size, off_t max_size, off_t min_size = MIN_MULTIPART_SIZE) const { return RowGetPart(start, size, max_size, min_size, true); } bool ClearParts(off_t start, off_t size); bool ClearAll() { return ClearParts(0, 0); } bool GetLastUpdatePart(off_t& start, off_t& size) const; bool ReplaceLastUpdatePart(off_t start, off_t size); bool RemoveLastUpdatePart(); bool Duplicate(untreated_list_t& list); void Dump(); }; #endif // S3FS_FDCACHE_UNTREATED_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/gnutls_auth.cpp000066400000000000000000000261261470675423500171420ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #ifdef USE_GNUTLS_NETTLE #include #include #include #endif #include #include #include "common.h" #include "s3fs.h" #include "s3fs_auth.h" #include "s3fs_logger.h" //------------------------------------------------------------------- // Utility Function for version //------------------------------------------------------------------- #ifdef USE_GNUTLS_NETTLE const char* s3fs_crypt_lib_name(void) { static constexpr char version[] = "GnuTLS(nettle)"; return version; } #else // USE_GNUTLS_NETTLE const char* s3fs_crypt_lib_name() { static constexpr char version[] = "GnuTLS(gcrypt)"; return version; } #endif // USE_GNUTLS_NETTLE //------------------------------------------------------------------- // Utility Function for global init //------------------------------------------------------------------- bool s3fs_init_global_ssl() { if(GNUTLS_E_SUCCESS != gnutls_global_init()){ return false; } #ifndef USE_GNUTLS_NETTLE if(nullptr == gcry_check_version(nullptr)){ return false; } #endif // USE_GNUTLS_NETTLE return true; } bool s3fs_destroy_global_ssl() { gnutls_global_deinit(); return true; } //------------------------------------------------------------------- // Utility Function for crypt lock //------------------------------------------------------------------- bool s3fs_init_crypt_mutex() { return true; } bool s3fs_destroy_crypt_mutex() { return true; } //------------------------------------------------------------------- // Utility Function for HMAC //------------------------------------------------------------------- #ifdef USE_GNUTLS_NETTLE std::unique_ptr s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { if(!key || !data || !digestlen){ return nullptr; } std::unique_ptr digest(new unsigned char[SHA1_DIGEST_SIZE]); struct hmac_sha1_ctx ctx_hmac; hmac_sha1_set_key(&ctx_hmac, keylen, reinterpret_cast(key)); hmac_sha1_update(&ctx_hmac, datalen, reinterpret_cast(data)); hmac_sha1_digest(&ctx_hmac, SHA1_DIGEST_SIZE, reinterpret_cast(digest.get())); *digestlen = SHA1_DIGEST_SIZE; return digest; } std::unique_ptr s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { if(!key || !data || !digestlen){ return nullptr; } std::unique_ptr digest(new unsigned char[SHA256_DIGEST_SIZE]); struct hmac_sha256_ctx ctx_hmac; hmac_sha256_set_key(&ctx_hmac, keylen, reinterpret_cast(key)); hmac_sha256_update(&ctx_hmac, datalen, reinterpret_cast(data)); hmac_sha256_digest(&ctx_hmac, SHA256_DIGEST_SIZE, reinterpret_cast(digest.get())); *digestlen = SHA256_DIGEST_SIZE; return digest; } #else // USE_GNUTLS_NETTLE std::unique_ptr s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { if(!key || !data || !digestlen){ return nullptr; } if(0 == (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA1))){ return nullptr; } std::unique_ptr digest(new unsigned char[*digestlen + 1]); if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA1, key, keylen, data, datalen, digest.get())){ return nullptr; } return digest; } std::unique_ptr s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { if(!key || !data || !digestlen){ return nullptr; } if(0 == (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){ return nullptr; } std::unique_ptr digest(new unsigned char[*digestlen + 1]); if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, digest.get())){ return nullptr; } return digest; } #endif // USE_GNUTLS_NETTLE //------------------------------------------------------------------- // Utility Function for MD5 //------------------------------------------------------------------- #ifdef USE_GNUTLS_NETTLE bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* result) { struct md5_ctx ctx_md5; md5_init(&ctx_md5); md5_update(&ctx_md5, datalen, data); md5_digest(&ctx_md5, result->size(), result->data()); return true; } bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result) { struct md5_ctx ctx_md5; off_t bytes; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } md5_init(&ctx_md5); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); return false; } md5_update(&ctx_md5, bytes, reinterpret_cast(buf.data())); } md5_digest(&ctx_md5, result->size(), result->data()); return true; } #else // USE_GNUTLS_NETTLE bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* digest) { gcry_md_hd_t ctx_md5; gcry_error_t err; if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_md5, GCRY_MD_MD5, 0))){ S3FS_PRN_ERR("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); return false; } gcry_md_write(ctx_md5, digest->data(), digest->size()); gcry_md_close(ctx_md5); return true; } bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result) { gcry_md_hd_t ctx_md5; gcry_error_t err; off_t bytes; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_md5, GCRY_MD_MD5, 0))){ S3FS_PRN_ERR("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); return false; } for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); gcry_md_close(ctx_md5); return false; } gcry_md_write(ctx_md5, buf.data(), bytes); } memcpy(result->data(), gcry_md_read(ctx_md5, 0), result->size()); gcry_md_close(ctx_md5); return true; } #endif // USE_GNUTLS_NETTLE //------------------------------------------------------------------- // Utility Function for SHA256 //------------------------------------------------------------------- #ifdef USE_GNUTLS_NETTLE bool s3fs_sha256(const unsigned char* data, size_t datalen, sha256_t* digest) { struct sha256_ctx ctx_sha256; sha256_init(&ctx_sha256); sha256_update(&ctx_sha256, datalen, data); sha256_digest(&ctx_sha256, digest->size(), digest->data()); return true; } bool s3fs_sha256_fd(int fd, off_t start, off_t size, sha256_t* result) { struct sha256_ctx ctx_sha256; off_t bytes; sha256_init(&ctx_sha256); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); return false; } sha256_update(&ctx_sha256, bytes, reinterpret_cast(buf.data())); } sha256_digest(&ctx_sha256, result->size(), result->data()); return true; } #else // USE_GNUTLS_NETTLE bool s3fs_sha256(const unsigned char* data, size_t datalen, sha256_t* digest) { gcry_md_hd_t ctx_sha256; gcry_error_t err; if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ S3FS_PRN_ERR("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); return false; } gcry_md_write(ctx_sha256, data, datalen); memcpy(digest->data(), gcry_md_read(ctx_sha256, 0), digest->size()); gcry_md_close(ctx_sha256); return true; } bool s3fs_sha256_fd(int fd, off_t start, off_t size, sha256_t* result) { gcry_md_hd_t ctx_sha256; gcry_error_t err; off_t bytes; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ S3FS_PRN_ERR("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); return false; } for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); gcry_md_close(ctx_sha256); return false; } gcry_md_write(ctx_sha256, buf.data(), bytes); } memcpy(result->data(), gcry_md_read(ctx_sha256, 0), result->size()); gcry_md_close(ctx_sha256); return true; } #endif // USE_GNUTLS_NETTLE /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/metaheader.cpp000066400000000000000000000257711470675423500167110ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include "common.h" #include "metaheader.h" #include "string_util.h" static constexpr struct timespec DEFAULT_TIMESPEC = {-1, 0}; //------------------------------------------------------------------- // Utility functions for convert //------------------------------------------------------------------- static struct timespec cvt_string_to_time(const char *str) { // [NOTE] // In rclone, there are cases where ns is set to x-amz-meta-mtime // with floating point number. s3fs uses x-amz-meta-mtime by // truncating the floating point or less (in seconds or less) to // correspond to this. // std::string strmtime; long nsec = 0; if(str && '\0' != *str){ strmtime = str; std::string::size_type pos = strmtime.find('.', 0); if(std::string::npos != pos){ nsec = cvt_strtoofft(strmtime.substr(pos + 1).c_str(), /*base=*/ 10); strmtime.erase(pos); } } struct timespec ts = {static_cast(cvt_strtoofft(strmtime.c_str(), /*base=*/ 10)), nsec}; return ts; } static struct timespec get_time(const headers_t& meta, const char *header) { headers_t::const_iterator iter; if(meta.cend() == (iter = meta.find(header))){ return DEFAULT_TIMESPEC; } return cvt_string_to_time((*iter).second.c_str()); } struct timespec get_mtime(const headers_t& meta, bool overcheck) { struct timespec t = get_time(meta, "x-amz-meta-mtime"); if(0 < t.tv_sec){ return t; } t = get_time(meta, "x-amz-meta-goog-reserved-file-mtime"); if(0 < t.tv_sec){ return t; } if(overcheck){ struct timespec ts = {get_lastmodified(meta), 0}; return ts; } return DEFAULT_TIMESPEC; } struct timespec get_ctime(const headers_t& meta, bool overcheck) { struct timespec t = get_time(meta, "x-amz-meta-ctime"); if(0 < t.tv_sec){ return t; } if(overcheck){ struct timespec ts = {get_lastmodified(meta), 0}; return ts; } return DEFAULT_TIMESPEC; } struct timespec get_atime(const headers_t& meta, bool overcheck) { struct timespec t = get_time(meta, "x-amz-meta-atime"); if(0 < t.tv_sec){ return t; } if(overcheck){ struct timespec ts = {get_lastmodified(meta), 0}; return ts; } return DEFAULT_TIMESPEC; } off_t get_size(const char *s) { return cvt_strtoofft(s, /*base=*/ 10); } off_t get_size(const headers_t& meta) { auto iter = meta.find("Content-Length"); if(meta.cend() == iter){ return 0; } return get_size((*iter).second.c_str()); } mode_t get_mode(const char *s, int base) { return static_cast(cvt_strtoofft(s, base)); } mode_t get_mode(const headers_t& meta, const std::string& strpath, bool checkdir, bool forcedir) { mode_t mode = 0; bool isS3sync = false; headers_t::const_iterator iter; if(meta.cend() != (iter = meta.find("x-amz-meta-mode"))){ mode = get_mode((*iter).second.c_str()); }else if(meta.cend() != (iter = meta.find("x-amz-meta-permissions"))){ // for s3sync mode = get_mode((*iter).second.c_str()); isS3sync = true; }else if(meta.cend() != (iter = meta.find("x-amz-meta-goog-reserved-posix-mode"))){ // for GCS mode = get_mode((*iter).second.c_str(), 8); }else{ // If another tool creates an object without permissions, default to owner // read-write and group readable. mode = (!strpath.empty() && '/' == *strpath.rbegin()) ? 0750 : 0640; } // Checking the bitmask, if the last 3 bits are all zero then process as a regular // file type (S_IFDIR or S_IFREG), otherwise return mode unmodified so that S_IFIFO, // S_IFSOCK, S_IFCHR, S_IFLNK and S_IFBLK devices can be processed properly by fuse. if(!(mode & S_IFMT)){ if(!isS3sync){ if(checkdir){ if(forcedir){ mode |= S_IFDIR; }else{ if(meta.cend() != (iter = meta.find("Content-Type"))){ std::string strConType = (*iter).second; // Leave just the mime type, remove any optional parameters (eg charset) std::string::size_type pos = strConType.find(';'); if(std::string::npos != pos){ strConType.erase(pos); } if(strConType == "application/x-directory" || strConType == "httpd/unix-directory"){ // Nextcloud uses this MIME type for directory objects when mounting bucket as external Storage mode |= S_IFDIR; }else if(!strpath.empty() && '/' == *strpath.rbegin()){ if(strConType == "binary/octet-stream" || strConType == "application/octet-stream"){ mode |= S_IFDIR; }else{ if(complement_stat){ // If complement lack stat mode, when the object has '/' character at end of name // and content type is text/plain and the object's size is 0 or 1, it should be // directory. off_t size = get_size(meta); if(strConType == "text/plain" && (0 == size || 1 == size)){ mode |= S_IFDIR; }else{ mode |= S_IFREG; } }else{ mode |= S_IFREG; } } }else{ mode |= S_IFREG; } }else{ mode |= S_IFREG; } } } // If complement lack stat mode, when it's mode is not set any permission, // the object is added minimal mode only for read permission. if(complement_stat && 0 == (mode & (S_IRWXU | S_IRWXG | S_IRWXO))){ mode |= (S_IRUSR | (0 == (mode & S_IFDIR) ? 0 : S_IXUSR)); } }else{ if(!checkdir){ // cut dir/reg flag. mode &= ~S_IFDIR; mode &= ~S_IFREG; } } } return mode; } uid_t get_uid(const char *s) { return static_cast(cvt_strtoofft(s, /*base=*/ 0)); } uid_t get_uid(const headers_t& meta) { headers_t::const_iterator iter; if(meta.cend() != (iter = meta.find("x-amz-meta-uid"))){ return get_uid((*iter).second.c_str()); }else if(meta.cend() != (iter = meta.find("x-amz-meta-owner"))){ // for s3sync return get_uid((*iter).second.c_str()); }else if(meta.cend() != (iter = meta.find("x-amz-meta-goog-reserved-posix-uid"))){ // for GCS return get_uid((*iter).second.c_str()); }else{ return geteuid(); } } gid_t get_gid(const char *s) { return static_cast(cvt_strtoofft(s, /*base=*/ 0)); } gid_t get_gid(const headers_t& meta) { headers_t::const_iterator iter; if(meta.cend() != (iter = meta.find("x-amz-meta-gid"))){ return get_gid((*iter).second.c_str()); }else if(meta.cend() != (iter = meta.find("x-amz-meta-group"))){ // for s3sync return get_gid((*iter).second.c_str()); }else if(meta.cend() != (iter = meta.find("x-amz-meta-goog-reserved-posix-gid"))){ // for GCS return get_gid((*iter).second.c_str()); }else{ return getegid(); } } blkcnt_t get_blocks(off_t size) { return (size / 512) + (0 == (size % 512) ? 0 : 1); } time_t cvtIAMExpireStringToTime(const char* s) { struct tm tm{}; if(!s){ return 0L; } strptime(s, "%Y-%m-%dT%H:%M:%S", &tm); return timegm(&tm); // GMT } time_t get_lastmodified(const char* s) { struct tm tm{}; if(!s){ return -1; } strptime(s, "%a, %d %b %Y %H:%M:%S %Z", &tm); return timegm(&tm); // GMT } time_t get_lastmodified(const headers_t& meta) { auto iter = meta.find("Last-Modified"); if(meta.cend() == iter){ return -1; } return get_lastmodified((*iter).second.c_str()); } // // Returns it whether it is an object with need checking in detail. // If this function returns true, the object is possible to be directory // and is needed checking detail(searching sub object). // bool is_need_check_obj_detail(const headers_t& meta) { headers_t::const_iterator iter; // directory object is Content-Length as 0. if(0 != get_size(meta)){ return false; } // if the object has x-amz-meta information, checking is no more. if(meta.cend() != meta.find("x-amz-meta-mode") || meta.cend() != meta.find("x-amz-meta-mtime") || meta.cend() != meta.find("x-amz-meta-ctime") || meta.cend() != meta.find("x-amz-meta-atime") || meta.cend() != meta.find("x-amz-meta-uid") || meta.cend() != meta.find("x-amz-meta-gid") || meta.cend() != meta.find("x-amz-meta-owner") || meta.cend() != meta.find("x-amz-meta-group") || meta.cend() != meta.find("x-amz-meta-permissions") ) { return false; } // if there is not Content-Type, or Content-Type is "x-directory", // checking is no more. if(meta.cend() == (iter = meta.find("Content-Type"))){ return false; } if("application/x-directory" == (*iter).second){ return false; } return true; } // [NOTE] // If add_noexist is false and the key does not exist, it will not be added. // bool merge_headers(headers_t& base, const headers_t& additional, bool add_noexist) { bool added = false; for(auto iter = additional.cbegin(); iter != additional.cend(); ++iter){ if(add_noexist || base.find(iter->first) != base.cend()){ base[iter->first] = iter->second; added = true; } } return added; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/metaheader.h000066400000000000000000000052071470675423500163460ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_METAHEADER_H_ #define S3FS_METAHEADER_H_ #include #include #include #include //------------------------------------------------------------------- // headers_t //------------------------------------------------------------------- struct header_nocase_cmp { bool operator()(const std::string &strleft, const std::string &strright) const { return (strcasecmp(strleft.c_str(), strright.c_str()) < 0); } }; typedef std::map headers_t; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- struct timespec get_mtime(const headers_t& meta, bool overcheck = true); struct timespec get_ctime(const headers_t& meta, bool overcheck = true); struct timespec get_atime(const headers_t& meta, bool overcheck = true); off_t get_size(const char *s); off_t get_size(const headers_t& meta); mode_t get_mode(const char *s, int base = 0); mode_t get_mode(const headers_t& meta, const std::string& strpath, bool checkdir = false, bool forcedir = false); uid_t get_uid(const char *s); uid_t get_uid(const headers_t& meta); gid_t get_gid(const char *s); gid_t get_gid(const headers_t& meta); blkcnt_t get_blocks(off_t size); time_t cvtIAMExpireStringToTime(const char* s); time_t get_lastmodified(const char* s); time_t get_lastmodified(const headers_t& meta); bool is_need_check_obj_detail(const headers_t& meta); bool merge_headers(headers_t& base, const headers_t& additional, bool add_noexist); bool simple_parse_xml(const char* data, size_t len, const char* key, std::string& value); #endif // S3FS_METAHEADER_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/mpu_util.cpp000066400000000000000000000123131470675423500164340ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include "s3fs_logger.h" #include "mpu_util.h" #include "curl.h" #include "s3fs_xml.h" #include "s3fs_auth.h" #include "string_util.h" //------------------------------------------------------------------- // Global variables //------------------------------------------------------------------- utility_incomp_type utility_mode = utility_incomp_type::NO_UTILITY_MODE; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- static void print_incomp_mpu_list(const incomp_mpu_list_t& list) { printf("\n"); printf("Lists the parts that have been uploaded for a specific multipart upload.\n"); printf("\n"); if(!list.empty()){ printf("---------------------------------------------------------------\n"); int cnt = 0; for(auto iter = list.cbegin(); iter != list.cend(); ++iter, ++cnt){ printf(" Path : %s\n", (*iter).key.c_str()); printf(" UploadId : %s\n", (*iter).id.c_str()); printf(" Date : %s\n", (*iter).date.c_str()); printf("\n"); } printf("---------------------------------------------------------------\n"); }else{ printf("There is no list.\n"); } } static bool abort_incomp_mpu_list(const incomp_mpu_list_t& list, time_t abort_time) { if(list.empty()){ return true; } time_t now_time = time(nullptr); // do removing. S3fsCurl s3fscurl; bool result = true; for(auto iter = list.cbegin(); iter != list.cend(); ++iter){ const char* tpath = (*iter).key.c_str(); std::string upload_id = (*iter).id; if(0 != abort_time){ // abort_time is 0, it means all. time_t date = 0; if(!get_unixtime_from_iso8601((*iter).date.c_str(), date)){ S3FS_PRN_DBG("date format is not ISO 8601 for %s multipart uploading object, skip this.", tpath); continue; } if(now_time <= (date + abort_time)){ continue; } } if(0 != s3fscurl.AbortMultipartUpload(tpath, upload_id)){ S3FS_PRN_EXIT("Failed to remove %s multipart uploading object.", tpath); result = false; }else{ printf("Succeed to remove %s multipart uploading object.\n", tpath); } // reset(initialize) curl object s3fscurl.DestroyCurlHandle(); } return result; } int s3fs_utility_processing(time_t abort_time) { if(utility_incomp_type::NO_UTILITY_MODE == utility_mode){ return EXIT_FAILURE; } printf("\n*** s3fs run as utility mode.\n\n"); S3fsCurl s3fscurl; std::string body; int result = EXIT_SUCCESS; if(0 != s3fscurl.MultipartListRequest(body)){ S3FS_PRN_EXIT("Could not get list multipart upload.\nThere is no incomplete multipart uploaded object in bucket.\n"); result = EXIT_FAILURE; }else{ // parse result(incomplete multipart upload information) S3FS_PRN_DBG("response body = {\n%s\n}", body.c_str()); std::unique_ptr doc(xmlReadMemory(body.c_str(), static_cast(body.size()), "", nullptr, 0), xmlFreeDoc); if(nullptr == doc){ S3FS_PRN_DBG("xmlReadMemory exited with error."); result = EXIT_FAILURE; }else{ // make incomplete uploads list incomp_mpu_list_t list; if(!get_incomp_mpu_list(doc.get(), list)){ S3FS_PRN_DBG("get_incomp_mpu_list exited with error."); result = EXIT_FAILURE; }else{ if(utility_incomp_type::INCOMP_TYPE_LIST == utility_mode){ // print list print_incomp_mpu_list(list); }else if(utility_incomp_type::INCOMP_TYPE_ABORT == utility_mode){ // remove if(!abort_incomp_mpu_list(list, abort_time)){ S3FS_PRN_DBG("an error occurred during removal process."); result = EXIT_FAILURE; } } } } } // ssl s3fs_destroy_global_ssl(); return result; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/mpu_util.h000066400000000000000000000040721470675423500161040ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_MPU_UTIL_H_ #define S3FS_MPU_UTIL_H_ #include #include #include #include //------------------------------------------------------------------- // Structure / Typedef //------------------------------------------------------------------- typedef struct incomplete_multipart_upload_info { std::string key; std::string id; std::string date; }INCOMP_MPU_INFO; typedef std::vector incomp_mpu_list_t; //------------------------------------------------------------------- // enum for utility process mode //------------------------------------------------------------------- enum class utility_incomp_type : uint8_t { NO_UTILITY_MODE = 0, // not utility mode INCOMP_TYPE_LIST, // list of incomplete mpu INCOMP_TYPE_ABORT // delete incomplete mpu }; extern utility_incomp_type utility_mode; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- int s3fs_utility_processing(time_t abort_time); #endif // S3FS_MPU_UTIL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/nss_auth.cpp000066400000000000000000000173511470675423500164310ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common.h" #include "s3fs.h" #include "s3fs_auth.h" #include "s3fs_logger.h" //------------------------------------------------------------------- // Utility Function for version //------------------------------------------------------------------- const char* s3fs_crypt_lib_name() { static constexpr char version[] = "NSS"; return version; } //------------------------------------------------------------------- // Utility Function for global init //------------------------------------------------------------------- bool s3fs_init_global_ssl() { PR_Init(PR_USER_THREAD, PR_PRIORITY_NORMAL, 0); if(SECSuccess != NSS_NoDB_Init(nullptr)){ S3FS_PRN_ERR("Failed NSS_NoDB_Init call."); return false; } return true; } bool s3fs_destroy_global_ssl() { NSS_Shutdown(); PL_ArenaFinish(); PR_Cleanup(); return true; } //------------------------------------------------------------------- // Utility Function for crypt lock //------------------------------------------------------------------- bool s3fs_init_crypt_mutex() { return true; } bool s3fs_destroy_crypt_mutex() { return true; } //------------------------------------------------------------------- // Utility Function for HMAC //------------------------------------------------------------------- static std::unique_ptr s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen, bool is_sha256) { if(!key || !data || !digestlen){ return nullptr; } PK11SlotInfo* Slot; PK11SymKey* pKey; PK11Context* Context; unsigned char tmpdigest[64]; SECItem KeySecItem = {siBuffer, reinterpret_cast(const_cast(key)), static_cast(keylen)}; SECItem NullSecItem = {siBuffer, nullptr, 0}; if(nullptr == (Slot = PK11_GetInternalKeySlot())){ return nullptr; } if(nullptr == (pKey = PK11_ImportSymKey(Slot, (is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, nullptr))){ PK11_FreeSlot(Slot); return nullptr; } if(nullptr == (Context = PK11_CreateContextBySymKey((is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), CKA_SIGN, pKey, &NullSecItem))){ PK11_FreeSymKey(pKey); PK11_FreeSlot(Slot); return nullptr; } *digestlen = 0; if(SECSuccess != PK11_DigestBegin(Context) || SECSuccess != PK11_DigestOp(Context, data, datalen) || SECSuccess != PK11_DigestFinal(Context, tmpdigest, digestlen, sizeof(tmpdigest)) ) { PK11_DestroyContext(Context, PR_TRUE); PK11_FreeSymKey(pKey); PK11_FreeSlot(Slot); return nullptr; } PK11_DestroyContext(Context, PR_TRUE); PK11_FreeSymKey(pKey); PK11_FreeSlot(Slot); std::unique_ptr digest(new unsigned char[*digestlen]); memcpy(digest.get(), tmpdigest, *digestlen); return digest; } std::unique_ptr s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { return s3fs_HMAC_RAW(key, keylen, data, datalen, digestlen, false); } std::unique_ptr s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { return s3fs_HMAC_RAW(key, keylen, data, datalen, digestlen, true); } //------------------------------------------------------------------- // Utility Function for MD5 //------------------------------------------------------------------- bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* result) { PK11Context* md5ctx; unsigned int md5outlen; md5ctx = PK11_CreateDigestContext(SEC_OID_MD5); PK11_DigestOp(md5ctx, data, datalen); PK11_DigestFinal(md5ctx, result->data(), &md5outlen, result->size()); PK11_DestroyContext(md5ctx, PR_TRUE); return true; } bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result) { PK11Context* md5ctx; off_t bytes; unsigned int md5outlen; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } md5ctx = PK11_CreateDigestContext(SEC_OID_MD5); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); PK11_DestroyContext(md5ctx, PR_TRUE); return false; } PK11_DigestOp(md5ctx, buf.data(), bytes); } PK11_DigestFinal(md5ctx, result->data(), &md5outlen, result->size()); PK11_DestroyContext(md5ctx, PR_TRUE); return true; } //------------------------------------------------------------------- // Utility Function for SHA256 //------------------------------------------------------------------- bool s3fs_sha256(const unsigned char* data, size_t datalen, sha256_t* digest) { PK11Context* sha256ctx; unsigned int sha256outlen; sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); PK11_DigestOp(sha256ctx, data, datalen); PK11_DigestFinal(sha256ctx, digest->data(), &sha256outlen, digest->size()); PK11_DestroyContext(sha256ctx, PR_TRUE); return true; } bool s3fs_sha256_fd(int fd, off_t start, off_t size, sha256_t* result) { PK11Context* sha256ctx; off_t bytes; unsigned int sha256outlen; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); PK11_DestroyContext(sha256ctx, PR_TRUE); return false; } PK11_DigestOp(sha256ctx, buf.data(), bytes); } PK11_DigestFinal(sha256ctx, result->data(), &sha256outlen, result->size()); PK11_DestroyContext(sha256ctx, PR_TRUE); return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/openssl_auth.cpp000066400000000000000000000270631470675423500173120ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "s3fs_auth.h" #include "s3fs_logger.h" //------------------------------------------------------------------- // Utility Function for version //------------------------------------------------------------------- const char* s3fs_crypt_lib_name() { static constexpr char version[] = "OpenSSL"; return version; } //------------------------------------------------------------------- // Utility Function for global init //------------------------------------------------------------------- bool s3fs_init_global_ssl() { ERR_load_crypto_strings(); // [NOTE] // OpenSSL 3.0 loads error strings automatically so these functions are not needed. // #ifndef USE_OPENSSL_30 ERR_load_BIO_strings(); #endif OpenSSL_add_all_algorithms(); return true; } bool s3fs_destroy_global_ssl() { EVP_cleanup(); ERR_free_strings(); return true; } //------------------------------------------------------------------- // Utility Function for crypt lock //------------------------------------------------------------------- // internal use struct for openssl struct CRYPTO_dynlock_value { std::mutex dyn_mutex; }; static std::unique_ptr s3fs_crypt_mutex; static void s3fs_crypt_mutex_lock(int mode, int pos, const char* file, int line) __attribute__ ((unused)) NO_THREAD_SAFETY_ANALYSIS; static void s3fs_crypt_mutex_lock(int mode, int pos, const char* file, int line) { if(s3fs_crypt_mutex){ if(mode & CRYPTO_LOCK){ s3fs_crypt_mutex[pos].lock(); }else{ s3fs_crypt_mutex[pos].unlock(); } } } static unsigned long s3fs_crypt_get_threadid() __attribute__ ((unused)); static unsigned long s3fs_crypt_get_threadid() { return static_cast(std::hash()(std::this_thread::get_id())); } static struct CRYPTO_dynlock_value* s3fs_dyn_crypt_mutex(const char* file, int line) __attribute__ ((unused)); static struct CRYPTO_dynlock_value* s3fs_dyn_crypt_mutex(const char* file, int line) { return new CRYPTO_dynlock_value(); } static void s3fs_dyn_crypt_mutex_lock(int mode, struct CRYPTO_dynlock_value* dyndata, const char* file, int line) __attribute__ ((unused)) NO_THREAD_SAFETY_ANALYSIS; static void s3fs_dyn_crypt_mutex_lock(int mode, struct CRYPTO_dynlock_value* dyndata, const char* file, int line) { if(dyndata){ if(mode & CRYPTO_LOCK){ dyndata->dyn_mutex.lock(); }else{ dyndata->dyn_mutex.unlock(); } } } static void s3fs_destroy_dyn_crypt_mutex(struct CRYPTO_dynlock_value* dyndata, const char* file, int line) __attribute__ ((unused)); static void s3fs_destroy_dyn_crypt_mutex(struct CRYPTO_dynlock_value* dyndata, const char* file, int line) { delete dyndata; } bool s3fs_init_crypt_mutex() { if(s3fs_crypt_mutex){ S3FS_PRN_DBG("s3fs_crypt_mutex is not nullptr, destroy it."); // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!s3fs_destroy_crypt_mutex()){ S3FS_PRN_ERR("Failed to s3fs_crypt_mutex"); return false; } } s3fs_crypt_mutex.reset(new std::mutex[CRYPTO_num_locks()]); // static lock CRYPTO_set_locking_callback(s3fs_crypt_mutex_lock); CRYPTO_set_id_callback(s3fs_crypt_get_threadid); // dynamic lock CRYPTO_set_dynlock_create_callback(s3fs_dyn_crypt_mutex); CRYPTO_set_dynlock_lock_callback(s3fs_dyn_crypt_mutex_lock); CRYPTO_set_dynlock_destroy_callback(s3fs_destroy_dyn_crypt_mutex); return true; } bool s3fs_destroy_crypt_mutex() { if(!s3fs_crypt_mutex){ return true; } CRYPTO_set_dynlock_destroy_callback(nullptr); CRYPTO_set_dynlock_lock_callback(nullptr); CRYPTO_set_dynlock_create_callback(nullptr); CRYPTO_set_id_callback(nullptr); CRYPTO_set_locking_callback(nullptr); CRYPTO_cleanup_all_ex_data(); s3fs_crypt_mutex.reset(); return true; } //------------------------------------------------------------------- // Utility Function for HMAC //------------------------------------------------------------------- static std::unique_ptr s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen, bool is_sha256) { if(!key || !data || !digestlen){ return nullptr; } (*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char); std::unique_ptr digest(new unsigned char[*digestlen]); if(is_sha256){ HMAC(EVP_sha256(), key, static_cast(keylen), data, datalen, digest.get(), digestlen); }else{ HMAC(EVP_sha1(), key, static_cast(keylen), data, datalen, digest.get(), digestlen); } return digest; } std::unique_ptr s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { return s3fs_HMAC_RAW(key, keylen, data, datalen, digestlen, false); } std::unique_ptr s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen) { return s3fs_HMAC_RAW(key, keylen, data, datalen, digestlen, true); } #ifdef USE_OPENSSL_30 //------------------------------------------------------------------- // Utility Function for MD5 (OpenSSL >= 3.0) //------------------------------------------------------------------- // [NOTE] // OpenSSL 3.0 deprecated the MD5_*** low-level encryption functions, // so we should use the high-level EVP API instead. // bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* digest) { auto digestlen = static_cast(digest->size()); const EVP_MD* md = EVP_get_digestbyname("md5"); EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(mdctx, md, nullptr); EVP_DigestUpdate(mdctx, data, datalen); EVP_DigestFinal_ex(mdctx, digest->data(), &digestlen); EVP_MD_CTX_destroy(mdctx); return true; } bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result) { auto md5_digest_len = static_cast(result->size()); off_t bytes; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } // instead of MD5_Init std::unique_ptr mdctx(EVP_MD_CTX_new(), EVP_MD_CTX_free); EVP_DigestInit_ex(mdctx.get(), EVP_md5(), nullptr); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); return false; } // instead of MD5_Update EVP_DigestUpdate(mdctx.get(), buf.data(), bytes); } // instead of MD5_Final EVP_DigestFinal_ex(mdctx.get(), result->data(), &md5_digest_len); return true; } #else //------------------------------------------------------------------- // Utility Function for MD5 (OpenSSL < 3.0) //------------------------------------------------------------------- // TODO: Does this fail on OpenSSL < 3.0 and we need to use MD5_CTX functions? bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* digest) { unsigned int digestlen = digest->size(); const EVP_MD* md = EVP_get_digestbyname("md5"); EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(mdctx, md, nullptr); EVP_DigestUpdate(mdctx, data, datalen); EVP_DigestFinal_ex(mdctx, digest->data(), &digestlen); EVP_MD_CTX_destroy(mdctx); return true; } bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result) { MD5_CTX md5ctx; off_t bytes; if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return false; } size = st.st_size; } MD5_Init(&md5ctx); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); return false; } MD5_Update(&md5ctx, buf.data(), bytes); } MD5_Final(result->data(), &md5ctx); return true; } #endif //------------------------------------------------------------------- // Utility Function for SHA256 //------------------------------------------------------------------- bool s3fs_sha256(const unsigned char* data, size_t datalen, sha256_t* digest) { const EVP_MD* md = EVP_get_digestbyname("sha256"); EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(mdctx, md, nullptr); EVP_DigestUpdate(mdctx, data, datalen); auto digestlen = static_cast(digest->size()); EVP_DigestFinal_ex(mdctx, digest->data(), &digestlen); EVP_MD_CTX_destroy(mdctx); return true; } bool s3fs_sha256_fd(int fd, off_t start, off_t size, sha256_t* result) { const EVP_MD* md = EVP_get_digestbyname("sha256"); EVP_MD_CTX* sha256ctx; off_t bytes; if(-1 == fd){ return false; } if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ S3FS_PRN_ERR("fstat error(%d)", errno); return false; } size = st.st_size; } sha256ctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(sha256ctx, md, nullptr); for(off_t total = 0; total < size; total += bytes){ std::array buf; bytes = std::min(static_cast(buf.size()), (size - total)); bytes = pread(fd, buf.data(), bytes, start + total); if(0 == bytes){ // end of file break; }else if(-1 == bytes){ // error S3FS_PRN_ERR("file read error(%d)", errno); EVP_MD_CTX_destroy(sha256ctx); return false; } EVP_DigestUpdate(sha256ctx, buf.data(), bytes); } EVP_DigestFinal_ex(sha256ctx, result->data(), nullptr); EVP_MD_CTX_destroy(sha256ctx); return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/psemaphore.h000066400000000000000000000060461470675423500164140ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_SEMAPHORE_H_ #define S3FS_SEMAPHORE_H_ //------------------------------------------------------------------- // Class Semaphore //------------------------------------------------------------------- // portability wrapper for sem_t since macOS does not implement it #ifdef __APPLE__ #include class Semaphore { public: explicit Semaphore(int value) : value(value), sem(dispatch_semaphore_create(value)) {} ~Semaphore() { // macOS cannot destroy a semaphore with posts less than the initializer for(int i = 0; i < get_value(); ++i){ post(); } dispatch_release(sem); } Semaphore(const Semaphore&) = delete; Semaphore(Semaphore&&) = delete; Semaphore& operator=(const Semaphore&) = delete; Semaphore& operator=(Semaphore&&) = delete; void wait() { dispatch_semaphore_wait(sem, DISPATCH_TIME_FOREVER); } bool try_wait() { if(0 == dispatch_semaphore_wait(sem, DISPATCH_TIME_NOW)){ return true; }else{ return false; } } void post() { dispatch_semaphore_signal(sem); } int get_value() const { return value; } private: int value; dispatch_semaphore_t sem; }; #else #include #include class Semaphore { public: explicit Semaphore(int value) : value(value) { sem_init(&mutex, 0, value); } ~Semaphore() { sem_destroy(&mutex); } void wait() { int r; do { r = sem_wait(&mutex); } while (r == -1 && errno == EINTR); } bool try_wait() { int result; do{ result = sem_trywait(&mutex); }while(result == -1 && errno == EINTR); return (0 == result); } void post() { sem_post(&mutex); } int get_value() const { return value; } private: int value; sem_t mutex; }; #endif #endif // S3FS_SEMAPHORE_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs.cpp000066400000000000000000006364671470675423500155020ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common.h" #include "s3fs.h" #include "s3fs_logger.h" #include "metaheader.h" #include "fdcache.h" #include "fdcache_auto.h" #include "fdcache_stat.h" #include "curl.h" #include "curl_multi.h" #include "s3objlist.h" #include "cache.h" #include "addhead.h" #include "sighandlers.h" #include "s3fs_xml.h" #include "string_util.h" #include "s3fs_auth.h" #include "s3fs_cred.h" #include "s3fs_help.h" #include "s3fs_util.h" #include "mpu_util.h" #include "threadpoolman.h" //------------------------------------------------------------------- // Symbols //------------------------------------------------------------------- #if !defined(ENOATTR) #define ENOATTR ENODATA #endif enum class dirtype : int8_t { UNKNOWN = -1, NEW = 0, OLD = 1, FOLDER = 2, NOOBJ = 3, }; //------------------------------------------------------------------- // Static variables //------------------------------------------------------------------- static uid_t mp_uid = 0; // owner of mount point(only not specified uid opt) static gid_t mp_gid = 0; // group of mount point(only not specified gid opt) static mode_t mp_mode = 0; // mode of mount point static mode_t mp_umask = 0; // umask for mount point static bool is_mp_umask = false;// default does not set. static std::string mountpoint; static std::unique_ptr ps3fscred; // using only in this file static std::string mimetype_file; static bool nocopyapi = false; static bool norenameapi = false; static bool nonempty = false; static bool allow_other = false; static uid_t s3fs_uid = 0; static gid_t s3fs_gid = 0; static mode_t s3fs_umask = 0; static bool is_s3fs_uid = false;// default does not set. static bool is_s3fs_gid = false;// default does not set. static bool is_s3fs_umask = false;// default does not set. static bool is_remove_cache = false; static bool is_use_xattr = false; static off_t multipart_threshold = 25 * 1024 * 1024; static int64_t singlepart_copy_limit = 512 * 1024 * 1024; static bool is_specified_endpoint = false; static int s3fs_init_deferred_exit_status = 0; static bool support_compat_dir = false;// default does not support compatibility directory type static int max_keys_list_object = 1000;// default is 1000 static off_t max_dirty_data = 5LL * 1024LL * 1024LL * 1024LL; static bool use_wtf8 = false; static off_t fake_diskfree_size = -1; // default is not set(-1) static int max_thread_count = 5; // default is 5 static bool update_parent_dir_stat= false; // default not updating parent directory stats static fsblkcnt_t bucket_block_count; // advertised block count of the bucket static unsigned long s3fs_block_size = 16 * 1024 * 1024; // s3fs block size is 16MB //------------------------------------------------------------------- // Global functions : prototype //------------------------------------------------------------------- int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size = true); // [NOTE] global function because this is called from FdEntity class //------------------------------------------------------------------- // Static functions : prototype //------------------------------------------------------------------- static bool is_special_name_folder_object(const char* path); static int chk_dir_object_type(const char* path, std::string& newpath, std::string& nowpath, std::string& nowcache, headers_t* pmeta = nullptr, dirtype* pDirType = nullptr); static int remove_old_type_dir(const std::string& path, dirtype type); static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t* pmeta = nullptr, bool overcheck = true, bool* pisforce = nullptr, bool add_no_truncate_cache = false); static int check_object_access(const char* path, int mask, struct stat* pstbuf); static int check_object_owner(const char* path, struct stat* pstbuf); static int check_parent_object_access(const char* path, int mask); static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags = O_RDONLY, bool is_load = false); static bool multi_head_callback(S3fsCurl* s3fscurl, void* param); static std::unique_ptr multi_head_retry_callback(S3fsCurl* s3fscurl); static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler); static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only = false); static int directory_empty(const char* path); static int rename_large_object(const char* from, const char* to); static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid); static int create_directory_object(const char* path, mode_t mode, const struct timespec& ts_atime, const struct timespec& ts_mtime, const struct timespec& ts_ctime, uid_t uid, gid_t gid, const char* pxattrvalue); static int rename_object(const char* from, const char* to, bool update_ctime); static int rename_object_nocopy(const char* from, const char* to, bool update_ctime); static int clone_directory_object(const char* from, const char* to, bool update_ctime, const char* pxattrvalue); static int rename_directory(const char* from, const char* to); static int update_mctime_parent_directory(const char* _path); static int remote_mountpath_exists(const char* path, bool compat_dir); static bool get_meta_xattr_value(const char* path, std::string& rawvalue); static bool get_parent_meta_xattr_value(const char* path, std::string& rawvalue); static bool get_xattr_posix_key_value(const char* path, std::string& xattrvalue, bool default_key); static bool build_inherited_xattr_value(const char* path, std::string& xattrvalue); static bool parse_xattr_keyval(const std::string& xattrpair, std::string& key, std::string* pval); static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs); static std::string raw_build_xattrs(const xattrs_t& xattrs); static std::string build_xattrs(const xattrs_t& xattrs); static int s3fs_check_service(); static bool set_mountpoint_attribute(struct stat& mpst); static int set_bucket(const std::string& arg); static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_args* outargs); static fsblkcnt_t parse_bucket_size(std::string max_size); static bool is_cmd_exists(const std::string& command); static int print_umount_message(const std::string& mp, bool force) __attribute__ ((unused)); //------------------------------------------------------------------- // fuse interface functions //------------------------------------------------------------------- static int s3fs_getattr(const char* path, struct stat* stbuf); static int s3fs_readlink(const char* path, char* buf, size_t size); static int s3fs_mknod(const char* path, mode_t mode, dev_t rdev); static int s3fs_mkdir(const char* path, mode_t mode); static int s3fs_unlink(const char* path); static int s3fs_rmdir(const char* path); static int s3fs_symlink(const char* from, const char* to); static int s3fs_rename(const char* from, const char* to); static int s3fs_link(const char* from, const char* to); static int s3fs_chmod(const char* path, mode_t mode); static int s3fs_chmod_nocopy(const char* path, mode_t mode); static int s3fs_chown(const char* path, uid_t uid, gid_t gid); static int s3fs_chown_nocopy(const char* path, uid_t uid, gid_t gid); static int s3fs_utimens(const char* path, const struct timespec ts[2]); static int s3fs_utimens_nocopy(const char* path, const struct timespec ts[2]); static int s3fs_truncate(const char* path, off_t size); static int s3fs_create(const char* path, mode_t mode, struct fuse_file_info* fi); static int s3fs_open(const char* path, struct fuse_file_info* fi); static int s3fs_read(const char* path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi); static int s3fs_write(const char* path, const char* buf, size_t size, off_t offset, struct fuse_file_info* fi); static int s3fs_statfs(const char* path, struct statvfs* stbuf); static int s3fs_flush(const char* path, struct fuse_file_info* fi); static int s3fs_fsync(const char* path, int datasync, struct fuse_file_info* fi); static int s3fs_release(const char* path, struct fuse_file_info* fi); static int s3fs_opendir(const char* path, struct fuse_file_info* fi); static int s3fs_readdir(const char* path, void* buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info* fi); static int s3fs_access(const char* path, int mask); static void* s3fs_init(struct fuse_conn_info* conn); static void s3fs_destroy(void*); #if defined(__APPLE__) static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags, uint32_t position); static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size, uint32_t position); #else static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags); static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size); #endif static int s3fs_listxattr(const char* path, char* list, size_t size); static int s3fs_removexattr(const char* path, const char* name); //------------------------------------------------------------------- // Classes //------------------------------------------------------------------- // // A flag indicating whether the mount point has a stat // // [NOTE] // The flag is accessed from child threads, so std::atomic is used for exclusive control of flags. static std::atomic has_mp_stat; // // A synchronous class that calls the fuse_fill_dir_t function that processes the readdir data // class SyncFiller { private: mutable std::mutex filler_lock; void* filler_buff; fuse_fill_dir_t filler_func; std::set filled; public: explicit SyncFiller(void* buff = nullptr, fuse_fill_dir_t filler = nullptr); ~SyncFiller() = default; SyncFiller(const SyncFiller&) = delete; SyncFiller(SyncFiller&&) = delete; SyncFiller& operator=(const SyncFiller&) = delete; SyncFiller& operator=(SyncFiller&&) = delete; int Fill(const char *name, const struct stat *stbuf, off_t off); int SufficiencyFill(const std::vector& pathlist); }; SyncFiller::SyncFiller(void* buff, fuse_fill_dir_t filler) : filler_buff(buff), filler_func(filler) { if(!filler_buff || !filler_func){ S3FS_PRN_CRIT("Internal error: SyncFiller constructor parameter is critical value."); abort(); } } // // See. prototype fuse_fill_dir_t in fuse.h // int SyncFiller::Fill(const char *name, const struct stat *stbuf, off_t off) { const std::lock_guard lock(filler_lock); int result = 0; if(filled.insert(name).second){ result = filler_func(filler_buff, name, stbuf, off); } return result; } int SyncFiller::SufficiencyFill(const std::vector& pathlist) { const std::lock_guard lock(filler_lock); int result = 0; for(auto it = pathlist.cbegin(); it != pathlist.cend(); ++it) { if(filled.insert(*it).second){ if(0 != filler_func(filler_buff, it->c_str(), nullptr, 0)){ result = 1; } } } return result; } //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- static bool IS_REPLACEDIR(dirtype type) { return dirtype::OLD == type || dirtype::FOLDER == type || dirtype::NOOBJ == type; } static bool IS_RMTYPEDIR(dirtype type) { return dirtype::OLD == type || dirtype::FOLDER == type; } static bool IS_CREATE_MP_STAT(const char* path) { // [NOTE] has_mp_stat is set in get_object_attribute() return (path != nullptr && 0 == strcmp(path, "/") && !has_mp_stat); } static bool is_special_name_folder_object(const char* path) { if(!support_compat_dir){ // s3fs does not support compatibility directory type("_$folder$" etc) now, // thus always returns false. return false; } if(!path || '\0' == path[0]){ return false; } if(0 == strcmp(path, "/") && mount_prefix.empty()){ // the path is the mount point which is the bucket root return false; } std::string strpath = path; headers_t header; if(std::string::npos == strpath.find("_$folder$", 0)){ if('/' == *strpath.rbegin()){ strpath.erase(strpath.length() - 1); } strpath += "_$folder$"; } S3fsCurl s3fscurl; if(0 != s3fscurl.HeadRequest(strpath.c_str(), header)){ return false; } header.clear(); S3FS_MALLOCTRIM(0); return true; } // [Detail] // This function is complicated for checking directory object type. // Arguments is used for deleting cache/path, and remake directory object. // Please see the codes which calls this function. // // path: target path // newpath: should be object path for making/putting/getting after checking // nowpath: now object name for deleting after checking // nowcache: now cache path for deleting after checking // pmeta: headers map // pDirType: directory object type // static int chk_dir_object_type(const char* path, std::string& newpath, std::string& nowpath, std::string& nowcache, headers_t* pmeta, dirtype* pDirType) { dirtype TypeTmp = dirtype::UNKNOWN; int result = -1; bool isforce = false; dirtype* pType = pDirType ? pDirType : &TypeTmp; // Normalize new path. newpath = path; if('/' != *newpath.rbegin()){ std::string::size_type Pos; if(std::string::npos != (Pos = newpath.find("_$folder$", 0))){ newpath.erase(Pos); } newpath += "/"; } // Always check "dir/" at first. if(0 == (result = get_object_attribute(newpath.c_str(), nullptr, pmeta, false, &isforce))){ // Found "dir/" cache --> Check for "_$folder$", "no dir object" nowcache = newpath; if(is_special_name_folder_object(newpath.c_str())){ // check support_compat_dir in this function // "_$folder$" type. (*pType) = dirtype::FOLDER; nowpath.erase(newpath.length() - 1); nowpath += "_$folder$"; // cut and add }else if(isforce){ // "no dir object" type. (*pType) = dirtype::NOOBJ; nowpath = ""; }else{ nowpath = newpath; if(!nowpath.empty() && '/' == *nowpath.rbegin()){ // "dir/" type (*pType) = dirtype::NEW; }else{ // "dir" type (*pType) = dirtype::OLD; } } }else if(support_compat_dir){ // Check "dir" when support_compat_dir is enabled nowpath.erase(newpath.length() - 1); if(0 == (result = get_object_attribute(nowpath.c_str(), nullptr, pmeta, false, &isforce))){ // Found "dir" cache --> this case is only "dir" type. // Because, if object is "_$folder$" or "no dir object", the cache is "dir/" type. // (But "no dir object" is checked here.) nowcache = nowpath; if(isforce){ (*pType) = dirtype::NOOBJ; nowpath = ""; }else{ (*pType) = dirtype::OLD; } }else{ // Not found cache --> check for "_$folder$" and "no dir object". // (come here is that support_compat_dir is enabled) nowcache = ""; // This case is no cache. nowpath += "_$folder$"; if(is_special_name_folder_object(nowpath.c_str())){ // "_$folder$" type. (*pType) = dirtype::FOLDER; result = 0; // result is OK. }else if(-ENOTEMPTY == directory_empty(newpath.c_str())){ // "no dir object" type. (*pType) = dirtype::NOOBJ; nowpath = ""; // now path. result = 0; // result is OK. }else{ // Error: Unknown type. (*pType) = dirtype::UNKNOWN; newpath = ""; nowpath = ""; } } } return result; } static int remove_old_type_dir(const std::string& path, dirtype type) { if(IS_RMTYPEDIR(type)){ S3fsCurl s3fscurl; int result = s3fscurl.DeleteRequest(path.c_str()); if(0 != result && -ENOENT != result){ return result; } // succeed removing or not found the directory }else{ // nothing to do } return 0; } // // Get object attributes with stat cache. // This function is base for s3fs_getattr(). // // [NOTICE] // Checking order is changed following list because of reducing the number of the requests. // 1) "dir" // 2) "dir/" // 3) "dir_$folder$" // // Special two case of the mount point directory: // [Case 1] the mount point is the root of the bucket: // 1) "/" // // [Case 2] the mount point is a directory path(ex. foo) below the bucket: // 1) "foo" // 2) "foo/" // 3) "foo_$folder$" // static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t* pmeta, bool overcheck, bool* pisforce, bool add_no_truncate_cache) { int result = -1; struct stat tmpstbuf; struct stat* pstat = pstbuf ? pstbuf : &tmpstbuf; headers_t tmpHead; headers_t* pheader = pmeta ? pmeta : &tmpHead; std::string strpath; S3fsCurl s3fscurl; bool forcedir = false; bool is_mountpoint = false; // path is the mount point bool is_bucket_mountpoint = false; // path is the mount point which is the bucket root std::string::size_type Pos; S3FS_PRN_DBG("[path=%s]", path); if(!path || '\0' == path[0]){ return -ENOENT; } *pstat = {}; // check mount point if(0 == strcmp(path, "/") || 0 == strcmp(path, ".")){ is_mountpoint = true; if(mount_prefix.empty()){ is_bucket_mountpoint = true; } // default stat for mount point if the directory stat file is not existed. pstat->st_mode = mp_mode; pstat->st_uid = is_s3fs_uid ? s3fs_uid : mp_uid; pstat->st_gid = is_s3fs_gid ? s3fs_gid : mp_gid; } // Check cache. pisforce = (nullptr != pisforce ? pisforce : &forcedir); (*pisforce) = false; strpath = path; if(support_compat_dir && overcheck && std::string::npos != (Pos = strpath.find("_$folder$", 0))){ strpath.erase(Pos); strpath += "/"; } // [NOTE] // For mount points("/"), the Stat cache key name is "/". // if(StatCache::getStatCacheData()->GetStat(strpath, pstat, pheader, overcheck, pisforce)){ if(is_mountpoint){ // if mount point, we need to set this. pstat->st_nlink = 1; // see fuse faq } return 0; } if(StatCache::getStatCacheData()->IsNoObjectCache(strpath)){ // there is the path in the cache for no object, it is no object. return -ENOENT; } // set query(head request) path if(is_bucket_mountpoint){ // [NOTE] // This is a special process for mount point // The path is "/" for mount points. // If the bucket mounted at a mount point, we try to find "/" object under // the bucket for mount point's stat. // In this case, we will send the request "HEAD // HTTP /1.1" to S3 server. // // If the directory under the bucket is mounted, it will be sent // "HEAD // HTTP/1.1", so we do not need to change path at // here. // strpath = "//"; // strpath is "//" }else{ strpath = path; } result = s3fscurl.HeadRequest(strpath.c_str(), (*pheader)); s3fscurl.DestroyCurlHandle(); // if not found target path object, do over checking if(-EPERM == result){ // [NOTE] // In case of a permission error, it exists in directory // file list but inaccessible. So there is a problem that // it will send a HEAD request every time, because it is // not registered in the Stats cache. // Therefore, even if the file has a permission error, it // should be registered in the Stats cache. However, if // the response without modifying is registered in the // cache, the file permission will be 0644(umask dependent) // because the meta header does not exist. // Thus, set the mode of 0000 here in the meta header so // that s3fs can print a permission error when the file // is actually accessed. // It is better not to set meta header other than mode, // so do not do it. // (*pheader)["x-amz-meta-mode"] = "0"; }else if(0 != result){ if(overcheck && !is_bucket_mountpoint){ // when support_compat_dir is disabled, strpath maybe have "_$folder$". if('/' != *strpath.rbegin() && std::string::npos == strpath.find("_$folder$", 0)){ // now path is "object", do check "object/" for over checking strpath += "/"; result = s3fscurl.HeadRequest(strpath.c_str(), (*pheader)); s3fscurl.DestroyCurlHandle(); } if(support_compat_dir && 0 != result){ // now path is "object/", do check "object_$folder$" for over checking strpath.erase(strpath.length() - 1); strpath += "_$folder$"; result = s3fscurl.HeadRequest(strpath.c_str(), (*pheader)); s3fscurl.DestroyCurlHandle(); if(0 != result){ // cut "_$folder$" for over checking "no dir object" after here if(std::string::npos != (Pos = strpath.find("_$folder$", 0))){ strpath.erase(Pos); } } } } if(0 != result && std::string::npos == strpath.find("_$folder$", 0)){ // now path is "object" or "object/", do check "no dir object" which is not object but has only children. // // [NOTE] // If the path is mount point and there is no Stat information file for it, we need this process. // if('/' == *strpath.rbegin()){ strpath.erase(strpath.length() - 1); } if(-ENOTEMPTY == directory_empty(strpath.c_str())){ // found "no dir object". strpath += "/"; *pisforce = true; result = 0; } } }else{ if('/' != *strpath.rbegin() && std::string::npos == strpath.find("_$folder$", 0) && is_need_check_obj_detail(*pheader)){ // check a case of that "object" does not have attribute and "object" is possible to be directory. if(-ENOTEMPTY == directory_empty(strpath.c_str())){ // found "no dir object". strpath += "/"; *pisforce = true; result = 0; } } } // set headers for mount point from default stat if(is_mountpoint){ if(0 != result || pheader->empty()){ has_mp_stat = false; // [NOTE] // If mount point and no stat information file, create header // information from the default stat. // (*pheader)["Content-Type"] = S3fsCurl::LookupMimeType(strpath); (*pheader)["x-amz-meta-uid"] = std::to_string(pstat->st_uid); (*pheader)["x-amz-meta-gid"] = std::to_string(pstat->st_gid); (*pheader)["x-amz-meta-mode"] = std::to_string(pstat->st_mode); (*pheader)["x-amz-meta-atime"] = std::to_string(pstat->st_atime); (*pheader)["x-amz-meta-ctime"] = std::to_string(pstat->st_ctime); (*pheader)["x-amz-meta-mtime"] = std::to_string(pstat->st_mtime); result = 0; }else{ has_mp_stat = true; } } // [NOTE] // If the file is listed but not allowed access, put it in // the positive cache instead of the negative cache. // // When mount points, the following error does not occur. // if(0 != result && -EPERM != result){ // finally, "path" object did not find. Add no object cache. strpath = path; // reset original StatCache::getStatCacheData()->AddNoObjectCache(strpath); return result; } // set cache key if(is_bucket_mountpoint){ strpath = "/"; }else if(std::string::npos != (Pos = strpath.find("_$folder$", 0))){ // if path has "_$folder$", need to cut it. strpath.erase(Pos); strpath += "/"; } // Set into cache // // [NOTE] // When add_no_truncate_cache is true, the stats is always cached. // This cached stats is only removed by DelStat(). // This is necessary for the case to access the attribute of opened file. // (ex. getxattr() is called while writing to the opened file.) // if(add_no_truncate_cache || 0 != StatCache::getStatCacheData()->GetCacheSize()){ // add into stat cache if(!StatCache::getStatCacheData()->AddStat(strpath, (*pheader), forcedir, add_no_truncate_cache)){ S3FS_PRN_ERR("failed adding stat cache [path=%s]", strpath.c_str()); return -ENOENT; } if(!StatCache::getStatCacheData()->GetStat(strpath, pstat, pheader, overcheck, pisforce)){ // There is not in cache.(why?) -> retry to convert. if(!convert_header_to_stat(strpath.c_str(), (*pheader), pstat, forcedir)){ S3FS_PRN_ERR("failed convert headers to stat[path=%s]", strpath.c_str()); return -ENOENT; } } }else{ // cache size is Zero -> only convert. if(!convert_header_to_stat(strpath.c_str(), (*pheader), pstat, forcedir)){ S3FS_PRN_ERR("failed convert headers to stat[path=%s]", strpath.c_str()); return -ENOENT; } } if(is_mountpoint){ // if mount point, we need to set this. pstat->st_nlink = 1; // see fuse faq } return 0; } // // Check the object uid and gid for write/read/execute. // The param "mask" is as same as access() function. // If there is not a target file, this function returns -ENOENT. // If the target file can be accessed, the result always is 0. // // path: the target object path // mask: bit field(F_OK, R_OK, W_OK, X_OK) like access(). // stat: nullptr or the pointer of struct stat. // static int check_object_access(const char* path, int mask, struct stat* pstbuf) { int result; struct stat st; struct stat* pst = (pstbuf ? pstbuf : &st); struct fuse_context* pcxt; S3FS_PRN_DBG("[path=%s]", path); if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } S3FS_PRN_DBG("[pid=%u,uid=%u,gid=%u]", (unsigned int)(pcxt->pid), (unsigned int)(pcxt->uid), (unsigned int)(pcxt->gid)); if(0 != (result = get_object_attribute(path, pst))){ // If there is not the target file(object), result is -ENOENT. return result; } if(0 == pcxt->uid){ // root is allowed all accessing. return 0; } if(is_s3fs_uid && s3fs_uid == pcxt->uid){ // "uid" user is allowed all accessing. return 0; } if(F_OK == mask){ // if there is a file, always return allowed. return 0; } // for "uid", "gid" option uid_t obj_uid = (is_s3fs_uid ? s3fs_uid : pst->st_uid); gid_t obj_gid = (is_s3fs_gid ? s3fs_gid : pst->st_gid); // compare file mode and uid/gid + mask. mode_t mode; mode_t base_mask = S_IRWXO; if(is_s3fs_umask){ // If umask is set, all object attributes set ~umask. mode = ((S_IRWXU | S_IRWXG | S_IRWXO) & ~s3fs_umask); }else{ mode = pst->st_mode; } if(pcxt->uid == obj_uid){ base_mask |= S_IRWXU; } if(pcxt->gid == obj_gid){ base_mask |= S_IRWXG; } else if(1 == is_uid_include_group(pcxt->uid, obj_gid)){ base_mask |= S_IRWXG; } mode &= base_mask; if(X_OK == (mask & X_OK)){ if(0 == (mode & (S_IXUSR | S_IXGRP | S_IXOTH))){ return -EACCES; } } if(W_OK == (mask & W_OK)){ if(0 == (mode & (S_IWUSR | S_IWGRP | S_IWOTH))){ return -EACCES; } } if(R_OK == (mask & R_OK)){ if(0 == (mode & (S_IRUSR | S_IRGRP | S_IROTH))){ return -EACCES; } } if(0 == mode){ return -EACCES; } return 0; } static int check_object_owner(const char* path, struct stat* pstbuf) { int result; struct stat st; struct stat* pst = (pstbuf ? pstbuf : &st); const struct fuse_context* pcxt; S3FS_PRN_DBG("[path=%s]", path); if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } if(0 != (result = get_object_attribute(path, pst))){ // If there is not the target file(object), result is -ENOENT. return result; } // check owner if(0 == pcxt->uid){ // root is allowed all accessing. return 0; } if(is_s3fs_uid && s3fs_uid == pcxt->uid){ // "uid" user is allowed all accessing. return 0; } if(pcxt->uid == pst->st_uid){ return 0; } return -EPERM; } // // Check accessing the parent directories of the object by uid and gid. // static int check_parent_object_access(const char* path, int mask) { std::string parent; int result; S3FS_PRN_DBG("[path=%s]", path); if(0 == strcmp(path, "/") || 0 == strcmp(path, ".")){ // path is mount point. return 0; } if(X_OK == (mask & X_OK)){ for(parent = mydirname(path); !parent.empty(); parent = mydirname(parent)){ if(parent == "."){ parent = "/"; } if(0 != (result = check_object_access(parent.c_str(), X_OK, nullptr))){ return result; } if(parent == "/" || parent == "."){ break; } } } mask = (mask & ~X_OK); if(0 != mask){ parent = mydirname(path); if(parent == "."){ parent = "/"; } if(0 != (result = check_object_access(parent.c_str(), mask, nullptr))){ return result; } } return 0; } // // ssevalue is MD5 for SSE-C type, or KMS id for SSE-KMS // bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue) { if(!path){ return false; } headers_t meta; if(0 != get_object_attribute(path, nullptr, &meta)){ S3FS_PRN_ERR("Failed to get object(%s) headers", path); return false; } ssetype = sse_type_t::SSE_DISABLE; ssevalue.clear(); for(auto iter = meta.cbegin(); iter != meta.cend(); ++iter){ std::string key = (*iter).first; if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption") && 0 == strcasecmp((*iter).second.c_str(), "AES256")){ ssetype = sse_type_t::SSE_S3; }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-aws-kms-key-id")){ ssetype = sse_type_t::SSE_KMS; ssevalue = (*iter).second; }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ ssetype = sse_type_t::SSE_C; ssevalue = (*iter).second; } } return true; } static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags, bool is_load) { int result; struct stat stobj; FdEntity* ent; headers_t meta; S3FS_PRN_INFO2("[path=%s]", path); if(0 != (result = get_object_attribute(path, &stobj, &meta))){ return result; } // open struct timespec st_mctime; if(!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)){ st_mctime = S3FS_OMIT_TS; }else{ set_stat_to_timespec(stobj, stat_time_type::MTIME, st_mctime); } bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true; if(nullptr == (ent = autoent.Open(path, &meta, stobj.st_size, st_mctime, flags, force_tmpfile, true, false))){ S3FS_PRN_ERR("Could not open file. errno(%d)", errno); return -EIO; } // load if(is_load && !ent->LoadAll(autoent.GetPseudoFd())){ S3FS_PRN_ERR("Could not load file. errno(%d)", errno); autoent.Close(); return -EIO; } *entity = ent; return 0; } // // create or update s3 meta // @return fuse return code // int put_headers(const char* path, headers_t& meta, bool is_copy, bool use_st_size) { int result; S3fsCurl s3fscurl(true); off_t size; std::string strpath; S3FS_PRN_INFO2("[path=%s]", path); if(0 == strcmp(path, "/") && mount_prefix.empty()){ strpath = "//"; // for the mount point that is bucket root, change "/" to "//". }else{ strpath = path; } // files larger than 5GB must be modified via the multipart interface // call use_st_size as false when the file does not exist(ex. rename object) if(use_st_size && '/' != *strpath.rbegin()){ // directory object("dir/") is always 0(Content-Length = 0) struct stat buf; if(0 != (result = get_object_attribute(path, &buf))){ return result; } size = buf.st_size; }else{ size = get_size(meta); } if(!nocopyapi && !nomultipart && size >= multipart_threshold){ if(0 != (result = s3fscurl.MultipartHeadRequest(strpath.c_str(), size, meta))){ return result; } }else{ if(0 != (result = s3fscurl.PutHeadRequest(strpath.c_str(), meta, is_copy))){ return result; } } return 0; } static int s3fs_getattr(const char* _path, struct stat* stbuf) { WTF8_ENCODE(path) int result; #if defined(__APPLE__) FUSE_CTX_DBG("[path=%s]", path); #else FUSE_CTX_INFO("[path=%s]", path); #endif // check parent directory attribute. if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_access(path, F_OK, stbuf))){ return result; } // If has already opened fd, the st_size should be instead. // (See: Issue 241) if(stbuf){ AutoFdEntity autoent; const FdEntity* ent; if(nullptr != (ent = autoent.OpenExistFdEntity(path))){ struct stat tmpstbuf; if(ent->GetStats(tmpstbuf)){ stbuf->st_size = tmpstbuf.st_size; } } if(0 == strcmp(path, "/")){ stbuf->st_size = 4096; } stbuf->st_blksize = 4096; stbuf->st_blocks = get_blocks(stbuf->st_size); S3FS_PRN_DBG("[path=%s] uid=%u, gid=%u, mode=%04o", path, (unsigned int)(stbuf->st_uid), (unsigned int)(stbuf->st_gid), stbuf->st_mode); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_readlink(const char* _path, char* buf, size_t size) { if(!_path || !buf || 0 == size){ return 0; } WTF8_ENCODE(path) std::string strValue; FUSE_CTX_INFO("[path=%s]", path); // check symbolic link cache if(!StatCache::getStatCacheData()->GetSymlink(path, strValue)){ // not found in cache, then open the path { // scope for AutoFdEntity AutoFdEntity autoent; FdEntity* ent; int result; if(0 != (result = get_local_fent(autoent, &ent, path, O_RDONLY))){ S3FS_PRN_ERR("could not get fent(file=%s)", path); return result; } // Get size off_t readsize; if(!ent->GetSize(readsize)){ S3FS_PRN_ERR("could not get file size(file=%s)", path); return -EIO; } if(static_cast(size) <= readsize){ readsize = size - 1; } // Read ssize_t ressize; if(0 > (ressize = ent->Read(autoent.GetPseudoFd(), buf, 0, readsize))){ S3FS_PRN_ERR("could not read file(file=%s, ressize=%zd)", path, ressize); return static_cast(ressize); } buf[ressize] = '\0'; } // check buf if it has space words. strValue = trim(buf); // decode wtf8. This will always be shorter if(use_wtf8){ strValue = s3fs_wtf8_decode(strValue); } // add symbolic link cache if(!StatCache::getStatCacheData()->AddSymlink(path, strValue)){ S3FS_PRN_ERR("failed to add symbolic link cache for %s", path); } } // copy result strncpy(buf, strValue.c_str(), size - 1); buf[size - 1] = '\0'; S3FS_MALLOCTRIM(0); return 0; } // common function for creation of a plain object static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid) { S3FS_PRN_INFO2("[path=%s][mode=%04o]", path, mode); std::string strnow = s3fs_str_realtime(); headers_t meta; meta["Content-Type"] = S3fsCurl::LookupMimeType(path); meta["x-amz-meta-uid"] = std::to_string(uid); meta["x-amz-meta-gid"] = std::to_string(gid); meta["x-amz-meta-mode"] = std::to_string(mode); meta["x-amz-meta-atime"] = strnow; meta["x-amz-meta-ctime"] = strnow; meta["x-amz-meta-mtime"] = strnow; S3fsCurl s3fscurl(true); return s3fscurl.PutRequest(path, meta, -1); // fd=-1 means for creating zero byte object. } static int s3fs_mknod(const char *_path, mode_t mode, dev_t rdev) { WTF8_ENCODE(path) int result; struct fuse_context* pcxt; FUSE_CTX_INFO("[path=%s][mode=%04o][dev=%llu]", path, mode, (unsigned long long)rdev); if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } if(0 != (result = create_file_object(path, mode, pcxt->uid, pcxt->gid))){ S3FS_PRN_ERR("could not create object for special file(result=%d)", result); return result; } StatCache::getStatCacheData()->DelStat(path); // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to mknod the file(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi) { WTF8_ENCODE(path) int result; const struct fuse_context* pcxt; FUSE_CTX_INFO("[path=%s][mode=%04o][flags=0x%x]", path, mode, fi->flags); if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } // check parent directory attribute. if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } result = check_object_access(path, W_OK, nullptr); if(-ENOENT == result){ if(0 != (result = check_parent_object_access(path, W_OK))){ return result; } }else if(0 != result){ return result; } std::string strnow = s3fs_str_realtime(); headers_t meta; meta["Content-Length"] = "0"; meta["x-amz-meta-uid"] = std::to_string(pcxt->uid); meta["x-amz-meta-gid"] = std::to_string(pcxt->gid); meta["x-amz-meta-mode"] = std::to_string(mode); meta["x-amz-meta-atime"] = strnow; meta["x-amz-meta-mtime"] = strnow; meta["x-amz-meta-ctime"] = strnow; std::string xattrvalue; if(build_inherited_xattr_value(path, xattrvalue)){ S3FS_PRN_DBG("Set xattrs = %s", urlDecode(xattrvalue).c_str()); meta["x-amz-meta-xattr"] = xattrvalue; } // [NOTE] set no_truncate flag // At this point, the file has not been created(uploaded) and // the data is only present in the Stats cache. // The Stats cache should not be deleted automatically by // timeout. If this stats is deleted, s3fs will try to get it // from the server with a Head request and will get an // unexpected error because the result object does not exist. // if(!StatCache::getStatCacheData()->AddStat(path, meta, false, true)){ return -EIO; } AutoFdEntity autoent; FdEntity* ent; int error = 0; if(nullptr == (ent = autoent.Open(path, &meta, 0, S3FS_OMIT_TS, fi->flags, false, true, false, &error))){ StatCache::getStatCacheData()->DelStat(path); return error; } ent->MarkDirtyNewFile(); fi->fh = autoent.Detach(); // KEEP fdentity open; S3FS_MALLOCTRIM(0); return 0; } static int create_directory_object(const char* path, mode_t mode, const struct timespec& ts_atime, const struct timespec& ts_mtime, const struct timespec& ts_ctime, uid_t uid, gid_t gid, const char* pxattrvalue) { S3FS_PRN_INFO1("[path=%s][mode=%04o][atime=%s][mtime=%s][ctime=%s][uid=%u][gid=%u]", path, mode, str(ts_atime).c_str(), str(ts_mtime).c_str(), str(ts_ctime).c_str(), (unsigned int)uid, (unsigned int)gid); if(!path || '\0' == path[0]){ return -EINVAL; } std::string tpath = path; if('/' != *tpath.rbegin()){ tpath += "/"; }else if("/" == tpath && mount_prefix.empty()){ tpath = "//"; // for the mount point that is bucket root, change "/" to "//". } headers_t meta; meta["x-amz-meta-uid"] = std::to_string(uid); meta["x-amz-meta-gid"] = std::to_string(gid); meta["x-amz-meta-mode"] = std::to_string(mode); meta["x-amz-meta-atime"] = str(ts_atime); meta["x-amz-meta-mtime"] = str(ts_mtime); meta["x-amz-meta-ctime"] = str(ts_ctime); if(pxattrvalue){ S3FS_PRN_DBG("Set xattrs = %s", urlDecode(pxattrvalue).c_str()); meta["x-amz-meta-xattr"] = pxattrvalue; } S3fsCurl s3fscurl; return s3fscurl.PutRequest(tpath.c_str(), meta, -1); // fd=-1 means for creating zero byte object. } static int s3fs_mkdir(const char* _path, mode_t mode) { WTF8_ENCODE(path) int result; struct fuse_context* pcxt; FUSE_CTX_INFO("[path=%s][mode=%04o]", path, mode); if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } // check parent directory attribute. if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){ return result; } if(-ENOENT != (result = check_object_access(path, F_OK, nullptr))){ if(0 == result){ result = -EEXIST; } return result; } std::string xattrvalue; const char* pxattrvalue; if(get_parent_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } struct timespec now; s3fs_realtime(now); result = create_directory_object(path, mode, now, now, now, pcxt->uid, pcxt->gid, pxattrvalue); StatCache::getStatCacheData()->DelStat(path); // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to create the directory(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_unlink(const char* _path) { WTF8_ENCODE(path) int result; FUSE_CTX_INFO("[path=%s]", path); if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){ return result; } S3fsCurl s3fscurl; result = s3fscurl.DeleteRequest(path); StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelSymlink(path); FdManager::DeleteCacheFile(path); // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to remove the file(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } S3FS_MALLOCTRIM(0); return result; } static int directory_empty(const char* path) { int result; S3ObjList head; if((result = list_bucket(path, head, "/", true)) != 0){ S3FS_PRN_ERR("list_bucket returns error."); return result; } if(!head.IsEmpty()){ return -ENOTEMPTY; } return 0; } static int s3fs_rmdir(const char* _path) { WTF8_ENCODE(path) int result; std::string strpath; struct stat stbuf; FUSE_CTX_INFO("[path=%s]", path); if(0 != (result = check_parent_object_access(path, W_OK | X_OK))){ return result; } // directory must be empty if(directory_empty(path) != 0){ return -ENOTEMPTY; } strpath = path; if('/' != *strpath.rbegin()){ strpath += "/"; } S3fsCurl s3fscurl; result = s3fscurl.DeleteRequest(strpath.c_str()); s3fscurl.DestroyCurlHandle(); StatCache::getStatCacheData()->DelStat(strpath); // double check for old version(before 1.63) // The old version makes "dir" object, newer version makes "dir/". // A case, there is only "dir", the first removing object is "dir/". // Then "dir/" is not exists, but curl_delete returns 0. // So need to check "dir" and should be removed it. if('/' == *strpath.rbegin()){ strpath.erase(strpath.length() - 1); } if(0 == get_object_attribute(strpath.c_str(), &stbuf, nullptr, false)){ if(S_ISDIR(stbuf.st_mode)){ // Found "dir" object. result = s3fscurl.DeleteRequest(strpath.c_str()); s3fscurl.DestroyCurlHandle(); StatCache::getStatCacheData()->DelStat(strpath); } } // If there is no "dir" and "dir/" object(this case is made by s3cmd/s3sync), // the cache key is "dir/". So we get error only once(delete "dir/"). // check for "_$folder$" object. // This processing is necessary for other S3 clients compatibility. if(is_special_name_folder_object(strpath.c_str())){ strpath += "_$folder$"; result = s3fscurl.DeleteRequest(strpath.c_str()); } // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to remove the directory(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_symlink(const char* _from, const char* _to) { WTF8_ENCODE(from) WTF8_ENCODE(to) int result; const struct fuse_context* pcxt; FUSE_CTX_INFO("[from=%s][to=%s]", from, to); if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){ return result; } if(-ENOENT != (result = check_object_access(to, F_OK, nullptr))){ if(0 == result){ result = -EEXIST; } return result; } std::string strnow = s3fs_str_realtime(); headers_t headers; headers["Content-Type"] = "application/octet-stream"; // Static headers["x-amz-meta-mode"] = std::to_string(S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO); headers["x-amz-meta-atime"] = strnow; headers["x-amz-meta-ctime"] = strnow; headers["x-amz-meta-mtime"] = strnow; headers["x-amz-meta-uid"] = std::to_string(pcxt->uid); headers["x-amz-meta-gid"] = std::to_string(pcxt->gid); // [NOTE] // Symbolic links do not set xattrs. // open tmpfile std::string strFrom; { // scope for AutoFdEntity AutoFdEntity autoent; FdEntity* ent; if(nullptr == (ent = autoent.Open(to, &headers, 0, S3FS_OMIT_TS, O_RDWR, true, true, false))){ S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno); return -errno; } // write(without space words) strFrom = trim(from); auto from_size = static_cast(strFrom.length()); ssize_t ressize; if(from_size != (ressize = ent->Write(autoent.GetPseudoFd(), strFrom.c_str(), 0, from_size))){ if(ressize < 0){ S3FS_PRN_ERR("could not write tmpfile(errno=%d)", static_cast(ressize)); return static_cast(ressize); }else{ S3FS_PRN_ERR("could not write tmpfile %zd byte(errno=%d)", ressize, errno); return (0 == errno ? -EIO : -errno); } } // upload if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_WARN("could not upload tmpfile(result=%d)", result); } } StatCache::getStatCacheData()->DelStat(to); if(!StatCache::getStatCacheData()->AddSymlink(to, strFrom)){ S3FS_PRN_ERR("failed to add symbolic link cache for %s", to); } // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(to))){ S3FS_PRN_ERR("succeed to create symbolic link(%s), but could not update timestamp of its parent directory(result=%d).", to, update_result); } S3FS_MALLOCTRIM(0); return result; } static int rename_object(const char* from, const char* to, bool update_ctime) { int result; headers_t meta; struct stat buf; S3FS_PRN_INFO1("[from=%s][to=%s]", from , to); if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){ // not permit writing "to" object parent dir. return result; } if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){ // not permit removing "from" object parent dir. return result; } if(0 != (result = get_object_attribute(from, &buf, &meta))){ return result; } std::string strSourcePath = (mount_prefix.empty() && 0 == strcmp("/", from)) ? "//" : from; if(update_ctime){ meta["x-amz-meta-ctime"] = s3fs_str_realtime(); } meta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); meta["Content-Type"] = S3fsCurl::LookupMimeType(to); meta["x-amz-metadata-directive"] = "REPLACE"; std::string xattrvalue; if(get_meta_xattr_value(from, xattrvalue)){ S3FS_PRN_DBG("Set xattrs = %s", urlDecode(xattrvalue).c_str()); meta["x-amz-meta-xattr"] = xattrvalue; } // [NOTE] // If it has a cache, open it first and leave it open until rename. // The cache is renamed after put_header, because it must be open // at the time of renaming. { // update time AutoFdEntity autoent; FdEntity* ent; if(nullptr == (ent = autoent.OpenExistFdEntity(from))){ // no opened fd // get mtime/ctime/atime from meta struct timespec mtime = get_mtime(meta); struct timespec ctime = get_ctime(meta); struct timespec atime = get_atime(meta); if(mtime.tv_sec < 0){ mtime.tv_sec = 0L; mtime.tv_nsec = 0L; } if(ctime.tv_sec < 0){ ctime.tv_sec = 0L; ctime.tv_nsec = 0L; } if(atime.tv_sec < 0){ atime.tv_sec = 0L; atime.tv_nsec = 0L; } if(FdManager::IsCacheDir()){ // create cache file if be needed // // [NOTE] // Do not specify "S3FS_OMIT_TS" for mctime parameter. // This is because if the cache file does not exist, the pagelist for it // will be recreated, but the entire file area indicated by this pagelist // will be in the "modified" state. // This does not affect the rename process, but the cache information in // the "modified" state remains, making it impossible to read the file correctly. // ent = autoent.Open(from, &meta, buf.st_size, mtime, O_RDONLY, false, true, false); } if(ent){ ent->SetMCtime(mtime, ctime); ent->SetAtime(atime); } } } // copy if(0 != (result = put_headers(to, meta, true, /* use_st_size= */ false))){ return result; } // rename FdManager::get()->Rename(from, to); // Remove file result = s3fs_unlink(from); StatCache::getStatCacheData()->DelStat(to); return result; } static int rename_object_nocopy(const char* from, const char* to, bool update_ctime) { int result; FUSE_CTX_INFO1("[from=%s][to=%s]", from , to); if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){ // not permit writing "to" object parent dir. return result; } if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){ // not permit removing "from" object parent dir. return result; } // open & load { // scope for AutoFdEntity AutoFdEntity autoent; FdEntity* ent; if(0 != (result = get_local_fent(autoent, &ent, from, O_RDWR, true))){ S3FS_PRN_ERR("could not open and read file(%s)", from); return result; } // Set header if(!ent->SetContentType(to)){ S3FS_PRN_ERR("could not set content-type for %s", to); return -EIO; } // update ctime if(update_ctime){ struct timespec ts; s3fs_realtime(ts); ent->SetCtime(ts); } // upload if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), to, true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); return result; } } FdManager::get()->Rename(from, to); // Remove file result = s3fs_unlink(from); // Stats StatCache::getStatCacheData()->DelStat(to); return result; } static int rename_large_object(const char* from, const char* to) { int result; struct stat buf; headers_t meta; S3FS_PRN_INFO1("[from=%s][to=%s]", from , to); if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){ // not permit writing "to" object parent dir. return result; } if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){ // not permit removing "from" object parent dir. return result; } if(0 != (result = get_object_attribute(from, &buf, &meta, false))){ return result; } S3fsCurl s3fscurl(true); if(0 != (result = s3fscurl.MultipartRenameRequest(from, to, meta, buf.st_size))){ return result; } s3fscurl.DestroyCurlHandle(); // Rename cache file FdManager::get()->Rename(from, to); // Remove file result = s3fs_unlink(from); // Stats StatCache::getStatCacheData()->DelStat(to); return result; } static int clone_directory_object(const char* from, const char* to, bool update_ctime, const char* pxattrvalue) { int result = -1; struct stat stbuf; S3FS_PRN_INFO1("[from=%s][to=%s]", from, to); // get target's attributes if(0 != (result = get_object_attribute(from, &stbuf))){ return result; } struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); if(update_ctime){ s3fs_realtime(ts_ctime); }else{ set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime); } result = create_directory_object(to, stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue); StatCache::getStatCacheData()->DelStat(to); return result; } static int rename_directory(const char* from, const char* to) { S3ObjList head; s3obj_list_t headlist; std::string strfrom = from ? from : ""; // from is without "/". std::string strto = to ? to : ""; // to is without "/" too. std::string basepath = strfrom + "/"; std::string newpath; // should be from name(not used) std::string nowcache; // now cache path(not used) dirtype DirType; bool normdir; std::vector mvnodes; struct stat stbuf; int result; bool is_dir; S3FS_PRN_INFO1("[from=%s][to=%s]", from, to); // // Initiate and Add base directory into mvnode struct. // strto += "/"; if(0 == chk_dir_object_type(from, newpath, strfrom, nowcache, nullptr, &DirType) && dirtype::UNKNOWN != DirType){ if(dirtype::NOOBJ != DirType){ normdir = false; }else{ normdir = true; strfrom = from; // from directory is not removed, but from directory attr is needed. } mvnodes.emplace_back(strfrom, strto, true, normdir); }else{ // Something wrong about "from" directory. } // // get a list of all the objects // // No delimiter is specified, the result(head) is all object keys. // (CommonPrefixes is empty, but all object is listed in Key.) if(0 != (result = list_bucket(basepath.c_str(), head, nullptr))){ S3FS_PRN_ERR("list_bucket returns error."); return result; } head.GetNameList(headlist); // get name without "/". StatCache::getStatCacheData()->GetNotruncateCache(basepath, headlist); // Add notruncate file name from stat cache S3ObjList::MakeHierarchizedList(headlist, false); // add hierarchized dir. s3obj_list_t::const_iterator liter; for(liter = headlist.cbegin(); headlist.cend() != liter; ++liter){ // make "from" and "to" object name. std::string from_name = basepath + (*liter); std::string to_name = strto + (*liter); std::string etag = head.GetETag((*liter).c_str()); // Check subdirectory. StatCache::getStatCacheData()->HasStat(from_name, etag.c_str()); // Check ETag if(0 != get_object_attribute(from_name.c_str(), &stbuf, nullptr)){ S3FS_PRN_WARN("failed to get %s object attribute.", from_name.c_str()); continue; } if(S_ISDIR(stbuf.st_mode)){ is_dir = true; if(0 != chk_dir_object_type(from_name.c_str(), newpath, from_name, nowcache, nullptr, &DirType) || dirtype::UNKNOWN == DirType){ S3FS_PRN_WARN("failed to get %s%s object directory type.", basepath.c_str(), (*liter).c_str()); continue; } if(dirtype::NOOBJ != DirType){ normdir = false; }else{ normdir = true; from_name = basepath + (*liter); // from directory is not removed, but from directory attr is needed. } }else{ is_dir = false; normdir = false; } // push this one onto the stack mvnodes.emplace_back(from_name, to_name, is_dir, normdir); } std::sort(mvnodes.begin(), mvnodes.end(), [](const mvnode& a, const mvnode& b) { return a.old_path < b.old_path; }); // // rename // // rename directory objects. for(auto mn_cur = mvnodes.cbegin(); mn_cur != mvnodes.cend(); ++mn_cur){ if(mn_cur->is_dir && !mn_cur->old_path.empty()){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(mn_cur->old_path.c_str(), xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } // [NOTE] // The ctime is updated only for the top (from) directory. // Other than that, it will not be updated. // if(0 != (result = clone_directory_object(mn_cur->old_path.c_str(), mn_cur->new_path.c_str(), (strfrom == mn_cur->old_path), pxattrvalue))){ S3FS_PRN_ERR("clone_directory_object returned an error(%d)", result); return result; } } } // iterate over the list - copy the files with rename_object // does a safe copy - copies first and then deletes old for(auto mn_cur = mvnodes.cbegin(); mn_cur != mvnodes.cend(); ++mn_cur){ if(!mn_cur->is_dir){ if(!nocopyapi && !norenameapi){ result = rename_object(mn_cur->old_path.c_str(), mn_cur->new_path.c_str(), false); // keep ctime }else{ result = rename_object_nocopy(mn_cur->old_path.c_str(), mn_cur->new_path.c_str(), false); // keep ctime } if(0 != result){ S3FS_PRN_ERR("rename_object returned an error(%d)", result); return result; } } } // Iterate over old the directories, bottoms up and remove for(auto mn_cur = mvnodes.rbegin(); mn_cur != mvnodes.rend(); ++mn_cur){ if(mn_cur->is_dir && !mn_cur->old_path.empty()){ if(!(mn_cur->is_normdir)){ if(0 != (result = s3fs_rmdir(mn_cur->old_path.c_str()))){ S3FS_PRN_ERR("s3fs_rmdir returned an error(%d)", result); return result; } }else{ // cache clear. StatCache::getStatCacheData()->DelStat(mn_cur->old_path); } } } return 0; } static int s3fs_rename(const char* _from, const char* _to) { WTF8_ENCODE(from) WTF8_ENCODE(to) struct stat buf; int result; FUSE_CTX_INFO("[from=%s][to=%s]", from, to); if(0 != (result = check_parent_object_access(to, W_OK | X_OK))){ // not permit writing "to" object parent dir. return result; } if(0 != (result = check_parent_object_access(from, W_OK | X_OK))){ // not permit removing "from" object parent dir. return result; } if(0 != (result = get_object_attribute(from, &buf, nullptr))){ return result; } if(0 != (result = directory_empty(to))){ return result; } // flush pending writes if file is open { // scope for AutoFdEntity AutoFdEntity autoent; FdEntity* ent; if(nullptr != (ent = autoent.OpenExistFdEntity(from, O_RDWR))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); return result; } StatCache::getStatCacheData()->DelStat(from); } } // files larger than 5GB must be modified via the multipart interface if(S_ISDIR(buf.st_mode)){ result = rename_directory(from, to); }else if(!nomultipart && buf.st_size >= singlepart_copy_limit){ result = rename_large_object(from, to); }else{ if(!nocopyapi && !norenameapi){ result = rename_object(from, to, true); // update ctime }else{ result = rename_object_nocopy(from, to, true); // update ctime } } // update parent directory timestamp // // [NOTE] // already updated timestamp for original path in above functions. // int update_result; if(0 != (update_result = update_mctime_parent_directory(to))){ S3FS_PRN_ERR("succeed to create the file/directory(%s), but could not update timestamp of its parent directory(result=%d).", to, update_result); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_link(const char* _from, const char* _to) { WTF8_ENCODE(from) WTF8_ENCODE(to) FUSE_CTX_INFO("[from=%s][to=%s]", from, to); return -ENOTSUP; } static int s3fs_chmod(const char* _path, mode_t mode) { WTF8_ENCODE(path) int result; std::string strpath; std::string newpath; std::string nowcache; headers_t meta; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; FUSE_CTX_INFO("[path=%s][mode=%04o]", path, mode); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_owner(path, &stbuf))){ return result; } if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, &meta); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode) && (IS_REPLACEDIR(nDirType) || IS_CREATE_MP_STAT(path))){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } if(IS_REPLACEDIR(nDirType)){ // Should rebuild directory object(except new type) // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); s3fs_realtime(ts_ctime); if(0 != (result = create_directory_object(newpath.c_str(), mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){ return result; } }else{ // normal object or directory object of newer version std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime(); updatemeta["x-amz-meta-mode"] = std::to_string(mode); updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. // // If the file starts uploading by multipart when the disk capacity is insufficient, // we need to put these header after finishing upload. // Or if the file is only open, we must update to FdEntity's internal meta. // AutoFdEntity autoent; FdEntity* ent; bool need_put_header = true; if(nullptr != (ent = autoent.OpenExistFdEntity(path))){ if(ent->MergeOrgMeta(updatemeta)){ // meta is changed, but now uploading. // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); need_put_header = false; // If there is data in the Stats cache, update the Stats cache. StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta); // [NOTE] // There are cases where this function is called during the process of // creating a new file (before uploading). // In this case, a temporary cache exists in the Stat cache. // So we need to update the cache, if it exists. (see. s3fs_create and s3fs_utimens) // if(!StatCache::getStatCacheData()->AddStat(strpath, updatemeta, false, true)){ return -EIO; } } } if(need_put_header){ // not found opened file. merge_headers(meta, updatemeta, true); // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); } } S3FS_MALLOCTRIM(0); return 0; } static int s3fs_chmod_nocopy(const char* _path, mode_t mode) { WTF8_ENCODE(path) int result; std::string strpath; std::string newpath; std::string nowcache; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; FUSE_CTX_INFO1("[path=%s][mode=%04o]", path, mode); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_owner(path, &stbuf))){ return result; } // Get attributes if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, nullptr, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, nullptr); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode)){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } if(IS_REPLACEDIR(nDirType)){ // Should rebuild all directory object // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); s3fs_realtime(ts_ctime); if(0 != (result = create_directory_object(newpath.c_str(), mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){ return result; } }else{ // normal object or directory object of newer version // open & load AutoFdEntity autoent; FdEntity* ent; if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){ S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); return result; } struct timespec ts; s3fs_realtime(ts); ent->SetCtime(ts); // Change file mode ent->SetMode(mode); // upload if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); return result; } StatCache::getStatCacheData()->DelStat(nowcache); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_chown(const char* _path, uid_t uid, gid_t gid) { WTF8_ENCODE(path) int result; std::string strpath; std::string newpath; std::string nowcache; headers_t meta; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; FUSE_CTX_INFO("[path=%s][uid=%u][gid=%u]", path, (unsigned int)uid, (unsigned int)gid); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_owner(path, &stbuf))){ return result; } if((uid_t)(-1) == uid){ uid = stbuf.st_uid; } if((gid_t)(-1) == gid){ gid = stbuf.st_gid; } if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, &meta); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode) && (IS_REPLACEDIR(nDirType) || IS_CREATE_MP_STAT(path))){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } if(IS_REPLACEDIR(nDirType)){ // Should rebuild directory object(except new type) // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); s3fs_realtime(ts_ctime); if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, uid, gid, pxattrvalue))){ return result; } }else{ std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime(); updatemeta["x-amz-meta-uid"] = std::to_string(uid); updatemeta["x-amz-meta-gid"] = std::to_string(gid); updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. // // If the file starts uploading by multipart when the disk capacity is insufficient, // we need to put these header after finishing upload. // Or if the file is only open, we must update to FdEntity's internal meta. // AutoFdEntity autoent; FdEntity* ent; bool need_put_header = true; if(nullptr != (ent = autoent.OpenExistFdEntity(path))){ if(ent->MergeOrgMeta(updatemeta)){ // meta is changed, but now uploading. // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); need_put_header = false; // If there is data in the Stats cache, update the Stats cache. StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta); // [NOTE] // There are cases where this function is called during the process of // creating a new file (before uploading). // In this case, a temporary cache exists in the Stat cache. // So we need to update the cache, if it exists. (see. s3fs_create and s3fs_utimens) // if(!StatCache::getStatCacheData()->AddStat(strpath, updatemeta, false, true)){ return -EIO; } } } if(need_put_header){ // not found opened file. merge_headers(meta, updatemeta, true); // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); } } S3FS_MALLOCTRIM(0); return 0; } static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid) { WTF8_ENCODE(path) int result; std::string strpath; std::string newpath; std::string nowcache; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; FUSE_CTX_INFO1("[path=%s][uid=%u][gid=%u]", path, (unsigned int)uid, (unsigned int)gid); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_owner(path, &stbuf))){ return result; } if((uid_t)(-1) == uid){ uid = stbuf.st_uid; } if((gid_t)(-1) == gid){ gid = stbuf.st_gid; } // Get attributes if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, nullptr, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, nullptr); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode)){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } if(IS_REPLACEDIR(nDirType)){ // Should rebuild all directory object // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); s3fs_realtime(ts_ctime); if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, uid, gid, pxattrvalue))){ return result; } }else{ // normal object or directory object of newer version // open & load AutoFdEntity autoent; FdEntity* ent; if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){ S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); return result; } struct timespec ts; s3fs_realtime(ts); ent->SetCtime(ts); // Change owner ent->SetUId(uid); ent->SetGId(gid); // upload if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); return result; } StatCache::getStatCacheData()->DelStat(nowcache); } S3FS_MALLOCTRIM(0); return result; } static timespec handle_utimens_special_values(timespec ts, timespec now, timespec orig) { if(ts.tv_nsec == UTIME_NOW){ return now; }else if(ts.tv_nsec == UTIME_OMIT){ return orig; }else{ return ts; } } static int update_mctime_parent_directory(const char* _path) { if(!update_parent_dir_stat){ // Disable updating parent directory stat. S3FS_PRN_DBG("Updating parent directory stats is disabled"); return 0; } WTF8_ENCODE(path) int result; std::string parentpath; // parent directory path std::string nowpath; // now directory object path("dir" or "dir/" or "xxx_$folder$", etc) std::string newpath; // directory path for the current version("dir/") std::string nowcache; headers_t meta; struct stat stbuf; struct timespec mctime; struct timespec atime; dirtype nDirType = dirtype::UNKNOWN; S3FS_PRN_INFO2("[path=%s]", path); // get parent directory path parentpath = mydirname(path); // check & get directory type if(0 != (result = chk_dir_object_type(parentpath.c_str(), newpath, nowpath, nowcache, &meta, &nDirType))){ return result; } // get directory stat // // [NOTE] // It is assumed that this function is called after the operation on // the file is completed, so there is no need to check the permissions // on the parent directory. // if(0 != (result = get_object_attribute(parentpath.c_str(), &stbuf))){ // If there is not the target file(object), result is -ENOENT. return result; } if(!S_ISDIR(stbuf.st_mode)){ S3FS_PRN_ERR("path(%s) is not parent directory.", parentpath.c_str()); return -EIO; } // make atime/mtime/ctime for updating s3fs_realtime(mctime); set_stat_to_timespec(stbuf, stat_time_type::ATIME, atime); if(0 == atime.tv_sec && 0 == atime.tv_nsec){ atime = mctime; } if(nocopyapi || IS_REPLACEDIR(nDirType) || IS_CREATE_MP_STAT(parentpath.c_str())){ // Should rebuild directory object(except new type) // Need to remove old dir("dir" etc) and make new dir("dir/") std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } // At first, remove directory old object if(!nowpath.empty()){ if(0 != (result = remove_old_type_dir(nowpath, nDirType))){ return result; } } if(!nowcache.empty()){ StatCache::getStatCacheData()->DelStat(nowcache); } // Make new directory object("dir/") if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, atime, mctime, mctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){ return result; } }else{ std::string strSourcePath = (mount_prefix.empty() && "/" == nowpath) ? "//" : nowpath; headers_t updatemeta; updatemeta["x-amz-meta-mtime"] = str(mctime); updatemeta["x-amz-meta-ctime"] = str(mctime); updatemeta["x-amz-meta-atime"] = str(atime); updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; merge_headers(meta, updatemeta, true); // upload meta for parent directory. if(0 != (result = put_headers(nowpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); } S3FS_MALLOCTRIM(0); return 0; } static int s3fs_utimens(const char* _path, const struct timespec ts[2]) { WTF8_ENCODE(path) int result; std::string strpath; std::string newpath; std::string nowcache; headers_t meta; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; FUSE_CTX_INFO("[path=%s][mtime=%s][ctime/atime=%s]", path, str(ts[1]).c_str(), str(ts[0]).c_str()); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_access(path, W_OK, &stbuf))){ if(0 != check_object_owner(path, &stbuf)){ return result; } } struct timespec now; struct timespec ts_atime; struct timespec ts_ctime; struct timespec ts_mtime; s3fs_realtime(now); set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); struct timespec atime = handle_utimens_special_values(ts[0], now, ts_atime); struct timespec ctime = handle_utimens_special_values(ts[0], now, ts_ctime); struct timespec mtime = handle_utimens_special_values(ts[1], now, ts_mtime); if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, &meta); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode) && (IS_REPLACEDIR(nDirType) || IS_CREATE_MP_STAT(path))){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } if(IS_REPLACEDIR(nDirType)){ // Should rebuild directory object(except new type) // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, atime, mtime, ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){ return result; } }else{ std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; updatemeta["x-amz-meta-mtime"] = str(mtime); updatemeta["x-amz-meta-ctime"] = str(ctime); updatemeta["x-amz-meta-atime"] = str(atime); updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. // // If the file starts uploading by multipart when the disk capacity is insufficient, // we need to put these header after finishing upload. // Or if the file is only open, we must update to FdEntity's internal meta. // AutoFdEntity autoent; FdEntity* ent; bool need_put_header = true; bool keep_mtime = false; if(nullptr != (ent = autoent.OpenExistFdEntity(path))){ if(ent->MergeOrgMeta(updatemeta)){ // meta is changed, but now uploading. // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); need_put_header = false; ent->SetHoldingMtime(mtime); // If there is data in the Stats cache, update the Stats cache. StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta); // [NOTE] // There are cases where this function is called during the process of // creating a new file (before uploading). // In this case, a temporary cache exists in the Stat cache.(see s3fs_create) // So we need to update the cache, if it exists. // // Previously, the process of creating a new file was to update the // file content after first uploading the file, but now the file is // not created until flushing. // So we need to create a temporary Stat cache for it. // if(!StatCache::getStatCacheData()->AddStat(strpath, updatemeta, false, true)){ return -EIO; } }else{ S3FS_PRN_INFO("meta is not pending, but need to keep current mtime."); // [NOTE] // Depending on the order in which write/flush and utimens are called, // the mtime updated here may be overwritten at the time of flush. // To avoid that, set a special flag. // keep_mtime = true; } } if(need_put_header){ // not found opened file. merge_headers(meta, updatemeta, true); // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); if(keep_mtime){ ent->SetHoldingMtime(mtime); } } } S3FS_MALLOCTRIM(0); return 0; } static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2]) { WTF8_ENCODE(path) int result; std::string strpath; std::string newpath; std::string nowcache; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; FUSE_CTX_INFO1("[path=%s][mtime=%s][atime/ctime=%s]", path, str(ts[1]).c_str(), str(ts[0]).c_str()); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_access(path, W_OK, &stbuf))){ if(0 != check_object_owner(path, &stbuf)){ return result; } } struct timespec now; struct timespec ts_atime; struct timespec ts_ctime; struct timespec ts_mtime; s3fs_realtime(now); set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); struct timespec atime = handle_utimens_special_values(ts[0], now, ts_atime); struct timespec ctime = handle_utimens_special_values(ts[0], now, ts_ctime); struct timespec mtime = handle_utimens_special_values(ts[1], now, ts_mtime); // Get attributes if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, nullptr, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, nullptr); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode)){ std::string xattrvalue; const char* pxattrvalue; if(get_meta_xattr_value(path, xattrvalue)){ pxattrvalue = xattrvalue.c_str(); }else{ pxattrvalue = nullptr; } if(IS_REPLACEDIR(nDirType)){ // Should rebuild all directory object // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, atime, mtime, ctime, stbuf.st_uid, stbuf.st_gid, pxattrvalue))){ return result; } }else{ // normal object or directory object of newer version // open & load AutoFdEntity autoent; FdEntity* ent; if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){ S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); return result; } // set mtime/ctime if(0 != (result = ent->SetMCtime(mtime, ctime))){ S3FS_PRN_ERR("could not set mtime and ctime to file(%s): result=%d", strpath.c_str(), result); return result; } // set atime if(0 != (result = ent->SetAtime(atime))){ S3FS_PRN_ERR("could not set atime to file(%s): result=%d", strpath.c_str(), result); return result; } // upload if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); return result; } StatCache::getStatCacheData()->DelStat(nowcache); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_truncate(const char* _path, off_t size) { WTF8_ENCODE(path) int result; headers_t meta; AutoFdEntity autoent; FdEntity* ent = nullptr; FUSE_CTX_INFO("[path=%s][size=%lld]", path, static_cast(size)); size = std::max(size, 0); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_access(path, W_OK, nullptr))){ return result; } // Get file information if(0 == (result = get_object_attribute(path, nullptr, &meta))){ // File exists // [NOTE] // If the file exists, the file has already been opened by FUSE before // truncate is called. Then the call below will change the file size. // (When an already open file is changed the file size, FUSE will not // reopen it.) // The Flush is called before this file is closed, so there is no need // to do it here. // // [NOTICE] // FdManager::Open() ignores changes that reduce the file size for the // file you are editing. However, if user opens only once, edit it, // and then shrink the file, it should be done. // When this function is called, the file is already open by FUSE or // some other operation. Therefore, if the number of open files is 1, // no edits other than that fd will be made, and the files will be // shrunk using ignore_modify flag even while editing. // See the comments when merging this code for FUSE2 limitations. // (In FUSE3, it will be possible to handle it reliably using fuse_file_info.) // bool ignore_modify; if(1 < FdManager::GetOpenFdCount(path)){ ignore_modify = false; }else{ ignore_modify = true; } if(nullptr == (ent = autoent.Open(path, &meta, size, S3FS_OMIT_TS, O_RDWR, false, true, ignore_modify))){ S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno); return -EIO; } ent->UpdateCtime(); #if defined(__APPLE__) // [NOTE] // Only for macos, this truncate calls to "size=0" do not reflect size. // The cause is unknown now, but it can be avoided by flushing the file. // if(0 == size){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); return result; } StatCache::getStatCacheData()->DelStat(path); } #endif }else{ // Not found -> Make tmpfile(with size) const struct fuse_context* pcxt; if(nullptr == (pcxt = fuse_get_context())){ return -EIO; } std::string strnow = s3fs_str_realtime(); meta["Content-Type"] = "application/octet-stream"; // Static meta["x-amz-meta-mode"] = std::to_string(S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO); meta["x-amz-meta-ctime"] = strnow; meta["x-amz-meta-mtime"] = strnow; meta["x-amz-meta-uid"] = std::to_string(pcxt->uid); meta["x-amz-meta-gid"] = std::to_string(pcxt->gid); if(nullptr == (ent = autoent.Open(path, &meta, size, S3FS_OMIT_TS, O_RDWR, true, true, false))){ S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno); return -EIO; } if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); return result; } StatCache::getStatCacheData()->DelStat(path); } S3FS_MALLOCTRIM(0); return result; } static int s3fs_open(const char* _path, struct fuse_file_info* fi) { WTF8_ENCODE(path) int result; struct stat st; bool needs_flush = false; FUSE_CTX_INFO("[path=%s][flags=0x%x]", path, fi->flags); if ((fi->flags & O_ACCMODE) == O_RDONLY && fi->flags & O_TRUNC) { return -EACCES; } // [NOTE] // Delete the Stats cache only if the file is not open. // If the file is open, the stats cache will not be deleted as // there are cases where the object does not exist on the server // and only the Stats cache exists. // if(StatCache::getStatCacheData()->HasStat(path)){ if(!FdManager::HasOpenEntityFd(path)){ StatCache::getStatCacheData()->DelStat(path); } } int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } result = check_object_access(path, mask, &st); if(-ENOENT == result){ if(0 != (result = check_parent_object_access(path, W_OK))){ return result; } }else if(0 != result){ return result; } AutoFdEntity autoent; FdEntity* ent; headers_t meta; if((unsigned int)fi->flags & O_TRUNC){ if(0 != st.st_size){ st.st_size = 0; needs_flush = true; } }else{ // [NOTE] // If the file has already been opened and edited, the file size in // the edited state is given priority. // This prevents the file size from being reset to its original size // if you keep the file open, shrink its size, and then read the file // from another process while it has not yet been flushed. // if(nullptr != (ent = autoent.OpenExistFdEntity(path)) && ent->IsModified()){ // sets the file size being edited. ent->GetSize(st.st_size); } } if(!S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)){ st.st_mtime = -1; } if(0 != (result = get_object_attribute(path, nullptr, &meta, true, nullptr, true))){ // no truncate cache return result; } struct timespec st_mctime; set_stat_to_timespec(st, stat_time_type::MTIME, st_mctime); if(nullptr == (ent = autoent.Open(path, &meta, st.st_size, st_mctime, fi->flags, false, true, false))){ StatCache::getStatCacheData()->DelStat(path); return -EIO; } if (needs_flush){ struct timespec ts; s3fs_realtime(ts); ent->SetMCtime(ts, ts); if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), path, true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); StatCache::getStatCacheData()->DelStat(path); return result; } } fi->fh = autoent.Detach(); // KEEP fdentity open; S3FS_MALLOCTRIM(0); return 0; } static int s3fs_read(const char* _path, char* buf, size_t size, off_t offset, struct fuse_file_info* fi) { WTF8_ENCODE(path) ssize_t res; FUSE_CTX_DBG("[path=%s][size=%zu][offset=%lld][pseudo_fd=%llu]", path, size, static_cast(offset), (unsigned long long)(fi->fh)); AutoFdEntity autoent; FdEntity* ent; if(nullptr == (ent = autoent.GetExistFdEntity(path, static_cast(fi->fh)))){ S3FS_PRN_ERR("could not find opened pseudo_fd(=%llu) for path(%s)", (unsigned long long)(fi->fh), path); return -EIO; } // check real file size off_t realsize = 0; if(!ent->GetSize(realsize) || 0 == realsize){ S3FS_PRN_DBG("file size is 0, so break to read."); return 0; } if(0 > (res = ent->Read(static_cast(fi->fh), buf, offset, size, false))){ S3FS_PRN_WARN("failed to read file(%s). result=%zd", path, res); } return static_cast(res); } static int s3fs_write(const char* _path, const char* buf, size_t size, off_t offset, struct fuse_file_info* fi) { WTF8_ENCODE(path) ssize_t res; FUSE_CTX_DBG("[path=%s][size=%zu][offset=%lld][pseudo_fd=%llu]", path, size, static_cast(offset), (unsigned long long)(fi->fh)); AutoFdEntity autoent; FdEntity* ent; if(nullptr == (ent = autoent.GetExistFdEntity(path, static_cast(fi->fh)))){ S3FS_PRN_ERR("could not find opened pseudo_fd(%llu) for path(%s)", (unsigned long long)(fi->fh), path); return -EIO; } if(0 > (res = ent->Write(static_cast(fi->fh), buf, offset, size))){ S3FS_PRN_WARN("failed to write file(%s). result=%zd", path, res); } if(max_dirty_data != -1 && ent->BytesModified() >= max_dirty_data){ int flushres; if(0 != (flushres = ent->RowFlush(static_cast(fi->fh), path, true))){ S3FS_PRN_ERR("could not upload file(%s): result=%d", path, flushres); StatCache::getStatCacheData()->DelStat(path); return flushres; } // Punch a hole in the file to recover disk space. if(!ent->PunchHole()){ S3FS_PRN_WARN("could not punching HOLEs to a cache file, but continue."); } } return static_cast(res); } static int s3fs_statfs(const char* _path, struct statvfs* stbuf) { // WTF8_ENCODE(path) stbuf->f_bsize = s3fs_block_size; stbuf->f_namemax = NAME_MAX; #if defined(__MSYS__) // WinFsp resolves the free space from f_bfree * f_frsize, and the total space from f_blocks * f_frsize (in bytes). stbuf->f_blocks = bucket_block_count; stbuf->f_frsize = stbuf->f_bsize; stbuf->f_bfree = stbuf->f_blocks; #elif defined(__APPLE__) stbuf->f_blocks = bucket_block_count; stbuf->f_frsize = stbuf->f_bsize; stbuf->f_bfree = stbuf->f_blocks; stbuf->f_files = UINT32_MAX; stbuf->f_ffree = UINT32_MAX; stbuf->f_favail = UINT32_MAX; #else stbuf->f_frsize = stbuf->f_bsize; stbuf->f_blocks = bucket_block_count; stbuf->f_bfree = stbuf->f_blocks; #endif stbuf->f_bavail = stbuf->f_blocks; return 0; } static int s3fs_flush(const char* _path, struct fuse_file_info* fi) { WTF8_ENCODE(path) int result; FUSE_CTX_INFO("[path=%s][pseudo_fd=%llu]", path, (unsigned long long)(fi->fh)); int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK); if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } result = check_object_access(path, mask, nullptr); if(-ENOENT == result){ if(0 != (result = check_parent_object_access(path, W_OK))){ return result; } }else if(0 != result){ return result; } AutoFdEntity autoent; FdEntity* ent; if(nullptr != (ent = autoent.GetExistFdEntity(path, static_cast(fi->fh)))){ bool is_new_file = ent->IsDirtyNewFile(); ent->UpdateMtime(true); // clear the flag not to update mtime. ent->UpdateCtime(); result = ent->Flush(static_cast(fi->fh), false); StatCache::getStatCacheData()->DelStat(path); if(is_new_file){ // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to create the file(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } } } S3FS_MALLOCTRIM(0); return result; } // [NOTICE] // Assumption is a valid fd. // static int s3fs_fsync(const char* _path, int datasync, struct fuse_file_info* fi) { WTF8_ENCODE(path) int result = 0; FUSE_CTX_INFO("[path=%s][datasync=%d][pseudo_fd=%llu]", path, datasync, (unsigned long long)(fi->fh)); AutoFdEntity autoent; FdEntity* ent; if(nullptr != (ent = autoent.GetExistFdEntity(path, static_cast(fi->fh)))){ bool is_new_file = ent->IsDirtyNewFile(); if(0 == datasync){ ent->UpdateMtime(); ent->UpdateCtime(); } result = ent->Flush(static_cast(fi->fh), false); if(0 != datasync){ // [NOTE] // The metadata are not updated when fdatasync is called. // Instead of it, these metadata are pended and set the dirty flag here. // Setting this flag allows metadata to be updated even if there is no // content update between the fdatasync call and the flush call. // ent->MarkDirtyMetadata(); } if(is_new_file){ // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to create the file(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } } } S3FS_MALLOCTRIM(0); // Issue 320: Delete stat cache entry because st_size may have changed. StatCache::getStatCacheData()->DelStat(path); return result; } static int s3fs_release(const char* _path, struct fuse_file_info* fi) { WTF8_ENCODE(path) FUSE_CTX_INFO("[path=%s][pseudo_fd=%llu]", path, (unsigned long long)(fi->fh)); { // scope for AutoFdEntity AutoFdEntity autoent; // [NOTE] // The pseudo fd stored in fi->fh is attached to AutoFdEntry so that it can be // destroyed here. // FdEntity* ent; if(nullptr == (ent = autoent.Attach(path, static_cast(fi->fh)))){ S3FS_PRN_ERR("could not find pseudo_fd(%llu) for path(%s)", (unsigned long long)(fi->fh), path); return -EIO; } // [NOTE] // There are cases when s3fs_flush is not called and s3fs_release is called. // (There have been reported cases where it is not called when exported as NFS.) // Therefore, Flush() is called here to try to upload the data. // Flush() will only perform an upload if the file has been updated. // int result; if(ent->IsModified()){ if(0 != (result = ent->Flush(static_cast(fi->fh), false))){ S3FS_PRN_ERR("failed to upload file contentsfor pseudo_fd(%llu) / path(%s) by result(%d)", (unsigned long long)(fi->fh), path, result); return result; } } // [NOTE] // All opened file's stats is cached with no truncate flag. // Thus we unset it here. StatCache::getStatCacheData()->ChangeNoTruncateFlag(path, false); // [NOTICE] // At first, we remove stats cache. // Because fuse does not wait for response from "release" function. :-( // And fuse runs next command before this function returns. // Thus we call deleting stats function ASAP. // if((fi->flags & O_RDWR) || (fi->flags & O_WRONLY)){ StatCache::getStatCacheData()->DelStat(path); } bool is_new_file = ent->IsDirtyNewFile(); if(0 != (result = ent->UploadPending(static_cast(fi->fh)))){ S3FS_PRN_ERR("could not upload pending data(meta, etc) for pseudo_fd(%llu) / path(%s)", (unsigned long long)(fi->fh), path); return result; } if(is_new_file){ // update parent directory timestamp int update_result; if(0 != (update_result = update_mctime_parent_directory(path))){ S3FS_PRN_ERR("succeed to create the file(%s), but could not update timestamp of its parent directory(result=%d).", path, update_result); } } } // check - for debug if(S3fsLog::IsS3fsLogDbg()){ if(FdManager::HasOpenEntityFd(path)){ S3FS_PRN_DBG("file(%s) is still opened(another pseudo fd is opened).", path); } } S3FS_MALLOCTRIM(0); return 0; } static int s3fs_opendir(const char* _path, struct fuse_file_info* fi) { WTF8_ENCODE(path) int result; int mask = (O_RDONLY != (fi->flags & O_ACCMODE) ? W_OK : R_OK); FUSE_CTX_INFO("[path=%s][flags=0x%x]", path, fi->flags); if(0 == (result = check_object_access(path, mask, nullptr))){ result = check_parent_object_access(path, X_OK); } S3FS_MALLOCTRIM(0); return result; } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress constParameterCallback static bool multi_head_callback(S3fsCurl* s3fscurl, void* param) { if(!s3fscurl){ return false; } // Add stat cache const std::string& saved_path = s3fscurl->GetSpecialSavedPath(); if(!StatCache::getStatCacheData()->AddStat(saved_path, *(s3fscurl->GetResponseHeaders()))){ S3FS_PRN_ERR("failed adding stat cache [path=%s]", saved_path.c_str()); return false; } // Get stats from stats cache(for converting from meta), and fill std::string bpath = mybasename(saved_path); if(use_wtf8){ bpath = s3fs_wtf8_decode(bpath); } if(param){ auto* pcbparam = reinterpret_cast(param); struct stat st; if(StatCache::getStatCacheData()->GetStat(saved_path, &st)){ pcbparam->Fill(bpath.c_str(), &st, 0); }else{ S3FS_PRN_INFO2("Could not find %s file in stat cache.", saved_path.c_str()); pcbparam->Fill(bpath.c_str(), nullptr, 0); } }else{ S3FS_PRN_WARN("param(multi_head_callback_param*) is nullptr, then can not call filler."); } return true; } struct multi_head_notfound_callback_param { std::mutex list_lock; s3obj_list_t notfound_list; }; static bool multi_head_notfound_callback(S3fsCurl* s3fscurl, void* param) { if(!s3fscurl){ return false; } S3FS_PRN_INFO("HEAD returned NotFound(404) for %s object, it maybe only the path exists and the object does not exist.", s3fscurl->GetPath().c_str()); if(!param){ S3FS_PRN_WARN("param(multi_head_notfound_callback_param*) is nullptr, then can not call filler."); return false; } // set path to not found list auto* pcbparam = reinterpret_cast(param); const std::lock_guard lock(pcbparam->list_lock); pcbparam->notfound_list.push_back(s3fscurl->GetBasePath()); return true; } static std::unique_ptr multi_head_retry_callback(S3fsCurl* s3fscurl) { if(!s3fscurl){ return nullptr; } size_t ssec_key_pos= s3fscurl->GetLastPreHeadSeecKeyPos(); int retry_count = s3fscurl->GetMultipartRetryCount(); // retry next sse key. // if end of sse key, set retry master count is up. ssec_key_pos = (ssec_key_pos == static_cast(-1) ? 0 : ssec_key_pos + 1); if(0 == S3fsCurl::GetSseKeyCount() || S3fsCurl::GetSseKeyCount() <= ssec_key_pos){ if(s3fscurl->IsOverMultipartRetryCount()){ S3FS_PRN_ERR("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpecialSavedPath().c_str()); return nullptr; } ssec_key_pos = -1; retry_count++; } std::unique_ptr newcurl(new S3fsCurl(s3fscurl->IsUseAhbe())); const std::string& path = s3fscurl->GetBasePath(); const std::string& base_path = s3fscurl->GetBasePath(); const std::string& saved_path = s3fscurl->GetSpecialSavedPath(); if(!newcurl->PreHeadRequest(path, base_path, saved_path, ssec_key_pos)){ S3FS_PRN_ERR("Could not duplicate curl object(%s).", saved_path.c_str()); return nullptr; } newcurl->SetMultipartRetryCount(retry_count); return newcurl; } static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler) { S3fsMultiCurl curlmulti(S3fsCurl::GetMaxMultiRequest(), true); // [NOTE] run all requests to completion even if some requests fail. s3obj_list_t headlist; int result = 0; S3FS_PRN_INFO1("[path=%s][list=%zu]", path, headlist.size()); // Make base path list. head.GetNameList(headlist, true, false); // get name with "/". StatCache::getStatCacheData()->GetNotruncateCache(std::string(path), headlist); // Add notruncate file name from stat cache // Initialize S3fsMultiCurl curlmulti.SetSuccessCallback(multi_head_callback); curlmulti.SetRetryCallback(multi_head_retry_callback); // Success Callback function parameter(SyncFiller object) SyncFiller syncfiller(buf, filler); curlmulti.SetSuccessCallbackParam(reinterpret_cast(&syncfiller)); // Not found Callback function parameter struct multi_head_notfound_callback_param notfound_param; if(support_compat_dir){ curlmulti.SetNotFoundCallback(multi_head_notfound_callback); curlmulti.SetNotFoundCallbackParam(reinterpret_cast(¬found_param)); } // Make single head request(with max). for(auto iter = headlist.cbegin(); headlist.cend() != iter; ++iter){ std::string disppath = path + (*iter); std::string etag = head.GetETag((*iter).c_str()); struct stat st; // [NOTE] // If there is a cache hit, file stat is filled by filler at here. // if(StatCache::getStatCacheData()->HasStat(disppath, &st, etag.c_str())){ std::string bpath = mybasename(disppath); if(use_wtf8){ bpath = s3fs_wtf8_decode(bpath); } syncfiller.Fill(bpath.c_str(), &st, 0); continue; } // First check for directory, start checking "not SSE-C". // If checking failed, retry to check with "SSE-C" by retry callback func when SSE-C mode. std::unique_ptr s3fscurl(new S3fsCurl()); if(!s3fscurl->PreHeadRequest(disppath, disppath, disppath)){ // target path = cache key path.(ex "dir/") S3FS_PRN_WARN("Could not make curl object for head request(%s).", disppath.c_str()); continue; } if(!curlmulti.SetS3fsCurlObject(std::move(s3fscurl))){ S3FS_PRN_WARN("Could not make curl object into multi curl(%s).", disppath.c_str()); continue; } } headlist.clear(); // Multi request if(0 != (result = curlmulti.Request())){ // If result is -EIO, it is something error occurred. // This case includes that the object is encrypting(SSE) and s3fs does not have keys. // So s3fs set result to 0 in order to continue the process. if(-EIO == result){ S3FS_PRN_WARN("error occurred in multi request(errno=%d), but continue...", result); result = 0; }else{ S3FS_PRN_ERR("error occurred in multi request(errno=%d).", result); return result; } } // [NOTE] // Objects that could not be found by HEAD request may exist only // as a path, so search for objects under that path.(a case of no dir object) // if(!support_compat_dir){ syncfiller.SufficiencyFill(head.GetCommonPrefixes()); } if(support_compat_dir && !notfound_param.notfound_list.empty()){ // [NOTE] not need to lock to access this here. // dummy header mode_t dirmask = umask(0); // macos does not have getumask() umask(dirmask); headers_t dummy_header; dummy_header["Content-Type"] = "application/x-directory"; // directory dummy_header["x-amz-meta-uid"] = std::to_string(is_s3fs_uid ? s3fs_uid : geteuid()); dummy_header["x-amz-meta-gid"] = std::to_string(is_s3fs_gid ? s3fs_gid : getegid()); dummy_header["x-amz-meta-mode"] = std::to_string(S_IFDIR | (~dirmask & (S_IRWXU | S_IRWXG | S_IRWXO))); dummy_header["x-amz-meta-atime"] = "0"; dummy_header["x-amz-meta-ctime"] = "0"; dummy_header["x-amz-meta-mtime"] = "0"; for(auto reiter = notfound_param.notfound_list.cbegin(); reiter != notfound_param.notfound_list.cend(); ++reiter){ int dir_result; const std::string& dirpath = *reiter; if(-ENOTEMPTY == (dir_result = directory_empty(dirpath.c_str()))){ // Found objects under the path, so the path is directory. // Add stat cache if(StatCache::getStatCacheData()->AddStat(dirpath, dummy_header, true)){ // set forcedir=true // Get stats from stats cache(for converting from meta), and fill std::string base_path = mybasename(dirpath); if(use_wtf8){ base_path = s3fs_wtf8_decode(base_path); } struct stat st; if(StatCache::getStatCacheData()->GetStat(dirpath, &st)){ syncfiller.Fill(base_path.c_str(), &st, 0); }else{ S3FS_PRN_INFO2("Could not find %s directory(no dir object) in stat cache.", dirpath.c_str()); syncfiller.Fill(base_path.c_str(), nullptr, 0); } }else{ S3FS_PRN_ERR("failed adding stat cache [path=%s], but dontinue...", dirpath.c_str()); } }else{ S3FS_PRN_WARN("%s object does not have any object under it(errno=%d),", reiter->c_str(), dir_result); } } } return result; } static int s3fs_readdir(const char* _path, void* buf, fuse_fill_dir_t filler, off_t offset, struct fuse_file_info* fi) { WTF8_ENCODE(path) S3ObjList head; int result; FUSE_CTX_INFO("[path=%s]", path); if(0 != (result = check_object_access(path, R_OK, nullptr))){ return result; } // get a list of all the objects if((result = list_bucket(path, head, "/")) != 0){ S3FS_PRN_ERR("list_bucket returns error(%d).", result); return result; } // force to add "." and ".." name. filler(buf, ".", nullptr, 0); filler(buf, "..", nullptr, 0); if(head.IsEmpty()){ return 0; } // Send multi head request for stats caching. std::string strpath = path; if(strcmp(path, "/") != 0){ strpath += "/"; } if(0 != (result = readdir_multi_head(strpath.c_str(), head, buf, filler))){ S3FS_PRN_ERR("readdir_multi_head returns error(%d).", result); } S3FS_MALLOCTRIM(0); return result; } static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, bool check_content_only) { std::string s3_realpath; std::string query_delimiter; std::string query_prefix; std::string query_maxkey; std::string next_continuation_token; std::string next_marker; bool truncated = true; S3fsCurl s3fscurl; S3FS_PRN_INFO1("[path=%s]", path); if(delimiter && 0 < strlen(delimiter)){ query_delimiter += "delimiter="; query_delimiter += delimiter; query_delimiter += "&"; } query_prefix += "&prefix="; s3_realpath = get_realpath(path); if(s3_realpath.empty() || '/' != *s3_realpath.rbegin()){ // last word must be "/" query_prefix += urlEncodePath(s3_realpath.substr(1) + "/"); }else{ query_prefix += urlEncodePath(s3_realpath.substr(1)); } if (check_content_only){ // Just need to know if there are child objects in dir // For dir with children, expect "dir/" and "dir/child" query_maxkey += "max-keys=2"; }else{ query_maxkey += "max-keys=" + std::to_string(max_keys_list_object); } while(truncated){ // append parameters to query in alphabetical order std::string each_query; if(!next_continuation_token.empty()){ each_query += "continuation-token=" + urlEncodePath(next_continuation_token) + "&"; next_continuation_token = ""; } each_query += query_delimiter; if(S3fsCurl::IsListObjectsV2()){ each_query += "list-type=2&"; } if(!next_marker.empty()){ each_query += "marker=" + urlEncodePath(next_marker) + "&"; next_marker = ""; } each_query += query_maxkey; each_query += query_prefix; // request int result; if(0 != (result = s3fscurl.ListBucketRequest(path, each_query.c_str()))){ S3FS_PRN_ERR("ListBucketRequest returns with error."); return result; } const std::string& body = s3fscurl.GetBodyData(); // [NOTE] // CR code(\r) is replaced with LF(\n) by xmlReadMemory() function. // To prevent that, only CR code is encoded by following function. // The encoded CR code is decoded with append_objects_from_xml(_ex). // std::string encbody = get_encoded_cr_code(body.c_str()); // xmlDocPtr std::unique_ptr doc(xmlReadMemory(encbody.c_str(), static_cast(encbody.size()), "", nullptr, 0), xmlFreeDoc); if(nullptr == doc){ S3FS_PRN_ERR("xmlReadMemory returns with error."); return -EIO; } if(0 != append_objects_from_xml(path, doc.get(), head)){ S3FS_PRN_ERR("append_objects_from_xml returns with error."); return -EIO; } if(true == (truncated = is_truncated(doc.get()))){ auto tmpch = get_next_continuation_token(doc.get()); if(nullptr != tmpch){ next_continuation_token = reinterpret_cast(tmpch.get()); }else if(nullptr != (tmpch = get_next_marker(doc.get()))){ next_marker = reinterpret_cast(tmpch.get()); } if(next_continuation_token.empty() && next_marker.empty()){ // If did not specify "delimiter", s3 did not return "NextMarker". // On this case, can use last name for next marker. // std::string lastname; if(!head.GetLastName(lastname)){ S3FS_PRN_WARN("Could not find next marker, thus break loop."); truncated = false; }else{ next_marker = s3_realpath.substr(1); if(s3_realpath.empty() || '/' != *s3_realpath.rbegin()){ next_marker += "/"; } next_marker += lastname; } } } // reset(initialize) curl object s3fscurl.DestroyCurlHandle(); if(check_content_only){ break; } } S3FS_MALLOCTRIM(0); return 0; } static int remote_mountpath_exists(const char* path, bool compat_dir) { struct stat stbuf; int result; S3FS_PRN_INFO1("[path=%s]", path); // getattr will prefix the path with the remote mountpoint if(0 != (result = get_object_attribute(path, &stbuf, nullptr))){ return result; } // [NOTE] // If there is no mount point(directory object) that s3fs can recognize, // an error will occur. // A mount point with a directory path(ex. "/...") // requires that directory object. // If the directory or object is created by a client other than s3fs, // s3fs may not be able to recognize it. If you specify such a directory // as a mount point, you can avoid the error by starting with "compat_dir" // specified. // if(!compat_dir && !has_mp_stat){ return -ENOENT; } return 0; } static bool get_meta_xattr_value(const char* path, std::string& rawvalue) { if(!path || '\0' == path[0]){ S3FS_PRN_ERR("path is empty."); return false; } S3FS_PRN_DBG("[path=%s]", path); rawvalue.clear(); headers_t meta; if(0 != get_object_attribute(path, nullptr, &meta)){ S3FS_PRN_ERR("Failed to get object(%s) headers", path); return false; } headers_t::const_iterator iter; if(meta.cend() == (iter = meta.find("x-amz-meta-xattr"))){ return false; } rawvalue = iter->second; return true; } static bool get_parent_meta_xattr_value(const char* path, std::string& rawvalue) { if(0 == strcmp(path, "/") || 0 == strcmp(path, ".")){ // path is mount point, thus does not have parent. return false; } std::string parent = mydirname(path); if(parent.empty()){ S3FS_PRN_ERR("Could not get parent path for %s.", path); return false; } return get_meta_xattr_value(parent.c_str(), rawvalue); } static bool get_xattr_posix_key_value(const char* path, std::string& xattrvalue, bool default_key) { xattrvalue.clear(); std::string rawvalue; if(!get_meta_xattr_value(path, rawvalue)){ return false; } xattrs_t xattrs; if(0 == parse_xattrs(rawvalue, xattrs)){ return false; } std::string targetkey; if(default_key){ targetkey = "system.posix_acl_default"; }else{ targetkey = "system.posix_acl_access"; } xattrs_t::iterator iter; if(xattrs.cend() == (iter = xattrs.find(targetkey))){ return false; } // convert value by base64 xattrvalue = s3fs_base64(reinterpret_cast(iter->second.c_str()), iter->second.length()); return true; } // [NOTE] // Converts and returns the POSIX ACL default(system.posix_acl_default) value of // the parent directory as a POSIX ACL(system.posix_acl_access) value. // Returns false if the parent directory has no POSIX ACL defaults. // static bool build_inherited_xattr_value(const char* path, std::string& xattrvalue) { S3FS_PRN_DBG("[path=%s]", path); xattrvalue.clear(); if(0 == strcmp(path, "/") || 0 == strcmp(path, ".")){ // path is mount point, thus does not have parent. return false; } std::string parent = mydirname(path); if(parent.empty()){ S3FS_PRN_ERR("Could not get parent path for %s.", path); return false; } // get parent's "system.posix_acl_default" value(base64'd). std::string parent_default_value; if(!get_xattr_posix_key_value(parent.c_str(), parent_default_value, true)){ return false; } // build "system.posix_acl_access" from parent's default value std::string raw_xattr_value; raw_xattr_value = "{\"system.posix_acl_access\":\""; raw_xattr_value += parent_default_value; raw_xattr_value += "\"}"; xattrvalue = urlEncodePath(raw_xattr_value); return true; } static bool parse_xattr_keyval(const std::string& xattrpair, std::string& key, std::string* pval) { // parse key and value size_t pos; std::string tmpval; if(std::string::npos == (pos = xattrpair.find_first_of(':'))){ S3FS_PRN_ERR("one of xattr pair(%s) is wrong format.", xattrpair.c_str()); return false; } key = xattrpair.substr(0, pos); tmpval = xattrpair.substr(pos + 1); if(!takeout_str_dquart(key) || !takeout_str_dquart(tmpval)){ S3FS_PRN_ERR("one of xattr pair(%s) is wrong format.", xattrpair.c_str()); return false; } *pval = s3fs_decode64(tmpval.c_str(), tmpval.size()); return true; } static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs) { xattrs.clear(); // decode std::string jsonxattrs = urlDecode(strxattrs); // get from "{" to "}" std::string restxattrs; { size_t startpos; size_t endpos = std::string::npos; if(std::string::npos != (startpos = jsonxattrs.find_first_of('{'))){ endpos = jsonxattrs.find_last_of('}'); } if(startpos == std::string::npos || endpos == std::string::npos || endpos <= startpos){ S3FS_PRN_WARN("xattr header(%s) is not json format.", jsonxattrs.c_str()); return 0; } restxattrs = jsonxattrs.substr(startpos + 1, endpos - (startpos + 1)); } // parse each key:val for(size_t pair_nextpos = restxattrs.find_first_of(','); !restxattrs.empty(); restxattrs = (pair_nextpos != std::string::npos ? restxattrs.substr(pair_nextpos + 1) : ""), pair_nextpos = restxattrs.find_first_of(',')){ std::string pair = pair_nextpos != std::string::npos ? restxattrs.substr(0, pair_nextpos) : restxattrs; std::string key; std::string val; if(!parse_xattr_keyval(pair, key, &val)){ // something format error, so skip this. continue; } xattrs[key] = val; } return xattrs.size(); } static std::string raw_build_xattrs(const xattrs_t& xattrs) { std::string strxattrs; bool is_set = false; for(auto iter = xattrs.cbegin(); iter != xattrs.cend(); ++iter){ if(is_set){ strxattrs += ','; }else{ is_set = true; strxattrs = "{"; } strxattrs += '\"'; strxattrs += iter->first; strxattrs += "\":\""; strxattrs += s3fs_base64(reinterpret_cast(iter->second.c_str()), iter->second.length()); strxattrs += '\"'; } if(is_set){ strxattrs += "}"; } return strxattrs; } static std::string build_xattrs(const xattrs_t& xattrs) { std::string strxattrs = raw_build_xattrs(xattrs); if(strxattrs.empty()){ strxattrs = "{}"; } strxattrs = urlEncodePath(strxattrs); return strxattrs; } static int set_xattrs_to_header(headers_t& meta, const char* name, const char* value, size_t size, int flags) { std::string strxattrs; xattrs_t xattrs; headers_t::iterator iter; if(meta.cend() == (iter = meta.find("x-amz-meta-xattr"))){ #if defined(XATTR_REPLACE) if(XATTR_REPLACE == (flags & XATTR_REPLACE)){ // there is no xattr header but flags is replace, so failure. return -ENOATTR; } #endif }else{ #if defined(XATTR_CREATE) if(XATTR_CREATE == (flags & XATTR_CREATE)){ // found xattr header but flags is only creating, so failure. return -EEXIST; } #endif strxattrs = iter->second; } // get map as xattrs_t parse_xattrs(strxattrs, xattrs); // add name(do not care overwrite and empty name/value) xattrs[name] = std::string(value, size); // build new strxattrs(not encoded) and set it to headers_t meta["x-amz-meta-xattr"] = build_xattrs(xattrs); S3FS_PRN_DBG("Set xattrs(after adding %s key) = %s", name, raw_build_xattrs(xattrs).c_str()); return 0; } #if defined(__APPLE__) static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags, uint32_t position) #else static int s3fs_setxattr(const char* path, const char* name, const char* value, size_t size, int flags) #endif { FUSE_CTX_INFO("[path=%s][name=%s][value=%p][size=%zu][flags=0x%x]", path, name, value, size, flags); if(!value && 0 < size){ S3FS_PRN_ERR("Wrong parameter: value(%p), size(%zu)", value, size); return 0; } #if defined(__APPLE__) if (position != 0) { // No resource fork support return -EINVAL; } #endif int result; std::string strpath; std::string newpath; std::string nowcache; headers_t meta; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_owner(path, &stbuf))){ return result; } if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, &meta); } if(0 != result){ return result; } if(S_ISDIR(stbuf.st_mode) && (IS_REPLACEDIR(nDirType) || IS_CREATE_MP_STAT(path))){ if(IS_REPLACEDIR(nDirType)){ // Should rebuild directory object(except new type) // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime); if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, nullptr))){ return result; } // need to set xattr header for directory. strpath = newpath; nowcache = strpath; } // set xattr all object std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime(); updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; // check opened file handle. // // If the file starts uploading by multipart when the disk capacity is insufficient, // we need to put these header after finishing upload. // Or if the file is only open, we must update to FdEntity's internal meta. // AutoFdEntity autoent; FdEntity* ent; bool need_put_header = true; if(nullptr != (ent = autoent.OpenExistFdEntity(path))){ // get xattr and make new xattr std::string strxattr; if(ent->GetXattr(strxattr)){ updatemeta["x-amz-meta-xattr"] = strxattr; }else{ // [NOTE] // Set an empty xattr. // This requires the key to be present in order to add xattr. ent->SetXattr(strxattr); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(0 != (result = set_xattrs_to_header(updatemeta, name, value, size, flags))){ return result; } if(ent->MergeOrgMeta(updatemeta)){ // meta is changed, but now uploading. // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); need_put_header = false; // If there is data in the Stats cache, update the Stats cache. StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta); // [NOTE] // There are cases where this function is called during the process of // creating a new file (before uploading). // In this case, a temporary cache exists in the Stat cache. // So we need to update the cache, if it exists. (see. s3fs_create and s3fs_utimens) // if(!StatCache::getStatCacheData()->AddStat(strpath, updatemeta, false, true)){ return -EIO; } } } if(need_put_header){ // not found opened file. // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(0 != (result = set_xattrs_to_header(meta, name, value, size, flags))){ return result; } merge_headers(meta, updatemeta, true); // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); } return 0; } #if defined(__APPLE__) static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size, uint32_t position) #else static int s3fs_getxattr(const char* path, const char* name, char* value, size_t size) #endif { #if defined(__APPLE__) FUSE_CTX_DBG("[path=%s][name=%s][value=%p][size=%zu]", path, name, value, size); #else FUSE_CTX_INFO("[path=%s][name=%s][value=%p][size=%zu]", path, name, value, size); #endif if(!path || !name){ return -EIO; } #if defined(__APPLE__) if (position != 0) { // No resource fork support return -EINVAL; } #endif int result; headers_t meta; xattrs_t xattrs; // check parent directory attribute. if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } // get headers if(0 != (result = get_object_attribute(path, nullptr, &meta))){ return result; } // get xattrs auto hiter = meta.find("x-amz-meta-xattr"); if(meta.cend() == hiter){ // object does not have xattrs return -ENOATTR; } std::string strxattrs = hiter->second; parse_xattrs(strxattrs, xattrs); S3FS_PRN_DBG("Get xattrs = %s", raw_build_xattrs(xattrs).c_str()); // search name std::string strname = name; auto xiter = xattrs.find(strname); if(xattrs.cend() == xiter){ // not found name in xattrs return -ENOATTR; } // decode size_t length = xiter->second.length(); const char* pvalue = xiter->second.c_str(); if(0 < size){ if(size < length){ // over buffer size return -ERANGE; } if(pvalue){ memcpy(value, pvalue, length); } } return static_cast(length); } static int s3fs_listxattr(const char* path, char* list, size_t size) { S3FS_PRN_INFO("[path=%s][list=%p][size=%zu]", path, list, size); if(!path){ return -EIO; } int result; headers_t meta; xattrs_t xattrs; // check parent directory attribute. if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } // get headers if(0 != (result = get_object_attribute(path, nullptr, &meta))){ return result; } // get xattrs headers_t::iterator iter; if(meta.cend() == (iter = meta.find("x-amz-meta-xattr"))){ // object does not have xattrs return 0; } std::string strxattrs = iter->second; parse_xattrs(strxattrs, xattrs); S3FS_PRN_DBG("Get xattrs = %s", raw_build_xattrs(xattrs).c_str()); // calculate total name length size_t total = 0; for(auto xiter = xattrs.cbegin(); xiter != xattrs.cend(); ++xiter){ if(!xiter->first.empty()){ total += xiter->first.length() + 1; } } if(0 == total){ return 0; } // check parameters if(0 == size){ return static_cast(total); } if(!list || size < total){ return -ERANGE; } // copy to list char* setpos = list; for(auto xiter = xattrs.cbegin(); xiter != xattrs.cend(); ++xiter){ if(!xiter->first.empty()){ strcpy(setpos, xiter->first.c_str()); setpos = &setpos[strlen(setpos) + 1]; } } return static_cast(total); } static int s3fs_removexattr(const char* path, const char* name) { FUSE_CTX_INFO("[path=%s][name=%s]", path, name); if(!path || !name){ return -EIO; } int result; std::string strpath; std::string newpath; std::string nowcache; headers_t meta; xattrs_t xattrs; struct stat stbuf; dirtype nDirType = dirtype::UNKNOWN; if(0 == strcmp(path, "/")){ S3FS_PRN_ERR("Could not change mode for mount point."); return -EIO; } if(0 != (result = check_parent_object_access(path, X_OK))){ return result; } if(0 != (result = check_object_owner(path, &stbuf))){ return result; } if(S_ISDIR(stbuf.st_mode)){ result = chk_dir_object_type(path, newpath, strpath, nowcache, &meta, &nDirType); }else{ strpath = path; nowcache = strpath; result = get_object_attribute(strpath.c_str(), nullptr, &meta); } if(0 != result){ return result; } // get xattrs auto hiter = meta.find("x-amz-meta-xattr"); if(meta.cend() == hiter){ // object does not have xattrs return -ENOATTR; } std::string strxattrs = hiter->second; parse_xattrs(strxattrs, xattrs); // check name xattrs std::string strname = name; auto xiter = xattrs.find(strname); if(xattrs.cend() == xiter){ return -ENOATTR; } // make new header_t after deleting name xattr xattrs.erase(xiter); S3FS_PRN_DBG("Reset xattrs(after delete %s key) = %s", name, raw_build_xattrs(xattrs).c_str()); if(S_ISDIR(stbuf.st_mode) && IS_REPLACEDIR(nDirType)){ // Should rebuild directory object(except new type) // Need to remove old dir("dir" etc) and make new dir("dir/") // At first, remove directory old object if(0 != (result = remove_old_type_dir(strpath, nDirType))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); // Make new directory object("dir/") struct timespec ts_atime; struct timespec ts_mtime; struct timespec ts_ctime; set_stat_to_timespec(stbuf, stat_time_type::ATIME, ts_atime); set_stat_to_timespec(stbuf, stat_time_type::MTIME, ts_mtime); set_stat_to_timespec(stbuf, stat_time_type::CTIME, ts_ctime); if(0 != (result = create_directory_object(newpath.c_str(), stbuf.st_mode, ts_atime, ts_mtime, ts_ctime, stbuf.st_uid, stbuf.st_gid, nullptr))){ return result; } // need to set xattr header for directory. strpath = newpath; nowcache = strpath; } // set xattr all object std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath; headers_t updatemeta; updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str())); updatemeta["x-amz-metadata-directive"] = "REPLACE"; if(!xattrs.empty()){ updatemeta["x-amz-meta-xattr"] = build_xattrs(xattrs); }else{ updatemeta["x-amz-meta-xattr"] = ""; // This is a special case. If empty, this header will eventually be removed. } // check opened file handle. // // If the file starts uploading by multipart when the disk capacity is insufficient, // we need to put these header after finishing upload. // Or if the file is only open, we must update to FdEntity's internal meta. // AutoFdEntity autoent; FdEntity* ent; bool need_put_header = true; if(nullptr != (ent = autoent.OpenExistFdEntity(path))){ if(ent->MergeOrgMeta(updatemeta)){ // meta is changed, but now uploading. // then the meta is pending and accumulated to be put after the upload is complete. S3FS_PRN_INFO("meta pending until upload is complete"); need_put_header = false; // If there is data in the Stats cache, update the Stats cache. StatCache::getStatCacheData()->UpdateMetaStats(strpath, updatemeta); } } if(need_put_header){ // not found opened file. if(updatemeta["x-amz-meta-xattr"].empty()){ updatemeta.erase("x-amz-meta-xattr"); } merge_headers(meta, updatemeta, true); // upload meta directly. if(0 != (result = put_headers(strpath.c_str(), meta, true))){ return result; } StatCache::getStatCacheData()->DelStat(nowcache); } return 0; } // s3fs_init calls this function to exit cleanly from the fuse event loop. // // There's no way to pass an exit status to the high-level event loop API, so // this function stores the exit value in a global for main() static void s3fs_exit_fuseloop(int exit_status) { S3FS_PRN_ERR("Exiting FUSE event loop due to errors\n"); s3fs_init_deferred_exit_status = exit_status; struct fuse_context *ctx = fuse_get_context(); if (nullptr != ctx) { fuse_exit(ctx->fuse); } } static void* s3fs_init(struct fuse_conn_info* conn) { S3FS_PRN_INIT_INFO("init v%s%s with %s, credential-library(%s)", VERSION, COMMIT_HASH_VAL, s3fs_crypt_lib_name(), ps3fscred->GetCredFuncVersion(false)); // cache(remove cache dirs at first) if(is_remove_cache && (!CacheFileStat::DeleteCacheFileStatDirectory() || !FdManager::DeleteCacheDirectory())){ S3FS_PRN_DBG("Could not initialize cache directory."); } // check loading IAM role name if(!ps3fscred->LoadIAMRoleFromMetaData()){ S3FS_PRN_CRIT("could not load IAM role name from meta data."); s3fs_exit_fuseloop(EXIT_FAILURE); return nullptr; } // Check Bucket { int result; if(EXIT_SUCCESS != (result = s3fs_check_service())){ s3fs_exit_fuseloop(result); return nullptr; } } // Investigate system capabilities #ifndef __APPLE__ if(conn->capable & FUSE_CAP_ATOMIC_O_TRUNC){ conn->want |= FUSE_CAP_ATOMIC_O_TRUNC; } #endif if(conn->capable & FUSE_CAP_BIG_WRITES){ conn->want |= FUSE_CAP_BIG_WRITES; } if(!ThreadPoolMan::Initialize(max_thread_count)){ S3FS_PRN_CRIT("Could not create thread pool(%d)", max_thread_count); s3fs_exit_fuseloop(EXIT_FAILURE); } // Signal object if(!S3fsSignals::Initialize()){ S3FS_PRN_ERR("Failed to initialize signal object, but continue..."); } return nullptr; } static void s3fs_destroy(void*) { S3FS_PRN_INFO("destroy"); // Signal object if(!S3fsSignals::Destroy()){ S3FS_PRN_WARN("Failed to clean up signal object."); } ThreadPoolMan::Destroy(); // cache(remove at last) if(is_remove_cache && (!CacheFileStat::DeleteCacheFileStatDirectory() || !FdManager::DeleteCacheDirectory())){ S3FS_PRN_WARN("Could not remove cache directory."); } } static int s3fs_access(const char* path, int mask) { FUSE_CTX_INFO("[path=%s][mask=%s%s%s%s]", path, ((mask & R_OK) == R_OK) ? "R_OK " : "", ((mask & W_OK) == W_OK) ? "W_OK " : "", ((mask & X_OK) == X_OK) ? "X_OK " : "", (mask == F_OK) ? "F_OK" : ""); int result = check_object_access(path, mask, nullptr); S3FS_MALLOCTRIM(0); return result; } // // If calling with wrong region, s3fs gets following error body as 400 error code. // " // AuthorizationHeaderMalformed // The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1' // ap-northeast-1 // ... // ... // " // // So this is cheap code but s3fs should get correct region automatically. // static bool check_region_error(const char* pbody, size_t len, std::string& expectregion) { if(!pbody){ return false; } std::string code; if(!simple_parse_xml(pbody, len, "Code", code) || code != "AuthorizationHeaderMalformed"){ return false; } if(!simple_parse_xml(pbody, len, "Region", expectregion)){ return false; } return true; } static bool check_endpoint_error(const char* pbody, size_t len, std::string& expectendpoint) { if(!pbody){ return false; } std::string code; if(!simple_parse_xml(pbody, len, "Code", code) || code != "PermanentRedirect"){ return false; } if(!simple_parse_xml(pbody, len, "Endpoint", expectendpoint)){ return false; } return true; } static bool check_invalid_sse_arg_error(const char* pbody, size_t len) { if(!pbody){ return false; } std::string code; if(!simple_parse_xml(pbody, len, "Code", code) || code != "InvalidArgument"){ return false; } std::string argname; if(!simple_parse_xml(pbody, len, "ArgumentName", argname) || argname != "x-amz-server-side-encryption"){ return false; } return true; } static bool check_error_message(const char* pbody, size_t len, std::string& message) { message.clear(); if(!pbody){ return false; } if(!simple_parse_xml(pbody, len, "Message", message)){ return false; } return true; } // [NOTE] // This function checks if the bucket is accessible when s3fs starts. // // The following patterns for mount points are supported by s3fs: // (1) Mount the bucket top // (2) Mount to a directory(folder) under the bucket. In this case: // (2A) Directories created by clients other than s3fs // (2B) Directory created by s3fs // // Both case of (1) and (2) check access permissions to the mount point // path(directory). // In the case of (2A), if the directory(object) for the mount point does // not exist, the check fails. However, launching s3fs with the "compat_dir" // option avoids this error and the check succeeds. If you do not specify // the "compat_dir" option in case (2A), please create a directory(object) // for the mount point before launching s3fs. // static int s3fs_check_service() { S3FS_PRN_INFO("check services."); // At first time for access S3, we check IAM role if it sets. if(!ps3fscred->CheckIAMCredentialUpdate()){ S3FS_PRN_CRIT("Failed to initialize IAM credential."); return EXIT_FAILURE; } S3fsCurl s3fscurl; int res; bool force_no_sse = false; while(0 > (res = s3fscurl.CheckBucket(get_realpath("/").c_str(), support_compat_dir, force_no_sse))){ // get response code bool do_retry = false; long responseCode = s3fscurl.GetLastResponseCode(); // check wrong endpoint, and automatically switch endpoint if(300 <= responseCode && responseCode < 500){ // check region error(for putting message or retrying) const std::string& body = s3fscurl.GetBodyData(); std::string expectregion; std::string expectendpoint; // Check if any case can be retried if(check_region_error(body.c_str(), body.size(), expectregion)){ // [NOTE] // If endpoint is not specified(using us-east-1 region) and // an error is encountered accessing a different region, we // will retry the check on the expected region. // see) https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro // if(s3host != "http://s3.amazonaws.com" && s3host != "https://s3.amazonaws.com"){ // specified endpoint for specified url is wrong. if(is_specified_endpoint){ S3FS_PRN_CRIT("The bucket region is not '%s'(specified) for specified url(%s), it is correctly '%s'. You should specify url(http(s)://s3-%s.amazonaws.com) and endpoint(%s) option.", endpoint.c_str(), s3host.c_str(), expectregion.c_str(), expectregion.c_str(), expectregion.c_str()); }else{ S3FS_PRN_CRIT("The bucket region is not '%s'(default) for specified url(%s), it is correctly '%s'. You should specify url(http(s)://s3-%s.amazonaws.com) and endpoint(%s) option.", endpoint.c_str(), s3host.c_str(), expectregion.c_str(), expectregion.c_str(), expectregion.c_str()); } }else if(is_specified_endpoint){ // specified endpoint is wrong. S3FS_PRN_CRIT("The bucket region is not '%s'(specified), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str()); }else if(S3fsCurl::GetSignatureType() == signature_type_t::V4_ONLY || S3fsCurl::GetSignatureType() == signature_type_t::V2_OR_V4){ // current endpoint and url are default value, so try to connect to expected region. S3FS_PRN_CRIT("Failed to connect region '%s'(default), so retry to connect region '%s' for url(http(s)://s3-%s.amazonaws.com).", endpoint.c_str(), expectregion.c_str(), expectregion.c_str()); // change endpoint endpoint = expectregion; // change url if(s3host == "http://s3.amazonaws.com"){ s3host = "http://s3-" + endpoint + ".amazonaws.com"; }else if(s3host == "https://s3.amazonaws.com"){ s3host = "https://s3-" + endpoint + ".amazonaws.com"; } // Retry with changed host s3fscurl.DestroyCurlHandle(); do_retry = true; }else{ S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str()); } }else if(check_endpoint_error(body.c_str(), body.size(), expectendpoint)){ // redirect error if(pathrequeststyle){ S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and endpoint(%s)). You need to specify correct url(http(s)://s3-.amazonaws.com) and endpoint option with use_path_request_style option.", s3host.c_str(), endpoint.c_str()); }else{ S3FS_PRN_CRIT("S3 service returned PermanentRedirect with %s (current is url(%s) and endpoint(%s)). You need to specify correct endpoint option.", expectendpoint.c_str(), s3host.c_str(), endpoint.c_str()); } return EXIT_FAILURE; }else if(check_invalid_sse_arg_error(body.c_str(), body.size())){ // SSE argument error, so retry it without SSE S3FS_PRN_CRIT("S3 service returned InvalidArgument(x-amz-server-side-encryption), so retry without adding x-amz-server-side-encryption."); // Retry without sse parameters s3fscurl.DestroyCurlHandle(); do_retry = true; force_no_sse = true; } } // Try changing signature from v4 to v2 // // [NOTE] // If there is no case to retry with the previous checks, and there // is a chance to retry with signature v2, prepare to retry with v2. // if(!do_retry && (responseCode == 400 || responseCode == 403) && S3fsCurl::GetSignatureType() == signature_type_t::V2_OR_V4){ // switch sigv2 S3FS_PRN_CRIT("Failed to connect by sigv4, so retry to connect by signature version 2. But you should to review url and endpoint option."); // retry to check with sigv2 s3fscurl.DestroyCurlHandle(); do_retry = true; S3fsCurl::SetSignatureType(signature_type_t::V2_ONLY); } // check errors(after retrying) if(!do_retry && responseCode != 200 && responseCode != 301){ // parse error message if existed std::string errMessage; const std::string& body = s3fscurl.GetBodyData(); check_error_message(body.c_str(), body.size(), errMessage); if(responseCode == 400){ S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Bad Request(host=%s, message=%s)", s3host.c_str(), errMessage.c_str()); }else if(responseCode == 403){ S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Invalid Credentials(host=%s, message=%s)", s3host.c_str(), errMessage.c_str()); }else if(responseCode == 404){ if(mount_prefix.empty()){ S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Bucket or directory not found(host=%s, message=%s)", s3host.c_str(), errMessage.c_str()); }else{ S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Bucket or directory(%s) not found(host=%s, message=%s) - You may need to specify the compat_dir option.", mount_prefix.c_str(), s3host.c_str(), errMessage.c_str()); } }else{ S3FS_PRN_CRIT("Failed to check bucket and directory for mount point : Unable to connect(host=%s, message=%s)", s3host.c_str(), errMessage.c_str()); } return EXIT_FAILURE; } } s3fscurl.DestroyCurlHandle(); // make sure remote mountpath exists and is a directory if(!mount_prefix.empty()){ if(remote_mountpath_exists("/", support_compat_dir) != 0){ S3FS_PRN_CRIT("Remote mountpath %s not found, this may be resolved with the compat_dir option.", mount_prefix.c_str()); return EXIT_FAILURE; } } S3FS_MALLOCTRIM(0); return EXIT_SUCCESS; } // // Check & Set attributes for mount point. // static bool set_mountpoint_attribute(struct stat& mpst) { mp_uid = geteuid(); mp_gid = getegid(); mp_mode = S_IFDIR | (allow_other ? (is_mp_umask ? (~mp_umask & (S_IRWXU | S_IRWXG | S_IRWXO)) : (S_IRWXU | S_IRWXG | S_IRWXO)) : S_IRWXU); // In MSYS2 environment with WinFsp, it is not supported to change mode of mount point. // Doing that forcely will occurs permission problem, so disabling it. #ifdef __MSYS__ return true; #else S3FS_PRN_INFO2("PROC(uid=%u, gid=%u) - MountPoint(uid=%u, gid=%u, mode=%04o)", (unsigned int)mp_uid, (unsigned int)mp_gid, (unsigned int)(mpst.st_uid), (unsigned int)(mpst.st_gid), mpst.st_mode); // check owner if(0 == mp_uid || mpst.st_uid == mp_uid){ return true; } // check group permission if(mpst.st_gid == mp_gid || 1 == is_uid_include_group(mp_uid, mpst.st_gid)){ if(S_IRWXG == (mpst.st_mode & S_IRWXG)){ return true; } } // check other permission if(S_IRWXO == (mpst.st_mode & S_IRWXO)){ return true; } return false; #endif } // // Set bucket and mount_prefix based on passed bucket name. // static int set_bucket(const std::string& arg) { size_t pos; if((pos = arg.find(':')) != std::string::npos){ if(arg.find("://") != std::string::npos){ S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg.c_str()); return -1; } std::string bucket_name = arg.substr(0, pos); if(!S3fsCred::SetBucket(bucket_name)){ S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg.c_str()); return -1; } std::string pmount_prefix = arg.substr(pos + 1); if(pmount_prefix.empty() || '/' != pmount_prefix[0]){ S3FS_PRN_EXIT("path(%s) must be prefix \"/\".", pmount_prefix.c_str()); return -1; } mount_prefix = pmount_prefix; // Trim the last consecutive '/' mount_prefix = trim_right(mount_prefix, "/"); }else{ if(!S3fsCred::SetBucket(arg)){ S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg.c_str()); return -1; } } return 0; } // // Utility function for parse "--bucket_size" option // // max_size: A string like 20000000, 30GiB, 20TB etc // return: An integer of type fsblkcnt_t corresponding to the number // of blocks with max_size calculated with the s3fs block size, // or 0 on error // static fsblkcnt_t parse_bucket_size(std::string max_size) { const unsigned long long ten00 = 1000L; const unsigned long long ten24 = 1024L; unsigned long long scale = 1; unsigned long long n_bytes = 0; size_t pos; if((pos = max_size.find("GB")) != std::string::npos){ scale = ten00 * ten00 * ten00; if(2 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("GiB")) != std::string::npos){ scale = ten24 * ten24 * ten24; if(3 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("TB")) != std::string::npos){ scale = ten00 * ten00 * ten00 * ten00; if(2 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("TiB")) != std::string::npos){ scale = ten24 * ten24 * ten24 * ten24; if(3 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("PB")) != std::string::npos){ scale = ten00 * ten00 * ten00 * ten00 * ten00; if(2 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("PiB")) != std::string::npos){ scale = ten24 * ten24 * ten24 * ten24 * ten24; if(3 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("EB")) != std::string::npos){ scale = ten00 * ten00 * ten00 * ten00 * ten00 * ten00; if(2 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); }else if((pos = max_size.find("EiB")) != std::string::npos){ scale = ten24 * ten24 * ten24 * ten24 * ten24 * ten24; if(3 < max_size.size() - pos){ return 0; // no trailing garbage } max_size.erase(pos); } // extra check for(size_t i = 0; i < pos; ++i){ if(!isdigit(max_size[i])){ return 0; // wrong number } n_bytes = strtoull(max_size.c_str(), nullptr, 10); if((INT64_MAX / scale) < n_bytes){ return 0; // overflow } n_bytes *= scale; } // [NOTE] // To round a number by s3fs block size. // And need to check the result value because fsblkcnt_t is 32bit in macos etc. // n_bytes /= s3fs_block_size; if(sizeof(fsblkcnt_t) <= 4){ if(INT32_MAX < n_bytes){ return 0; // overflow } } return static_cast(n_bytes); // cast to fsblkcnt_t } static bool is_cmd_exists(const std::string& command) { // The `command -v` is a POSIX-compliant method for checking the existence of a program. std::string cmd = "command -v " + command + " >/dev/null 2>&1"; int result = system(cmd.c_str()); return (result !=-1 && WIFEXITED(result) && WEXITSTATUS(result) == 0); } static int print_umount_message(const std::string& mp, bool force) { std::string cmd; if (is_cmd_exists("fusermount")){ if (force){ cmd = "fusermount -uz " + mp; } else { cmd = "fusermount -u " + mp; } }else{ if (force){ cmd = "umount -l " + mp; } else { cmd = "umount " + mp; } } S3FS_PRN_EXIT("MOUNTPOINT %s is stale, you could use this command to fix: %s", mp.c_str(), cmd.c_str()); return 0; } // This is repeatedly called by the fuse option parser // if the key is equal to FUSE_OPT_KEY_OPT, it's an option passed in prefixed by // '-' or '--' e.g.: -f -d -ousecache=/tmp // // if the key is equal to FUSE_OPT_KEY_NONOPT, it's either the bucket name // or the mountpoint. The bucket name will always come before the mountpoint // static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_args* outargs) { int ret; if(key == FUSE_OPT_KEY_NONOPT){ // the first NONOPT option is the bucket name if(S3fsCred::GetBucket().empty()){ if ((ret = set_bucket(arg)) != 0){ return ret; } return 0; }else if (!strcmp(arg, "s3fs")) { return 0; } // the second NONOPT option is the mountpoint(not utility mode) if(mountpoint.empty() && utility_incomp_type::NO_UTILITY_MODE == utility_mode){ // save the mountpoint and do some basic error checking mountpoint = arg; struct stat stbuf; // In MSYS2 environment with WinFsp, it is not needed to create the mount point before mounting. // Also it causes a conflict with WinFsp's validation, so disabling it. #ifdef __MSYS__ stbuf = {}; set_mountpoint_attribute(stbuf); #else if(stat(arg, &stbuf) == -1){ // check stale mountpoint if(errno == ENOTCONN){ print_umount_message(mountpoint, true); } else { S3FS_PRN_EXIT("unable to access MOUNTPOINT %s: %s", mountpoint.c_str(), strerror(errno)); } return -1; } if(!(S_ISDIR(stbuf.st_mode))){ S3FS_PRN_EXIT("MOUNTPOINT: %s is not a directory.", mountpoint.c_str()); return -1; } if(!set_mountpoint_attribute(stbuf)){ S3FS_PRN_EXIT("MOUNTPOINT: %s permission denied.", mountpoint.c_str()); return -1; } if(!nonempty){ const struct dirent *ent; DIR *dp = opendir(mountpoint.c_str()); if(dp == nullptr){ S3FS_PRN_EXIT("failed to open MOUNTPOINT: %s: %s", mountpoint.c_str(), strerror(errno)); return -1; } while((ent = readdir(dp)) != nullptr){ if(strcmp(ent->d_name, ".") != 0 && strcmp(ent->d_name, "..") != 0){ closedir(dp); S3FS_PRN_EXIT("MOUNTPOINT directory %s is not empty. if you are sure this is safe, can use the 'nonempty' mount option.", mountpoint.c_str()); return -1; } } closedir(dp); } #endif return 1; } // Unknown option if(utility_incomp_type::NO_UTILITY_MODE == utility_mode){ S3FS_PRN_EXIT("specified unknown third option(%s).", arg); }else{ S3FS_PRN_EXIT("specified unknown second option(%s). you don't need to specify second option(mountpoint) for utility mode(-u).", arg); } return -1; }else if(key == FUSE_OPT_KEY_OPT){ if(is_prefix(arg, "uid=")){ s3fs_uid = get_uid(strchr(arg, '=') + sizeof(char)); if(0 != geteuid() && 0 == s3fs_uid){ S3FS_PRN_EXIT("root user can only specify uid=0."); return -1; } is_s3fs_uid = true; return 1; // continue for fuse option } else if(is_prefix(arg, "gid=")){ s3fs_gid = get_gid(strchr(arg, '=') + sizeof(char)); if(0 != getegid() && 0 == s3fs_gid){ S3FS_PRN_EXIT("root user can only specify gid=0."); return -1; } is_s3fs_gid = true; return 1; // continue for fuse option } else if(is_prefix(arg, "bucket_size=")){ bucket_block_count = parse_bucket_size(strchr(arg, '=') + sizeof(char)); if(0 == bucket_block_count){ S3FS_PRN_EXIT("invalid bucket_size option."); return -1; } return 0; } else if(is_prefix(arg, "umask=")){ off_t s3fs_umask_tmp = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 8); s3fs_umask = s3fs_umask_tmp & (S_IRWXU | S_IRWXG | S_IRWXO); is_s3fs_umask = true; return 1; // continue for fuse option } else if(0 == strcmp(arg, "allow_other")){ allow_other = true; return 1; // continue for fuse option } else if(is_prefix(arg, "mp_umask=")){ off_t mp_umask_tmp = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 8); mp_umask = mp_umask_tmp & (S_IRWXU | S_IRWXG | S_IRWXO); is_mp_umask = true; return 0; } else if(is_prefix(arg, "default_acl=")){ const char* acl_string = strchr(arg, '=') + sizeof(char); acl_t acl = to_acl(acl_string); if(acl == acl_t::UNKNOWN){ S3FS_PRN_EXIT("unknown value for default_acl: %s", acl_string); return -1; } S3fsCurl::SetDefaultAcl(acl); return 0; } else if(is_prefix(arg, "retries=")){ off_t retries = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10); if(retries == 0){ S3FS_PRN_EXIT("retries must be greater than zero"); return -1; } S3fsCurl::SetRetries(static_cast(retries)); return 0; } else if(is_prefix(arg, "tmpdir=")){ FdManager::SetTmpDir(strchr(arg, '=') + sizeof(char)); return 0; } else if(is_prefix(arg, "use_cache=")){ FdManager::SetCacheDir(strchr(arg, '=') + sizeof(char)); return 0; } else if(0 == strcmp(arg, "check_cache_dir_exist")){ FdManager::SetCheckCacheDirExist(true); return 0; } else if(0 == strcmp(arg, "del_cache")){ is_remove_cache = true; return 0; } else if(is_prefix(arg, "multireq_max=")){ int maxreq = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); S3fsCurl::SetMaxMultiRequest(maxreq); return 0; } else if(0 == strcmp(arg, "nonempty")){ nonempty = true; return 1; // need to continue for fuse. } else if(0 == strcmp(arg, "nomultipart")){ nomultipart = true; return 0; } // old format for storage_class else if(0 == strcmp(arg, "use_rrs") || is_prefix(arg, "use_rrs=")){ off_t rrs = 1; // for an old format. if(is_prefix(arg, "use_rrs=")){ rrs = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10); } if(0 == rrs){ S3fsCurl::SetStorageClass("STANDARD"); }else if(1 == rrs){ S3fsCurl::SetStorageClass("REDUCED_REDUNDANCY"); }else{ S3FS_PRN_EXIT("poorly formed argument to option: use_rrs"); return -1; } return 0; } else if(is_prefix(arg, "storage_class=")){ const char *storage_class = strchr(arg, '=') + sizeof(char); S3fsCurl::SetStorageClass(storage_class); return 0; } // // [NOTE] // use_sse Set Server Side Encrypting type to SSE-S3 // use_sse=1 // use_sse=file Set Server Side Encrypting type to Custom key(SSE-C) and load custom keys // use_sse=custom(c):file // use_sse=custom(c) Set Server Side Encrypting type to Custom key(SSE-C) // use_sse=kmsid(k):kms-key-id Set Server Side Encrypting type to AWS Key Management key id(SSE-KMS) and load KMS id // use_sse=kmsid(k) Set Server Side Encrypting type to AWS Key Management key id(SSE-KMS) // // load_sse_c=file Load Server Side Encrypting custom keys // // AWSSSECKEYS Loading Environment for Server Side Encrypting custom keys // AWSSSEKMSID Loading Environment for Server Side Encrypting Key id // else if(is_prefix(arg, "use_sse")){ if(0 == strcmp(arg, "use_sse") || 0 == strcmp(arg, "use_sse=1")){ // use_sse=1 is old type parameter // sse type is SSE_S3 if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseS3Type()){ S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment."); return -1; } S3fsCurl::SetSseType(sse_type_t::SSE_S3); }else if(0 == strcmp(arg, "use_sse=kmsid") || 0 == strcmp(arg, "use_sse=k")){ // sse type is SSE_KMS with out kmsid(expecting id is loaded by environment) if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseKmsType()){ S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment."); return -1; } if(!S3fsCurl::IsSetSseKmsId()){ S3FS_PRN_EXIT("use_sse=kms but not loaded kms id by environment."); return -1; } S3fsCurl::SetSseType(sse_type_t::SSE_KMS); }else if(is_prefix(arg, "use_sse=kmsid:") || is_prefix(arg, "use_sse=k:")){ // sse type is SSE_KMS with kmsid if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseKmsType()){ S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment."); return -1; } const char* kmsid; if(is_prefix(arg, "use_sse=kmsid:")){ kmsid = &arg[strlen("use_sse=kmsid:")]; }else{ kmsid = &arg[strlen("use_sse=k:")]; } if(!S3fsCurl::SetSseKmsid(kmsid)){ S3FS_PRN_EXIT("failed to load use_sse kms id."); return -1; } S3fsCurl::SetSseType(sse_type_t::SSE_KMS); }else if(0 == strcmp(arg, "use_sse=custom") || 0 == strcmp(arg, "use_sse=c")){ // sse type is SSE_C with out custom keys(expecting keys are loaded by environment or load_sse_c option) if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseCType()){ S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment."); return -1; } // [NOTE] // do not check ckeys exists here. // S3fsCurl::SetSseType(sse_type_t::SSE_C); }else if(is_prefix(arg, "use_sse=custom:") || is_prefix(arg, "use_sse=c:")){ // sse type is SSE_C with custom keys if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseCType()){ S3FS_PRN_EXIT("already set SSE another type, so conflict use_sse option or environment."); return -1; } const char* ssecfile; if(is_prefix(arg, "use_sse=custom:")){ ssecfile = &arg[strlen("use_sse=custom:")]; }else{ ssecfile = &arg[strlen("use_sse=c:")]; } if(!S3fsCurl::SetSseCKeys(ssecfile)){ S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile); return -1; } S3fsCurl::SetSseType(sse_type_t::SSE_C); }else if(0 == strcmp(arg, "use_sse=")){ // this type is old style(parameter is custom key file path) // SSE_C with custom keys. const char* ssecfile = &arg[strlen("use_sse=")]; if(!S3fsCurl::SetSseCKeys(ssecfile)){ S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile); return -1; } S3fsCurl::SetSseType(sse_type_t::SSE_C); }else{ // never come here. S3FS_PRN_EXIT("something wrong use_sse option."); return -1; } return 0; } // [NOTE] // Do only load SSE custom keys, care for set without set sse type. else if(is_prefix(arg, "load_sse_c=")){ const char* ssecfile = &arg[strlen("load_sse_c=")]; if(!S3fsCurl::SetSseCKeys(ssecfile)){ S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile); return -1; } return 0; } else if(is_prefix(arg, "ssl_verify_hostname=")){ long sslvh = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); if(-1 == S3fsCurl::SetSslVerifyHostname(sslvh)){ S3FS_PRN_EXIT("poorly formed argument to option: ssl_verify_hostname."); return -1; } return 0; } else if(is_prefix(arg, "ssl_client_cert=")){ std::string values = strchr(arg, '=') + sizeof(char); if(!S3fsCurl::SetSSLClientCertOptions(values)){ S3FS_PRN_EXIT("failed to set SSL client certification options."); return -1; } return 0; } // // Detect options for credential // else if(0 >= (ret = ps3fscred->DetectParam(arg))){ if(0 > ret){ return -1; } return 0; } else if(is_prefix(arg, "public_bucket=")){ off_t pubbucket = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10); if(1 == pubbucket){ S3fsCurl::SetPublicBucket(true); // [NOTE] // if bucket is public(without credential), s3 do not allow copy api. // so s3fs sets nocopyapi mode. // nocopyapi = true; }else if(0 == pubbucket){ S3fsCurl::SetPublicBucket(false); }else{ S3FS_PRN_EXIT("poorly formed argument to option: public_bucket."); return -1; } return 0; } else if(is_prefix(arg, "bucket=")){ std::string bname = strchr(arg, '=') + sizeof(char); if ((ret = set_bucket(bname))){ return ret; } return 0; } else if(0 == strcmp(arg, "no_check_certificate")){ S3fsCurl::SetCheckCertificate(false); return 0; } else if(is_prefix(arg, "connect_timeout=")){ long contimeout = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); S3fsCurl::SetConnectTimeout(contimeout); return 0; } else if(is_prefix(arg, "readwrite_timeout=")){ auto rwtimeout = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); S3fsCurl::SetReadwriteTimeout(rwtimeout); return 0; } else if(is_prefix(arg, "list_object_max_keys=")){ int max_keys = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); if(max_keys < 1000){ S3FS_PRN_EXIT("argument should be over 1000: list_object_max_keys"); return -1; } max_keys_list_object = max_keys; return 0; } else if(is_prefix(arg, "max_stat_cache_size=")){ auto cache_size = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), 10)); StatCache::getStatCacheData()->SetCacheSize(cache_size); return 0; } else if(is_prefix(arg, "stat_cache_expire=")){ auto expr_time = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), 10)); StatCache::getStatCacheData()->SetExpireTime(expr_time); return 0; } // [NOTE] // This option is for compatibility old version. else if(is_prefix(arg, "stat_cache_interval_expire=")){ auto expr_time = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); StatCache::getStatCacheData()->SetExpireTime(expr_time, true); return 0; } else if(0 == strcmp(arg, "enable_noobj_cache")){ S3FS_PRN_WARN("enable_noobj_cache is enabled by default and a future version will remove this option."); StatCache::getStatCacheData()->EnableCacheNoObject(); return 0; } else if(0 == strcmp(arg, "disable_noobj_cache")){ StatCache::getStatCacheData()->DisableCacheNoObject(); return 0; } else if(0 == strcmp(arg, "nodnscache")){ S3fsCurl::SetDnsCache(false); return 0; } else if(0 == strcmp(arg, "nosscache")){ S3fsCurl::SetSslSessionCache(false); return 0; } else if(is_prefix(arg, "parallel_count=") || is_prefix(arg, "parallel_upload=")){ int maxpara = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); if(0 >= maxpara){ S3FS_PRN_EXIT("argument should be over 1: parallel_count"); return -1; } S3fsCurl::SetMaxParallelCount(maxpara); return 0; } else if(is_prefix(arg, "max_thread_count=")){ int max_thcount = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); if(0 >= max_thcount){ S3FS_PRN_EXIT("argument should be over 1: max_thread_count"); return -1; } max_thread_count = max_thcount; S3FS_PRN_WARN("The max_thread_count option is not a formal option. Please note that it will change in the future."); return 0; } else if(is_prefix(arg, "fd_page_size=")){ S3FS_PRN_ERR("option fd_page_size is no longer supported, so skip this option."); return 0; } else if(is_prefix(arg, "multipart_size=")){ off_t size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10); if(!S3fsCurl::SetMultipartSize(size)){ S3FS_PRN_EXIT("multipart_size option must be at least 5 MB."); return -1; } return 0; } else if(is_prefix(arg, "multipart_copy_size=")){ off_t size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10); if(!S3fsCurl::SetMultipartCopySize(size)){ S3FS_PRN_EXIT("multipart_copy_size option must be at least 5 MB."); return -1; } return 0; } else if(is_prefix(arg, "max_dirty_data=")){ off_t size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10); if(size >= 50){ size *= 1024 * 1024; }else if(size != -1){ S3FS_PRN_EXIT("max_dirty_data option must be at least 50 MB."); return -1; } max_dirty_data = size; return 0; } if(is_prefix(arg, "free_space_ratio=")){ int ratio = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)); if(FdManager::GetEnsureFreeDiskSpace()!=0){ S3FS_PRN_EXIT("option free_space_ratio conflicts with ensure_diskfree, please set only one of them."); return -1; } if(ratio < 0 || ratio > 100){ S3FS_PRN_EXIT("option free_space_ratio must between 0 to 100, which is: %d", ratio); return -1; } off_t dfsize = FdManager::GetTotalDiskSpaceByRatio(ratio); S3FS_PRN_INFO("Free space ratio set to %d %%, ensure the available disk space is greater than %.3f MB", ratio, static_cast(dfsize) / 1024 / 1024); if(dfsize < S3fsCurl::GetMultipartSize()){ S3FS_PRN_WARN("specified size to ensure disk free space is smaller than multipart size, so set multipart size to it."); dfsize = S3fsCurl::GetMultipartSize(); } FdManager::SetEnsureFreeDiskSpace(dfsize); return 0; } else if(is_prefix(arg, "ensure_diskfree=")){ off_t dfsize = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10) * 1024 * 1024; if(FdManager::GetEnsureFreeDiskSpace()!=0){ S3FS_PRN_EXIT("option free_space_ratio conflicts with ensure_diskfree, please set only one of them."); return -1; } S3FS_PRN_INFO("Set and ensure the available disk space is greater than %.3f MB.", static_cast(dfsize) / 1024 / 1024); if(dfsize < S3fsCurl::GetMultipartSize()){ S3FS_PRN_WARN("specified size to ensure disk free space is smaller than multipart size, so set multipart size to it."); dfsize = S3fsCurl::GetMultipartSize(); } FdManager::SetEnsureFreeDiskSpace(dfsize); return 0; } else if(is_prefix(arg, "fake_diskfree=")){ S3FS_PRN_WARN("The fake_diskfree option was specified. Use this option for testing or debugging."); // [NOTE] This value is used for initializing to FdManager after parsing all options. fake_diskfree_size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10) * 1024 * 1024; return 0; } else if(is_prefix(arg, "multipart_threshold=")){ multipart_threshold = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)) * 1024 * 1024; if(multipart_threshold <= MIN_MULTIPART_SIZE){ S3FS_PRN_EXIT("multipart_threshold must be at least %lld, was: %lld", static_cast(MIN_MULTIPART_SIZE), static_cast(multipart_threshold)); return -1; } return 0; } else if(is_prefix(arg, "singlepart_copy_limit=")){ singlepart_copy_limit = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)) * 1024 * 1024; return 0; } else if(is_prefix(arg, "ahbe_conf=")){ std::string ahbe_conf = strchr(arg, '=') + sizeof(char); if(!AdditionalHeader::get()->Load(ahbe_conf.c_str())){ S3FS_PRN_EXIT("failed to load ahbe_conf file(%s).", ahbe_conf.c_str()); return -1; } AdditionalHeader::get()->Dump(); return 0; } else if(0 == strcmp(arg, "noxmlns")){ noxmlns = true; return 0; } else if(0 == strcmp(arg, "nomixupload")){ FdEntity::SetNoMixMultipart(); return 0; } else if(0 == strcmp(arg, "nocopyapi")){ nocopyapi = true; return 0; } else if(0 == strcmp(arg, "streamupload")){ FdEntity::SetStreamUpload(true); S3FS_PRN_WARN("The streamupload option is not a formal option. Please note that it will change in the future."); return 0; } else if(0 == strcmp(arg, "norenameapi")){ norenameapi = true; return 0; } else if(0 == strcmp(arg, "complement_stat")){ complement_stat = true; return 0; } else if(0 == strcmp(arg, "notsup_compat_dir")){ S3FS_PRN_WARN("notsup_compat_dir is enabled by default and a future version will remove this option."); support_compat_dir = false; return 0; } else if(0 == strcmp(arg, "compat_dir")){ support_compat_dir = true; return 0; } else if(0 == strcmp(arg, "enable_content_md5")){ S3fsCurl::SetContentMd5(true); return 0; } else if(0 == strcmp(arg, "enable_unsigned_payload")){ S3fsCurl::SetUnsignedPayload(true); return 0; } else if(0 == strcmp(arg, "update_parent_dir_stat")){ update_parent_dir_stat = true; return 0; } else if(is_prefix(arg, "host=")){ s3host = strchr(arg, '=') + sizeof(char); return 0; } else if(is_prefix(arg, "servicepath=")){ service_path = strchr(arg, '=') + sizeof(char); return 0; } else if(is_prefix(arg, "url=")){ s3host = strchr(arg, '=') + sizeof(char); // strip the trailing '/', if any, off the end of the host // std::string size_t found, length; found = s3host.find_last_of('/'); length = s3host.length(); while(found == (length - 1) && length > 0){ s3host.erase(found); found = s3host.find_last_of('/'); length = s3host.length(); } // Check url for http / https protocol std::string if(!is_prefix(s3host.c_str(), "https://") && !is_prefix(s3host.c_str(), "http://")){ S3FS_PRN_EXIT("option url has invalid format, missing http / https protocol"); return -1; } return 0; } else if(0 == strcmp(arg, "sigv2")){ S3fsCurl::SetSignatureType(signature_type_t::V2_ONLY); return 0; } else if(0 == strcmp(arg, "sigv4")){ S3fsCurl::SetSignatureType(signature_type_t::V4_ONLY); return 0; } else if(is_prefix(arg, "endpoint=")){ endpoint = strchr(arg, '=') + sizeof(char); is_specified_endpoint = true; return 0; } else if(0 == strcmp(arg, "use_path_request_style")){ pathrequeststyle = true; return 0; } else if(0 == strcmp(arg, "noua")){ S3fsCurl::SetUserAgentFlag(false); return 0; } else if(0 == strcmp(arg, "listobjectsv2")){ S3fsCurl::SetListObjectsV2(true); return 0; } else if(0 == strcmp(arg, "use_xattr")){ is_use_xattr = true; return 0; }else if(is_prefix(arg, "use_xattr=")){ const char* strflag = strchr(arg, '=') + sizeof(char); if(0 == strcmp(strflag, "1")){ is_use_xattr = true; }else if(0 == strcmp(strflag, "0")){ is_use_xattr = false; }else{ S3FS_PRN_EXIT("option use_xattr has unknown parameter(%s).", strflag); return -1; } return 0; } else if(is_prefix(arg, "cipher_suites=")){ cipher_suites = strchr(arg, '=') + sizeof(char); return 0; } else if(is_prefix(arg, "instance_name=")){ instance_name = strchr(arg, '=') + sizeof(char); instance_name = "[" + instance_name + "]"; return 0; } else if(is_prefix(arg, "mime=")){ mimetype_file = strchr(arg, '=') + sizeof(char); return 0; } else if(is_prefix(arg, "proxy=")){ const char* url = &arg[strlen("proxy=")]; if(!S3fsCurl::SetProxy(url)){ S3FS_PRN_EXIT("failed to set proxy(%s).", url); return -1; } return 0; } else if(is_prefix(arg, "proxy_cred_file=")){ const char* file = &arg[strlen("proxy_cred_file=")]; if(!S3fsCurl::SetProxyUserPwd(file)){ S3FS_PRN_EXIT("failed to set proxy user and passphrase from file(%s).", file); return -1; } return 0; } else if(is_prefix(arg, "ipresolve=")){ const char* pipresolve = &arg[strlen("ipresolve=")]; if(!S3fsCurl::SetIPResolveType(pipresolve)){ S3FS_PRN_EXIT("failed to ip resolve option value(%s).", pipresolve); return -1; } return 0; } // // log file option // else if(is_prefix(arg, "logfile=")){ const char* strlogfile = strchr(arg, '=') + sizeof(char); if(!S3fsLog::SetLogfile(strlogfile)){ S3FS_PRN_EXIT("The file(%s) specified by logfile option could not be opened.", strlogfile); return -1; } return 0; } // // debug level option // else if(is_prefix(arg, "dbglevel=")){ const char* strlevel = strchr(arg, '=') + sizeof(char); if(0 == strcasecmp(strlevel, "silent") || 0 == strcasecmp(strlevel, "critical") || 0 == strcasecmp(strlevel, "crit")){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_CRIT); }else if(0 == strcasecmp(strlevel, "error") || 0 == strcasecmp(strlevel, "err")){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_ERR); }else if(0 == strcasecmp(strlevel, "wan") || 0 == strcasecmp(strlevel, "warn") || 0 == strcasecmp(strlevel, "warning")){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_WARN); }else if(0 == strcasecmp(strlevel, "inf") || 0 == strcasecmp(strlevel, "info") || 0 == strcasecmp(strlevel, "information")){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_INFO); }else if(0 == strcasecmp(strlevel, "dbg") || 0 == strcasecmp(strlevel, "debug")){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_DBG); }else{ S3FS_PRN_EXIT("option dbglevel has unknown parameter(%s).", strlevel); return -1; } return 0; } // // debug option // // S3fsLog level is LEVEL_INFO, after second -d is passed to fuse. // else if(0 == strcmp(arg, "-d") || 0 == strcmp(arg, "--debug")){ if(!S3fsLog::IsS3fsLogInfo() && !S3fsLog::IsS3fsLogDbg()){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_INFO); return 0; } if(0 == strcmp(arg, "--debug")){ // fuse doesn't understand "--debug", but it understands -d. // but we can't pass -d back to fuse. return 0; } } // "f2" is not used no more. // (set S3fsLog::LEVEL_DBG) else if(0 == strcmp(arg, "f2")){ S3fsLog::SetLogLevel(S3fsLog::LEVEL_DBG); return 0; } else if(0 == strcmp(arg, "curldbg")){ S3fsCurl::SetVerbose(true); return 0; }else if(is_prefix(arg, "curldbg=")){ const char* strlevel = strchr(arg, '=') + sizeof(char); if(0 == strcasecmp(strlevel, "normal")){ S3fsCurl::SetVerbose(true); }else if(0 == strcasecmp(strlevel, "body")){ S3fsCurl::SetVerbose(true); S3fsCurl::SetDumpBody(true); }else{ S3FS_PRN_EXIT("option curldbg has unknown parameter(%s).", strlevel); return -1; } return 0; } // // no time stamp in debug message // else if(0 == strcmp(arg, "no_time_stamp_msg")){ S3fsLog::SetTimeStamp(false); return 0; } // // Check cache file, using SIGUSR1 // else if(0 == strcmp(arg, "set_check_cache_sigusr1")){ if(!S3fsSignals::SetUsr1Handler(nullptr)){ S3FS_PRN_EXIT("could not set sigusr1 for checking cache."); return -1; } return 0; }else if(is_prefix(arg, "set_check_cache_sigusr1=")){ const char* strfilepath = strchr(arg, '=') + sizeof(char); if(!S3fsSignals::SetUsr1Handler(strfilepath)){ S3FS_PRN_EXIT("could not set sigusr1 for checking cache and output file(%s).", strfilepath); return -1; } return 0; } else if(is_prefix(arg, "accessKeyId=")){ S3FS_PRN_EXIT("option accessKeyId is no longer supported."); return -1; } else if(is_prefix(arg, "secretAccessKey=")){ S3FS_PRN_EXIT("option secretAccessKey is no longer supported."); return -1; } else if(0 == strcmp(arg, "use_wtf8")){ use_wtf8 = true; return 0; } else if(0 == strcmp(arg, "requester_pays")){ S3fsCurl::SetRequesterPays(true); return 0; } // [NOTE] // following option will be discarding, because these are not for fuse. // (Referenced sshfs.c) // else if(0 == strcmp(arg, "auto") || 0 == strcmp(arg, "noauto") || 0 == strcmp(arg, "user") || 0 == strcmp(arg, "nouser") || 0 == strcmp(arg, "users") || 0 == strcmp(arg, "_netdev")) { return 0; } } return 1; } int main(int argc, char* argv[]) { int ch; int fuse_res; int option_index = 0; struct fuse_operations s3fs_oper{}; time_t incomp_abort_time = (24 * 60 * 60); S3fsLog singletonLog; static constexpr struct option long_opts[] = { {"help", no_argument, nullptr, 'h'}, {"version", no_argument, nullptr, 0}, {"debug", no_argument, nullptr, 'd'}, {"incomplete-mpu-list", no_argument, nullptr, 'u'}, {"incomplete-mpu-abort", optional_argument, nullptr, 'a'}, // 'a' is only identifier and is not option. {nullptr, 0, nullptr, 0} }; // Use uint32_t instead of fsblkcnt_t::max since the latter overflows and confuses df bucket_block_count = std::numeric_limits::max(); // init xml2 xmlInitParser(); LIBXML_TEST_VERSION init_sysconf_vars(); // get program name - emulate basename program_name = argv[0]; size_t found = program_name.find_last_of('/'); if(found != std::string::npos){ program_name.replace(0, found+1, ""); } // set credential object // ps3fscred.reset(new S3fsCred()); if(!S3fsCurl::InitCredentialObject(ps3fscred.get())){ S3FS_PRN_EXIT("Failed to setup credential object to s3fs curl."); exit(EXIT_FAILURE); } while((ch = getopt_long(argc, argv, "dho:fsu", long_opts, &option_index)) != -1){ switch(ch){ case 0: if(strcmp(long_opts[option_index].name, "version") == 0){ show_version(); exit(EXIT_SUCCESS); } break; case 'h': show_help(); exit(EXIT_SUCCESS); case 'o': break; case 'd': break; case 'f': foreground = true; break; case 's': break; case 'u': // --incomplete-mpu-list if(utility_incomp_type::NO_UTILITY_MODE != utility_mode){ S3FS_PRN_EXIT("already utility mode option is specified."); exit(EXIT_FAILURE); } utility_mode = utility_incomp_type::INCOMP_TYPE_LIST; break; case 'a': // --incomplete-mpu-abort if(utility_incomp_type::NO_UTILITY_MODE != utility_mode){ S3FS_PRN_EXIT("already utility mode option is specified."); exit(EXIT_FAILURE); } utility_mode = utility_incomp_type::INCOMP_TYPE_ABORT; // check expire argument if(nullptr != optarg && 0 == strcasecmp(optarg, "all")){ // all is 0s incomp_abort_time = 0; }else if(nullptr != optarg){ if(!convert_unixtime_from_option_arg(optarg, incomp_abort_time)){ S3FS_PRN_EXIT("--incomplete-mpu-abort option argument is wrong."); exit(EXIT_FAILURE); } } // if optarg is null, incomp_abort_time is 24H(default) break; default: exit(EXIT_FAILURE); } } // print launch message print_launch_message(argc, argv); // Load SSE environment if(!S3fsCurl::LoadEnvSse()){ S3FS_PRN_EXIT("something wrong about SSE environment."); exit(EXIT_FAILURE); } // ssl init if(!s3fs_init_global_ssl()){ S3FS_PRN_EXIT("could not initialize for ssl libraries."); exit(EXIT_FAILURE); } // init curl (without mime types) // // [NOTE] // The curl initialization here does not load mime types. // The mime types file parameter are dynamic values according // to the user's environment, and are analyzed by the my_fuse_opt_proc // function. // The my_fuse_opt_proc function is executed after this curl // initialization. Because the curl method is used in the // my_fuse_opt_proc function, then it must be called here to // initialize. Fortunately, the processing using mime types // is only PUT/POST processing, and it is not used until the // call of my_fuse_opt_proc function is completed. Therefore, // the mime type is loaded just after calling the my_fuse_opt_proc // function. // if(!S3fsCurl::InitS3fsCurl()){ S3FS_PRN_EXIT("Could not initiate curl library."); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } // This is the fuse-style parser for the arguments // after which the bucket name and mountpoint names // should have been set struct fuse_args custom_args = FUSE_ARGS_INIT(argc, argv); if(0 != fuse_opt_parse(&custom_args, nullptr, nullptr, my_fuse_opt_proc)){ S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } // init mime types for curl if(!S3fsCurl::InitMimeType(mimetype_file)){ S3FS_PRN_WARN("Missing MIME types prevents setting Content-Type on uploaded objects."); } // [NOTE] // exclusive option check here. // if(strcasecmp(S3fsCurl::GetStorageClass().c_str(), "REDUCED_REDUNDANCY") == 0 && !S3fsCurl::IsSseDisable()){ S3FS_PRN_EXIT("use_sse option could not be specified with storage class reduced_redundancy."); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } if(!S3fsCurl::FinalCheckSse()){ S3FS_PRN_EXIT("something wrong about SSE options."); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } if(S3fsCurl::GetSignatureType() == signature_type_t::V2_ONLY && S3fsCurl::GetUnsignedPayload()){ S3FS_PRN_WARN("Ignoring enable_unsigned_payload with sigv2"); } if(!FdEntity::GetNoMixMultipart() && max_dirty_data != -1){ S3FS_PRN_WARN("Setting max_dirty_data to -1 when nomixupload is enabled"); max_dirty_data = -1; } // // Check the combination of parameters for credential // if(!ps3fscred->CheckAllParams()){ S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } // The second plain argument is the mountpoint // if the option was given, we all ready checked for a // readable, non-empty directory, this checks determines // if the mountpoint option was ever supplied if(utility_incomp_type::NO_UTILITY_MODE == utility_mode){ if(mountpoint.empty()){ S3FS_PRN_EXIT("missing MOUNTPOINT argument."); show_usage(); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } } // check tmp dir permission if(!FdManager::CheckTmpDirExist()){ S3FS_PRN_EXIT("temporary directory doesn't exists."); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } // check cache dir permission if(!FdManager::CheckCacheDirExist() || !FdManager::CheckCacheTopDir() || !CacheFileStat::CheckCacheFileStatTopDir()){ S3FS_PRN_EXIT("could not allow cache directory permission, check permission of cache directories."); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } // set user agent S3fsCurl::InitUserAgent(); // There's room for more command line error checking // Check to see if the bucket name contains periods and https (SSL) is // being used. This is a known limitation: // https://docs.amazonwebservices.com/AmazonS3/latest/dev/ // The Developers Guide suggests that either use HTTP of for us to write // our own certificate verification logic. // For now, this will be unsupported unless we get a request for it to // be supported. In that case, we have a couple of options: // - implement a command line option that bypasses the verify host // but doesn't bypass verifying the certificate // - write our own host verification (this might be complex) // See issue #128strncasecmp /* if(1 == S3fsCurl::GetSslVerifyHostname()){ found = S3fsCred::GetBucket().find_first_of('.'); if(found != std::string::npos){ found = s3host.find("https:"); if(found != std::string::npos){ S3FS_PRN_EXIT("Using https and a bucket name with periods is unsupported."); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } } } */ if(utility_incomp_type::NO_UTILITY_MODE != utility_mode){ int exitcode = s3fs_utility_processing(incomp_abort_time); S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(exitcode); } // Check multipart / copy api for mix multipart uploading if(nomultipart || nocopyapi || norenameapi){ FdEntity::SetNoMixMultipart(); max_dirty_data = -1; } // Free disk space if(-1 != fake_diskfree_size){ // Case: Set fake disk space // Not check free disk space for maultipart request FdManager::InitFakeUsedDiskSize(fake_diskfree_size); }else{ if(0 == FdManager::GetEnsureFreeDiskSpace()){ // Case: Not set any ensure disk free space // Set default value of free_space_ratio to 10% // int ratio = 10; off_t dfsize = FdManager::GetTotalDiskSpaceByRatio(ratio); S3FS_PRN_INFO("Free space ratio default to %d %%, ensure the available disk space is greater than %.3f MB", ratio, static_cast(dfsize) / 1024 / 1024); if(dfsize < S3fsCurl::GetMultipartSize()){ S3FS_PRN_WARN("specified size to ensure disk free space is smaller than multipart size, so set multipart size to it."); dfsize = S3fsCurl::GetMultipartSize(); } FdManager::SetEnsureFreeDiskSpace(dfsize); } // Check free disk space for maultipart request if(!FdManager::IsSafeDiskSpace(nullptr, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){ // Try to clean cache dir and retry S3FS_PRN_WARN("No enough disk space for s3fs, try to clean cache dir"); FdManager::get()->CleanupCacheDir(); if(!FdManager::IsSafeDiskSpace(nullptr, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount(), true)){ S3fsCurl::DestroyS3fsCurl(); s3fs_destroy_global_ssl(); exit(EXIT_FAILURE); } } } s3fs_oper.getattr = s3fs_getattr; s3fs_oper.readlink = s3fs_readlink; s3fs_oper.mknod = s3fs_mknod; s3fs_oper.mkdir = s3fs_mkdir; s3fs_oper.unlink = s3fs_unlink; s3fs_oper.rmdir = s3fs_rmdir; s3fs_oper.symlink = s3fs_symlink; s3fs_oper.rename = s3fs_rename; s3fs_oper.link = s3fs_link; if(!nocopyapi){ s3fs_oper.chmod = s3fs_chmod; s3fs_oper.chown = s3fs_chown; s3fs_oper.utimens = s3fs_utimens; }else{ s3fs_oper.chmod = s3fs_chmod_nocopy; s3fs_oper.chown = s3fs_chown_nocopy; s3fs_oper.utimens = s3fs_utimens_nocopy; } s3fs_oper.truncate = s3fs_truncate; s3fs_oper.open = s3fs_open; s3fs_oper.read = s3fs_read; s3fs_oper.write = s3fs_write; s3fs_oper.statfs = s3fs_statfs; s3fs_oper.flush = s3fs_flush; s3fs_oper.fsync = s3fs_fsync; s3fs_oper.release = s3fs_release; s3fs_oper.opendir = s3fs_opendir; s3fs_oper.readdir = s3fs_readdir; s3fs_oper.init = s3fs_init; s3fs_oper.destroy = s3fs_destroy; s3fs_oper.access = s3fs_access; s3fs_oper.create = s3fs_create; // extended attributes if(is_use_xattr){ s3fs_oper.setxattr = s3fs_setxattr; s3fs_oper.getxattr = s3fs_getxattr; s3fs_oper.listxattr = s3fs_listxattr; s3fs_oper.removexattr = s3fs_removexattr; } s3fs_oper.flag_utime_omit_ok = true; // now passing things off to fuse, fuse will finish evaluating the command line args fuse_res = fuse_main(custom_args.argc, custom_args.argv, &s3fs_oper, nullptr); if(fuse_res == 0){ fuse_res = s3fs_init_deferred_exit_status; } fuse_opt_free_args(&custom_args); // Destroy curl if(!S3fsCurl::DestroyS3fsCurl()){ S3FS_PRN_WARN("Could not release curl library."); } s3fs_destroy_global_ssl(); // cleanup xml2 xmlCleanupParser(); S3FS_MALLOCTRIM(0); exit(fuse_res); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs.h000066400000000000000000000043501470675423500151230ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_S3FS_H_ #define S3FS_S3FS_H_ #define FUSE_USE_VERSION 26 #include #define S3FS_FUSE_EXIT() \ do{ \ struct fuse_context* pcxt = fuse_get_context(); \ if(pcxt){ \ fuse_exit(pcxt->fuse); \ } \ }while(0) // [NOTE] // s3fs use many small allocated chunk in heap area for stats // cache and parsing xml, etc. The OS may decide that giving // this little memory back to the kernel will cause too much // overhead and delay the operation. // Address of gratitude, this workaround quotes a document of // libxml2.( http://xmlsoft.org/xmlmem.html ) // // When valgrind is used to test memory leak of s3fs, a large // amount of chunk may be reported. You can check the memory // release accurately by defining the S3FS_MALLOC_TRIM flag // and building it. Also, when executing s3fs, you can define // the MMAP_THRESHOLD environment variable and check more // accurate memory leak.( see, man 3 free ) // #ifdef S3FS_MALLOC_TRIM #ifdef HAVE_MALLOC_TRIM #include #define S3FS_MALLOCTRIM(pad) malloc_trim(pad) #else // HAVE_MALLOC_TRIM #define S3FS_MALLOCTRIM(pad) #endif // HAVE_MALLOC_TRIM #else // S3FS_MALLOC_TRIM #define S3FS_MALLOCTRIM(pad) #endif // S3FS_MALLOC_TRIM #endif // S3FS_S3FS_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_auth.h000066400000000000000000000043741470675423500161520ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_AUTH_H_ #define S3FS_AUTH_H_ #include #include #include #include typedef std::array md5_t; typedef std::array sha256_t; //------------------------------------------------------------------- // Utility functions for Authentication //------------------------------------------------------------------- // // in common_auth.cpp // std::string s3fs_get_content_md5(int fd); std::string s3fs_sha256_hex_fd(int fd, off_t start, off_t size); // // in xxxxxx_auth.cpp // const char* s3fs_crypt_lib_name(); bool s3fs_init_global_ssl(); bool s3fs_destroy_global_ssl(); bool s3fs_init_crypt_mutex(); bool s3fs_destroy_crypt_mutex(); std::unique_ptr s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen); std::unique_ptr s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned int* digestlen); bool s3fs_md5(const unsigned char* data, size_t datalen, md5_t* result); bool s3fs_md5_fd(int fd, off_t start, off_t size, md5_t* result); bool s3fs_sha256(const unsigned char* data, size_t datalen, sha256_t* digest); bool s3fs_sha256_fd(int fd, off_t start, off_t size, sha256_t* result); #endif // S3FS_AUTH_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_cred.cpp000066400000000000000000001425141470675423500164600ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include "common.h" #include "s3fs_cred.h" #include "s3fs_help.h" #include "s3fs_logger.h" #include "curl.h" #include "string_util.h" #include "metaheader.h" //------------------------------------------------------------------- // Symbols //------------------------------------------------------------------- static constexpr char DEFAULT_AWS_PROFILE_NAME[] = "default"; //------------------------------------------------------------------- // External Credential dummy function //------------------------------------------------------------------- // [NOTE] // This function expects the following values: // // detail=false ex. "Custom AWS Credential Library - v1.0.0" // detail=true ex. "Custom AWS Credential Library - v1.0.0 // s3fs-fuse credential I/F library for S3 compatible storage X. // Copyright(C) 2022 Foo" // const char* VersionS3fsCredential(bool detail) { static constexpr char version[] = "built-in"; static constexpr char detail_version[] = "s3fs-fuse built-in Credential I/F Function\n" "Copyright(C) 2007 s3fs-fuse\n"; if(detail){ return detail_version; }else{ return version; } } bool InitS3fsCredential(const char* popts, char** pperrstr) { if(popts && 0 < strlen(popts)){ S3FS_PRN_WARN("The external credential library does not have InitS3fsCredential function, but credlib_opts value is not empty(%s)", popts); } if(pperrstr){ *pperrstr = strdup("The external credential library does not have InitS3fsCredential function, so built-in function was called."); }else{ S3FS_PRN_INFO("The external credential library does not have InitS3fsCredential function, so built-in function was called."); } return true; } bool FreeS3fsCredential(char** pperrstr) { if(pperrstr){ *pperrstr = strdup("The external credential library does not have FreeS3fsCredential function, so built-in function was called."); }else{ S3FS_PRN_INFO("The external credential library does not have FreeS3fsCredential function, so built-in function was called."); } return true; } bool UpdateS3fsCredential(char** ppaccess_key_id, char** ppserect_access_key, char** ppaccess_token, long long* ptoken_expire, char** pperrstr) { S3FS_PRN_INFO("Parameters : ppaccess_key_id=%p, ppserect_access_key=%p, ppaccess_token=%p, ptoken_expire=%p", ppaccess_key_id, ppserect_access_key, ppaccess_token, ptoken_expire); if(pperrstr){ *pperrstr = strdup("Check why built-in function was called, the external credential library must have UpdateS3fsCredential function."); }else{ S3FS_PRN_CRIT("Check why built-in function was called, the external credential library must have UpdateS3fsCredential function."); } if(ppaccess_key_id){ *ppaccess_key_id = nullptr; } if(ppserect_access_key){ *ppserect_access_key = nullptr; } if(ppaccess_token){ *ppaccess_token = nullptr; } return false; // always false } //------------------------------------------------------------------- // Class Variables //------------------------------------------------------------------- constexpr char S3fsCred::ALLBUCKET_FIELDS_TYPE[]; constexpr char S3fsCred::KEYVAL_FIELDS_TYPE[]; constexpr char S3fsCred::AWS_ACCESSKEYID[]; constexpr char S3fsCred::AWS_SECRETKEY[]; constexpr char S3fsCred::ECS_IAM_ENV_VAR[]; constexpr char S3fsCred::IAMCRED_ACCESSKEYID[]; constexpr char S3fsCred::IAMCRED_SECRETACCESSKEY[]; constexpr char S3fsCred::IAMCRED_ROLEARN[]; constexpr char S3fsCred::IAMv2_token_url[]; constexpr char S3fsCred::IAMv2_token_ttl_hdr[]; constexpr char S3fsCred::IAMv2_token_hdr[]; std::string S3fsCred::bucket_name; //------------------------------------------------------------------- // Class Methods //------------------------------------------------------------------- bool S3fsCred::SetBucket(const std::string& bucket) { if(bucket.empty()){ return false; } S3fsCred::bucket_name = bucket; return true; } const std::string& S3fsCred::GetBucket() { return S3fsCred::bucket_name; } bool S3fsCred::ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename) { if(!response){ return false; } // [NOTE] // expected following strings. // // myrolename // std::istringstream ssrole(response); std::string oneline; if (getline(ssrole, oneline, '\n')){ rolename = oneline; return !rolename.empty(); } return false; } //------------------------------------------------------------------- // Methods : Constructor / Destructor //------------------------------------------------------------------- S3fsCred::S3fsCred() : aws_profile(DEFAULT_AWS_PROFILE_NAME), load_iamrole(false), AWSAccessTokenExpire(0), is_ecs(false), is_use_session_token(false), is_ibm_iam_auth(false), IAM_cred_url("http://169.254.169.254/latest/meta-data/iam/security-credentials/"), IAM_api_version(2), IAM_field_count(4), IAM_token_field("Token"), IAM_expiry_field("Expiration"), set_builtin_cred_opts(false), hExtCredLib(nullptr), pFuncCredVersion(VersionS3fsCredential), pFuncCredInit(InitS3fsCredential), pFuncCredFree(FreeS3fsCredential), pFuncCredUpdate(UpdateS3fsCredential) { } S3fsCred::~S3fsCred() { UnloadExtCredLib(); } //------------------------------------------------------------------- // Methods : Access member variables //------------------------------------------------------------------- bool S3fsCred::SetS3fsPasswdFile(const char* file) { if(!file || strlen(file) == 0){ return false; } passwd_file = file; return true; } bool S3fsCred::IsSetPasswdFile() const { return !passwd_file.empty(); } bool S3fsCred::SetAwsProfileName(const char* name) { if(!name || strlen(name) == 0){ return false; } aws_profile = name; return true; } bool S3fsCred::SetIAMRoleMetadataType(bool flag) { bool old = load_iamrole; load_iamrole = flag; return old; } bool S3fsCred::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey) { if((!is_ibm_iam_auth && (!AccessKeyId || '\0' == AccessKeyId[0])) || !SecretAccessKey || '\0' == SecretAccessKey[0]){ return false; } AWSAccessKeyId = AccessKeyId; AWSSecretAccessKey = SecretAccessKey; return true; } bool S3fsCred::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken) { bool access_key_is_empty = AccessKeyId == nullptr || '\0' == AccessKeyId[0]; bool secret_access_key_is_empty = SecretAccessKey == nullptr || '\0' == SecretAccessKey[0]; bool session_token_is_empty = SessionToken == nullptr || '\0' == SessionToken[0]; if((!is_ibm_iam_auth && access_key_is_empty) || secret_access_key_is_empty || session_token_is_empty){ return false; } AWSAccessKeyId = AccessKeyId; AWSSecretAccessKey = SecretAccessKey; AWSAccessToken = SessionToken; is_use_session_token= true; return true; } bool S3fsCred::IsSetAccessKeys() const { return IsSetIAMRole() || ((!AWSAccessKeyId.empty() || is_ibm_iam_auth) && !AWSSecretAccessKey.empty()); } bool S3fsCred::SetIsECS(bool flag) { bool old = is_ecs; is_ecs = flag; return old; } bool S3fsCred::SetIsUseSessionToken(bool flag) { bool old = is_use_session_token; is_use_session_token = flag; return old; } bool S3fsCred::SetIsIBMIAMAuth(bool flag) { bool old = is_ibm_iam_auth; is_ibm_iam_auth = flag; return old; } bool S3fsCred::SetIAMRole(const char* role) { IAM_role = role ? role : ""; return true; } const std::string& S3fsCred::GetIAMRole() const { return IAM_role; } bool S3fsCred::IsSetIAMRole() const { return !IAM_role.empty(); } size_t S3fsCred::SetIAMFieldCount(size_t field_count) { size_t old = IAM_field_count; IAM_field_count = field_count; return old; } std::string S3fsCred::SetIAMCredentialsURL(const char* url) { std::string old = IAM_cred_url; IAM_cred_url = url ? url : ""; return old; } std::string S3fsCred::SetIAMTokenField(const char* token_field) { std::string old = IAM_token_field; IAM_token_field = token_field ? token_field : ""; return old; } std::string S3fsCred::SetIAMExpiryField(const char* expiry_field) { std::string old = IAM_expiry_field; IAM_expiry_field = expiry_field ? expiry_field : ""; return old; } bool S3fsCred::GetIAMCredentialsURL(std::string& url, bool check_iam_role) { // check if(check_iam_role && !is_ecs && !IsIBMIAMAuth()){ if(!IsSetIAMRole()) { S3FS_PRN_ERR("IAM role name is empty."); return false; } S3FS_PRN_INFO3("[IAM role=%s]", GetIAMRole().c_str()); } if(is_ecs){ const char *env = std::getenv(S3fsCred::ECS_IAM_ENV_VAR); if(env == nullptr){ S3FS_PRN_ERR("%s is not set.", S3fsCred::ECS_IAM_ENV_VAR); return false; } url = IAM_cred_url + env; }else if(IsIBMIAMAuth()){ url = IAM_cred_url; }else{ // [NOTE] // To avoid deadlocking, do not manipulate the S3fsCred object // in the S3fsCurl::GetIAMv2ApiToken method (when retrying). // if(GetIMDSVersion() > 1){ S3fsCurl s3fscurl; std::string token; int result = s3fscurl.GetIAMv2ApiToken(S3fsCred::IAMv2_token_url, S3fsCred::IAMv2_token_ttl, S3fsCred::IAMv2_token_ttl_hdr, token); if(-ENOENT == result){ // If we get a 404 back when requesting the token service, // then it's highly likely we're running in an environment // that doesn't support the AWS IMDSv2 API, so we'll skip // the token retrieval in the future. SetIMDSVersion(1); }else if(result != 0){ // If we get an unexpected error when retrieving the API // token, log it but continue. Requirement for including // an API token with the metadata request may or may not // be required, so we should not abort here. S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result); }else{ // Set token if(!SetIAMv2APIToken(token)){ S3FS_PRN_ERR("Error storing IMDSv2 API token(%s).", token.c_str()); } } } if(check_iam_role){ url = IAM_cred_url + GetIAMRole(); }else{ url = IAM_cred_url; } } return true; } int S3fsCred::SetIMDSVersion(int version) { int old = IAM_api_version; IAM_api_version = version; return old; } int S3fsCred::GetIMDSVersion() const { return IAM_api_version; } bool S3fsCred::SetIAMv2APIToken(const std::string& token) { S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", token.c_str()); if(token.empty()){ return false; } IAMv2_api_token = token; return true; } const std::string& S3fsCred::GetIAMv2APIToken() const { return IAMv2_api_token; } // [NOTE] // Currently, token_lock is always locked before calling this method, // and this method calls the S3fsCurl::GetIAMCredentials method. // Currently, when the request fails and retries in the process of // S3fsCurl::GetIAMCredentials, does not use the S3fsCred object in // retry logic. // Be careful not to deadlock whenever you change this logic. // bool S3fsCred::LoadIAMCredentials() { // url(check iam role) std::string url; if(!GetIAMCredentialsURL(url, true)){ return false; } const char* iam_v2_token = nullptr; std::string str_iam_v2_token; if(GetIMDSVersion() > 1){ str_iam_v2_token = GetIAMv2APIToken(); iam_v2_token = str_iam_v2_token.c_str(); } const char* ibm_secret_access_key = nullptr; std::string str_ibm_secret_access_key; if(IsIBMIAMAuth()){ str_ibm_secret_access_key = AWSSecretAccessKey; ibm_secret_access_key = str_ibm_secret_access_key.c_str(); } S3fsCurl s3fscurl; std::string response; if(!s3fscurl.GetIAMCredentials(url.c_str(), iam_v2_token, ibm_secret_access_key, response)){ return false; } if(!SetIAMCredentials(response.c_str())){ S3FS_PRN_ERR("Something error occurred, could not set IAM role name."); return false; } return true; } // // load IAM role name from http://169.254.169.254/latest/meta-data/iam/security-credentials // bool S3fsCred::LoadIAMRoleFromMetaData() { const std::lock_guard lock(token_lock); if(load_iamrole){ // url(not check iam role) std::string url; if(!GetIAMCredentialsURL(url, false)){ return false; } const char* iam_v2_token = nullptr; std::string str_iam_v2_token; if(GetIMDSVersion() > 1){ str_iam_v2_token = GetIAMv2APIToken(); iam_v2_token = str_iam_v2_token.c_str(); } S3fsCurl s3fscurl; std::string token; if(!s3fscurl.GetIAMRoleFromMetaData(url.c_str(), iam_v2_token, token)){ return false; } if(!SetIAMRoleFromMetaData(token.c_str())){ S3FS_PRN_ERR("Something error occurred, could not set IAM role name."); return false; } S3FS_PRN_INFO("loaded IAM role name = %s", GetIAMRole().c_str()); } return true; } bool S3fsCred::SetIAMCredentials(const char* response) { S3FS_PRN_INFO3("IAM credential response = \"%s\"", response); iamcredmap_t keyval; if(!ParseIAMCredentialResponse(response, keyval)){ return false; } if(IAM_field_count != keyval.size()){ return false; } AWSAccessToken = keyval[IAM_token_field]; if(is_ibm_iam_auth){ off_t tmp_expire = 0; if(!s3fs_strtoofft(&tmp_expire, keyval[IAM_expiry_field].c_str(), /*base=*/ 10)){ return false; } AWSAccessTokenExpire = static_cast(tmp_expire); }else{ AWSAccessKeyId = keyval[S3fsCred::IAMCRED_ACCESSKEYID]; AWSSecretAccessKey = keyval[S3fsCred::IAMCRED_SECRETACCESSKEY]; AWSAccessTokenExpire = cvtIAMExpireStringToTime(keyval[IAM_expiry_field].c_str()); } return true; } bool S3fsCred::SetIAMRoleFromMetaData(const char* response) { S3FS_PRN_INFO3("IAM role name response = \"%s\"", response ? response : "(null)"); std::string rolename; if(!S3fsCred::ParseIAMRoleFromMetaDataResponse(response, rolename)){ return false; } SetIAMRole(rolename.c_str()); return true; } //------------------------------------------------------------------- // Methods : for Credentials //------------------------------------------------------------------- // // Check passwd file readable // bool S3fsCred::IsReadableS3fsPasswdFile() const { if(passwd_file.empty()){ return false; } std::ifstream PF(passwd_file.c_str()); if(!PF.good()){ return false; } PF.close(); return true; } // // S3fsCred::CheckS3fsPasswdFilePerms // // expect that global passwd_file variable contains // a non-empty value and is readable by the current user // // Check for too permissive access to the file // help save users from themselves via a security hole // // only two options: return or error out // bool S3fsCred::CheckS3fsPasswdFilePerms() { struct stat info; // let's get the file info if(stat(passwd_file.c_str(), &info) != 0){ S3FS_PRN_EXIT("unexpected error from stat(%s): %s", passwd_file.c_str(), strerror(errno)); return false; } // Check readable if(!IsReadableS3fsPasswdFile()){ S3FS_PRN_EXIT("S3fs passwd file \"%s\" is not readable.", passwd_file.c_str()); return false; } // return error if any file has others permissions if( (info.st_mode & S_IROTH) || (info.st_mode & S_IWOTH) || (info.st_mode & S_IXOTH)) { S3FS_PRN_EXIT("credentials file %s should not have others permissions.", passwd_file.c_str()); return false; } // Any local file should not have any group permissions // /etc/passwd-s3fs can have group permissions if(passwd_file != "/etc/passwd-s3fs"){ if( (info.st_mode & S_IRGRP) || (info.st_mode & S_IWGRP) || (info.st_mode & S_IXGRP)) { S3FS_PRN_EXIT("credentials file %s should not have group permissions.", passwd_file.c_str()); return false; } }else{ // "/etc/passwd-s3fs" does not allow group write. if((info.st_mode & S_IWGRP)){ S3FS_PRN_EXIT("credentials file %s should not have group writable permissions.", passwd_file.c_str()); return false; } } if((info.st_mode & S_IXUSR) || (info.st_mode & S_IXGRP)){ S3FS_PRN_EXIT("credentials file %s should not have executable permissions.", passwd_file.c_str()); return false; } return true; } // // Read and Parse passwd file // // The line of the password file is one of the following formats: // (1) "accesskey:secretkey" : AWS format for default(all) access key/secret key // (2) "bucket:accesskey:secretkey" : AWS format for bucket's access key/secret key // (3) "key=value" : Content-dependent KeyValue contents // // This function sets result into bucketkvmap_t, it bucket name and key&value mapping. // If bucket name is empty(1 or 3 format), bucket name for mapping is set "\t" or "". // // Return: true - Succeed parsing // false - Should shutdown immediately // bool S3fsCred::ParseS3fsPasswdFile(bucketkvmap_t& resmap) { std::string line; size_t first_pos; readline_t linelist; // open passwd file std::ifstream PF(passwd_file.c_str()); if(!PF.good()){ S3FS_PRN_EXIT("could not open passwd file : %s", passwd_file.c_str()); return false; } // read each line while(getline(PF, line)){ line = trim(line); if(line.empty()){ continue; } if('#' == line[0]){ continue; } if(std::string::npos != line.find_first_of(" \t")){ S3FS_PRN_EXIT("invalid line in passwd file, found whitespace character."); return false; } if('[' == line[0]){ S3FS_PRN_EXIT("invalid line in passwd file, found a bracket \"[\" character."); return false; } linelist.push_back(line); } // read '=' type kvmap_t kv; for(auto iter = linelist.cbegin(); iter != linelist.cend(); ++iter){ first_pos = iter->find_first_of('='); if(first_pos == std::string::npos){ continue; } // formatted by "key=val" std::string key = trim(iter->substr(0, first_pos)); std::string val = trim(iter->substr(first_pos + 1, std::string::npos)); if(key.empty()){ continue; } if(kv.cend() != kv.find(key)){ S3FS_PRN_WARN("same key name(%s) found in passwd file, skip this.", key.c_str()); continue; } kv[key] = val; } // set special key name resmap[S3fsCred::KEYVAL_FIELDS_TYPE] = kv; // read ':' type for(auto iter = linelist.cbegin(); iter != linelist.cend(); ++iter){ first_pos = iter->find_first_of(':'); size_t last_pos = iter->find_last_of(':'); if(first_pos == std::string::npos){ continue; } std::string bucketname; std::string accesskey; std::string secret; if(first_pos != last_pos){ // formatted by "bucket:accesskey:secretkey" bucketname= trim(iter->substr(0, first_pos)); accesskey = trim(iter->substr(first_pos + 1, last_pos - first_pos - 1)); secret = trim(iter->substr(last_pos + 1, std::string::npos)); }else{ // formatted by "accesskey:secretkey" bucketname= S3fsCred::ALLBUCKET_FIELDS_TYPE; accesskey = trim(iter->substr(0, first_pos)); secret = trim(iter->substr(first_pos + 1, std::string::npos)); } if(resmap.cend() != resmap.find(bucketname)){ S3FS_PRN_EXIT("there are multiple entries for the same bucket(%s) in the passwd file.", (bucketname.empty() ? "default" : bucketname.c_str())); return false; } kv.clear(); kv[S3fsCred::AWS_ACCESSKEYID] = accesskey; kv[S3fsCred::AWS_SECRETKEY] = secret; resmap[bucketname] = kv; } return true; } // // ReadS3fsPasswdFile // // Support for per bucket credentials // // Format for the credentials file: // [bucket:]AccessKeyId:SecretAccessKey // // Lines beginning with # are considered comments // and ignored, as are empty lines // // Uncommented lines without the ":" character are flagged as // an error, so are lines with spaces or tabs // // only one default key pair is allowed, but not required // bool S3fsCred::ReadS3fsPasswdFile() { bucketkvmap_t bucketmap; kvmap_t keyval; // if you got here, the password file // exists and is readable by the // current user, check for permissions if(!CheckS3fsPasswdFilePerms()){ return false; } // // parse passwd file // if(!ParseS3fsPasswdFile(bucketmap)){ return false; } // // check key=value type format. // auto it = bucketmap.find(S3fsCred::KEYVAL_FIELDS_TYPE); if(bucketmap.cend() != it){ // aws format std::string access_key_id; std::string secret_access_key; int result = CheckS3fsCredentialAwsFormat(it->second, access_key_id, secret_access_key); if(-1 == result){ return false; }else if(1 == result){ // found ascess(secret) keys if(!SetAccessKey(access_key_id.c_str(), secret_access_key.c_str())){ S3FS_PRN_EXIT("failed to set access key/secret key."); return false; } return true; } } std::string bucket_key = S3fsCred::ALLBUCKET_FIELDS_TYPE; if(!S3fsCred::bucket_name.empty() && bucketmap.cend() != bucketmap.find(S3fsCred::bucket_name)){ bucket_key = S3fsCred::bucket_name; } it = bucketmap.find(bucket_key); if(bucketmap.cend() == it){ S3FS_PRN_EXIT("Not found access key/secret key in passwd file."); return false; } keyval = it->second; auto aws_accesskeyid_it = keyval.find(S3fsCred::AWS_ACCESSKEYID); auto aws_secretkey_it = keyval.find(S3fsCred::AWS_SECRETKEY); if(keyval.cend() == aws_accesskeyid_it || keyval.end() == aws_secretkey_it){ S3FS_PRN_EXIT("Not found access key/secret key in passwd file."); return false; } if(!SetAccessKey(aws_accesskeyid_it->second.c_str(), aws_secretkey_it->second.c_str())){ S3FS_PRN_EXIT("failed to set internal data for access key/secret key from passwd file."); return false; } return true; } // // Return: 1 - OK(could read and set accesskey etc.) // 0 - NG(could not read) // -1 - Should shutdown immediately // int S3fsCred::CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap, std::string& access_key_id, std::string& secret_access_key) { std::string str1(S3fsCred::AWS_ACCESSKEYID); std::string str2(S3fsCred::AWS_SECRETKEY); if(kvmap.empty()){ return 0; } auto str1_it = kvmap.find(str1); auto str2_it = kvmap.find(str2); if(kvmap.cend() == str1_it && kvmap.end() == str2_it){ return 0; } if(kvmap.cend() == str1_it || kvmap.end() == str2_it){ S3FS_PRN_EXIT("AWSAccesskey or AWSSecretkey is not specified."); return -1; } access_key_id = str1_it->second; secret_access_key = str2_it->second; return 1; } // // Read Aws Credential File // bool S3fsCred::ReadAwsCredentialFile(const std::string &filename) { // open passwd file std::ifstream PF(filename.c_str()); if(!PF.good()){ return false; } std::string profile; std::string accesskey; std::string secret; std::string session_token; // read each line std::string line; while(getline(PF, line)){ line = trim(line); if(line.empty()){ continue; } if('#' == line[0]){ continue; } if(line.size() > 2 && line[0] == '[' && line[line.size() - 1] == ']') { if(profile == aws_profile){ break; } profile = line.substr(1, line.size() - 2); accesskey.clear(); secret.clear(); session_token.clear(); } size_t pos = line.find_first_of('='); if(pos == std::string::npos){ continue; } std::string key = trim(line.substr(0, pos)); std::string value = trim(line.substr(pos + 1, std::string::npos)); if(key == "aws_access_key_id"){ accesskey = value; }else if(key == "aws_secret_access_key"){ secret = value; }else if(key == "aws_session_token"){ session_token = value; } } if(profile != aws_profile){ return false; } if(session_token.empty()){ if(is_use_session_token){ S3FS_PRN_EXIT("AWS session token was expected but wasn't provided in aws/credentials file for profile: %s.", aws_profile.c_str()); return false; } if(!SetAccessKey(accesskey.c_str(), secret.c_str())){ S3FS_PRN_EXIT("failed to set internal data for access key/secret key from aws credential file."); return false; } }else{ if(!SetAccessKeyWithSessionToken(accesskey.c_str(), secret.c_str(), session_token.c_str())){ S3FS_PRN_EXIT("session token is invalid."); return false; } } return true; } // // InitialS3fsCredentials // // called only when were are not mounting a // public bucket // // Here is the order precedence for getting the // keys: // // 1 - from the command line (security risk) // 2 - from a password file specified on the command line // 3 - from environment variables // 3a - from the AWS_CREDENTIAL_FILE environment variable // 3b - from ${HOME}/.aws/credentials // 4 - from the users ~/.passwd-s3fs // 5 - from /etc/passwd-s3fs // bool S3fsCred::InitialS3fsCredentials() { // should be redundant if(S3fsCurl::IsPublicBucket()){ return true; } // access key loading is deferred if(load_iamrole || IsSetExtCredLib() || is_ecs){ return true; } // 1 - keys specified on the command line if(IsSetAccessKeys()){ return true; } // 2 - was specified on the command line if(IsSetPasswdFile()){ if(!ReadS3fsPasswdFile()){ return false; } return true; } // 3 - environment variables const char* AWSACCESSKEYID = getenv("AWS_ACCESS_KEY_ID") ? getenv("AWS_ACCESS_KEY_ID") : getenv("AWSACCESSKEYID"); const char* AWSSECRETACCESSKEY = getenv("AWS_SECRET_ACCESS_KEY") ? getenv("AWS_SECRET_ACCESS_KEY") : getenv("AWSSECRETACCESSKEY"); const char* AWSSESSIONTOKEN = getenv("AWS_SESSION_TOKEN") ? getenv("AWS_SESSION_TOKEN") : getenv("AWSSESSIONTOKEN"); if(AWSACCESSKEYID != nullptr || AWSSECRETACCESSKEY != nullptr){ if( (AWSACCESSKEYID == nullptr && AWSSECRETACCESSKEY != nullptr) || (AWSACCESSKEYID != nullptr && AWSSECRETACCESSKEY == nullptr) ){ S3FS_PRN_EXIT("both environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set together."); return false; } S3FS_PRN_INFO2("access key from env variables"); if(AWSSESSIONTOKEN != nullptr){ S3FS_PRN_INFO2("session token is available"); if(!SetAccessKeyWithSessionToken(AWSACCESSKEYID, AWSSECRETACCESSKEY, AWSSESSIONTOKEN)){ S3FS_PRN_EXIT("session token is invalid."); return false; } }else{ S3FS_PRN_INFO2("session token is not available"); if(is_use_session_token){ S3FS_PRN_EXIT("environment variable AWS_SESSION_TOKEN is expected to be set."); return false; } } if(!SetAccessKey(AWSACCESSKEYID, AWSSECRETACCESSKEY)){ S3FS_PRN_EXIT("if one access key is specified, both keys need to be specified."); return false; } return true; } // 3a - from the AWS_CREDENTIAL_FILE environment variable char* AWS_CREDENTIAL_FILE = getenv("AWS_CREDENTIAL_FILE"); if(AWS_CREDENTIAL_FILE != nullptr){ passwd_file = AWS_CREDENTIAL_FILE; if(IsSetPasswdFile()){ if(!IsReadableS3fsPasswdFile()){ S3FS_PRN_EXIT("AWS_CREDENTIAL_FILE: \"%s\" is not readable.", passwd_file.c_str()); return false; } if(!ReadS3fsPasswdFile()){ return false; } return true; } } // 3b - check ${HOME}/.aws/credentials std::string aws_credentials = std::string(getpwuid(getuid())->pw_dir) + "/.aws/credentials"; if(ReadAwsCredentialFile(aws_credentials)){ return true; }else if(aws_profile != DEFAULT_AWS_PROFILE_NAME){ S3FS_PRN_EXIT("Could not find profile: %s in file: %s", aws_profile.c_str(), aws_credentials.c_str()); return false; } // 4 - from the default location in the users home directory char* HOME = getenv("HOME"); if(HOME != nullptr){ passwd_file = HOME; passwd_file += "/.passwd-s3fs"; if(IsReadableS3fsPasswdFile()){ if(!ReadS3fsPasswdFile()){ return false; } // It is possible that the user's file was there but // contained no key pairs i.e. commented out // in that case, go look in the final location if(IsSetAccessKeys()){ return true; } } } // 5 - from the system default location passwd_file = "/etc/passwd-s3fs"; if(IsReadableS3fsPasswdFile()){ if(!ReadS3fsPasswdFile()){ return false; } return true; } S3FS_PRN_EXIT("could not determine how to establish security credentials."); return false; } //------------------------------------------------------------------- // Methods : for IAM //------------------------------------------------------------------- bool S3fsCred::ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval) { if(!response){ return false; } std::istringstream sscred(response); std::string oneline; keyval.clear(); while(getline(sscred, oneline, ',')){ std::string::size_type pos; std::string key; std::string val; if(std::string::npos != (pos = oneline.find(S3fsCred::IAMCRED_ACCESSKEYID))){ key = S3fsCred::IAMCRED_ACCESSKEYID; }else if(std::string::npos != (pos = oneline.find(S3fsCred::IAMCRED_SECRETACCESSKEY))){ key = S3fsCred::IAMCRED_SECRETACCESSKEY; }else if(std::string::npos != (pos = oneline.find(IAM_token_field))){ key = IAM_token_field; }else if(std::string::npos != (pos = oneline.find(IAM_expiry_field))){ key = IAM_expiry_field; }else if(std::string::npos != (pos = oneline.find(S3fsCred::IAMCRED_ROLEARN))){ key = S3fsCred::IAMCRED_ROLEARN; }else{ continue; } if(std::string::npos == (pos = oneline.find(':', pos + key.length()))){ continue; } if(is_ibm_iam_auth && key == IAM_expiry_field){ // parse integer value if(std::string::npos == (pos = oneline.find_first_of("0123456789", pos))){ continue; } oneline.erase(0, pos); if(std::string::npos == (pos = oneline.find_last_of("0123456789"))){ continue; } val = oneline.substr(0, pos+1); }else{ // parse std::string value (starts and ends with quotes) if(std::string::npos == (pos = oneline.find('\"', pos))){ continue; } oneline.erase(0, pos+1); if(std::string::npos == (pos = oneline.find('\"'))){ continue; } val = oneline.substr(0, pos); } keyval[key] = val; } return true; } bool S3fsCred::CheckIAMCredentialUpdate(std::string* access_key_id, std::string* secret_access_key, std::string* access_token) { const std::lock_guard lock(token_lock); if(IsIBMIAMAuth() || IsSetExtCredLib() || is_ecs || IsSetIAMRole()){ if(AWSAccessTokenExpire < (time(nullptr) + S3fsCred::IAM_EXPIRE_MERGING)){ S3FS_PRN_INFO("IAM Access Token refreshing..."); // update if(!IsSetExtCredLib()){ if(!LoadIAMCredentials()){ S3FS_PRN_ERR("Access Token refresh by built-in failed"); return false; } }else{ if(!UpdateExtCredentials()){ S3FS_PRN_ERR("Access Token refresh by %s(external credential library) failed", credlib.c_str()); return false; } } S3FS_PRN_INFO("IAM Access Token refreshed"); } } // set if(access_key_id){ *access_key_id = AWSAccessKeyId; } if(secret_access_key){ *secret_access_key = AWSSecretAccessKey; } if(access_token){ if(IsIBMIAMAuth() || IsSetExtCredLib() || is_ecs || is_use_session_token || IsSetIAMRole()){ *access_token = AWSAccessToken; }else{ access_token->clear(); } } return true; } const char* S3fsCred::GetCredFuncVersion(bool detail) const { static constexpr char errVersion[] = "unknown"; if(!pFuncCredVersion){ return errVersion; } return (*pFuncCredVersion)(detail); } //------------------------------------------------------------------- // Methods : External Credential Library //------------------------------------------------------------------- bool S3fsCred::SetExtCredLib(const char* arg) { if(!arg || strlen(arg) == 0){ return false; } credlib = arg; return true; } bool S3fsCred::IsSetExtCredLib() const { return !credlib.empty(); } bool S3fsCred::SetExtCredLibOpts(const char* args) { if(!args || strlen(args) == 0){ return false; } credlib_opts = args; return true; } bool S3fsCred::IsSetExtCredLibOpts() const { return !credlib_opts.empty(); } bool S3fsCred::InitExtCredLib() { if(!LoadExtCredLib()){ return false; } // Initialize library if(!pFuncCredInit){ S3FS_PRN_CRIT("\"InitS3fsCredential\" function pointer is nullptr, why?"); UnloadExtCredLib(); return false; } const char* popts = credlib_opts.empty() ? nullptr : credlib_opts.c_str(); char* perrstr = nullptr; if(!(*pFuncCredInit)(popts, &perrstr)){ S3FS_PRN_ERR("Could not initialize %s(external credential library) by \"InitS3fsCredential\" function : %s", credlib.c_str(), perrstr ? perrstr : "unknown"); // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(perrstr){ free(perrstr); } UnloadExtCredLib(); return false; } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(perrstr){ free(perrstr); } return true; } bool S3fsCred::LoadExtCredLib() { if(credlib.empty()){ return false; } UnloadExtCredLib(); S3FS_PRN_INFO("Load External Credential Library : %s", credlib.c_str()); // Open Library // // Search Library: (RPATH ->) LD_LIBRARY_PATH -> (RUNPATH ->) /etc/ld.so.cache -> /lib -> /usr/lib // if(nullptr == (hExtCredLib = dlopen(credlib.c_str(), RTLD_LAZY))){ const char* preason = dlerror(); S3FS_PRN_ERR("Could not load %s(external credential library) by error : %s", credlib.c_str(), preason ? preason : "unknown"); return false; } // Set function pointers if(nullptr == (pFuncCredVersion = reinterpret_cast(dlsym(hExtCredLib, "VersionS3fsCredential")))){ S3FS_PRN_ERR("%s(external credential library) does not have \"VersionS3fsCredential\" function which is required.", credlib.c_str()); UnloadExtCredLib(); return false; } if(nullptr == (pFuncCredUpdate = reinterpret_cast(dlsym(hExtCredLib, "UpdateS3fsCredential")))){ S3FS_PRN_ERR("%s(external credential library) does not have \"UpdateS3fsCredential\" function which is required.", credlib.c_str()); UnloadExtCredLib(); return false; } if(nullptr == (pFuncCredInit = reinterpret_cast(dlsym(hExtCredLib, "InitS3fsCredential")))){ S3FS_PRN_INFO("%s(external credential library) does not have \"InitS3fsCredential\" function which is optional.", credlib.c_str()); pFuncCredInit = InitS3fsCredential; // set built-in function } if(nullptr == (pFuncCredFree = reinterpret_cast(dlsym(hExtCredLib, "FreeS3fsCredential")))){ S3FS_PRN_INFO("%s(external credential library) does not have \"FreeS3fsCredential\" function which is optional.", credlib.c_str()); pFuncCredFree = FreeS3fsCredential; // set built-in function } S3FS_PRN_INFO("Succeed loading External Credential Library : %s", credlib.c_str()); return true; } bool S3fsCred::UnloadExtCredLib() { if(hExtCredLib){ S3FS_PRN_INFO("Unload External Credential Library : %s", credlib.c_str()); // Uninitialize library if(!pFuncCredFree){ S3FS_PRN_CRIT("\"FreeS3fsCredential\" function pointer is nullptr, why?"); }else{ char* perrstr = nullptr; if(!(*pFuncCredFree)(&perrstr)){ S3FS_PRN_ERR("Could not uninitialize by \"FreeS3fsCredential\" function : %s", perrstr ? perrstr : "unknown"); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(perrstr){ free(perrstr); } } // reset pFuncCredVersion = VersionS3fsCredential; pFuncCredInit = InitS3fsCredential; pFuncCredFree = FreeS3fsCredential; pFuncCredUpdate = UpdateS3fsCredential; // close dlclose(hExtCredLib); hExtCredLib = nullptr; } return true; } bool S3fsCred::UpdateExtCredentials() { if(!hExtCredLib){ S3FS_PRN_CRIT("External Credential Library is not loaded, why?"); return false; } char* paccess_key_id = nullptr; char* pserect_access_key = nullptr; char* paccess_token = nullptr; char* perrstr = nullptr; long long token_expire = 0; bool result = (*pFuncCredUpdate)(&paccess_key_id, &pserect_access_key, &paccess_token, &token_expire, &perrstr); if(!result){ // error occurred S3FS_PRN_ERR("Could not update credential by \"UpdateS3fsCredential\" function : %s", perrstr ? perrstr : "unknown"); // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse }else if(!paccess_key_id || !pserect_access_key || !paccess_token || token_expire <= 0){ // some variables are wrong S3FS_PRN_ERR("After updating credential by \"UpdateS3fsCredential\" function, but some variables are wrong : paccess_key_id=%p, pserect_access_key=%p, paccess_token=%p, token_expire=%lld", paccess_key_id, pserect_access_key, paccess_token, token_expire); result = false; }else{ // succeed updating AWSAccessKeyId = paccess_key_id; AWSSecretAccessKey = pserect_access_key; AWSAccessToken = paccess_token; AWSAccessTokenExpire = token_expire; } // clean // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(paccess_key_id){ free(paccess_key_id); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(pserect_access_key){ free(pserect_access_key); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(paccess_token){ free(paccess_token); } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(perrstr){ free(perrstr); } return result; } //------------------------------------------------------------------- // Methods: Option detection //------------------------------------------------------------------- // return value: 1 = Not processed as it is not a option for this class // 0 = The option was detected and processed appropriately // -1 = Processing cannot be continued because a fatal error was detected // int S3fsCred::DetectParam(const char* arg) { const std::lock_guard lock(token_lock); if(!arg){ S3FS_PRN_EXIT("parameter arg is empty(null)"); return -1; } if(is_prefix(arg, "passwd_file=")){ SetS3fsPasswdFile(strchr(arg, '=') + sizeof(char)); set_builtin_cred_opts = true; return 0; } if(0 == strcmp(arg, "ibm_iam_auth")){ SetIsIBMIAMAuth(true); SetIAMCredentialsURL("https://iam.cloud.ibm.com/identity/token"); SetIAMTokenField("\"access_token\""); SetIAMExpiryField("\"expiration\""); SetIAMFieldCount(2); SetIMDSVersion(1); set_builtin_cred_opts = true; return 0; } if(0 == strcmp(arg, "use_session_token")){ SetIsUseSessionToken(true); set_builtin_cred_opts = true; return 0; } if(is_prefix(arg, "ibm_iam_endpoint=")){ std::string endpoint_url; const char* iam_endpoint = strchr(arg, '=') + sizeof(char); // Check url for http / https protocol std::string if(!is_prefix(iam_endpoint, "https://") && !is_prefix(iam_endpoint, "http://")){ S3FS_PRN_EXIT("option ibm_iam_endpoint has invalid format, missing http / https protocol"); return -1; } endpoint_url = std::string(iam_endpoint) + "/identity/token"; SetIAMCredentialsURL(endpoint_url.c_str()); set_builtin_cred_opts = true; return 0; } if(0 == strcmp(arg, "imdsv1only")){ SetIMDSVersion(1); set_builtin_cred_opts = true; return 0; } if(0 == strcmp(arg, "ecs")){ if(IsIBMIAMAuth()){ S3FS_PRN_EXIT("option ecs cannot be used in conjunction with ibm"); return -1; } SetIsECS(true); SetIMDSVersion(1); SetIAMCredentialsURL("http://169.254.170.2"); SetIAMFieldCount(5); set_builtin_cred_opts = true; return 0; } if(is_prefix(arg, "iam_role")){ if(is_ecs || IsIBMIAMAuth()){ S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs or ibm"); return -1; } if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){ // loading IAM role name in s3fs_init(), because we need to wait initializing curl. // SetIAMRoleMetadataType(true); set_builtin_cred_opts = true; return 0; }else if(is_prefix(arg, "iam_role=")){ const char* role = strchr(arg, '=') + sizeof(char); SetIAMRole(role); SetIAMRoleMetadataType(false); set_builtin_cred_opts = true; return 0; } } if(is_prefix(arg, "profile=")){ SetAwsProfileName(strchr(arg, '=') + sizeof(char)); set_builtin_cred_opts = true; return 0; } if(is_prefix(arg, "credlib=")){ if(!SetExtCredLib(strchr(arg, '=') + sizeof(char))){ S3FS_PRN_EXIT("failed to set credlib option : %s", (strchr(arg, '=') + sizeof(char))); return -1; } return 0; } if(is_prefix(arg, "credlib_opts=")){ if(!SetExtCredLibOpts(strchr(arg, '=') + sizeof(char))){ S3FS_PRN_EXIT("failed to set credlib_opts option : %s", (strchr(arg, '=') + sizeof(char))); return -1; } return 0; } return 1; } //------------------------------------------------------------------- // Methods : check parameters //------------------------------------------------------------------- // // Checking forbidden parameters for bucket // bool S3fsCred::CheckForbiddenBucketParams() { // The first plain argument is the bucket if(bucket_name.empty()){ S3FS_PRN_EXIT("missing BUCKET argument."); show_usage(); return false; } // bucket names cannot contain upper case characters in virtual-hosted style if(!pathrequeststyle && (lower(bucket_name) != bucket_name)){ S3FS_PRN_EXIT("BUCKET %s, name not compatible with virtual-hosted style.", bucket_name.c_str()); return false; } // check bucket name for illegal characters size_t found = bucket_name.find_first_of("/:\\;!@#$%^&*?|+="); if(found != std::string::npos){ S3FS_PRN_EXIT("BUCKET %s -- bucket name contains an illegal character: '%c' at position %zu", bucket_name.c_str(), bucket_name[found], found); return false; } if(!pathrequeststyle && is_prefix(s3host.c_str(), "https://") && bucket_name.find_first_of('.') != std::string::npos) { S3FS_PRN_EXIT("BUCKET %s -- cannot mount bucket with . while using HTTPS without use_path_request_style", bucket_name.c_str()); return false; } return true; } // // Check the combination of parameters // bool S3fsCred::CheckAllParams() { const std::lock_guard lock(token_lock); // // Checking forbidden parameters for bucket // if(!CheckForbiddenBucketParams()){ return false; } // error checking of command line arguments for compatibility if(S3fsCurl::IsPublicBucket() && IsSetAccessKeys()){ S3FS_PRN_EXIT("specifying both public_bucket and the access keys options is invalid."); return false; } if(IsSetPasswdFile() && IsSetAccessKeys()){ S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid."); return false; } if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs && !IsSetExtCredLib()){ if(!InitialS3fsCredentials()){ return false; } if(!IsSetAccessKeys()){ S3FS_PRN_EXIT("could not establish security credentials, check documentation."); return false; } // More error checking on the access key pair can be done // like checking for appropriate lengths and characters } // check IBM IAM requirements if(IsIBMIAMAuth()){ // check that default ACL is either public-read or private acl_t defaultACL = S3fsCurl::GetDefaultAcl(); if(defaultACL != acl_t::PRIVATE && defaultACL != acl_t::PUBLIC_READ){ S3FS_PRN_EXIT("can only use 'public-read' or 'private' ACL while using ibm_iam_auth"); return false; } } // check External Credential Library // // [NOTE] // If credlib(_opts) option (for External Credential Library) is specified, // no other Credential related options can be specified. It is exclusive. // if(set_builtin_cred_opts && (IsSetExtCredLib() || IsSetExtCredLibOpts())){ S3FS_PRN_EXIT("The \"credlib\" or \"credlib_opts\" option and other credential-related options(passwd_file, iam_role, profile, use_session_token, ecs, imdsv1only, ibm_iam_auth, ibm_iam_endpoint, etc) cannot be specified together."); return false; } // Load and Initialize external credential library if(IsSetExtCredLib() || IsSetExtCredLibOpts()){ if(!IsSetExtCredLib()){ S3FS_PRN_EXIT("The \"credlib_opts\"(%s) is specified but \"credlib\" option is not specified.", credlib_opts.c_str()); return false; } if(!InitExtCredLib()){ S3FS_PRN_EXIT("failed to load the library specified by the option credlib(%s, %s).", credlib.c_str(), credlib_opts.c_str()); return false; } S3FS_PRN_INFO("Loaded External Credential Library:\n%s", GetCredFuncVersion(true)); } return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_cred.h000066400000000000000000000174541470675423500161310ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_CRED_H_ #define S3FS_CRED_H_ #include #include #include #include "common.h" #include "s3fs_extcred.h" #include "types.h" //---------------------------------------------- // Typedefs //---------------------------------------------- typedef std::map iamcredmap_t; //------------------------------------------------ // class S3fsCred //------------------------------------------------ // This is a class for operating and managing Credentials(accesskey, // secret key, tokens, etc.) used by S3fs. // Operations related to Credentials are aggregated in this class. // // cppcheck-suppress ctuOneDefinitionRuleViolation ; for stub in test_curl_util.cpp class S3fsCred { private: static constexpr char ALLBUCKET_FIELDS_TYPE[] = ""; // special key for mapping(This name is absolutely not used as a bucket name) static constexpr char KEYVAL_FIELDS_TYPE[] = "\t"; // special key for mapping(This name is absolutely not used as a bucket name) static constexpr char AWS_ACCESSKEYID[] = "AWSAccessKeyId"; static constexpr char AWS_SECRETKEY[] = "AWSSecretKey"; static constexpr int IAM_EXPIRE_MERGING = 20 * 60; // update timing static constexpr char ECS_IAM_ENV_VAR[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; static constexpr char IAMCRED_ACCESSKEYID[] = "AccessKeyId"; static constexpr char IAMCRED_SECRETACCESSKEY[] = "SecretAccessKey"; static constexpr char IAMCRED_ROLEARN[] = "RoleArn"; static std::string bucket_name; mutable std::mutex token_lock; std::string passwd_file; std::string aws_profile; bool load_iamrole; std::string AWSAccessKeyId GUARDED_BY(token_lock); std::string AWSSecretAccessKey GUARDED_BY(token_lock); std::string AWSAccessToken GUARDED_BY(token_lock); time_t AWSAccessTokenExpire GUARDED_BY(token_lock); bool is_ecs; bool is_use_session_token; bool is_ibm_iam_auth; std::string IAM_cred_url; int IAM_api_version GUARDED_BY(token_lock); std::string IAMv2_api_token GUARDED_BY(token_lock); size_t IAM_field_count; std::string IAM_token_field; std::string IAM_expiry_field; std::string IAM_role GUARDED_BY(token_lock); bool set_builtin_cred_opts; // true if options other than "credlib" is set std::string credlib; // credlib(name or path) std::string credlib_opts; // options for credlib void* hExtCredLib; fp_VersionS3fsCredential pFuncCredVersion; fp_InitS3fsCredential pFuncCredInit; fp_FreeS3fsCredential pFuncCredFree; fp_UpdateS3fsCredential pFuncCredUpdate; public: static constexpr char IAMv2_token_url[] = "http://169.254.169.254/latest/api/token"; static constexpr int IAMv2_token_ttl = 21600; static constexpr char IAMv2_token_ttl_hdr[] = "X-aws-ec2-metadata-token-ttl-seconds"; static constexpr char IAMv2_token_hdr[] = "X-aws-ec2-metadata-token"; private: static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename); bool SetS3fsPasswdFile(const char* file); bool IsSetPasswdFile() const; bool SetAwsProfileName(const char* profile_name); bool SetIAMRoleMetadataType(bool flag); bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey) REQUIRES(S3fsCred::token_lock); bool SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken) REQUIRES(S3fsCred::token_lock); bool IsSetAccessKeys() const REQUIRES(S3fsCred::token_lock); bool SetIsECS(bool flag); bool SetIsUseSessionToken(bool flag); bool SetIsIBMIAMAuth(bool flag); int SetIMDSVersion(int version) REQUIRES(S3fsCred::token_lock); int GetIMDSVersion() const REQUIRES(S3fsCred::token_lock); bool SetIAMv2APIToken(const std::string& token) REQUIRES(S3fsCred::token_lock); const std::string& GetIAMv2APIToken() const REQUIRES(S3fsCred::token_lock); bool SetIAMRole(const char* role) REQUIRES(S3fsCred::token_lock); const std::string& GetIAMRole() const REQUIRES(S3fsCred::token_lock); bool IsSetIAMRole() const REQUIRES(S3fsCred::token_lock); size_t SetIAMFieldCount(size_t field_count); std::string SetIAMCredentialsURL(const char* url); std::string SetIAMTokenField(const char* token_field); std::string SetIAMExpiryField(const char* expiry_field); bool IsReadableS3fsPasswdFile() const; bool CheckS3fsPasswdFilePerms(); bool ParseS3fsPasswdFile(bucketkvmap_t& resmap); bool ReadS3fsPasswdFile() REQUIRES(S3fsCred::token_lock); static int CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap, std::string& access_key_id, std::string& secret_access_key); bool ReadAwsCredentialFile(const std::string &filename) REQUIRES(S3fsCred::token_lock); bool InitialS3fsCredentials() REQUIRES(S3fsCred::token_lock); bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval); bool GetIAMCredentialsURL(std::string& url, bool check_iam_role) REQUIRES(S3fsCred::token_lock); bool LoadIAMCredentials() REQUIRES(S3fsCred::token_lock); bool SetIAMCredentials(const char* response) REQUIRES(S3fsCred::token_lock); bool SetIAMRoleFromMetaData(const char* response) REQUIRES(S3fsCred::token_lock); bool SetExtCredLib(const char* arg); bool IsSetExtCredLib() const; bool SetExtCredLibOpts(const char* args); bool IsSetExtCredLibOpts() const; bool InitExtCredLib(); bool LoadExtCredLib(); bool UnloadExtCredLib(); bool UpdateExtCredentials() REQUIRES(S3fsCred::token_lock); static bool CheckForbiddenBucketParams(); public: static bool SetBucket(const std::string& bucket); static const std::string& GetBucket(); S3fsCred(); ~S3fsCred(); S3fsCred(const S3fsCred&) = delete; S3fsCred(S3fsCred&&) = delete; S3fsCred& operator=(const S3fsCred&) = delete; S3fsCred& operator=(S3fsCred&&) = delete; bool IsIBMIAMAuth() const { return is_ibm_iam_auth; } bool LoadIAMRoleFromMetaData(); bool CheckIAMCredentialUpdate(std::string* access_key_id = nullptr, std::string* secret_access_key = nullptr, std::string* access_token = nullptr); const char* GetCredFuncVersion(bool detail) const; int DetectParam(const char* arg); bool CheckAllParams(); }; #endif // S3FS_CRED_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_extcred.h000066400000000000000000000131541470675423500166430ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_EXTCRED_H_ #define S3FS_EXTCRED_H_ //------------------------------------------------------------------- // Attributes(weak) : use only in s3fs-fuse internally //------------------------------------------------------------------- // [NOTE] // This macro is only used inside s3fs-fuse. // External projects that utilize this header file substitute empty //values as follows: // #ifndef S3FS_FUNCATTR_WEAK #define S3FS_FUNCATTR_WEAK #endif extern "C" { //------------------------------------------------------------------- // Prototype for External Credential 4 functions //------------------------------------------------------------------- // // [Required] VersionS3fsCredential // // Returns the library name and version as a string. // extern const char* VersionS3fsCredential(bool detail) S3FS_FUNCATTR_WEAK; // // [Optional] InitS3fsCredential // // A function that does the necessary initialization after the library is // loaded. This function is called only once immediately after loading the // library. // If there is a required initialization inside the library, implement it. // Implementation of this function is optional and not required. If not // implemented, it will not be called. // // const char* popts : String passed with the credlib_opts option. If the // credlib_opts option is not specified, nullptr will be // passed. // char** pperrstr : pperrstr is used to pass the error message to the // caller when an error occurs. // If this pointer is not nullptr, you can allocate memory // and set an error message to it. The allocated memory // area is freed by the caller. // extern bool InitS3fsCredential(const char* popts, char** pperrstr) S3FS_FUNCATTR_WEAK; // // [Optional] FreeS3fsCredential // // A function that is called only once just before the library is unloaded. // If there is a required discard process in the library, implement it. // Implementation of this feature is optional and not required. // If not implemented, it will not be called. // // char** pperrstr : pperrstr is used to pass the error message to the // caller when an error occurs. // If this pointer is not nullptr, you can allocate memory // and set an error message to it. The allocated memory // area is freed by the caller. // extern bool FreeS3fsCredential(char** pperrstr) S3FS_FUNCATTR_WEAK; // // [Required] UpdateS3fsCredential // // A function that updates the token. // // char** ppaccess_key_id : Allocate and set "Access Key ID" string // area to *ppaccess_key_id. // char** ppserect_access_key : Allocate and set "Access Secret Key ID" // string area to *ppserect_access_key. // char** ppaccess_token : Allocate and set "Token" string area to // *ppaccess_token. // long long* ptoken_expire : Set token expire time(time_t) value to // *ptoken_expire. // This is essentially a time_t* variable. // To avoid system differences about time_t // size, long long* is used. // When setting the value, cast from time_t // to long long to set the value. // char** pperrstr : pperrstr is used to pass the error message to the // caller when an error occurs. // // For all argument of the character string pointer(char **) set the // allocated string area. The allocated area is freed by the caller. // extern bool UpdateS3fsCredential(char** ppaccess_key_id, char** ppserect_access_key, char** ppaccess_token, long long* ptoken_expire, char** pperrstr) S3FS_FUNCATTR_WEAK; //--------------------------------------------------------- // Typedef Prototype function //--------------------------------------------------------- // // const char* VersionS3fsCredential() // typedef const char* (*fp_VersionS3fsCredential)(bool detail); // // bool InitS3fsCredential(char** pperrstr) // typedef bool (*fp_InitS3fsCredential)(const char* popts, char** pperrstr); // // bool FreeS3fsCredential(char** pperrstr) // typedef bool (*fp_FreeS3fsCredential)(char** pperrstr); // // bool UpdateS3fsCredential(char** ppaccess_key_id, char** ppserect_access_key, char** ppaccess_token, long long* ptoken_expire, char** pperrstr) // typedef bool (*fp_UpdateS3fsCredential)(char** ppaccess_key_id, char** ppserect_access_key, char** ppaccess_token, long long* ptoken_expire, char** pperrstr); } // extern "C" #endif // S3FS_EXTCRED_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_global.cpp000066400000000000000000000031331470675423500167740ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include "common.h" //------------------------------------------------------------------- // Global variables //------------------------------------------------------------------- bool foreground = false; bool nomultipart = false; bool pathrequeststyle = false; bool complement_stat = false; bool noxmlns = false; std::string program_name; std::string service_path = "/"; std::string s3host = "https://s3.amazonaws.com"; std::string endpoint = "us-east-1"; std::string cipher_suites; std::string instance_name; /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_help.cpp000066400000000000000000001046241470675423500164730ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include "common.h" #include "s3fs_help.h" #include "s3fs_auth.h" //------------------------------------------------------------------- // Contents //------------------------------------------------------------------- static constexpr char help_string[] = "\n" "Mount an Amazon S3 bucket as a file system.\n" "\n" "Usage:\n" " mounting\n" " s3fs bucket[:/path] mountpoint [options]\n" " s3fs mountpoint [options (must specify bucket= option)]\n" "\n" " unmounting\n" " umount mountpoint\n" "\n" " General forms for s3fs and FUSE/mount options:\n" " -o opt[,opt...]\n" " -o opt [-o opt] ...\n" "\n" " utility mode (remove interrupted multipart uploading objects)\n" " s3fs --incomplete-mpu-list (-u) bucket\n" " s3fs --incomplete-mpu-abort[=all | =] bucket\n" "\n" "s3fs Options:\n" "\n" " Most s3fs options are given in the form where \"opt\" is:\n" "\n" " =\n" "\n" " bucket\n" " - if it is not specified bucket name (and path) in command line,\n" " must specify this option after -o option for bucket name.\n" "\n" " default_acl (default=\"private\")\n" " - the default canned acl to apply to all written s3 objects,\n" " e.g., private, public-read. see\n" " https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl\n" " for the full list of canned ACLs\n" "\n" " retries (default=\"5\")\n" " - number of times to retry a failed S3 transaction\n" "\n" " tmpdir (default=\"/tmp\")\n" " - local folder for temporary files.\n" "\n" " use_cache (default=\"\" which means disabled)\n" " - local folder to use for local file cache\n" "\n" " check_cache_dir_exist (default is disable)\n" " - if use_cache is set, check if the cache directory exists.\n" " If this option is not specified, it will be created at runtime\n" " when the cache directory does not exist.\n" "\n" " del_cache (delete local file cache)\n" " - delete local file cache when s3fs starts and exits.\n" "\n" " storage_class (default=\"standard\")\n" " - store object with specified storage class. Possible values:\n" " standard, standard_ia, onezone_ia, reduced_redundancy,\n" " intelligent_tiering, glacier, glacier_ir, and deep_archive.\n" "\n" " use_rrs (default is disable)\n" " - use Amazon's Reduced Redundancy Storage.\n" " this option can not be specified with use_sse.\n" " (can specify use_rrs=1 for old version)\n" " this option has been replaced by new storage_class option.\n" "\n" " use_sse (default is disable)\n" " - Specify three type Amazon's Server-Site Encryption: SSE-S3,\n" " SSE-C or SSE-KMS. SSE-S3 uses Amazon S3-managed encryption\n" " keys, SSE-C uses customer-provided encryption keys, and\n" " SSE-KMS uses the master key which you manage in AWS KMS.\n" " You can specify \"use_sse\" or \"use_sse=1\" enables SSE-S3\n" " type (use_sse=1 is old type parameter).\n" " Case of setting SSE-C, you can specify \"use_sse=custom\",\n" " \"use_sse=custom:\" or\n" " \"use_sse=\" (only \n" " specified is old type parameter). You can use \"c\" for\n" " short \"custom\".\n" " The custom key file must be 600 permission. The file can\n" " have some lines, each line is one SSE-C key. The first line\n" " in file is used as Customer-Provided Encryption Keys for\n" " uploading and changing headers etc. If there are some keys\n" " after first line, those are used downloading object which\n" " are encrypted by not first key. So that, you can keep all\n" " SSE-C keys in file, that is SSE-C key history.\n" " If you specify \"custom\" (\"c\") without file path, you\n" " need to set custom key by load_sse_c option or AWSSSECKEYS\n" " environment. (AWSSSECKEYS environment has some SSE-C keys\n" " with \":\" separator.) This option is used to decide the\n" " SSE type. So that if you do not want to encrypt a object\n" " object at uploading, but you need to decrypt encrypted\n" " object at downloading, you can use load_sse_c option instead\n" " of this option.\n" " For setting SSE-KMS, specify \"use_sse=kmsid\" or\n" " \"use_sse=kmsid:\". You can use \"k\" for short \"kmsid\".\n" " If you san specify SSE-KMS type with your in AWS\n" " KMS, you can set it after \"kmsid:\" (or \"k:\"). If you\n" " specify only \"kmsid\" (\"k\"), you need to set AWSSSEKMSID\n" " environment which value is . You must be careful\n" " about that you can not use the KMS id which is not same EC2\n" " region.\n" " Additionally, if you specify SSE-KMS, your endpoints must use\n" " Secure Sockets Layer(SSL) or Transport Layer Security(TLS).\n" "\n" " load_sse_c - specify SSE-C keys\n" " Specify the custom-provided encryption keys file path for decrypting\n" " at downloading.\n" " If you use the custom-provided encryption key at uploading, you\n" " specify with \"use_sse=custom\". The file has many lines, one line\n" " means one custom key. So that you can keep all SSE-C keys in file,\n" " that is SSE-C key history. AWSSSECKEYS environment is as same as this\n" " file contents.\n" "\n" " public_bucket (default=\"\" which means disabled)\n" " - anonymously mount a public bucket when set to 1, ignores the \n" " $HOME/.passwd-s3fs and /etc/passwd-s3fs files.\n" " S3 does not allow copy object api for anonymous users, then\n" " s3fs sets nocopyapi option automatically when public_bucket=1\n" " option is specified.\n" "\n" " passwd_file (default=\"\")\n" " - specify which s3fs password file to use\n" "\n" " ahbe_conf (default=\"\" which means disabled)\n" " - This option specifies the configuration file path which\n" " file is the additional HTTP header by file (object) extension.\n" " The configuration file format is below:\n" " -----------\n" " line = [file suffix or regex] HTTP-header [HTTP-values]\n" " file suffix = file (object) suffix, if this field is empty,\n" " it means \"reg:(.*)\".(=all object).\n" " regex = regular expression to match the file (object) path.\n" " this type starts with \"reg:\" prefix.\n" " HTTP-header = additional HTTP header name\n" " HTTP-values = additional HTTP header value\n" " -----------\n" " Sample:\n" " -----------\n" " .gz Content-Encoding gzip\n" " .Z Content-Encoding compress\n" " reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2\n" " -----------\n" " A sample configuration file is uploaded in \"test\" directory.\n" " If you specify this option for set \"Content-Encoding\" HTTP \n" " header, please take care for RFC 2616.\n" "\n" " profile (default=\"default\")\n" " - Choose a profile from ${HOME}/.aws/credentials to authenticate\n" " against S3. Note that this format matches the AWS CLI format and\n" " differs from the s3fs passwd format.\n" "\n" " connect_timeout (default=\"300\" seconds)\n" " - time to wait for connection before giving up\n" "\n" " readwrite_timeout (default=\"120\" seconds)\n" " - time to wait between read/write activity before giving up\n" "\n" " list_object_max_keys (default=\"1000\")\n" " - specify the maximum number of keys returned by S3 list object\n" " API. The default is 1000. you can set this value to 1000 or more.\n" "\n" " max_stat_cache_size (default=\"100,000\" entries (about 40MB))\n" " - maximum number of entries in the stat cache, and this maximum is\n" " also treated as the number of symbolic link cache.\n" "\n" " stat_cache_expire (default is 900))\n" " - specify expire time (seconds) for entries in the stat cache.\n" " This expire time indicates the time since stat cached. and this\n" " is also set to the expire time of the symbolic link cache.\n" "\n" " stat_cache_interval_expire (default is 900)\n" " - specify expire time (seconds) for entries in the stat cache(and\n" " symbolic link cache).\n" " This expire time is based on the time from the last access time\n" " of the stat cache. This option is exclusive with stat_cache_expire,\n" " and is left for compatibility with older versions.\n" "\n" " disable_noobj_cache (default is enable)\n" " - By default s3fs memorizes when an object does not exist up until\n" " the stat cache timeout. This caching can cause staleness for\n" " applications. If disabled, s3fs will not memorize objects and may\n" " cause extra HeadObject requests and reduce performance.\n" "\n" " no_check_certificate\n" " - server certificate won't be checked against the available \n" " certificate authorities.\n" "\n" " ssl_verify_hostname (default=\"2\")\n" " - When 0, do not verify the SSL certificate against the hostname.\n" "\n" " ssl_client_cert (default=\"\")\n" " - Specify an SSL client certificate.\n" " Specify this optional parameter in the following format:\n" " \"[:[:[:\n" " [:]]]]\"\n" " : Client certificate.\n" " Specify the file path or NickName(for NSS, etc.).\n" " : Type of certificate, default is \"PEM\"(optional).\n" " : Certificate's private key file(optional).\n" " : Type of private key, default is \"PEM\"(optional).\n" " : Passphrase of the private key(optional).\n" " It is also possible to omit this value and specify\n" " it using the environment variable\n" " \"S3FS_SSL_PRIVKEY_PASSWORD\".\n" "\n" " nodnscache (disable DNS cache)\n" " - s3fs is always using DNS cache, this option make DNS cache disable.\n" "\n" " nosscache (disable SSL session cache)\n" " - s3fs is always using SSL session cache, this option make SSL \n" " session cache disable.\n" "\n" " multireq_max (default=\"20\")\n" " - maximum number of parallel request for listing objects.\n" "\n" " parallel_count (default=\"5\")\n" " - number of parallel request for uploading big objects.\n" " s3fs uploads large object (over 20MB) by multipart post request, \n" " and sends parallel requests.\n" " This option limits parallel request count which s3fs requests \n" " at once. It is necessary to set this value depending on a CPU \n" " and a network band.\n" "\n" " multipart_size (default=\"10\")\n" " - part size, in MB, for each multipart request.\n" " The minimum value is 5 MB and the maximum value is 5 GB.\n" "\n" " multipart_copy_size (default=\"512\")\n" " - part size, in MB, for each multipart copy request, used for\n" " renames and mixupload.\n" " The minimum value is 5 MB and the maximum value is 5 GB.\n" " Must be at least 512 MB to copy the maximum 5 TB object size\n" " but lower values may improve performance.\n" "\n" " max_dirty_data (default=\"5120\")\n" " - flush dirty data to S3 after a certain number of MB written.\n" " The minimum value is 50 MB. -1 value means disable.\n" " Cannot be used with nomixupload.\n" "\n" " bucket_size (default=maximum long unsigned integer value)\n" " - The size of the bucket with which the corresponding\n" " elements of the statvfs structure will be filled. The option\n" " argument is an integer optionally followed by a\n" " multiplicative suffix (GB, GiB, TB, TiB, PB, PiB,\n" " EB, EiB) (no spaces in between). If no suffix is supplied,\n" " bytes are assumed; eg: 20000000, 30GB, 45TiB. Note that\n" " s3fs does not compute the actual volume size (too\n" " expensive): by default it will assume the maximum possible\n" " size; however, since this may confuse other software which\n" " uses s3fs, the advertised bucket size can be set with this\n" " option.\n" "\n" " ensure_diskfree (default 0)\n" " - sets MB to ensure disk free space. This option means the\n" " threshold of free space size on disk which is used for the\n" " cache file by s3fs. s3fs makes file for\n" " downloading, uploading and caching files. If the disk free\n" " space is smaller than this value, s3fs do not use disk space\n" " as possible in exchange for the performance.\n" "\n" " free_space_ratio (default=\"10\")\n" " - sets min free space ratio of the disk.\n" " The value of this option can be between 0 and 100. It will control\n" " the size of the cache according to this ratio to ensure that the\n" " idle ratio of the disk is greater than this value.\n" " For example, when the disk space is 50GB, the default value will\n" " ensure that the disk will reserve at least 50GB * 10%% = 5GB of\n" " remaining space.\n" "\n" " multipart_threshold (default=\"25\")\n" " - threshold, in MB, to use multipart upload instead of\n" " single-part. Must be at least 5 MB.\n" "\n" " singlepart_copy_limit (default=\"512\")\n" " - maximum size, in MB, of a single-part copy before trying \n" " multipart copy.\n" "\n" " host (default=\"https://s3.amazonaws.com\")\n" " - Set a non-Amazon host, e.g., https://example.com.\n" "\n" " servicepath (default=\"/\")\n" " - Set a service path when the non-Amazon host requires a prefix.\n" "\n" " url (default=\"https://s3.amazonaws.com\")\n" " - sets the url to use to access Amazon S3. If you want to use HTTP,\n" " then you can set \"url=http://s3.amazonaws.com\".\n" " If you do not use https, please specify the URL with the url\n" " option.\n" "\n" " endpoint (default=\"us-east-1\")\n" " - sets the endpoint to use on signature version 4\n" " If this option is not specified, s3fs uses \"us-east-1\" region as\n" " the default. If the s3fs could not connect to the region specified\n" " by this option, s3fs could not run. But if you do not specify this\n" " option, and if you can not connect with the default region, s3fs\n" " will retry to automatically connect to the other region. So s3fs\n" " can know the correct region name, because s3fs can find it in an\n" " error from the S3 server.\n" "\n" " sigv2 (default is signature version 4 falling back to version 2)\n" " - sets signing AWS requests by using only signature version 2\n" "\n" " sigv4 (default is signature version 4 falling back to version 2)\n" " - sets signing AWS requests by using only signature version 4\n" "\n" " mp_umask (default is \"0000\")\n" " - sets umask for the mount point directory.\n" " If allow_other option is not set, s3fs allows access to the mount\n" " point only to the owner. In the opposite case s3fs allows access\n" " to all users as the default. But if you set the allow_other with\n" " this option, you can control the permissions of the\n" " mount point by this option like umask.\n" "\n" " umask (default is \"0000\")\n" " - sets umask for files under the mountpoint. This can allow\n" " users other than the mounting user to read and write to files\n" " that they did not create.\n" "\n" " nomultipart (disable multipart uploads)\n" "\n" " streamupload (default is disable)\n" " - Enable stream upload.\n" " If this option is enabled, a sequential upload will be performed\n" " in parallel with the write from the part that has been written\n" " during a multipart upload.\n" " This is expected to give better performance than other upload\n" " functions.\n" " Note that this option is still experimental and may change in the\n" " future.\n" "\n" " max_thread_count (default is \"5\")\n" " - Specifies the number of threads waiting for stream uploads.\n" " Note that this option and Stream Upload are still experimental\n" " and subject to change in the future.\n" " This option will be merged with \"parallel_count\" in the future.\n" "\n" " enable_content_md5 (default is disable)\n" " - Allow S3 server to check data integrity of uploads via the\n" " Content-MD5 header. This can add CPU overhead to transfers.\n" "\n" " enable_unsigned_payload (default is disable)\n" " - Do not calculate Content-SHA256 for PutObject and UploadPart\n" " payloads. This can reduce CPU overhead to transfers.\n" "\n" " ecs (default is disable)\n" " - This option instructs s3fs to query the ECS container credential\n" " metadata address instead of the instance metadata address.\n" "\n" " iam_role (default is no IAM role)\n" " - This option requires the IAM role name or \"auto\". If you specify\n" " \"auto\", s3fs will automatically use the IAM role names that are set\n" " to an instance. If you specify this option without any argument, it\n" " is the same as that you have specified the \"auto\".\n" "\n" " imdsv1only (default is to use IMDSv2 with fallback to v1)\n" " - AWS instance metadata service, used with IAM role authentication,\n" " supports the use of an API token. If you're using an IAM role\n" " in an environment that does not support IMDSv2, setting this flag\n" " will skip retrieval and usage of the API token when retrieving\n" " IAM credentials.\n" "\n" " ibm_iam_auth (default is not using IBM IAM authentication)\n" " - This option instructs s3fs to use IBM IAM authentication.\n" " In this mode, the AWSAccessKey and AWSSecretKey will be used as\n" " IBM's Service-Instance-ID and APIKey, respectively.\n" "\n" " ibm_iam_endpoint (default is https://iam.cloud.ibm.com)\n" " - sets the URL to use for IBM IAM authentication.\n" "\n" " credlib (default=\"\" which means disabled)\n" " - Specifies the shared library that handles the credentials\n" " containing the authentication token.\n" " If this option is specified, the specified credential and token\n" " processing provided by the shared library ant will be performed\n" " instead of the built-in credential processing.\n" " This option cannot be specified with passwd_file, profile,\n" " use_session_token, ecs, ibm_iam_auth, ibm_iam_endpoint, imdsv1only\n" " and iam_role option.\n" "\n" " credlib_opts (default=\"\" which means disabled)\n" " - Specifies the options to pass when the shared library specified\n" " in credlib is loaded and then initialized.\n" " For the string specified in this option, specify the string defined\n" " by the shared library.\n" "\n" " use_xattr (default is not handling the extended attribute)\n" " Enable to handle the extended attribute (xattrs).\n" " If you set this option, you can use the extended attribute.\n" " For example, encfs and ecryptfs need to support the extended attribute.\n" " Notice: if s3fs handles the extended attribute, s3fs can not work to\n" " copy command with preserve=mode.\n" "\n" " noxmlns (disable registering xml name space)\n" " disable registering xml name space for response of \n" " ListBucketResult and ListVersionsResult etc. Default name \n" " space is looked up from \"http://s3.amazonaws.com/doc/2006-03-01\".\n" " This option should not be specified now, because s3fs looks up\n" " xmlns automatically after v1.66.\n" "\n" " nomixupload (disable copy in multipart uploads)\n" " Disable to use PUT (copy api) when multipart uploading large size objects.\n" " By default, when doing multipart upload, the range of unchanged data\n" " will use PUT (copy api) whenever possible.\n" " When nocopyapi or norenameapi is specified, use of PUT (copy api) is\n" " invalidated even if this option is not specified.\n" "\n" " nocopyapi (for other incomplete compatibility object storage)\n" " Enable compatibility with S3-like APIs which do not support\n" " PUT (copy api).\n" " If you set this option, s3fs do not use PUT with \n" " \"x-amz-copy-source\" (copy api). Because traffic is increased\n" " 2-3 times by this option, we do not recommend this.\n" "\n" " norenameapi (for other incomplete compatibility object storage)\n" " Enable compatibility with S3-like APIs which do not support\n" " PUT (copy api).\n" " This option is a subset of nocopyapi option. The nocopyapi\n" " option does not use copy-api for all command (ex. chmod, chown,\n" " touch, mv, etc), but this option does not use copy-api for\n" " only rename command (ex. mv). If this option is specified with\n" " nocopyapi, then s3fs ignores it.\n" "\n" " use_path_request_style (use legacy API calling style)\n" " Enable compatibility with S3-like APIs which do not support\n" " the virtual-host request style, by using the older path request\n" " style.\n" "\n" " listobjectsv2 (use ListObjectsV2)\n" " Issue ListObjectsV2 instead of ListObjects, useful on object\n" " stores without ListObjects support.\n" "\n" " noua (suppress User-Agent header)\n" " Usually s3fs outputs of the User-Agent in \"s3fs/ (commit\n" " hash ; )\" format.\n" " If this option is specified, s3fs suppresses the output of the\n" " User-Agent.\n" "\n" " cipher_suites\n" " Customize the list of TLS cipher suites.\n" " Expects a colon separated list of cipher suite names.\n" " A list of available cipher suites, depending on your TLS engine,\n" " can be found on the CURL library documentation:\n" " https://curl.haxx.se/docs/ssl-ciphers.html\n" "\n" " instance_name - The instance name of the current s3fs mountpoint.\n" " This name will be added to logging messages and user agent headers sent by s3fs.\n" "\n" " complement_stat (complement lack of file/directory mode)\n" " s3fs complements lack of information about file/directory mode\n" " if a file or a directory object does not have x-amz-meta-mode\n" " header. As default, s3fs does not complements stat information\n" " for a object, then the object will not be able to be allowed to\n" " list/modify.\n" "\n" " compat_dir (enable support of alternative directory names)\n" " s3fs supports two different naming schemas \"dir/\" and\n" " \"dir\" to map directory names to S3 objects and\n" " vice versa by default. As a third variant, directories can be\n" " determined indirectly if there is a file object with a path (e.g.\n" " \"/dir/file\") but without the parent directory.\n" " This option enables a fourth variant, \"dir_$folder$\", created by\n" " older applications.\n" " \n" " S3fs uses only the first schema \"dir/\" to create S3 objects for\n" " directories." " \n" " The support for these different naming schemas causes an increased\n" " communication effort.\n" "\n" " use_wtf8 - support arbitrary file system encoding.\n" " S3 requires all object names to be valid UTF-8. But some\n" " clients, notably Windows NFS clients, use their own encoding.\n" " This option re-encodes invalid UTF-8 object names into valid\n" " UTF-8 by mapping offending codes into a 'private' codepage of the\n" " Unicode set.\n" " Useful on clients not using UTF-8 as their file system encoding.\n" "\n" " use_session_token - indicate that session token should be provided.\n" " If credentials are provided by environment variables this switch\n" " forces presence check of AWSSESSIONTOKEN variable.\n" " Otherwise an error is returned.\n" "\n" " requester_pays (default is disable)\n" " This option instructs s3fs to enable requests involving\n" " Requester Pays buckets.\n" " It includes the 'x-amz-request-payer=requester' entry in the\n" " request header.\n" "\n" " mime (default is \"/etc/mime.types\")\n" " Specify the path of the mime.types file.\n" " If this option is not specified, the existence of \"/etc/mime.types\"\n" " is checked, and that file is loaded as mime information.\n" " If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n" " is checked as well.\n" "\n" " proxy (default=\"\")\n" " This option specifies a proxy to S3 server.\n" " Specify the proxy with '[]' formatted.\n" " '://' can be omitted, and 'http://' is used when omitted.\n" " Also, ':' can also be omitted. If omitted, port 443 is used for\n" " HTTPS schema, and port 1080 is used otherwise.\n" " This option is the same as the curl command's '--proxy(-x)' option and\n" " libcurl's 'CURLOPT_PROXY' flag.\n" " This option is equivalent to and takes precedence over the environment\n" " variables 'http_proxy', 'all_proxy', etc.\n" "\n" " proxy_cred_file (default=\"\")\n" " This option specifies the file that describes the username and\n" " passphrase for authentication of the proxy when the HTTP schema\n" " proxy is specified by the 'proxy' option.\n" " Username and passphrase are valid only for HTTP schema. If the HTTP\n" " proxy does not require authentication, this option is not required.\n" " Separate the username and passphrase with a ':' character and\n" " specify each as a URL-encoded string.\n" "\n" " ipresolve (default=\"whatever\")\n" " Select what type of IP addresses to use when establishing a\n" " connection.\n" " Default('whatever') can use addresses of all IP versions(IPv4 and\n" " IPv6) that your system allows. If you specify 'IPv4', only IPv4\n" " addresses are used. And when 'IPv6'is specified, only IPv6 addresses\n" " will be used.\n" "\n" " logfile - specify the log output file.\n" " s3fs outputs the log file to syslog. Alternatively, if s3fs is\n" " started with the \"-f\" option specified, the log will be output\n" " to the stdout/stderr.\n" " You can use this option to specify the log file that s3fs outputs.\n" " If you specify a log file with this option, it will reopen the log\n" " file when s3fs receives a SIGHUP signal. You can use the SIGHUP\n" " signal for log rotation.\n" "\n" " dbglevel (default=\"crit\")\n" " Set the debug message level. set value as crit (critical), err\n" " (error), warn (warning), info (information) to debug level.\n" " default debug level is critical. If s3fs run with \"-d\" option,\n" " the debug level is set information. When s3fs catch the signal\n" " SIGUSR2, the debug level is bump up.\n" "\n" " curldbg - put curl debug message\n" " Put the debug message from libcurl when this option is specified.\n" " Specify \"normal\" or \"body\" for the parameter.\n" " If the parameter is omitted, it is the same as \"normal\".\n" " If \"body\" is specified, some API communication body data will be\n" " output in addition to the debug message output as \"normal\".\n" "\n" " no_time_stamp_msg - no time stamp in debug message\n" " The time stamp is output to the debug message by default.\n" " If this option is specified, the time stamp will not be output\n" " in the debug message.\n" " It is the same even if the environment variable \"S3FS_MSGTIMESTAMP\"\n" " is set to \"no\".\n" "\n" " set_check_cache_sigusr1 (default is stdout)\n" " If the cache is enabled, you can check the integrity of the\n" " cache file and the cache file's stats info file.\n" " This option is specified and when sending the SIGUSR1 signal\n" " to the s3fs process checks the cache status at that time.\n" " This option can take a file path as parameter to output the\n" " check result to that file. The file path parameter can be omitted.\n" " If omitted, the result will be output to stdout or syslog.\n" "\n" " update_parent_dir_stat (default is disable)\n" " The parent directory's mtime and ctime are updated when a file or\n" " directory is created or deleted (when the parent directory's inode is\n" " updated).\n" " By default, parent directory statistics are not updated.\n" "\n" "FUSE/mount Options:\n" "\n" " Most of the generic mount options described in 'man mount' are\n" " supported (ro, rw, suid, nosuid, dev, nodev, exec, noexec, atime,\n" " noatime, sync async, dirsync). Filesystems are mounted with\n" " '-onodev,nosuid' by default, which can only be overridden by a\n" " privileged user.\n" " \n" " There are many FUSE specific mount options that can be specified.\n" " e.g. allow_other See the FUSE's README for the full set.\n" "\n" "Utility mode Options:\n" "\n" " -u, --incomplete-mpu-list\n" " Lists multipart incomplete objects uploaded to the specified\n" " bucket.\n" " --incomplete-mpu-abort (=all or =)\n" " Delete the multipart incomplete object uploaded to the specified\n" " bucket.\n" " If \"all\" is specified for this option, all multipart incomplete\n" " objects will be deleted. If you specify no argument as an option,\n" " objects older than 24 hours (24H) will be deleted (This is the\n" " default value). You can specify an optional date format. It can\n" " be specified as year, month, day, hour, minute, second, and it is\n" " expressed as \"Y\", \"M\", \"D\", \"h\", \"m\", \"s\" respectively.\n" " For example, \"1Y6M10D12h30m30s\".\n" "\n" "Miscellaneous Options:\n" "\n" " -h, --help Output this help.\n" " --version Output version info.\n" " -d --debug Turn on DEBUG messages to syslog. Specifying -d\n" " twice turns on FUSE debug messages to STDOUT.\n" " -f FUSE foreground option - do not run as daemon.\n" " -s FUSE single-threaded option\n" " disable multi-threaded operation\n" "\n" "\n" "s3fs home page: \n" ; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- void show_usage() { printf("Usage: %s BUCKET:[PATH] MOUNTPOINT [OPTION]...\n", program_name.c_str()); } void show_help() { show_usage(); printf(help_string); } void show_version() { printf( "Amazon Simple Storage Service File System V%s%s with %s\n" "Copyright (C) 2010 Randy Rizun \n" "License GPL2: GNU GPL version 2 \n" "This is free software: you are free to change and redistribute it.\n" "There is NO WARRANTY, to the extent permitted by law.\n", VERSION, COMMIT_HASH_VAL, s3fs_crypt_lib_name()); } const char* short_version() { static constexpr char short_ver[] = "s3fs version " VERSION "" COMMIT_HASH_VAL; return short_ver; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_help.h000066400000000000000000000024241470675423500161330ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_S3FS_HELP_H_ #define S3FS_S3FS_HELP_H_ //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- void show_usage(); void show_help(); void show_version(); const char* short_version(); #endif // S3FS_S3FS_HELP_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_logger.cpp000066400000000000000000000250261470675423500170200ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include "common.h" #include "s3fs_logger.h" //------------------------------------------------------------------- // S3fsLog class : variables //------------------------------------------------------------------- constexpr char S3fsLog::LOGFILEENV[]; constexpr const char* S3fsLog::nest_spaces[]; constexpr char S3fsLog::MSGTIMESTAMP[]; S3fsLog* S3fsLog::pSingleton = nullptr; S3fsLog::s3fs_log_level S3fsLog::debug_level = S3fsLog::LEVEL_CRIT; FILE* S3fsLog::logfp = nullptr; std::string S3fsLog::logfile; bool S3fsLog::time_stamp = true; //------------------------------------------------------------------- // S3fsLog class : class methods //------------------------------------------------------------------- bool S3fsLog::IsS3fsLogLevel(s3fs_log_level level) { return (level == (S3fsLog::debug_level & level)); } std::string S3fsLog::GetCurrentTime() { std::ostringstream current_time; if(time_stamp){ struct timeval now; struct timespec tsnow; struct tm res; char tmp[32]; if(-1 == clock_gettime(S3FS_CLOCK_MONOTONIC, &tsnow)){ now.tv_sec = tsnow.tv_sec; now.tv_usec = (tsnow.tv_nsec / 1000); }else{ gettimeofday(&now, nullptr); } strftime(tmp, sizeof(tmp), "%Y-%m-%dT%H:%M:%S", gmtime_r(&now.tv_sec, &res)); current_time << tmp << "." << std::setfill('0') << std::setw(3) << (now.tv_usec / 1000) << "Z "; } return current_time.str(); } bool S3fsLog::SetLogfile(const char* pfile) { if(!S3fsLog::pSingleton){ S3FS_PRN_CRIT("S3fsLog::pSingleton is nullptr."); return false; } return S3fsLog::pSingleton->LowSetLogfile(pfile); } bool S3fsLog::ReopenLogfile() { if(!S3fsLog::pSingleton){ S3FS_PRN_CRIT("S3fsLog::pSingleton is nullptr."); return false; } if(!S3fsLog::logfp){ S3FS_PRN_INFO("Currently the log file is output to stdout/stderr."); return true; } if(!S3fsLog::logfile.empty()){ S3FS_PRN_ERR("There is a problem with the path to the log file being empty."); return false; } std::string tmp = S3fsLog::logfile; return S3fsLog::pSingleton->LowSetLogfile(tmp.c_str()); } S3fsLog::s3fs_log_level S3fsLog::SetLogLevel(s3fs_log_level level) { if(!S3fsLog::pSingleton){ S3FS_PRN_CRIT("S3fsLog::pSingleton is nullptr."); return S3fsLog::debug_level; // Although it is an error, it returns the current value. } return S3fsLog::pSingleton->LowSetLogLevel(level); } S3fsLog::s3fs_log_level S3fsLog::BumpupLogLevel() { if(!S3fsLog::pSingleton){ S3FS_PRN_CRIT("S3fsLog::pSingleton is nullptr."); return S3fsLog::debug_level; // Although it is an error, it returns the current value. } return S3fsLog::pSingleton->LowBumpupLogLevel(); } bool S3fsLog::SetTimeStamp(bool value) { bool old = S3fsLog::time_stamp; S3fsLog::time_stamp = value; return old; } //------------------------------------------------------------------- // S3fsLog class : methods //------------------------------------------------------------------- S3fsLog::S3fsLog() { if(!S3fsLog::pSingleton){ S3fsLog::pSingleton = this; // init syslog(default CRIT) openlog("s3fs", LOG_PID | LOG_ODELAY | LOG_NOWAIT, LOG_USER); LowLoadEnv(); }else{ S3FS_PRN_ERR("Already set singleton object for S3fsLog."); } } S3fsLog::~S3fsLog() { if(S3fsLog::pSingleton == this){ FILE* oldfp = S3fsLog::logfp; S3fsLog::logfp = nullptr; if(oldfp && 0 != fclose(oldfp)){ S3FS_PRN_ERR("Could not close old log file(%s), but continue...", (S3fsLog::logfile.empty() ? S3fsLog::logfile.c_str() : "null")); } S3fsLog::logfile.clear(); S3fsLog::pSingleton = nullptr; S3fsLog::debug_level = S3fsLog::LEVEL_CRIT; closelog(); }else{ S3FS_PRN_ERR("This object is not singleton S3fsLog object."); } } bool S3fsLog::LowLoadEnv() { if(S3fsLog::pSingleton != this){ S3FS_PRN_ERR("This object is not as same as S3fsLog::pSingleton."); return false; } char* pEnvVal; if(nullptr != (pEnvVal = getenv(S3fsLog::LOGFILEENV))){ if(!SetLogfile(pEnvVal)){ return false; } } if(nullptr != (pEnvVal = getenv(S3fsLog::MSGTIMESTAMP))){ if(0 == strcasecmp(pEnvVal, "true") || 0 == strcasecmp(pEnvVal, "yes") || 0 == strcasecmp(pEnvVal, "1")){ S3fsLog::time_stamp = true; }else if(0 == strcasecmp(pEnvVal, "false") || 0 == strcasecmp(pEnvVal, "no") || 0 == strcasecmp(pEnvVal, "0")){ S3fsLog::time_stamp = false; }else{ S3FS_PRN_WARN("Unknown %s environment value(%s) is specified, skip to set time stamp mode.", S3fsLog::MSGTIMESTAMP, pEnvVal); } } return true; } bool S3fsLog::LowSetLogfile(const char* pfile) { if(S3fsLog::pSingleton != this){ S3FS_PRN_ERR("This object is not as same as S3fsLog::pSingleton."); return false; } if(!pfile){ // close log file if it is opened if(S3fsLog::logfp && 0 != fclose(S3fsLog::logfp)){ S3FS_PRN_ERR("Could not close log file(%s).", (S3fsLog::logfile.empty() ? S3fsLog::logfile.c_str() : "null")); return false; } S3fsLog::logfp = nullptr; S3fsLog::logfile.clear(); }else{ // open new log file // // [NOTE] // It will reopen even if it is the same file. // FILE* newfp; if(nullptr == (newfp = fopen(pfile, "a+"))){ S3FS_PRN_ERR("Could not open log file(%s).", pfile); return false; } // switch new log file and close old log file if it is opened FILE* oldfp = S3fsLog::logfp; if(oldfp && 0 != fclose(oldfp)){ S3FS_PRN_ERR("Could not close old log file(%s).", (!S3fsLog::logfile.empty() ? S3fsLog::logfile.c_str() : "null")); fclose(newfp); return false; } S3fsLog::logfp = newfp; S3fsLog::logfile = pfile; } return true; } S3fsLog::s3fs_log_level S3fsLog::LowSetLogLevel(s3fs_log_level level) { if(S3fsLog::pSingleton != this){ S3FS_PRN_ERR("This object is not as same as S3fsLog::pSingleton."); return S3fsLog::debug_level; // Although it is an error, it returns the current value. } if(level == S3fsLog::debug_level){ return S3fsLog::debug_level; } s3fs_log_level old = S3fsLog::debug_level; S3fsLog::debug_level = level; setlogmask(LOG_UPTO(GetSyslogLevel(S3fsLog::debug_level))); S3FS_PRN_CRIT("change debug level from %sto %s", GetLevelString(old), GetLevelString(S3fsLog::debug_level)); return old; } S3fsLog::s3fs_log_level S3fsLog::LowBumpupLogLevel() { if(S3fsLog::pSingleton != this){ S3FS_PRN_ERR("This object is not as same as S3fsLog::pSingleton."); return S3fsLog::debug_level; // Although it is an error, it returns the current value. } s3fs_log_level old = S3fsLog::debug_level; S3fsLog::debug_level = ( LEVEL_CRIT == S3fsLog::debug_level ? LEVEL_ERR : LEVEL_ERR == S3fsLog::debug_level ? LEVEL_WARN : LEVEL_WARN == S3fsLog::debug_level ? LEVEL_INFO : LEVEL_INFO == S3fsLog::debug_level ? LEVEL_DBG : LEVEL_CRIT ); setlogmask(LOG_UPTO(GetSyslogLevel(S3fsLog::debug_level))); S3FS_PRN_CRIT("change debug level from %sto %s", GetLevelString(old), GetLevelString(S3fsLog::debug_level)); return old; } void s3fs_low_logprn(S3fsLog::s3fs_log_level level, const char* file, const char *func, int line, const char *fmt, ...) { if(S3fsLog::IsS3fsLogLevel(level)){ va_list va; va_start(va, fmt); size_t len = vsnprintf(nullptr, 0, fmt, va) + 1; va_end(va); std::unique_ptr message(new char[len]); va_start(va, fmt); vsnprintf(message.get(), len, fmt, va); va_end(va); if(foreground || S3fsLog::IsSetLogFile()){ S3fsLog::SeekEnd(); fprintf(S3fsLog::GetOutputLogFile(), "%s%s%s:%s(%d): %s\n", S3fsLog::GetCurrentTime().c_str(), S3fsLog::GetLevelString(level), file, func, line, message.get()); S3fsLog::Flush(); }else{ // TODO: why does this differ from s3fs_low_logprn2? syslog(S3fsLog::GetSyslogLevel(level), "%s%s:%s(%d): %s", instance_name.c_str(), file, func, line, message.get()); } } } void s3fs_low_logprn2(S3fsLog::s3fs_log_level level, int nest, const char* file, const char *func, int line, const char *fmt, ...) { if(S3fsLog::IsS3fsLogLevel(level)){ va_list va; va_start(va, fmt); size_t len = vsnprintf(nullptr, 0, fmt, va) + 1; va_end(va); std::unique_ptr message(new char[len]); va_start(va, fmt); vsnprintf(message.get(), len, fmt, va); va_end(va); if(foreground || S3fsLog::IsSetLogFile()){ S3fsLog::SeekEnd(); fprintf(S3fsLog::GetOutputLogFile(), "%s%s%s%s:%s(%d): %s\n", S3fsLog::GetCurrentTime().c_str(), S3fsLog::GetLevelString(level), S3fsLog::GetS3fsLogNest(nest), file, func, line, message.get()); S3fsLog::Flush(); }else{ syslog(S3fsLog::GetSyslogLevel(level), "%s%s%s", instance_name.c_str(), S3fsLog::GetS3fsLogNest(nest), message.get()); } } } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_logger.h000066400000000000000000000254031470675423500164640ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_LOGGER_H_ #define S3FS_LOGGER_H_ #include #include #include #include #include #include #include "common.h" #ifdef CLOCK_MONOTONIC_COARSE #define S3FS_CLOCK_MONOTONIC CLOCK_MONOTONIC_COARSE #else // case of OSX #define S3FS_CLOCK_MONOTONIC CLOCK_MONOTONIC #endif //------------------------------------------------------------------- // S3fsLog class //------------------------------------------------------------------- class S3fsLog { public: enum s3fs_log_level : uint8_t { LEVEL_CRIT = 0, // LEVEL_CRIT LEVEL_ERR = 1, // LEVEL_ERR LEVEL_WARN = 3, // LEVEL_WARNING LEVEL_INFO = 7, // LEVEL_INFO LEVEL_DBG = 15 // LEVEL_DEBUG }; protected: static constexpr int NEST_MAX = 4; static constexpr const char* nest_spaces[NEST_MAX] = {"", " ", " ", " "}; static constexpr char LOGFILEENV[] = "S3FS_LOGFILE"; static constexpr char MSGTIMESTAMP[] = "S3FS_MSGTIMESTAMP"; static S3fsLog* pSingleton; static s3fs_log_level debug_level; static FILE* logfp; static std::string logfile; static bool time_stamp; protected: bool LowLoadEnv(); bool LowSetLogfile(const char* pfile); s3fs_log_level LowSetLogLevel(s3fs_log_level level); s3fs_log_level LowBumpupLogLevel(); public: static bool IsS3fsLogLevel(s3fs_log_level level); static bool IsS3fsLogCrit() { return IsS3fsLogLevel(LEVEL_CRIT); } static bool IsS3fsLogErr() { return IsS3fsLogLevel(LEVEL_ERR); } static bool IsS3fsLogWarn() { return IsS3fsLogLevel(LEVEL_WARN); } static bool IsS3fsLogInfo() { return IsS3fsLogLevel(LEVEL_INFO); } static bool IsS3fsLogDbg() { return IsS3fsLogLevel(LEVEL_DBG); } static constexpr int GetSyslogLevel(s3fs_log_level level) { return ( LEVEL_DBG == (level & LEVEL_DBG) ? LOG_DEBUG : LEVEL_INFO == (level & LEVEL_DBG) ? LOG_INFO : LEVEL_WARN == (level & LEVEL_DBG) ? LOG_WARNING : LEVEL_ERR == (level & LEVEL_DBG) ? LOG_ERR : LOG_CRIT ); } static std::string GetCurrentTime(); static constexpr const char* GetLevelString(s3fs_log_level level) { return ( LEVEL_DBG == (level & LEVEL_DBG) ? "[DBG] " : LEVEL_INFO == (level & LEVEL_DBG) ? "[INF] " : LEVEL_WARN == (level & LEVEL_DBG) ? "[WAN] " : LEVEL_ERR == (level & LEVEL_DBG) ? "[ERR] " : "[CRT] " ); } static constexpr const char* GetS3fsLogNest(int nest) { return nest_spaces[nest < NEST_MAX ? nest : NEST_MAX - 1]; } static bool IsSetLogFile() { return (nullptr != logfp); } static FILE* GetOutputLogFile() { return (logfp ? logfp : stdout); } static FILE* GetErrorLogFile() { return (logfp ? logfp : stderr); } static void SeekEnd() { if(logfp){ fseek(logfp, 0, SEEK_END); } } static void Flush() { if(logfp){ fflush(logfp); } } static bool SetLogfile(const char* pfile); static bool ReopenLogfile(); static s3fs_log_level SetLogLevel(s3fs_log_level level); static s3fs_log_level BumpupLogLevel(); static bool SetTimeStamp(bool value); explicit S3fsLog(); ~S3fsLog(); S3fsLog(const S3fsLog&) = delete; S3fsLog(S3fsLog&&) = delete; S3fsLog& operator=(const S3fsLog&) = delete; S3fsLog& operator=(S3fsLog&&) = delete; }; //------------------------------------------------------------------- // Debug macros //------------------------------------------------------------------- void s3fs_low_logprn(S3fsLog::s3fs_log_level level, const char* file, const char *func, int line, const char *fmt, ...) __attribute__ ((format (printf, 5, 6))); #define S3FS_LOW_LOGPRN(level, fmt, ...) \ do{ \ s3fs_low_logprn(level, __FILE__, __func__, __LINE__, fmt, ##__VA_ARGS__); \ }while(0) void s3fs_low_logprn2(S3fsLog::s3fs_log_level level, int nest, const char* file, const char *func, int line, const char *fmt, ...) __attribute__ ((format (printf, 6, 7))); #define S3FS_LOW_LOGPRN2(level, nest, fmt, ...) \ do{ \ s3fs_low_logprn2(level, nest, __FILE__, __func__, __LINE__, fmt, ##__VA_ARGS__); \ }while(0) #define S3FS_LOW_CURLDBG(fmt, ...) \ do{ \ if(foreground || S3fsLog::IsSetLogFile()){ \ S3fsLog::SeekEnd(); \ fprintf(S3fsLog::GetOutputLogFile(), "%s[CURL DBG] " fmt "%s\n", S3fsLog::GetCurrentTime().c_str(), __VA_ARGS__); \ S3fsLog::Flush(); \ }else{ \ syslog(S3fsLog::GetSyslogLevel(S3fsLog::LEVEL_CRIT), "%s" fmt "%s", instance_name.c_str(), __VA_ARGS__); \ } \ }while(0) #define S3FS_LOW_LOGPRN_EXIT(fmt, ...) \ do{ \ if(foreground || S3fsLog::IsSetLogFile()){ \ S3fsLog::SeekEnd(); \ fprintf(S3fsLog::GetErrorLogFile(), "s3fs: " fmt "%s\n", __VA_ARGS__); \ S3fsLog::Flush(); \ }else{ \ fprintf(S3fsLog::GetErrorLogFile(), "s3fs: " fmt "%s\n", __VA_ARGS__); \ syslog(S3fsLog::GetSyslogLevel(S3fsLog::LEVEL_CRIT), "%ss3fs: " fmt "%s", instance_name.c_str(), __VA_ARGS__); \ } \ }while(0) // Special macro for init message #define S3FS_PRN_INIT_INFO(fmt, ...) \ do{ \ if(foreground || S3fsLog::IsSetLogFile()){ \ S3fsLog::SeekEnd(); \ fprintf(S3fsLog::GetOutputLogFile(), "%s%s%s%s:%s(%d): " fmt "%s\n", S3fsLog::GetCurrentTime().c_str(), S3fsLog::GetLevelString(S3fsLog::LEVEL_INFO), S3fsLog::GetS3fsLogNest(0), __FILE__, __func__, __LINE__, __VA_ARGS__, ""); \ S3fsLog::Flush(); \ }else{ \ syslog(S3fsLog::GetSyslogLevel(S3fsLog::LEVEL_INFO), "%s%s" fmt "%s", instance_name.c_str(), S3fsLog::GetS3fsLogNest(0), __VA_ARGS__, ""); \ } \ }while(0) #define S3FS_PRN_LAUNCH_INFO(fmt, ...) \ do{ \ if(foreground || S3fsLog::IsSetLogFile()){ \ S3fsLog::SeekEnd(); \ fprintf(S3fsLog::GetOutputLogFile(), "%s%s" fmt "%s\n", S3fsLog::GetCurrentTime().c_str(), S3fsLog::GetLevelString(S3fsLog::LEVEL_INFO), __VA_ARGS__, ""); \ S3fsLog::Flush(); \ }else{ \ syslog(S3fsLog::GetSyslogLevel(S3fsLog::LEVEL_INFO), "%s" fmt "%s", instance_name.c_str(), __VA_ARGS__, ""); \ } \ }while(0) // Special macro for checking cache files #define S3FS_LOW_CACHE(fp, fmt, ...) \ do{ \ if(foreground || S3fsLog::IsSetLogFile()){ \ S3fsLog::SeekEnd(); \ fprintf(fp, fmt "%s\n", __VA_ARGS__); \ S3fsLog::Flush(); \ }else{ \ syslog(S3fsLog::GetSyslogLevel(S3fsLog::LEVEL_INFO), "%s: " fmt "%s", instance_name.c_str(), __VA_ARGS__); \ } \ }while(0) // [NOTE] // small trick for VA_ARGS // #define S3FS_PRN_EXIT(fmt, ...) S3FS_LOW_LOGPRN_EXIT(fmt, ##__VA_ARGS__, "") #define S3FS_PRN_CRIT(fmt, ...) S3FS_LOW_LOGPRN(S3fsLog::LEVEL_CRIT, fmt, ##__VA_ARGS__) #define S3FS_PRN_ERR(fmt, ...) S3FS_LOW_LOGPRN(S3fsLog::LEVEL_ERR, fmt, ##__VA_ARGS__) #define S3FS_PRN_WARN(fmt, ...) S3FS_LOW_LOGPRN(S3fsLog::LEVEL_WARN, fmt, ##__VA_ARGS__) #define S3FS_PRN_DBG(fmt, ...) S3FS_LOW_LOGPRN(S3fsLog::LEVEL_DBG, fmt, ##__VA_ARGS__) #define S3FS_PRN_INFO(fmt, ...) S3FS_LOW_LOGPRN2(S3fsLog::LEVEL_INFO, 0, fmt, ##__VA_ARGS__) #define S3FS_PRN_INFO1(fmt, ...) S3FS_LOW_LOGPRN2(S3fsLog::LEVEL_INFO, 1, fmt, ##__VA_ARGS__) #define S3FS_PRN_INFO2(fmt, ...) S3FS_LOW_LOGPRN2(S3fsLog::LEVEL_INFO, 2, fmt, ##__VA_ARGS__) #define S3FS_PRN_INFO3(fmt, ...) S3FS_LOW_LOGPRN2(S3fsLog::LEVEL_INFO, 3, fmt, ##__VA_ARGS__) #define S3FS_PRN_CURL(fmt, ...) S3FS_LOW_CURLDBG(fmt, ##__VA_ARGS__, "") #define S3FS_PRN_CACHE(fp, ...) S3FS_LOW_CACHE(fp, ##__VA_ARGS__, "") // Macros to print log with fuse context #define PRINT_FUSE_CTX(level, indent, fmt, ...) do { \ if(S3fsLog::IsS3fsLogLevel(level)){ \ struct fuse_context *ctx = fuse_get_context(); \ if(ctx == NULL){ \ S3FS_LOW_LOGPRN2(level, indent, fmt, ##__VA_ARGS__); \ }else{ \ S3FS_LOW_LOGPRN2(level, indent, fmt"[pid=%u,uid=%u,gid=%u]",\ ##__VA_ARGS__, \ (unsigned int)(ctx->pid), \ (unsigned int)(ctx->uid), \ (unsigned int)(ctx->gid)); \ } \ } \ } while (0) #define FUSE_CTX_INFO(fmt, ...) do { \ PRINT_FUSE_CTX(S3fsLog::LEVEL_INFO, 0, fmt, ##__VA_ARGS__); \ } while (0) #define FUSE_CTX_INFO1(fmt, ...) do { \ PRINT_FUSE_CTX(S3fsLog::LEVEL_INFO, 1, fmt, ##__VA_ARGS__); \ } while (0) #define FUSE_CTX_DBG(fmt, ...) do { \ PRINT_FUSE_CTX(S3fsLog::LEVEL_DBG, 0, fmt, ##__VA_ARGS__); \ } while (0) #endif // S3FS_LOGGER_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_util.cpp000066400000000000000000000345311470675423500165170ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "s3fs_logger.h" #include "s3fs_util.h" #include "string_util.h" #include "s3fs_help.h" //------------------------------------------------------------------- // Global variables //------------------------------------------------------------------- std::string mount_prefix; static size_t max_password_size; static size_t max_group_name_length; //------------------------------------------------------------------- // Utilities //------------------------------------------------------------------- std::string get_realpath(const char *path) { std::string realpath = mount_prefix; realpath += path; return realpath; } void init_sysconf_vars() { // SUSv4tc1 says the following about _SC_GETGR_R_SIZE_MAX and // _SC_GETPW_R_SIZE_MAX: // Note that sysconf(_SC_GETGR_R_SIZE_MAX) may return -1 if // there is no hard limit on the size of the buffer needed to // store all the groups returned. errno = 0; long res = sysconf(_SC_GETPW_R_SIZE_MAX); if(0 > res){ if (errno != 0){ S3FS_PRN_ERR("could not get max password length."); abort(); } res = 1024; // default initial length } max_password_size = res; errno = 0; res = sysconf(_SC_GETGR_R_SIZE_MAX); if(0 > res) { if (errno != 0) { S3FS_PRN_ERR("could not get max group name length."); abort(); } res = 1024; // default initial length } max_group_name_length = res; } //------------------------------------------------------------------- // Utility for UID/GID //------------------------------------------------------------------- // get user name from uid std::string get_username(uid_t uid) { size_t maxlen = max_password_size; int result; struct passwd pwinfo; struct passwd* ppwinfo = nullptr; // make buffer std::unique_ptr pbuf(new char[maxlen]); // get pw information while(ERANGE == (result = getpwuid_r(uid, &pwinfo, pbuf.get(), maxlen, &ppwinfo))){ maxlen *= 2; pbuf.reset(new char[maxlen]); } if(0 != result){ S3FS_PRN_ERR("could not get pw information(%d).", result); return ""; } // check pw if(nullptr == ppwinfo){ return ""; } std::string name = SAFESTRPTR(ppwinfo->pw_name); return name; } int is_uid_include_group(uid_t uid, gid_t gid) { size_t maxlen = max_group_name_length; int result; struct group ginfo; struct group* pginfo = nullptr; // make buffer std::unique_ptr pbuf(new char[maxlen]); // get group information while(ERANGE == (result = getgrgid_r(gid, &ginfo, pbuf.get(), maxlen, &pginfo))){ maxlen *= 2; pbuf.reset(new char[maxlen]); } if(0 != result){ S3FS_PRN_ERR("could not get group information(%d).", result); return -result; } // check group if(nullptr == pginfo){ // there is not gid in group. return -EINVAL; } std::string username = get_username(uid); char** ppgr_mem; for(ppgr_mem = pginfo->gr_mem; ppgr_mem && *ppgr_mem; ppgr_mem++){ if(username == *ppgr_mem){ // Found username in group. return 1; } } return 0; } //------------------------------------------------------------------- // Utility for file and directory //------------------------------------------------------------------- // [NOTE] // basename/dirname returns a static variable pointer as the return value. // Normally this shouldn't be a problem, but in macos10 we found a case // where dirname didn't receive its return value correctly due to thread // conflicts. // To avoid this, exclusive control is performed by mutex. // static std::mutex basename_lock; std::string mydirname(const std::string& path) { const std::lock_guard lock(basename_lock); return mydirname(path.c_str()); } // safe variant of dirname // dirname clobbers path so let it operate on a tmp copy std::string mydirname(const char* path) { if(!path || '\0' == path[0]){ return ""; } char *buf = strdup(path); std::string result = dirname(buf); free(buf); return result; } std::string mybasename(const std::string& path) { const std::lock_guard data_lock(basename_lock); return mybasename(path.c_str()); } // safe variant of basename // basename clobbers path so let it operate on a tmp copy std::string mybasename(const char* path) { if(!path || '\0' == path[0]){ return ""; } char *buf = strdup(path); std::string result = basename(buf); free(buf); return result; } // mkdir --parents int mkdirp(const std::string& path, mode_t mode) { std::string base; std::string component; std::istringstream ss(path); while (getline(ss, component, '/')) { base += component + "/"; struct stat st; if(0 == stat(base.c_str(), &st)){ if(!S_ISDIR(st.st_mode)){ return EPERM; } }else{ if(0 != mkdir(base.c_str(), mode) && errno != EEXIST){ return errno; } } } return 0; } // get existed directory path std::string get_exist_directory_path(const std::string& path) { std::string existed("/"); // "/" is existed. std::string base; std::string component; std::istringstream ss(path); while (getline(ss, component, '/')) { if(base != "/"){ base += "/"; } base += component; struct stat st; if(0 == stat(base.c_str(), &st) && S_ISDIR(st.st_mode)){ existed = base; }else{ break; } } return existed; } bool check_exist_dir_permission(const char* dirpath) { if(!dirpath || '\0' == dirpath[0]){ return false; } // exists struct stat st; if(0 != stat(dirpath, &st)){ if(ENOENT == errno){ // dir does not exist return true; } if(EACCES == errno){ // could not access directory return false; } // something error occurred return false; } // check type if(!S_ISDIR(st.st_mode)){ // path is not directory return false; } // check permission uid_t myuid = geteuid(); if(myuid == st.st_uid){ if(S_IRWXU != (st.st_mode & S_IRWXU)){ return false; } }else{ if(1 == is_uid_include_group(myuid, st.st_gid)){ if(S_IRWXG != (st.st_mode & S_IRWXG)){ return false; } }else{ if(S_IRWXO != (st.st_mode & S_IRWXO)){ return false; } } } return true; } bool delete_files_in_dir(const char* dir, bool is_remove_own) { DIR* dp; struct dirent* dent; if(nullptr == (dp = opendir(dir))){ S3FS_PRN_ERR("could not open dir(%s) - errno(%d)", dir, errno); return false; } for(dent = readdir(dp); dent; dent = readdir(dp)){ if(0 == strcmp(dent->d_name, "..") || 0 == strcmp(dent->d_name, ".")){ continue; } std::string fullpath = dir; fullpath += "/"; fullpath += dent->d_name; struct stat st; if(0 != lstat(fullpath.c_str(), &st)){ S3FS_PRN_ERR("could not get stats of file(%s) - errno(%d)", fullpath.c_str(), errno); closedir(dp); return false; } if(S_ISDIR(st.st_mode)){ // dir -> Reentrant if(!delete_files_in_dir(fullpath.c_str(), true)){ S3FS_PRN_ERR("could not remove sub dir(%s) - errno(%d)", fullpath.c_str(), errno); closedir(dp); return false; } }else{ if(0 != unlink(fullpath.c_str())){ S3FS_PRN_ERR("could not remove file(%s) - errno(%d)", fullpath.c_str(), errno); closedir(dp); return false; } } } closedir(dp); if(is_remove_own && 0 != rmdir(dir)){ S3FS_PRN_ERR("could not remove dir(%s) - errno(%d)", dir, errno); return false; } return true; } //------------------------------------------------------------------- // Utility for system information //------------------------------------------------------------------- bool compare_sysname(const char* target) { // [NOTE] // The buffer size of sysname member in struct utsname is // OS dependent, but 512 bytes is sufficient for now. // static const char* psysname = nullptr; static char sysname[512]; if(!psysname){ struct utsname sysinfo; if(0 != uname(&sysinfo)){ S3FS_PRN_ERR("could not initialize system name to internal buffer(errno:%d), thus use \"Linux\".", errno); strcpy(sysname, "Linux"); }else{ S3FS_PRN_INFO("system name is %s", sysinfo.sysname); sysname[sizeof(sysname) - 1] = '\0'; strncpy(sysname, sysinfo.sysname, sizeof(sysname) - 1); } psysname = &sysname[0]; } if(!target || 0 != strcmp(psysname, target)){ return false; } return true; } //------------------------------------------------------------------- // Utility for print message at launching //------------------------------------------------------------------- void print_launch_message(int argc, char** argv) { std::string message = short_version(); if(argv){ message += " :"; for(int cnt = 0; cnt < argc; ++cnt){ if(argv[cnt]){ message += " "; if(0 == cnt){ message += basename(argv[cnt]); }else{ message += argv[cnt]; } } } } S3FS_PRN_LAUNCH_INFO("%s", message.c_str()); } // // result: -1 ts1 < ts2 // 0 ts1 == ts2 // 1 ts1 > ts2 // int compare_timespec(const struct timespec& ts1, const struct timespec& ts2) { if(ts1.tv_sec < ts2.tv_sec){ return -1; }else if(ts1.tv_sec > ts2.tv_sec){ return 1; }else{ if(ts1.tv_nsec < ts2.tv_nsec){ return -1; }else if(ts1.tv_nsec > ts2.tv_nsec){ return 1; } } return 0; } // // result: -1 st < ts // 0 st == ts // 1 st > ts // int compare_timespec(const struct stat& st, stat_time_type type, const struct timespec& ts) { struct timespec st_ts; set_stat_to_timespec(st, type, st_ts); return compare_timespec(st_ts, ts); } void set_timespec_to_stat(struct stat& st, stat_time_type type, const struct timespec& ts) { if(stat_time_type::ATIME == type){ #if defined(__APPLE__) st.st_atime = ts.tv_sec; st.st_atimespec.tv_nsec = ts.tv_nsec; #else st.st_atim.tv_sec = ts.tv_sec; st.st_atim.tv_nsec = ts.tv_nsec; #endif }else if(stat_time_type::MTIME == type){ #if defined(__APPLE__) st.st_mtime = ts.tv_sec; st.st_mtimespec.tv_nsec = ts.tv_nsec; #else st.st_mtim.tv_sec = ts.tv_sec; st.st_mtim.tv_nsec = ts.tv_nsec; #endif }else if(stat_time_type::CTIME == type){ #if defined(__APPLE__) st.st_ctime = ts.tv_sec; st.st_ctimespec.tv_nsec = ts.tv_nsec; #else st.st_ctim.tv_sec = ts.tv_sec; st.st_ctim.tv_nsec = ts.tv_nsec; #endif }else{ S3FS_PRN_ERR("unknown type(%d), so skip to set value.", static_cast(type)); } } struct timespec* set_stat_to_timespec(const struct stat& st, stat_time_type type, struct timespec& ts) { if(stat_time_type::ATIME == type){ #if defined(__APPLE__) ts.tv_sec = st.st_atime; ts.tv_nsec = st.st_atimespec.tv_nsec; #else ts = st.st_atim; #endif }else if(stat_time_type::MTIME == type){ #if defined(__APPLE__) ts.tv_sec = st.st_mtime; ts.tv_nsec = st.st_mtimespec.tv_nsec; #else ts = st.st_mtim; #endif }else if(stat_time_type::CTIME == type){ #if defined(__APPLE__) ts.tv_sec = st.st_ctime; ts.tv_nsec = st.st_ctimespec.tv_nsec; #else ts = st.st_ctim; #endif }else{ S3FS_PRN_ERR("unknown type(%d), so use 0 as timespec.", static_cast(type)); ts.tv_sec = 0; ts.tv_nsec = 0; } return &ts; } std::string str_stat_time(const struct stat& st, stat_time_type type) { struct timespec ts; return str(*set_stat_to_timespec(st, type, ts)); } struct timespec* s3fs_realtime(struct timespec& ts) { if(-1 == clock_gettime(static_cast(CLOCK_REALTIME), &ts)){ S3FS_PRN_WARN("failed to clock_gettime by errno(%d)", errno); ts.tv_sec = time(nullptr); ts.tv_nsec = 0; } return &ts; } std::string s3fs_str_realtime() { struct timespec ts; return str(*s3fs_realtime(ts)); } int s3fs_fclose(FILE* fp) { if(fp == nullptr){ return 0; } return fclose(fp); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_util.h000066400000000000000000000072251470675423500161640ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_S3FS_UTIL_H_ #define S3FS_S3FS_UTIL_H_ #include #include #include #include #ifndef CLOCK_REALTIME #define CLOCK_REALTIME 0 #endif #ifndef CLOCK_MONOTONIC #define CLOCK_MONOTONIC CLOCK_REALTIME #endif #ifndef CLOCK_MONOTONIC_COARSE #define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC #endif //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- std::string get_realpath(const char *path); void init_sysconf_vars(); std::string get_username(uid_t uid); int is_uid_include_group(uid_t uid, gid_t gid); std::string mydirname(const char* path); std::string mydirname(const std::string& path); std::string mybasename(const char* path); std::string mybasename(const std::string& path); int mkdirp(const std::string& path, mode_t mode); std::string get_exist_directory_path(const std::string& path); bool check_exist_dir_permission(const char* dirpath); bool delete_files_in_dir(const char* dir, bool is_remove_own); bool compare_sysname(const char* target); void print_launch_message(int argc, char** argv); // // Utility for nanosecond time(timespec) // enum class stat_time_type : uint8_t { ATIME, MTIME, CTIME }; //------------------------------------------------------------------- // Utility for nanosecond time(timespec) //------------------------------------------------------------------- static constexpr struct timespec S3FS_OMIT_TS = {0, UTIME_OMIT}; int compare_timespec(const struct timespec& ts1, const struct timespec& ts2); int compare_timespec(const struct stat& st, stat_time_type type, const struct timespec& ts); void set_timespec_to_stat(struct stat& st, stat_time_type type, const struct timespec& ts); struct timespec* set_stat_to_timespec(const struct stat& st, stat_time_type type, struct timespec& ts); std::string str_stat_time(const struct stat& st, stat_time_type type); struct timespec* s3fs_realtime(struct timespec& ts); std::string s3fs_str_realtime(); // Wrap fclose since it is illegal to take the address of a stdlib function int s3fs_fclose(FILE* fp); class scope_guard { public: template explicit scope_guard(Callable&& undo_func) : func(std::forward(undo_func)) {} ~scope_guard() { if(func != nullptr) { func(); } } void dismiss() { func = nullptr; } scope_guard(const scope_guard&) = delete; scope_guard(scope_guard&& other) = delete; scope_guard& operator=(const scope_guard&) = delete; scope_guard& operator=(scope_guard&&) = delete; private: std::function func; }; #endif // S3FS_S3FS_UTIL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_xml.cpp000066400000000000000000000410651470675423500163420ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include "common.h" #include "s3fs_logger.h" #include "s3fs_xml.h" #include "s3fs_util.h" #include "s3objlist.h" #include "string_util.h" //------------------------------------------------------------------- // Symbols //------------------------------------------------------------------- enum class get_object_name_result : std::uint8_t { SUCCESS, FAILURE, FILE_OR_SUBDIR_IN_DIR }; //------------------------------------------------------------------- // Variables //------------------------------------------------------------------- // [NOTE] // mutex for static variables in GetXmlNsUrl // static std::mutex xml_parser_mutex; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- static bool GetXmlNsUrl(xmlDocPtr doc, std::string& nsurl) { bool result = false; if(!doc){ return result; } std::string tmpNs; { static time_t tmLast = 0; // cache for 60 sec. static std::string strNs; const std::lock_guard lock(xml_parser_mutex); if((tmLast + 60) < time(nullptr)){ // refresh tmLast = time(nullptr); strNs = ""; xmlNodePtr pRootNode = xmlDocGetRootElement(doc); if(pRootNode){ std::unique_ptr nslist(xmlGetNsList(doc, pRootNode), xmlFree); if(nslist){ if(*nslist && (*nslist)[0].href){ int len = xmlStrlen((*nslist)[0].href); if(0 < len){ strNs = std::string(reinterpret_cast((*nslist)[0].href), len); } } } } } tmpNs = strNs; } if(!tmpNs.empty()){ nsurl = tmpNs; result = true; } return result; } static unique_ptr_xmlChar get_base_exp(xmlDocPtr doc, const char* exp) { std::string xmlnsurl; std::string exp_string; if(!doc){ return {nullptr, xmlFree}; } unique_ptr_xmlXPathContext ctx(xmlXPathNewContext(doc), xmlXPathFreeContext); if(!noxmlns && GetXmlNsUrl(doc, xmlnsurl)){ xmlXPathRegisterNs(ctx.get(), reinterpret_cast("s3"), reinterpret_cast(xmlnsurl.c_str())); exp_string = "/s3:ListBucketResult/s3:"; } else { exp_string = "/ListBucketResult/"; } exp_string += exp; unique_ptr_xmlXPathObject marker_xp(xmlXPathEvalExpression(reinterpret_cast(exp_string.c_str()), ctx.get()), xmlXPathFreeObject); if(nullptr == marker_xp){ return {nullptr, xmlFree}; } if(xmlXPathNodeSetIsEmpty(marker_xp->nodesetval)){ S3FS_PRN_INFO("marker_xp->nodesetval is empty."); return {nullptr, xmlFree}; } xmlNodeSetPtr nodes = marker_xp->nodesetval; unique_ptr_xmlChar result(xmlNodeListGetString(doc, nodes->nodeTab[0]->xmlChildrenNode, 1), xmlFree); return result; } static unique_ptr_xmlChar get_prefix(xmlDocPtr doc) { return get_base_exp(doc, "Prefix"); } unique_ptr_xmlChar get_next_continuation_token(xmlDocPtr doc) { return get_base_exp(doc, "NextContinuationToken"); } unique_ptr_xmlChar get_next_marker(xmlDocPtr doc) { return get_base_exp(doc, "NextMarker"); } static std::pair get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path) { // Get full path unique_ptr_xmlChar fullpath(xmlNodeListGetString(doc, node, 1), xmlFree); if(!fullpath){ S3FS_PRN_ERR("could not get object full path name.."); return {get_object_name_result::FAILURE, ""}; } // basepath(path) is as same as fullpath. if(0 == strcmp(reinterpret_cast(fullpath.get()), path)){ return {get_object_name_result::FILE_OR_SUBDIR_IN_DIR, ""}; } // Make dir path and filename std::string strdirpath = mydirname(reinterpret_cast(fullpath.get())); std::string strmybpath = mybasename(reinterpret_cast(fullpath.get())); const char* dirpath = strdirpath.c_str(); const char* mybname = strmybpath.c_str(); const char* basepath= (path && '/' == path[0]) ? &path[1] : path; if('\0' == mybname[0]){ return {get_object_name_result::FAILURE, ""}; } // check subdir & file in subdir if(0 < strlen(dirpath)){ // case of "/" if(0 == strcmp(mybname, "/") && 0 == strcmp(dirpath, "/")){ return {get_object_name_result::FILE_OR_SUBDIR_IN_DIR, ""}; } // case of "." if(0 == strcmp(mybname, ".") && 0 == strcmp(dirpath, ".")){ return {get_object_name_result::FILE_OR_SUBDIR_IN_DIR, ""}; } // case of ".." if(0 == strcmp(mybname, "..") && 0 == strcmp(dirpath, ".")){ return {get_object_name_result::FILE_OR_SUBDIR_IN_DIR, ""}; } // case of "name" if(0 == strcmp(dirpath, ".")){ // OK return {get_object_name_result::SUCCESS, mybname}; }else{ if(basepath && 0 == strcmp(dirpath, basepath)){ // OK return {get_object_name_result::SUCCESS, mybname}; }else if(basepath && 0 < strlen(basepath) && '/' == basepath[strlen(basepath) - 1] && 0 == strncmp(dirpath, basepath, strlen(basepath) - 1)){ std::string withdirname; if(strlen(dirpath) > strlen(basepath)){ withdirname = &dirpath[strlen(basepath)]; } // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!withdirname.empty() && '/' != *withdirname.rbegin()){ withdirname += "/"; } withdirname += mybname; return {get_object_name_result::SUCCESS, withdirname}; } } } // case of something wrong return {get_object_name_result::FILE_OR_SUBDIR_IN_DIR, ""}; } static unique_ptr_xmlChar get_exp_value_xml(xmlDocPtr doc, xmlXPathContextPtr ctx, const char* exp_key) { if(!doc || !ctx || !exp_key){ return {nullptr, xmlFree}; } xmlNodeSetPtr exp_nodes; // search exp_key tag unique_ptr_xmlXPathObject exp(xmlXPathEvalExpression(reinterpret_cast(exp_key), ctx), xmlXPathFreeObject); if(nullptr == exp){ S3FS_PRN_ERR("Could not find key(%s).", exp_key); return {nullptr, xmlFree}; } if(xmlXPathNodeSetIsEmpty(exp->nodesetval)){ S3FS_PRN_ERR("Key(%s) node is empty.", exp_key); return {nullptr, xmlFree}; } // get exp_key value & set in struct exp_nodes = exp->nodesetval; unique_ptr_xmlChar exp_value(xmlNodeListGetString(doc, exp_nodes->nodeTab[0]->xmlChildrenNode, 1), xmlFree); if(nullptr == exp_value){ S3FS_PRN_ERR("Key(%s) value is empty.", exp_key); return {nullptr, xmlFree}; } return exp_value; } bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list) { if(!doc){ return false; } unique_ptr_xmlXPathContext ctx(xmlXPathNewContext(doc), xmlXPathFreeContext); std::string xmlnsurl; std::string ex_upload = "//"; std::string ex_key; std::string ex_id; std::string ex_date; if(!noxmlns && GetXmlNsUrl(doc, xmlnsurl)){ xmlXPathRegisterNs(ctx.get(), reinterpret_cast("s3"), reinterpret_cast(xmlnsurl.c_str())); ex_upload += "s3:"; ex_key += "s3:"; ex_id += "s3:"; ex_date += "s3:"; } ex_upload += "Upload"; ex_key += "Key"; ex_id += "UploadId"; ex_date += "Initiated"; // get "Upload" Tags unique_ptr_xmlXPathObject upload_xp(xmlXPathEvalExpression(reinterpret_cast(ex_upload.c_str()), ctx.get()), xmlXPathFreeObject); if(nullptr == upload_xp){ S3FS_PRN_ERR("xmlXPathEvalExpression returns null."); return false; } if(xmlXPathNodeSetIsEmpty(upload_xp->nodesetval)){ S3FS_PRN_INFO("upload_xp->nodesetval is empty."); return true; } // Make list int cnt; xmlNodeSetPtr upload_nodes; list.clear(); for(cnt = 0, upload_nodes = upload_xp->nodesetval; cnt < upload_nodes->nodeNr; cnt++){ ctx->node = upload_nodes->nodeTab[cnt]; INCOMP_MPU_INFO part; // search "Key" tag unique_ptr_xmlChar ex_value(get_exp_value_xml(doc, ctx.get(), ex_key.c_str())); if(nullptr == ex_value){ continue; } if('/' != *(reinterpret_cast(ex_value.get()))){ part.key = "/"; }else{ part.key = ""; } part.key += reinterpret_cast(ex_value.get()); // search "UploadId" tag if(nullptr == (ex_value = get_exp_value_xml(doc, ctx.get(), ex_id.c_str()))){ continue; } part.id = reinterpret_cast(ex_value.get()); // search "Initiated" tag if(nullptr == (ex_value = get_exp_value_xml(doc, ctx.get(), ex_date.c_str()))){ continue; } part.date = reinterpret_cast(ex_value.get()); list.push_back(part); } return true; } bool is_truncated(xmlDocPtr doc) { unique_ptr_xmlChar strTruncate(get_base_exp(doc, "IsTruncated")); if(!strTruncate){ return false; } return 0 == strcasecmp(reinterpret_cast(strTruncate.get()), "true"); } int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head, bool prefix) { xmlNodeSetPtr content_nodes; unique_ptr_xmlXPathObject contents_xp(xmlXPathEvalExpression(reinterpret_cast(ex_contents), ctx), xmlXPathFreeObject); if(nullptr == contents_xp){ S3FS_PRN_ERR("xmlXPathEvalExpression returns null."); return -1; } if(xmlXPathNodeSetIsEmpty(contents_xp->nodesetval)){ S3FS_PRN_DBG("contents_xp->nodesetval is empty."); return 0; } content_nodes = contents_xp->nodesetval; bool is_dir; std::string stretag; int i; for(i = 0; i < content_nodes->nodeNr; i++){ ctx->node = content_nodes->nodeTab[i]; // object name unique_ptr_xmlXPathObject key(xmlXPathEvalExpression(reinterpret_cast(ex_key), ctx), xmlXPathFreeObject); if(nullptr == key){ S3FS_PRN_WARN("key is null. but continue."); continue; } if(xmlXPathNodeSetIsEmpty(key->nodesetval)){ S3FS_PRN_WARN("node is empty. but continue."); continue; } xmlNodeSetPtr key_nodes = key->nodesetval; auto result = get_object_name(doc, key_nodes->nodeTab[0]->xmlChildrenNode, path); switch(result.first){ case get_object_name_result::FAILURE: S3FS_PRN_WARN("name is something wrong. but continue."); break; case get_object_name_result::SUCCESS: { is_dir = isCPrefix ? true : false; stretag = ""; if(!isCPrefix && ex_etag){ // Get ETag unique_ptr_xmlXPathObject ETag(xmlXPathEvalExpression(reinterpret_cast(ex_etag), ctx), xmlXPathFreeObject); if(nullptr != ETag){ if(xmlXPathNodeSetIsEmpty(ETag->nodesetval)){ S3FS_PRN_INFO("ETag->nodesetval is empty."); }else{ xmlNodeSetPtr etag_nodes = ETag->nodesetval; unique_ptr_xmlChar petag(xmlNodeListGetString(doc, etag_nodes->nodeTab[0]->xmlChildrenNode, 1), xmlFree); if(petag){ stretag = reinterpret_cast(petag.get()); } } } } // [NOTE] // The XML data passed to this function is CR code(\r) encoded. // The function below decodes that encoded CR code. // std::string decname = get_decoded_cr_code(result.second.c_str()); if(prefix){ head.AddCommonPrefix(decname); } if(!head.insert(decname.c_str(), (!stretag.empty() ? stretag.c_str() : nullptr), is_dir)){ S3FS_PRN_ERR("insert_object returns with error."); return -1; } break; } case get_object_name_result::FILE_OR_SUBDIR_IN_DIR: S3FS_PRN_DBG("name is file or subdir in dir. but continue."); break; } } return 0; } int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head) { std::string xmlnsurl; std::string ex_contents = "//"; std::string ex_key; std::string ex_cprefix = "//"; std::string ex_prefix; std::string ex_etag; if(!doc){ return -1; } // If there is not , use path instead of it. auto pprefix = get_prefix(doc); std::string prefix = (pprefix ? reinterpret_cast(pprefix.get()) : path ? path : ""); unique_ptr_xmlXPathContext ctx(xmlXPathNewContext(doc), xmlXPathFreeContext); if(!noxmlns && GetXmlNsUrl(doc, xmlnsurl)){ xmlXPathRegisterNs(ctx.get(), reinterpret_cast("s3"), reinterpret_cast(xmlnsurl.c_str())); ex_contents+= "s3:"; ex_key += "s3:"; ex_cprefix += "s3:"; ex_prefix += "s3:"; ex_etag += "s3:"; } ex_contents+= "Contents"; ex_key += "Key"; ex_cprefix += "CommonPrefixes"; ex_prefix += "Prefix"; ex_etag += "ETag"; if(-1 == append_objects_from_xml_ex(prefix.c_str(), doc, ctx.get(), ex_contents.c_str(), ex_key.c_str(), ex_etag.c_str(), 0, head, /*prefix=*/ false) || -1 == append_objects_from_xml_ex(prefix.c_str(), doc, ctx.get(), ex_cprefix.c_str(), ex_prefix.c_str(), nullptr, 1, head, /*prefix=*/ true) ) { S3FS_PRN_ERR("append_objects_from_xml_ex returns with error."); return -1; } return 0; } //------------------------------------------------------------------- // Utility functions //------------------------------------------------------------------- bool simple_parse_xml(const char* data, size_t len, const char* key, std::string& value) { bool result = false; if(!data || !key){ return false; } value.clear(); std::unique_ptr doc(xmlReadMemory(data, static_cast(len), "", nullptr, 0), xmlFreeDoc); if(nullptr == doc){ return false; } if(nullptr == doc->children){ return false; } for(xmlNodePtr cur_node = doc->children->children; nullptr != cur_node; cur_node = cur_node->next){ // For DEBUG // std::string cur_node_name(reinterpret_cast(cur_node->name)); // printf("cur_node_name: %s\n", cur_node_name.c_str()); if(XML_ELEMENT_NODE == cur_node->type){ std::string elementName = reinterpret_cast(cur_node->name); // For DEBUG // printf("elementName: %s\n", elementName.c_str()); if(cur_node->children){ if(XML_TEXT_NODE == cur_node->children->type){ if(elementName == key) { value = reinterpret_cast(cur_node->children->content); result = true; break; } } } } } return result; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3fs_xml.h000066400000000000000000000044371470675423500160110ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_S3FS_XML_H_ #define S3FS_S3FS_XML_H_ #include #include // [NOTE] nessetially include this header in some environments #include #include #include "mpu_util.h" class S3ObjList; typedef std::unique_ptr unique_ptr_xmlChar; typedef std::unique_ptr unique_ptr_xmlXPathObject; typedef std::unique_ptr unique_ptr_xmlXPathContext; typedef std::unique_ptr unique_ptr_xmlDoc; //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- bool is_truncated(xmlDocPtr doc); int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head, bool prefix); int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head); unique_ptr_xmlChar get_next_continuation_token(xmlDocPtr doc); unique_ptr_xmlChar get_next_marker(xmlDocPtr doc); bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list); bool simple_parse_xml(const char* data, size_t len, const char* key, std::string& value); #endif // S3FS_S3FS_XML_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3objlist.cpp000066400000000000000000000172771470675423500165300ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include "s3objlist.h" //------------------------------------------------------------------- // Class S3ObjList //------------------------------------------------------------------- // New class S3ObjList is base on old s3_object struct. // This class is for S3 compatible clients. // // If name is terminated by "/", it is forced dir type. // If name is terminated by "_$folder$", it is forced dir type. // If is_dir is true and name is not terminated by "/", the name is added "/". // bool S3ObjList::insert(const char* name, const char* etag, bool is_dir) { if(!name || '\0' == name[0]){ return false; } s3obj_t::iterator iter; std::string newname; std::string orgname = name; // Normalization std::string::size_type pos = orgname.find("_$folder$"); if(std::string::npos != pos){ newname = orgname.substr(0, pos); is_dir = true; }else{ newname = orgname; } if(is_dir){ if('/' != *newname.rbegin()){ newname += "/"; } }else{ if('/' == *newname.rbegin()){ is_dir = true; } } // Check derived name object. if(is_dir){ std::string chkname = newname.substr(0, newname.length() - 1); if(objects.cend() != (iter = objects.find(chkname))){ // found "dir" object --> remove it. objects.erase(iter); } }else{ std::string chkname = newname + "/"; if(objects.cend() != (iter = objects.find(chkname))){ // found "dir/" object --> not add new object. // and add normalization return insert_normalized(orgname.c_str(), chkname.c_str(), true); } } // Add object if(objects.cend() != (iter = objects.find(newname))){ // Found same object --> update information. (*iter).second.normalname.clear(); (*iter).second.orgname = orgname; (*iter).second.is_dir = is_dir; if(etag){ (*iter).second.etag = etag; // over write } }else{ // add new object s3obj_entry newobject; newobject.orgname = orgname; newobject.is_dir = is_dir; if(etag){ newobject.etag = etag; } objects[newname] = newobject; } // add normalization return insert_normalized(orgname.c_str(), newname.c_str(), is_dir); } bool S3ObjList::insert_normalized(const char* name, const char* normalized, bool is_dir) { if(!name || '\0' == name[0] || !normalized || '\0' == normalized[0]){ return false; } if(0 == strcmp(name, normalized)){ return true; } s3obj_t::iterator iter; if(objects.cend() != (iter = objects.find(name))){ // found name --> over write iter->second.orgname.clear(); iter->second.etag.clear(); iter->second.normalname = normalized; iter->second.is_dir = is_dir; }else{ // not found --> add new object s3obj_entry newobject; newobject.normalname = normalized; newobject.is_dir = is_dir; objects[name] = newobject; } return true; } const s3obj_entry* S3ObjList::GetS3Obj(const char* name) const { s3obj_t::const_iterator iter; if(!name || '\0' == name[0]){ return nullptr; } if(objects.cend() == (iter = objects.find(name))){ return nullptr; } return &((*iter).second); } std::string S3ObjList::GetOrgName(const char* name) const { const s3obj_entry* ps3obj; if(!name || '\0' == name[0]){ return ""; } if(nullptr == (ps3obj = GetS3Obj(name))){ return ""; } return ps3obj->orgname; } std::string S3ObjList::GetNormalizedName(const char* name) const { const s3obj_entry* ps3obj; if(!name || '\0' == name[0]){ return ""; } if(nullptr == (ps3obj = GetS3Obj(name))){ return ""; } if(ps3obj->normalname.empty()){ return name; } return ps3obj->normalname; } std::string S3ObjList::GetETag(const char* name) const { const s3obj_entry* ps3obj; if(!name || '\0' == name[0]){ return ""; } if(nullptr == (ps3obj = GetS3Obj(name))){ return ""; } return ps3obj->etag; } bool S3ObjList::IsDir(const char* name) const { const s3obj_entry* ps3obj; if(nullptr == (ps3obj = GetS3Obj(name))){ return false; } return ps3obj->is_dir; } bool S3ObjList::GetLastName(std::string& lastname) const { bool result = false; lastname = ""; for(auto iter = objects.cbegin(); iter != objects.cend(); ++iter){ if(!iter->second.orgname.empty()){ if(lastname.compare(iter->second.orgname) < 0){ lastname = (*iter).second.orgname; result = true; } }else{ if(lastname.compare(iter->second.normalname) < 0){ lastname = (*iter).second.normalname; result = true; } } } return result; } bool S3ObjList::GetNameList(s3obj_list_t& list, bool OnlyNormalized, bool CutSlash) const { s3obj_t::const_iterator iter; for(iter = objects.cbegin(); objects.cend() != iter; ++iter){ if(OnlyNormalized && !iter->second.normalname.empty()){ continue; } std::string name = (*iter).first; if(CutSlash && 1 < name.length() && '/' == *name.rbegin()){ // only "/" std::string is skipped this. name.erase(name.length() - 1); } list.push_back(name); } return true; } typedef std::map s3obj_h_t; bool S3ObjList::MakeHierarchizedList(s3obj_list_t& list, bool haveSlash) { s3obj_h_t h_map; for(auto liter = list.cbegin(); list.cend() != liter; ++liter){ std::string strtmp = (*liter); if(1 < strtmp.length() && '/' == *strtmp.rbegin()){ strtmp.erase(strtmp.length() - 1); } h_map[strtmp] = true; // check hierarchized directory for(std::string::size_type pos = strtmp.find_last_of('/'); std::string::npos != pos; pos = strtmp.find_last_of('/')){ strtmp.erase(pos); if(strtmp.empty() || "/" == strtmp){ break; } if(h_map.cend() == h_map.find(strtmp)){ // not found h_map[strtmp] = false; } } } // check map and add lost hierarchized directory. for(auto hiter = h_map.cbegin(); hiter != h_map.cend(); ++hiter){ if(false == (*hiter).second){ // add hierarchized directory. std::string strtmp = (*hiter).first; if(haveSlash){ strtmp += "/"; } list.push_back(strtmp); } } return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/s3objlist.h000066400000000000000000000057061470675423500161670ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_S3OBJLIST_H_ #define S3FS_S3OBJLIST_H_ #include #include #include #include //------------------------------------------------------------------- // Structure / Typedef //------------------------------------------------------------------- struct s3obj_entry{ std::string normalname; // normalized name: if empty, object is normalized name. std::string orgname; // original name: if empty, object is original name. std::string etag; bool is_dir = false; }; typedef std::map s3obj_t; typedef std::vector s3obj_list_t; //------------------------------------------------------------------- // Class S3ObjList //------------------------------------------------------------------- class S3ObjList { private: s3obj_t objects; std::vector common_prefixes; bool insert_normalized(const char* name, const char* normalized, bool is_dir); const s3obj_entry* GetS3Obj(const char* name) const; s3obj_t::const_iterator cbegin() const { return objects.cbegin(); } s3obj_t::const_iterator cend() const { return objects.cend(); } public: bool IsEmpty() const { return objects.empty(); } bool insert(const char* name, const char* etag = nullptr, bool is_dir = false); std::string GetOrgName(const char* name) const; std::string GetNormalizedName(const char* name) const; std::string GetETag(const char* name) const; const std::vector& GetCommonPrefixes() const { return common_prefixes; } void AddCommonPrefix(std::string prefix) { common_prefixes.push_back(std::move(prefix)); } bool IsDir(const char* name) const; bool GetNameList(s3obj_list_t& list, bool OnlyNormalized = true, bool CutSlash = true) const; bool GetLastName(std::string& lastname) const; static bool MakeHierarchizedList(s3obj_list_t& list, bool haveSlash); }; #endif // S3FS_S3OBJLIST_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/sighandlers.cpp000066400000000000000000000143371470675423500171110ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include "psemaphore.h" #include "s3fs_logger.h" #include "sighandlers.h" #include "fdcache.h" //------------------------------------------------------------------- // Class S3fsSignals //------------------------------------------------------------------- std::unique_ptr S3fsSignals::pSingleton; bool S3fsSignals::enableUsr1 = false; //------------------------------------------------------------------- // Class methods //------------------------------------------------------------------- bool S3fsSignals::Initialize() { if(!S3fsSignals::pSingleton){ S3fsSignals::pSingleton.reset(new S3fsSignals); } return true; } bool S3fsSignals::Destroy() { S3fsSignals::pSingleton.reset(); return true; } void S3fsSignals::HandlerUSR1(int sig) { if(SIGUSR1 != sig){ S3FS_PRN_ERR("The handler for SIGUSR1 received signal(%d)", sig); return; } S3fsSignals* pSigobj = S3fsSignals::get(); if(!pSigobj){ S3FS_PRN_ERR("S3fsSignals object is not initialized."); return; } if(!pSigobj->WakeupUsr1Thread()){ S3FS_PRN_ERR("Failed to wakeup the thread for SIGUSR1."); return; } } bool S3fsSignals::SetUsr1Handler(const char* path) { if(!FdManager::HaveLseekHole()){ S3FS_PRN_ERR("Could not set SIGUSR1 for checking cache, because this system does not support SEEK_DATA/SEEK_HOLE in lseek function."); return false; } // set output file if(!FdManager::SetCacheCheckOutput(path)){ S3FS_PRN_ERR("Could not set output file(%s) for checking cache.", path ? path : "null(stdout)"); return false; } S3fsSignals::enableUsr1 = true; return true; } void S3fsSignals::CheckCacheWorker(Semaphore* pSem) { if(!pSem){ return; } if(!S3fsSignals::enableUsr1){ return; } // wait and loop while(S3fsSignals::enableUsr1){ // wait pSem->wait(); // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!S3fsSignals::enableUsr1){ break; // assap } // check all cache if(!FdManager::get()->CheckAllCache()){ S3FS_PRN_ERR("Processing failed due to some problem."); } // do not allow request queuing while(pSem->try_wait()); } } void S3fsSignals::HandlerUSR2(int sig) { if(SIGUSR2 == sig){ S3fsLog::BumpupLogLevel(); }else{ S3FS_PRN_ERR("The handler for SIGUSR2 received signal(%d)", sig); } } bool S3fsSignals::InitUsr2Handler() { struct sigaction sa{}; sa.sa_handler = S3fsSignals::HandlerUSR2; sa.sa_flags = SA_RESTART; if(0 != sigaction(SIGUSR2, &sa, nullptr)){ return false; } return true; } void S3fsSignals::HandlerHUP(int sig) { if(SIGHUP == sig){ S3fsLog::ReopenLogfile(); }else{ S3FS_PRN_ERR("The handler for SIGHUP received signal(%d)", sig); } } bool S3fsSignals::InitHupHandler() { struct sigaction sa{}; sa.sa_handler = S3fsSignals::HandlerHUP; sa.sa_flags = SA_RESTART; if(0 != sigaction(SIGHUP, &sa, nullptr)){ return false; } return true; } //------------------------------------------------------------------- // Methods //------------------------------------------------------------------- S3fsSignals::S3fsSignals() { if(S3fsSignals::enableUsr1){ if(!InitUsr1Handler()){ S3FS_PRN_ERR("failed creating thread for SIGUSR1 handler, but continue..."); } } if(!S3fsSignals::InitUsr2Handler()){ S3FS_PRN_ERR("failed to initialize SIGUSR2 handler for bumping log level, but continue..."); } if(!S3fsSignals::InitHupHandler()){ S3FS_PRN_ERR("failed to initialize SIGHUP handler for reopen log file, but continue..."); } } S3fsSignals::~S3fsSignals() { if(S3fsSignals::enableUsr1){ if(!DestroyUsr1Handler()){ S3FS_PRN_ERR("failed stopping thread for SIGUSR1 handler, but continue..."); } } } bool S3fsSignals::InitUsr1Handler() { if(pThreadUsr1 || pSemUsr1){ S3FS_PRN_ERR("Already run thread for SIGUSR1"); return false; } // create thread std::unique_ptr pSemUsr1_tmp(new Semaphore(0)); pThreadUsr1.reset(new std::thread(S3fsSignals::CheckCacheWorker, pSemUsr1_tmp.get())); pSemUsr1 = std::move(pSemUsr1_tmp); // set handler struct sigaction sa{}; sa.sa_handler = S3fsSignals::HandlerUSR1; sa.sa_flags = SA_RESTART; if(0 != sigaction(SIGUSR1, &sa, nullptr)){ S3FS_PRN_ERR("Could not set signal handler for SIGUSR1"); DestroyUsr1Handler(); return false; } return true; } bool S3fsSignals::DestroyUsr1Handler() { if(!pThreadUsr1 || !pSemUsr1){ return false; } // for thread exit S3fsSignals::enableUsr1 = false; // wakeup thread pSemUsr1->post(); // wait for thread exiting pThreadUsr1->join(); pSemUsr1.reset(); pThreadUsr1.reset(); return true; } bool S3fsSignals::WakeupUsr1Thread() { if(!pThreadUsr1 || !pSemUsr1){ S3FS_PRN_ERR("The thread for SIGUSR1 is not setup."); return false; } pSemUsr1->post(); return true; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/sighandlers.h000066400000000000000000000044011470675423500165450ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_SIGHANDLERS_H_ #define S3FS_SIGHANDLERS_H_ #include #include class Semaphore; //---------------------------------------------- // class S3fsSignals //---------------------------------------------- class S3fsSignals { private: static std::unique_ptr pSingleton; static bool enableUsr1; std::unique_ptr pThreadUsr1; std::unique_ptr pSemUsr1; protected: static S3fsSignals* get() { return pSingleton.get(); } static void HandlerUSR1(int sig); static void CheckCacheWorker(Semaphore* pSem); static void HandlerUSR2(int sig); static bool InitUsr2Handler(); static void HandlerHUP(int sig); static bool InitHupHandler(); S3fsSignals(); bool InitUsr1Handler(); bool DestroyUsr1Handler(); bool WakeupUsr1Thread(); public: ~S3fsSignals(); S3fsSignals(const S3fsSignals&) = delete; S3fsSignals(S3fsSignals&&) = delete; S3fsSignals& operator=(const S3fsSignals&) = delete; S3fsSignals& operator=(S3fsSignals&&) = delete; static bool Initialize(); static bool Destroy(); static bool SetUsr1Handler(const char* path); }; #endif // S3FS_SIGHANDLERS_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/string_util.cpp000066400000000000000000000511661470675423500171520ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include "s3fs_logger.h" #include "string_util.h" //------------------------------------------------------------------- // Global variables //------------------------------------------------------------------- //------------------------------------------------------------------- // Functions //------------------------------------------------------------------- std::string str(const struct timespec& value) { std::ostringstream s; s << value.tv_sec; if(value.tv_nsec != 0){ s << "." << std::setfill('0') << std::setw(9) << value.tv_nsec; } return s.str(); } #ifdef __MSYS__ /* * Polyfill for strptime function * * This source code is from https://gist.github.com/jeremyfromearth/5694aa3a66714254752179ecf3c95582 . */ char* strptime(const char* s, const char* f, struct tm* tm) { std::istringstream input(s); input.imbue(std::locale(setlocale(LC_ALL, nullptr))); input >> std::get_time(tm, f); if (input.fail()) { return nullptr; } return (char*)(s + input.tellg()); } #endif bool s3fs_strtoofft(off_t* value, const char* str, int base) { if(value == nullptr || str == nullptr){ return false; } errno = 0; char *temp; long long result = strtoll(str, &temp, base); if(temp == str || *temp != '\0'){ return false; } if((result == LLONG_MIN || result == LLONG_MAX) && errno == ERANGE){ return false; } *value = result; return true; } off_t cvt_strtoofft(const char* str, int base) { off_t result = 0; if(!s3fs_strtoofft(&result, str, base)){ S3FS_PRN_WARN("something error is occurred in convert std::string(%s) to off_t, thus return 0 as default.", (str ? str : "null")); return 0; } return result; } std::string lower(std::string s) { // change each character of the std::string to lower case for(size_t i = 0; i < s.length(); i++){ s[i] = static_cast(tolower(s[i])); } return s; } std::string trim_left(std::string d, const char *t /* = SPACES */) { return d.erase(0, d.find_first_not_of(t)); } std::string trim_right(std::string d, const char *t /* = SPACES */) { std::string::size_type i(d.find_last_not_of(t)); if(i == std::string::npos){ return ""; }else{ return d.erase(d.find_last_not_of(t) + 1); } } std::string trim(std::string s, const char *t /* = SPACES */) { return trim_left(trim_right(std::move(s), t), t); } std::string peeloff(const std::string& s) { if(s.size() < 2 || *s.cbegin() != '"' || *s.rbegin() != '"'){ return s; } return s.substr(1, s.size() - 2); } // // Three url encode functions // // urlEncodeGeneral: A general URL encoding function. // urlEncodePath : A function that URL encodes by excluding the path // separator('/'). // urlEncodeQuery : A function that does URL encoding by excluding // some characters('=', '&' and '%'). // This function can be used when the target string // contains already URL encoded strings. It also // excludes the character () used in query strings. // Therefore, it is a function to use as URL encoding // for use in query strings. // static constexpr char encode_general_except_chars[] = ".-_~"; // For general URL encode static constexpr char encode_path_except_chars[] = ".-_~/"; // For fuse(included path) URL encode static constexpr char encode_query_except_chars[] = ".-_~=&%"; // For query params(and encoded string) static std::string rawUrlEncode(const std::string &s, const char* except_chars) { std::string result; for (size_t i = 0; i < s.length(); ++i) { unsigned char c = s[i]; if((except_chars && nullptr != strchr(except_chars, c)) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ) { result += c; }else{ result += "%"; result += s3fs_hex_upper(&c, 1); } } return result; } std::string urlEncodeGeneral(const std::string &s) { return rawUrlEncode(s, encode_general_except_chars); } std::string urlEncodePath(const std::string &s) { return rawUrlEncode(s, encode_path_except_chars); } std::string urlEncodeQuery(const std::string &s) { return rawUrlEncode(s, encode_query_except_chars); } std::string urlDecode(const std::string& s) { std::string result; for(size_t i = 0; i < s.length(); ++i){ if(s[i] != '%'){ result += s[i]; }else{ int ch = 0; if(s.length() <= ++i){ break; // wrong format. } ch += ('0' <= s[i] && s[i] <= '9') ? (s[i] - '0') : ('A' <= s[i] && s[i] <= 'F') ? (s[i] - 'A' + 0x0a) : ('a' <= s[i] && s[i] <= 'f') ? (s[i] - 'a' + 0x0a) : 0x00; if(s.length() <= ++i){ break; // wrong format. } ch *= 16; ch += ('0' <= s[i] && s[i] <= '9') ? (s[i] - '0') : ('A' <= s[i] && s[i] <= 'F') ? (s[i] - 'A' + 0x0a) : ('a' <= s[i] && s[i] <= 'f') ? (s[i] - 'a' + 0x0a) : 0x00; result += static_cast(ch); } } return result; } bool takeout_str_dquart(std::string& str) { size_t pos; // '"' for start if(std::string::npos != (pos = str.find_first_of('\"'))){ str.erase(0, pos + 1); // '"' for end if(std::string::npos == (pos = str.find_last_of('\"'))){ return false; } str.erase(pos); if(std::string::npos != str.find_first_of('\"')){ return false; } } return true; } // // ex. target="http://......?keyword=value&..." // bool get_keyword_value(const std::string& target, const char* keyword, std::string& value) { if(!keyword){ return false; } size_t spos; size_t epos; if(std::string::npos == (spos = target.find(keyword))){ return false; } spos += strlen(keyword); if('=' != target[spos]){ return false; } spos++; if(std::string::npos == (epos = target.find('&', spos))){ value = target.substr(spos); }else{ value = target.substr(spos, (epos - spos)); } return true; } // // Returns the current date // in a format suitable for a HTTP request header. // std::string get_date_rfc850() { char buf[100]; time_t t = time(nullptr); struct tm res; strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S GMT", gmtime_r(&t, &res)); return buf; } void get_date_sigv3(std::string& date, std::string& date8601) { time_t tm = time(nullptr); date = get_date_string(tm); date8601 = get_date_iso8601(tm); } std::string get_date_string(time_t tm) { char buf[100]; struct tm res; strftime(buf, sizeof(buf), "%Y%m%d", gmtime_r(&tm, &res)); return buf; } std::string get_date_iso8601(time_t tm) { char buf[100]; struct tm res; strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime_r(&tm, &res)); return buf; } bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime) { if(!pdate){ return false; } struct tm tm; const char* prest = strptime(pdate, "%Y-%m-%dT%T", &tm); if(prest == pdate){ // wrong format return false; } unixtime = mktime(&tm); return true; } // // Convert to unixtime from std::string which formatted by following: // "12Y12M12D12h12m12s", "86400s", "9h30m", etc // bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime) { if(!argv){ return false; } unixtime = 0; const char* ptmp; int last_unit_type = 0; // unit flag. bool is_last_number; time_t tmptime; for(ptmp = argv, is_last_number = true, tmptime = 0; ptmp && *ptmp; ++ptmp){ if('0' <= *ptmp && *ptmp <= '9'){ tmptime *= 10; tmptime += static_cast(*ptmp - '0'); is_last_number = true; }else if(is_last_number){ if('Y' == *ptmp && 1 > last_unit_type){ unixtime += (tmptime * (60 * 60 * 24 * 365)); // average 365 day / year last_unit_type = 1; }else if('M' == *ptmp && 2 > last_unit_type){ unixtime += (tmptime * (60 * 60 * 24 * 30)); // average 30 day / month last_unit_type = 2; }else if('D' == *ptmp && 3 > last_unit_type){ unixtime += (tmptime * (60 * 60 * 24)); last_unit_type = 3; }else if('h' == *ptmp && 4 > last_unit_type){ unixtime += (tmptime * (60 * 60)); last_unit_type = 4; }else if('m' == *ptmp && 5 > last_unit_type){ unixtime += (tmptime * 60); last_unit_type = 5; }else if('s' == *ptmp && 6 > last_unit_type){ unixtime += tmptime; last_unit_type = 6; }else{ return false; } tmptime = 0; is_last_number = false; }else{ return false; } } if(is_last_number){ return false; } return true; } static std::string s3fs_hex(const unsigned char* input, size_t length, const char *hexAlphabet) { std::string hex; for(size_t pos = 0; pos < length; ++pos){ hex += hexAlphabet[input[pos] / 16]; hex += hexAlphabet[input[pos] % 16]; } return hex; } std::string s3fs_hex_lower(const unsigned char* input, size_t length) { return s3fs_hex(input, length, "0123456789abcdef"); } std::string s3fs_hex_upper(const unsigned char* input, size_t length) { return s3fs_hex(input, length, "0123456789ABCDEF"); } std::string s3fs_base64(const unsigned char* input, size_t length) { static constexpr char base[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="; std::string result; result.reserve(((length + 3 - 1) / 3) * 4 + 1); unsigned char parts[4]; size_t rpos; for(rpos = 0; rpos < length; rpos += 3){ parts[0] = (input[rpos] & 0xfc) >> 2; parts[1] = ((input[rpos] & 0x03) << 4) | ((((rpos + 1) < length ? input[rpos + 1] : 0x00) & 0xf0) >> 4); parts[2] = (rpos + 1) < length ? (((input[rpos + 1] & 0x0f) << 2) | ((((rpos + 2) < length ? input[rpos + 2] : 0x00) & 0xc0) >> 6)) : 0x40; parts[3] = (rpos + 2) < length ? (input[rpos + 2] & 0x3f) : 0x40; result += base[parts[0]]; result += base[parts[1]]; result += base[parts[2]]; result += base[parts[3]]; } return result; } static inline unsigned char char_decode64(const char ch) { unsigned char by; if('A' <= ch && ch <= 'Z'){ // A - Z by = static_cast(ch - 'A'); }else if('a' <= ch && ch <= 'z'){ // a - z by = static_cast(ch - 'a' + 26); }else if('0' <= ch && ch <= '9'){ // 0 - 9 by = static_cast(ch - '0' + 52); }else if('+' == ch){ // + by = 62; }else if('/' == ch){ // / by = 63; }else if('=' == ch){ // = by = 64; }else{ // something wrong by = UCHAR_MAX; } return by; } std::string s3fs_decode64(const char* input, size_t input_len) { std::string result; result.reserve(input_len / 4 * 3); unsigned char parts[4]; size_t rpos; for(rpos = 0; rpos < input_len; rpos += 4){ parts[0] = char_decode64(input[rpos]); parts[1] = (rpos + 1) < input_len ? char_decode64(input[rpos + 1]) : 64; parts[2] = (rpos + 2) < input_len ? char_decode64(input[rpos + 2]) : 64; parts[3] = (rpos + 3) < input_len ? char_decode64(input[rpos + 3]) : 64; result += static_cast(((parts[0] << 2) & 0xfc) | ((parts[1] >> 4) & 0x03)); if(64 == parts[2]){ break; } result += static_cast(((parts[1] << 4) & 0xf0) | ((parts[2] >> 2) & 0x0f)); if(64 == parts[3]){ break; } result += static_cast(((parts[2] << 6) & 0xc0) | (parts[3] & 0x3f)); } return result; } // // detect and rewrite invalid utf8. We take invalid bytes // and encode them into a private region of the unicode // space. This is sometimes known as wtf8, wobbly transformation format. // it is necessary because S3 validates the utf8 used for identifiers for // correctness, while some clients may provide invalid utf, notably // windows using cp1252. // // Base location for transform. The range 0xE000 - 0xF8ff // is a private range, se use the start of this range. static constexpr unsigned int escape_base = 0xe000; // encode bytes into wobbly utf8. // 'result' can be null. returns true if transform was needed. bool s3fs_wtf8_encode(const char *s, std::string *result) { bool invalid = false; // Pass valid utf8 code through for (; *s; s++) { const unsigned char c = *s; // single byte encoding if (c <= 0x7f) { if (result) { *result += c; } continue; } // otherwise, it must be one of the valid start bytes if ( c >= 0xc2 && c <= 0xf5 ) { // two byte encoding // don't need bounds check, std::string is zero terminated if ((c & 0xe0) == 0xc0 && (s[1] & 0xc0) == 0x80) { // all two byte encodings starting higher than c1 are valid if (result) { *result += c; *result += *(++s); } continue; } // three byte encoding if ((c & 0xf0) == 0xe0 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80) { const unsigned code = ((c & 0x0f) << 12) | ((s[1] & 0x3f) << 6) | (s[2] & 0x3f); if (code >= 0x800 && ! (code >= 0xd800 && code <= 0xd8ff)) { // not overlong and not a surrogate pair if (result) { *result += c; *result += *(++s); *result += *(++s); } continue; } } // four byte encoding if ((c & 0xf8) == 0xf0 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80 && (s[3] & 0xc0) == 0x80) { const unsigned code = ((c & 0x07) << 18) | ((s[1] & 0x3f) << 12) | ((s[2] & 0x3f) << 6) | (s[3] & 0x3f); if (code >= 0x10000 && code <= 0x10ffff) { // not overlong and in defined unicode space if (result) { *result += c; *result += *(++s); *result += *(++s); *result += *(++s); } continue; } } } // printf("invalid %02x at %d\n", c, i); // Invalid utf8 code. Convert it to a private two byte area of unicode // e.g. the e000 - f8ff area. This will be a three byte encoding invalid = true; if (result) { unsigned escape = escape_base + c; *result += static_cast(0xe0 | ((escape >> 12) & 0x0f)); *result += static_cast(0x80 | ((escape >> 06) & 0x3f)); *result += static_cast(0x80 | ((escape >> 00) & 0x3f)); } } return invalid; } std::string s3fs_wtf8_encode(const std::string &s) { std::string result; s3fs_wtf8_encode(s.c_str(), &result); return result; } // The reverse operation, turn encoded bytes back into their original values // The code assumes that we map to a three-byte code point. bool s3fs_wtf8_decode(const char *s, std::string *result) { bool encoded = false; for (; *s; s++) { unsigned char c = *s; // look for a three byte tuple matching our encoding code if ((c & 0xf0) == 0xe0 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80) { unsigned code = (c & 0x0f) << 12; code |= (s[1] & 0x3f) << 6; code |= (s[2] & 0x3f) << 0; if (code >= escape_base && code <= escape_base + 0xff) { // convert back encoded = true; if(result){ *result += static_cast(code - escape_base); } s+=2; continue; } } if (result) { *result += c; } } return encoded; } std::string s3fs_wtf8_decode(const std::string &s) { std::string result; s3fs_wtf8_decode(s.c_str(), &result); return result; } // // Encode only CR('\r'=0x0D) and it also encodes the '%' character accordingly. // // The xmlReadMemory() function in libxml2 replaces CR code with LF code('\n'=0x0A) // due to the XML specification. // s3fs uses libxml2 to parse the S3 response, and this automatic substitution // of libxml2 may change the object name(file/dir name). Therefore, before passing // the response to the xmlReadMemory() function, we need the string encoded by // this function. // // [NOTE] // Normally the quotes included in the XML content data are HTML encoded("""). // Encoding for CR can also be HTML encoded as binary code (ex, " "), but // if the same string content(as file name) as this encoded string exists, the // original string cannot be distinguished whichever encoded or not encoded. // Therefore, CR is encoded in the same manner as URL encoding("%0A"). // And it is assumed that there is no CR code in the S3 response tag etc.(actually // it shouldn't exist) // std::string get_encoded_cr_code(const char* pbase) { std::string result; if(!pbase){ return result; } std::string strbase(pbase); size_t baselength = strbase.length(); size_t startpos = 0; size_t foundpos; while(startpos < baselength && std::string::npos != (foundpos = strbase.find_first_of("%\r", startpos))){ if(0 < (foundpos - startpos)){ result += strbase.substr(startpos, foundpos - startpos); } if('%' == strbase[foundpos]){ result += "%45"; }else if('\r' == strbase[foundpos]){ result += "%0D"; } startpos = foundpos + 1; } if(startpos < baselength){ result += strbase.substr(startpos); } return result; } // // Decode a string encoded with get_encoded_cr_code(). // std::string get_decoded_cr_code(const char* pencode) { std::string result; if(!pencode){ return result; } std::string strencode(pencode); size_t encodelength = strencode.length(); size_t startpos = 0; size_t foundpos; while(startpos < encodelength && std::string::npos != (foundpos = strencode.find('%', startpos))){ if(0 < (foundpos - startpos)){ result += strencode.substr(startpos, foundpos - startpos); } if((foundpos + 2) < encodelength && 0 == strencode.compare(foundpos, 3, "%45")){ result += '%'; startpos = foundpos + 3; }else if((foundpos + 2) < encodelength && 0 == strencode.compare(foundpos, 3, "%0D")){ result += '\r'; startpos = foundpos + 3; }else if((foundpos + 1) < encodelength && 0 == strencode.compare(foundpos, 2, "%%")){ result += '%'; startpos = foundpos + 2; }else{ result += '%'; startpos = foundpos + 1; } } if(startpos < encodelength){ result += strencode.substr(startpos); } return result; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/string_util.h000066400000000000000000000107411470675423500166110ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_STRING_UTIL_H_ #define S3FS_STRING_UTIL_H_ #include #include #include // // A collection of string utilities for manipulating URLs and HTTP responses. // //------------------------------------------------------------------- // Global variables //------------------------------------------------------------------- static constexpr char SPACES[] = " \t\r\n"; //------------------------------------------------------------------- // Inline functions //------------------------------------------------------------------- static inline bool is_prefix(const char *str, const char *prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } static inline const char* SAFESTRPTR(const char *strptr) { return strptr ? strptr : ""; } //------------------------------------------------------------------- // Macros(WTF8) //------------------------------------------------------------------- #define WTF8_ENCODE(ARG) \ std::string ARG##_buf; \ const char * ARG = _##ARG; \ if (use_wtf8 && s3fs_wtf8_encode( _##ARG, 0 )) { \ s3fs_wtf8_encode( _##ARG, &ARG##_buf); \ ARG = ARG##_buf.c_str(); \ } //------------------------------------------------------------------- // Utilities //------------------------------------------------------------------- // TODO: rename to to_string? std::string str(const struct timespec& value); #ifdef __MSYS__ // // Polyfill for strptime function. // char* strptime(const char* s, const char* f, struct tm* tm); #endif // // Convert string to off_t. Returns false on bad input. // Replacement for C++11 std::stoll. // bool s3fs_strtoofft(off_t* value, const char* str, int base = 0); // // This function returns 0 if a value that cannot be converted is specified. // Only call if 0 is considered an error and the operation can continue. // off_t cvt_strtoofft(const char* str, int base); // // String Manipulation // std::string trim_left(std::string s, const char *t = SPACES); std::string trim_right(std::string s, const char *t = SPACES); std::string trim(std::string s, const char *t = SPACES); std::string lower(std::string s); std::string peeloff(const std::string& s); // // Date string // std::string get_date_rfc850(); void get_date_sigv3(std::string& date, std::string& date8601); std::string get_date_string(time_t tm); std::string get_date_iso8601(time_t tm); bool get_unixtime_from_iso8601(const char* pdate, time_t& unixtime); bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime); // // For encoding // std::string urlEncodeGeneral(const std::string &s); std::string urlEncodePath(const std::string &s); std::string urlEncodeQuery(const std::string &s); std::string urlDecode(const std::string& s); bool takeout_str_dquart(std::string& str); bool get_keyword_value(const std::string& target, const char* keyword, std::string& value); // // For binary string // std::string s3fs_hex_lower(const unsigned char* input, size_t length); std::string s3fs_hex_upper(const unsigned char* input, size_t length); std::string s3fs_base64(const unsigned char* input, size_t length); std::string s3fs_decode64(const char* input, size_t input_len); // // WTF8 // bool s3fs_wtf8_encode(const char *s, std::string *result); std::string s3fs_wtf8_encode(const std::string &s); bool s3fs_wtf8_decode(const char *s, std::string *result); std::string s3fs_wtf8_decode(const std::string &s); // // For CR in XML // std::string get_encoded_cr_code(const char* pbase); std::string get_decoded_cr_code(const char* pencode); #endif // S3FS_STRING_UTIL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/test_curl_util.cpp000066400000000000000000000121771470675423500176470ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2020 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include "curl_util.h" #include "test_util.h" //--------------------------------------------------------- // S3fsCred Stub // // [NOTE] // This test program links curl_util.cpp just to use the // curl_slist_sort_insert function. // This file has a call to S3fsCred::GetBucket(), which // results in a link error. That method is not used in // this test file, so define a stub class. Linking all // implementation of the S3fsCred class or making all // stubs is not practical, so this is the best answer. // class S3fsCred { private: static std::string bucket_name; public: static const std::string& GetBucket(); }; std::string S3fsCred::bucket_name; const std::string& S3fsCred::GetBucket() { return S3fsCred::bucket_name; } //--------------------------------------------------------- #define ASSERT_IS_SORTED(x) assert_is_sorted((x), __FILE__, __LINE__) void assert_is_sorted(const struct curl_slist* list, const char *file, int line) { for(; list != nullptr; list = list->next){ std::string key1 = list->data; key1.erase(key1.find(':')); std::string key2 = list->data; key2.erase(key2.find(':')); std::cerr << "key1: " << key1 << " key2: " << key2 << std::endl; if(strcasecmp(key1.c_str(), key2.c_str()) > 0){ std::cerr << "not sorted: " << key1 << " " << key2 << " at " << file << ":" << line << std::endl; std::exit(1); } } std::cerr << std::endl; } size_t curl_slist_length(const struct curl_slist* list) { size_t len = 0; for(; list != nullptr; list = list->next){ ++len; } return len; } void test_sort_insert() { struct curl_slist* list = nullptr; ASSERT_IS_SORTED(list); // add to head list = curl_slist_sort_insert(list, "2", "val"); ASSERT_IS_SORTED(list); // add to tail list = curl_slist_sort_insert(list, "4", "val"); ASSERT_IS_SORTED(list); // add in between list = curl_slist_sort_insert(list, "3", "val"); ASSERT_IS_SORTED(list); // add to head list = curl_slist_sort_insert(list, "1", "val"); ASSERT_IS_SORTED(list); ASSERT_STREQUALS("1: val", list->data); // replace head list = curl_slist_sort_insert(list, "1", "val2"); ASSERT_IS_SORTED(list); ASSERT_EQUALS(static_cast(4), curl_slist_length(list)); ASSERT_STREQUALS("1: val2", list->data); curl_slist_free_all(list); } void test_slist_remove() { struct curl_slist* list = nullptr; // remove no elements ASSERT_EQUALS(static_cast(0), curl_slist_length(list)); list = curl_slist_remove(list, "1"); ASSERT_EQUALS(static_cast(0), curl_slist_length(list)); // remove only element list = nullptr; list = curl_slist_sort_insert(list, "1", "val"); ASSERT_EQUALS(static_cast(1), curl_slist_length(list)); list = curl_slist_remove(list, "1"); ASSERT_EQUALS(static_cast(0), curl_slist_length(list)); // remove head element list = nullptr; list = curl_slist_sort_insert(list, "1", "val"); list = curl_slist_sort_insert(list, "2", "val"); ASSERT_EQUALS(static_cast(2), curl_slist_length(list)); list = curl_slist_remove(list, "1"); ASSERT_EQUALS(static_cast(1), curl_slist_length(list)); curl_slist_free_all(list); // remove tail element list = nullptr; list = curl_slist_sort_insert(list, "1", "val"); list = curl_slist_sort_insert(list, "2", "val"); ASSERT_EQUALS(static_cast(2), curl_slist_length(list)); list = curl_slist_remove(list, "2"); ASSERT_EQUALS(static_cast(1), curl_slist_length(list)); curl_slist_free_all(list); // remove middle element list = nullptr; list = curl_slist_sort_insert(list, "1", "val"); list = curl_slist_sort_insert(list, "2", "val"); list = curl_slist_sort_insert(list, "3", "val"); ASSERT_EQUALS(static_cast(3), curl_slist_length(list)); list = curl_slist_remove(list, "2"); ASSERT_EQUALS(static_cast(2), curl_slist_length(list)); curl_slist_free_all(list); } int main(int argc, const char *argv[]) { test_sort_insert(); test_slist_remove(); return 0; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/test_page_list.cpp000066400000000000000000000047661470675423500176210ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2021 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "fdcache_page.h" #include "fdcache_stat.h" #include "test_util.h" bool CacheFileStat::Open() { return false; } // NOLINT(readability-convert-member-functions-to-static) void test_compress() { PageList list; ASSERT_EQUALS(off_t(0), list.Size()); list.Init(42, /*is_loaded=*/ false, /*is_modified=*/ false); ASSERT_EQUALS(off_t(42), list.Size()); ASSERT_FALSE(list.IsPageLoaded(0, 1)); list.SetPageLoadedStatus(0, 1, /*pstatus=*/ PageList::page_status::LOADED); ASSERT_TRUE(list.IsPageLoaded(0, 1)); ASSERT_FALSE(list.IsPageLoaded(0, 2)); off_t start = 0; off_t size = 0; ASSERT_TRUE(list.FindUnloadedPage(0, start, size)); ASSERT_EQUALS(off_t(1), start); ASSERT_EQUALS(off_t(41), size); // test adding subsequent page then compressing list.SetPageLoadedStatus(1, 3, /*pstatus=*/ PageList::page_status::LOADED); list.Compress(); ASSERT_TRUE(list.IsPageLoaded(0, 3)); ASSERT_TRUE(list.FindUnloadedPage(0, start, size)); ASSERT_EQUALS(off_t(4), start); ASSERT_EQUALS(off_t(38), size); // test adding non-contiguous page then compressing list.SetPageLoadedStatus(5, 1, /*pstatus=*/ PageList::page_status::LOADED); list.Compress(); ASSERT_TRUE(list.FindUnloadedPage(0, start, size)); ASSERT_EQUALS(off_t(4), start); ASSERT_EQUALS(off_t(1), size); list.Dump(); printf("\n"); // test adding page between two pages then compressing list.SetPageLoadedStatus(4, 1, /*pstatus=*/ PageList::page_status::LOADED); list.Compress(); list.Dump(); ASSERT_TRUE(list.FindUnloadedPage(0, start, size)); ASSERT_EQUALS(off_t(6), start); ASSERT_EQUALS(off_t(36), size); } int main(int argc, const char *argv[]) { test_compress(); return 0; } s3fs-fuse-1.95/src/test_string_util.cpp000066400000000000000000000216051470675423500202040ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2014 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include "s3fs_logger.h" #include "string_util.h" #include "test_util.h" //------------------------------------------------------------------- // Global variables for test_string_util //------------------------------------------------------------------- bool foreground = false; std::string instance_name; void test_trim() { ASSERT_EQUALS(std::string("1234"), trim(" 1234 ")); ASSERT_EQUALS(std::string("1234"), trim("1234 ")); ASSERT_EQUALS(std::string("1234"), trim(" 1234")); ASSERT_EQUALS(std::string("1234"), trim("1234")); ASSERT_EQUALS(std::string("1234 "), trim_left(" 1234 ")); ASSERT_EQUALS(std::string("1234 "), trim_left("1234 ")); ASSERT_EQUALS(std::string("1234"), trim_left(" 1234")); ASSERT_EQUALS(std::string("1234"), trim_left("1234")); ASSERT_EQUALS(std::string(" 1234"), trim_right(" 1234 ")); ASSERT_EQUALS(std::string("1234"), trim_right("1234 ")); ASSERT_EQUALS(std::string(" 1234"), trim_right(" 1234")); ASSERT_EQUALS(std::string("1234"), trim_right("1234")); ASSERT_EQUALS(std::string("1234"), peeloff("\"1234\"")); // "1234" -> 1234 ASSERT_EQUALS(std::string("\"1234\""), peeloff("\"\"1234\"\"")); // ""1234"" -> "1234" ASSERT_EQUALS(std::string("\"1234"), peeloff("\"\"1234\"")); // ""1234" -> "1234 ASSERT_EQUALS(std::string("1234\""), peeloff("\"1234\"\"")); // "1234"" -> 1234" ASSERT_EQUALS(std::string("\"1234"), peeloff("\"1234")); // "1234 -> "1234 ASSERT_EQUALS(std::string("1234\""), peeloff("1234\"")); // 1234" -> 1234" ASSERT_EQUALS(std::string(" \"1234\""), peeloff(" \"1234\"")); // _"1234" -> _"1234" ASSERT_EQUALS(std::string("\"1234\" "), peeloff("\"1234\" ")); // "1234"_ -> "1234"_ } void test_base64() { std::string buf; char tmpbuf = '\0'; ASSERT_EQUALS(s3fs_base64(nullptr, 0), std::string("")); buf = s3fs_decode64(nullptr, 0); ASSERT_BUFEQUALS(buf.c_str(), buf.length(), &tmpbuf, 0); ASSERT_EQUALS(s3fs_base64(reinterpret_cast(""), 0), std::string("")); buf = s3fs_decode64("", 0); ASSERT_BUFEQUALS(buf.c_str(), buf.length(), &tmpbuf, 0); ASSERT_EQUALS(s3fs_base64(reinterpret_cast("1"), 1), std::string("MQ==")); buf = s3fs_decode64("MQ==", 4); ASSERT_BUFEQUALS(buf.c_str(), buf.length(), "1", 1); ASSERT_EQUALS(buf.length(), static_cast(1)); ASSERT_EQUALS(s3fs_base64(reinterpret_cast("12"), 2), std::string("MTI=")); buf = s3fs_decode64("MTI=", 4); ASSERT_BUFEQUALS(buf.c_str(), buf.length(), "12", 2); ASSERT_EQUALS(buf.length(), static_cast(2)); ASSERT_EQUALS(s3fs_base64(reinterpret_cast("123"), 3), std::string("MTIz")); buf = s3fs_decode64("MTIz", 4); ASSERT_BUFEQUALS(buf.c_str(), buf.length(), "123", 3); ASSERT_EQUALS(buf.length(), static_cast(3)); ASSERT_EQUALS(s3fs_base64(reinterpret_cast("1234"), 4), std::string("MTIzNA==")); buf = s3fs_decode64("MTIzNA==", 8); ASSERT_BUFEQUALS(buf.c_str(), buf.length(), "1234", 4); ASSERT_EQUALS(buf.length(), static_cast(4)); // TODO: invalid input } void test_strtoofft() { off_t value; ASSERT_TRUE(s3fs_strtoofft(&value, "0")); ASSERT_EQUALS(value, static_cast(0L)); ASSERT_TRUE(s3fs_strtoofft(&value, "9")); ASSERT_EQUALS(value, static_cast(9L)); ASSERT_FALSE(s3fs_strtoofft(&value, "A")); ASSERT_TRUE(s3fs_strtoofft(&value, "A", /*base=*/ 16)); ASSERT_EQUALS(value, static_cast(10L)); ASSERT_TRUE(s3fs_strtoofft(&value, "F", /*base=*/ 16)); ASSERT_EQUALS(value, static_cast(15L)); ASSERT_TRUE(s3fs_strtoofft(&value, "a", /*base=*/ 16)); ASSERT_EQUALS(value, static_cast(10L)); ASSERT_TRUE(s3fs_strtoofft(&value, "f", /*base=*/ 16)); ASSERT_EQUALS(value, static_cast(15L)); ASSERT_TRUE(s3fs_strtoofft(&value, "deadbeef", /*base=*/ 16)); ASSERT_EQUALS(value, static_cast(3735928559L)); } void test_wtf8_encoding() { std::string ascii("normal std::string"); std::string utf8("Hyld\xc3\xbdpi \xc3\xbej\xc3\xb3\xc3\xb0""f\xc3\xa9lagsins vex \xc3\xbar k\xc3\xa6rkomnu b\xc3\xb6li \xc3\xad \xc3\xa1st"); std::string cp1252("Hyld\xfdpi \xfej\xf3\xf0""f\xe9lagsins vex \xfar k\xe6rkomnu b\xf6li \xed \xe1st"); std::string broken = utf8; broken[14] = '\x97'; std::string mixed = ascii + utf8 + cp1252; ASSERT_EQUALS(s3fs_wtf8_encode(ascii), ascii); ASSERT_EQUALS(s3fs_wtf8_decode(ascii), ascii); ASSERT_EQUALS(s3fs_wtf8_encode(utf8), utf8); ASSERT_EQUALS(s3fs_wtf8_decode(utf8), utf8); ASSERT_NEQUALS(s3fs_wtf8_encode(cp1252), cp1252); ASSERT_EQUALS(s3fs_wtf8_decode(s3fs_wtf8_encode(cp1252)), cp1252); ASSERT_NEQUALS(s3fs_wtf8_encode(broken), broken); ASSERT_EQUALS(s3fs_wtf8_decode(s3fs_wtf8_encode(broken)), broken); ASSERT_NEQUALS(s3fs_wtf8_encode(mixed), mixed); ASSERT_EQUALS(s3fs_wtf8_decode(s3fs_wtf8_encode(mixed)), mixed); } void test_cr_encoding() { // bse strings std::string base_no("STR"); std::string base_end_cr1("STR\r"); std::string base_mid_cr1("STR\rSTR"); std::string base_end_cr2("STR\r\r"); std::string base_mid_cr2("STR\r\rSTR"); std::string base_end_per1("STR%"); std::string base_mid_per1("STR%STR"); std::string base_end_per2("STR%%"); std::string base_mid_per2("STR%%STR"); std::string base_end_crlf1("STR\r\n"); std::string base_mid_crlf1("STR\r\nSTR"); std::string base_end_crlf2("STR\r\n\r\n"); std::string base_mid_crlf2("STR\r\n\r\nSTR"); std::string base_end_crper1("STR%\r"); std::string base_mid_crper1("STR%\rSTR"); std::string base_end_crper2("STR%\r%\r"); std::string base_mid_crper2("STR%\r%\rSTR"); // encode->decode->compare ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_no.c_str()).c_str()), base_no); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_cr1.c_str()).c_str()), base_end_cr1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_cr1.c_str()).c_str()), base_mid_cr1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_cr2.c_str()).c_str()), base_end_cr2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_cr2.c_str()).c_str()), base_mid_cr2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_per1.c_str()).c_str()), base_end_per1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_per1.c_str()).c_str()), base_mid_per1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_per2.c_str()).c_str()), base_end_per2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_per2.c_str()).c_str()), base_mid_per2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_crlf1.c_str()).c_str()), base_end_crlf1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_crlf1.c_str()).c_str()), base_mid_crlf1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_crlf2.c_str()).c_str()), base_end_crlf2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_crlf2.c_str()).c_str()), base_mid_crlf2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_crper1.c_str()).c_str()), base_end_crper1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_crper1.c_str()).c_str()), base_mid_crper1); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_end_crper2.c_str()).c_str()), base_end_crper2); ASSERT_EQUALS(get_decoded_cr_code(get_encoded_cr_code(base_mid_crper2.c_str()).c_str()), base_mid_crper2); } int main(int argc, const char *argv[]) { S3fsLog singletonLog; test_trim(); test_base64(); test_strtoofft(); test_wtf8_encoding(); test_cr_encoding(); return 0; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/test_util.h000066400000000000000000000075631470675423500162720ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2014 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_TEST_UTIL_H_ #define S3FS_TEST_UTIL_H_ #include #include #include #include #include "string_util.h" template inline void assert_equals(const T &x, const T &y, const char *file, int line) { if (x != y) { std::cerr << x << " != " << y << " at " << file << ":" << line << std::endl; std::cerr << std::endl; abort(); } } template <> inline void assert_equals(const std::string &x, const std::string &y, const char *file, int line) { if (x != y) { std::cerr << x << " != " << y << " at " << file << ":" << line << std::endl; std::cerr << s3fs_hex_lower(reinterpret_cast(x.c_str()), x.size()) << std::endl; std::cerr << s3fs_hex_lower(reinterpret_cast(y.c_str()), y.size()) << std::endl; abort(); } } template inline void assert_nequals(const T &x, const T &y, const char *file, int line) { if (x == y) { std::cerr << x << " == " << y << " at " << file << ":" << line << std::endl; abort(); } } template <> inline void assert_nequals(const std::string &x, const std::string &y, const char *file, int line) { if (x == y) { std::cerr << x << " == " << y << " at " << file << ":" << line << std::endl; std::cerr << s3fs_hex_lower(reinterpret_cast(x.c_str()), x.size()) << std::endl; std::cerr << s3fs_hex_lower(reinterpret_cast(y.c_str()), y.size()) << std::endl; abort(); } } inline void assert_strequals(const char *x, const char *y, const char *file, int line) { if(x == nullptr && y == nullptr){ return; // cppcheck-suppress nullPointerRedundantCheck } else if(x == nullptr || y == nullptr || strcmp(x, y) != 0){ std::cerr << (x ? x : "null") << " != " << (y ? y : "null") << " at " << file << ":" << line << std::endl; abort(); } } inline void assert_bufequals(const char *x, size_t len1, const char *y, size_t len2, const char *file, int line) { if(x == nullptr && y == nullptr){ return; // cppcheck-suppress nullPointerRedundantCheck } else if(x == nullptr || y == nullptr || len1 != len2 || memcmp(x, y, len1) != 0){ std::cerr << (x ? std::string(x, len1) : "null") << " != " << (y ? std::string(y, len2) : "null") << " at " << file << ":" << line << std::endl; abort(); } } #define ASSERT_TRUE(x) assert_equals((x), true, __FILE__, __LINE__) #define ASSERT_FALSE(x) assert_equals((x), false, __FILE__, __LINE__) #define ASSERT_EQUALS(x, y) assert_equals((x), (y), __FILE__, __LINE__) #define ASSERT_NEQUALS(x, y) assert_nequals((x), (y), __FILE__, __LINE__) #define ASSERT_STREQUALS(x, y) assert_strequals((x), (y), __FILE__, __LINE__) #define ASSERT_BUFEQUALS(x, len1, y, len2) assert_bufequals((x), (len1), (y), (len2), __FILE__, __LINE__) #endif // S3FS_TEST_UTIL_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/threadpoolman.cpp000066400000000000000000000136441470675423500174430ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Takeshi Nakatani * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include "s3fs_logger.h" #include "threadpoolman.h" //------------------------------------------------ // ThreadPoolMan class variables //------------------------------------------------ std::unique_ptr ThreadPoolMan::singleton; //------------------------------------------------ // ThreadPoolMan class methods //------------------------------------------------ bool ThreadPoolMan::Initialize(int count) { if(ThreadPoolMan::singleton){ S3FS_PRN_CRIT("Already singleton for Thread Manager exists."); abort(); } ThreadPoolMan::singleton.reset(new ThreadPoolMan(count)); return true; } void ThreadPoolMan::Destroy() { ThreadPoolMan::singleton.reset(); } bool ThreadPoolMan::Instruct(const thpoolman_param& param) { if(!ThreadPoolMan::singleton){ S3FS_PRN_WARN("The singleton object is not initialized yet."); return false; } ThreadPoolMan::singleton->SetInstruction(param); return true; } // // Thread worker // void ThreadPoolMan::Worker(ThreadPoolMan* psingleton, std::promise promise) { if(!psingleton){ S3FS_PRN_ERR("The parameter for worker thread is invalid."); promise.set_value(-EIO); return; } S3FS_PRN_INFO3("Start worker thread in ThreadPoolMan."); while(!psingleton->IsExit()){ // wait psingleton->thpoolman_sem.wait(); if(psingleton->IsExit()){ break; } // get instruction thpoolman_param param; { const std::lock_guard lock(psingleton->thread_list_lock); if(psingleton->instruction_list.empty()){ S3FS_PRN_DBG("Got a semaphore, but the instruction is empty."); continue; }else{ param = psingleton->instruction_list.front(); psingleton->instruction_list.pop_front(); } } void* retval = param.pfunc(param.args); if(nullptr != retval){ S3FS_PRN_WARN("The instruction function returned with something error code(%ld).", reinterpret_cast(retval)); } if(param.psem){ param.psem->post(); } } promise.set_value(0); } //------------------------------------------------ // ThreadPoolMan methods //------------------------------------------------ ThreadPoolMan::ThreadPoolMan(int count) : is_exit(false), thpoolman_sem(0) { if(count < 1){ S3FS_PRN_CRIT("Failed to creating singleton for Thread Manager, because thread count(%d) is under 1.", count); abort(); } if(ThreadPoolMan::singleton){ S3FS_PRN_CRIT("Already singleton for Thread Manager exists."); abort(); } // create threads if(!StartThreads(count)){ S3FS_PRN_ERR("Failed starting threads at initializing."); abort(); } } ThreadPoolMan::~ThreadPoolMan() { StopThreads(); } bool ThreadPoolMan::IsExit() const { return is_exit; } void ThreadPoolMan::SetExitFlag(bool exit_flag) { is_exit = exit_flag; } bool ThreadPoolMan::StopThreads() { const std::lock_guard lock(thread_list_lock); if(thread_list.empty()){ S3FS_PRN_INFO("Any threads are running now, then nothing to do."); return true; } // all threads to exit SetExitFlag(true); for(size_t waitcnt = thread_list.size(); 0 < waitcnt; --waitcnt){ thpoolman_sem.post(); } // wait for threads exiting for(auto& pair : thread_list){ pair.first.join(); long retval = pair.second.get(); S3FS_PRN_DBG("join succeeded - return code(%ld)", reinterpret_cast(retval)); } thread_list.clear(); // reset semaphore(to zero) while(thpoolman_sem.try_wait()){ } return true; } bool ThreadPoolMan::StartThreads(int count) { if(count < 1){ S3FS_PRN_ERR("Failed to creating threads, because thread count(%d) is under 1.", count); return false; } // stop all thread if they are running. // cppcheck-suppress unmatchedSuppression // cppcheck-suppress knownConditionTrueFalse if(!StopThreads()){ S3FS_PRN_ERR("Failed to stop existed threads."); return false; } // create all threads SetExitFlag(false); for(int cnt = 0; cnt < count; ++cnt){ // run thread std::promise promise; std::future future = promise.get_future(); std::thread thread(ThreadPoolMan::Worker, this, std::move(promise)); const std::lock_guard lock(thread_list_lock); thread_list.emplace_back(std::move(thread), std::move(future)); } return true; } void ThreadPoolMan::SetInstruction(const thpoolman_param& param) { // set parameter to list { const std::lock_guard lock(thread_list_lock); instruction_list.push_back(param); } // run thread thpoolman_sem.post(); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/threadpoolman.h000066400000000000000000000060761470675423500171110ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_THREADPOOLMAN_H_ #define S3FS_THREADPOOLMAN_H_ #include #include #include #include #include #include "common.h" #include "psemaphore.h" //------------------------------------------------ // Typedefs for functions and structures //------------------------------------------------ // // Prototype function // typedef void* (*thpoolman_worker)(void*); // // Parameter structure // // [NOTE] // The args member is a value that is an argument of the worker function. // The psem member is allowed nullptr. If it is not nullptr, the post() method is // called when finishing the function. // struct thpoolman_param { void* args = nullptr; Semaphore* psem = nullptr; thpoolman_worker pfunc = nullptr; }; typedef std::list thpoolman_params_t; //------------------------------------------------ // Class ThreadPoolMan //------------------------------------------------ class ThreadPoolMan { private: static std::unique_ptr singleton; std::atomic is_exit; Semaphore thpoolman_sem; std::mutex thread_list_lock; std::vector>> thread_list GUARDED_BY(thread_list_lock); thpoolman_params_t instruction_list GUARDED_BY(thread_list_lock); private: static void Worker(ThreadPoolMan* psingleton, std::promise promise); explicit ThreadPoolMan(int count = 1); bool IsExit() const; void SetExitFlag(bool exit_flag); bool StopThreads(); bool StartThreads(int count); void SetInstruction(const thpoolman_param& pparam); public: ~ThreadPoolMan(); ThreadPoolMan(const ThreadPoolMan&) = delete; ThreadPoolMan(ThreadPoolMan&&) = delete; ThreadPoolMan& operator=(const ThreadPoolMan&) = delete; ThreadPoolMan& operator=(ThreadPoolMan&&) = delete; static bool Initialize(int count); static void Destroy(); static bool Instruct(const thpoolman_param& pparam); }; #endif // S3FS_THREADPOOLMAN_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/src/types.h000066400000000000000000000234751470675423500154220ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef S3FS_TYPES_H_ #define S3FS_TYPES_H_ #include #include #include #include #include #include #include #include // // For extended attribute // (HAVE_XXX symbols are defined in config.h) // #ifdef HAVE_SYS_EXTATTR_H #include #elif HAVE_ATTR_XATTR_H #include #elif HAVE_SYS_XATTR_H #include #endif //------------------------------------------------------------------- // xattrs_t //------------------------------------------------------------------- // // Header "x-amz-meta-xattr" is for extended attributes. // This header is url encoded string which is json formatted. // x-amz-meta-xattr:urlencode({"xattr-1":"base64(value-1)","xattr-2":"base64(value-2)","xattr-3":"base64(value-3)"}) // typedef std::map xattrs_t; //------------------------------------------------------------------- // acl_t //------------------------------------------------------------------- enum class acl_t : uint8_t { PRIVATE, PUBLIC_READ, PUBLIC_READ_WRITE, AWS_EXEC_READ, AUTHENTICATED_READ, BUCKET_OWNER_READ, BUCKET_OWNER_FULL_CONTROL, LOG_DELIVERY_WRITE, UNKNOWN }; inline const char* str(acl_t value) { switch(value){ case acl_t::PRIVATE: return "private"; case acl_t::PUBLIC_READ: return "public-read"; case acl_t::PUBLIC_READ_WRITE: return "public-read-write"; case acl_t::AWS_EXEC_READ: return "aws-exec-read"; case acl_t::AUTHENTICATED_READ: return "authenticated-read"; case acl_t::BUCKET_OWNER_READ: return "bucket-owner-read"; case acl_t::BUCKET_OWNER_FULL_CONTROL: return "bucket-owner-full-control"; case acl_t::LOG_DELIVERY_WRITE: return "log-delivery-write"; case acl_t::UNKNOWN: return nullptr; } abort(); } inline acl_t to_acl(const char *acl) { if(0 == strcmp(acl, "private")){ return acl_t::PRIVATE; }else if(0 == strcmp(acl, "public-read")){ return acl_t::PUBLIC_READ; }else if(0 == strcmp(acl, "public-read-write")){ return acl_t::PUBLIC_READ_WRITE; }else if(0 == strcmp(acl, "aws-exec-read")){ return acl_t::AWS_EXEC_READ; }else if(0 == strcmp(acl, "authenticated-read")){ return acl_t::AUTHENTICATED_READ; }else if(0 == strcmp(acl, "bucket-owner-read")){ return acl_t::BUCKET_OWNER_READ; }else if(0 == strcmp(acl, "bucket-owner-full-control")){ return acl_t::BUCKET_OWNER_FULL_CONTROL; }else if(0 == strcmp(acl, "log-delivery-write")){ return acl_t::LOG_DELIVERY_WRITE; }else{ return acl_t::UNKNOWN; } } //------------------------------------------------------------------- // sse_type_t //------------------------------------------------------------------- enum class sse_type_t : uint8_t { SSE_DISABLE = 0, // not use server side encrypting SSE_S3, // server side encrypting by S3 key SSE_C, // server side encrypting by custom key SSE_KMS // server side encrypting by kms id }; enum class signature_type_t : uint8_t { V2_ONLY, V4_ONLY, V2_OR_V4 }; //---------------------------------------------- // etaglist_t / filepart / untreatedpart //---------------------------------------------- // // Etag string and part number pair // struct etagpair { std::string etag; // expected etag value int part_num; // part number explicit etagpair(const char* petag = nullptr, int part = -1) : etag(petag ? petag : ""), part_num(part) {} ~etagpair() { clear(); } void clear() { etag.clear(); part_num = -1; } }; // Requires pointer stability and thus must be a list not a vector typedef std::list etaglist_t; struct petagpool { // Requires pointer stability and thus must be a list not a vector std::list petaglist; ~petagpool() { clear(); } void clear() { petaglist.clear(); } etagpair* add(const etagpair& etag_entity) { petaglist.push_back(etag_entity); return &petaglist.back(); } }; // // Each part information for Multipart upload // struct filepart { bool uploaded = false; // does finish uploading std::string etag; // expected etag value int fd; // base file(temporary full file) descriptor off_t startpos; // seek fd point for uploading off_t size; // uploading size bool is_copy; // whether is copy multipart etagpair* petag; // use only parallel upload explicit filepart(bool is_uploaded = false, int _fd = -1, off_t part_start = 0, off_t part_size = -1, bool is_copy_part = false, etagpair* petagpair = nullptr) : fd(_fd), startpos(part_start), size(part_size), is_copy(is_copy_part), petag(petagpair) {} ~filepart() { clear(); } void clear() { uploaded = false; etag = ""; fd = -1; startpos = 0; size = -1; is_copy = false; petag = nullptr; } void add_etag_list(etaglist_t& list, int partnum = -1) { if(-1 == partnum){ partnum = static_cast(list.size()) + 1; } list.emplace_back(nullptr, partnum); petag = &list.back(); } void set_etag(etagpair* petagobj) { petag = petagobj; } int get_part_number() const { if(!petag){ return -1; } return petag->part_num; } }; typedef std::vector filepart_list_t; // // Each part information for Untreated parts // struct untreatedpart { off_t start; // untreated start position off_t size; // number of untreated bytes long untreated_tag; // untreated part tag explicit untreatedpart(off_t part_start = 0, off_t part_size = 0, long part_untreated_tag = 0) : start(part_start), size(part_size), untreated_tag(part_untreated_tag) { if(part_start < 0 || part_size <= 0){ clear(); // wrong parameter, so clear value. } } ~untreatedpart() { clear(); } void clear() { start = 0; size = 0; untreated_tag = 0; } // [NOTE] // Check if the areas overlap // However, even if the areas do not overlap, this method returns true if areas are adjacent. // bool check_overlap(off_t chk_start, off_t chk_size) const { if(chk_start < 0 || chk_size <= 0 || start < 0 || size <= 0 || (chk_start + chk_size) < start || (start + size) < chk_start){ return false; } return true; } bool stretch(off_t add_start, off_t add_size, long tag) { if(!check_overlap(add_start, add_size)){ return false; } off_t new_start = std::min(start, add_start); off_t new_next_start = std::max((start + size), (add_start + add_size)); start = new_start; size = new_next_start - new_start; untreated_tag = tag; return true; } }; typedef std::vector untreated_list_t; // // Information on each part of multipart upload // struct mp_part { off_t start; off_t size; int part_num; // Set only for information to upload explicit mp_part(off_t set_start = 0, off_t set_size = 0, int part = 0) : start(set_start), size(set_size), part_num(part) {} }; typedef std::vector mp_part_list_t; inline off_t total_mp_part_list(const mp_part_list_t& mplist) { off_t size = 0; for(auto iter = mplist.cbegin(); iter != mplist.cend(); ++iter){ size += iter->size; } return size; } // // Rename directory struct // struct mvnode { mvnode(std::string old_path, std::string new_path, bool is_dir, bool is_normdir) : old_path(std::move(old_path)) , new_path(std::move(new_path)) , is_dir(is_dir) , is_normdir(is_normdir) {} std::string old_path; std::string new_path; bool is_dir; bool is_normdir; }; //------------------------------------------------------------------- // mimes_t //------------------------------------------------------------------- struct case_insensitive_compare_func { bool operator()(const std::string& a, const std::string& b) const { return strcasecmp(a.c_str(), b.c_str()) < 0; } }; typedef std::map mimes_t; //------------------------------------------------------------------- // Typedefs specialized for use //------------------------------------------------------------------- typedef std::vector readline_t; typedef std::map kvmap_t; typedef std::map bucketkvmap_t; #endif // S3FS_TYPES_H_ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/test/000077500000000000000000000000001470675423500142625ustar00rootroot00000000000000s3fs-fuse-1.95/test/Makefile.am000066400000000000000000000034741470675423500163260ustar00rootroot00000000000000###################################################################### # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ###################################################################### TESTS=small-integration-test.sh EXTRA_DIST = \ integration-test-common.sh \ small-integration-test.sh \ mergedir.sh \ sample_delcache.sh \ sample_ahbe.conf testdir = test noinst_PROGRAMS = \ junk_data \ write_multiblock \ mknod_test \ truncate_read_file \ cr_filename junk_data_SOURCES = junk_data.cc write_multiblock_SOURCES = write_multiblock.cc mknod_test_SOURCES = mknod_test.cc truncate_read_file_SOURCES = truncate_read_file.cc cr_filename_SOURCES = cr_filename.cc clang-tidy: clang-tidy \ $(junk_data_SOURCES) \ $(write_multiblock_SOURCES) \ $(mknod_test_SOURCES) \ $(truncate_read_file_SOURCES) \ $(cr_filename_SOURCES) \ -- $(DEPS_CFLAGS) $(CPPFLAGS) # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/chaos-http-proxy.conf000066400000000000000000000001271470675423500203620ustar00rootroot00000000000000com.bouncestorage.chaoshttpproxy.http_503=1 com.bouncestorage.chaoshttpproxy.success=9 s3fs-fuse-1.95/test/compile_all_targets.sh000077500000000000000000000032041470675423500206310ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # set -o errexit set -o nounset set -o pipefail COMMON_FLAGS='-O -Wall -Werror' make clean CXXFLAGS="$COMMON_FLAGS" ./configure --with-gnutls make --jobs "$(nproc)" make clean CXXFLAGS="$COMMON_FLAGS" ./configure --with-gnutls --with-nettle make --jobs "$(nproc)" make clean CXXFLAGS="$COMMON_FLAGS" ./configure --with-nss make --jobs "$(nproc)" make clean CXXFLAGS="$COMMON_FLAGS" ./configure --with-openssl make --jobs "$(nproc)" make clean CXXFLAGS="$COMMON_FLAGS -std=c++23" ./configure make --jobs "$(nproc)" make clean CXXFLAGS="$COMMON_FLAGS -m32" ./configure make --jobs "$(nproc)" make clean CXX=clang++ CXXFLAGS="$COMMON_FLAGS -Wshorten-64-to-32" ./configure make --jobs "$(nproc)" # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/cr_filename.cc000066400000000000000000000044031470675423500170360ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2021 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include // [NOTE] // This is a program used for file size inspection. // File size checking should be done by the caller of this program. // This program truncates the file and reads the file in another process // between truncate and flush(close file). // int main(int argc, const char *argv[]) { if(argc != 2){ fprintf(stderr, "[ERROR] Wrong paraemters\n"); fprintf(stdout, "[Usage] cr_filename \n"); exit(EXIT_FAILURE); } int fd; char filepath[4096]; snprintf(filepath, sizeof(filepath), "%s\r", argv[1]); filepath[sizeof(filepath) - 1] = '\0'; // for safety // create empty file if(-1 == (fd = open(filepath, O_CREAT|O_RDWR, 0644))){ fprintf(stderr, "[ERROR] Could not open file(%s)\n", filepath); exit(EXIT_FAILURE); } close(fd); // stat struct stat buf; if(0 != stat(filepath, &buf)){ fprintf(stderr, "[ERROR] Could not get stat for file(%s)\n", filepath); exit(EXIT_FAILURE); } // remove file if(0 != unlink(filepath)){ fprintf(stderr, "[ERROR] Could not remove file(%s)\n", filepath); exit(EXIT_FAILURE); } exit(EXIT_SUCCESS); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/test/filter-suite-log.sh000077500000000000000000000116151470675423500200200ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # func_usage() { echo "" echo "Usage: $1 [-h] " echo " -h print help" echo " log file path path for test-suite.log" echo "" } PRGNAME=$(basename "$0") SCRIPTDIR=$(dirname "$0") S3FSDIR=$(cd "${SCRIPTDIR}"/.. || exit 1; pwd) TOPDIR=$(cd "${S3FSDIR}"/test || exit 1; pwd) SUITELOG="${TOPDIR}/test-suite.log" TMP_LINENO_FILE="/tmp/.lineno.tmp" while [ $# -ne 0 ]; do if [ "X$1" = "X" ]; then break elif [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ] || [ "X$1" = "X--help" ] || [ "X$1" = "X--HELP" ]; then func_usage "${PRGNAME}" exit 0 else SUITELOG=$1 fi shift done if [ ! -f "${SUITELOG}" ]; then echo "[ERROR] not found ${SUITELOG} log file." exit 1 fi # # Extract keyword line numbers and types # # 0 : normal line # 1 : start line for one small test(specified in integration-test-main.sh) # 2 : passed line of end of one small test(specified in test-utils.sh) # 3 : failed line of end of one small test(specified in test-utils.sh) # grep -n -e 'test_.*: ".*"' -o -e 'test_.* passed' -o -e 'test_.* failed' "${SUITELOG}" 2>/dev/null | sed 's/:test_.*: ".*"/ 1/g' | sed 's/:test_.* passed/ 2/g' | sed 's/:test_.* failed/ 3/g' > "${TMP_LINENO_FILE}" # # Loop for printing result # prev_line_type=0 prev_line_number=1 while read -r line; do # line is " " # # shellcheck disable=SC2206 number_type=(${line}) head_line_cnt=$((number_type[0] - 1)) tail_line_cnt=$((number_type[0] - prev_line_number)) if [ "${number_type[1]}" -eq 2 ]; then echo "" fi if [ "${prev_line_type}" -eq 1 ]; then if [ "${number_type[1]}" -eq 2 ]; then # if passed, cut s3fs information messages head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' elif [ "${number_type[1]}" -eq 3 ]; then # if failed, print all head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' else # there is start keyword but not end keyword, so print all head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' fi elif [ "${prev_line_type}" -eq 2 ] || [ "${prev_line_type}" -eq 3 ]; then if [ "${number_type[1]}" -eq 2 ] || [ "${number_type[1]}" -eq 3 ]; then # previous is end of chmpx, but this type is end of chmpx without start keyword. then print all head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' else # this area is not from start to end, cut s3fs information messages head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' fi else if [ "${number_type[1]}" -eq 2 ] || [ "${number_type[1]}" -eq 3 ]; then # previous is normal, but this type is end of chmpx without start keyword. then print all head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' else # this area is normal, cut s3fs information messages head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' fi fi if [ "${number_type[1]}" -eq 3 ]; then echo "" fi prev_line_type="${number_type[1]}" prev_line_number="${number_type[0]}" done < "${TMP_LINENO_FILE}" # # Print rest lines # file_line_cnt=$(wc -l "${SUITELOG}" | awk '{print $1}') tail_line_cnt=$((file_line_cnt - prev_line_number)) if [ "${prev_line_type}" -eq 1 ]; then tail "-${tail_line_cnt}" "${SUITELOG}" | grep -v -e '[0-9]\+%' else tail "-${tail_line_cnt}" "${SUITELOG}" | grep -v -e '[0-9]\+%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' fi # # Remove temp file # rm -f "${TMP_LINENO_FILE}" exit 0 # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/integration-test-common.sh000066400000000000000000000305001470675423500214020ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # Common code for starting an s3fs-fuse mountpoint and an S3Proxy instance # to run tests against S3Proxy locally. # # To run against an Amazon S3 or other S3 provider, specify the following # environment variables: # # S3FS_CREDENTIALS_FILE=keyfile s3fs format key file # S3FS_PROFILE=name s3fs profile to use (overrides key file) # TEST_BUCKET_1=bucketname Name of bucket to use # S3PROXY_BINARY="" Specify empty string to skip S3Proxy start # S3_URL="https://s3.amazonaws.com" Specify Amazon AWS as the S3 provider # S3_ENDPOINT="us-east-1" Specify region # TMPDIR="/var/tmp" Set to use a temporary directory different # from /var/tmp # CHAOS_HTTP_PROXY=1 Test proxy(environment) by CHAOS HTTP PROXY # CHAOS_HTTP_PROXY_OPT=1 Test proxy(option) by CHAOS HTTP PROXY # # Example of running against Amazon S3 using a bucket named "bucket": # # S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="https://s3.amazonaws.com" ./small-integration-test.sh # # To change the s3fs-fuse debug level: # # DBGLEVEL=debug ./small-integration-test.sh # # To stop and wait after the mount point is up for manual interaction. This allows you to # explore the mounted file system exactly as it would have been started for the test case # # INTERACT=1 DBGLEVEL=debug ./small-integration-test.sh # # Run all of the tests from the makefile # # S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="https://s3.amazonaws.com" make check # # Run the tests with request auth turned off in both S3Proxy and s3fs-fuse. This can be # useful for poking around with plain old curl # # PUBLIC=1 INTERACT=1 ./small-integration-test.sh # # A valgrind tool can be specified # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh set -o errexit set -o pipefail S3FS=../src/s3fs # Allow these defaulted values to be overridden # # [NOTE] # CHAOS HTTP PROXY does not support HTTPS. # if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then : "${S3_URL:="https://127.0.0.1:8080"}" else : "${S3_URL:="http://127.0.0.1:8080"}" fi : "${S3_ENDPOINT:="us-east-1"}" : "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}" : "${TEST_BUCKET_1:="s3fs-integration-test"}" export TEST_BUCKET_1 export S3_URL export S3_ENDPOINT TEST_SCRIPT_DIR=$(pwd) export TEST_SCRIPT_DIR export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1} S3PROXY_VERSION="2.0.0" S3PROXY_BINARY="${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}" CHAOS_HTTP_PROXY_VERSION="1.1.0" CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}" if [ ! -f "$S3FS_CREDENTIALS_FILE" ] then echo "Missing credentials file: ${S3FS_CREDENTIALS_FILE}" exit 1 fi chmod 600 "${S3FS_CREDENTIALS_FILE}" if [ -z "${S3FS_PROFILE}" ]; then AWS_ACCESS_KEY_ID=$(cut -d: -f1 "${S3FS_CREDENTIALS_FILE}") export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 "${S3FS_CREDENTIALS_FILE}") export AWS_SECRET_ACCESS_KEY fi if [ ! -d "${TEST_BUCKET_MOUNT_POINT_1}" ]; then mkdir -p "${TEST_BUCKET_MOUNT_POINT_1}" fi # This function execute the function parameters $1 times # before giving up, with 1 second delays. function retry { local N="$1" shift rc=0 for _ in $(seq "${N}"); do echo "Trying: $*" # shellcheck disable=SC2068,SC2294 eval $@ rc=$? if [ "${rc}" -eq 0 ]; then break fi sleep 1 echo "Retrying: $*" done if [ "${rc}" -ne 0 ]; then echo "timeout waiting for $*" fi return "${rc}" } # Proxy is not started if S3PROXY_BINARY is an empty string # PUBLIC unset: use s3proxy.conf # PUBLIC=1: use s3proxy-noauth.conf (no request signing) # function start_s3proxy { if [ -n "${PUBLIC}" ]; then local S3PROXY_CONFIG="s3proxy-noauth.conf" else if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then local S3PROXY_CONFIG="s3proxy.conf" else local S3PROXY_CONFIG="s3proxy_http.conf" fi fi if [ -n "${S3PROXY_BINARY}" ] then if [ ! -e "${S3PROXY_BINARY}" ]; then curl "https://github.com/gaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy" \ --fail --location --silent --output "${S3PROXY_BINARY}" chmod +x "${S3PROXY_BINARY}" fi # generate self-signed SSL certificate # # [NOTE] # The PROXY test is HTTP only, so do not create CA certificates. # if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then S3PROXY_CACERT_FILE="/tmp/keystore.pem" rm -f /tmp/keystore.jks "${S3PROXY_CACERT_FILE}" printf 'password\npassword\n\n\n\n\n\n\ny' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1 echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file "${S3PROXY_CACERT_FILE}" else S3PROXY_CACERT_FILE="" fi "${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" & S3PROXY_PID=$! # wait for S3Proxy to start wait_for_port 8080 fi if [ -n "${CHAOS_HTTP_PROXY}" ] || [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then curl "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \ --fail --location --silent --output "${CHAOS_HTTP_PROXY_BINARY}" chmod +x "${CHAOS_HTTP_PROXY_BINARY}" fi "${STDBUF_BIN}" -oL -eL java -jar "${CHAOS_HTTP_PROXY_BINARY}" --properties chaos-http-proxy.conf & CHAOS_HTTP_PROXY_PID=$! # wait for Chaos HTTP Proxy to start wait_for_port 1080 fi } function stop_s3proxy { if [ -n "${S3PROXY_PID}" ] then kill "${S3PROXY_PID}" fi if [ -n "${CHAOS_HTTP_PROXY_PID}" ] then kill "${CHAOS_HTTP_PROXY_PID}" fi } # Mount the bucket, function arguments passed to s3fs in addition to # a set of common arguments. function start_s3fs { # Public bucket if PUBLIC is set if [ -n "${PUBLIC}" ]; then local AUTH_OPT="-o public_bucket=1" elif [ -n "${S3FS_PROFILE}" ]; then local AUTH_OPT="-o profile=${S3FS_PROFILE}" else local AUTH_OPT="-o passwd_file=${S3FS_CREDENTIALS_FILE}" fi # If VALGRIND is set, pass it as options to valgrind. # start valgrind-listener in another shell. # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh # Start valgrind-listener (default port is 1500) if [ -n "${VALGRIND}" ]; then VALGRIND_EXEC="valgrind ${VALGRIND} --log-socket=127.0.1.1" fi # On OSX only, we need to specify the direct_io and auto_cache flag. # # And Turn off creation and reference of spotlight index. # (Leaving spotlight ON will result in a lot of wasted requests, # which will affect test execution time) # if [ "$(uname)" = "Darwin" ]; then local DIRECT_IO_OPT="-o direct_io -o auto_cache" # disable spotlight sudo mdutil -a -i off else local DIRECT_IO_OPT="" fi # Set environment variables or options for proxy. # And the PROXY test is HTTP only and does not set CA certificates. # if [ -n "${CHAOS_HTTP_PROXY}" ]; then export http_proxy="127.0.0.1:1080" S3FS_HTTP_PROXY_OPT="" elif [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then S3FS_HTTP_PROXY_OPT="-o proxy=http://127.0.0.1:1080" else S3FS_HTTP_PROXY_OPT="" fi # [NOTE] # For macos fuse-t, we need to specify the "noattrcache" option to # disable NFS caching. # if [ "$(uname)" = "Darwin" ]; then local FUSE_T_ATTRCACHE_OPT="-o noattrcache" else local FUSE_T_ATTRCACHE_OPT="" fi # [NOTE] # On macOS we may get a VERIFY error for the self-signed certificate used by s3proxy. # We can specify NO_CHECK_CERT=1 to avoid this. # if [ -n "${NO_CHECK_CERT}" ] && [ "${NO_CHECK_CERT}" -eq 1 ]; then local NO_CHECK_CERT_OPT="-o no_check_certificate" else local NO_CHECK_CERT_OPT="" fi # Common s3fs options: # # TODO: Allow all these options to be overridden with env variables # # use_path_request_style # The test env doesn't have virtual hosts # $AUTH_OPT # Will be either "-o public_bucket=1" # or # "-o passwd_file=${S3FS_CREDENTIALS_FILE}" # dbglevel # error by default. override with DBGLEVEL env variable # -f # Keep s3fs in foreground instead of daemonizing # # subshell with set -x to log exact invocation of s3fs-fuse # shellcheck disable=SC2086 ( set -x CURL_CA_BUNDLE="${S3PROXY_CACERT_FILE}" \ ${STDBUF_BIN} -oL -eL \ ${VALGRIND_EXEC} \ ${S3FS} \ ${TEST_BUCKET_1} \ ${TEST_BUCKET_MOUNT_POINT_1} \ -o use_path_request_style \ -o url="${S3_URL}" \ -o endpoint="${S3_ENDPOINT}" \ -o use_xattr=1 \ -o enable_unsigned_payload \ ${AUTH_OPT} \ ${DIRECT_IO_OPT} \ ${S3FS_HTTP_PROXY_OPT} \ ${NO_CHECK_CERT_OPT} \ ${FUSE_T_ATTRCACHE_OPT} \ -o stat_cache_expire=1 \ -o stat_cache_interval_expire=1 \ -o dbglevel="${DBGLEVEL:=info}" \ -o no_time_stamp_msg \ -o retries=3 \ -f \ "${@}" & echo $! >&3 ) 3>pid | "${STDBUF_BIN}" -oL -eL "${SED_BIN}" "${SED_BUFFER_FLAG}" "s/^/s3fs: /" & sleep 1 S3FS_PID=$(/dev/null) if echo "${_DF_RESULT}" | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"; then break; fi sleep 1 TRYCOUNT=$((TRYCOUNT + 1)) done if [ "${TRYCOUNT}" -gt "${RETRIES}" ]; then echo "Waited ${TRYCOUNT} seconds, but it could not be mounted." exit 1 fi else retry "${RETRIES:=20}" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts || exit 1 fi # Quick way to start system up for manual testing with options under test if [[ -n "${INTERACT}" ]]; then echo "Mountpoint ${TEST_BUCKET_MOUNT_POINT_1} is ready" echo "control-C to quit" sleep infinity exit 0 fi } function stop_s3fs { # Retry in case file system is in use if [ "$(uname)" = "Darwin" ]; then if df | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"; then retry 10 df "|" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" "&&" umount "${TEST_BUCKET_MOUNT_POINT_1}" fi else if grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts; then retry 10 grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts "&&" fusermount -u "${TEST_BUCKET_MOUNT_POINT_1}" fi fi } # trap handlers do not stack. If a test sets its own, the new handler should call common_exit_handler function common_exit_handler { stop_s3fs stop_s3proxy } trap common_exit_handler EXIT # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/integration-test-main.sh000077500000000000000000003167571470675423500210660ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # set -o errexit set -o pipefail source test-utils.sh function test_create_empty_file { describe "Testing creating an empty file ..." local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" touch "${TEST_TEXT_FILE}" check_file_size "${TEST_TEXT_FILE}" 0 aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${OBJECT_NAME}" rm_test_file } function test_append_file { describe "Testing append to file ..." local TEST_INPUT="echo ${TEST_TEXT} to ${TEST_TEXT_FILE}" # Write a small test file for x in $(seq 1 "${TEST_TEXT_FILE_LENGTH}"); do echo "${TEST_INPUT}" done > "${TEST_TEXT_FILE}" check_file_size "${TEST_TEXT_FILE}" $((TEST_TEXT_FILE_LENGTH * $((${#TEST_INPUT} + 1)) )) rm_test_file } function test_truncate_file { describe "Testing truncate file ..." # Write a small test file echo "${TEST_TEXT}" > "${TEST_TEXT_FILE}" # Truncate file to 0 length. This should trigger open(path, O_RDWR | O_TRUNC...) : > "${TEST_TEXT_FILE}" check_file_size "${TEST_TEXT_FILE}" 0 rm_test_file } function test_truncate_upload { describe "Testing truncate file for uploading ..." # This file size uses multipart, mix upload when uploading. # We will test these cases. rm_test_file "${BIG_FILE}" "${TRUNCATE_BIN}" "${BIG_FILE}" -s "${BIG_FILE_LENGTH}" rm_test_file "${BIG_FILE}" } function test_truncate_empty_file { describe "Testing truncate empty file ..." # Write an empty test file touch "${TEST_TEXT_FILE}" # Truncate the file to 1024 length local t_size=1024 "${TRUNCATE_BIN}" "${TEST_TEXT_FILE}" -s "${t_size}" check_file_size "${TEST_TEXT_FILE}" "${t_size}" rm_test_file } function test_truncate_shrink_file { describe "Testing truncate shrinking large binary file ..." local BIG_TRUNCATE_TEST_FILE="big-truncate-test.bin" local t_size=$((1024 * 1024 * 32 + 64)) dd if=/dev/urandom of="${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" bs=1024 count=$((1024 * 64)) cp "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" "${BIG_TRUNCATE_TEST_FILE}" "${TRUNCATE_BIN}" "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" -s "${t_size}" "${TRUNCATE_BIN}" "${BIG_TRUNCATE_TEST_FILE}" -s "${t_size}" if ! cmp "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" "${BIG_TRUNCATE_TEST_FILE}"; then return 1 fi rm -f "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" rm_test_file "${BIG_TRUNCATE_TEST_FILE}" } function test_truncate_shrink_read_file { describe "Testing truncate(shrink) and read file ..." # Initiate file size is 1024, and shrink file size is 512 local init_size=1024 local shrink_size=512 # create file dd if=/dev/urandom of="${TEST_TEXT_FILE}" bs="${init_size}" count=1 # truncate(shrink) file and read it before flushing ../../truncate_read_file "${TEST_TEXT_FILE}" "${shrink_size}" # check file size check_file_size "${TEST_TEXT_FILE}" "${shrink_size}" # Truncate the file to 1024 length local t_size=1024 rm_test_file } function test_mv_file { describe "Testing mv file function ..." # if the rename file exists, delete it if [ -e "${ALT_TEST_TEXT_FILE}" ] then rm "${ALT_TEST_TEXT_FILE}" fi if [ -e "${ALT_TEST_TEXT_FILE}" ] then echo "Could not delete file ${ALT_TEST_TEXT_FILE}, it still exists" return 1 fi # create the test file again mk_test_file # save file length local ALT_TEXT_LENGTH; ALT_TEXT_LENGTH=$(wc -c "${TEST_TEXT_FILE}" | awk '{print $1}') #rename the test file mv "${TEST_TEXT_FILE}" "${ALT_TEST_TEXT_FILE}" if [ ! -e "${ALT_TEST_TEXT_FILE}" ] then echo "Could not move file" return 1 fi #check the renamed file content-type if [ -f "/etc/mime.types" ] then check_content_type "$1/${ALT_TEST_TEXT_FILE}" "text/plain" fi # Check the contents of the alt file local ALT_FILE_LENGTH; ALT_FILE_LENGTH=$(wc -c "${ALT_TEST_TEXT_FILE}" | awk '{print $1}') if [ "$ALT_FILE_LENGTH" -ne "$ALT_TEXT_LENGTH" ] then echo "moved file length is not as expected expected: $ALT_TEXT_LENGTH got: $ALT_FILE_LENGTH" return 1 fi # clean up rm_test_file "${ALT_TEST_TEXT_FILE}" } function test_mv_to_exist_file { describe "Testing mv file to exist file function ..." local BIG_MV_FILE_BLOCK_SIZE=$((BIG_FILE_BLOCK_SIZE + 1)) ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${BIG_FILE}" ../../junk_data $((BIG_MV_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${BIG_FILE}-mv" mv "${BIG_FILE}" "${BIG_FILE}-mv" rm_test_file "${BIG_FILE}-mv" } function test_mv_empty_directory { describe "Testing mv directory function ..." if [ -e "${TEST_DIR}" ]; then echo "Unexpected, this file/directory exists: ${TEST_DIR}" return 1 fi mk_test_dir mv "${TEST_DIR}" "${TEST_DIR}_rename" if [ ! -d "${TEST_DIR}_rename" ]; then echo "Directory ${TEST_DIR} was not renamed" return 1 fi rmdir "${TEST_DIR}_rename" if [ -e "${TEST_DIR}_rename" ]; then echo "Could not remove the test directory, it still exists: ${TEST_DIR}_rename" return 1 fi } function test_mv_nonempty_directory { describe "Testing mv directory function ..." if [ -e "${TEST_DIR}" ]; then echo "Unexpected, this file/directory exists: ${TEST_DIR}" return 1 fi mk_test_dir touch "${TEST_DIR}"/file mv "${TEST_DIR}" "${TEST_DIR}_rename" if [ ! -d "${TEST_DIR}_rename" ]; then echo "Directory ${TEST_DIR} was not renamed" return 1 fi rm -r "${TEST_DIR}_rename" if [ -e "${TEST_DIR}_rename" ]; then echo "Could not remove the test directory, it still exists: ${TEST_DIR}_rename" return 1 fi } function test_redirects { describe "Testing redirects ..." mk_test_file "ABCDEF" local CONTENT; CONTENT=$(cat "${TEST_TEXT_FILE}") if [ "${CONTENT}" != "ABCDEF" ]; then echo "CONTENT read is unexpected, got ${CONTENT}, expected ABCDEF" return 1 fi echo "XYZ" > "${TEST_TEXT_FILE}" CONTENT=$(cat "${TEST_TEXT_FILE}") if [ "${CONTENT}" != "XYZ" ]; then echo "CONTENT read is unexpected, got ${CONTENT}, expected XYZ" return 1 fi echo "123456" >> "${TEST_TEXT_FILE}" local LINE1; LINE1=$("${SED_BIN}" -n '1,1p' "${TEST_TEXT_FILE}") local LINE2; LINE2=$("${SED_BIN}" -n '2,2p' "${TEST_TEXT_FILE}") if [ "${LINE1}" != "XYZ" ]; then echo "LINE1 was not as expected, got ${LINE1}, expected XYZ" return 1 fi if [ "${LINE2}" != "123456" ]; then echo "LINE2 was not as expected, got ${LINE2}, expected 123456" return 1 fi # clean up rm_test_file } function test_mkdir_rmdir { describe "Testing creation/removal of a directory ..." if [ -e "${TEST_DIR}" ]; then echo "Unexpected, this file/directory exists: ${TEST_DIR}" return 1 fi mk_test_dir rm_test_dir } function test_chmod { describe "Testing chmod file function ..." # create the test file again mk_test_file local ORIGINAL_PERMISSIONS; ORIGINAL_PERMISSIONS=$(get_permissions "${TEST_TEXT_FILE}") chmod 777 "${TEST_TEXT_FILE}"; # if they're the same, we have a problem. local CHANGED_PERMISSIONS; CHANGED_PERMISSIONS=$(get_permissions "${TEST_TEXT_FILE}") if [ "${CHANGED_PERMISSIONS}" = "${ORIGINAL_PERMISSIONS}" ] then echo "Could not modify ${TEST_TEXT_FILE} permissions" return 1 fi # clean up rm_test_file } function test_chown { describe "Testing chown file function ..." # create the test file again mk_test_file local ORIGINAL_PERMISSIONS ORIGINAL_PERMISSIONS=$(get_user_and_group "${TEST_TEXT_FILE}") # [NOTE] # Prevents test interruptions due to permission errors, etc. # If the chown command fails, an error will occur with the # following judgment statement. So skip the chown command error. # '|| true' was added due to a problem with Travis CI and MacOS # and ensure_diskfree option. # chown 1000:1000 "${TEST_TEXT_FILE}" || true # if they're the same, we have a problem. local CHANGED_PERMISSIONS CHANGED_PERMISSIONS=$(get_user_and_group "${TEST_TEXT_FILE}") if [ "${CHANGED_PERMISSIONS}" = "${ORIGINAL_PERMISSIONS}" ] then if [ "${ORIGINAL_PERMISSIONS}" = "1000:1000" ] then echo "Could not be strict check because original file permission 1000:1000" else echo "Could not modify ${TEST_TEXT_FILE} ownership($ORIGINAL_PERMISSIONS to 1000:1000)" return 1 fi fi # clean up rm_test_file } function test_list { describe "Testing list ..." mk_test_file mk_test_dir local file_list=(*) local file_cnt=${#file_list[@]} if [ "${file_cnt}" -ne 2 ]; then echo "Expected 2 file but got ${file_cnt}" echo "Files: " "${file_list[@]}" return 1 fi rm_test_file rm_test_dir } function test_remove_nonempty_directory { describe "Testing removing a non-empty directory ..." mk_test_dir touch "${TEST_DIR}/file" ( set +o pipefail rmdir "${TEST_DIR}" 2>&1 | grep -q "Directory not empty" ) rm "${TEST_DIR}/file" rm_test_dir } function test_external_directory_creation { describe "Test external directory creation ..." local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/directory/"${TEST_TEXT_FILE}" echo "data" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" # shellcheck disable=SC2010 ls | grep -q directory stat directory >/dev/null 2>&1 get_permissions directory | grep -q 750$ ls directory cmp <(echo "data") directory/"${TEST_TEXT_FILE}" rm -f directory/"${TEST_TEXT_FILE}" } function test_external_modification { describe "Test external modification to an object ..." echo "old" > "${TEST_TEXT_FILE}" # [NOTE] # If the stat and file cache directory are enabled, an error will # occur if the unixtime(sec) value does not change. # If mtime(ctime/atime) when updating from the external program # (awscli) is the same unixtime value as immediately before, the # cache will be read out. # Therefore, we need to wait over 1 second here. # sleep 1 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo "new new" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" cmp "${TEST_TEXT_FILE}" <(echo "new new") rm -f "${TEST_TEXT_FILE}" } function test_external_creation { describe "Test external creation of an object ..." local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" # [NOTE] # If noobj_cache is enabled, register that cache here. # [ ! -e "${TEST_TEXT_FILE}" ] # [NOTE] # If noobj_cache is enabled, we cannot be sure that it is registered in that cache. # That's because an error will occur if the upload by aws cli takes more than 1 second. # echo "data" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" wait_ostype 1 [ -e "${TEST_TEXT_FILE}" ] rm -f "${TEST_TEXT_FILE}" } function test_read_external_object() { describe "create objects via aws CLI and read via s3fs ..." local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo "test" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" cmp "${TEST_TEXT_FILE}" <(echo "test") rm -f "${TEST_TEXT_FILE}" } function test_read_external_dir_object() { describe "create directory objects via aws CLI and read via s3fs ..." local SUB_DIR_NAME; SUB_DIR_NAME="subdir" local SUB_DIR_TEST_FILE; SUB_DIR_TEST_FILE="${SUB_DIR_NAME}/${TEST_TEXT_FILE}" local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${SUB_DIR_TEST_FILE}" echo "test" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" if stat "${SUB_DIR_NAME}" | grep -q '1969-12-31[[:space:]]23:59:59[.]000000000'; then echo "sub directory a/c/m time is underflow(-1)." return 1 fi rm -rf "${SUB_DIR_NAME}" } function test_update_metadata_external_small_object() { describe "update meta to small file after created file by aws cli" # [NOTE] # Use the only filename in the test to avoid being affected by noobjcache. # local TEST_FILE_EXT; TEST_FILE_EXT=$(make_random_string) local TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}" local TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}" local TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}" local TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" local TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" local TEST_INPUT="TEST_STRING_IN_SMALL_FILE" # # chmod # local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_CHMOD_FILE}" echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" chmod +x "${TEST_CHMOD_FILE}" cmp "${TEST_CHMOD_FILE}" <(echo "${TEST_INPUT}") # # chown # OBJECT_NAME=$(basename "${PWD}")/"${TEST_CHOWN_FILE}" echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" chown "${UID}" "${TEST_CHOWN_FILE}" cmp "${TEST_CHOWN_FILE}" <(echo "${TEST_INPUT}") # # utimens # OBJECT_NAME=$(basename "${PWD}")/"${TEST_UTIMENS_FILE}" echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" touch "${TEST_UTIMENS_FILE}" cmp "${TEST_UTIMENS_FILE}" <(echo "${TEST_INPUT}") # # set xattr # OBJECT_NAME=$(basename "${PWD}")/"${TEST_SETXATTR_FILE}" echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" set_xattr key value "${TEST_SETXATTR_FILE}" cmp "${TEST_SETXATTR_FILE}" <(echo "${TEST_INPUT}") # # remove xattr # # "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"} # OBJECT_NAME=$(basename "${PWD}")/"${TEST_RMXATTR_FILE}" echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D del_xattr key "${TEST_RMXATTR_FILE}" cmp "${TEST_RMXATTR_FILE}" <(echo "${TEST_INPUT}") rm -f "${TEST_CHMOD_FILE}" rm -f "${TEST_CHOWN_FILE}" rm -f "${TEST_UTIMENS_FILE}" rm -f "${TEST_SETXATTR_FILE}" rm -f "${TEST_RMXATTR_FILE}" } function test_update_metadata_external_large_object() { describe "update meta to large file after created file by aws cli" # [NOTE] # Use the only filename in the test to avoid being affected by noobjcache. # local TEST_FILE_EXT; TEST_FILE_EXT=$(make_random_string) local TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}" local TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}" local TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}" local TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" local TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}" ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEMP_DIR}/${BIG_FILE}" # # chmod # local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_CHMOD_FILE}" aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress chmod +x "${TEST_CHMOD_FILE}" cmp "${TEST_CHMOD_FILE}" "${TEMP_DIR}/${BIG_FILE}" # # chown # OBJECT_NAME=$(basename "${PWD}")/"${TEST_CHOWN_FILE}" aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress chown "${UID}" "${TEST_CHOWN_FILE}" cmp "${TEST_CHOWN_FILE}" "${TEMP_DIR}/${BIG_FILE}" # # utimens # OBJECT_NAME=$(basename "${PWD}")/"${TEST_UTIMENS_FILE}" aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress touch "${TEST_UTIMENS_FILE}" cmp "${TEST_UTIMENS_FILE}" "${TEMP_DIR}/${BIG_FILE}" # # set xattr # OBJECT_NAME=$(basename "${PWD}")/"${TEST_SETXATTR_FILE}" aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress set_xattr key value "${TEST_SETXATTR_FILE}" cmp "${TEST_SETXATTR_FILE}" "${TEMP_DIR}/${BIG_FILE}" # # remove xattr # # "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"} # OBJECT_NAME=$(basename "${PWD}")/"${TEST_RMXATTR_FILE}" aws_cli s3 cp "${TEMP_DIR}/${BIG_FILE}" "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D del_xattr key "${TEST_RMXATTR_FILE}" cmp "${TEST_RMXATTR_FILE}" "${TEMP_DIR}/${BIG_FILE}" rm -f "${TEMP_DIR}/${BIG_FILE}" rm -f "${TEST_CHMOD_FILE}" rm -f "${TEST_CHOWN_FILE}" rm -f "${TEST_UTIMENS_FILE}" rm -f "${TEST_SETXATTR_FILE}" rm -f "${TEST_RMXATTR_FILE}" } function test_rename_before_close { describe "Testing rename before close ..." # shellcheck disable=SC2094 ( echo foo mv "${TEST_TEXT_FILE}" "${TEST_TEXT_FILE}.new" ) > "${TEST_TEXT_FILE}" if ! cmp <(echo "foo") "${TEST_TEXT_FILE}.new"; then echo "rename before close failed" return 1 fi rm_test_file "${TEST_TEXT_FILE}.new" rm -f "${TEST_TEXT_FILE}" } function test_multipart_upload { describe "Testing multi-part upload ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEMP_DIR}/${BIG_FILE}" dd if="${TEMP_DIR}/${BIG_FILE}" of="${BIG_FILE}" bs="${BIG_FILE_BLOCK_SIZE}" count="${BIG_FILE_COUNT}" # Verify contents of file echo "Comparing test file" if ! cmp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}" then return 1 fi rm -f "${TEMP_DIR}/${BIG_FILE}" rm_test_file "${BIG_FILE}" } function test_multipart_copy { describe "Testing multi-part copy ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEMP_DIR}/${BIG_FILE}" dd if="${TEMP_DIR}/${BIG_FILE}" of="${BIG_FILE}" bs="${BIG_FILE_BLOCK_SIZE}" count="${BIG_FILE_COUNT}" mv "${BIG_FILE}" "${BIG_FILE}-copy" # Verify contents of file echo "Comparing test file" if ! cmp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}-copy" then return 1 fi #check the renamed file content-type check_content_type "$1/${BIG_FILE}-copy" "application/octet-stream" rm -f "${TEMP_DIR}/${BIG_FILE}" rm_test_file "${BIG_FILE}-copy" } function test_multipart_mix { describe "Testing multi-part mix ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEMP_DIR}/${BIG_FILE}" dd if="${TEMP_DIR}/${BIG_FILE}" of="${BIG_FILE}" bs="${BIG_FILE_BLOCK_SIZE}" count="${BIG_FILE_COUNT}" # (1) Edit the middle of an existing file # modify directly(seek 7.5MB offset) # In the case of nomultipart and nocopyapi, # it makes no sense, but copying files is because it leaves no cache. # cp "${TEMP_DIR}/${BIG_FILE}" "${TEMP_DIR}/${BIG_FILE}-mix" cp "${BIG_FILE}" "${BIG_FILE}-mix" local MODIFY_START_BLOCK=$((15*1024*1024/2/4)) echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek="${MODIFY_START_BLOCK}" conv=notrunc echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek="${MODIFY_START_BLOCK}" conv=notrunc # Verify contents of file echo "Comparing test file (1)" if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix" then return 1 fi # (2) Write to an area larger than the size of the existing file # modify directly(over file end offset) # cp "${TEMP_DIR}/${BIG_FILE}" "${TEMP_DIR}/${BIG_FILE}-mix" cp "${BIG_FILE}" "${BIG_FILE}-mix" local OVER_FILE_BLOCK_POS=$((26*1024*1024/4)) echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek="${OVER_FILE_BLOCK_POS}" conv=notrunc echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek="${OVER_FILE_BLOCK_POS}" conv=notrunc # Verify contents of file echo "Comparing test file (2)" if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix" then return 1 fi # (3) Writing from the 0th byte # cp "${TEMP_DIR}/${BIG_FILE}" "${TEMP_DIR}/${BIG_FILE}-mix" cp "${BIG_FILE}" "${BIG_FILE}-mix" echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek=0 conv=notrunc echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek=0 conv=notrunc # Verify contents of file echo "Comparing test file (3)" if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix" then return 1 fi # (4) Write to the area within 5MB from the top # modify directly(seek 1MB offset) # cp "${TEMP_DIR}/${BIG_FILE}" "${TEMP_DIR}/${BIG_FILE}-mix" cp "${BIG_FILE}" "${BIG_FILE}-mix" local MODIFY_START_BLOCK=$((1*1024*1024)) echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek="${MODIFY_START_BLOCK}" conv=notrunc echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek="${MODIFY_START_BLOCK}" conv=notrunc # Verify contents of file echo "Comparing test file (4)" if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix" then return 1 fi # [NOTE] # For macos, in order to free up disk space for statvfs (or df command), # it is necessary to zero out the file size, delete it, and sync it. # In the case of macos, even if you delete a file, there seems to be a # delay in the free space being reflected. # Testing the ensure_diskfree option shows that if this is not done, free # disk space will be exhausted. # if [ "$(uname)" = "Darwin" ]; then cat /dev/null > "${TEMP_DIR}/${BIG_FILE}" cat /dev/null > "${TEMP_DIR}/${BIG_FILE}-mix" cat /dev/null > "${BIG_FILE}" cat /dev/null > "${BIG_FILE}-mix" fi rm -f "${TEMP_DIR}/${BIG_FILE}" rm -f "${TEMP_DIR}/${BIG_FILE}-mix" rm_test_file "${BIG_FILE}" rm_test_file "${BIG_FILE}-mix" if [ "$(uname)" = "Darwin" ]; then sync fi } function test_utimens_during_multipart { describe "Testing utimens calling during multipart copy ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEMP_DIR}/${BIG_FILE}" cp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}" # The second copy of the "-p" option calls utimens during multipart upload. cp -p "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}" rm -f "${TEMP_DIR}/${BIG_FILE}" rm_test_file "${BIG_FILE}" } function test_special_characters { describe "Testing special characters ..." ( set +o pipefail # shellcheck disable=SC2010 ls 'special' 2>&1 | grep -q 'No such file or directory' # shellcheck disable=SC2010 ls 'special?' 2>&1 | grep -q 'No such file or directory' # shellcheck disable=SC2010 ls 'special*' 2>&1 | grep -q 'No such file or directory' # shellcheck disable=SC2010 ls 'special~' 2>&1 | grep -q 'No such file or directory' # shellcheck disable=SC2010 ls 'specialμ' 2>&1 | grep -q 'No such file or directory' ) mkdir "TOYOTA TRUCK 8.2.2" rm -rf "TOYOTA TRUCK 8.2.2" } function test_hardlink { describe "Testing hardlinks ..." rm -f "${TEST_TEXT_FILE}" rm -f "${ALT_TEST_TEXT_FILE}" echo foo > "${TEST_TEXT_FILE}" ( # [NOTE] # macos-fuse-t returns 'Input/output error' # set +o pipefail ln "${TEST_TEXT_FILE}" "${ALT_TEST_TEXT_FILE}" 2>&1 | grep -q -e 'Operation not supported' -e 'Not supported' -e 'Input/output error' ) rm_test_file rm_test_file "${ALT_TEST_TEXT_FILE}" } function test_mknod { describe "Testing mknod system call function ..." local MKNOD_TEST_FILE_BASENAME="mknod_testfile" rm -f "${MKNOD_TEST_FILE_BASENAME}*" ../../mknod_test "${MKNOD_TEST_FILE_BASENAME}" } function test_symlink { describe "Testing symlinks ..." rm -f "${TEST_TEXT_FILE}" rm -f "${ALT_TEST_TEXT_FILE}" echo foo > "${TEST_TEXT_FILE}" ln -s "${TEST_TEXT_FILE}" "${ALT_TEST_TEXT_FILE}" cmp "${TEST_TEXT_FILE}" "${ALT_TEST_TEXT_FILE}" rm -f "${TEST_TEXT_FILE}" [ -L "${ALT_TEST_TEXT_FILE}" ] [ ! -f "${ALT_TEST_TEXT_FILE}" ] rm -f "${ALT_TEST_TEXT_FILE}" } function test_extended_attributes { describe "Testing extended attributes ..." rm -f "${TEST_TEXT_FILE}" touch "${TEST_TEXT_FILE}" # set value set_xattr key1 value0 "${TEST_TEXT_FILE}" get_xattr key1 "${TEST_TEXT_FILE}" | grep -q '^value0$' # over write value set_xattr key1 value1 "${TEST_TEXT_FILE}" get_xattr key1 "${TEST_TEXT_FILE}" | grep -q '^value1$' # [NOTE] # macOS still caches extended attributes even when told not to. # Thus we need to wait one second here. wait_ostype 1 "Darwin" # append value set_xattr key2 value2 "${TEST_TEXT_FILE}" get_xattr key1 "${TEST_TEXT_FILE}" | grep -q '^value1$' get_xattr key2 "${TEST_TEXT_FILE}" | grep -q '^value2$' # remove value del_xattr key1 "${TEST_TEXT_FILE}" get_xattr key1 "${TEST_TEXT_FILE}" && return 1 get_xattr key2 "${TEST_TEXT_FILE}" | grep -q '^value2$' rm_test_file } function test_mtime_file { describe "Testing mtime preservation function ..." # if the rename file exists, delete it if [ -e "${ALT_TEST_TEXT_FILE}" ] || [ -L "${ALT_TEST_TEXT_FILE}" ] then rm "${ALT_TEST_TEXT_FILE}" fi if [ -e "${ALT_TEST_TEXT_FILE}" ] then echo "Could not delete file ${ALT_TEST_TEXT_FILE}, it still exists" return 1 fi # create the test file again mk_test_file #copy the test file with preserve mode cp -p "${TEST_TEXT_FILE}" "${ALT_TEST_TEXT_FILE}" local testmtime; testmtime=$(get_mtime "${TEST_TEXT_FILE}") local testctime; testctime=$(get_ctime "${TEST_TEXT_FILE}") local testatime; testatime=$(get_atime "${TEST_TEXT_FILE}") local altmtime; altmtime=$(get_mtime "${ALT_TEST_TEXT_FILE}") local altctime; altctime=$(get_ctime "${ALT_TEST_TEXT_FILE}") local altatime; altatime=$(get_atime "${ALT_TEST_TEXT_FILE}") if [ "${testmtime}" != "${altmtime}" ] || [ "${testctime}" = "${altctime}" ] || [ "${testatime}" != "${altatime}" ]; then echo "cp(-p) expected times: mtime( ${testmtime} == ${altmtime} ), ctime( ${testctime} != ${altctime} ), atime( ${testatime} == ${altatime} )" fi rm_test_file rm_test_file "${ALT_TEST_TEXT_FILE}" } # [NOTE] # If it mounted with relatime or noatime options , the "touch -a" # command may not update the atime. # In ubuntu:xenial, atime was updated even if relatime was granted. # However, it was not updated in bionic/focal. # We can probably update atime by explicitly specifying the strictatime # option and running the "touch -a" command. However, the strictatime # option cannot be set. # Therefore, if the relatime option is set, the test with the "touch -a" # command is bypassed. # We do not know why atime is not updated may or not be affected by # these options.(can't say for sure) # However, if atime has not been updated, the s3fs_utimens entry point # will not be called from FUSE library. We added this bypass because # the test became unstable. # function test_update_time_chmod() { describe "Testing update time function chmod..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # chmod -> update only ctime # chmod +x "${TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TEST_TEXT_FILE}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "chmod expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi rm_test_file } function test_update_time_chown() { describe "Testing update time function chown..." # # chown -> update only ctime # local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # [NOTE] # In this test, chown is called with the same UID. # chown "${UID}" "${TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TEST_TEXT_FILE}") if ! uname | grep -q Darwin; then if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi else # [FIXME] macos fuse-t # macos fuse-t doesn't update stat if UID doesn't change. # There is a way to specify "uid=1000" with aws cli and use sudo when chown is executed, but the # test is not finished. # For now, we are just leaving the chown call with the same UID as the parameter. # This test will be fixed in the future. if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then if [ "${base_atime}" = "${atime}" ] && [ "${base_ctime}" = "${ctime}" ] && [ "${base_mtime}" = "${mtime}" ]; then echo "[FIXME] Doing a temporary test bypass : same ctime $base_ctime = $ctime and same mtime: $base_mtime = $mtime and same atime: $base_atime = $atime" else echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime != $mtime, atime: $base_atime != $atime" return 1 fi fi fi rm_test_file } function test_update_time_xattr() { describe "Testing update time function set_xattr..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # set_xattr -> update only ctime # set_xattr key value "${TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TEST_TEXT_FILE}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi rm_test_file } function test_update_time_touch() { describe "Testing update time function touch..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # touch -> update ctime/atime/mtime # touch "${TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TEST_TEXT_FILE}") if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then echo "touch expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime, atime: $base_atime != $atime" return 1 fi rm_test_file } function test_update_time_touch_a() { describe "Testing update time function touch -a..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # "touch -a" -> update ctime/atime, not update mtime # touch -a "${TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TEST_TEXT_FILE}") if ! uname | grep -q Darwin; then if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime == $mtime" return 1 fi else # [macos] fuse-t # atime/ctime/mtime are all updated. # if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime != $mtime" return 1 fi fi rm_test_file } function test_update_time_append() { describe "Testing update time function append..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # append -> update ctime/mtime, not update atime # echo foo >> "${TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TEST_TEXT_FILE}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then echo "append expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime and same atime: $base_atime == $atime" return 1 fi rm_test_file } function test_update_time_cp_p() { describe "Testing update time function cp -p..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # cp -p -> update ctime, not update atime/mtime # local TIME_TEST_TEXT_FILE=test-s3fs-time.txt cp -p "${TEST_TEXT_FILE}" "${TIME_TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TIME_TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TIME_TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TIME_TEST_TEXT_FILE}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "cp with -p option expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi rm_test_file rm_test_file "${TIME_TEST_TEXT_FILE}" } function test_update_time_mv() { describe "Testing update time function mv..." local t0=1000000000 # 9 September 2001 local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/"${TEST_TEXT_FILE}" echo data | aws_cli s3 cp --metadata="atime=${t0},ctime=${t0},mtime=${t0}" - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" local base_atime; base_atime=$(get_atime "${TEST_TEXT_FILE}") local base_ctime; base_ctime=$(get_ctime "${TEST_TEXT_FILE}") local base_mtime; base_mtime=$(get_mtime "${TEST_TEXT_FILE}") # # mv -> update ctime, not update atime/mtime # local TIME2_TEST_TEXT_FILE=test-s3fs-time2.txt mv "${TEST_TEXT_FILE}" "${TIME2_TEST_TEXT_FILE}" local atime; atime=$(get_atime "${TIME2_TEST_TEXT_FILE}") local ctime; ctime=$(get_ctime "${TIME2_TEST_TEXT_FILE}") local mtime; mtime=$(get_mtime "${TIME2_TEST_TEXT_FILE}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "mv expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi rm_test_file "${TIME_TEST_TEXT_FILE}" rm_test_file "${TIME2_TEST_TEXT_FILE}" } # [NOTE] # See the description of test_update_time () for notes about the # "touch -a" command and atime. # function test_update_directory_time_chmod() { describe "Testing update time for directory mv..." # # create the directory and sub-directory and a file in directory # local t0=1000000000 # 9 September 2001 local DIRECTORY_NAME; DIRECTORY_NAME=$(basename "${PWD}")/"${TEST_DIR}" aws_cli s3api put-object --content-type="application/x-directory" --metadata="atime=${t0},ctime=${t0},mtime=${t0}" --bucket "${TEST_BUCKET_1}" --key "$DIRECTORY_NAME/" local base_atime; base_atime=$(get_atime "${TEST_DIR}") local base_ctime; base_ctime=$(get_ctime "${TEST_DIR}") local base_mtime; base_mtime=$(get_mtime "${TEST_DIR}") # # chmod -> update only ctime # chmod 0777 "${TEST_DIR}" local atime; atime=$(get_atime "${TEST_DIR}") local ctime; ctime=$(get_ctime "${TEST_DIR}") local mtime; mtime=$(get_mtime "${TEST_DIR}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "chmod expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi rm -rf "${TEST_DIR}" } function test_update_directory_time_chown { describe "Testing update time for directory chown..." local t0=1000000000 # 9 September 2001 local DIRECTORY_NAME; DIRECTORY_NAME=$(basename "${PWD}")/"${TEST_DIR}" aws_cli s3api put-object --content-type="application/x-directory" --metadata="atime=${t0},ctime=${t0},mtime=${t0}" --bucket "${TEST_BUCKET_1}" --key "$DIRECTORY_NAME/" local base_atime; base_atime=$(get_atime "${TEST_DIR}") local base_ctime; base_ctime=$(get_ctime "${TEST_DIR}") local base_mtime; base_mtime=$(get_mtime "${TEST_DIR}") # # chown -> update only ctime # chown $UID "${TEST_DIR}" local atime; atime=$(get_atime "${TEST_DIR}") local ctime; ctime=$(get_ctime "${TEST_DIR}") local mtime; mtime=$(get_mtime "${TEST_DIR}") if ! uname | grep -q Darwin; then if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi else # [macos] fuse-t # atime/ctime/mtime are not updated. # if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" != "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "touch with -a option expected updated ctime: $base_ctime == $ctime, atime: $base_atime == $atime and same mtime: $base_mtime == $mtime" return 1 fi fi rm -rf "${TEST_DIR}" } function test_update_directory_time_set_xattr { describe "Testing update time for directory set_xattr..." local t0=1000000000 # 9 September 2001 local DIRECTORY_NAME; DIRECTORY_NAME=$(basename "${PWD}")/"${TEST_DIR}" aws_cli s3api put-object --content-type="application/x-directory" --metadata="atime=${t0},ctime=${t0},mtime=${t0}" --bucket "${TEST_BUCKET_1}" --key "$DIRECTORY_NAME/" local base_atime; base_atime=$(get_atime "${TEST_DIR}") local base_ctime; base_ctime=$(get_ctime "${TEST_DIR}") local base_mtime; base_mtime=$(get_mtime "${TEST_DIR}") # # set_xattr -> update only ctime # set_xattr key value "${TEST_DIR}" local atime; atime=$(get_atime "${TEST_DIR}") local ctime; ctime=$(get_ctime "${TEST_DIR}") local mtime; mtime=$(get_mtime "${TEST_DIR}") if ! uname | grep -q Darwin; then if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi else # [macos] fuse-t # atime/mtime are not updated. # if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi fi rm -rf "${TEST_DIR}" } function test_update_directory_time_touch { describe "Testing update time for directory touch..." local t0=1000000000 # 9 September 2001 local DIRECTORY_NAME; DIRECTORY_NAME=$(basename "${PWD}")/"${TEST_DIR}" aws_cli s3api put-object --content-type="application/x-directory" --metadata="atime=${t0},ctime=${t0},mtime=${t0}" --bucket "${TEST_BUCKET_1}" --key "$DIRECTORY_NAME/" local base_atime; base_atime=$(get_atime "${TEST_DIR}") local base_ctime; base_ctime=$(get_ctime "${TEST_DIR}") local base_mtime; base_mtime=$(get_mtime "${TEST_DIR}") # # touch -> update ctime/atime/mtime # touch "${TEST_DIR}" local atime; atime=$(get_atime "${TEST_DIR}") local ctime; ctime=$(get_ctime "${TEST_DIR}") local mtime; mtime=$(get_mtime "${TEST_DIR}") if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then echo "touch expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime, atime: $base_atime != $atime" return 1 fi rm -rf "${TEST_DIR}" } function test_update_directory_time_touch_a { describe "Testing update time for directory touch -a..." local t0=1000000000 # 9 September 2001 local DIRECTORY_NAME; DIRECTORY_NAME=$(basename "${PWD}")/"${TEST_DIR}" aws_cli s3api put-object --content-type="application/x-directory" --metadata="atime=${t0},ctime=${t0},mtime=${t0}" --bucket "${TEST_BUCKET_1}" --key "$DIRECTORY_NAME/" local base_atime; base_atime=$(get_atime "${TEST_DIR}") local base_ctime; base_ctime=$(get_ctime "${TEST_DIR}") local base_mtime; base_mtime=$(get_mtime "${TEST_DIR}") # # "touch -a" -> update ctime/atime, not update mtime # touch -a "${TEST_DIR}" local atime; atime=$(get_atime "${TEST_DIR}") local ctime; ctime=$(get_ctime "${TEST_DIR}") local mtime; mtime=$(get_mtime "${TEST_DIR}") if ! uname | grep -q Darwin; then if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime == $mtime" return 1 fi else # [macos] fuse-t # atime/ctime/mtime are all updated. # if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime != $mtime" return 1 fi fi rm -rf "${TEST_DIR}" } function test_update_directory_time_subdir() { describe "Testing update time for directory subdirectory..." local TIME_TEST_SUBDIR="${TEST_DIR}/testsubdir" local TIME_TEST_FILE_INDIR="${TEST_DIR}/testfile" mk_test_dir mkdir "${TIME_TEST_SUBDIR}" touch "${TIME_TEST_FILE_INDIR}" # TODO: remove sleep after improving AWS CLI speed sleep 1 local base_atime; base_atime=$(get_atime "${TEST_DIR}") local base_ctime; base_ctime=$(get_ctime "${TEST_DIR}") local base_mtime; base_mtime=$(get_mtime "${TEST_DIR}") local subdir_atime; subdir_atime=$(get_atime "${TIME_TEST_SUBDIR}") local subdir_ctime; subdir_ctime=$(get_ctime "${TIME_TEST_SUBDIR}") local subdir_mtime; subdir_mtime=$(get_mtime "${TIME_TEST_SUBDIR}") local subfile_atime; subfile_atime=$(get_atime "${TIME_TEST_FILE_INDIR}") local subfile_ctime; subfile_ctime=$(get_ctime "${TIME_TEST_FILE_INDIR}") local subfile_mtime; subfile_mtime=$(get_mtime "${TIME_TEST_FILE_INDIR}") # # mv -> update ctime, not update atime/mtime for target directory # not update any for sub-directory and a file # local TIME_TEST_DIR=timetestdir local TIME2_TEST_SUBDIR="${TIME_TEST_DIR}/testsubdir" local TIME2_TEST_FILE_INDIR="${TIME_TEST_DIR}/testfile" mv "${TEST_DIR}" "${TIME_TEST_DIR}" local atime; atime=$(get_atime "${TIME_TEST_DIR}") local ctime; ctime=$(get_ctime "${TIME_TEST_DIR}") local mtime; mtime=$(get_mtime "${TIME_TEST_DIR}") if [ "${base_atime}" != "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" != "${mtime}" ]; then echo "mv expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi atime=$(get_atime "${TIME2_TEST_SUBDIR}") ctime=$(get_ctime "${TIME2_TEST_SUBDIR}") mtime=$(get_mtime "${TIME2_TEST_SUBDIR}") if [ "${subdir_atime}" != "${atime}" ] || [ "${subdir_ctime}" != "${ctime}" ] || [ "${subdir_mtime}" != "${mtime}" ]; then echo "mv for sub-directory expected same ctime: $subdir_ctime == $ctime, mtime: $subdir_mtime == $mtime, atime: $subdir_atime == $atime" return 1 fi atime=$(get_atime "${TIME2_TEST_FILE_INDIR}") ctime=$(get_ctime "${TIME2_TEST_FILE_INDIR}") mtime=$(get_mtime "${TIME2_TEST_FILE_INDIR}") if [ "${subfile_atime}" != "${atime}" ] || [ "${subfile_ctime}" != "${ctime}" ] || [ "${subfile_mtime}" != "${mtime}" ]; then echo "mv for a file in directory expected same ctime: $subfile_ctime == $ctime, mtime: $subfile_mtime == $mtime, atime: $subfile_atime == $atime" return 1 fi rm -rf "${TIME_TEST_SUBDIR}" rm -rf "${TIME_TEST_DIR}" rm -rf "${TEST_DIR}" } # [NOTE] # This test changes the file mode while creating/editing a new file, # and finally closes it. # Test with the sed command as it occurs when in place mode of the sed # command. (If trying it with a standard C function(and shell script), # it will be not the same result of sed, so sed is used.) # function test_update_chmod_opened_file() { describe "Testing create, modify the file by sed in place mode" # test file local BEFORE_STRING_DATA; BEFORE_STRING_DATA="sed in place test : BEFORE DATA" local AFTER_STRING_DATA; AFTER_STRING_DATA="sed in place test : AFTER DATA" echo "${BEFORE_STRING_DATA}" > "${TEST_TEXT_FILE}" # sed in place sed -i -e 's/BEFORE DATA/AFTER DATA/g' "${TEST_TEXT_FILE}" # compare result local RESULT_STRING; RESULT_STRING=$(cat "${TEST_TEXT_FILE}") if [ -z "${RESULT_STRING}" ] || [ "${RESULT_STRING}" != "${AFTER_STRING_DATA}" ]; then echo "the file conversion by sed in place command failed." return 1 fi # clean up rm_test_file "${ALT_TEST_TEXT_FILE}" } function test_update_parent_directory_time_sub() { if [ $# -ne 1 ]; then echo "Internal error: parameter is wrong." return 1 fi # [NOTE] # Skip test for mknod/mkfifo command. # If run them, ctime/mtime of the parent directory will be updated. # local TEST_PARENTDIR_PARENT="${1}" local TEST_PARENTDIR_FILE="${TEST_PARENTDIR_PARENT}/testfile" local TEST_PARENTDIR_SYMFILE_BASE="testfile2" local TEST_PARENTDIR_FILE_MV="${TEST_PARENTDIR_PARENT}/${TEST_PARENTDIR_SYMFILE_BASE}" local TEST_PARENTDIR_SYMFILE="${TEST_PARENTDIR_PARENT}/symfile" local TEST_PARENTDIR_SYMFILE_MV="${TEST_PARENTDIR_PARENT}/symfile2" local TEST_PARENTDIR_DIR="${TEST_PARENTDIR_PARENT}/testdir" local TEST_PARENTDIR_DIR_MV="${TEST_PARENTDIR_PARENT}/testdir2" # # Create file -> Darwin: Not update any # -> Others: Update parent directory's mtime/ctime # local base_atime; base_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local base_ctime; base_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local base_mtime; base_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") touch "${TEST_PARENTDIR_FILE}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "creating file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Update file -> Not update parent directory's atime/mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" touch "${TEST_PARENTDIR_FILE}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" != "${after_ctime}" ] || [ "${base_mtime}" != "${after_mtime}" ]; then echo "updating file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} == ${after_ctime} ), mtime( ${base_mtime} == ${after_mtime} )" return 1 fi # # Rename file -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" mv "${TEST_PARENTDIR_FILE}" "${TEST_PARENTDIR_FILE_MV}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "renaming file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Create symbolic link -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" ln -s "${TEST_PARENTDIR_SYMFILE_BASE}" "${TEST_PARENTDIR_SYMFILE}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "creating symbolic file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Update symbolic file -> Not update parent directory's atime/mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" touch "${TEST_PARENTDIR_SYMFILE}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" != "${after_ctime}" ] || [ "${base_mtime}" != "${after_mtime}" ]; then echo "updating symbolic file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} == ${after_ctime} ), mtime( ${base_mtime} == ${after_mtime} )" return 1 fi # # Rename symbolic link -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" mv "${TEST_PARENTDIR_SYMFILE}" "${TEST_PARENTDIR_SYMFILE_MV}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "renaming symbolic file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Delete symbolic link -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" rm "${TEST_PARENTDIR_SYMFILE_MV}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "deleting symbolic file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Delete file -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" rm "${TEST_PARENTDIR_FILE_MV}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "deleting file expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Create directory -> Update parent directory's mtime/ctime # local base_atime; base_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local base_ctime; base_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local base_mtime; base_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") mkdir "${TEST_PARENTDIR_DIR}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "creating directory expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Update directory -> Not update parent directory's atime/mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" touch "${TEST_PARENTDIR_DIR}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" != "${after_ctime}" ] || [ "${base_mtime}" != "${after_mtime}" ]; then echo "updating directory expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} == ${after_ctime} ), mtime( ${base_mtime} == ${after_mtime} )" return 1 fi # # Rename directory -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" mv "${TEST_PARENTDIR_DIR}" "${TEST_PARENTDIR_DIR_MV}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "renaming directory expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi # # Delete directory -> Update parent directory's mtime/ctime # base_atime="${after_atime}" base_ctime="${after_ctime}" base_mtime="${after_mtime}" rm -r "${TEST_PARENTDIR_DIR_MV}" local after_atime; after_atime=$(get_atime "${TEST_PARENTDIR_PARENT}") local after_ctime; after_ctime=$(get_ctime "${TEST_PARENTDIR_PARENT}") local after_mtime; after_mtime=$(get_mtime "${TEST_PARENTDIR_PARENT}") if [ "${base_atime}" != "${after_atime}" ] || [ "${base_ctime}" = "${after_ctime}" ] || [ "${base_mtime}" = "${after_mtime}" ]; then echo "deleting directory expected updating ctime/mtime: atime( ${base_atime} == ${after_atime} ), ctime( ${base_ctime} != ${after_ctime} ), mtime( ${base_mtime} != ${after_mtime} )" return 1 fi return 0 } function test_update_parent_directory_time() { describe "Testing update time of parent directory..." # # Test sub directory # mk_test_dir if ! test_update_parent_directory_time_sub "${TEST_DIR}"; then echo "failed test about updating time of parent directory: ${TEST_DIR}" return 1 fi rm -rf "${TEST_DIR}" # # Test bucket top directory # # [NOTE] # The current directory for test execution is "/testrun-xxxx". # This test checks in the directory at the top of the bucket. # if ! test_update_parent_directory_time_sub ".."; then echo "failed test about updating time of parent directory: ${TEST_DIR}" return 1 fi return 0 } function test_rm_rf_dir { describe "Test that rm -rf will remove directory with contents ..." # Create a dir with some files and directories mkdir dir1 mkdir dir1/dir2 touch dir1/file1 touch dir1/dir2/file2 # Remove the dir with recursive rm rm -rf dir1 if [ -e dir1 ]; then echo "rm -rf did not remove $PWD/dir1" return 1 fi } function test_posix_acl { describe "Testing posix acl function ..." #------------------------------------------------------ # Directory #------------------------------------------------------ local POSIX_ACL_TEST_DIR1="posix_acl_dir1" local POSIX_ACL_TEST_DIR2="posix_acl_dir2" rm -rf "${POSIX_ACL_TEST_DIR1}" rm -rf "${POSIX_ACL_TEST_DIR2}" mkdir "${POSIX_ACL_TEST_DIR1}" # [NOTE] # Let the target username be "root". # On some systems the USER environment variable may be # empty, so using ${USER} will not give an accurate test. # TARGET_ACL_USER="root" # # Set posix acl(not default) # setfacl -m "u:${TARGET_ACL_USER}:rwx" "${POSIX_ACL_TEST_DIR1}" if ! getfacl "${POSIX_ACL_TEST_DIR1}" | grep -q "^user:${TARGET_ACL_USER}:rwx"; then echo "Could not set posix acl(not default) to ${POSIX_ACL_TEST_DIR1} directory" return 1 fi # # Set posix acl(default) # setfacl -d -m "u:${TARGET_ACL_USER}:rwx" "${POSIX_ACL_TEST_DIR1}" if ! getfacl "${POSIX_ACL_TEST_DIR1}" | grep -q "^default:user:${TARGET_ACL_USER}:rwx"; then echo "Could not set posix acl(default) to ${POSIX_ACL_TEST_DIR1} directory" return 1 fi # # Rename # mv "${POSIX_ACL_TEST_DIR1}" "${POSIX_ACL_TEST_DIR2}" if ! getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^user:${TARGET_ACL_USER}:rwx"; then echo "Could not move with posix acl(not default) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi if ! getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^default:user:${TARGET_ACL_USER}:rwx"; then echo "Could not move with posix acl(default) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi # # Copy with permission # cp -rp "${POSIX_ACL_TEST_DIR2}" "${POSIX_ACL_TEST_DIR1}" if ! getfacl "${POSIX_ACL_TEST_DIR1}" | grep -q "^user:${TARGET_ACL_USER}:rwx"; then echo "Could not copy with posix acl(not default) to ${POSIX_ACL_TEST_DIR1} directory" return 1 fi if ! getfacl "${POSIX_ACL_TEST_DIR1}" | grep -q "^default:user:${TARGET_ACL_USER}:rwx"; then echo "Could not copy with posix acl(default) to ${POSIX_ACL_TEST_DIR1} directory" return 1 fi # # Overwrite posix acl(not default) # setfacl -m "u:${TARGET_ACL_USER}:r-x" "${POSIX_ACL_TEST_DIR2}" if ! getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^user:${TARGET_ACL_USER}:r-x"; then echo "Could not set posix acl(not default) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi # # Overwrite posix acl(default) # setfacl -d -m "u:${TARGET_ACL_USER}:r-x" "${POSIX_ACL_TEST_DIR2}" if ! getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^default:user:${TARGET_ACL_USER}:r-x"; then echo "Could not set posix acl(default) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi # # Remove posix acl(default) # setfacl -k "${POSIX_ACL_TEST_DIR2}" if getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^default:user:${TARGET_ACL_USER}"; then echo "Could not remove posix acl(default) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi # # Remove posix acl(all) # setfacl -b "${POSIX_ACL_TEST_DIR2}" if getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^user:${TARGET_ACL_USER}"; then echo "Could not remove posix acl(all) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi # # Copy without permission # rm -rf "${POSIX_ACL_TEST_DIR2}" cp -r "${POSIX_ACL_TEST_DIR1}" "${POSIX_ACL_TEST_DIR2}" if getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^default:user:${TARGET_ACL_USER}"; then echo "Could not copy without posix acl(default) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi if getfacl "${POSIX_ACL_TEST_DIR2}" | grep -q "^user:${TARGET_ACL_USER}"; then echo "Could not copy without posix acl(all) to ${POSIX_ACL_TEST_DIR2} directory" return 1 fi #------------------------------------------------------ # File #------------------------------------------------------ local POSIX_ACL_TEST_FILE1="posix_acl_dir1/posix_acl_file1" local POSIX_ACL_TEST_FILE2="posix_acl_dir1/posix_acl_file2" local POSIX_ACL_TEST_FILE3="posix_acl_dir2/posix_acl_file3" local POSIX_ACL_TEST_FILE4="posix_acl_dir2/posix_acl_file4" mkdir "${POSIX_ACL_TEST_DIR2}" touch "${POSIX_ACL_TEST_FILE1}" # # Check default inherited posix acl # if ! getfacl "${POSIX_ACL_TEST_FILE1}" | grep -q "^user:${TARGET_ACL_USER}:rwx"; then echo "Could not set posix acl(inherited default) to ${POSIX_ACL_TEST_FILE1} file" return 1 fi # # Overwrite posix acl # setfacl -m "u:${TARGET_ACL_USER}:r-x" "${POSIX_ACL_TEST_FILE1}" if ! getfacl "${POSIX_ACL_TEST_FILE1}" | grep -q "^user:${TARGET_ACL_USER}:r-x"; then echo "Could not overwrite posix acl to ${POSIX_ACL_TEST_FILE1} file" return 1 fi # # Rename # mv "${POSIX_ACL_TEST_FILE1}" "${POSIX_ACL_TEST_FILE2}" if ! getfacl "${POSIX_ACL_TEST_FILE2}" | grep -q "^user:${TARGET_ACL_USER}:r-x"; then echo "Could not move with posix acl to ${POSIX_ACL_TEST_FILE2} file" return 1 fi # # Copy with permission # cp -p "${POSIX_ACL_TEST_FILE2}" "${POSIX_ACL_TEST_FILE1}" if ! getfacl "${POSIX_ACL_TEST_FILE1}" | grep -q "^user:${TARGET_ACL_USER}:r-x"; then echo "Could not copy with posix acl to ${POSIX_ACL_TEST_FILE1} file" return 1 fi # # Remove posix acl # setfacl -b "${POSIX_ACL_TEST_FILE2}" if getfacl "${POSIX_ACL_TEST_FILE2}" | grep -q "^default:user:${TARGET_ACL_USER}"; then echo "Could not remove posix acl to ${POSIX_ACL_TEST_FILE2} file" return 1 fi # # Copy without permission(set parent directory default acl) # rm -f "${POSIX_ACL_TEST_FILE2}" cp "${POSIX_ACL_TEST_FILE1}" "${POSIX_ACL_TEST_FILE2}" if ! getfacl "${POSIX_ACL_TEST_FILE2}" | grep -q "^user:${TARGET_ACL_USER}:rwx"; then echo "Could not copy without posix acl(inherited parent) to ${POSIX_ACL_TEST_FILE2} file" return 1 fi # # Copy with permission(to no-acl directory) # cp -p "${POSIX_ACL_TEST_FILE1}" "${POSIX_ACL_TEST_FILE3}" if ! getfacl "${POSIX_ACL_TEST_FILE3}" | grep -q "^user:${TARGET_ACL_USER}:r-x"; then echo "Could not copy with posix acl to ${POSIX_ACL_TEST_FILE3} file in no-acl directory" return 1 fi # # Copy without permission(to no-acl directory) # cp "${POSIX_ACL_TEST_FILE1}" "${POSIX_ACL_TEST_FILE4}" if getfacl "${POSIX_ACL_TEST_FILE4}" | grep -q "^user:${TARGET_ACL_USER}"; then echo "Could not copy without posix acl to ${POSIX_ACL_TEST_FILE4} file in no-acl directory" return 1 fi rm -rf "${POSIX_ACL_TEST_DIR1}" rm -rf "${POSIX_ACL_TEST_DIR2}" } function test_copy_file { describe "Test simple copy ..." dd if=/dev/urandom of=/tmp/simple_file bs=1024 count=1 cp /tmp/simple_file copied_simple_file cmp /tmp/simple_file copied_simple_file rm_test_file /tmp/simple_file rm_test_file copied_simple_file } function test_write_after_seek_ahead { describe "Test writes succeed after a seek ahead ..." dd if=/dev/zero of=testfile seek=1 count=1 bs=1024 rm_test_file testfile } function test_overwrite_existing_file_range { describe "Test overwrite range succeeds ..." dd if=<(seq 1000) of="${TEST_TEXT_FILE}" dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=1 count=1 bs=1024 conv=notrunc # [NOTE] # In ALPINE, comparisons using redirects often fail. # Therefore, create a real file for comparison and use it. # dd if=<(seq 1000) of=/tmp/cmp_base_file dd if=/dev/zero of=/tmp/cmp_base_file seek=1 count=1 bs=1024 conv=notrunc cmp "${TEST_TEXT_FILE}" /tmp/cmp_base_file rm_test_file /tmp/cmp_base_file rm_test_file } function test_concurrent_directory_updates { describe "Test concurrent updates to a directory ..." for i in $(seq 5); do echo foo > "${i}" done for _ in $(seq 10); do for i in $(seq 5); do local file # shellcheck disable=SC2012,SC2046 file=$(ls $(seq 5) | "${SED_BIN}" -n "$((RANDOM % 5 + 1))p") cat "${file}" >/dev/null || true rm -f "${file}" echo "foo" > "${file}" || true done & done wait # shellcheck disable=SC2046 rm -f $(seq 5) } function test_concurrent_reads { describe "Test concurrent reads from a file ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEST_TEXT_FILE}" for _ in $(seq 10); do dd if="${TEST_TEXT_FILE}" of=/dev/null seek=$((RANDOM % BIG_FILE_LENGTH)) count=16 bs=1024 & done wait rm_test_file } function test_concurrent_writes { describe "Test concurrent writes to a file ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${TEST_TEXT_FILE}" NUM_PROCS=10 PIDS=() for _ in $(seq "${NUM_PROCS}"); do dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=$((RANDOM % BIG_FILE_LENGTH)) count=16 bs=1024 conv=notrunc & PIDS+=($!) done GRC=0 for PID in "${PIDS[@]}"; do wait "${PID}" RC=$? [ $RC -ne 0 ] && GRC="${RC}" done rm_test_file if [ "${GRC}" -ne 0 ]; then echo "unexpected return code: $GRC" return 1 fi } function test_open_second_fd { describe "read from an open fd ..." rm_test_file second_fd_file local RESULT # shellcheck disable=SC2094 RESULT=$( (echo foo ; wc -c < second_fd_file >&2) 2>& 1>second_fd_file) if [ "${RESULT}" -ne 4 ]; then echo "size mismatch, expected: 4, was: ${RESULT}" return 1 fi rm_test_file second_fd_file } function test_write_multiple_offsets { describe "test writing to multiple offsets ..." ../../write_multiblock -f "${TEST_TEXT_FILE}" -p "1024:1" -p "$((16 * 1024 * 1024)):1" -p "$((18 * 1024 * 1024)):1" rm_test_file "${TEST_TEXT_FILE}" } function test_write_multiple_offsets_backwards { describe "test writing to multiple offsets ..." ../../write_multiblock -f "${TEST_TEXT_FILE}" -p "$((20 * 1024 * 1024 + 1)):1" -p "$((10 * 1024 * 1024)):1" rm_test_file "${TEST_TEXT_FILE}" } function test_clean_up_cache() { describe "Test clean up cache ..." local dir="many_files" local count=25 mkdir -p "${dir}" for x in $(seq "${count}"); do ../../junk_data 10485760 > "${dir}"/file-"${x}" done local file_list=("${dir}"/*); local file_cnt="${#file_list[@]}" if [ "${file_cnt}" != "${count}" ]; then echo "Expected $count files but got ${file_cnt}" echo "Files: " "${file_list[@]}" rm -rf "${dir}" return 1 fi local CACHE_DISK_AVAIL_SIZE; CACHE_DISK_AVAIL_SIZE=$(get_disk_avail_size "${CACHE_DIR}") if [ "${CACHE_DISK_AVAIL_SIZE}" -lt "${ENSURE_DISKFREE_SIZE}" ];then echo "Cache disk avail size:${CACHE_DISK_AVAIL_SIZE} less than ensure_diskfree size:${ENSURE_DISKFREE_SIZE}" rm -rf "${dir}" return 1 fi rm -rf "${dir}" } function test_content_type() { describe "Test Content-Type detection ..." local DIR_NAME; DIR_NAME=$(basename "${PWD}") touch "test.txt" check_content_type "${DIR_NAME}/test.txt" "text/plain" touch "test.jpg" check_content_type "${DIR_NAME}/test.jpg" "image/jpeg" touch "test.bin" check_content_type "${DIR_NAME}/test.bin" "application/octet-stream" mkdir "test.dir" check_content_type "${DIR_NAME}/test.dir/" "application/x-directory" rm -f test.txt rm -f test.jpg rm -f test.bin rm -rf test.dir } # create more files than -o max_stat_cache_size function test_truncate_cache() { describe "Test make cache files over max cache file size ..." for dir in $(seq 2); do mkdir "${dir}" for file in $(seq 75); do touch "${dir}/${file}" done # FIXME: # In the case of macos-fuse-t, if you do not enter a wait here, the following error may occur: # "ls: fts_read: Input/output error" # Currently, we have not yet been able to establish a solution to this problem. # Please pay attention to future developments in macos-fuse-t. # wait_ostype 1 "Darwin" ls "${dir}" done # shellcheck disable=SC2046 rm -rf $(seq 2) } function test_cache_file_stat() { describe "Test cache file stat ..." ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${BIG_FILE}" # # The first argument of the script is "testrun-" the directory name. # local CACHE_TESTRUN_DIR=$1 # # get cache file inode number # local CACHE_FILE_INODE CACHE_FILE_INODE=$(get_inode "${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}") if [ -z "${CACHE_FILE_INODE}" ]; then echo "Not found cache file or failed to get inode: ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}" return 1; fi # # get lines from cache stat file # local CACHE_FILE_STAT_LINE_1; CACHE_FILE_STAT_LINE_1=$("${SED_BIN}" -n 1p "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}") local CACHE_FILE_STAT_LINE_2; CACHE_FILE_STAT_LINE_2=$("${SED_BIN}" -n 2p "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}") if [ -z "${CACHE_FILE_STAT_LINE_1}" ] || [ -z "${CACHE_FILE_STAT_LINE_2}" ]; then echo "could not get first or second line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}" return 1; fi # # compare # if [ "${CACHE_FILE_STAT_LINE_1}" != "${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}" ]; then echo "first line(cache file stat) is different: \"${CACHE_FILE_STAT_LINE_1}\" != \"${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}\"" return 1; fi if [ "${CACHE_FILE_STAT_LINE_2}" != "0:${BIG_FILE_LENGTH}:1:0" ]; then echo "last line(cache file stat) is different: \"${CACHE_FILE_STAT_LINE_2}\" != \"0:${BIG_FILE_LENGTH}:1:0\"" return 1; fi # # remove cache files directly # rm -f "${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}" rm -f "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}" # # write a byte into the middle(not the boundary) of the file # local CHECK_UPLOAD_OFFSET=$((10 * 1024 * 1024 + 17)) dd if=/dev/urandom of="${BIG_FILE}" bs=1 count=1 seek="${CHECK_UPLOAD_OFFSET}" conv=notrunc # # get cache file inode number # CACHE_FILE_INODE=$(get_inode "${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}") if [ -z "${CACHE_FILE_INODE}" ]; then echo "Not found cache file or failed to get inode: ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}" return 1; fi # # get lines from cache stat file # CACHE_FILE_STAT_LINE_1=$("${SED_BIN}" -n 1p "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}") local CACHE_FILE_STAT_LINE_E; CACHE_FILE_STAT_LINE_E=$(tail -1 "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}" 2>/dev/null) if [ -z "${CACHE_FILE_STAT_LINE_1}" ] || [ -z "${CACHE_FILE_STAT_LINE_E}" ]; then echo "could not get first or end line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}" return 1; fi # # check first and cache file length from last line # # we should check all stat lines, but there are cases where the value # differs depending on the processing system etc., then the cache file # size is calculated and compared. # local CACHE_LAST_OFFSET; CACHE_LAST_OFFSET=$(echo "${CACHE_FILE_STAT_LINE_E}" | cut -d ":" -f1) local CACHE_LAST_SIZE; CACHE_LAST_SIZE=$(echo "${CACHE_FILE_STAT_LINE_E}" | cut -d ":" -f2) local CACHE_TOTAL_SIZE=$((CACHE_LAST_OFFSET + CACHE_LAST_SIZE)) if [ "${CACHE_FILE_STAT_LINE_1}" != "${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}" ]; then echo "first line(cache file stat) is different: \"${CACHE_FILE_STAT_LINE_1}\" != \"${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}\"" return 1; fi if [ "${BIG_FILE_LENGTH}" -ne "${CACHE_TOTAL_SIZE}" ]; then echo "the file size indicated by the cache stat file is different: \"${BIG_FILE_LENGTH}\" != \"${CACHE_TOTAL_SIZE}\"" return 1; fi rm_test_file "${BIG_FILE}" } function test_zero_cache_file_stat() { describe "Test zero byte cache file stat ..." rm_test_file "${TEST_TEXT_FILE}" # # create empty file # touch "${TEST_TEXT_FILE}" # # The first argument of the script is "testrun-" the directory name. # local CACHE_TESTRUN_DIR=$1 # [NOTE] # The stat file is a one-line text file, expecting for ":0"(ex. "4543937: 0"). # if ! head -1 "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${TEST_TEXT_FILE}" 2>/dev/null | grep -q ':0$' 2>/dev/null; then echo "The cache file stat after creating an empty file is incorrect : ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${TEST_TEXT_FILE}" return 1; fi rm_test_file "${TEST_TEXT_FILE}" } function test_upload_sparsefile { describe "Testing upload sparse file ..." rm_test_file "${BIG_FILE}" rm -f "${TEMP_DIR}/${BIG_FILE}" # # Make all HOLE file # "${TRUNCATE_BIN}" "${BIG_FILE}" -s "${BIG_FILE_LENGTH}" # # Write some bytes to ABOUT middle in the file # (Dare to remove the block breaks) # local WRITE_POS=$((BIG_FILE_LENGTH / 2 - 128)) echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}" bs=1 count=16 seek="${WRITE_POS}" conv=notrunc # # copy(upload) the file # cp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}" # # check # cmp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}" rm_test_file "${BIG_FILE}" rm -f "${TEMP_DIR}/${BIG_FILE}" } function test_mix_upload_entities() { describe "Testing upload sparse files ..." # # Make test file # ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${BIG_FILE}" # # If the cache option is enabled, delete the cache of uploaded files. # if [ -f "${CACHE_DIR}/${TEST_BUCKET_1}/${BIG_FILE}" ]; then rm -f "${CACHE_DIR}/${TEST_BUCKET_1}/${BIG_FILE}" fi if [ -f "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${BIG_FILE}" ]; then rm -f "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${BIG_FILE}" fi # # Do a partial write to the file. # echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}" bs=1 count=16 seek=0 conv=notrunc echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}" bs=1 count=16 seek=8192 conv=notrunc echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}" bs=1 count=16 seek=1073152 conv=notrunc echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}" bs=1 count=16 seek=26214400 conv=notrunc echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}" bs=1 count=16 seek=26222592 conv=notrunc rm_test_file "${BIG_FILE}" } # # [NOTE] # This test runs last because it uses up disk space and may not recover. # This may be a problem, especially on MacOS. (See the comment near the definition # line for the ENSURE_DISKFREE_SIZE variable) # function test_ensurespace_move_file() { describe "Testing upload(mv) file when disk space is not enough ..." # # Make test file which is not under mountpoint # mkdir -p "${CACHE_DIR}/.s3fs_test_tmpdir" ../../junk_data $((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) > "${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}" # # Backup file stat # local ORIGINAL_PERMISSIONS ORIGINAL_PERMISSIONS=$(get_user_and_group "${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}") # # Fill the disk size # local NOW_CACHE_DISK_AVAIL_SIZE; NOW_CACHE_DISK_AVAIL_SIZE=$(get_disk_avail_size "${CACHE_DIR}") local TMP_FILE_NO=0 while true; do local ALLOWED_USING_SIZE=$((NOW_CACHE_DISK_AVAIL_SIZE - ENSURE_DISKFREE_SIZE)) if [ "${ALLOWED_USING_SIZE}" -gt "${BIG_FILE_LENGTH}" ]; then cp -p "${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}" "${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}_${TMP_FILE_NO}" local TMP_FILE_NO=$((TMP_FILE_NO + 1)) else break; fi done # # move file # mv "${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}" "${BIG_FILE}" # # file stat # local MOVED_PERMISSIONS MOVED_PERMISSIONS=$(get_user_and_group "${BIG_FILE}") local MOVED_FILE_LENGTH MOVED_FILE_LENGTH=$(get_size "${BIG_FILE}") # # check # if [ "${MOVED_PERMISSIONS}" != "${ORIGINAL_PERMISSIONS}" ]; then echo "Failed to move file with permission" return 1 fi if [ "${MOVED_FILE_LENGTH}" -ne "${BIG_FILE_LENGTH}" ]; then echo "Failed to move file with file length: ${MOVED_FILE_LENGTH} ${BIG_FILE_LENGTH}" return 1 fi rm_test_file "${BIG_FILE}" rm -rf "${CACHE_DIR}/.s3fs_test_tmpdir" } function test_not_existed_dir_obj() { describe "Test not existed directory object..." local DIR_NAME; DIR_NAME=$(basename "${PWD}") # # Create files under not existed directory by aws command # local OBJECT_NAME_1; OBJECT_NAME_1="${DIR_NAME}/not_existed_dir_single/${TEST_TEXT_FILE}" local OBJECT_NAME_2; OBJECT_NAME_2="${DIR_NAME}/not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}" echo data1 | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME_1}" echo data2 | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME_2}" # Top directory # shellcheck disable=SC2010 if ! ls -1 | grep -q '^not_existed_dir_single$'; then echo "Expect to find \"not_existed_dir_single\" directory, but it is not found" return 1 fi # shellcheck disable=SC2010 if ! ls -1 | grep -q '^not_existed_dir_parent$'; then echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found" return 1 fi # Single nest directory if ! stat not_existed_dir_single; then echo "Expect to find \"not_existed_dir_single\" directory, but it is not found" return 1 fi # shellcheck disable=SC2010 if ! ls -1 not_existed_dir_single | grep -q "^${TEST_TEXT_FILE}\$"; then echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found" return 1 fi # shellcheck disable=SC2010 if ! ls -1 "not_existed_dir_single/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_single/${TEST_TEXT_FILE}\$"; then echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found" return 1 fi # Double nest directory if ! stat not_existed_dir_parent; then echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found" return 1 fi # shellcheck disable=SC2010 if ! ls -1 not_existed_dir_parent | grep -q '^not_existed_dir_child'; then echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found" return 1 fi if ! stat not_existed_dir_parent/not_existed_dir_child; then echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found" return 1 fi # shellcheck disable=SC2010 if ! ls -1 not_existed_dir_parent/not_existed_dir_child | grep -q "^${TEST_TEXT_FILE}\$"; then echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found" return 1 fi # shellcheck disable=SC2010 if ! ls -1 "not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\$"; then echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found" return 1 fi rm -rf not_existed_dir_single rm -rf not_existed_dir_parent } function test_ut_ossfs { describe "Testing ossfs python ut..." # shellcheck disable=SC2153 export TEST_BUCKET_MOUNT_POINT="${TEST_BUCKET_MOUNT_POINT_1}" ../../ut_test.py } function test_cr_filename { describe "Testing filename with CR code ..." # The following tests create a file, test it, and delete it. # So this test just calls the following program. # ../../cr_filename "${TEST_TEXT_FILE}" } # # This test opens a file and writes multiple sets of data. # The file is opened only once and multiple blocks of data are written # to the file descriptor with a gap. # # That is, the data sets are written discontinuously. # The data to be written uses multiple data that is less than or larger # than the part size of the multi-part upload. # The gap should be at least the part size of the multi-part upload. # Write as shown below: # ............ # # There are two types of tests: new files and existing files. # For existing files, the file size must be larger than where this test # writes last position. # ............... # function test_write_data_with_skip() { describe "Testing write data block with skipping block..." # # The first argument of the script is "testrun-" the directory name. # local CACHE_TESTRUN_DIR=$1 local _SKIPWRITE_FILE="test_skipwrite" local _TMP_SKIPWRITE_FILE="/tmp/${_SKIPWRITE_FILE}" #------------------------------------------------------ # (1) test new file #------------------------------------------------------ # # Clean files # rm_test_file "${_SKIPWRITE_FILE}" rm_test_file "${_TMP_SKIPWRITE_FILE}" # # Create new file in bucket and temporary directory(/tmp) # # Writing to the file is as follows: # |<-- skip(12MB) --><-- write(1MB) --><-- skip(22MB) --><-- write(20MB) --><-- skip(23MB) --><-- write(1MB) -->| (79MB) # # As a result, areas that are not written to the file are mixed. # The part that is not written has a HOLE that is truncate and filled # with 0x00. # Assuming that multipart upload is performed on a part-by-part basis, # it will be as follows: # part 1) 0x0.. 0x9FFFFF : # part 2) 0xA00000..0x13FFFFF : 0xA00000..0xBFFFFF # 0xC00000..0xCFFFFF # 0xD00000..0x13FFFFF # part 3) 0x1400000..0x1DFFFFF : # part 4) 0x1E00000..0x27FFFFF : 0x1E00000..0x22FFFFF # 0x2300000..0x27FFFFF # part 5) 0x2800000..0x31FFFFF : # part 6) 0x3200000..0x3BFFFFF : 0x3200000..0x36FFFFF # 0x3700000..0x3BFFFFF # part 7) 0x3C00000..0x45FFFFF : # part 8) 0x4600000..0x4BFFFFF : 0x4600000..0x4AFFFFF # 0x4B00000..0x4BFFFFF # ../../write_multiblock -f "${_SKIPWRITE_FILE}" -f "${_TMP_SKIPWRITE_FILE}" -p 12582912:65536 -p 36700160:20971520 -p 78643200:65536 # # delete cache file if using cache # if s3fs_args | grep -q use_cache; then rm -f "${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}" rm -f "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}" fi # # Compare # cmp "${_SKIPWRITE_FILE}" "${_TMP_SKIPWRITE_FILE}" #------------------------------------------------------ # (2) test existed file #------------------------------------------------------ # [NOTE] # This test uses the file used in the previous test as an existing file. # if s3fs_args | grep -q use_cache; then rm -f "${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}" rm -f "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}" fi # # Over write data to existed file in bucket and temporary directory(/tmp) # # Writing to the file is as follows: # |<----------------------------------------------- existed file ----------------------------------------------------------->| (79MB) # |<-- skip(12MB) --><-- write(1MB) --><-- skip(22MB) --><-- write(20MB) --><-- skip(22MB) --><-- write(1MB) --><-- 1MB -->| (79MB) # # As a result, areas that are not written to the file are mixed. # The part that is not written has a HOLE that is truncate and filled # with 0x00. # Assuming that multipart upload is performed on a part-by-part basis, # it will be as follows: # part 1) 0x0.. 0x9FFFFF : # part 2) 0xA00000..0x13FFFFF : 0xA00000..0xBFFFFF # 0xC00000..0xCFFFFF # 0xD00000..0x13FFFFF # part 3) 0x1400000..0x1DFFFFF : # part 4) 0x1E00000..0x27FFFFF : 0x1E00000..0x22FFFFF # 0x2300000..0x27FFFFF # part 5) 0x2800000..0x31FFFFF : # part 6) 0x3200000..0x3BFFFFF : 0x3200000..0x36FFFFF # 0x3700000..0x3BFFFFF # part 7) 0x3C00000..0x45FFFFF : # part 8) 0x4600000..0x4BFFFFF : 0x4600000..0x49FFFFF # part 8) 0x4600000..0x4BFFFFF : 0x4A00000..0x4AFFFFF # 0x4B00000..0x4BFFFFF # ../../write_multiblock -f "${_SKIPWRITE_FILE}" -f "${_TMP_SKIPWRITE_FILE}" -p 12582912:65536 -p 36700160:20971520 -p 77594624:65536 # # delete cache file if using cache # if s3fs_args | grep -q use_cache; then rm -f "${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}" rm -f "${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${_SKIPWRITE_FILE}" fi # # Compare # cmp "${_SKIPWRITE_FILE}" "${_TMP_SKIPWRITE_FILE}" # # Clean files # rm_test_file "${_SKIPWRITE_FILE}" rm_test_file "${_TMP_SKIPWRITE_FILE}" } function test_not_boundary_writes { describe "Test non-boundary write ..." # [MEMO] # Files used in this test, multipart related sizes, etc. # # Test file size: 25MB(25 * 1024 * 1024) # Multipart size: 10MB # Multipart minimum upload size: 5MB # # The multipart upload part that should be executed here is as follows: # Part number 1: 0 - 10,485,759 (size = 10MB) # Part number 2: 10,485,760 - 20,971,519 (size = 10MB) # Part number 3: 20,971,520 - 26,214,399 (size = 5MB) # local BOUNDARY_TEST_FILE_SIZE; BOUNDARY_TEST_FILE_SIZE=$((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) ../../junk_data "${BOUNDARY_TEST_FILE_SIZE}" > "${TEST_TEXT_FILE}" # # Write in First boundary # # Write 0 - 3,145,727(3MB) : less than the multipart minimum size from the beginning dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=0 count=3072 bs=1024 conv=notrunc # Write 0 - 7,340,031(7MB) : multipart exceeding the minimum size from the beginning dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=0 count=7168 bs=1024 conv=notrunc # Write 0 - 12,582,911(12MB) : beyond the multipart size boundary from the beginning dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=0 count=12288 bs=1024 conv=notrunc # # Write in First and second boundary # # Write 3,145,728 - 4,194,303(1MB) : less than the minimum multipart size from the middle of the first multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=3072 count=1024 bs=1024 conv=notrunc # Write 3,145,728 - 9,437,183(6MB) : exceeding the minimum multipart size from the middle of the first multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=3072 count=6144 bs=1024 conv=notrunc # Write 3,145,728 - 12,582,911(9MB) : beyond the multipart boundary from the middle of the first multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=3072 count=9216 bs=1024 conv=notrunc # # Write in Second boundary # # Write 12,582,912 - 14,680,063(2MB) : below the minimum multipart size from the middle of the multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=12288 count=2048 bs=1024 conv=notrunc # Write 12,582,912 - 18,874,367(6MB) : data exceeding the minimum multipart size from the middle of the multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=12288 count=6144 bs=1024 conv=notrunc # Write 12,582,912 - 23,068,671(10MB) : beyond the multipart boundary from the middle of the multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=12288 count=10240 bs=1024 conv=notrunc # Write 12,582,912 - 26,214,399(13MB) : beyond the multipart boundary(last) from the middle of the multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=12288 count=13312 bs=1024 conv=notrunc # # Write in Last boundary # # Write 23,068,672 - 24,117,247(1MB) : below the minimum multipart size from the middle of the final multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=22528 count=1024 bs=1024 conv=notrunc # Write 23,068,672 - 26,214,399(3MB) : beyond the multipart boundary(last) from the middle of the final multipart area dd if=/dev/zero of="${TEST_TEXT_FILE}" seek=22528 count=3072 bs=1024 conv=notrunc rm_test_file } function test_chmod_mountpoint { describe "Testing chmod to mount point..." local MOUNTPOINT_DIR; MOUNTPOINT_DIR=$(cd ..; pwd) local ORIGINAL_PERMISSIONS; ORIGINAL_PERMISSIONS=$(get_permissions "${MOUNTPOINT_DIR}") chmod 0777 "${MOUNTPOINT_DIR}"; # if they're the same, we have a problem. local CHANGED_PERMISSIONS; CHANGED_PERMISSIONS=$(get_permissions "${MOUNTPOINT_DIR}") if [ "${CHANGED_PERMISSIONS}" = "${ORIGINAL_PERMISSIONS}" ] then echo "Could not modify mount point(${MOUNTPOINT_DIR}) permissions" return 1 fi } function test_chown_mountpoint { describe "Testing chown mount point..." local MOUNTPOINT_DIR; MOUNTPOINT_DIR=$(cd ..; pwd) local ORIGINAL_PERMISSIONS ORIGINAL_PERMISSIONS=$(get_user_and_group "${MOUNTPOINT_DIR}") # [NOTE] # Prevents test interruptions due to permission errors, etc. # If the chown command fails, an error will occur with the # following judgment statement. So skip the chown command error. # '|| true' was added due to a problem with MacOS and ensure_diskfree option. # chown 1000:1000 "${MOUNTPOINT_DIR}" || true # if they're the same, we have a problem. local CHANGED_PERMISSIONS CHANGED_PERMISSIONS=$(get_user_and_group "${MOUNTPOINT_DIR}") if [ "${CHANGED_PERMISSIONS}" = "${ORIGINAL_PERMISSIONS}" ] then if [ "${ORIGINAL_PERMISSIONS}" = "1000:1000" ] then echo "Could not be strict check because original file permission 1000:1000" else echo "Could not modify mount point(${MOUNTPOINT_DIR}) ownership($ORIGINAL_PERMISSIONS to 1000:1000)" return 1 fi fi } function test_time_mountpoint { describe "Testing atime/ctime/mtime to mount point..." local MOUNTPOINT_DIR; MOUNTPOINT_DIR=$(cd ..; pwd) local base_atime; base_atime=$(get_atime "${MOUNTPOINT_DIR}") local base_ctime; base_ctime=$(get_ctime "${MOUNTPOINT_DIR}") local base_mtime; base_mtime=$(get_mtime "${MOUNTPOINT_DIR}") touch "${MOUNTPOINT_DIR}" local atime; atime=$(get_atime "${MOUNTPOINT_DIR}") local ctime; ctime=$(get_ctime "${MOUNTPOINT_DIR}") local mtime; mtime=$(get_mtime "${MOUNTPOINT_DIR}") if [ "${base_atime}" = "${atime}" ] || [ "${base_ctime}" = "${ctime}" ] || [ "${base_mtime}" = "${mtime}" ]; then echo "chmod expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime == $mtime, atime: $base_atime == $atime" return 1 fi } function test_file_names_longer_than_posix() { local DIR_NAME; DIR_NAME=$(basename "${PWD}") a256="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" #a256="aaaa" if ! touch "${a256}"; then echo "could not create long file name" return 1 fi rm -f "${a256}" echo data | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${DIR_NAME}/${a256}" files=(*) if [ "${#files[@]}" = 0 ]; then echo "failed to list long file name" return 1 fi rm -f "${a256}" } function test_statvfs() { describe "Testing the free/available size on mount point(statvfs)..." # [NOTE] # The df command result format is different between Linux and macos, # but the order of Total/Used/Available size is the same. # local MOUNTPOINT_DIR; MOUNTPOINT_DIR=$(cd ..; pwd) local DF_RESULT; DF_RESULT=$(df "${MOUNTPOINT_DIR}" 2>/dev/null | tail -n +2) local TOTAL_SIZE; TOTAL_SIZE=$(echo "${DF_RESULT}" | awk '{print $2}') local USED_SIZE; USED_SIZE=$(echo "${DF_RESULT}" | awk '{print $3}') local AVAIL_SIZE; AVAIL_SIZE=$(echo "${DF_RESULT}" | awk '{print $4}') # [NOTE] # In the disk information (statvfs) provided by s3fs, Total size and # Available size are always the same and not 0, and used size is always 0. # if [ -z "${TOTAL_SIZE}" ] || [ -z "${AVAIL_SIZE}" ] || [ -z "${USED_SIZE}" ] || [ "${TOTAL_SIZE}" = "0" ] || [ "${AVAIL_SIZE}" = "0" ] || [ "${TOTAL_SIZE}" != "${AVAIL_SIZE}" ] || [ "${USED_SIZE}" != "0" ]; then echo "The result of df command is wrong: Total=${TOTAL_SIZE}, Used=${USED_SIZE}, Available=${AVAIL_SIZE}" return 1 fi } function add_all_tests { if s3fs_args | grep -q use_cache; then add_tests test_cache_file_stat add_tests test_zero_cache_file_stat else add_tests test_file_names_longer_than_posix fi if ! s3fs_args | grep -q ensure_diskfree && ! uname | grep -q Darwin; then add_tests test_clean_up_cache fi add_tests test_create_empty_file add_tests test_append_file add_tests test_truncate_file add_tests test_truncate_upload add_tests test_truncate_empty_file add_tests test_truncate_shrink_file add_tests test_truncate_shrink_read_file add_tests test_mv_file add_tests test_mv_to_exist_file add_tests test_mv_empty_directory add_tests test_mv_nonempty_directory add_tests test_redirects add_tests test_mkdir_rmdir add_tests test_chmod add_tests test_chown add_tests test_list add_tests test_remove_nonempty_directory add_tests test_external_directory_creation add_tests test_external_modification add_tests test_external_creation add_tests test_read_external_object add_tests test_read_external_dir_object add_tests test_update_metadata_external_small_object add_tests test_update_metadata_external_large_object add_tests test_rename_before_close add_tests test_multipart_upload add_tests test_multipart_copy if ! uname | grep -q Darwin || ! s3fs_args | grep -q nocopyapi; then # FIXME: # If you specify the nocopyapi option with macos-fuse-t, the following error will # occur when manipulating the xattr of the copied object: # "could not copy extended attributes to : Result too large" # As no solution has been found at this time, this test is bypassed on macos with # nocopyapi. # Please pay attention to future developments in macos-fuse-t. # add_tests test_multipart_mix fi add_tests test_utimens_during_multipart add_tests test_special_characters add_tests test_hardlink add_tests test_symlink if ! uname | grep -q Darwin; then add_tests test_mknod fi add_tests test_extended_attributes add_tests test_mtime_file add_tests test_update_time_chmod add_tests test_update_time_chown add_tests test_update_time_xattr add_tests test_update_time_touch if ! mount -t fuse.s3fs | grep "$TEST_BUCKET_MOUNT_POINT_1 " | grep -q -e noatime -e relatime ; then add_tests test_update_time_touch_a fi add_tests test_update_time_append add_tests test_update_time_cp_p add_tests test_update_time_mv add_tests test_update_directory_time_chmod add_tests test_update_directory_time_chown add_tests test_update_directory_time_touch if ! mount -t fuse.s3fs | grep "$TEST_BUCKET_MOUNT_POINT_1 " | grep -q -e noatime -e relatime ; then add_tests test_update_directory_time_touch_a fi if ! uname | grep -q Darwin; then # FIXME: # These test fail in macos-fuse-t because mtime/ctime/atime are not updated. # Currently, these are not an issue with s3fs, so we will bypass this test for macos. # Please pay attention to future developments in macos-fuse-t. # add_tests test_update_directory_time_set_xattr add_tests test_update_directory_time_subdir fi add_tests test_update_chmod_opened_file if s3fs_args | grep -q update_parent_dir_stat; then if ! uname | grep -q Darwin; then # FIXME: # In macos-fuse-t, this test can sometimes succeed if the test waits for more # than one second while it is processing. # However, the results are currently unstable, thus this test is bypassed on macos. # Please pay attention to future developments in macos-fuse-t. # add_tests test_update_parent_directory_time fi fi if ! s3fs_args | grep -q use_xattr; then add_tests test_posix_acl fi add_tests test_rm_rf_dir add_tests test_copy_file add_tests test_write_after_seek_ahead add_tests test_overwrite_existing_file_range add_tests test_concurrent_directory_updates add_tests test_concurrent_reads add_tests test_concurrent_writes add_tests test_open_second_fd add_tests test_write_multiple_offsets add_tests test_write_multiple_offsets_backwards add_tests test_content_type add_tests test_truncate_cache add_tests test_upload_sparsefile add_tests test_mix_upload_entities # TODO: investigate why only Alpine cannot see the implicit directory objects. if ! test -f /etc/os-release || ! grep -q -i -e 'ID=alpine' -e 'ID="alpine"' /etc/os-release; then add_tests test_not_existed_dir_obj fi add_tests test_ut_ossfs add_tests test_cr_filename if ! s3fs_args | grep -q ensure_diskfree && ! uname | grep -q Darwin; then add_tests test_ensurespace_move_file fi add_tests test_write_data_with_skip add_tests test_not_boundary_writes # [NOTE] # The test on CI will fail depending on the permissions, so skip these(chmod/chown). # # add_tests test_chmod_mountpoint # add_tests test_chown_mountpoint add_tests test_time_mountpoint add_tests test_statvfs } init_suite add_all_tests run_suite # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/junk_data.cc000066400000000000000000000030621470675423500165320ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2021 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ // Generate junk data at high speed. An alternative to dd if=/dev/urandom. #include #include #include int main(int argc, const char *argv[]) { if (argc != 2) { return 1; } uint64_t count = strtoull(argv[1], nullptr, 10); char buf[128 * 1024]; for (uint64_t i = 0; i < count; i += sizeof(buf)) { for (uint64_t j = 0; j < sizeof(buf) / sizeof(i); ++j) { *(reinterpret_cast(buf) + j) = i / sizeof(i) + j; } fwrite(buf, 1, sizeof(buf) > count - i ? count - i : sizeof(buf), stdout); } return 0; } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/test/mergedir.sh000077500000000000000000000126001470675423500164160ustar00rootroot00000000000000#!/bin/sh # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # Merge old directory object to new. # For s3fs after v1.64 # ### ### UsageFunction ### UsageFunction() { echo "Usage: $1 [-h] [-y] [-all] " echo " -h print usage" echo " -y no confirm" echo " -all force all directories" echo " There is no -all option is only to merge for other S3 client." echo " If -all is specified, this shell script merge all directory" echo " for s3fs old version." echo "" } ### Check parameters WHOAMI=$(whoami) OWNNAME=$(basename "$0") AUTOYES="no" ALLYES="no" DIRPARAM="" while [ "$1" != "" ]; do if [ "X$1" = "X-help" ] || [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ]; then UsageFunction "${OWNNAME}" exit 0 elif [ "X$1" = "X-y" ] || [ "X$1" = "X-Y" ]; then AUTOYES="yes" elif [ "X$1" = "X-all" ] || [ "X$1" = "X-ALL" ]; then ALLYES="yes" else if [ "X$DIRPARAM" != "X" ]; then echo "*** Input error." echo "" UsageFunction "${OWNNAME}" exit 1 fi DIRPARAM=$1 fi shift done if [ "X$DIRPARAM" = "X" ]; then echo "*** Input error." echo "" UsageFunction "${OWNNAME}" exit 1 fi if [ "$WHOAMI" != "root" ]; then echo "" echo "Warning: You run this script by $WHOAMI, should be root." echo "" fi ### Caution echo "#############################################################################" echo "[CAUTION]" echo "This program merges a directory made in s3fs which is older than version 1.64." echo "And made in other S3 client application." echo "This program may be have bugs which are not fixed yet." echo "Please execute this program by responsibility of your own." echo "#############################################################################" echo "" DATE=$(date +'%Y%m%d-%H%M%S') LOGFILE="${OWNNAME}-${DATE}.log" echo "Start to merge directory object... [${DIRPARAM}]" { echo "# Start to merge directory object... [${DIRPARAM}]" echo "# DATE : $(date)" echo "# BASEDIR : $(pwd)" echo "# TARGET PATH : ${DIRPARAM}" echo "" } > "${LOGFILE}" if [ "$AUTOYES" = "yes" ]; then echo "(no confirmation)" else echo "" fi echo "" ### Get Directory list DIRLIST=$(find "${DIRPARAM}" -type d -print | grep -v ^\.$) # # Main loop # for DIR in $DIRLIST; do ### Skip "." and ".." directories BASENAME=$(basename "${DIR}") if [ "${BASENAME}" = "." ] || [ "${BASENAME}" = ".." ]; then continue fi if [ "${ALLYES}" = "no" ]; then ### Skip "d---------" directories. ### Other clients make directory object "dir/" which don't have ### "x-amz-meta-mode" attribute. ### Then these directories is "d---------", it is target directory. # shellcheck disable=SC2012 DIRPERMIT=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $1}') if [ "${DIRPERMIT}" != "d---------" ]; then continue fi fi ### Confirm ANSWER="" if [ "${AUTOYES}" = "yes" ]; then ANSWER="y" fi while [ "X${ANSWER}" != "XY" ] && [ "X${ANSWER}" != "Xy" ] && [ "X${ANSWER}" != "XN" ] && [ "X${ANSWER}" != "Xn" ]; do printf "%s" "Do you merge ${DIR} ? (y/n): " read -r ANSWER done if [ "X${ANSWER}" != "XY" ] && [ "X${ANSWER}" != "Xy" ]; then continue fi ### Do # shellcheck disable=SC2012 CHOWN=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $3":"$4" "$7}') # shellcheck disable=SC2012 CHMOD=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $7}') # shellcheck disable=SC2012 TOUCH=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $6" "$7}') printf "%s" "*** Merge ${DIR} : " printf "%s" " ${DIR} : " >> "${LOGFILE}" chmod 755 "${CHMOD}" > /dev/null 2>&1 RESULT=$? if [ "${RESULT}" -ne 0 ]; then echo "Failed(chmod)" echo "Failed(chmod)" >> "${LOGFILE}" continue fi chown "${CHOWN}" > /dev/null 2>&1 RESULT=$? if [ "${RESULT}" -ne 0 ]; then echo "Failed(chown)" echo "Failed(chown)" >> "${LOGFILE}" continue fi touch -t "${TOUCH}" > /dev/null 2>&1 RESULT=$? if [ "${RESULT}" -ne 0 ]; then echo "Failed(touch)" echo "Failed(touch)" >> "${LOGFILE}" continue fi echo "Succeed" echo "Succeed" >> "${LOGFILE}" done echo "" echo "" >> "${LOGFILE}" echo "Finished." echo "# Finished : $(date)" >> "${LOGFILE}" # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/mknod_test.cc000066400000000000000000000140771470675423500167510ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2021 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #if !defined(__APPLE__) && !defined(__FreeBSD__) #include #endif //--------------------------------------------------------- // Const //--------------------------------------------------------- static constexpr char usage_string[] = "Usage : \"mknod_test \""; static constexpr char str_mode_reg[] = "REGULAR"; static constexpr char str_mode_chr[] = "CHARACTER"; static constexpr char str_mode_blk[] = "BLOCK"; static constexpr char str_mode_fifo[] = "FIFO"; static constexpr char str_mode_sock[] = "SOCK"; static constexpr char str_ext_reg[] = "reg"; static constexpr char str_ext_chr[] = "chr"; static constexpr char str_ext_blk[] = "blk"; static constexpr char str_ext_fifo[] = "fifo"; static constexpr char str_ext_sock[] = "sock"; // [NOTE] // It would be nice if PATH_MAX could be used as is, but since there are // issues using on Linux and we also must support for macos, this simple // test program defines a fixed value for simplicity. // static constexpr size_t S3FS_TEST_PATH_MAX = 255; static constexpr size_t MAX_BASE_PATH_LENGTH = S3FS_TEST_PATH_MAX - 5; //--------------------------------------------------------- // Test function //--------------------------------------------------------- bool TestMknod(const char* basepath, mode_t mode) { if(!basepath){ fprintf(stderr, "[ERROR] Called function with wrong basepath argument.\n"); return false; } const char* str_mode; dev_t dev; char filepath[S3FS_TEST_PATH_MAX]; switch(mode){ case S_IFREG: str_mode = str_mode_reg; dev = 0; snprintf(filepath, sizeof(filepath), "%s.%s", basepath, str_ext_reg); filepath[S3FS_TEST_PATH_MAX - 1] = '\0'; // for safety break; case S_IFCHR: str_mode = str_mode_chr; dev = makedev(0, 0); snprintf(filepath, sizeof(filepath), "%s.%s", basepath, str_ext_chr); filepath[S3FS_TEST_PATH_MAX - 1] = '\0'; // for safety break; case S_IFBLK: str_mode = str_mode_blk; dev = makedev((unsigned int)(259), 0); // temporary value snprintf(filepath, sizeof(filepath), "%s.%s", basepath, str_ext_blk); filepath[S3FS_TEST_PATH_MAX - 1] = '\0'; // for safety break; case S_IFIFO: str_mode = str_mode_fifo; dev = 0; snprintf(filepath, sizeof(filepath), "%s.%s", basepath, str_ext_fifo); filepath[S3FS_TEST_PATH_MAX - 1] = '\0'; // for safety break; case S_IFSOCK: str_mode = str_mode_sock; dev = 0; snprintf(filepath, sizeof(filepath), "%s.%s", basepath, str_ext_sock); filepath[S3FS_TEST_PATH_MAX - 1] = '\0'; // for safety break; default: fprintf(stderr, "[ERROR] Called function with wrong mode argument.\n"); return false; } // // Create // if(0 != mknod(filepath, mode | S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH, dev)){ fprintf(stderr, "[ERROR] Could not create %s file(%s) : errno = %d\n", str_mode, filepath, errno); return false; } // // Check // struct stat st; if(0 != stat(filepath, &st)){ fprintf(stderr, "[ERROR] Could not get stat from %s file(%s) : errno = %d\n", str_mode, filepath, errno); return false; } if(mode != (st.st_mode & S_IFMT)){ fprintf(stderr, "[ERROR] Created %s file(%s) does not have 0%o stat\n", str_mode, filepath, mode); return false; } // // Remove // if(0 != unlink(filepath)){ fprintf(stderr, "[WARNING] Could not remove %s file(%s) : errno = %d\n", str_mode, filepath, mode); } return true; } //--------------------------------------------------------- // Main //--------------------------------------------------------- int main(int argc, const char *argv[]) { // Parse parameters if(2 != argc){ fprintf(stderr, "[ERROR] No parameter is specified.\n"); fprintf(stderr, "%s\n", usage_string); exit(EXIT_FAILURE); } if(0 == strcmp("-h", argv[1]) || 0 == strcmp("--help", argv[1])){ fprintf(stdout, "%s\n", usage_string); exit(EXIT_SUCCESS); } if(MAX_BASE_PATH_LENGTH < strlen(argv[1])){ fprintf(stderr, "[ERROR] Base file path is too long, it must be less than %zu\n", MAX_BASE_PATH_LENGTH); exit(EXIT_FAILURE); } // Test // // [NOTE] // Privilege is required to execute S_IFBLK. // if(0 != geteuid()){ fprintf(stderr, "[WARNING] Skipping mknod(S_IFBLK) due to missing root privileges.\n"); } if(!TestMknod(argv[1], S_IFREG) || !TestMknod(argv[1], S_IFCHR) || !TestMknod(argv[1], S_IFIFO) || !TestMknod(argv[1], S_IFSOCK) || (0 == geteuid() && !TestMknod(argv[1], S_IFBLK))) { exit(EXIT_FAILURE); } exit(EXIT_SUCCESS); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/test/passwd-s3fs000066400000000000000000000000401470675423500163540ustar00rootroot00000000000000local-identity:local-credential s3fs-fuse-1.95/test/run_tests_using_sanitizers.sh000077500000000000000000000052461470675423500223360ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # set -o errexit set -o nounset set -o pipefail # Disable preprocessor warnings from _FORTIFY_SOURCE and -O0 COMMON_FLAGS="-g -O0 -Wno-cpp" # run tests with libstc++ debug mode, https://gcc.gnu.org/onlinedocs/libstdc++/manual/debug_mode.html make clean ./configure CXXFLAGS="$COMMON_FLAGS -D_GLIBCXX_DEBUG" make --jobs="$(nproc)" ALL_TESTS=1 make check -C test/ # run tests under AddressSanitizer, https://clang.llvm.org/docs/AddressSanitizer.html make clean ./configure CXX=clang++ CXXFLAGS="$COMMON_FLAGS -fsanitize=address -fsanitize-address-use-after-scope" make --jobs="$(nproc)" ALL_TESTS=1 ASAN_OPTIONS='detect_leaks=1,detect_stack_use_after_return=1' make check -C test/ # run tests under MemorySanitizer, https://clang.llvm.org/docs/MemorySanitizer.html # TODO: this requires a custom libc++ #make clean #./configure CXX=clang++ CXXFLAGS="$COMMON_FLAGS -fsanitize=memory" #make --jobs="$(nproc)" #ALL_TESTS=1 make check -C test/ # run tests under ThreadSanitizer, https://clang.llvm.org/docs/ThreadSanitizer.html make clean ./configure CXX=clang++ CXXFLAGS="$COMMON_FLAGS -fsanitize=thread" make --jobs="$(nproc)" ALL_TESTS=1 TSAN_OPTIONS='halt_on_error=1,suppressions=threadsanitizer_suppressions.txt' make check -C test/ # run tests under UndefinedBehaviorSanitizer, https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html make clean ./configure CXX=clang++ CXXFLAGS="$COMMON_FLAGS -fsanitize=undefined,implicit-conversion,local-bounds,unsigned-integer-overflow" make --jobs="$(nproc)" ALL_TESTS=1 make check -C test/ # run tests with Valgrind make clean ./configure CXXFLAGS="$COMMON_FLAGS" make --jobs="$(nproc)" ALL_TESTS=1 RETRIES=100 VALGRIND="--leak-check=full --error-exitcode=1" S3_URL=http://127.0.0.1:8081 make check -C test/ # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/s3proxy.conf000066400000000000000000000005521470675423500165620ustar00rootroot00000000000000s3proxy.endpoint=http://127.0.0.1:8081 s3proxy.secure-endpoint=https://127.0.0.1:8080 s3proxy.authorization=aws-v2-or-v4 s3proxy.identity=local-identity s3proxy.credential=local-credential s3proxy.keystore-path=/tmp/keystore.jks s3proxy.keystore-password=password jclouds.provider=transient jclouds.identity=remote-identity jclouds.credential=remote-credential s3fs-fuse-1.95/test/s3proxy_http.conf000066400000000000000000000003601470675423500176160ustar00rootroot00000000000000s3proxy.endpoint=http://127.0.0.1:8080 s3proxy.authorization=aws-v2-or-v4 s3proxy.identity=local-identity s3proxy.credential=local-credential jclouds.provider=transient jclouds.identity=remote-identity jclouds.credential=remote-credential s3fs-fuse-1.95/test/sample_ahbe.conf000066400000000000000000000036171470675423500174000ustar00rootroot00000000000000# S3FS: Sample ahbe_conf parameter file. # # This file is configuration file for additional header by extension(ahbe). # s3fs loads this file at starting. # # Format: # line = [file suffix or regex] HTTP-header [HTTP-header-values] # file suffix = file(object) suffix, if this field is empty, # it means "reg:(.*)".(=all object). # regex = regular expression to match the file(object) path. # this type starts with "reg:" prefix. # HTTP-header = additional HTTP header name # HTTP-header-values = additional HTTP header value # # # # Verification is done in the order in which they are described in the file. # That order is very important. # # Example: # " Content-Encoding gzip" --> all object # ".gz Content-Encoding gzip" --> only ".gz" extension file # "reg:^/DIR/(.*).t2$ Content-Encoding text2" --> "/DIR/*.t2" extension file # # Notice: # If you need to set all object, you can specify without "suffix" or regex # type "reg:(.*)". Then all of object(file) is added additional header. # If you have this configuration file for Content-Encoding, you should # know about RFC 2616. # # "The default (identity) encoding; the use of no transformation # whatsoever. This content-coding is used only in the Accept- # Encoding header, and SHOULD NOT be used in the Content-Encoding # header." # # file suffix type .gz Content-Encoding gzip .Z Content-Encoding compress .bz2 Content-Encoding bzip2 .svgz Content-Encoding gzip .svg.gz Content-Encoding gzip .tgz Content-Encoding gzip .tar.gz Content-Encoding gzip .taz Content-Encoding gzip .tz Content-Encoding gzip .tbz2 Content-Encoding gzip gz.js Content-Encoding gzip # regex type(test) reg:^/MYDIR/(.*)[.]t2$ Content-Encoding text2 s3fs-fuse-1.95/test/sample_delcache.sh000077500000000000000000000071651470675423500177230ustar00rootroot00000000000000#!/bin/sh # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # This is unsupported sample deleting cache files script. # So s3fs's local cache files(stats and objects) grow up, # you need to delete these. # This script deletes these files with total size limit # by sorted atime of files. # You can modify this script for your system. # # [Usage] script [-silent] # func_usage() { echo "" echo "Usage: $1 [-silent]" echo " $1 -h" echo "Sample: $1 mybucket /tmp/s3fs/cache 1073741824" echo "" echo " bucket name = bucket name which specified s3fs option" echo " cache path = cache directory path which specified by" echo " use_cache s3fs option." echo " limit size = limit for total cache files size." echo " specify by BYTE" echo " -silent = silent mode" echo "" } PRGNAME=$(basename "$0") if [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ]; then func_usage "${PRGNAME}" exit 0 fi if [ "X$1" = "X" ] || [ "X$2" = "X" ] || [ "X$3" = "X" ]; then func_usage "${PRGNAME}" exit 1 fi BUCKET="$1" CDIR="$2" LIMIT="$3" SILENT=0 if [ "X$4" = "X-silent" ]; then SILENT=1 fi FILES_CDIR="${CDIR}/${BUCKET}" STATS_CDIR="${CDIR}/.${BUCKET}.stat" CURRENT_CACHE_SIZE=$(du -sb "${FILES_CDIR}" | awk '{print $1}') # # Check total size # if [ "${LIMIT}" -ge "${CURRENT_CACHE_SIZE}" ]; then if [ $SILENT -ne 1 ]; then echo "${FILES_CDIR} (${CURRENT_CACHE_SIZE}) is below allowed ${LIMIT}" fi exit 0 fi # # Remove loop # TMP_ATIME=0 TMP_STATS="" TMP_CFILE="" # # Make file list by sorted access time # find "${STATS_CDIR}" -type f -exec stat -c "%X:%n" "{}" \; | sort | while read -r part do echo "Looking at ${part}" TMP_ATIME=$(echo "${part}" | cut -d: -f1) TMP_STATS=$(echo "${part}" | cut -d: -f2-) TMP_CFILE=$(echo "${TMP_STATS}" | sed -e "s/\\.${BUCKET}\\.stat/${BUCKET}/") if [ "$(stat -c %X "${TMP_STATS}")" -eq "${TMP_ATIME}" ]; then if ! rm "${TMP_STATS}" "${TMP_CFILE}" > /dev/null 2>&1; then if [ "${SILENT}" -ne 1 ]; then echo "ERROR: Could not remove files(${TMP_STATS},${TMP_CFILE})" fi exit 1 else if [ "${SILENT}" -ne 1 ]; then echo "remove file: ${TMP_CFILE} ${TMP_STATS}" fi fi fi if [ "${LIMIT}" -ge "$(du -sb "${FILES_CDIR}" | awk '{print $1}')" ]; then if [ "${SILENT}" -ne 1 ]; then echo "finish removing files" fi break fi done if [ "${SILENT}" -ne 1 ]; then TOTAL_SIZE=$(du -sb "${FILES_CDIR}" | awk '{print $1}') echo "Finish: ${FILES_CDIR} total size is ${TOTAL_SIZE}" fi exit 0 # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/small-integration-test.sh000077500000000000000000000051761470675423500212400ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # Test s3fs-fuse file system operations with # set -o errexit set -o pipefail source integration-test-common.sh CACHE_DIR="/tmp/s3fs-cache" rm -rf "${CACHE_DIR}" mkdir "${CACHE_DIR}" source test-utils.sh #reserve 200MB for data cache FAKE_FREE_DISK_SIZE=200 ENSURE_DISKFREE_SIZE=10 # set up client-side encryption keys head -c 32 < /dev/urandom > /tmp/ssekey.bin base64 < /tmp/ssekey.bin > /tmp/ssekey openssl md5 -binary < /tmp/ssekey.bin | base64 > /tmp/ssekeymd5 chmod 600 /tmp/ssekey /tmp/ssekey.bin /tmp/ssekeymd5 export CACHE_DIR export ENSURE_DISKFREE_SIZE if [ -n "${ALL_TESTS}" ]; then FLAGS=( "use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE} -o use_xattr -o update_parent_dir_stat" enable_content_md5 disable_noobj_cache "max_stat_cache_size=100" nocopyapi nomultipart sigv2 sigv4 "singlepart_copy_limit=10" # limit size to exercise multipart code paths #use_sse # TODO: S3Proxy does not support SSE #use_sse=custom:/tmp/ssekey # TODO: S3Proxy does not support SSE "use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE} -o streamupload" ) else FLAGS=( sigv4 ) fi start_s3proxy if ! aws_cli s3api head-bucket --bucket "${TEST_BUCKET_1}" --region "${S3_ENDPOINT}"; then aws_cli s3 mb "s3://${TEST_BUCKET_1}" --region "${S3_ENDPOINT}" fi for flag in "${FLAGS[@]}"; do echo "testing s3fs flag: ${flag}" # shellcheck disable=SC2086 start_s3fs -o ${flag} ./integration-test-main.sh stop_s3fs done stop_s3proxy echo "$0: tests complete." # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/test-utils.sh000066400000000000000000000261171470675423500167420ustar00rootroot00000000000000#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # #### Test utils set -o errexit set -o pipefail # # Configuration # TEST_TEXT="HELLO WORLD" TEST_TEXT_FILE=test-s3fs.txt TEST_DIR=testdir # shellcheck disable=SC2034 ALT_TEST_TEXT_FILE=test-s3fs-ALT.txt # shellcheck disable=SC2034 TEST_TEXT_FILE_LENGTH=15 # shellcheck disable=SC2034 BIG_FILE=big-file-s3fs.txt # shellcheck disable=SC2034 TEMP_DIR="${TMPDIR:-"/var/tmp"}" # /dev/urandom can only return 32 MB per block maximum BIG_FILE_BLOCK_SIZE=$((25 * 1024 * 1024)) BIG_FILE_COUNT=1 # This should be greater than the multipart size # shellcheck disable=SC2034 BIG_FILE_LENGTH=$((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT)) # Set locale because some tests check for English expressions export LC_ALL=en_US.UTF-8 export RUN_DIR # [NOTE] # stdbuf, truncate and sed installed on macos do not work as # expected(not compatible with Linux). # Therefore, macos installs a brew package such as coreutils # and uses gnu commands(gstdbuf, gtruncate, gsed). # Set your PATH appropriately so that you can find these commands. # if [ "$(uname)" = "Darwin" ]; then export STDBUF_BIN="gstdbuf" export TRUNCATE_BIN="gtruncate" export SED_BIN="gsed" export BASE64_BIN="gbase64" else export STDBUF_BIN="stdbuf" export TRUNCATE_BIN="truncate" export SED_BIN="sed" export BASE64_BIN="base64" fi export SED_BUFFER_FLAG="--unbuffered" # [NOTE] # Specifying cache disable option depending on stat(coreutils) version # TODO: investigate why this is necessary #2327 # if stat --cached=never / >/dev/null 2>&1; then STAT_BIN=(stat --cache=never) else STAT_BIN=(stat) fi function get_xattr() { if [ "$(uname)" = "Darwin" ]; then xattr -p "$1" "$2" else getfattr -n "$1" --only-values "$2" fi } function set_xattr() { if [ "$(uname)" = "Darwin" ]; then xattr -w "$1" "$2" "$3" else setfattr -n "$1" -v "$2" "$3" fi } function del_xattr() { if [ "$(uname)" = "Darwin" ]; then xattr -d "$1" "$2" else setfattr -x "$1" "$2" fi } function get_inode() { if [ "$(uname)" = "Darwin" ]; then "${STAT_BIN[@]}" -f "%i" "$1" else "${STAT_BIN[@]}" --format "%i" "$1" fi } function get_size() { if [ "$(uname)" = "Darwin" ]; then "${STAT_BIN[@]}" -f "%z" "$1" else "${STAT_BIN[@]}" --format "%s" "$1" fi } function check_file_size() { local FILE_NAME="$1" local EXPECTED_SIZE="$2" # Verify file length via metadata local size size=$(get_size "${FILE_NAME}") if [ "${size}" -ne "${EXPECTED_SIZE}" ] then echo "error: expected ${FILE_NAME} to be ${EXPECTED_SIZE} length but was ${size} via metadata" return 1 fi # Verify file length via data # shellcheck disable=SC2002 size=$(cat "${FILE_NAME}" | wc -c) if [ "${size}" -ne "${EXPECTED_SIZE}" ] then echo "error: expected ${FILE_NAME} to be ${EXPECTED_SIZE} length, got ${size} via data" return 1 fi } function mk_test_file { if [ $# = 0 ]; then local TEXT="${TEST_TEXT}" else local TEXT="$1" fi echo "${TEXT}" > "${TEST_TEXT_FILE}" if [ ! -e "${TEST_TEXT_FILE}" ] then echo "Could not create file ${TEST_TEXT_FILE}, it does not exist" exit 1 fi } function rm_test_file { if [ $# = 0 ]; then local FILE="${TEST_TEXT_FILE}" else local FILE="$1" fi rm -f "${FILE}" if [ -e "${FILE}" ] then echo "Could not cleanup file ${TEST_TEXT_FILE}" exit 1 fi } function mk_test_dir { mkdir "${TEST_DIR}" if [ ! -d "${TEST_DIR}" ]; then echo "Directory ${TEST_DIR} was not created" exit 1 fi } function rm_test_dir { rmdir "${TEST_DIR}" if [ -e "${TEST_DIR}" ]; then echo "Could not remove the test directory, it still exists: ${TEST_DIR}" exit 1 fi } # Create and cd to a unique directory for this test run # Sets RUN_DIR to the name of the created directory function cd_run_dir { if [ "${TEST_BUCKET_MOUNT_POINT_1}" = "" ]; then echo "TEST_BUCKET_MOUNT_POINT_1 variable not set" exit 1 fi local RUN_DIR="${TEST_BUCKET_MOUNT_POINT_1}/${1}" mkdir -p "${RUN_DIR}" cd "${RUN_DIR}" } function clean_run_dir { if [ -d "${RUN_DIR}" ]; then rm -rf "${RUN_DIR}" || echo "Error removing ${RUN_DIR}" fi } # Resets test suite function init_suite { TEST_LIST=() TEST_FAILED_LIST=() TEST_PASSED_LIST=() } # Report a passing test case # report_pass TEST_NAME function report_pass { echo "$1 passed" TEST_PASSED_LIST+=("$1") } # Report a failing test case # report_fail TEST_NAME function report_fail { echo "$1 failed" TEST_FAILED_LIST+=("$1") } # Add tests to the suite # add_tests TEST_NAME... function add_tests { TEST_LIST+=("$@") } # Log test name and description # describe [DESCRIPTION] function describe { echo "${FUNCNAME[1]}: \"$*\"" } # Runs each test in a suite and summarizes results. The list of # tests added by add_tests() is called with CWD set to a tmp # directory in the bucket. An attempt to clean this directory is # made after the test run. function run_suite { orig_dir="${PWD}" key_prefix="testrun-${RANDOM}" cd_run_dir "${key_prefix}" for t in "${TEST_LIST[@]}"; do # Ensure test input name differs every iteration TEST_TEXT_FILE="test-s3fs-${RANDOM}.txt" TEST_DIR="testdir-${RANDOM}" # shellcheck disable=SC2034 ALT_TEST_TEXT_FILE="test-s3fs-ALT-${RANDOM}.txt" # shellcheck disable=SC2034 BIG_FILE="big-file-s3fs-${RANDOM}.txt" # The following sequence runs tests in a subshell to allow continuation # on test failure, but still allowing errexit to be in effect during # the test. # # See: # https://groups.google.com/d/msg/gnu.bash.bug/NCK_0GmIv2M/dkeZ9MFhPOIJ # Other ways of trying to capture the return value will also disable # errexit in the function due to bash... compliance with POSIX? set +o errexit (set -o errexit; $t $key_prefix) # shellcheck disable=SC2181 if [ $? == 0 ]; then report_pass "${t}" else report_fail "${t}" fi set -o errexit done cd "${orig_dir}" clean_run_dir for t in "${TEST_PASSED_LIST[@]}"; do echo "PASS: ${t}" done for t in "${TEST_FAILED_LIST[@]}"; do echo "FAIL: ${t}" done local passed=${#TEST_PASSED_LIST[@]} local failed=${#TEST_FAILED_LIST[@]} echo "SUMMARY for $0: ${passed} tests passed. ${failed} tests failed." if [[ "${failed}" != 0 ]]; then return 1 else return 0 fi } function get_ctime() { # ex: "1657504903.019784214" if [ "$(uname)" = "Darwin" ]; then "${STAT_BIN[@]}" -f "%Fc" "$1" else "${STAT_BIN[@]}" --format "%.9Z" "$1" fi } function get_mtime() { # ex: "1657504903.019784214" if [ "$(uname)" = "Darwin" ]; then "${STAT_BIN[@]}" -f "%Fm" "$1" else "${STAT_BIN[@]}" --format "%.9Y" "$1" fi } function get_atime() { # ex: "1657504903.019784214" if [ "$(uname)" = "Darwin" ]; then "${STAT_BIN[@]}" -f "%Fa" "$1" else "${STAT_BIN[@]}" --format "%.9X" "$1" fi } function get_permissions() { if [ "$(uname)" = "Darwin" ]; then "${STAT_BIN[@]}" -f "%p" "$1" else "${STAT_BIN[@]}" --format "%a" "$1" fi } function get_user_and_group() { if [ "$(uname)" = "Darwin" ]; then stat -f "%u:%g" "$1" else "${STAT_BIN[@]}" --format "%u:%g" "$1" fi } function check_content_type() { local INFO_STR INFO_STR=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "$1" | jq -r .ContentType) if [ "${INFO_STR}" != "$2" ] then echo "Expected Content-Type: $2 but got: ${INFO_STR}" return 1 fi } function get_disk_avail_size() { local DISK_AVAIL_SIZE DISK_AVAIL_SIZE=$(BLOCKSIZE=$((1024 * 1024)) df "$1" | awk '{print $4}' | tail -n 1) echo "${DISK_AVAIL_SIZE}" } function aws_cli() { local FLAGS="" if [ -n "${S3FS_PROFILE}" ]; then FLAGS="--profile ${S3FS_PROFILE}" fi if [ "$1" = "s3" ] && [ "$2" != "ls" ] && [ "$2" != "mb" ]; then if s3fs_args | grep -q use_sse=custom; then FLAGS="${FLAGS} --sse-c AES256 --sse-c-key fileb:///tmp/ssekey.bin" fi elif [ "$1" = "s3api" ] && [ "$2" != "head-bucket" ]; then if s3fs_args | grep -q use_sse=custom; then FLAGS="${FLAGS} --sse-customer-algorithm AES256 --sse-customer-key $(cat /tmp/ssekey) --sse-customer-key-md5 $(cat /tmp/ssekeymd5)" fi fi # [NOTE] # AWS_EC2_METADATA_DISABLED for preventing the metadata service(to 169.254.169.254). # shellcheck disable=SC2086,SC2068 AWS_EC2_METADATA_DISABLED=true aws $@ --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem ${FLAGS} } function wait_for_port() { local PORT="$1" for _ in $(seq 30); do if exec 3<>"/dev/tcp/127.0.0.1/${PORT}"; then exec 3<&- # Close for read exec 3>&- # Close for write break fi sleep 1 done } function make_random_string() { if [ -n "$1" ]; then local END_POS="$1" else local END_POS=8 fi if [ "$(uname)" = "Darwin" ]; then local BASE64_OPT="--break=0" else local BASE64_OPT="--wrap=0" fi "${BASE64_BIN}" "${BASE64_OPT}" < /dev/urandom 2>/dev/null | tr -d /+ | head -c "${END_POS}" return 0 } function s3fs_args() { if [ "$(uname)" = "Darwin" ]; then ps -o args -p "${S3FS_PID}" | tail -n +2 else ps -o args -p "${S3FS_PID}" --no-headers fi } # # $1: sleep seconds # $2: OS type(ex. 'Darwin', unset(means all os type)) # # [NOTE] macos fuse-t # macos fuse-t mounts over NFS, and the mtime/ctime/atime attribute # values are in seconds(not m/u/n-sec). # Therefore, unlike tests on other OSs, we have to wait at least 1 # second. # This function is called primarily for this purpose. # function wait_ostype() { if [ -z "$2" ] || uname | grep -q "$2"; then if [ -n "$1" ] && ! (echo "$1" | grep -q '[^0-9]'); then sleep "$1" fi fi } # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/threadsanitizer_suppressions.txt000066400000000000000000000000251470675423500230550ustar00rootroot00000000000000race:OPENSSL_sk_free s3fs-fuse-1.95/test/truncate_read_file.cc000066400000000000000000000047761470675423500204260ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2021 Andrew Gaul * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include // [NOTE] // This is a program used for file size inspection. // File size checking should be done by the caller of this program. // This program truncates the file and reads the file in another process // between truncate and flush(close file). // int main(int argc, const char *argv[]) { if(argc != 3){ fprintf(stderr, "[ERROR] Wrong paraemters\n"); fprintf(stdout, "[Usage] truncate_read_file \n"); exit(EXIT_FAILURE); } const char* filepath = argv[1]; auto size = static_cast(strtoull(argv[2], nullptr, 10)); int fd; // open file if(-1 == (fd = open(filepath, O_RDWR))){ fprintf(stderr, "[ERROR] Could not open file(%s)\n", filepath); exit(EXIT_FAILURE); } // truncate if(0 != ftruncate(fd, size)){ fprintf(stderr, "[ERROR] Could not truncate file(%s) to %lld byte.\n", filepath, (long long)size); close(fd); exit(EXIT_FAILURE); } // run sub-process for reading file(cat) char szCommand[1024]; snprintf(szCommand, sizeof(szCommand), "cat %s >/dev/null 2>&1", filepath); szCommand[sizeof(szCommand) - 1] = '\0'; // for safety if(0 != system(szCommand)){ fprintf(stderr, "[ERROR] Failed to run sub-process(cat).\n"); close(fd); exit(EXIT_FAILURE); } // close file(flush) close(fd); exit(EXIT_SUCCESS); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */ s3fs-fuse-1.95/test/ut_test.py000077500000000000000000000065431470675423500163360ustar00rootroot00000000000000#!/usr/bin/env python3 # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # import os import unittest import random import sys import time class OssfsUnitTest(unittest.TestCase): def setUp(self): pass def tearDown(self): pass def random_string(self, len): char_set = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g'] list = [] for i in range(0, len): list.append(random.choice(char_set)) return "".join(list) def test_read_file(self): filename = "%s" % (self.random_string(10)) print(filename) f = open(filename, 'w') data = self.random_string(1000) f.write(data) f.close() f = open(filename, 'r') data = f.read(100) self.assertEqual(len(data), 100) data = f.read(100) self.assertEqual(len(data), 100) f.close() os.remove(filename) def test_rename_file(self): filename1 = "%s" % (self.random_string(10)) filename2 = "%s" % (self.random_string(10)) print(filename1, filename2) f = open(filename1, 'w+') data1 = self.random_string(1000) f.write(data1) os.rename(filename1, filename2) f.seek(0, 0) data2 = f.read() f.close() self.assertEqual(len(data1), len(data2)) self.assertEqual(data1, data2) os.remove(filename2) def test_rename_file2(self): filename1 = "%s" % (self.random_string(10)) filename2 = "%s" % (self.random_string(10)) print(filename1, filename2) f = open(filename1, 'w') data1 = self.random_string(1000) f.write(data1) f.close() os.rename(filename1, filename2) f = open(filename2, 'r') f.seek(0, 0) data2 = f.read() f.close() self.assertEqual(len(data1), len(data2)) self.assertEqual(data1, data2) os.remove(filename2) def test_truncate_open_file(self): filename = "%s" % (self.random_string(10)) fd = os.open(filename, os.O_CREAT|os.O_RDWR) try: os.write(fd, b'a' * 42) self.assertEqual(os.fstat(fd).st_size, 42) os.ftruncate(fd, 100) self.assertEqual(os.fstat(fd).st_size, 100) finally: os.close(fd) self.assertEqual(100, os.stat(filename).st_size) os.remove(filename) if __name__ == '__main__': unittest.main() # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 # s3fs-fuse-1.95/test/write_multiblock.cc000066400000000000000000000175451470675423500201640ustar00rootroot00000000000000/* * s3fs - FUSE-based file system backed by Amazon S3 * * Copyright(C) 2007 Randy Rizun * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version 2 * of the License, or (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include //--------------------------------------------------------- // Structures and Typedefs //--------------------------------------------------------- struct write_block_part { off_t start; off_t size; }; typedef std::vector wbpart_list_t; typedef std::list strlist_t; //--------------------------------------------------------- // Const //--------------------------------------------------------- static constexpr char usage_string[] = "Usage : \"write_multiblock -f -p \" (allows -f and -p multiple times.)"; //--------------------------------------------------------- // Utility functions //--------------------------------------------------------- static std::unique_ptr create_random_data(off_t size) { int fd; if(-1 == (fd = open("/dev/urandom", O_RDONLY))){ std::cerr << "[ERROR] Could not open /dev/urandom" << std::endl; return nullptr; } std::unique_ptr pbuff(new unsigned char[size]); for(ssize_t readpos = 0, readcnt = 0; readpos < size; readpos += readcnt){ if(-1 == (readcnt = read(fd, &(pbuff[readpos]), static_cast(size - readpos)))){ if(EAGAIN != errno && EWOULDBLOCK != errno && EINTR != errno){ std::cerr << "[ERROR] Failed reading from /dev/urandom with errno: " << errno << std::endl; close(fd); return nullptr; } readcnt = 0; } } close(fd); return pbuff; } static off_t cvt_string_to_number(const char* pstr) { if(!pstr){ return -1; } errno = 0; char* ptemp = nullptr; long long result = strtoll(pstr, &ptemp, 10); if(!ptemp || ptemp == pstr || *ptemp != '\0'){ return -1; } if((result == LLONG_MIN || result == LLONG_MAX) && errno == ERANGE){ return -1; } return static_cast(result); } static bool parse_string(const char* pstr, char delim, strlist_t& strlist) { if(!pstr){ return false; } std::string strAll(pstr); while(!strAll.empty()){ size_t pos = strAll.find_first_of(delim); if(std::string::npos != pos){ strlist.push_back(strAll.substr(0, pos)); strAll = strAll.substr(pos + 1); }else{ strlist.push_back(strAll); strAll.clear(); } } return true; } static bool parse_write_blocks(const char* pstr, wbpart_list_t& wbparts, off_t& max_size) { if(!pstr){ return false; } strlist_t partlist; if(!parse_string(pstr, ',', partlist)){ return false; } for(auto iter = partlist.cbegin(); iter != partlist.cend(); ++iter){ strlist_t partpair; if(parse_string(iter->c_str(), ':', partpair) && 2 == partpair.size()){ write_block_part tmp_part; tmp_part.start = cvt_string_to_number(partpair.front().c_str()); partpair.pop_front(); tmp_part.size = cvt_string_to_number(partpair.front().c_str()); if(tmp_part.start < 0 || tmp_part.size <= 0){ std::cerr << "[ERROR] -p option parameter(" << pstr << ") is something wrong." << std::endl; return false; } max_size = std::max(max_size, tmp_part.size); wbparts.push_back(tmp_part); }else{ std::cerr << "[ERROR] -p option parameter(" << pstr << ") is something wrong." << std::endl; return false; } } return true; } static bool parse_arguments(int argc, char** argv, strlist_t& files, wbpart_list_t& wbparts, off_t& max_size) { if(argc < 2 || !argv){ std::cerr << "[ERROR] The -f option and -p option are required as arguments." << std::endl; std::cerr << usage_string << std::endl; return false; } files.clear(); wbparts.clear(); max_size = 0; int opt; while(-1 != (opt = getopt(argc, argv, "f:p:"))){ switch(opt){ case 'f': files.emplace_back(optarg); break; case 'p': if(!parse_write_blocks(optarg, wbparts, max_size)){ return false; } break; default: std::cerr << usage_string << std::endl; return false; } } if(files.empty() || wbparts.empty()){ std::cerr << "[ERROR] The -f option and -p option are required as arguments." << std::endl; std::cerr << usage_string << std::endl; return false; } return true; } //--------------------------------------------------------- // Main //--------------------------------------------------------- int main(int argc, char** argv) { // parse arguments strlist_t files; wbpart_list_t wbparts; off_t max_size = 0; if(!parse_arguments(argc, argv, files, wbparts, max_size)){ exit(EXIT_FAILURE); } // make data and buffer std::unique_ptr pData = create_random_data(max_size); for(auto fiter = files.cbegin(); fiter != files.cend(); ++fiter){ // open/create file int fd; struct stat st; if(0 == stat(fiter->c_str(), &st)){ if(!S_ISREG(st.st_mode)){ std::cerr << "[ERROR] File " << *fiter << " is existed, but it is not regular file." << std::endl; exit(EXIT_FAILURE); } if(-1 == (fd = open(fiter->c_str(), O_WRONLY))){ std::cerr << "[ERROR] Could not open " << *fiter << std::endl; exit(EXIT_FAILURE); } }else{ if(-1 == (fd = open(fiter->c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644))){ std::cerr << "[ERROR] Could not create " << *fiter << std::endl; exit(EXIT_FAILURE); } } // write blocks for(auto piter = wbparts.cbegin(); piter != wbparts.cend(); ++piter){ // write one block for(ssize_t writepos = 0, writecnt = 0; writepos < piter->size; writepos += writecnt){ if(-1 == (writecnt = pwrite(fd, &(pData[writepos]), static_cast(piter->size - writepos), (piter->start + writepos)))){ if(EAGAIN != errno && EWOULDBLOCK != errno && EINTR != errno){ std::cerr << "[ERROR] Failed writing to " << *fiter << " by errno : " << errno << std::endl; close(fd); exit(EXIT_FAILURE); } writecnt = 0; } } } // close file close(fd); } exit(EXIT_SUCCESS); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: expandtab sw=4 ts=4 fdm=marker * vim<600: expandtab sw=4 ts=4 */