pax_global_header00006660000000000000000000000064147665431030014523gustar00rootroot0000000000000052 comment=555054e5096fdbaab608601671373bf2e628d4ea lgogdownloader-3.17/000077500000000000000000000000001476654310300144645ustar00rootroot00000000000000lgogdownloader-3.17/.github/000077500000000000000000000000001476654310300160245ustar00rootroot00000000000000lgogdownloader-3.17/.github/workflows/000077500000000000000000000000001476654310300200615ustar00rootroot00000000000000lgogdownloader-3.17/.github/workflows/linux.yml000066400000000000000000000024031476654310300217420ustar00rootroot00000000000000name: linux on: push: branches: - "*" paths-ignore: - "**.md" pull_request: branches: - "*" workflow_dispatch: jobs: build: name: ${{matrix.cxx_compiler}} USE_QT_GUI=${{matrix.qt_gui}} runs-on: ubuntu-latest strategy: fail-fast: false matrix: cxx_compiler: [g++, clang++] qt_gui: [YES, NO] steps: - name: "Checkout Code" uses: actions/checkout@v4 with: submodules: "recursive" fetch-depth: 0 - name: Dependencies run: | sudo apt -y update sudo apt -y install ninja-build build-essential libcurl4-openssl-dev libboost-regex-dev \ libjsoncpp-dev librhash-dev libtinyxml2-dev libtidy-dev \ libboost-system-dev libboost-filesystem-dev libboost-program-options-dev \ libboost-date-time-dev libboost-iostreams-dev cmake \ pkg-config zlib1g-dev qtwebengine5-dev - name: Configure env: CXX: ${{matrix.cxx_compiler}} run: | cmake -Bbuild -DCMAKE_BUILD_TYPE=Release \ -DUSE_QT_GUI=${{matrix.qt_gui}} -GNinja - name: Build run: ninja -Cbuild - name: Run run: cd build && ./lgogdownloader --help lgogdownloader-3.17/.gitignore000066400000000000000000000001511476654310300164510ustar00rootroot00000000000000*.layout *~ *.[oa] bin/* obj/* *.gz Makefile CMakeCache.txt CMakeFiles/ cmake_install.cmake build/ *.cbp lgogdownloader-3.17/CMakeLists.txt000066400000000000000000000117661476654310300172370ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.18.0 FATAL_ERROR) project (lgogdownloader LANGUAGES C CXX VERSION 3.17) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") option(USE_QT_GUI "Build with Qt GUI login support" OFF) if(USE_QT_GUI) add_definitions(-DUSE_QT_GUI_LOGIN=1) set(CMAKE_AUTOMOC ON) set(CMAKE_AUTOUIC ON) endif(USE_QT_GUI) find_package(Boost CONFIG REQUIRED system filesystem regex program_options date_time iostreams ) find_package(CURL 7.55.0 REQUIRED) find_package(Jsoncpp REQUIRED) find_package(Tinyxml2 REQUIRED) find_package(Rhash REQUIRED) find_package(Threads REQUIRED) find_package(ZLIB REQUIRED) find_package(Tidy REQUIRED) file(GLOB SRC_FILES main.cpp src/website.cpp src/downloader.cpp src/progressbar.cpp src/util.cpp src/blacklist.cpp src/gamefile.cpp src/gamedetails.cpp src/galaxyapi.cpp src/ziputil.cpp ) if(USE_QT_GUI) set(QT Qt6) find_package(Qt6 COMPONENTS Widgets WebEngineWidgets CONFIG) if(NOT Qt6_FOUND) set(QT Qt5) find_package(Qt5 REQUIRED COMPONENTS Widgets WebEngineWidgets CONFIG) endif() file(GLOB QT_GUI_SRC_FILES src/gui_login.cpp ) list(APPEND SRC_FILES ${QT_GUI_SRC_FILES}) endif(USE_QT_GUI) set(GIT_CHECKOUT FALSE) if(EXISTS ${PROJECT_SOURCE_DIR}/.git) if(NOT EXISTS ${PROJECT_SOURCE_DIR}/.git/shallow) find_package(Git) if(GIT_FOUND) set(GIT_CHECKOUT TRUE) else(GIT_FOUND) message(WARNING "Git executable not found") endif(GIT_FOUND) else(NOT EXISTS ${PROJECT_SOURCE_DIR}/.git/shallow) message(STATUS "Shallow Git clone detected, not attempting to retrieve version info") endif(NOT EXISTS ${PROJECT_SOURCE_DIR}/.git/shallow) endif(EXISTS ${PROJECT_SOURCE_DIR}/.git) if(GIT_CHECKOUT) execute_process(COMMAND ${GIT_EXECUTABLE} diff --shortstat WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_SHORTSTAT OUTPUT_STRIP_TRAILING_WHITESPACE ) execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_REV_PARSE OUTPUT_STRIP_TRAILING_WHITESPACE ) if(GIT_SHORTSTAT) set(GIT_DIRTY ON) endif(GIT_SHORTSTAT) if(GIT_DIRTY) set(PROJECT_VERSION_MINOR ${PROJECT_VERSION_MINOR}M) endif(GIT_DIRTY) set(PROJECT_VERSION_PATCH ${GIT_REV_PARSE}) set(PROJECT_VERSION ${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}) endif(GIT_CHECKOUT) set(VERSION_NUMBER ${PROJECT_VERSION}) set(VERSION_STRING "LGOGDownloader ${VERSION_NUMBER}") set(DEFAULT_USER_AGENT "LGOGDownloader/${VERSION_NUMBER} (${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR})") add_definitions(-D_FILE_OFFSET_BITS=64 -DVERSION_NUMBER="${VERSION_NUMBER}" -DVERSION_STRING="${VERSION_STRING}" -DDEFAULT_USER_AGENT="${DEFAULT_USER_AGENT}") add_executable (${PROJECT_NAME} ${SRC_FILES}) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include PRIVATE ${Boost_INCLUDE_DIRS} PRIVATE ${CURL_INCLUDE_DIRS} PRIVATE ${OAuth_INCLUDE_DIRS} PRIVATE ${Jsoncpp_INCLUDE_DIRS} PRIVATE ${Tinyxml2_INCLUDE_DIRS} PRIVATE ${Rhash_INCLUDE_DIRS} PRIVATE ${ZLIB_INCLUDE_DIRS} PRIVATE ${Tidy_INCLUDE_DIRS} ) target_link_libraries(${PROJECT_NAME} PRIVATE ${Boost_LIBRARIES} PRIVATE ${CURL_LIBRARIES} PRIVATE ${OAuth_LIBRARIES} PRIVATE ${Jsoncpp_LIBRARIES} PRIVATE ${Tinyxml2_LIBRARIES} PRIVATE ${Rhash_LIBRARIES} PRIVATE ${CMAKE_THREAD_LIBS_INIT} PRIVATE ${ZLIB_LIBRARIES} PRIVATE ${Tidy_LIBRARIES} ) # Check if libatomic is needed in order to use std::atomic, and add # it to the list of JavaScriptCore libraries. file(WRITE ${CMAKE_BINARY_DIR}/test_atomic.cpp "#include \n" "int main() { std::atomic i(0); i++; return 0; }\n") try_compile(ATOMIC_BUILD_SUCCEEDED ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}/test_atomic.cpp) if (NOT ATOMIC_BUILD_SUCCEEDED) target_link_libraries(${PROJECT_NAME} PRIVATE -latomic ) endif () file(REMOVE ${CMAKE_BINARY_DIR}/test_atomic.cpp) if(USE_QT_GUI) target_link_libraries(${PROJECT_NAME} PRIVATE ${QT}::Widgets PRIVATE ${QT}::WebEngineWidgets ) endif(USE_QT_GUI) if(Qt6_FOUND) set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17) else() set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 11) endif(Qt6_FOUND) if(MSVC) # Force to always compile with W4 if(CMAKE_CXX_FLAGS MATCHES "/W[0-4]") string(REGEX REPLACE "/W[0-4]" "/W4" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4") endif() elseif(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") # Update if necessary set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-long-long -fexceptions") endif() set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables") set(INSTALL_SHARE_DIR share CACHE PATH "Installation directory for resource files") install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}${CMAKE_EXECUTABLE_SUFFIX} DESTINATION ${INSTALL_BIN_DIR}) add_subdirectory(man) lgogdownloader-3.17/COPYING000066400000000000000000000007441476654310300155240ustar00rootroot00000000000000 DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE Version 2, December 2004 Copyright (C) 2004 Sam Hocevar Everyone is permitted to copy and distribute verbatim or modified copies of this license document, and changing it is allowed as long as the name is changed. DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. You just DO WHAT THE FUCK YOU WANT TO. lgogdownloader-3.17/README.md000066400000000000000000000062611476654310300157500ustar00rootroot00000000000000# LGOGDownloader This repository contains the code of LGOGDownloader which is unofficial open source downloader for [GOG.com](https://www.gog.com/). It uses the same API as GOG Galaxy which doesn't have Linux support at the moment. ## Dependencies * [libcurl](https://curl.haxx.se/libcurl/) >= 7.55.0 * [librhash](https://github.com/rhash/RHash) * [jsoncpp](https://github.com/open-source-parsers/jsoncpp) * [libtidy](https://www.html-tidy.org/) * [tinyxml2](https://github.com/leethomason/tinyxml2) * [boost](http://www.boost.org/) (regex, date-time, system, filesystem, program-options, iostreams) * [zlib](https://www.zlib.net/) * [qtwebengine](https://www.qt.io/) if built with -DUSE_QT_GUI=ON ## Make dependencies * [cmake](https://cmake.org/) >= 3.18.0 * [ninja](https://github.com/ninja-build/ninja) ## Debian/Ubuntu # apt install build-essential libcurl4-openssl-dev libboost-regex-dev \ libjsoncpp-dev librhash-dev libtinyxml2-dev libtidy-dev \ libboost-system-dev libboost-filesystem-dev libboost-program-options-dev \ libboost-date-time-dev libboost-iostreams-dev cmake \ pkg-config zlib1g-dev qtwebengine5-dev ninja-build ### Build and install $ cmake -B build -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DUSE_QT_GUI=ON -GNinja $ ninja -Cbuild install ## Fedora ``` sudo dnf install cmake make gcc gcc-c++ glibc tinyxml2-devel rhash-devel \ libtidy-devel tinyxml-devel jsoncpp-devel libcurl-devel \ boost-devel ``` ### Build and Install ``` cmake .. make ``` ## Usage examples - **Login** lgogdownloader --login - **Listing games and details for specific games** lgogdownloader --list lgogdownloader --list details --game witcher - **Downloading files** lgogdownloader --download lgogdownloader --download --game stardew_valley --exclude extras lgogdownloader --download --threads 6 --platform linux --language en+de,fr lgogdownloader --download-file tyrian_2000/9543 - **Repairing files** lgogdownloader --repair --game beneath_a_steel_sky lgogdownloader --repair --download --game "^a" - **Using Galaxy API for listing and installing game builds** lgogdownloader --galaxy-platform windows --galaxy-show-builds stardew_valley lgogdownloader --galaxy-platform windows --galaxy-install stardew_valley/0 lgogdownloader --galaxy-platform windows --galaxy-install beneath_a_steel_sky/0 --galaxy-no-dependencies - **See man page or help text for more** lgogdownloader --help man lgogdownloader ## Links - [LGOGDownloader website](https://sites.google.com/site/gogdownloader/) - [GOG forum thread](https://www.gog.com/forum/general/lgogdownloader_gogdownloader_for_linux) - [LGOGDownloader @ AUR](https://aur.archlinux.org/packages/lgogdownloader/) - [LGOGDownloader @ AUR (git version)](https://aur.archlinux.org/packages/lgogdownloader-git/) - [LGOGDownloader @ Debian](https://tracker.debian.org/lgogdownloader) - [LGOGDownloader @ Ubuntu](https://launchpad.net/ubuntu/+source/lgogdownloader) [![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=PT95NXVLQU6WG&source=url) lgogdownloader-3.17/cmake/000077500000000000000000000000001476654310300155445ustar00rootroot00000000000000lgogdownloader-3.17/cmake/FindJsoncpp.cmake000066400000000000000000000014041476654310300207620ustar00rootroot00000000000000# - Try to find Jsoncpp # # Once done, this will define # Jsoncpp_FOUND - system has Jsoncpp # Jsoncpp_INCLUDE_DIRS - the Jsoncpp include directories # Jsoncpp_LIBRARIES - link these to use Jsoncpp find_package(PkgConfig) pkg_check_modules(PC_JSONCPP REQUIRED jsoncpp) find_path(JSONCPP_INCLUDE_DIR NAMES json/allocator.h HINTS ${PC_JSONCPP_INCLUDEDIR} ${PC_JSONCPP_INCLUDEDIRS} PATH_SUFFIXES jsoncpp PATHS ${PC_JSONCPP_INCLUDE_DIRS} ) find_library(JSONCPP_LIBRARY jsoncpp PATHS ${PC_JSONCPP_LIBRARY_DIRS} ) mark_as_advanced(JSONCPP_INCLUDE_DIR JSONCPP_LIBRARY) if(PC_JSONCPP_FOUND) set(Jsoncpp_FOUND ON) set(Jsoncpp_INCLUDE_DIRS ${JSONCPP_INCLUDE_DIR}) set(Jsoncpp_LIBRARIES ${JSONCPP_LIBRARY}) endif(PC_JSONCPP_FOUND) lgogdownloader-3.17/cmake/FindRhash.cmake000066400000000000000000000012051476654310300204120ustar00rootroot00000000000000# - Try to find rhash # # Once done this will define # Rhash_FOUND - System has rhash # Rhash_INCLUDE_DIRS - The rhash include directories # Rhash_LIBRARIES - The libraries needed to use rhash find_path(RHASH_INCLUDE_DIR rhash.h) find_library(RHASH_LIBRARY rhash) mark_as_advanced(RHASH_INCLUDE_DIR RHASH_LIBRARY) if(RHASH_LIBRARY AND RHASH_INCLUDE_DIR) set(Rhash_FOUND ON) set(Rhash_LIBRARIES ${RHASH_LIBRARY}) set(Rhash_INCLUDE_DIRS ${RHASH_INCLUDE_DIR}) else() set(Rhash_FOUND OFF) if(Rhash_FIND_REQUIRED) message(FATAL_ERROR "Could not find rhash") endif(Rhash_FIND_REQUIRED) endif(RHASH_LIBRARY AND RHASH_INCLUDE_DIR) lgogdownloader-3.17/cmake/FindTidy.cmake000066400000000000000000000013521476654310300202610ustar00rootroot00000000000000# - Try to find tidy # # Once done this will define # Tidy_FOUND - System has tidy # Tidy_INCLUDE_DIRS - The tidy include directories # Tidy_LIBRARIES - The libraries needed to use tidy find_package(PkgConfig) pkg_check_modules(PC_TIDY tidy REQUIRED) find_path(TIDY_INCLUDE_DIR tidy.h HINTS ${PC_TIDY_INCLUDEDIR} ${PC_TIDY_INCLUDE_DIRS} PATH_SUFFIXES tidy PATHS ${PC_TIDY_INCLUDE_DIRS} ) find_library(TIDY_LIBRARY tidy HINTS ${PC_TIDY_LIBDIR} ${PC_TIDY_LIBRARY_DIRS} PATHS ${PC_TIDY_LIBRARY_DIRS} ) mark_as_advanced(TIDY_INCLUDE_DIR TIDY_LIBRARY) if(TIDY_INCLUDE_DIR) set(Tidy_FOUND ON) set(Tidy_INCLUDE_DIRS ${TIDY_INCLUDE_DIR}) set(Tidy_LIBRARIES ${TIDY_LIBRARY}) endif(TIDY_INCLUDE_DIR) lgogdownloader-3.17/cmake/FindTinyxml2.cmake000066400000000000000000000014671476654310300211050ustar00rootroot00000000000000# - Try to find tinyxml2 # # Once done this will define # Tinyxml2_FOUND - System has tinyxml2 # Tinyxml2_INCLUDE_DIRS - The tinyxml2 include directories # Tinyxml2_LIBRARIES - The libraries needed to use tinyxml find_package(PkgConfig) pkg_check_modules(PC_TINYXML2 tinyxml2) find_path(TINYXML2_INCLUDE_DIR tinyxml2.h HINTS ${PC_TINYXML2_INCLUDEDIR} ${PC_TINYXML2_INCLUDE_DIRS} PATHS ${PC_TINYXML2_INCLUDE_DIRS} ) find_library(TINYXML2_LIBRARY tinyxml2 HINTS ${PC_TINYXML2_LIBDIR} ${PC_TINYXML2_LIBRARY_DIRS} PATHS ${PC_TINYXML2_LIBRARY_DIRS} ) mark_as_advanced(TINYXML2_INCLUDE_DIR TINYXML2_LIBRARY) if(TINYXML2_INCLUDE_DIR) set(Tinyxml2_FOUND ON) set(Tinyxml2_INCLUDE_DIRS ${TINYXML2_INCLUDE_DIR}) set(Tinyxml2_LIBRARIES ${TINYXML2_LIBRARY}) endif(TINYXML2_INCLUDE_DIR) lgogdownloader-3.17/include/000077500000000000000000000000001476654310300161075ustar00rootroot00000000000000lgogdownloader-3.17/include/blacklist.h000066400000000000000000000021161476654310300202300ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef BLACKLIST_H__ #define BLACKLIST_H__ #include #include #include class Config; class gameFile; class BlacklistItem { public: unsigned int linenr; // where the blacklist item is defined in blacklist.txt unsigned int flags; std::string source; // source representation of the item boost::regex regex; }; class Blacklist { public: Blacklist() {}; void initialize(const std::vector& lines); bool isBlacklisted(const std::string& path); std::vector::size_type size() const { return blacklist_.size(); } bool empty() { return blacklist_.empty(); } private: std::vector blacklist_; }; #endif // BLACKLIST_H_ lgogdownloader-3.17/include/config.h000066400000000000000000000215431476654310300175320ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef CONFIG_H__ #define CONFIG_H__ #include #include #include #include #include #include "blacklist.h" struct DirectoryConfig { bool bSubDirectories; std::string sDirectory; std::string sWinePrefix; std::string sGameSubdir; std::string sInstallersSubdir; std::string sExtrasSubdir; std::string sPatchesSubdir; std::string sLanguagePackSubdir; std::string sDLCSubdir; std::string sGalaxyInstallSubdir; }; struct DownloadConfig { unsigned int iInstallerPlatform; unsigned int iInstallerLanguage; unsigned int iGalaxyCDN; std::vector vPlatformPriority; std::vector vLanguagePriority; std::vector vGalaxyCDNPriority; std::vector vTags; unsigned int iInclude; unsigned int iGalaxyPlatform; unsigned int iGalaxyLanguage; unsigned int iGalaxyArch; bool bRemoteXML; bool bSaveChangelogs; bool bSaveSerials; bool bSaveGameDetailsJson; bool bSaveProductJson; bool bSaveLogo; bool bSaveIcon; bool bAutomaticXMLCreation; bool bFreeSpaceCheck; bool bIgnoreDLCCount; bool bDuplicateHandler; bool bGalaxyDependencies; bool bDeleteOrphans; bool bGalaxyLowercasePath; }; struct gameSpecificConfig { DownloadConfig dlConf; DirectoryConfig dirConf; }; class GalaxyConfig { public: bool isExpired() { std::unique_lock lock(m); bool bExpired = true; // assume that token is expired intmax_t time_now = time(NULL); if (this->token_json.isMember("expires_at")) bExpired = (time_now > this->token_json["expires_at"].asLargestInt()); return bExpired; } std::string getAccessToken() { std:: string access_token; std::unique_lock lock(m); if (this->token_json.isMember("access_token")) access_token = this->token_json["access_token"].asString(); return access_token; } std::string getRefreshToken() { std::string refresh_token; std::unique_lock lock(m); if (this->token_json.isMember("refresh_token")) refresh_token = this->token_json["refresh_token"].asString(); return refresh_token; } Json::Value getJSON() { std::unique_lock lock(m); return this->token_json; } std::string getUserId() { std::unique_lock lock(m); if(this->token_json.isMember("user_id")) { return this->token_json["user_id"].asString(); } return {}; } void setJSON(Json::Value json) { std::unique_lock lock(m); if (!json.isMember("expires_at")) { intmax_t time_now = time(NULL); Json::Value::LargestInt expires_in = 3600; if (json.isMember("expires_in")) if (!json["expires_in"].isNull()) expires_in = json["expires_in"].asLargestInt(); Json::Value::LargestInt expires_at = time_now + expires_in; json["expires_at"] = expires_at; } this->token_json = json; } void setFilepath(const std::string& path) { std::unique_lock lock(m); this->filepath = path; } std::string getFilepath() { std::unique_lock lock(m); return this->filepath; } void resetClient() { std::lock_guard lock(m); if(token_json.isMember("client_id")) { token_json["client_id"] = default_client_id; } if(token_json.isMember("client_secret")) { token_json["client_secret"] = default_client_secret; } } std::string getClientId() { std::lock_guard lock(m); if(token_json.isMember("client_id")) { return token_json["client_id"].asString(); } return default_client_id; } std::string getClientSecret() { std::lock_guard lock(m); if(token_json.isMember("client_secret")) { return token_json["client_secret"].asString(); } return default_client_secret; } std::string getRedirectUri() { std::unique_lock lock(m); return this->redirect_uri; } GalaxyConfig() = default; GalaxyConfig(const GalaxyConfig& other) { std::lock_guard guard(other.m); redirect_uri = other.redirect_uri; filepath = other.filepath; token_json = other.token_json; } GalaxyConfig& operator= (GalaxyConfig& other) { if(&other == this) return *this; std::unique_lock lock1(m, std::defer_lock); std::unique_lock lock2(other.m, std::defer_lock); std::lock(lock1, lock2); redirect_uri = other.redirect_uri; filepath = other.filepath; token_json = other.token_json; return *this; } protected: private: const std::string default_client_id = "46899977096215655"; const std::string default_client_secret = "9d85c43b1482497dbbce61f6e4aa173a433796eeae2ca8c5f6129f2dc4de46d9"; std::string redirect_uri = "https://embed.gog.com/on_login_success?origin=client"; std::string filepath; Json::Value token_json; mutable std::mutex m; }; struct CurlConfig { bool bVerifyPeer; bool bVerbose; std::string sCACertPath; std::string sCookiePath; std::string sUserAgent; long int iTimeout; curl_off_t iDownloadRate; long int iLowSpeedTimeout; long int iLowSpeedTimeoutRate; std::string sInterface; }; class Config { public: Config() {}; virtual ~Config() {}; // Booleans bool bLogin; bool bForceBrowserLogin; bool bSaveConfig; bool bResetConfig; bool bDownload; bool bRepair; bool bUpdated; bool bNew; bool bCheckStatus; bool bNotifications; bool bIncludeHiddenProducts; bool bSizeOnly; bool bUnicode; // use Unicode in console output bool bColor; // use colors bool bReport; bool bRespectUmask; bool bPlatformDetection; #ifdef USE_QT_GUI_LOGIN bool bEnableLoginGUI; bool bForceGUILogin; #endif bool bUseFastCheck; bool bTrustAPIForExtras; bool bGalaxyListCDNs; // Cache bool bUseCache; bool bUpdateCache; int iCacheValid; // Download with file id options std::string sFileIdString; std::string sOutputFilename; // Curl CurlConfig curlConf; // Download DownloadConfig dlConf; // Directories DirectoryConfig dirConf; std::string sCacheDirectory; std::string sXMLDirectory; std::string sConfigDirectory; // File paths std::string sConfigFilePath; std::string sBlacklistFilePath; std::string sIgnorelistFilePath; std::string sGameHasDLCListFilePath; std::string sReportFilePath; std::string sTransformConfigFilePath; std::string sXMLFile; // Regex std::string sGameRegex; std::string sOrphanRegex; std::string sIgnoreDLCCountRegex; // Priorities std::string sPlatformPriority; std::string sLanguagePriority; // General strings std::string sVersionString; std::string sVersionNumber; std::string sEmail; std::string sPassword; // Lists Blacklist blacklist; Blacklist ignorelist; // Cloud save options std::vector cloudWhiteList; std::vector cloudBlackList; bool bCloudForce; // Integers int iRetries; unsigned int iThreads; unsigned int iInfoThreads; int iWait; size_t iChunkSize; int iProgressInterval; int iMsgLevel; unsigned int iListFormat; Json::Value transformationsJSON; }; #endif // CONFIG_H__ lgogdownloader-3.17/include/downloader.h000066400000000000000000000217201476654310300204200ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef DOWNLOADER_H #define DOWNLOADER_H #if __GNUC__ # if !(__x86_64__ || __ppc64__ || __LP64__) # ifndef _LARGEFILE_SOURCE # define _LARGEFILE_SOURCE # endif # ifndef _LARGEFILE64_SOURCE # define _LARGEFILE64_SOURCE # endif # if !defined(_FILE_OFFSET_BITS) || (_FILE_OFFSET_BITS == 32) # define _FILE_OFFSET_BITS 64 # endif # endif #endif #include "config.h" #include "progressbar.h" #include "website.h" #include "threadsafequeue.h" #include "galaxyapi.h" #include "globals.h" #include "util.h" #include #include #include #include #include #include class cloudSaveFile; class Timer { public: Timer() { this->reset(); }; void reset() { gettimeofday(&(this->last_update), NULL); }; double getTimeBetweenUpdates() { // Returns time elapsed between updates in milliseconds struct timeval time_now; gettimeofday(&time_now, NULL); double time_between = ( (time_now.tv_sec+(time_now.tv_usec/1000000.0))*1000.0 - (this->last_update.tv_sec+(this->last_update.tv_usec/1000000.0))*1000.0 ); return time_between; }; ~Timer() {}; private: struct timeval last_update; }; struct xferInfo { unsigned int tid; CURL* curlhandle; Timer timer; std::deque< std::pair > TimeAndSize; curl_off_t offset; bool isChunk = false; curl_off_t chunk_file_total = 0; curl_off_t chunk_file_offset = 0; }; typedef struct { std::string filepath; off_t comp_size; off_t uncomp_size; off_t start_offset_zip; off_t start_offset_mojosetup; off_t end_offset; uint16_t file_attributes; uint32_t crc32; time_t timestamp; std::string installer_url; // For split file handling bool isSplitFile = false; std::string splitFileBasePath; std::string splitFilePartExt; off_t splitFileStartOffset; off_t splitFileEndOffset; } zipFileEntry; typedef std::map> splitFilesMap; class Downloader { public: Downloader(); virtual ~Downloader(); bool isLoggedIn(); int init(); int login(); int listGames(); void checkNotifications(); void clearUpdateNotifications(); void repair(); void download(); void downloadCloudSaves(const std::string& product_id, const std::string& build_id = std::string()); void downloadCloudSavesById(const std::string& product_id, const std::string& build_id = std::string()); void uploadCloudSaves(const std::string& product_id, const std::string& build_id = std::string()); void uploadCloudSavesById(const std::string& product_id, const std::string& build_id = std::string()); void deleteCloudSaves(const std::string& product_id, const std::string& build_id = std::string()); void deleteCloudSavesById(const std::string& product_id, const std::string& build_id = std::string()); void checkOrphans(); void checkStatus(); void updateCache(); int downloadFileWithId(const std::string& fileid_string, const std::string& output_filepath); void showWishlist(); CURL* curlhandle; Timer timer; ProgressBar* progressbar; std::deque< std::pair > TimeAndSize; void saveGalaxyJSON(); void galaxyInstallGame(const std::string& product_id, const std::string& build_id = std::string(), const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64); void galaxyInstallGameById(const std::string& product_id, const std::string& build_id = std::string(), const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64); void galaxyListCDNs(const std::string& product_id, const std::string& build_id = std::string()); void galaxyListCDNsById(const std::string& product_id, const std::string& build_id = std::string()); void galaxyShowBuilds(const std::string& product_id, const std::string& build_id = std::string()); void galaxyShowCloudSaves(const std::string& product_id, const std::string& build_id = std::string()); void galaxyShowLocalCloudSaves(const std::string& product_id, const std::string& build_id = std::string()); void galaxyShowLocalCloudSavesById(const std::string& product_id, const std::string& build_id = std::string()); void galaxyShowBuildsById(const std::string& product_id, const std::string& build_id = std::string()); void galaxyShowCloudSavesById(const std::string& product_id, const std::string& build_id = std::string()); protected: private: std::map cloudSaveLocations(const std::string& product_id, const std::string& build_id = std::string()); int cloudSaveListByIdForEach(const std::string& product_id, const std::string& build_id, const std::function &f); CURLcode downloadFile(const std::string& url, const std::string& filepath, const std::string& xml_data = std::string(), const std::string& gamename = std::string()); int repairFile(const std::string& url, const std::string& filepath, const std::string& xml_data = std::string(), const std::string& gamename = std::string()); int getGameDetails(); void getGameList(); uintmax_t getResumePosition(); CURLcode beginDownload(); std::string getResponse(const std::string& url); std::string getLocalFileHash(const std::string& filepath, const std::string& gamename = std::string()); std::string getRemoteFileHash(const gameFile& gf); void addStatusLine(const std::string& statusCode, const std::string& gamename, const std::string& filepath, const uintmax_t& filesize, const std::string& localHash); int loadGameDetailsCache(); int saveGameDetailsCache(); std::vector getGameDetailsFromJsonNode(Json::Value root, const int& recursion_level = 0); static std::string getSerialsFromJSON(const Json::Value& json); void saveSerials(const std::string& serials, const std::string& filepath); static std::string getChangelogFromJSON(const Json::Value& json); void saveJsonFile(const std::string& json, const std::string& filepath); void saveChangelog(const std::string& changelog, const std::string& filepath); static void processDownloadQueue(Config conf, const unsigned int& tid); static void processCloudSaveDownloadQueue(Config conf, const unsigned int& tid); static void processCloudSaveUploadQueue(Config conf, const unsigned int& tid); static int progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); template void printProgress(const ThreadSafeQueue& download_queue); static void getGameDetailsThread(Config config, const unsigned int& tid); void printGameDetailsAsText(gameDetails& game); void printGameFileDetailsAsText(gameFile& gf); static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow); static size_t writeData(void *ptr, size_t size, size_t nmemb, FILE *stream); static size_t readData(void *ptr, size_t size, size_t nmemb, FILE *stream); std::vector galaxyGetOrphanedFiles(const std::vector& items, const std::string& install_path); static void processGalaxyDownloadQueue(const std::string& install_path, Config conf, const unsigned int& tid); void galaxyInstallGame_MojoSetupHack(const std::string& product_id); void galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendtoFirst = false); static void processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid); int mojoSetupGetFileVector(const gameFile& gf, std::vector& vFiles); std::string getGalaxyInstallDirectory(galaxyAPI *galaxyHandle, const Json::Value& manifest); bool galaxySelectProductIdHelper(const std::string& product_id, std::string& selected_product); std::vector galaxyGetDepotItemVectorFromJson(const Json::Value& json, const unsigned int& iGalaxyArch = GlobalConstants::ARCH_X64); int galaxyGetBuildIndexWithBuildId(Json::Value json, const std::string& build_id = std::string()); Website *gogWebsite; galaxyAPI *gogGalaxy; std::vector gameItems; std::vector games; off_t resume_position; int retries; std::ofstream report_ofs; }; #endif // DOWNLOADER_H lgogdownloader-3.17/include/downloadinfo.h000066400000000000000000000047461476654310300207560ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef DOWNLOADINFO_H #define DOWNLOADINFO_H #include #include const unsigned int DLSTATUS_NOTSTARTED = 0; const unsigned int DLSTATUS_STARTING = 1 << 0; const unsigned int DLSTATUS_RUNNING = 1 << 1; const unsigned int DLSTATUS_FINISHED = 1 << 2; struct progressInfo { curl_off_t dlnow; curl_off_t dltotal; double rate; double rate_avg; }; class DownloadInfo { public: void setFilename(const std::string& filename_) { std::unique_lock lock(m); filename = filename_; } std::string getFilename() { std::unique_lock lock(m); return filename; } void setStatus(const unsigned int& status_) { std::unique_lock lock(m); status = status_; } unsigned int getStatus() { std::unique_lock lock(m); return status; } void setProgressInfo(const progressInfo& info) { std::unique_lock lock(m); progress_info = info; } progressInfo getProgressInfo() { std::unique_lock lock(m); return progress_info; } DownloadInfo()=default; DownloadInfo(const DownloadInfo& other) { std::lock_guard guard(other.m); filename = other.filename; status = other.status; progress_info = other.progress_info; } DownloadInfo& operator= (DownloadInfo& other) { if(&other == this) return *this; std::unique_lock lock1(m, std::defer_lock); std::unique_lock lock2(other.m, std::defer_lock); std::lock(lock1, lock2); filename = other.filename; status = other.status; progress_info = other.progress_info; return *this; } private: std::string filename; unsigned int status; progressInfo progress_info; mutable std::mutex m; }; #endif // DOWNLOADINFO_H lgogdownloader-3.17/include/galaxyapi.h000066400000000000000000000070651476654310300202470ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef GALAXYAPI_H #define GALAXYAPI_H #include "globalconstants.h" #include "globals.h" #include "config.h" #include "util.h" #include "gamedetails.h" #include #include #include #include #include struct galaxyDepotItemChunk { std::string md5_compressed; std::string md5_uncompressed; uintmax_t size_compressed; uintmax_t size_uncompressed; uintmax_t offset_compressed; uintmax_t offset_uncompressed; }; struct galaxyDepotItem { std::string path; std::vector chunks; uintmax_t totalSizeCompressed; uintmax_t totalSizeUncompressed; std::string md5; std::string product_id; bool isDependency = false; bool isSmallFilesContainer = false; bool isInSFC = false; uintmax_t sfc_offset; uintmax_t sfc_size; }; class galaxyAPI { public: galaxyAPI(CurlConfig& conf); virtual ~galaxyAPI(); int init(); bool isTokenExpired(); bool refreshLogin(); bool refreshLogin(const std::string &clientId, const std::string &clientSecret, const std::string &refreshToken, bool newSession); Json::Value getProductBuilds(const std::string& product_id, const std::string& platform = "windows", const std::string& generation = "2"); Json::Value getManifestV1(const std::string& product_id, const std::string& build_id, const std::string& manifest_id = "repository", const std::string& platform = "windows"); Json::Value getManifestV1(const std::string& manifest_url); Json::Value getManifestV2(std::string manifest_hash, const bool& is_dependency = false); Json::Value getCloudPathAsJson(const std::string &clientId); Json::Value getSecureLink(const std::string& product_id, const std::string& path); Json::Value getDependencyLink(const std::string& path); std::string getResponse(const std::string& url, const char *encoding = nullptr); Json::Value getResponseJson(const std::string& url, const char *encoding = nullptr); std::string hashToGalaxyPath(const std::string& hash); std::vector getDepotItemsVector(const std::string& hash, const bool& is_dependency = false); Json::Value getProductInfo(const std::string& product_id); gameDetails productInfoJsonToGameDetails(const Json::Value& json, const DownloadConfig& dlConf); Json::Value getUserData(); Json::Value getDependenciesJson(); std::vector getFilteredDepotItemsVectorFromJson(const Json::Value& depot_json, const std::string& galaxy_language, const std::string& galaxy_arch, const bool& is_dependency = false); std::string getPathFromDownlinkUrl(const std::string& downlink_url, const std::string& gamename); std::vector cdnUrlTemplatesFromJson(const Json::Value& json, const std::vector& cdnPriority); protected: private: CurlConfig curlConf; static size_t writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp); CURL* curlhandle; std::vector fileJsonNodeToGameFileVector(const std::string& gamename, const Json::Value& json, const unsigned int& type, const DownloadConfig& dlConf); }; #endif // GALAXYAPI_H lgogdownloader-3.17/include/gamedetails.h000066400000000000000000000046551476654310300205510ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef GAMEDETAILS_H #define GAMEDETAILS_H #include "globalconstants.h" #include "globals.h" #include "gamefile.h" #include "config.h" #include "util.h" #include #include #include class gameDetails { public: gameDetails(); std::vector extras; std::vector installers; std::vector patches; std::vector languagepacks; std::vector dlcs; std::string gamename; std::string gamename_basegame; std::string product_id; std::string title; std::string title_basegame; std::string icon; std::string serials; std::string changelog; std::string logo; std::string gameDetailsJson; std::string productJson; void filterWithPriorities(const gameSpecificConfig& config); void makeFilepaths(const DirectoryConfig& config); std::string getSerialsFilepath(); std::string getLogoFilepath(); std::string getIconFilepath(); std::string getChangelogFilepath(); std::string getGameDetailsJsonFilepath(); std::string getProductJsonFilepath(); Json::Value getDetailsAsJson(); std::vector getGameFileVector(); std::vector getGameFileVectorFiltered(const unsigned int& iType); void filterWithType(const unsigned int& iType); std::string makeCustomFilepath(const std::string& filename, const gameDetails& gd, const DirectoryConfig& dirConf); virtual ~gameDetails(); protected: void filterListWithPriorities(std::vector& list, const gameSpecificConfig& config); void filterListWithType(std::vector& list, const unsigned int& iType); std::string makeFilepath(const gameFile& gf, const DirectoryConfig& dirConf); private: std::string serialsFilepath; std::string logoFilepath; std::string iconFilepath; std::string changelogFilepath; std::string gameDetailsJsonFilepath; std::string productJsonFilepath; }; #endif // GAMEDETAILS_H lgogdownloader-3.17/include/gamefile.h000066400000000000000000000023071476654310300200330ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef GAMEFILE_H #define GAMEFILE_H #include "globalconstants.h" #include "globals.h" #include #include #include class gameFile { public: gameFile(); int updated; std::string gamename; std::string id; std::string name; std::string path; std::string size; std::string galaxy_downlink_json_url; std::string version; std::string title; std::string gamename_basegame = ""; std::string title_basegame = ""; unsigned int platform; unsigned int language; unsigned int type; int score; int silent; void setFilepath(const std::string& path); std::string getFilepath() const; Json::Value getAsJson(); virtual ~gameFile(); protected: private: std::string filepath; }; #endif // GAMEFILE_H lgogdownloader-3.17/include/globalconstants.h000066400000000000000000000215001476654310300214530ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef GLOBALCONSTANTS_H_INCLUDED #define GLOBALCONSTANTS_H_INCLUDED #include #include namespace GlobalConstants { const int GAMEDETAILS_CACHE_VERSION = 7; const int ZLIB_WINDOW_SIZE = 15; struct optionsStruct {const unsigned int id; const std::string code; const std::string str; const std::string regexp;}; const std::string PROTOCOL_PREFIX = "gogdownloader://"; // Language constants const unsigned int LANGUAGE_EN = 1 << 0; const unsigned int LANGUAGE_DE = 1 << 1; const unsigned int LANGUAGE_FR = 1 << 2; const unsigned int LANGUAGE_PL = 1 << 3; const unsigned int LANGUAGE_RU = 1 << 4; const unsigned int LANGUAGE_CN = 1 << 5; const unsigned int LANGUAGE_CZ = 1 << 6; const unsigned int LANGUAGE_ES = 1 << 7; const unsigned int LANGUAGE_HU = 1 << 8; const unsigned int LANGUAGE_IT = 1 << 9; const unsigned int LANGUAGE_JP = 1 << 10; const unsigned int LANGUAGE_TR = 1 << 11; const unsigned int LANGUAGE_PT = 1 << 12; const unsigned int LANGUAGE_KO = 1 << 13; const unsigned int LANGUAGE_NL = 1 << 14; const unsigned int LANGUAGE_SV = 1 << 15; const unsigned int LANGUAGE_NO = 1 << 16; const unsigned int LANGUAGE_DA = 1 << 17; const unsigned int LANGUAGE_FI = 1 << 18; const unsigned int LANGUAGE_PT_BR = 1 << 19; const unsigned int LANGUAGE_SK = 1 << 20; const unsigned int LANGUAGE_BL = 1 << 21; const unsigned int LANGUAGE_UK = 1 << 22; const unsigned int LANGUAGE_ES_419 = 1 << 23; const unsigned int LANGUAGE_AR = 1 << 24; const unsigned int LANGUAGE_RO = 1 << 25; const unsigned int LANGUAGE_HE = 1 << 26; const unsigned int LANGUAGE_TH = 1 << 27; const std::vector LANGUAGES = { { LANGUAGE_EN, "en", "English", "en|eng|english|en[_-]US" }, { LANGUAGE_DE, "de", "German", "de|deu|ger|german|de[_-]DE" }, { LANGUAGE_FR, "fr", "French", "fr|fra|fre|french|fr[_-]FR" }, { LANGUAGE_PL, "pl", "Polish", "pl|pol|polish|pl[_-]PL" }, { LANGUAGE_RU, "ru", "Russian", "ru|rus|russian|ru[_-]RU" }, { LANGUAGE_CN, "cn", "Chinese", "cn|zh|zho|chi|chinese|zh[_-](CN|Hans)" }, { LANGUAGE_CZ, "cz", "Czech", "cz|cs|ces|cze|czech|cs[_-]CZ" }, { LANGUAGE_ES, "es", "Spanish", "es|spa|spanish|es[_-]ES" }, { LANGUAGE_HU, "hu", "Hungarian", "hu|hun|hungarian|hu[_-]HU" }, { LANGUAGE_IT, "it", "Italian", "it|ita|italian|it[_-]IT" }, { LANGUAGE_JP, "jp", "Japanese", "jp|ja|jpn|japanese|ja[_-]JP" }, { LANGUAGE_TR, "tr", "Turkish", "tr|tur|turkish|tr[_-]TR" }, { LANGUAGE_PT, "pt", "Portuguese", "pt|por|portuguese|pt[_-]PT" }, { LANGUAGE_KO, "ko", "Korean", "ko|kor|korean|ko[_-]KR" }, { LANGUAGE_NL, "nl", "Dutch", "nl|nld|dut|dutch|nl[_-]NL" }, { LANGUAGE_SV, "sv", "Swedish", "sv|swe|swedish|sv[_-]SE" }, { LANGUAGE_NO, "no", "Norwegian", "no|nor|norwegian|nb[_-]no|nn[_-]NO" }, { LANGUAGE_DA, "da", "Danish", "da|dan|danish|da[_-]DK" }, { LANGUAGE_FI, "fi", "Finnish", "fi|fin|finnish|fi[_-]FI" }, { LANGUAGE_PT_BR, "br", "Brazilian Portuguese", "br|pt_br|pt-br|ptbr|brazilian_portuguese" }, { LANGUAGE_SK, "sk", "Slovak", "sk|slk|slo|slovak|sk[_-]SK" }, { LANGUAGE_BL, "bl", "Bulgarian", "bl|bg|bul|bulgarian|bg[_-]BG" }, { LANGUAGE_UK, "uk", "Ukrainian", "uk|ukr|ukrainian|uk[_-]UA" }, { LANGUAGE_ES_419, "es_mx", "Spanish (Latin American)", "es_mx|es-mx|esmx|es-419|spanish_latin_american" }, { LANGUAGE_AR, "ar", "Arabic", "ar|ara|arabic|ar[_-][A-Z]{2}" }, { LANGUAGE_RO, "ro", "Romanian", "ro|ron|rum|romanian|ro[_-][RM]O" }, { LANGUAGE_HE, "he", "Hebrew", "he|heb|hebrew|he[_-]IL" }, { LANGUAGE_TH, "th", "Thai", "th|tha|thai|th[_-]TH" } }; // Platform constants const unsigned int PLATFORM_WINDOWS = 1 << 0; const unsigned int PLATFORM_MAC = 1 << 1; const unsigned int PLATFORM_LINUX = 1 << 2; const std::vector PLATFORMS = { { PLATFORM_WINDOWS, "win", "Windows" , "w|win|windows" }, { PLATFORM_MAC, "mac", "Mac" , "m|mac|osx" }, { PLATFORM_LINUX, "linux", "Linux" , "l|lin|linux" } }; // Galaxy platform arch const unsigned int ARCH_X86 = 1 << 0; const unsigned int ARCH_X64 = 1 << 1; const std::vector GALAXY_ARCHS = { { ARCH_X86, "32", "32-bit", "32|x86|32bit|32-bit" }, { ARCH_X64, "64", "64-bit", "64|x64|64bit|64-bit" } }; const unsigned int LIST_FORMAT_GAMES = 1 << 0; const unsigned int LIST_FORMAT_DETAILS_TEXT = 1 << 1; const unsigned int LIST_FORMAT_DETAILS_JSON = 1 << 2; const unsigned int LIST_FORMAT_TAGS = 1 << 3; const unsigned int LIST_FORMAT_TRANSFORMATIONS = 1 << 4; const unsigned int LIST_FORMAT_USERDATA = 1 << 5; const unsigned int LIST_FORMAT_WISHLIST = 1 << 6; const std::vector LIST_FORMAT = { { LIST_FORMAT_GAMES, "games", "Games", "g|games" }, { LIST_FORMAT_DETAILS_TEXT, "details", "Details", "d|details" }, { LIST_FORMAT_DETAILS_JSON, "json", "JSON", "j|json" }, { LIST_FORMAT_TAGS, "tags", "Tags", "t|tags" }, { LIST_FORMAT_TRANSFORMATIONS, "transform", "Transformations", "tr|transform|transformations" }, { LIST_FORMAT_USERDATA, "userdata", "User data", "ud|userdata" }, { LIST_FORMAT_WISHLIST, "wishlist", "Wishlist", "w|wishlist" } }; const unsigned int GFTYPE_BASE_INSTALLER = 1 << 0; const unsigned int GFTYPE_BASE_EXTRA = 1 << 1; const unsigned int GFTYPE_BASE_PATCH = 1 << 2; const unsigned int GFTYPE_BASE_LANGPACK = 1 << 3; const unsigned int GFTYPE_DLC_INSTALLER = 1 << 4; const unsigned int GFTYPE_DLC_EXTRA = 1 << 5; const unsigned int GFTYPE_DLC_PATCH = 1 << 6; const unsigned int GFTYPE_DLC_LANGPACK = 1 << 7; const unsigned int GFTYPE_CUSTOM_BASE = 1 << 8; const unsigned int GFTYPE_CUSTOM_DLC = 1 << 9; const unsigned int GFTYPE_DLC = GFTYPE_DLC_INSTALLER | GFTYPE_DLC_EXTRA | GFTYPE_DLC_PATCH | GFTYPE_DLC_LANGPACK | GFTYPE_CUSTOM_DLC; const unsigned int GFTYPE_BASE = GFTYPE_BASE_INSTALLER | GFTYPE_BASE_EXTRA | GFTYPE_BASE_PATCH | GFTYPE_BASE_LANGPACK | GFTYPE_CUSTOM_BASE; const unsigned int GFTYPE_INSTALLER = GFTYPE_BASE_INSTALLER | GFTYPE_DLC_INSTALLER; const unsigned int GFTYPE_EXTRA = GFTYPE_BASE_EXTRA | GFTYPE_DLC_EXTRA; const unsigned int GFTYPE_PATCH = GFTYPE_BASE_PATCH | GFTYPE_DLC_PATCH; const unsigned int GFTYPE_LANGPACK = GFTYPE_BASE_LANGPACK | GFTYPE_DLC_LANGPACK; const unsigned int GFTYPE_CUSTOM = GFTYPE_CUSTOM_BASE | GFTYPE_CUSTOM_DLC; const std::vector INCLUDE_OPTIONS = { { GFTYPE_BASE_INSTALLER, "bi", "Base game installers", "bi|basegame_installers" }, { GFTYPE_BASE_EXTRA, "be", "Base game extras", "be|basegame_extras" }, { GFTYPE_BASE_PATCH, "bp", "Base game patches", "bp|basegame_patches" }, { GFTYPE_BASE_LANGPACK, "bl", "Base game language packs", "bl|basegame_languagepacks|basegame_langpacks" }, { GFTYPE_DLC_INSTALLER, "di", "DLC installers", "di|dlc_installers" }, { GFTYPE_DLC_EXTRA, "de", "DLC extras", "de|dlc_extras" }, { GFTYPE_DLC_PATCH, "dp", "DLC patches", "dp|dlc_patches" }, { GFTYPE_DLC_LANGPACK, "dl", "DLC language packs", "dl|dlc_languagepacks|dlc_langpacks" }, { GFTYPE_DLC, "d", "DLCs", "d|dlc|dlcs" }, { GFTYPE_BASE, "b", "Basegame", "b|bg|basegame" }, { GFTYPE_INSTALLER, "i", "All installers", "i|installers" }, { GFTYPE_EXTRA, "e", "All extras", "e|extras" }, { GFTYPE_PATCH, "p", "All patches", "p|patches" }, { GFTYPE_LANGPACK, "l", "All language packs", "l|languagepacks|langpacks" } }; } #endif // GLOBALCONSTANTS_H_INCLUDED lgogdownloader-3.17/include/globals.h000066400000000000000000000011341476654310300177020ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef GLOBALS_H_INCLUDED #define GLOBALS_H_INCLUDED #include "config.h" #include #include namespace Globals { extern GalaxyConfig galaxyConf; extern Config globalConfig; extern std::vector vOwnedGamesIds; } #endif // GLOBALS_H_INCLUDED lgogdownloader-3.17/include/gui_login.h000066400000000000000000000021661476654310300202410ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef GUI_LOGIN_H #define GUI_LOGIN_H #include "config.h" #include "util.h" #include "globals.h" #include #include #include #include class GuiLogin : public QObject { Q_OBJECT public: GuiLogin(); virtual ~GuiLogin(); void Login(); void Login(const std::string& username, const std::string& password); std::string getCode(); std::vector getCookies(); private: QWebEngineCookieStore *cookiestore; std::vector cookies; std::string auth_code; std::string login_username; std::string login_password; public slots: void loadFinished(bool success); void cookieAdded(const QNetworkCookie &cookie); }; #endif // GUI_LOGIN_H lgogdownloader-3.17/include/message.h000066400000000000000000000063451476654310300177140ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef MESSAGE_H #define MESSAGE_H #include const unsigned int MSGTYPE_INFO = 1 << 0; const unsigned int MSGTYPE_WARNING = 1 << 1; const unsigned int MSGTYPE_ERROR = 1 << 2; const unsigned int MSGTYPE_SUCCESS = 1 << 3; const int MSGLEVEL_ALWAYS = -1; const int MSGLEVEL_DEFAULT = 0; const int MSGLEVEL_VERBOSE = 1; const int MSGLEVEL_DEBUG = 2; class Message { public: Message() = default; Message(std::string msg, const unsigned int& type = MSGTYPE_INFO, const std::string& prefix = std::string(), const int& level = MSGLEVEL_DEFAULT) { prefix_ = prefix; msg_ = msg; type_ = type; timestamp_ = boost::posix_time::second_clock::local_time(); level_ = level; } void setMessage(const std::string& msg) { msg_ = msg; } void setType(const unsigned int& type) { type_ = type; } void setTimestamp(const boost::posix_time::ptime& timestamp) { timestamp_ = timestamp; } void setPrefix(const std::string& prefix) { prefix_ = prefix; } void setLevel(const int& level) { level_ = level; } std::string getMessage() { return msg_; } unsigned int getType() { return type_; } boost::posix_time::ptime getTimestamp() { return timestamp_; } std::string getTimestampString() { return boost::posix_time::to_simple_string(timestamp_); } std::string getPrefix() { return prefix_; } int getLevel() { return level_; } std::string getFormattedString(const bool& bColor = true, const bool& bPrefix = true) { std::string str; std::string color_value = "\033[39m"; // Default foreground color std::string color_reset = "\033[0m"; if (type_ == MSGTYPE_INFO) color_value = "\033[39m"; // Default foreground color else if (type_ == MSGTYPE_WARNING) color_value = "\033[33m"; // Yellow else if (type_ == MSGTYPE_ERROR) color_value = "\033[31m"; // Red else if (type_ == MSGTYPE_SUCCESS) color_value = "\033[32m"; // Green str = msg_; if (!prefix_.empty() && bPrefix) str = prefix_ + " " + str; str = getTimestampString() + " " + str; if (bColor) str = color_value + str + color_reset; return str; } private: std::string msg_; boost::posix_time::ptime timestamp_; unsigned int type_; std::string prefix_; int level_; }; #endif // MESSAGE_H lgogdownloader-3.17/include/progressbar.h000066400000000000000000000022601476654310300206110ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef PROGRESSBAR_H #define PROGRESSBAR_H #include #include class ProgressBar { public: ProgressBar(bool bUnicode, bool bColor); virtual ~ProgressBar(); void draw(unsigned int length, double fraction); std::string createBarString(unsigned int length, double fraction); protected: private: std::vector const m_bar_chars; std::string const m_left_border; std::string const m_right_border; std::string const m_simple_left_border; std::string const m_simple_right_border; std::string const m_simple_empty_fill; std::string const m_simple_bar_char; std::string const m_bar_color; std::string const m_border_color; std::string const COLOR_RESET; bool m_use_unicode; bool m_use_color; }; #endif // PROGRESSBAR_H lgogdownloader-3.17/include/threadsafequeue.h000066400000000000000000000041111476654310300214300ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef THREADSAFEQUEUE_H #define THREADSAFEQUEUE_H #include #include #include template class ThreadSafeQueue { public: void push(const T& item) { std::unique_lock lock(m); q.push(item); lock.unlock(); cvar.notify_one(); } bool empty() const { std::unique_lock lock(m); return q.empty(); } typename std::queue::size_type size() const { std::unique_lock lock(m); return q.size(); } bool try_pop(T& item) { std::unique_lock lock(m); if(q.empty()) return false; item = q.front(); q.pop(); return true; } void wait_and_pop(T& item) { std::unique_lock lock(m); while(q.empty()) cvar.wait(lock); item = q.front(); q.pop(); } ThreadSafeQueue() = default; ThreadSafeQueue(const ThreadSafeQueue& other) { std::lock_guard guard(other.m); q = other.q; } ThreadSafeQueue& operator= (ThreadSafeQueue& other) { if(&other == this) return *this; std::unique_lock lock1(m, std::defer_lock); std::unique_lock lock2(other.m, std::defer_lock); std::lock(lock1, lock2); q = other.q; return *this; } private: std::queue q; mutable std::mutex m; std::condition_variable cvar; }; #endif // THREADSAFEQUEUE_H lgogdownloader-3.17/include/util.h000066400000000000000000000116641476654310300172450ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef UTIL_H #define UTIL_H #include "globalconstants.h" #include "config.h" #include "globals.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include typedef struct { char *memory; curl_off_t size; } ChunkMemoryStruct; struct gameItem { std::string name; std::string id; std::vector dlcnames; Json::Value gamedetailsjson; int updates = 0; bool isnew; }; struct wishlistItem { std::string title; unsigned int platform; std::vector tags; time_t release_date_time; std::string currency; std::string price; std::string discount_percent; std::string discount; std::string store_credit; std::string url; bool bIsBonusStoreCreditIncluded; bool bIsDiscounted; }; namespace Util { std::string getFileHash(const std::string& filename, unsigned hash_id); std::string getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start = 0, off_t range_end = 0); std::string getChunkHash(unsigned char* chunk, uintmax_t chunk_size, unsigned hash_id); int createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir = std::string()); int getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory = std::string()); int replaceString(std::string& str, const std::string& to_replace, const std::string& replace_with); int replaceAllString(std::string& str, const std::string& to_replace, const std::string& replace_with); void setFilePermissions(const boost::filesystem::path& path, const boost::filesystem::perms& permissions); int getTerminalWidth(); void getManualUrlsFromJSON(const Json::Value &root, std::vector &urls); std::vector getDLCNamesFromJSON(const Json::Value &root); std::string getHomeDir(); std::string getConfigHome(); std::string getCacheHome(); std::vector tokenize(const std::string& str, const std::string& separator = ","); unsigned int getOptionValue(const std::string& str, const std::vector& options, const bool& bAllowStringToIntConversion = true); std::string getOptionNameString(const unsigned int& value, const std::vector& options); void parseOptionString(const std::string &option_string, std::vector &priority, unsigned int &type, const std::vector& options); std::string getLocalFileHash(const std::string& xml_dir, const std::string& filepath, const std::string& gamename = std::string(), const bool& useFastCheck = true); void shortenStringToTerminalWidth(std::string& str); std::string getJsonUIntValueAsString(const Json::Value& json); std::string getStrippedString(std::string str); std::string makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate); std::string makeEtaString(const boost::posix_time::time_duration& duration); std::string CurlHandleGetInfoString(CURL* curlhandle, CURLINFO info); void CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf); CURLcode CurlGetResponse(const std::string& url, std::string& response, int max_retries = -1); CURLcode CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries = -1); curl_off_t CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp); curl_off_t CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp); curl_off_t CurlReadChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, ChunkMemoryStruct *userp); std::string makeSizeString(const unsigned long long& iSizeInBytes); template std::string formattedString(const std::string& format, Args ... args) { std::size_t sz = std::snprintf(nullptr, 0, format.c_str(), args ...) + 1; // +1 for null terminator std::unique_ptr buf(new char[sz]); std::snprintf(buf.get(), sz, format.c_str(), args ...); return std::string(buf.get(), buf.get() + sz - 1); // -1 because we don't want the null terminator } Json::Value readJsonFile(const std::string& path); std::string transformGamename(const std::string& gamename); std::string htmlToXhtml(const std::string& html); tinyxml2::XMLNode* nextXMLNode(tinyxml2::XMLNode* node); } #endif // UTIL_H lgogdownloader-3.17/include/website.h000066400000000000000000000032471476654310300177300ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef WEBSITE_H #define WEBSITE_H #include "config.h" #include "util.h" #include "globals.h" #include #include #include class Website { public: Website(); int Login(const std::string& email, const std::string& password); std::string getResponse(const std::string& url); Json::Value getResponseJson(const std::string& url); Json::Value getGameDetailsJSON(const std::string& gameid); std::vector getGames(); std::vector getWishlistItems(); bool IsLoggedIn(); std::map getTags(); std::vector getOwnedGamesIds(); virtual ~Website(); protected: private: CURL* curlhandle; bool IsloggedInSimple(); std::map getTagsFromJson(const Json::Value& json); int retries; std::string LoginGetAuthCode(const std::string& email, const std::string& password); std::string LoginGetAuthCodeCurl(const std::string& login_form_html, const std::string& email, const std::string& password); std::string LoginGetAuthCodeBrowser(const std::string& auth_url); #ifdef USE_QT_GUI_LOGIN std::string LoginGetAuthCodeGUI(const std::string& email, const std::string& password); #endif }; #endif // WEBSITE_H lgogdownloader-3.17/include/ziputil.h000066400000000000000000000064221476654310300177640ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #ifndef ZIPUTIL_H #define ZIPUTIL_H #include #include #include #include #include #define ZIP_LOCAL_HEADER_SIGNATURE 0x04034b50 #define ZIP_CD_HEADER_SIGNATURE 0x02014b50 #define ZIP_EOCD_HEADER_SIGNATURE 0x06054b50 #define ZIP_EOCD_HEADER_SIGNATURE64 0x06064b50 #define ZIP_EXTENSION_ZIP64 0x0001 #define ZIP_EXTENDED_TIMESTAMP 0x5455 #define ZIP_INFOZIP_UNIX_NEW 0x7875 typedef struct { uint32_t header = 0; uint16_t disk = 0; uint16_t cd_start_disk = 0; uint16_t cd_records = 0; uint16_t total_cd_records = 0; uint32_t cd_size = 0; uint32_t cd_start_offset = 0; uint16_t comment_length = 0; std::string comment; } zipEOCD; typedef struct { uint32_t header = 0; uint64_t directory_record_size = 0; uint16_t version_made_by = 0; uint16_t version_needed = 0; uint32_t cd = 0; uint32_t cd_start = 0; uint64_t cd_total_disk = 0; uint64_t cd_total = 0; uint64_t cd_size = 0; uint64_t cd_offset = 0; std::string comment; } zip64EOCD; typedef struct { uint32_t header = 0; uint16_t version_made_by = 0; uint16_t version_needed = 0; uint16_t flag = 0; uint16_t compression_method = 0; uint16_t mod_date = 0; uint16_t mod_time = 0; uint32_t crc32 = 0; uint64_t comp_size = 0; uint64_t uncomp_size = 0; uint16_t filename_length = 0; uint16_t extra_length = 0; uint16_t comment_length = 0; uint32_t disk_num = 0; uint16_t internal_file_attr = 0; uint32_t external_file_attr = 0; uint64_t disk_offset = 0; std::string filename; std::string extra; std::string comment; time_t timestamp = 0; bool isLocalCDEntry = false; } zipCDEntry; namespace ZipUtil { off_t getMojoSetupScriptSize(std::stringstream *stream); off_t getMojoSetupInstallerSize(std::stringstream *stream); struct tm date_time_to_tm(uint64_t date, uint64_t time); bool isValidDate(struct tm timeinfo); uint64_t readValue(std::istream *stream, uint32_t len); uint64_t readUInt64(std::istream *stream); uint32_t readUInt32(std::istream *stream); uint16_t readUInt16(std::istream *stream); uint8_t readUInt8(std::istream *stream); off_t getZipEOCDOffsetSignature(std::istream *stream, const uint32_t& signature); off_t getZipEOCDOffset(std::istream *stream); off_t getZip64EOCDOffset(std::istream *stream); zipEOCD readZipEOCDStruct(std::istream *stream, const off_t& eocd_start_pos = 0); zip64EOCD readZip64EOCDStruct(std::istream *stream, const off_t& eocd_start_pos = 0); zipCDEntry readZipCDEntry(std::istream *stream); int extractFile(const std::string& input_file_path, const std::string& output_file_path); int extractStream(std::istream* input_stream, std::ostream* output_stream); boost::filesystem::perms getBoostFilePermission(const uint16_t& attributes); bool isSymlink(const uint16_t& attributes); } #endif // ZIPUTIL_H lgogdownloader-3.17/main.cpp000066400000000000000000001452031476654310300161210ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "downloader.h" #include "config.h" #include "util.h" #include "globalconstants.h" #include "galaxyapi.h" #include "globals.h" #include #include #include #include namespace bpo = boost::program_options; Config Globals::globalConfig; template void set_vm_value(std::map& vm, const std::string& option, const T& value) { vm[option].value() = boost::any(value); } void ensure_trailing_slash(std::string &path, const char *default_ = nullptr) { if (!path.empty()) { if (path.at(path.length()-1)!='/') path += "/"; } else { path = default_; // Directory wasn't specified, use current directory } } int main(int argc, char *argv[]) { struct sigaction act; act.sa_handler = SIG_IGN; act.sa_flags = SA_RESTART; sigemptyset(&act.sa_mask); if (sigaction(SIGPIPE, &act, NULL) < 0) return 1; rhash_library_init(); Globals::globalConfig.sVersionString = VERSION_STRING; Globals::globalConfig.sVersionNumber = VERSION_NUMBER; Globals::globalConfig.curlConf.sUserAgent = DEFAULT_USER_AGENT; Globals::globalConfig.sCacheDirectory = Util::getCacheHome() + "/lgogdownloader"; Globals::globalConfig.sXMLDirectory = Globals::globalConfig.sCacheDirectory + "/xml"; Globals::globalConfig.sConfigDirectory = Util::getConfigHome() + "/lgogdownloader"; Globals::globalConfig.curlConf.sCookiePath = Globals::globalConfig.sConfigDirectory + "/cookies.txt"; Globals::globalConfig.sConfigFilePath = Globals::globalConfig.sConfigDirectory + "/config.cfg"; Globals::globalConfig.sBlacklistFilePath = Globals::globalConfig.sConfigDirectory + "/blacklist.txt"; Globals::globalConfig.sIgnorelistFilePath = Globals::globalConfig.sConfigDirectory + "/ignorelist.txt"; Globals::globalConfig.sTransformConfigFilePath = Globals::globalConfig.sConfigDirectory + "/transformations.json"; Globals::galaxyConf.setFilepath(Globals::globalConfig.sConfigDirectory + "/galaxy_tokens.json"); std::string sDefaultBlacklistFilePath = Globals::globalConfig.sConfigDirectory + "/blacklist.txt"; std::string sDefaultIgnorelistFilePath = Globals::globalConfig.sConfigDirectory + "/ignorelist.txt"; std::string priority_help_text = "Set priority by separating values with \",\"\nCombine values by separating with \"+\""; // Create help text for --platform option std::string platform_text = "Select which installers are downloaded\n"; for (unsigned int i = 0; i < GlobalConstants::PLATFORMS.size(); ++i) { platform_text += GlobalConstants::PLATFORMS[i].str + " = " + GlobalConstants::PLATFORMS[i].regexp + "\n"; } platform_text += "All = all"; platform_text += "\n\n" + priority_help_text; platform_text += "\nExample: Linux if available otherwise Windows and Mac: l,w+m"; // Create help text for --galaxy-platform option std::string galaxy_platform_text = "Select platform\n"; for (unsigned int i = 0; i < GlobalConstants::PLATFORMS.size(); ++i) { galaxy_platform_text += GlobalConstants::PLATFORMS[i].str + " = " + GlobalConstants::PLATFORMS[i].regexp + "\n"; } // Create help text for --language option std::string language_text = "Select which language installers are downloaded\n"; for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i) { language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n"; } language_text += "All = all"; language_text += "\n\n" + priority_help_text; language_text += "\nExample: German if available otherwise English and French: de,en+fr"; // Create help text for --galaxy-language option std::string galaxy_language_text = "Select language\n"; for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i) { galaxy_language_text += GlobalConstants::LANGUAGES[i].str + " = " + GlobalConstants::LANGUAGES[i].regexp + "\n"; } // Create help text for --galaxy-arch option std::string galaxy_arch_text = "Select architecture\n"; for (unsigned int i = 0; i < GlobalConstants::GALAXY_ARCHS.size(); ++i) { galaxy_arch_text += GlobalConstants::GALAXY_ARCHS[i].str + " = " + GlobalConstants::GALAXY_ARCHS[i].regexp + "\n"; } // Create help text for --subdir-galaxy-install option std::string galaxy_install_subdir_text = "Set subdirectory for galaxy install\n" "\nTemplates:\n" "- %install_dir% = Installation directory from Galaxy API response\n" "- %gamename% = Game name\n" "- %title% = Title of the game\n" "- %product_id% = Product id of the game\n" "- %install_dir_stripped% = %install_dir% with some characters stripped\n" "- %title_stripped% = %title% with some characters stripped\n" "\n\"stripped\" means that every character that doesn't match the following list is removed:\n" "> alphanumeric\n" "> space\n" "> - _ . ( ) [ ] { }"; // Create help text for --galaxy-cdn-priority option std::string galaxy_cdn_priority_text = "Set priority for used CDNs\n"; galaxy_cdn_priority_text += "Use --galaxy-list-cdns to list available CDNs\n"; galaxy_cdn_priority_text += "Set priority by separating values with \",\""; // Create help text for --check-orphans std::string orphans_regex_default = ".*\\.(zip|exe|bin|dmg|old|deb|tar\\.gz|pkg|sh|mp4)$"; // Limit to files with these extensions (".old" is for renamed older version files) std::string check_orphans_text = "Check for orphaned files (files found on local filesystem that are not found on GOG servers). Sets regular expression filter (Perl syntax) for files to check. If no argument is given then the regex defaults to '" + orphans_regex_default + "'"; // Help text for subdir options std::string subdir_help_text = "\nTemplates:\n" "- %platform%\n" "- %gamename%\n" "- %gamename_firstletter%\n" "- %dlcname%\n" "- %gamename_transformed%\n" "- %gamename_transformed_firstletter%\n" "- %title%\n" "- %title_stripped%\n" "- %dlc_title%\n" "- %dlc_title_stripped%"; // Help text for include and exclude options std::string include_options_text; for (unsigned int i = 0; i < GlobalConstants::INCLUDE_OPTIONS.size(); ++i) { include_options_text += GlobalConstants::INCLUDE_OPTIONS[i].str + " = " + GlobalConstants::INCLUDE_OPTIONS[i].regexp + "\n"; } include_options_text += "All = all\n"; include_options_text += "Separate with \",\" to use multiple values"; // Create help text for --list-format option std::string list_format_text = "List games/tags\n"; for (unsigned int i = 0; i < GlobalConstants::LIST_FORMAT.size(); ++i) { list_format_text += GlobalConstants::LIST_FORMAT[i].str + " = " + GlobalConstants::LIST_FORMAT[i].regexp + "\n"; } std::string galaxy_product_id_install; std::string galaxy_product_id_list_cdns; std::string galaxy_product_id_show_builds; std::string galaxy_product_id_show_cloud_paths; std::string galaxy_product_id_show_local_cloud_paths; std::string galaxy_product_cloud_saves; std::string galaxy_product_cloud_saves_delete; std::string galaxy_upload_product_cloud_saves; std::string tags; std::vector vFileIdStrings; std::vector unrecognized_options_cfg; std::vector unrecognized_options_cli; bpo::variables_map vm; bpo::options_description options_cli_all("Options"); bpo::options_description options_cli_no_cfg; bpo::options_description options_cli_no_cfg_hidden; bpo::options_description options_cli_all_include_hidden; bpo::options_description options_cli_experimental("Experimental"); bpo::options_description options_cli_cfg; bpo::options_description options_cfg_only; bpo::options_description options_cfg_all("Configuration"); bool bClearUpdateNotifications = false; bool bList = false; bool bCheckLoginStatus = false; try { bool bInsecure = false; bool bNoColor = false; bool bNoUnicode = false; bool bNoDuplicateHandler = false; bool bNoRemoteXML = false; bool bNoSubDirectories = false; bool bNoPlatformDetection = false; bool bNoGalaxyDependencies = false; bool bNoFastStatusCheck = false; std::string sInstallerPlatform; std::string sInstallerLanguage; std::string sIncludeOptions; std::string sExcludeOptions; std::string sGalaxyPlatform; std::string sGalaxyLanguage; std::string sGalaxyArch; std::string sGalaxyCDN; std::string sListFormat; Globals::globalConfig.bReport = false; // Commandline options (no config file) options_cli_no_cfg.add_options() ("help,h", "Print help message") ("version", "Print version information") ("login", bpo::value(&Globals::globalConfig.bLogin)->zero_tokens()->default_value(false), "Login") #ifdef USE_QT_GUI_LOGIN ("gui-login", bpo::value(&Globals::globalConfig.bForceGUILogin)->zero_tokens()->default_value(false), "Login (force GUI login)\nImplies --enable-login-gui") #endif ("browser-login", bpo::value(&Globals::globalConfig.bForceBrowserLogin)->zero_tokens()->default_value(false), "Login (force browser login)") ("check-login-status", bpo::value(&bCheckLoginStatus)->zero_tokens()->default_value(false), "Check login status") ("list", bpo::value(&sListFormat)->implicit_value("games"), list_format_text.c_str()) ("download", bpo::value(&Globals::globalConfig.bDownload)->zero_tokens()->default_value(false), "Download") ("repair", bpo::value(&Globals::globalConfig.bRepair)->zero_tokens()->default_value(false), "Repair downloaded files\nUse --repair --download to redownload files when filesizes don't match (possibly different version). Redownload will rename the old file (appends .old to filename)") ("game", bpo::value(&Globals::globalConfig.sGameRegex)->default_value(""), "Set regular expression filter\nfor download/list/repair (Perl syntax)") ("create-xml", bpo::value(&Globals::globalConfig.sXMLFile)->implicit_value("automatic"), "Create GOG XML for file\n\"automatic\" to enable automatic XML creation") ("notifications", bpo::value(&Globals::globalConfig.bNotifications)->zero_tokens()->default_value(false), "Check notifications") ("updated", bpo::value(&Globals::globalConfig.bUpdated)->zero_tokens()->default_value(false), "List/download only games with update flag set") ("new", bpo::value(&Globals::globalConfig.bNew)->zero_tokens()->default_value(false), "List/download only games with new flag set") ("clear-update-flags", bpo::value(&bClearUpdateNotifications)->zero_tokens()->default_value(false), "Clear update notification flags") ("check-orphans", bpo::value(&Globals::globalConfig.sOrphanRegex)->implicit_value(""), check_orphans_text.c_str()) ("delete-orphans", bpo::value(&Globals::globalConfig.dlConf.bDeleteOrphans)->zero_tokens()->default_value(false), "Delete orphaned files during --check-orphans and --galaxy-install") ("status", bpo::value(&Globals::globalConfig.bCheckStatus)->zero_tokens()->default_value(false), "Show status of files\n\nOutput format:\nstatuscode gamename filename filesize filehash\n\nStatus codes:\nOK - File is OK\nND - File is not downloaded\nMD5 - MD5 mismatch, different version\nFS - File size mismatch, incomplete download\n\nSee also --no-fast-status-check option") ("save-config", bpo::value(&Globals::globalConfig.bSaveConfig)->zero_tokens()->default_value(false), "Create config file with current settings") ("reset-config", bpo::value(&Globals::globalConfig.bResetConfig)->zero_tokens()->default_value(false), "Reset config settings to default") ("report", bpo::value(&Globals::globalConfig.sReportFilePath)->implicit_value("lgogdownloader-report.log"), "Save report of downloaded/repaired files to specified file\nDefault filename: lgogdownloader-report.log") ("update-cache", bpo::value(&Globals::globalConfig.bUpdateCache)->zero_tokens()->default_value(false), "Update game details cache") ("no-platform-detection", bpo::value(&bNoPlatformDetection)->zero_tokens()->default_value(false), "Don't try to detect supported platforms from game shelf.\nSkips the initial fast platform detection and detects the supported platforms from game details which is slower but more accurate.\nUseful in case platform identifier is missing for some games in the game shelf.\nUsing --platform with --list doesn't work with this option.") ("download-file", bpo::value(&Globals::globalConfig.sFileIdString)->default_value(""), "Download files using fileid\n\nFormat:\n\"gamename/fileid\"\n\"gamename/dlc_gamename/fileid\"\n\"gogdownloader://gamename/fileid\"\n\"gogdownloader://gamename/dlc_name/fileid\"\n\nMultiple files:\n\"gamename1/fileid1,gamename2/fileid2,gamename2/dlcname/fileid1\"\n\nThis option ignores all subdir options. The files are downloaded to directory specified with --directory option.") ("output-file,o", bpo::value(&Globals::globalConfig.sOutputFilename)->default_value(""), "Set filename of file downloaded with --download-file.") ("cacert", bpo::value(&Globals::globalConfig.curlConf.sCACertPath)->default_value(""), "Path to CA certificate bundle in PEM format") ("respect-umask", bpo::value(&Globals::globalConfig.bRespectUmask)->zero_tokens()->default_value(false), "Do not adjust permissions of sensitive files") ("user-agent", bpo::value(&Globals::globalConfig.curlConf.sUserAgent)->default_value(DEFAULT_USER_AGENT), "Set user agent") ("wine-prefix", bpo::value(&Globals::globalConfig.dirConf.sWinePrefix)->default_value("."), "Set wineprefix directory") ("cloud-whitelist", bpo::value>(&Globals::globalConfig.cloudWhiteList)->multitoken(), "Include this list of cloud saves, by default all cloud saves are included\n Example: --cloud-whitelist saves/AutoSave-0 saves/AutoSave-1/screenshot.png") ("cloud-blacklist", bpo::value>(&Globals::globalConfig.cloudBlackList)->multitoken(), "Exclude this list of cloud saves\n Example: --cloud-blacklist saves/AutoSave-0 saves/AutoSave-1/screenshot.png") ("cloud-force", bpo::value(&Globals::globalConfig.bCloudForce)->zero_tokens()->default_value(false), "Download or Upload cloud saves even if they're up-to-date\nDelete remote cloud saves even if no saves are whitelisted") #ifdef USE_QT_GUI_LOGIN ("enable-login-gui", bpo::value(&Globals::globalConfig.bEnableLoginGUI)->zero_tokens()->default_value(false), "Enable login GUI when encountering reCAPTCHA on login form") #endif ("tag", bpo::value(&tags)->default_value(""), "Filter using tags. Separate with \",\" to use multiple values") ("blacklist", bpo::value(&Globals::globalConfig.sBlacklistFilePath)->default_value(sDefaultBlacklistFilePath), "Filepath to blacklist") ("ignorelist", bpo::value(&Globals::globalConfig.sIgnorelistFilePath)->default_value(sDefaultIgnorelistFilePath), "Filepath to ignorelist") ; // Commandline options (config file) options_cli_cfg.add_options() ("directory", bpo::value(&Globals::globalConfig.dirConf.sDirectory)->default_value("."), "Set download directory") ("limit-rate", bpo::value(&Globals::globalConfig.curlConf.iDownloadRate)->default_value(0), "Limit download rate to value in kB\n0 = unlimited") ("xml-directory", bpo::value(&Globals::globalConfig.sXMLDirectory), "Set directory for GOG XML files") ("chunk-size", bpo::value(&Globals::globalConfig.iChunkSize)->default_value(10), "Chunk size (in MB) when creating XML") ("platform", bpo::value(&sInstallerPlatform)->default_value("w+l"), platform_text.c_str()) ("language", bpo::value(&sInstallerLanguage)->default_value("en"), language_text.c_str()) ("no-remote-xml", bpo::value(&bNoRemoteXML)->zero_tokens()->default_value(false), "Don't use remote XML for repair") ("no-unicode", bpo::value(&bNoUnicode)->zero_tokens()->default_value(false), "Don't use Unicode in the progress bar") ("no-color", bpo::value(&bNoColor)->zero_tokens()->default_value(false), "Don't use coloring in the progress bar or status messages") ("no-duplicate-handling", bpo::value(&bNoDuplicateHandler)->zero_tokens()->default_value(false), "Don't use duplicate handler for installers\nDuplicate installers from different languages are handled separately") ("no-subdirectories", bpo::value(&bNoSubDirectories)->zero_tokens()->default_value(false), "Don't create subdirectories for extras, patches and language packs") ("curl-verbose", bpo::value(&Globals::globalConfig.curlConf.bVerbose)->zero_tokens()->default_value(false), "Set libcurl to verbose mode") ("insecure", bpo::value(&bInsecure)->zero_tokens()->default_value(false), "Don't verify authenticity of SSL certificates") ("timeout", bpo::value(&Globals::globalConfig.curlConf.iTimeout)->default_value(10), "Set timeout for connection\nMaximum time in seconds that connection phase is allowed to take") ("retries", bpo::value(&Globals::globalConfig.iRetries)->default_value(3), "Set maximum number of retries on failed download") ("wait", bpo::value(&Globals::globalConfig.iWait)->default_value(0), "Time to wait between requests (milliseconds)") ("subdir-installers", bpo::value(&Globals::globalConfig.dirConf.sInstallersSubdir)->default_value(""), ("Set subdirectory for installers" + subdir_help_text).c_str()) ("subdir-extras", bpo::value(&Globals::globalConfig.dirConf.sExtrasSubdir)->default_value("extras"), ("Set subdirectory for extras" + subdir_help_text).c_str()) ("subdir-patches", bpo::value(&Globals::globalConfig.dirConf.sPatchesSubdir)->default_value("patches"), ("Set subdirectory for patches" + subdir_help_text).c_str()) ("subdir-language-packs", bpo::value(&Globals::globalConfig.dirConf.sLanguagePackSubdir)->default_value("languagepacks"), ("Set subdirectory for language packs" + subdir_help_text).c_str()) ("subdir-dlc", bpo::value(&Globals::globalConfig.dirConf.sDLCSubdir)->default_value("dlc/%dlcname%"), ("Set subdirectory for dlc" + subdir_help_text).c_str()) ("subdir-game", bpo::value(&Globals::globalConfig.dirConf.sGameSubdir)->default_value("%gamename%"), ("Set subdirectory for game" + subdir_help_text).c_str()) ("use-cache", bpo::value(&Globals::globalConfig.bUseCache)->zero_tokens()->default_value(false), ("Use game details cache")) ("cache-valid", bpo::value(&Globals::globalConfig.iCacheValid)->default_value(2880), ("Set how long cached game details are valid (in minutes)\nDefault: 2880 minutes (48 hours)")) ("save-serials", bpo::value(&Globals::globalConfig.dlConf.bSaveSerials)->zero_tokens()->default_value(false), "Save serial numbers when downloading") ("save-game-details-json", bpo::value(&Globals::globalConfig.dlConf.bSaveGameDetailsJson)->zero_tokens()->default_value(false), "Save game details JSON data as-is to \"game-details.json\"") ("save-product-json", bpo::value(&Globals::globalConfig.dlConf.bSaveProductJson)->zero_tokens()->default_value(false), "Save product info JSON data from the API as-is to \"product.json\"") ("save-logo", bpo::value(&Globals::globalConfig.dlConf.bSaveLogo)->zero_tokens()->default_value(false), "Save logo when downloading") ("save-icon", bpo::value(&Globals::globalConfig.dlConf.bSaveIcon)->zero_tokens()->default_value(false), "Save icon when downloading") ("ignore-dlc-count", bpo::value(&Globals::globalConfig.sIgnoreDLCCountRegex)->implicit_value(".*"), "Set regular expression filter for games to ignore DLC count information\nIgnoring DLC count information helps in situations where the account page doesn't provide accurate information about DLCs") ("include", bpo::value(&sIncludeOptions)->default_value("all"), ("Select what to download/list/repair\n" + include_options_text).c_str()) ("exclude", bpo::value(&sExcludeOptions)->default_value(""), ("Select what not to download/list/repair\n" + include_options_text).c_str()) ("automatic-xml-creation", bpo::value(&Globals::globalConfig.dlConf.bAutomaticXMLCreation)->zero_tokens()->default_value(false), "Automatically create XML data after download has completed") ("save-changelogs", bpo::value(&Globals::globalConfig.dlConf.bSaveChangelogs)->zero_tokens()->default_value(false), "Save changelogs when downloading") ("threads", bpo::value(&Globals::globalConfig.iThreads)->default_value(4), "Number of download threads") ("info-threads", bpo::value(&Globals::globalConfig.iInfoThreads)->default_value(4), "Number of threads for getting product info") ("progress-interval", bpo::value(&Globals::globalConfig.iProgressInterval)->default_value(100), "Set interval for progress bar update (milliseconds)\nValue must be between 1 and 10000") ("lowspeed-timeout", bpo::value(&Globals::globalConfig.curlConf.iLowSpeedTimeout)->default_value(30), "Set time in number seconds that the transfer speed should be below the rate set with --lowspeed-rate for it to considered too slow and aborted") ("lowspeed-rate", bpo::value(&Globals::globalConfig.curlConf.iLowSpeedTimeoutRate)->default_value(200), "Set average transfer speed in bytes per second that the transfer should be below during time specified with --lowspeed-timeout for it to be considered too slow and aborted") ("include-hidden-products", bpo::value(&Globals::globalConfig.bIncludeHiddenProducts)->zero_tokens()->default_value(false), "Include games that have been set hidden in account page") ("size-only", bpo::value(&Globals::globalConfig.bSizeOnly)->zero_tokens()->default_value(false), "Don't check the hashes of the files whose size matches that on the server") ("verbosity", bpo::value(&Globals::globalConfig.iMsgLevel)->default_value(0), "Set message verbosity level\n -1 = Less verbose\n 0 = Default\n 1 = Verbose\n 2 = Debug") ("check-free-space", bpo::value(&Globals::globalConfig.dlConf.bFreeSpaceCheck)->zero_tokens()->default_value(false), "Check for available free space before starting download") ("no-fast-status-check", bpo::value(&bNoFastStatusCheck)->zero_tokens()->default_value(false), "Don't use fast status check.\nMakes --status much slower but able to catch corrupted files by calculating local file hash for all files.") ("trust-api-for-extras", bpo::value(&Globals::globalConfig.bTrustAPIForExtras)->zero_tokens()->default_value(false), "Trust API responses for extras to be correct.") ("interface", bpo::value(&Globals::globalConfig.curlConf.sInterface)->default_value(""), "Perform operations using a specified network interface") ; options_cli_no_cfg_hidden.add_options() ("login-email", bpo::value(&Globals::globalConfig.sEmail)->default_value(""), "login email") ("login-password", bpo::value(&Globals::globalConfig.sPassword)->default_value(""), "login password") ; options_cli_experimental.add_options() ("galaxy-install", bpo::value(&galaxy_product_id_install)->default_value(""), "Install game using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-show-builds", bpo::value(&galaxy_product_id_show_builds)->default_value(""), "Show game builds using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build\nLists available builds if build index is not specified\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-download-cloud-saves", bpo::value(&galaxy_product_cloud_saves)->default_value(""), "Download cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-upload-cloud-saves", bpo::value(&galaxy_upload_product_cloud_saves)->default_value(""), "Upload cloud saves using product-id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-show-cloud-saves", bpo::value(&galaxy_product_id_show_cloud_paths)->default_value(""), "Show game cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-show-local-cloud-saves", bpo::value(&galaxy_product_id_show_local_cloud_paths)->default_value(""), "Show local cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-delete-cloud-saves", bpo::value(&galaxy_product_cloud_saves_delete)->default_value(""), "Delete cloud-saves using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-platform", bpo::value(&sGalaxyPlatform)->default_value("w"), galaxy_platform_text.c_str()) ("galaxy-language", bpo::value(&sGalaxyLanguage)->default_value("en"), galaxy_language_text.c_str()) ("galaxy-arch", bpo::value(&sGalaxyArch)->default_value("x64"), galaxy_arch_text.c_str()) ("galaxy-no-dependencies", bpo::value(&bNoGalaxyDependencies)->zero_tokens()->default_value(false), "Don't download dependencies during --galaxy-install") ("subdir-galaxy-install", bpo::value(&Globals::globalConfig.dirConf.sGalaxyInstallSubdir)->default_value("%install_dir%"), galaxy_install_subdir_text.c_str()) ("galaxy-cdn-priority", bpo::value(&sGalaxyCDN)->default_value("edgecast,akamai_edgecast_proxy,fastly"), galaxy_cdn_priority_text.c_str()) ("galaxy-list-cdns", bpo::value(&galaxy_product_id_list_cdns)->default_value(""), "List available CDNs for game using product id [product_id/build_index] or gamename regex [gamename/build_id]\nBuild index is used to select a build and defaults to 0 if not specified.\n\nExample: 12345/2 selects build 2 for product 12345") ("galaxy-lowercase-path", bpo::value(&Globals::globalConfig.dlConf.bGalaxyLowercasePath)->zero_tokens()->default_value(false), "Make filepath lowercase for Windows game files") ; options_cli_all.add(options_cli_no_cfg).add(options_cli_cfg).add(options_cli_experimental); options_cfg_all.add(options_cfg_only).add(options_cli_cfg); options_cli_all_include_hidden.add(options_cli_all).add(options_cli_no_cfg_hidden); bpo::parsed_options parsed = bpo::parse_command_line(argc, argv, options_cli_all_include_hidden); bpo::store(parsed, vm); unrecognized_options_cli = bpo::collect_unrecognized(parsed.options, bpo::include_positional); bpo::notify(vm); if (vm.count("help")) { std::cout << Globals::globalConfig.sVersionString << std::endl << options_cli_all << std::endl; return 0; } if (vm.count("version")) { std::cout << VERSION_STRING << std::endl; return 0; } if (vm.count("list")) { bList = true; } // Create lgogdownloader directories boost::filesystem::path path = Globals::globalConfig.sXMLDirectory; if (!boost::filesystem::exists(path)) { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; return 1; } } path = Globals::globalConfig.sConfigDirectory; if (!boost::filesystem::exists(path)) { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; return 1; } } path = Globals::globalConfig.sCacheDirectory; if (!boost::filesystem::exists(path)) { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; return 1; } } if (boost::filesystem::exists(Globals::globalConfig.sConfigFilePath)) { std::ifstream ifs(Globals::globalConfig.sConfigFilePath.c_str()); if (!ifs) { std::cerr << "Could not open config file: " << Globals::globalConfig.sConfigFilePath << std::endl; return 1; } else { bpo::parsed_options parsed = bpo::parse_config_file(ifs, options_cfg_all, true); bpo::store(parsed, vm); bpo::notify(vm); ifs.close(); unrecognized_options_cfg = bpo::collect_unrecognized(parsed.options, bpo::include_positional); } } if (boost::filesystem::exists(Globals::globalConfig.sBlacklistFilePath)) { std::ifstream ifs(Globals::globalConfig.sBlacklistFilePath.c_str()); if (!ifs) { std::cerr << "Could not open blacklist file: " << Globals::globalConfig.sBlacklistFilePath << std::endl; return 1; } else { std::string line; std::vector lines; while (!ifs.eof()) { std::getline(ifs, line); lines.push_back(std::move(line)); } Globals::globalConfig.blacklist.initialize(lines); } } if (boost::filesystem::exists(Globals::globalConfig.sIgnorelistFilePath)) { std::ifstream ifs(Globals::globalConfig.sIgnorelistFilePath.c_str()); if (!ifs) { std::cerr << "Could not open ignorelist file: " << Globals::globalConfig.sIgnorelistFilePath << std::endl; return 1; } else { std::string line; std::vector lines; while (!ifs.eof()) { std::getline(ifs, line); lines.push_back(std::move(line)); } Globals::globalConfig.ignorelist.initialize(lines); } } if (boost::filesystem::exists(Globals::globalConfig.sTransformConfigFilePath)) { Globals::globalConfig.transformationsJSON = Util::readJsonFile(Globals::globalConfig.sTransformConfigFilePath); } #ifdef USE_QT_GUI_LOGIN if (Globals::globalConfig.bForceGUILogin) { Globals::globalConfig.bLogin = true; Globals::globalConfig.bEnableLoginGUI = true; } #endif if (Globals::globalConfig.bForceBrowserLogin) { Globals::globalConfig.bLogin = true; } if (Globals::globalConfig.bLogin) { std::string login_conf = Globals::globalConfig.sConfigDirectory + "/login.txt"; if (boost::filesystem::exists(login_conf)) { std::ifstream ifs(login_conf); if (!ifs) { std::cerr << "Could not open login conf: " << login_conf << std::endl; return 1; } else { std::string line; std::vector lines; while (!ifs.eof()) { std::getline(ifs, line); lines.push_back(std::move(line)); } Globals::globalConfig.sEmail = lines[0]; Globals::globalConfig.sPassword = lines[1]; } } } if (vm.count("chunk-size")) Globals::globalConfig.iChunkSize <<= 20; // Convert chunk size from bytes to megabytes if (vm.count("limit-rate")) Globals::globalConfig.curlConf.iDownloadRate <<= 10; // Convert download rate from bytes to kilobytes if (vm.count("check-orphans")) if (Globals::globalConfig.sOrphanRegex.empty()) Globals::globalConfig.sOrphanRegex = orphans_regex_default; if (vm.count("report")) Globals::globalConfig.bReport = true; if (Globals::globalConfig.iWait > 0) Globals::globalConfig.iWait *= 1000; if (Globals::globalConfig.iProgressInterval < 1) Globals::globalConfig.iProgressInterval = 1; else if (Globals::globalConfig.iProgressInterval > 10000) Globals::globalConfig.iProgressInterval = 10000; if (Globals::globalConfig.iThreads < 1) { Globals::globalConfig.iThreads = 1; set_vm_value(vm, "threads", Globals::globalConfig.iThreads); } if (Globals::globalConfig.iMsgLevel < -1) { Globals::globalConfig.iMsgLevel = -1; set_vm_value(vm, "verbosity", Globals::globalConfig.iMsgLevel); } Globals::globalConfig.curlConf.bVerifyPeer = !bInsecure; Globals::globalConfig.bColor = !bNoColor; Globals::globalConfig.bUnicode = !bNoUnicode; Globals::globalConfig.dlConf.bDuplicateHandler = !bNoDuplicateHandler; Globals::globalConfig.dlConf.bRemoteXML = !bNoRemoteXML; Globals::globalConfig.dirConf.bSubDirectories = !bNoSubDirectories; Globals::globalConfig.bPlatformDetection = !bNoPlatformDetection; Globals::globalConfig.dlConf.bGalaxyDependencies = !bNoGalaxyDependencies; Globals::globalConfig.bUseFastCheck = !bNoFastStatusCheck; for (auto i = unrecognized_options_cli.begin(); i != unrecognized_options_cli.end(); ++i) if (i->compare(0, GlobalConstants::PROTOCOL_PREFIX.length(), GlobalConstants::PROTOCOL_PREFIX) == 0) Globals::globalConfig.sFileIdString = *i; if (!Globals::globalConfig.sFileIdString.empty()) { if (Globals::globalConfig.sFileIdString.compare(0, GlobalConstants::PROTOCOL_PREFIX.length(), GlobalConstants::PROTOCOL_PREFIX) == 0) { Globals::globalConfig.sFileIdString.replace(0, GlobalConstants::PROTOCOL_PREFIX.length(), ""); } vFileIdStrings = Util::tokenize(Globals::globalConfig.sFileIdString, ","); } if (!tags.empty()) Globals::globalConfig.dlConf.vTags = Util::tokenize(tags, ","); if (!Globals::globalConfig.sOutputFilename.empty() && vFileIdStrings.size() > 1) { std::cerr << "Cannot specify an output file name when downloading multiple files." << std::endl; return 1; } if (Globals::globalConfig.sXMLFile == "automatic") Globals::globalConfig.dlConf.bAutomaticXMLCreation = true; Util::parseOptionString(sInstallerLanguage, Globals::globalConfig.dlConf.vLanguagePriority, Globals::globalConfig.dlConf.iInstallerLanguage, GlobalConstants::LANGUAGES); Util::parseOptionString(sInstallerPlatform, Globals::globalConfig.dlConf.vPlatformPriority, Globals::globalConfig.dlConf.iInstallerPlatform, GlobalConstants::PLATFORMS); Globals::globalConfig.dlConf.iGalaxyPlatform = Util::getOptionValue(sGalaxyPlatform, GlobalConstants::PLATFORMS); Globals::globalConfig.dlConf.iGalaxyLanguage = Util::getOptionValue(sGalaxyLanguage, GlobalConstants::LANGUAGES); Globals::globalConfig.dlConf.iGalaxyArch = Util::getOptionValue(sGalaxyArch, GlobalConstants::GALAXY_ARCHS, false); if (Globals::globalConfig.dlConf.iGalaxyArch == 0 || Globals::globalConfig.dlConf.iGalaxyArch == Util::getOptionValue("all", GlobalConstants::GALAXY_ARCHS, false)) Globals::globalConfig.dlConf.iGalaxyArch = GlobalConstants::ARCH_X64; Globals::globalConfig.dlConf.vGalaxyCDNPriority = Util::tokenize(sGalaxyCDN, ","); unsigned int include_value = 0; unsigned int exclude_value = 0; std::vector vInclude = Util::tokenize(sIncludeOptions, ","); std::vector vExclude = Util::tokenize(sExcludeOptions, ","); for (std::vector::iterator it = vInclude.begin(); it != vInclude.end(); it++) { include_value |= Util::getOptionValue(*it, GlobalConstants::INCLUDE_OPTIONS); } for (std::vector::iterator it = vExclude.begin(); it != vExclude.end(); it++) { exclude_value |= Util::getOptionValue(*it, GlobalConstants::INCLUDE_OPTIONS); } Globals::globalConfig.dlConf.iInclude = include_value & ~exclude_value; Globals::globalConfig.iListFormat = Util::getOptionValue(sListFormat, GlobalConstants::LIST_FORMAT, false); } catch (std::exception& e) { std::cerr << "Error: " << e.what() << std::endl; return 1; } catch (...) { std::cerr << "Exception of unknown type!" << std::endl; return 1; } if (Globals::globalConfig.dlConf.iInstallerPlatform < GlobalConstants::PLATFORMS[0].id || Globals::globalConfig.dlConf.iInstallerPlatform > Util::getOptionValue("all", GlobalConstants::PLATFORMS)) { std::cerr << "Invalid value for --platform" << std::endl; return 1; } if (Globals::globalConfig.dlConf.iInstallerLanguage < GlobalConstants::LANGUAGES[0].id || Globals::globalConfig.dlConf.iInstallerLanguage > Util::getOptionValue("all", GlobalConstants::LANGUAGES)) { std::cerr << "Invalid value for --language" << std::endl; return 1; } if (!Globals::globalConfig.sXMLDirectory.empty()) { // Make sure that xml directory doesn't have trailing slash if (Globals::globalConfig.sXMLDirectory.at(Globals::globalConfig.sXMLDirectory.length()-1)=='/') Globals::globalConfig.sXMLDirectory.assign(Globals::globalConfig.sXMLDirectory.begin(), Globals::globalConfig.sXMLDirectory.end()-1); } // Create GOG XML for a file if (!Globals::globalConfig.sXMLFile.empty() && (Globals::globalConfig.sXMLFile != "automatic")) { Util::createXML(Globals::globalConfig.sXMLFile, Globals::globalConfig.iChunkSize, Globals::globalConfig.sXMLDirectory); return 0; } // Make sure that directory has trailing slash ensure_trailing_slash(Globals::globalConfig.dirConf.sDirectory, "./"); ensure_trailing_slash(Globals::globalConfig.dirConf.sWinePrefix, "./"); // CA certificate bundle if (Globals::globalConfig.curlConf.sCACertPath.empty()) { // Use CURL_CA_BUNDLE environment variable for CA certificate path if it is set char *ca_bundle = getenv("CURL_CA_BUNDLE"); if (ca_bundle) Globals::globalConfig.curlConf.sCACertPath = (std::string)ca_bundle; } if (!unrecognized_options_cfg.empty() && (!Globals::globalConfig.bSaveConfig || !Globals::globalConfig.bResetConfig)) { std::cerr << "Unrecognized options in " << Globals::globalConfig.sConfigFilePath << std::endl; for (unsigned int i = 0; i < unrecognized_options_cfg.size(); i+=2) { std::cerr << unrecognized_options_cfg[i] << " = " << unrecognized_options_cfg[i+1] << std::endl; } std::cerr << std::endl; } // Init curl globally curl_global_init(CURL_GLOBAL_ALL); struct CurlCleanup { ~CurlCleanup() { curl_global_cleanup(); } }; CurlCleanup _curl_cleanup; Downloader downloader; bool bLoginOK = false; // Login because --login was used if (Globals::globalConfig.bLogin) { bLoginOK = downloader.login(); } else { bool bIsLoggedin = downloader.isLoggedIn(); if (bCheckLoginStatus) { if (bIsLoggedin) { std::cout << "Login status: Logged in" << std::endl; return 0; } else { std::cout << "Login status: Not logged in" << std::endl; return 1; } } if (!bIsLoggedin) { Globals::globalConfig.bLogin = true; bLoginOK = downloader.login(); if (bLoginOK) { bIsLoggedin = downloader.isLoggedIn(); } } // Login failed, cleanup if (!bLoginOK && !bIsLoggedin) { return 1; } } // Make sure that config file and cookie file are only readable/writable by owner if (!Globals::globalConfig.bRespectUmask) { Util::setFilePermissions(Globals::globalConfig.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write); Util::setFilePermissions(Globals::globalConfig.curlConf.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write); Util::setFilePermissions(Globals::galaxyConf.getFilepath(), boost::filesystem::owner_read | boost::filesystem::owner_write); } if (Globals::globalConfig.bSaveConfig || bLoginOK) { std::ofstream ofs(Globals::globalConfig.sConfigFilePath.c_str()); if (ofs) { std::cerr << "Saving config: " << Globals::globalConfig.sConfigFilePath << std::endl; for (bpo::variables_map::iterator it = vm.begin(); it != vm.end(); ++it) { std::string option = it->first; std::string option_value_string; const bpo::variable_value& option_value = it->second; try { if (options_cfg_all.find(option, false).long_name() == option) { if (!option_value.empty()) { const std::type_info& type = option_value.value().type() ; if ( type == typeid(std::string) ) option_value_string = option_value.as(); else if ( type == typeid(int) ) option_value_string = std::to_string(option_value.as()); else if ( type == typeid(size_t) ) option_value_string = std::to_string(option_value.as()); else if ( type == typeid(unsigned int) ) option_value_string = std::to_string(option_value.as()); else if ( type == typeid(long int) ) option_value_string = std::to_string(option_value.as()); else if ( type == typeid(bool) ) { if (option_value.as() == true) option_value_string = "true"; else option_value_string = "false"; } } } } catch (...) { continue; } ofs << option << " = " << option_value_string << std::endl; } ofs.close(); if (!Globals::globalConfig.bRespectUmask) Util::setFilePermissions(Globals::globalConfig.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write); if (Globals::globalConfig.bSaveConfig) { return 0; } } else { std::cerr << "Failed to create config: " << Globals::globalConfig.sConfigFilePath << std::endl; return 1; } } else if (Globals::globalConfig.bResetConfig) { std::ofstream ofs(Globals::globalConfig.sConfigFilePath.c_str()); if (ofs) { ofs.close(); if (!Globals::globalConfig.bRespectUmask) Util::setFilePermissions(Globals::globalConfig.sConfigFilePath, boost::filesystem::owner_read | boost::filesystem::owner_write); return 0; } else { std::cerr << "Failed to create config: " << Globals::globalConfig.sConfigFilePath << std::endl; return 1; } } bool bInitOK = downloader.init(); if (!bInitOK) { return 1; } int res = 0; if (Globals::globalConfig.bUpdateCache) downloader.updateCache(); else if (Globals::globalConfig.bNotifications) downloader.checkNotifications(); else if (bClearUpdateNotifications) downloader.clearUpdateNotifications(); else if (!vFileIdStrings.empty()) { for (std::vector::iterator it = vFileIdStrings.begin(); it != vFileIdStrings.end(); it++) { res |= downloader.downloadFileWithId(*it, Globals::globalConfig.sOutputFilename) ? 1 : 0; } } else if (Globals::globalConfig.bRepair) // Repair file downloader.repair(); else if (Globals::globalConfig.bDownload) // Download games downloader.download(); else if (bList) // List games/extras/tags res = downloader.listGames(); else if (!Globals::globalConfig.sOrphanRegex.empty()) // Check for orphaned files if regex for orphans is set downloader.checkOrphans(); else if (Globals::globalConfig.bCheckStatus) downloader.checkStatus(); else if (!galaxy_product_id_show_builds.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_id_show_builds, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.galaxyShowBuilds(product_id, build_id); } else if (!galaxy_product_id_show_cloud_paths.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_id_show_cloud_paths, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.galaxyShowCloudSaves(product_id, build_id); } else if (!galaxy_product_id_show_local_cloud_paths.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_id_show_local_cloud_paths, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.galaxyShowLocalCloudSaves(product_id, build_id); } else if (!galaxy_product_cloud_saves_delete.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_cloud_saves_delete, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.deleteCloudSaves(product_id, build_id); } else if (!galaxy_product_id_install.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_id_install, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.galaxyInstallGame(product_id, build_id, Globals::globalConfig.dlConf.iGalaxyArch); } else if (!galaxy_product_id_list_cdns.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_id_list_cdns, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.galaxyListCDNs(product_id, build_id); } else if (!galaxy_product_cloud_saves.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_product_cloud_saves, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.downloadCloudSaves(product_id, build_id); } else if (!galaxy_upload_product_cloud_saves.empty()) { std::string build_id; std::vector tokens = Util::tokenize(galaxy_upload_product_cloud_saves, "/"); std::string product_id = tokens[0]; if (tokens.size() == 2) { build_id = tokens[1]; } downloader.uploadCloudSaves(product_id, build_id); } else { if (!Globals::globalConfig.bLogin) { // Show help message std::cerr << Globals::globalConfig.sVersionString << std::endl << options_cli_all << std::endl; } } // Orphan check was called at the same time as download. Perform it after download has finished if (!Globals::globalConfig.sOrphanRegex.empty() && Globals::globalConfig.bDownload) downloader.checkOrphans(); return res; } lgogdownloader-3.17/man/000077500000000000000000000000001476654310300152375ustar00rootroot00000000000000lgogdownloader-3.17/man/CMakeLists.txt000066400000000000000000000012411476654310300177750ustar00rootroot00000000000000find_program(GZIP gzip DOC "Location of the gzip program") mark_as_advanced(GZIP) include(GNUInstallDirs) if(GZIP) set(MAN_PAGE "${CMAKE_CURRENT_SOURCE_DIR}/${PROJECT_NAME}.1") set(MAN_FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.1.gz") add_custom_command( OUTPUT ${MAN_FILE} COMMAND ${GZIP} -c -9 ${MAN_PAGE} > ${MAN_FILE} MAIN_DEPENDENCY ${MAN_PAGE} COMMENT "Building man page" VERBATIM ) add_custom_target(manpage ALL DEPENDS ${MAN_FILE} ${PROJECT_NAME}) install(FILES ${MAN_FILE} DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) else(GZIP) message("WARNING: One of the following is missing: gzip; man page will not be generated") endif(GZIP) lgogdownloader-3.17/man/lgogdownloader.1000066400000000000000000000521401476654310300203320ustar00rootroot00000000000000.TH LGOGDOWNLOADER "1" "2025-03-19" "LGOGDownloader 3.17" "User Commands" .SH NAME LGOGDownloader \- manual page for LGOGDownloader 3.17 .SH SYNOPSIS .B lgogdownloader [\fIOPTION\fP]... .SH DESCRIPTION An open-source GOG.com downloader for Linux users which uses the same API as GOG Galaxy. .PP LGOGDownloader can download purchased games, query GOG.com to see if game files have changed, as well as downloading extras such as artwork and manuals. It is capable of downloading language-specific installers for games where they exist. .PP LGOGDownloader Options: .TP \fB\-h\fR [ \fB\-\-help\fR ] Print help message .TP \fB\-\-version\fR Print version information .TP \fB\-\-login\fR Login .TP \fB\-\-gui\-login\fR Login (force GUI login) .br Implies \fB\-\-enable\-login\-gui\fR .TP \fB\-\-browser\-login\fR Login (force browser login) .TP \fB\-\-check\-login\-status\fR Check login status .TP \fB\-\-list\fR [=arg(=games)] List games/tags .br Games = g|games .br Details = d|details .br JSON = j|json .br Tags = t|tags .br Transformations = tr|transform|transformations .br User data = ud|userdata .br Wishlist = w|wishlist . .TP \fB\-\-download\fR Download .TP \fB\-\-repair\fR Repair downloaded files Use \fB\-\-repair\fR \fB\-\-download\fR to redownload files when filesizes don't match (possibly different version). Redownload will rename the old file (appends .old to filename) .TP \fB\-\-game\fR arg Set regular expression filter for download/list/repair (Perl syntax) .TP \fB\-\-create\-xml\fR [=arg(=automatic)] Create GOG XML for file .br "automatic" to enable automatic XML creation .TP \fB\-\-notifications\fR Check notifications .TP \fB\-\-updated\fR List/download only games with update flag set .TP \fB\-\-new\fR List/download only games with new flag set .TP \fB\-\-clear\-update\-flags\fR Clear update notification flags .TP \fB\-\-check\-orphans\fR arg Check for orphaned files (files found on local filesystem that are not found on GOG servers). Sets regular expression filter (Perl syntax) for files to check. If no argument is given then the regex defaults to \&'.*\e.(zip|exe|bin|dmg|old|deb|tar\e.gz|pkg|sh|mp4)$' .TP \fB\-\-delete\-orphans\fR Delete orphaned files during \fB\-\-check\-orphans\fR and \fB\-\-galaxy\-install\fR .TP \fB\-\-status\fR Show status of files .sp 1 Output format: .br statuscode gamename filename filesize filehash .sp 1 Status codes: .br OK \- File is OK .br ND \- File is not downloaded .br MD5 \- MD5 mismatch, different version .br FS \- File size mismatch, incomplete download .nf .IP See also \fB\-\-no\-fast\-status\-check\fR option .TP \fB\-\-save\-config\fR Create config file with current settings .TP \fB\-\-reset\-config\fR Reset config settings to default .TP \fB\-\-report\fR [=arg(=lgogdownloader\-report.log)] Save report of downloaded/repaired files to specified file .br Default filename: lgogdownloader\-report\&.log .TP \fB\-\-update\-cache\fR Update game details cache .TP \fB\-\-no\-platform\-detection\fR Don't try to detect supported platforms from game shelf. .br Skips the initial fast platform detection and detects the supported platforms from game details which is slower but more accurate. .br Useful in case platform identifier is missing for some games in the game shelf. .br Using \fB\-\-platform\fR with \fB\-\-list\fR doesn't work with this option. .TP \fB\-\-download\-file\fR arg Download files using fileid .sp 1 Format: .br "gamename/fileid" .br "gamename/dlc_gamename/fileid" .br "gogdownloader://gamename/fileid" .br "gogdownloader://gamename/dlc_name/fileid" .sp 1 Multiple files: .br "gamename1/fileid1,gamename2/fileid2,gamename2/dlcname/fileid1" .sp 1 This option ignores all subdir options. .br The files are downloaded to directory specified with \fB\-\-directory\fR option. .TP \fB\-o\fR [ \fB\-\-output\-file\fR ] arg Set filename of file downloaded with \fB\-\-download\-file\fR. .TP \fB\-\-cacert\fR arg Path to CA certificate bundle in PEM format .TP \fB\-\-respect\-umask\fR Do not adjust permissions of sensitive files .TP \fB\-\-user\-agent\fR arg (=LGOGDownloader/3.17 (Linux x86_64)) Set user agent .TP \fB\-\-wine\-prefix\fR arg (=.) Set wineprefix directory .TP \fB\-\-cloud\-whitelist\fR arg Include this list of cloud saves, by default all cloud saves are included .sp 1 Example: \fB\-\-cloud\-whitelist\fR saves/AutoSave\-0 saves/AutoSave\-1/screenshot.png .TP \fB\-\-cloud\-blacklist\fR arg Exclude this list of cloud saves .br Example: \fB\-\-cloud\-blacklist\fR saves/AutoSave\-0 saves/AutoSave\-1/screenshot.png .TP \fB\-\-cloud\-force\fR Download or Upload cloud saves even if they're up\-to\-date .br Delete remote cloud saves even if no saves are whitelisted .TP \fB\-\-enable\-login\-gui\fR Enable login GUI when encountering reCAPTCHA on login form .TP \fB\-\-tag\fR arg Filter using tags. Separate with "," to use multiple values .TP \fB\-\-blacklist\fR arg (=$XDG_CONFIG_HOME/lgogdownloader/blacklist.txt) Filepath to blacklist .TP \fB\-\-ignorelist\fR arg (=$XDG_CONFIG_HOME/lgogdownloader/ignorelist.txt) Filepath to ignorelist .TP \fB\-\-directory\fR arg (=.) Set download directory .TP \fB\-\-limit\-rate\fR arg (=0) Limit download rate to value in kB .br 0 = unlimited .TP \fB\-\-xml\-directory\fR arg Set directory for GOG XML files .TP \fB\-\-chunk\-size\fR arg (=10) Chunk size (in MB) when creating XML .TP \fB\-\-platform\fR arg (=w+l) Select which installers are downloaded .br Windows = w|win|windows .br Mac = m|mac|osx .br Linux = l|lin|linux .br All = all .sp 1 Set priority by separating values with "," .br Combine values by separating with "+" .br Example: Linux if available otherwise Windows and Mac: l,w+m .TP \fB\-\-language\fR arg (=en) Select which language installers are downloaded .br .br See \fBLANGUAGES\fR section for available values .br All languages = all .sp 1 Set priority by separating values with "," .br Combine values by separating with "+" .br Example: German if available otherwise English and French: .br \-\-language de,en+fr .TP \fB\-\-no\-remote\-xml\fR Don't use remote XML for repair .TP \fB\-\-no\-unicode\fR Don't use Unicode in the progress bar .TP \fB\-\-no\-color\fR Don't use coloring in the progress bar or status messages .TP \fB\-\-no\-duplicate\-handling\fR Don't use duplicate handler for installers .br Duplicate installers from different languages are handled separately .TP \fB\-\-no\-subdirectories\fR Don't create subdirectories for extras, patches and language packs .TP \fB\-\-curl\-verbose\fR Set libcurl to verbose mode .TP \fB\-\-insecure\fR Don't verify authenticity of SSL certificates .TP \fB\-\-timeout\fR arg (=10) Set timeout for connection .br Maximum time in seconds that connection phase is allowed to take .TP \fB\-\-retries\fR arg (=3) Set maximum number of retries on failed download .TP \fB\-\-wait\fR arg (=0) Time to wait between requests (milliseconds) .TP \fB\-\-subdir\-installers\fR arg Set subdirectory for installers .br Templates: .br \- %platform% .br \- %gamename% .br \- %gamename_firstletter% .br \- %dlcname% .br \- %gamename_transformed% .br \- %gamename_transformed_firstletter% .br \- %title% .br \- %title_stripped% .br \- %dlc_title% .br \- %dlc_title_stripped% .TP \fB\-\-subdir\-extras\fR arg (=extras) Set subdirectory for extras .br See \fB\-\-subdir\-installers\fR for template values .TP \fB\-\-subdir\-patches\fR arg (=patches) Set subdirectory for patches .br See \fB\-\-subdir\-installers\fR for template values .TP \fB\-\-subdir\-language\-packs\fR arg (=languagepacks) Set subdirectory for language packs .br See \fB\-\-subdir\-installers\fR for template values .TP \fB\-\-subdir\-dlc\fR arg (=dlc/%dlcname%) Set subdirectory for dlc .br See \fB\-\-subdir\-installers\fR for template values .TP \fB\-\-subdir\-game\fR arg (=%gamename%) Set subdirectory for game .br See \fB\-\-subdir\-installers\fR for template values .TP \fB\-\-use\-cache\fR Use game details cache .TP \fB\-\-cache\-valid\fR arg (=2880) Set how long cached game details are valid (in minutes) .br Default: 2880 minutes (48 hours) .TP \fB\-\-save\-serials\fR Save serial numbers when downloading .TP \fB\-\-save\-game\-details\-json\fR Save game details JSON data as\-is to "game\-details.json" .TP \fB\-\-save\-product\-json\fR Save product info JSON data from the API as\-is to "product.json" .TP \fB\-\-save\-logo\fR Save logo when downloading .TP \fB\-\-save\-icon\fR Save icon when downloading .TP \fB\-\-ignore\-dlc\-count\fR [=arg(=.*)] Set regular expression filter for games to ignore DLC count information .br Ignoring DLC count information helps in situations where the account page doesn't provide accurate information about DLCs .TP \fB\-\-include\fR arg (=all) Select what to download/list/repair .br Base game installers = bi|basegame_installers .br Base game extras = be|basegame_extras .br Base game patches = bp|basegame_patches .br Base game language packs = bl|basegame_languagepacks|basegame_langpacks .br DLC installers = di|dlc_installers .br DLC extras = de|dlc_extras .br DLC patches = dp|dlc_patches .br DLC language packs = dl|dlc_languagepacks|dlc_langpacks .br DLCs = d|dlc|dlcs .br Basegame = b|bg|basegame .br All installers = i|installers .br All extras = e|extras .br All patches = p|patches .br All language packs = l|languagepacks|langpacks .br All = all .br Separate with "," to use multiple values .TP \fB\-\-exclude\fR arg Select what not to download/list/repair .br See \fB\-\-include\fR for option values .TP \fB\-\-automatic\-xml\-creation\fR Automatically create XML data after download has completed .TP \fB\-\-save\-changelogs\fR Save changelogs when downloading .TP \fB\-\-threads\fR arg (=4) Number of download threads .TP \fB\-\-info\-threads\fR arg (=4) Number of threads for getting product info .TP \fB\-\-progress\-interval\fR arg (=100) Set interval for progress bar update (milliseconds) .br Value must be between 1 and 10000 .TP \fB\-\-lowspeed\-timeout\fR arg (=30) Set time in number seconds that the transfer speed should be below the rate .br Set with \fB\-\-lowspeed\-rate\fR for it to considered too slow and aborted .TP \fB\-\-lowspeed\-rate\fR arg (=200) Set average transfer speed in bytes per second that the transfer should be below during time specified with \fB\-\-lowspeed\-timeout\fR for it to be considered too slow and aborted .TP \fB\-\-include\-hidden\-products\fR Include games that have been set hidden in account page .TP \fB\-\-size\-only\fR Don't check the hashes of the files whose size matches that on the server .TP \fB\-\-verbosity\fR arg (=0) Set message verbosity level .br \-1 = Less verbose .br 0 = Default .br 1 = Verbose .br 2 = Debug .TP \fB\-\-check\-free\-space\fR Check for available free space before starting download .TP \fB\-\-no\-fast\-status\-check\fR Don't use fast status check. .br Makes \fB\-\-status\fR much slower but able to catch corrupted files by calculating local file hash for all files. .TP \fB\-\-trust\-api\-for\-extras\fR Trust API responses for extras to be correct. .TP \fB\-\-interface\fR arg Perform operations using a specified network interface .SS "Experimental:" .TP \fB\-\-galaxy\-install\fR arg Install game using product id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-show\-builds\fR arg Show game builds using product id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build. .br Lists available builds if build is not specified .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-download\-cloud\-saves\fR arg Download cloud saves using product\-id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-upload\-cloud\-saves\fR arg Upload cloud saves using product\-id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-show\-cloud\-saves\fR arg Show game cloud\-saves using product id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-show\-local\-cloud\-saves\fR arg Show local cloud\-saves using product id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-delete\-cloud\-saves\fR arg Delete cloud\-saves using product id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .TP \fB\-\-galaxy\-platform\fR arg (=w) Select platform .br Windows = w|win|windows .br Mac = m|mac|osx .br Linux = l|lin|linux .br .TP \fB\-\-galaxy\-language\fR arg (=en) Select language .br See \fBLANGUAGES\fR section for available values .TP \fB\-\-galaxy\-arch\fR arg (=x64) Select architecture .br 32\-bit = 32|x86|32bit|32\-bit .br 64\-bit = 64|x64|64bit|64\-bit .TP \fB\-\-galaxy\-no\-dependencies\fR Don't download dependencies during \fB\-\-galaxy\-install\fR .TP \fB\-\-subdir\-galaxy\-install\fR arg (=%install_dir%) Set subdirectory for galaxy install .sp 1 Templates: .br \- %install_dir% = Installation directory from Galaxy API response .br \- %gamename% = Game name .br \- %title% = Title of the game .br \- %product_id% = Product id of the game .br \- %install_dir_stripped% = %install_dir% with some characters stripped .br \- %title_stripped% = %title% with some characters stripped .sp 1 "stripped" means that every character that doesn't match the following list is removed: .br > alphanumeric .br > space .br > \- _ . ( ) [ ] { } .TP \fB\-\-galaxy\-cdn\-priority\fR arg (=edgecast,akamai_edgecast_proxy,fastly) Set priority for used CDNs .br Use \-\-galaxy\-list\-cdns to list available CDNs .br Set priority by separating values with "," .TP \fB\-\-galaxy\-list-cdns\fR arg List available CDNs for a game using product id [product_id/build] or gamename regex [gamename/build] .br Build (build index or build id) is used to select a build and defaults to 0 if not specified. .br Example: 12345/2 selects build 2 for product 12345 .br .TP \fB\-\-galaxy\-lowercase\-path\fR arg Make filepath lowercase for Windows game files .br .SH LANGUAGES Languages available to select with \fB\-\-language\fR and \fB\-\-galaxy\-language\fR options .br English = en|eng|english|en[_\-]US .br German = de|deu|ger|german|de[_\-]DE .br French = fr|fra|fre|french|fr[_\-]FR .br Polish = pl|pol|polish|pl[_\-]PL .br Russian = ru|rus|russian|ru[_\-]RU .br Chinese = cn|zh|zho|chi|chinese|zh[_\-](CN|Hans) .br Czech = cz|cs|ces|cze|czech|cs[_\-]CZ .br Spanish = es|spa|spanish|es[_\-]ES .br Hungarian = hu|hun|hungarian|hu[_\-]HU .br Italian = it|ita|italian|it[_\-]IT .br Japanese = jp|ja|jpn|japanese|ja[_\-]JP .br Turkish = tr|tur|turkish|tr[_\-]TR .br Portuguese = pt|por|portuguese|pt[_\-]PT .br Korean = ko|kor|korean|ko[_\-]KR .br Dutch = nl|nld|dut|dutch|nl[_\-]NL .br Swedish = sv|swe|swedish|sv[_\-]SE .br Norwegian = no|nor|norwegian|nb[_\-]no|nn[_\-]NO .br Danish = da|dan|danish|da[_\-]DK .br Finnish = fi|fin|finnish|fi[_\-]FI .br Brazilian Portuguese = br|pt_br|pt\-br|ptbr|brazilian_portuguese .br Slovak = sk|slk|slo|slovak|sk[_\-]SK .br Bulgarian = bl|bg|bul|bulgarian|bg[_\-]BG .br Ukrainian = uk|ukr|ukrainian|uk[_\-]UA .br Spanish (Latin American) = es_mx|es\-mx|esmx|es\-419|spanish_latin_american .br Arabic = ar|ara|arabic|ar[_\-][A\-Z]{2} .br Romanian = ro|ron|rum|romanian|ro[_\-][RM]O .br Hebrew = he|heb|hebrew|he[_\-]IL .br Thai = th|tha|thai|th[_\-]TH .SH BLACKLIST .fi \fI$XDG_CONFIG_HOME/lgogdownloader/blacklist.txt\fP .br Allows user to specify individual files that should not be downloaded or mentioned as orphans. See also \fBIGNORELIST\fP for ignoring files during orphan check. .sp 1 Each line in the file specifies one blacklist expression, except for empty lines and lines starting with #. .br First few characters specify blacklist item type and flags. So far, only regular expression (perl variant) are supported, so each line must start with "Rp" characters. After a space comes the expression itself. Expressions are matched against file path relative to what was specified as \fI--directory\fP. \fIExample black list\fP .br # used to store manually downloaded mods/patches/maps/, don't mention it as orphans .br Rp ^[^/]*/manual/.* .br # included with every *divinity game, once is enough .br Rp beyond_divinity/extras/bd_ladymageknight\.zip .br Rp divinity_2_developers_cut/extras/divinity_2_ladymageknight\.zip .sp # extra 6GB is A LOT of space if you don't actually plan to mod your game .br Rp the_witcher_2/extras/the_witcher_2_redkit\.zip .br Rp the_witcher_2/extras/extras_pack_3_hu_pl_ru_tr_zh_\.zip .br Rp the_witcher_2/extras/extras_pack_2_fr_it_jp_\.zip .SH IGNORELIST .fi \fI$XDG_CONFIG_HOME/lgogdownloader/ignorelist.txt\fP .br Allows user to specify individual files that should not be mentioned as orphans. .br Basically the same as blacklist but is used only when checking for orphaned files. .br See \fBBLACKLIST\fP for details about formatting. .SH PRIORITIES Separating values with "," when using \fBlanguage\fP and \fBplatform\fP switches enables a priority-based mode: only the first matching one will be downloaded. .PP For example, setting \fBlanguage\fP to \fBfr+en\fP means both French and English will be downloaded (if available) for all games. Setting \fBlanguage\fP to \fBfr,en\fP means that the French version (and only that one) will be downloaded if available, and if not, the English version will be downloaded. .PP You're allowed to "stack" codes in the priority string if needed. If you set \fBlanguage\fP to \fBes+fr,en\fP it means it'll download both Spanish (es) and French (fr) versions if they are available, and the English (en) one only if none of French and Spanish are available. .SH AVAILABILITY The latest version of this distribution is available from \fIhttps://github.com/Sude-/lgogdownloader\fP .SH FILES .fi .TP \fI$XDG_CONFIG_HOME/lgogdownloader/\fP Storage for configuration files and cookies .br If \fB$XDG_CONFIG_HOME\fP is not set, it will use \fI$HOME/.config/lgogdownloader/\fP. .TP \fI$XDG_CACHE_HOME/lgogdownloader/xml/\fP Storage for XML files .br If \fB$XDG_CACHE_HOME\fP is not set, it will use \fI$HOME/.cache/lgogdownloader/xml/\fP. .TP \fI$XDG_CONFIG_HOME/lgogdownloader/blacklist.txt\fP Allows user to specify individual files that should not be downloaded. .br It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader. .TP \fI$XDG_CONFIG_HOME/lgogdownloader/ignorelist.txt\fP Allows user to specify individual files that should not be mentioned as orphans. The file has the same format and interpretation as a blacklist. .br It doesn't have to exist, but if it does exist, it must be readable to lgogdownloader. .TP \fI$XDG_CONFIG_HOME/lgogdownloader/gamespecific/gamename.conf\fP JSON formatted file. Sets game specific settings for \fBgamename\fP. .br Allowed settings are \fBlanguage\fP, \fBplatform\fP, \fBinclude\fP, \fBignore-dlc-count\fP, \fBsubdirectories\fP, \fBdirectory\fP, \fBsubdir-game\fP, \fBsubdir-installers\fP, \fBsubdir-extras\fP, \fBsubdir-patches\fP, \fBsubdir-language-packs\fP and \fBsubdir-dlc\fP. .br Must be in the following format: .br { "language" : , "platform" : , "include" : , "ignore-dlc-count" : , "subdirectories" : , "directory" : , "subdir-game" : , "subdir-installers" : , "subdir-extras" : , "subdir-patches" : , "subdir-language-packs" : , "subdir-dlc" : .br } .TP \fI$XDG_CONFIG_HOME/lgogdownloader/transformations.json\fP JSON formatted file. Used to transform gamenames. .br Must be in the following format: .br { : { "regex" : , "replacement" : , "exceptions" : [ , , ], }, : { "regex" : , "replacement" : , }, .br } .br Member names are used to match the gamename (regex). Member names must be unique. .br For example if the file contains 2 rules with "^x" then only the last one is applied. However if user really wants multiple different rules for everything starting with "x" then adding wild wildcard matches makes them unique ("^x", "^x.*", "^x.*.*") .br If it matches then \fBregex\fP is used for the actual replacement using the value in \fBreplacement\fP. .br "\fBexceptions\fP" is an optional array of gamenames excluded from the rule. These are matched exactly, no regex. .br \fBExample:\fP .br match all games beginning with "\fBb\fP" and if they end with "\fB_the\fP" then remove "\fB_the\fP" at the end and prefix it with "\fBthe_\fP" with exception of "\fBblackwell_epiphany_the\fP" .br { "^b" : { "regex" : "(.*)_the$", "replacement" : "the_\\\\1", "exceptions" : [ "blackwell_epiphany_the", ], }, .br } lgogdownloader-3.17/src/000077500000000000000000000000001476654310300152535ustar00rootroot00000000000000lgogdownloader-3.17/src/blacklist.cpp000066400000000000000000000044631476654310300177360ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "blacklist.h" #include "config.h" #include "util.h" #include #include enum { BLFLAG_RX = 1 << 0, BLFLAG_PERL = 1 << 1 }; void Blacklist::initialize(const std::vector& lines) { int linenr = 1; for (auto it = lines.begin(); it != lines.end(); ++it, ++linenr) { BlacklistItem item; const std::string& s = *it; if (s.length() == 0 || s[0] == '#') continue; std::size_t i; for (i = 0; i < s.length() && s[i] != '\x20'; ++i) { switch (s[i]) { case 'R': item.flags |= BLFLAG_RX; break; case 'p': item.flags |= BLFLAG_PERL; break; default: std::cout << "unknown flag '" << s[i] << "' in blacklist line " << linenr << std::endl; break; } } ++i; if (i == s.length()) { std::cout << "empty expression in blacklist line " << linenr << std::endl; continue; } if (item.flags & BLFLAG_RX) { boost::regex::flag_type rx_flags = boost::regex::normal; // we only support perl-like syntax for now, which is boost default (normal). Add further flag processing // here if that changes. rx_flags |= boost::regex::nosubs; item.linenr = linenr; item.source.assign(s.substr(i).c_str()); item.regex.assign(item.source, rx_flags); blacklist_.push_back(std::move(item)); } else { std::cout << "unknown expression type in blacklist line " << linenr << std::endl; } } } bool Blacklist::isBlacklisted(const std::string& path) { for (auto it = blacklist_.begin(); it != blacklist_.end(); ++it) { const BlacklistItem& item = *it; if (item.flags & BLFLAG_RX && boost::regex_search(path, item.regex)) return true; } return false; } lgogdownloader-3.17/src/downloader.cpp000066400000000000000000007646751476654310300201460ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "downloader.h" #include "util.h" #include "globals.h" #include "downloadinfo.h" #include "message.h" #include "ziputil.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace bptime = boost::posix_time; struct cloudSaveFile { boost::posix_time::ptime lastModified; unsigned long long fileSize; std::string path; std::string location; }; std::vector Globals::vOwnedGamesIds; std::vector vDownloadInfo; ThreadSafeQueue dlQueue; ThreadSafeQueue dlCloudSaveQueue; ThreadSafeQueue msgQueue; ThreadSafeQueue createXMLQueue; ThreadSafeQueue gameItemQueue; ThreadSafeQueue gameDetailsQueue; ThreadSafeQueue dlQueueGalaxy; ThreadSafeQueue dlQueueGalaxy_MojoSetupHack; std::mutex mtx_create_directories; // Mutex for creating directories in Downloader::processDownloadQueue std::atomic iTotalRemainingBytes(0); std::string username() { auto user = std::getenv("USER"); return user ? user : std::string(); } void dirForEachHelper(const boost::filesystem::path &location, std::function &f) { boost::filesystem::directory_iterator begin { location }; boost::filesystem::directory_iterator end; for(boost::filesystem::directory_iterator curr_dir { begin }; curr_dir != end; ++curr_dir) { if(boost::filesystem::is_directory(*curr_dir)) { dirForEachHelper(*curr_dir, f); } else { f(curr_dir); } } } void dirForEach(const std::string &location, std::function &&f) { dirForEachHelper(location, f); } bool whitelisted(const std::string &path) { auto &whitelist = Globals::globalConfig.cloudWhiteList; auto &blacklist = Globals::globalConfig.cloudBlackList; // Check if path is whitelisted if(!whitelist.empty()) { return std::any_of(std::begin(whitelist), std::end(whitelist), [&path](const std::string &whitelisted) { return path.rfind(whitelisted, 0) == 0 && (path.size() == whitelisted.size() || path[whitelisted.size()] == '/'); }); } // Check if blacklisted if(!blacklist.empty()) { return !std::any_of(std::begin(blacklist), std::end(blacklist), [&path](const std::string &blacklisted) { return path.rfind(blacklisted, 0) == 0 && (path.size() == blacklisted.size() || path[blacklisted.size()] == '/'); }); } return true; } Downloader::Downloader() { if (Globals::globalConfig.bLogin) { if (boost::filesystem::exists(Globals::globalConfig.curlConf.sCookiePath)) if (!boost::filesystem::remove(Globals::globalConfig.curlConf.sCookiePath)) std::cerr << "Failed to delete " << Globals::globalConfig.curlConf.sCookiePath << std::endl; if (boost::filesystem::exists(Globals::galaxyConf.getFilepath())) if (!boost::filesystem::remove(Globals::galaxyConf.getFilepath())) std::cerr << "Failed to delete " << Globals::galaxyConf.getFilepath() << std::endl; } this->resume_position = 0; this->retries = 0; // Initialize curl and set curl options curlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(curlhandle, Globals::globalConfig.curlConf); curl_easy_setopt(curlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallback); curl_easy_setopt(curlhandle, CURLOPT_XFERINFODATA, this); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(curlhandle, CURLOPT_READFUNCTION, Downloader::readData); // Create new GOG website handle gogWebsite = new Website(); progressbar = new ProgressBar(Globals::globalConfig.bUnicode, Globals::globalConfig.bColor); if (boost::filesystem::exists(Globals::galaxyConf.getFilepath())) { Json::Value json = Util::readJsonFile(Globals::galaxyConf.getFilepath()); if (!json.isMember("expires_at")) { std::time_t last_modified = boost::filesystem::last_write_time(Globals::galaxyConf.getFilepath()); Json::Value::LargestInt expires_in = json["expires_in"].asLargestInt(); json["expires_at"] = expires_in + last_modified; } Globals::galaxyConf.setJSON(json); } gogGalaxy = new galaxyAPI(Globals::globalConfig.curlConf); } Downloader::~Downloader() { if (Globals::globalConfig.bReport) if (this->report_ofs) this->report_ofs.close(); if (!gogGalaxy->isTokenExpired()) this->saveGalaxyJSON(); delete progressbar; delete gogGalaxy; delete gogWebsite; curl_easy_cleanup(curlhandle); // Make sure that cookie file is only readable/writable by owner if (!Globals::globalConfig.bRespectUmask) { Util::setFilePermissions(Globals::globalConfig.curlConf.sCookiePath, boost::filesystem::owner_read | boost::filesystem::owner_write); } } /* Login check returns false if not logged in returns true if logged in */ bool Downloader::isLoggedIn() { bool bIsLoggedIn = false; bool bWebsiteIsLoggedIn = gogWebsite->IsLoggedIn(); bool bGalaxyIsLoggedIn = !gogGalaxy->isTokenExpired(); if (!bGalaxyIsLoggedIn) { if (gogGalaxy->refreshLogin()) bGalaxyIsLoggedIn = true; } if (bWebsiteIsLoggedIn && bGalaxyIsLoggedIn) bIsLoggedIn = true; return bIsLoggedIn; } /* Initialize the downloader returns 0 if failed returns 1 if successful */ int Downloader::init() { if (!gogGalaxy->init()) { if (gogGalaxy->refreshLogin()) { this->saveGalaxyJSON(); } else return 0; } if (!Globals::galaxyConf.getJSON().empty()) { if (Globals::galaxyConf.isExpired()) { // Access token has expired, refresh if (gogGalaxy->refreshLogin()) { this->saveGalaxyJSON(); } } } if (Globals::globalConfig.bReport && (Globals::globalConfig.bDownload || Globals::globalConfig.bRepair)) { this->report_ofs.open(Globals::globalConfig.sReportFilePath); if (!this->report_ofs) { Globals::globalConfig.bReport = false; std::cerr << "Failed to create " << Globals::globalConfig.sReportFilePath << std::endl; return 0; } } return 1; } /* Login returns 0 if login fails returns 1 if successful */ int Downloader::login() { std::string email; std::string password; bool headless = false; bool bForceGUI = false; #ifdef USE_QT_GUI_LOGIN bForceGUI = Globals::globalConfig.bForceGUILogin; #endif if (!Globals::globalConfig.sEmail.empty() && !Globals::globalConfig.sPassword.empty()) { email = Globals::globalConfig.sEmail; password = Globals::globalConfig.sPassword; } else if (!(bForceGUI || Globals::globalConfig.bForceBrowserLogin)) { if (!isatty(STDIN_FILENO)) { /* Attempt to read this stuff from elsewhere */ bool cookie_gone = !(boost::filesystem::exists(Globals::globalConfig.curlConf.sCookiePath)); bool tokens_gone = !(boost::filesystem::exists(Globals::globalConfig.sConfigDirectory + "/galaxy_tokens.json")); std::cout << Globals::globalConfig.curlConf.sCookiePath << std::endl; std::cout << (Globals::globalConfig.sConfigDirectory + "/galaxy_tokens.json") << std::endl; if(cookie_gone || tokens_gone) { std::cerr << "Unable to read email and password" << std::endl; return 0; } else headless = true; } else { std::cerr << "Email: "; std::getline(std::cin,email); std::cerr << "Password: "; struct termios termios_old, termios_new; tcgetattr(STDIN_FILENO, &termios_old); // Get current terminal attributes termios_new = termios_old; termios_new.c_lflag &= ~ECHO; // Set ECHO off tcsetattr(STDIN_FILENO, TCSANOW, &termios_new); // Set terminal attributes std::getline(std::cin, password); tcsetattr(STDIN_FILENO, TCSANOW, &termios_old); // Restore old terminal attributes std::cerr << std::endl; } } if ((email.empty() || password.empty()) && !(Globals::globalConfig.bForceBrowserLogin || headless || bForceGUI) ) { std::cerr << "Email and/or password empty" << std::endl; return 0; } // Login to website and Galaxy API if (Globals::globalConfig.bLogin) { // Delete old cookies if (boost::filesystem::exists(Globals::globalConfig.curlConf.sCookiePath)) if (!boost::filesystem::remove(Globals::globalConfig.curlConf.sCookiePath)) std::cerr << "Failed to delete " << Globals::globalConfig.curlConf.sCookiePath << std::endl; int iLoginResult = gogWebsite->Login(email, password); if (iLoginResult < 1) { std::cerr << "Galaxy: Login failed" << std::endl; return 0; } else { std::cerr << "Galaxy: Login successful" << std::endl; if (!Globals::galaxyConf.getJSON().empty()) { this->saveGalaxyJSON(); } } if (gogWebsite->IsLoggedIn()) { std::cerr << "HTTP: Login successful" << std::endl; } else { std::cerr << "HTTP: Login failed" << std::endl; return 0; } } return 1; } void Downloader::checkNotifications() { Json::Value userData = gogGalaxy->getUserData(); if (userData.empty()) { std::cout << "Empty JSON response" << std::endl; return; } if (!userData.isMember("updates")) { std::cout << "Invalid JSON response" << std::endl; return; } std::cout << "New forum replies: " << userData["updates"]["messages"].asInt() << std::endl; std::cout << "Updated games: " << userData["updates"]["products"].asInt() << std::endl; std::cout << "Unread chat messages: " << userData["updates"]["unreadChatMessages"].asInt() << std::endl; std::cout << "Pending friend requests: " << userData["updates"]["pendingFriendRequests"].asInt() << std::endl; } void Downloader::clearUpdateNotifications() { Json::Value userData = gogGalaxy->getUserData(); if (userData.empty()) { return; } if (!userData.isMember("updates")) { return; } if (userData["updates"]["products"].asInt() < 1) { std::cout << "No updates" << std::endl; return; } Globals::globalConfig.bUpdated = true; this->getGameList(); for (unsigned int i = 0; i < gameItems.size(); ++i) { // Getting game details should remove the update flag std::cerr << "\033[KClearing update flags " << i+1 << " / " << gameItems.size() << "\r" << std::flush; Json::Value details = gogWebsite->getGameDetailsJSON(gameItems[i].id); } std::cerr << std::endl; } void Downloader::getGameList() { gameItems = gogWebsite->getGames(); } /* Get detailed info about the games returns 0 if successful returns 1 if fails */ int Downloader::getGameDetails() { // Set default game specific directory options to values from config DirectoryConfig dirConfDefault = Globals::globalConfig.dirConf; if (Globals::globalConfig.bUseCache && !Globals::globalConfig.bUpdateCache) { int result = this->loadGameDetailsCache(); if (result == 0) { for (unsigned int i = 0; i < this->games.size(); ++i) { gameSpecificConfig conf; conf.dirConf = dirConfDefault; Util::getGameSpecificConfig(games[i].gamename, &conf); this->games[i].makeFilepaths(conf.dirConf); } return 0; } else { if (result == 1) { std::cerr << "Cache doesn't exist." << std::endl; std::cerr << "Create cache with --update-cache" << std::endl; } else if (result == 3) { std::cerr << "Cache is too old." << std::endl; std::cerr << "Update cache with --update-cache or use bigger --cache-valid" << std::endl; } else if (result == 5) { std::cerr << "Cache version doesn't match current version." << std::endl; std::cerr << "Update cache with --update-cache" << std::endl; } return 1; } } if (gameItems.empty()) this->getGameList(); if (!gameItems.empty()) { for (unsigned int i = 0; i < gameItems.size(); ++i) { gameItemQueue.push(gameItems[i]); } // Create threads unsigned int threads = std::min(Globals::globalConfig.iInfoThreads, static_cast(gameItemQueue.size())); std::vector vThreads; for (unsigned int i = 0; i < threads; ++i) { DownloadInfo dlInfo; dlInfo.setStatus(DLSTATUS_NOTSTARTED); vDownloadInfo.push_back(dlInfo); vThreads.push_back(std::thread(Downloader::getGameDetailsThread, Globals::globalConfig, i)); } unsigned int dl_status = DLSTATUS_NOTSTARTED; while (dl_status != DLSTATUS_FINISHED) { dl_status = DLSTATUS_NOTSTARTED; // Print progress information once per 100ms std::this_thread::sleep_for(std::chrono::milliseconds(Globals::globalConfig.iProgressInterval)); std::cerr << "\033[J\r" << std::flush; // Clear screen from the current line down to the bottom of the screen // Print messages from message queue first Message msg; while (msgQueue.try_pop(msg)) { if (msg.getLevel() <= Globals::globalConfig.iMsgLevel) std::cerr << msg.getFormattedString(Globals::globalConfig.bColor, true) << std::endl; if (Globals::globalConfig.bReport) { this->report_ofs << msg.getTimestampString() << ": " << msg.getMessage() << std::endl; } } for (unsigned int i = 0; i < vDownloadInfo.size(); ++i) { unsigned int status = vDownloadInfo[i].getStatus(); dl_status |= status; } std::cerr << "Getting game info " << (gameItems.size() - gameItemQueue.size()) << " / " << gameItems.size() << std::endl; if (dl_status != DLSTATUS_FINISHED) { std::cerr << "\033[1A\r" << std::flush; // Move cursor up by 1 row } } // Join threads for (unsigned int i = 0; i < vThreads.size(); ++i) vThreads[i].join(); vThreads.clear(); vDownloadInfo.clear(); gameDetails details; while (gameDetailsQueue.try_pop(details)) { this->games.push_back(details); } std::sort(this->games.begin(), this->games.end(), [](const gameDetails& i, const gameDetails& j) -> bool { return i.gamename < j.gamename; }); } return 0; } int Downloader::listGames() { if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_GAMES || Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_TRANSFORMATIONS) { if (gameItems.empty()) this->getGameList(); for (unsigned int i = 0; i < gameItems.size(); ++i) { std::string gamename = gameItems[i].name; if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_TRANSFORMATIONS) { std::cout << gamename << " -> " << Util::transformGamename(gamename) << std::endl; } else { if (gameItems[i].updates > 0) { gamename += " [" + std::to_string(gameItems[i].updates) + "]"; std::string color = gameItems[i].isnew ? "01;34" : "32"; if (Globals::globalConfig.bColor) gamename = "\033[" + color + "m" + gamename + "\033[0m"; } else { if (Globals::globalConfig.bColor && gameItems[i].isnew) gamename = "\033[01;34m" + gamename + "\033[0m"; } std::cout << gamename << std::endl; for (unsigned int j = 0; j < gameItems[i].dlcnames.size(); ++j) std::cout << "+> " << gameItems[i].dlcnames[j] << std::endl; } } } else if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_TAGS) { std::map tags; tags = gogWebsite->getTags(); if (!tags.empty()) { for (auto tag : tags) { std::cout << tag.first << " = " << tag.second << std::endl; } } } else if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_USERDATA) { Json::Value userdata; std::istringstream response(gogWebsite->getResponse("https://embed.gog.com/userData.json")); try { response >> userdata; } catch(const Json::Exception& exc) { std::cerr << "Failed to get user data" << std::endl; return 1; } std::cout << userdata << std::endl; } else if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_WISHLIST) { this->showWishlist(); } else { if (this->games.empty()) { int res = this->getGameDetails(); if (res > 0) return res; } if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_DETAILS_JSON) { Json::Value json(Json::arrayValue); for (auto game : this->games) json.append(game.getDetailsAsJson()); std::cout << json << std::endl; } else if (Globals::globalConfig.iListFormat == GlobalConstants::LIST_FORMAT_DETAILS_TEXT) { for (auto game : this->games) printGameDetailsAsText(game); } } return 0; } void Downloader::repair() { if (this->games.empty()) this->getGameDetails(); // Create a vector containing all game files std::vector vGameFiles; for (unsigned int i = 0; i < games.size(); ++i) { std::vector vec = games[i].getGameFileVector(); vGameFiles.insert(std::end(vGameFiles), std::begin(vec), std::end(vec)); } for (unsigned int i = 0; i < vGameFiles.size(); ++i) { gameSpecificConfig conf; conf.dlConf = Globals::globalConfig.dlConf; conf.dirConf = Globals::globalConfig.dirConf; Util::getGameSpecificConfig(vGameFiles[i].gamename, &conf); unsigned int type = vGameFiles[i].type; if (!(type & conf.dlConf.iInclude)) continue; std::string filepath = vGameFiles[i].getFilepath(); if (Globals::globalConfig.blacklist.isBlacklisted(filepath)) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cerr << "skipped blacklisted file " << filepath << std::endl; continue; } // Refresh Galaxy login if token is expired if (gogGalaxy->isTokenExpired()) { if (!gogGalaxy->refreshLogin()) { std::cerr << "Galaxy API failed to refresh login" << std::endl; break; } } Json::Value downlinkJson = gogGalaxy->getResponseJson(vGameFiles[i].galaxy_downlink_json_url); if (downlinkJson.empty()) { std::cerr << "Empty JSON response, skipping file" << std::endl; continue; } if (!downlinkJson.isMember("downlink")) { std::cerr << "Invalid JSON response, skipping file" << std::endl; continue; } std::string xml_url; if (downlinkJson.isMember("checksum")) if (!downlinkJson["checksum"].empty()) xml_url = downlinkJson["checksum"].asString(); // Get XML data std::string XML = ""; if (conf.dlConf.bRemoteXML && !xml_url.empty()) XML = gogGalaxy->getResponse(xml_url); // Repair bool bUseLocalXML = !conf.dlConf.bRemoteXML; // Use local XML data for extras if (XML.empty() && (type & GlobalConstants::GFTYPE_EXTRA)) bUseLocalXML = true; if (!XML.empty() || bUseLocalXML) { std::string url = downlinkJson["downlink"].asString(); std::cout << "Repairing file " << filepath << std::endl; this->repairFile(url, filepath, XML, vGameFiles[i].gamename); std::cout << std::endl; } } } void Downloader::download() { if (this->games.empty()) this->getGameDetails(); for (unsigned int i = 0; i < games.size(); ++i) { gameSpecificConfig conf; conf.dlConf = Globals::globalConfig.dlConf; conf.dirConf = Globals::globalConfig.dirConf; Util::getGameSpecificConfig(games[i].gamename, &conf); if (conf.dlConf.bSaveSerials && !games[i].serials.empty()) { std::string filepath = games[i].getSerialsFilepath(); this->saveSerials(games[i].serials, filepath); } if (conf.dlConf.bSaveLogo && !games[i].logo.empty()) { std::string filepath = games[i].getLogoFilepath(); this->downloadFile(games[i].logo, filepath, "", games[i].gamename); } if (conf.dlConf.bSaveIcon && !games[i].icon.empty()) { std::string filepath = games[i].getIconFilepath(); this->downloadFile(games[i].icon, filepath, "", games[i].gamename); } if (conf.dlConf.bSaveChangelogs && !games[i].changelog.empty()) { std::string filepath = games[i].getChangelogFilepath(); this->saveChangelog(games[i].changelog, filepath); } if (conf.dlConf.bSaveGameDetailsJson && !games[i].gameDetailsJson.empty()) { std::string filepath = games[i].getGameDetailsJsonFilepath(); this->saveJsonFile(games[i].gameDetailsJson, filepath); } if (conf.dlConf.bSaveProductJson && !games[i].productJson.empty()) { std::string filepath = games[i].getProductJsonFilepath(); this->saveJsonFile(games[i].productJson, filepath); } if ((conf.dlConf.iInclude & GlobalConstants::GFTYPE_DLC) && !games[i].dlcs.empty()) { for (unsigned int j = 0; j < games[i].dlcs.size(); ++j) { if (conf.dlConf.bSaveSerials && !games[i].dlcs[j].serials.empty()) { std::string filepath = games[i].dlcs[j].getSerialsFilepath(); this->saveSerials(games[i].dlcs[j].serials, filepath); } if (conf.dlConf.bSaveLogo && !games[i].dlcs[j].logo.empty()) { std::string filepath = games[i].dlcs[j].getLogoFilepath(); this->downloadFile(games[i].dlcs[j].logo, filepath, "", games[i].dlcs[j].gamename); } if (conf.dlConf.bSaveIcon && !games[i].dlcs[j].icon.empty()) { std::string filepath = games[i].dlcs[j].getIconFilepath(); this->downloadFile(games[i].dlcs[j].icon, filepath, "", games[i].dlcs[j].gamename); } if (conf.dlConf.bSaveChangelogs && !games[i].dlcs[j].changelog.empty()) { std::string filepath = games[i].dlcs[j].getChangelogFilepath(); this->saveChangelog(games[i].dlcs[j].changelog, filepath); } if (conf.dlConf.bSaveProductJson && !games[i].dlcs[j].productJson.empty()) { std::string filepath = games[i].dlcs[j].getProductJsonFilepath(); this->saveJsonFile(games[i].dlcs[j].productJson, filepath); } } } auto vFiles = games[i].getGameFileVectorFiltered(conf.dlConf.iInclude); for (auto gf : vFiles) { dlQueue.push(gf); unsigned long long filesize = 0; try { filesize = std::stoll(gf.size); } catch (std::invalid_argument& e) { filesize = 0; } iTotalRemainingBytes.fetch_add(filesize); } } if (!dlQueue.empty()) { unsigned long long totalSizeBytes = iTotalRemainingBytes.load(); std::cout << "Total size: " << Util::makeSizeString(totalSizeBytes) << std::endl; if (Globals::globalConfig.dlConf.bFreeSpaceCheck) { boost::filesystem::path path = boost::filesystem::absolute(Globals::globalConfig.dirConf.sDirectory); while(!boost::filesystem::exists(path) && !path.empty()) { path = path.parent_path(); } if(boost::filesystem::exists(path) && !path.empty()) { boost::filesystem::space_info space = boost::filesystem::space(path); if (space.available < totalSizeBytes) { std::cerr << "Not enough free space in " << boost::filesystem::canonical(path) << " (" << Util::makeSizeString(space.available) << ")"<< std::endl; exit(1); } } } // Limit thread count to number of items in download queue unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlQueue.size())); // Create download threads std::vector vThreads; for (unsigned int i = 0; i < iThreads; ++i) { DownloadInfo dlInfo; dlInfo.setStatus(DLSTATUS_NOTSTARTED); vDownloadInfo.push_back(dlInfo); vThreads.push_back(std::thread(Downloader::processDownloadQueue, Globals::globalConfig, i)); } this->printProgress(dlQueue); // Join threads for (unsigned int i = 0; i < vThreads.size(); ++i) vThreads[i].join(); vThreads.clear(); vDownloadInfo.clear(); } // Create xml data for all files in the queue if (!createXMLQueue.empty()) { std::cout << "Starting XML creation" << std::endl; gameFile gf; while (createXMLQueue.try_pop(gf)) { std::string xml_directory = Globals::globalConfig.sXMLDirectory + "/" + gf.gamename; Util::createXML(gf.getFilepath(), Globals::globalConfig.iChunkSize, xml_directory); } } } // Download a file, resume if possible CURLcode Downloader::downloadFile(const std::string& url, const std::string& filepath, const std::string& xml_data, const std::string& gamename) { CURLcode res = CURLE_RECV_ERROR; // assume network error bool bResume = false; FILE *outfile; off_t offset=0; // Get directory from filepath boost::filesystem::path pathname = filepath; pathname = boost::filesystem::absolute(pathname, boost::filesystem::current_path()); std::string directory = pathname.parent_path().string(); std::string filenameXML = pathname.filename().string() + ".xml"; std::string xml_directory; if (!gamename.empty()) xml_directory = Globals::globalConfig.sXMLDirectory + "/" + gamename; else xml_directory = Globals::globalConfig.sXMLDirectory; // Using local XML data for version check before resuming boost::filesystem::path local_xml_file; local_xml_file = xml_directory + "/" + filenameXML; bool bSameVersion = true; // assume same version bool bLocalXMLExists = boost::filesystem::exists(local_xml_file); // This is additional check to see if remote xml should be saved to speed up future version checks if (!xml_data.empty()) { std::string localHash = this->getLocalFileHash(filepath, gamename); // Do version check if local hash exists if (!localHash.empty()) { tinyxml2::XMLDocument remote_xml; remote_xml.Parse(xml_data.c_str()); tinyxml2::XMLElement *fileElemRemote = remote_xml.FirstChildElement("file"); if (fileElemRemote) { std::string remoteHash = fileElemRemote->Attribute("md5"); if (remoteHash != localHash) bSameVersion = false; } } } // Check that directory exists and create subdirectories boost::filesystem::path path = directory; if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { std::cerr << path << " is not directory" << std::endl; return res; } } else { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; return res; } } // Check if file exists if ((outfile=fopen(filepath.c_str(), "r"))!=NULL) { if (bSameVersion) { // Check if file is complete so we can skip it instead of resuming if (!xml_data.empty()) { off_t filesize_xml; off_t filesize_local = boost::filesystem::file_size(filepath); tinyxml2::XMLDocument remote_xml; remote_xml.Parse(xml_data.c_str()); tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); if (fileElem) { std::string total_size = fileElem->Attribute("total_size"); try { filesize_xml = std::stoull(total_size); } catch (std::invalid_argument& e) { filesize_xml = 0; } if (filesize_local == filesize_xml) { std::cout << "Skipping complete file: " + filepath << std::endl; fclose(outfile); // Save remote XML if (!xml_data.empty()) { if ((bLocalXMLExists && (!bSameVersion || Globals::globalConfig.bRepair)) || !bLocalXMLExists) { // Check that directory exists and create subdirectories boost::filesystem::path path = xml_directory; if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { std::cerr << path << " is not directory" << std::endl; } } else { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; } } std::ofstream ofs(local_xml_file.string().c_str()); if (ofs) { ofs << xml_data; ofs.close(); } else { std::cerr << "Can't create " << local_xml_file.string() << std::endl; } } } res = CURLE_OK; return res; } } } // File exists, resume if ((outfile = freopen(filepath.c_str(), "r+", outfile))!=NULL ) { bResume = true; fseek(outfile, 0, SEEK_END); // use ftello to support large files on 32 bit platforms offset = ftello(outfile); curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM_LARGE, offset); this->resume_position = offset; } else { std::cerr << "Failed to reopen " << filepath << std::endl; return res; } } else { // File exists but is not the same version fclose(outfile); std::cerr << "Remote file is different, renaming local file" << std::endl; std::string date_old = "." + bptime::to_iso_string(bptime::second_clock::local_time()) + ".old"; boost::filesystem::path new_name = filepath + date_old; // Rename old file by appending date and ".old" to filename boost::system::error_code ec; boost::filesystem::rename(pathname, new_name, ec); // Rename the file if (ec) { std::cerr << "Failed to rename " << filepath << " to " << new_name.string() << std::endl; std::cerr << "Skipping file" << std::endl; return res; } else { // Create new file if ((outfile=fopen(filepath.c_str(), "w"))!=NULL) { curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM, 0); // start downloading from the beginning of file this->resume_position = 0; } else { std::cerr << "Failed to create " << filepath << std::endl; return res; } } } } else { // File doesn't exist, create new file if ((outfile=fopen(filepath.c_str(), "w"))!=NULL) { curl_easy_setopt(curlhandle, CURLOPT_RESUME_FROM, 0); // start downloading from the beginning of file this->resume_position = 0; } else { std::cerr << "Failed to create " << filepath << std::endl; return res; } } // Save remote XML if (!xml_data.empty()) { if ((bLocalXMLExists && (!bSameVersion || Globals::globalConfig.bRepair)) || !bLocalXMLExists) { // Check that directory exists and create subdirectories boost::filesystem::path path = xml_directory; if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { std::cerr << path << " is not directory" << std::endl; } } else { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; } } std::ofstream ofs(local_xml_file.string().c_str()); if (ofs) { ofs << xml_data; ofs.close(); } else { std::cerr << "Can't create " << local_xml_file.string() << std::endl; } } } curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, outfile); curl_easy_setopt(curlhandle, CURLOPT_FILETIME, 1L); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0); res = this->beginDownload(); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); fclose(outfile); // Download failed and was not a resume attempt so delete the file if (res != CURLE_OK && res != CURLE_PARTIAL_FILE && !bResume && res != CURLE_OPERATION_TIMEDOUT) { boost::filesystem::path path = filepath; if (boost::filesystem::exists(path)) if (!boost::filesystem::remove(path)) std::cerr << "Failed to delete " << path << std::endl; } if (Globals::globalConfig.bReport) { std::string status = static_cast(curl_easy_strerror(res)); if (bResume && res == CURLE_RANGE_ERROR) // CURLE_RANGE_ERROR on resume attempts is not an error that user needs to know about status = "No error"; std::string report_line = "Downloaded [" + status + "] " + filepath; this->report_ofs << report_line << std::endl; } // Retry partially downloaded file // Retry if we aborted the transfer due to low speed limit if ((res == CURLE_PARTIAL_FILE || res == CURLE_OPERATION_TIMEDOUT || res == CURLE_RECV_ERROR) && (this->retries < Globals::globalConfig.iRetries) ) { this->retries++; std::cerr << std::endl << "Retry " << this->retries << "/" << Globals::globalConfig.iRetries; if (res == CURLE_PARTIAL_FILE) std::cerr << " (partial download)"; else if (res == CURLE_OPERATION_TIMEDOUT) std::cerr << " (timeout)"; else if (res == CURLE_RECV_ERROR) std::cerr << " (failed receiving network data)"; std::cerr << std::endl; res = this->downloadFile(url, filepath, xml_data, gamename); } else { this->retries = 0; // Reset retries counter // Set timestamp for downloaded file to same value as file on server long filetime = -1; CURLcode result = curl_easy_getinfo(curlhandle, CURLINFO_FILETIME, &filetime); if (result == CURLE_OK && filetime >= 0) { std::time_t timestamp = (std::time_t)filetime; try { boost::filesystem::last_write_time(filepath, timestamp); } catch(const boost::filesystem::filesystem_error& e) { std::cerr << e.what() << std::endl; } } } curl_easy_setopt(curlhandle, CURLOPT_FILETIME, 0L); return res; } // Repair file int Downloader::repairFile(const std::string& url, const std::string& filepath, const std::string& xml_data, const std::string& gamename) { int res = 0; FILE *outfile; off_t offset=0, from_offset, to_offset, filesize; std::string filehash; int chunks; std::vector chunk_from, chunk_to; std::vector chunk_hash; bool bParsingFailed = false; // Get filename boost::filesystem::path pathname = filepath; std::string filename = pathname.filename().string(); std::string xml_directory; if (!gamename.empty()) xml_directory = Globals::globalConfig.sXMLDirectory + "/" + gamename; else xml_directory = Globals::globalConfig.sXMLDirectory; std::string xml_file = xml_directory + "/" + filename + ".xml"; bool bFileExists = boost::filesystem::exists(pathname); bool bLocalXMLExists = boost::filesystem::exists(xml_file); tinyxml2::XMLDocument xml; if (!xml_data.empty()) // Parse remote XML data { std::cout << "XML: Using remote file" << std::endl; xml.Parse(xml_data.c_str()); } else { // Parse local XML data std::cout << "XML: Using local file" << std::endl; if (!bLocalXMLExists) std::cout << "XML: File doesn't exist (" << xml_file << ")" << std::endl; xml.LoadFile(xml_file.c_str()); } // Check if file node exists in XML data tinyxml2::XMLElement *fileElem = xml.FirstChildElement("file"); if (!fileElem) { // File node doesn't exist std::cout << "XML: Parsing failed / not valid XML" << std::endl; if (Globals::globalConfig.bDownload) bParsingFailed = true; else return res; } else { // File node exists --> valid XML std::cout << "XML: Valid XML" << std::endl; filename = fileElem->Attribute("name"); filehash = fileElem->Attribute("md5"); std::stringstream(fileElem->Attribute("chunks")) >> chunks; std::stringstream(fileElem->Attribute("total_size")) >> filesize; //Iterate through all chunk nodes tinyxml2::XMLElement *chunkElem = fileElem->FirstChildElement("chunk"); while (chunkElem) { std::stringstream(chunkElem->Attribute("from")) >> from_offset; std::stringstream(chunkElem->Attribute("to")) >> to_offset; chunk_from.push_back(from_offset); chunk_to.push_back(to_offset); chunk_hash.push_back(chunkElem->GetText()); chunkElem = chunkElem->NextSiblingElement("chunk"); } std::cout << "XML: Parsing finished" << std::endl << std::endl << filename << std::endl << "\tMD5:\t" << filehash << std::endl << "\tChunks:\t" << chunks << std::endl << "\tSize:\t" << filesize << " bytes" << std::endl << std::endl; } // No local XML file and parsing failed. if (bParsingFailed && !bLocalXMLExists) { if (Globals::globalConfig.bDownload) { std::cout << "Downloading: " << filepath << std::endl; CURLcode result = this->downloadFile(url, filepath, xml_data, gamename); std::cout << std::endl; long int response_code = 0; if (result == CURLE_HTTP_RETURNED_ERROR) { curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); } if ( /* File doesn't exist so only accept if everything was OK */ (!bFileExists && result == CURLE_OK) || /* File exists so also accept CURLE_RANGE_ERROR and response code 416 */ (bFileExists && (result == CURLE_OK || result == CURLE_RANGE_ERROR || response_code == 416)) ) { bLocalXMLExists = boost::filesystem::exists(xml_file); // Check to see if downloadFile saved XML data if (Globals::globalConfig.dlConf.bAutomaticXMLCreation && !bLocalXMLExists) { std::cout << "Starting automatic XML creation" << std::endl; Util::createXML(filepath, Globals::globalConfig.iChunkSize, xml_directory); } res = 1; } } else { std::cout << "Can't repair file." << std::endl; } return res; } // Check if file exists if (bFileExists) { // File exists if ((outfile = fopen(filepath.c_str(), "r+"))!=NULL ) { fseek(outfile, 0, SEEK_END); // use ftello to support large files on 32 bit platforms offset = ftello(outfile); } else { std::cout << "Failed to open " << filepath << std::endl; return res; } } else { std::cout << "File doesn't exist " << filepath << std::endl; if (Globals::globalConfig.bDownload) { std::cout << "Downloading: " << filepath << std::endl; CURLcode result = this->downloadFile(url, filepath, xml_data, gamename); std::cout << std::endl; if (result == CURLE_OK) { if (Globals::globalConfig.dlConf.bAutomaticXMLCreation && bParsingFailed) { std::cout << "Starting automatic XML creation" << std::endl; Util::createXML(filepath, Globals::globalConfig.iChunkSize, xml_directory); } res = 1; } } return res; } // check if file sizes match if (offset != filesize) { std::cout << "Filesizes don't match" << std::endl << "Incomplete download or different version" << std::endl; fclose(outfile); if (Globals::globalConfig.bDownload) { std::cout << "Redownloading file" << std::endl; std::string date_old = "." + bptime::to_iso_string(bptime::second_clock::local_time()) + ".old"; boost::filesystem::path new_name = filepath + date_old; // Rename old file by appending date and ".old" to filename std::cout << "Renaming old file to " << new_name.string() << std::endl; boost::system::error_code ec; boost::filesystem::rename(pathname, new_name, ec); // Rename the file if (ec) { std::cout << "Failed to rename " << filepath << " to " << new_name.string() << std::endl; std::cout << "Skipping file" << std::endl; res = 0; } else { if (bLocalXMLExists) { std::cout << "Deleting old XML data" << std::endl; boost::filesystem::remove(xml_file, ec); // Delete old XML data if (ec) { std::cout << "Failed to delete " << xml_file << std::endl; } } CURLcode result = this->downloadFile(url, filepath, xml_data, gamename); std::cout << std::endl; if (result == CURLE_OK) { bLocalXMLExists = boost::filesystem::exists(xml_file); // Check to see if downloadFile saved XML data if (!bLocalXMLExists) { std::cout << "Starting automatic XML creation" << std::endl; Util::createXML(filepath, Globals::globalConfig.iChunkSize, xml_directory); } res = 1; } else { res = 0; } } } return res; } // Check all chunks int iChunksRepaired = 0; int iChunkRetryCount = 0; int iChunkRetryLimit = 3; bool bChunkRetryLimitReached = false; for (int i=0; ibeginDownload(); //begin chunk download std::cout << std::endl; if (Globals::globalConfig.bReport) iChunksRepaired++; i--; //verify downloaded chunk iChunkRetryCount++; if (iChunkRetryCount >= iChunkRetryLimit) { bChunkRetryLimitReached = true; } } else { std::cout << "OK\r" << std::flush; iChunkRetryCount = 0; // reset retry count bChunkRetryLimitReached = false; } free(chunk); res = 1; } std::cout << std::endl; fclose(outfile); if (Globals::globalConfig.bReport) { std::string report_line; if (bChunkRetryLimitReached) report_line = "Repair failed: " + filepath; else report_line = "Repaired [" + std::to_string(iChunksRepaired) + "/" + std::to_string(chunks) + "] " + filepath; this->report_ofs << report_line << std::endl; } if (bChunkRetryLimitReached) return res; // Set timestamp for downloaded file to same value as file on server long filetime = -1; CURLcode result = curl_easy_getinfo(curlhandle, CURLINFO_FILETIME, &filetime); if (result == CURLE_OK && filetime >= 0) { std::time_t timestamp = (std::time_t)filetime; try { boost::filesystem::last_write_time(filepath, timestamp); } catch(const boost::filesystem::filesystem_error& e) { std::cerr << e.what() << std::endl; } } curl_easy_setopt(curlhandle, CURLOPT_FILETIME, 0L); return res; } CURLcode Downloader::beginDownload() { this->TimeAndSize.clear(); this->timer.reset(); CURLcode result = curl_easy_perform(curlhandle); this->resume_position = 0; return result; } std::string Downloader::getResponse(const std::string& url) { std::ostringstream memory; std::string response; curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); int max_retries = std::min(3, Globals::globalConfig.iRetries); CURLcode result = Util::CurlHandleGetResponse(curlhandle, response, max_retries); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 0); if (result != CURLE_OK) { std::cout << curl_easy_strerror(result) << std::endl; if (result == CURLE_HTTP_RETURNED_ERROR) { long int response_code = 0; result = curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); std::cout << "HTTP ERROR: "; if (result == CURLE_OK) std::cout << response_code << " (" << url << ")" << std::endl; else std::cout << "failed to get error code: " << curl_easy_strerror(result) << " (" << url << ")" << std::endl; } } return response; } int Downloader::progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { // unused so lets prevent warnings and be more pedantic (void) ulnow; (void) ultotal; // on entry: dltotal - how much remains to download till the end of the file (bytes) // dlnow - how much was downloaded from the start of the program (bytes) int bar_length = 26; int min_bar_length = 5; Downloader* downloader = static_cast(clientp); double rate; // average download speed in B/s curl_off_t curl_rate; // trying to get rate and setting to NaN if it fails if (CURLE_OK != curl_easy_getinfo(downloader->curlhandle, CURLINFO_SPEED_DOWNLOAD_T, &curl_rate)) rate = std::numeric_limits::quiet_NaN(); else rate = static_cast(curl_rate); // (Shmerl): this flag is needed to catch the case before anything was downloaded on resume, // and there is no way to calculate the fraction, so we set to 0 (otherwise it'd be 1). // This is to prevent the progress bar from jumping to 100% and then to lower value. // It's visually better to jump from 0% to higher one. bool starting = ((0 == dlnow) && (0 == dltotal)); // (Shmerl): DEBUG: strange thing - when resuming a file which is already downloaded, dlnow is correctly 0.0 // but dltotal is 389.0! This messes things up in the progress bar not showing the very last bar as full. // enable this debug line to test the problem: // // printf("\r\033[0K dlnow: %0.2f, dltotal: %0.2f\r", dlnow, dltotal); fflush(stdout); return 0; // // For now making a quirky workaround and setting dltotal to 0.0 in that case. // It's probably better to find a real fix. if ((0 == dlnow) && (389 == dltotal)) dltotal = 0; // setting full dlwnow and dltotal curl_off_t offset = static_cast(downloader->getResumePosition()); if (offset>0) { dlnow += offset; dltotal += offset; } // Update progress bar every 100ms if (downloader->timer.getTimeBetweenUpdates()>=100 || dlnow == dltotal) { downloader->timer.reset(); int iTermWidth = Util::getTerminalWidth(); // 10 second average download speed // Don't use static value of 10 seconds because update interval depends on when and how often progress callback is called downloader->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast(dlnow))); if (downloader->TimeAndSize.size() > 100) // 100 * 100ms = 10s { downloader->TimeAndSize.pop_front(); time_t time_first = downloader->TimeAndSize.front().first; uintmax_t size_first = downloader->TimeAndSize.front().second; time_t time_last = downloader->TimeAndSize.back().first; uintmax_t size_last = downloader->TimeAndSize.back().second; rate = (size_last - size_first) / static_cast((time_last - time_first)); } std::string etastring = Util::makeEtaString((dltotal - dlnow), rate); // Create progressbar double fraction = starting ? 0.0 : static_cast(dlnow) / static_cast(dltotal); std::cout << Util::formattedString("\033[0K\r%3.0f%% ", fraction * 100); // Download rate unit conversion std::string rate_unit; if (rate > 1048576) // 1 MB { rate /= 1048576; rate_unit = "MB/s"; } else { rate /= 1024; rate_unit = "kB/s"; } std::string status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s\r", static_cast(dlnow)/1024/1024, static_cast(dltotal)/1024/1024, rate, rate_unit.c_str(), etastring.c_str()); int status_text_length = status_text.length() + 6; if ((status_text_length + bar_length) > iTermWidth) bar_length -= (status_text_length + bar_length) - iTermWidth; // Don't draw progressbar if length is less than min_bar_length if (bar_length >= min_bar_length) downloader->progressbar->draw(bar_length, fraction); std::cout << status_text << std::flush; } return 0; } size_t Downloader::writeData(void *ptr, size_t size, size_t nmemb, FILE *stream) { return fwrite(ptr, size, nmemb, stream); } size_t Downloader::readData(void *ptr, size_t size, size_t nmemb, FILE *stream) { return fread(ptr, size, nmemb, stream); } uintmax_t Downloader::getResumePosition() { return this->resume_position; } std::string Downloader::getSerialsFromJSON(const Json::Value& json) { std::ostringstream serials; if (!json.isMember("cdKey")) return std::string(); std::string cdkey = json["cdKey"].asString(); if (cdkey.empty()) return std::string(); if (cdkey.find("") == std::string::npos) { boost::regex expression(""); std::string text = boost::regex_replace(cdkey, expression, "\n"); serials << text << std::endl; } else { std::string xhtml = Util::htmlToXhtml(cdkey); tinyxml2::XMLDocument doc; doc.Parse(xhtml.c_str()); tinyxml2::XMLNode* node = doc.FirstChildElement("html"); while(node) { tinyxml2::XMLElement *element = node->ToElement(); const char* text = element->GetText(); if (text) serials << text << std::endl; node = Util::nextXMLNode(node); } } return serials.str(); } std::string Downloader::getChangelogFromJSON(const Json::Value& json) { std::string changelog; std::string title = "Changelog"; if (!json.isMember("changelog")) return std::string(); changelog = json["changelog"].asString(); if (changelog.empty()) return std::string(); if (json.isMember("title")) title = title + ": " + json["title"].asString(); changelog = "\n\n\n\n" + title + "\n\n" + changelog + "\n"; return changelog; } // Linear search. Good thing computers are fast and lists are small. static int isPresent(std::vector& list, const boost::filesystem::path& path, Blacklist& blacklist) { if(blacklist.isBlacklisted(path.native())) return false; for (unsigned int k = 0; k < list.size(); ++k) if (list[k].getFilepath() == path.native()) return true; return false; } void Downloader::checkOrphans() { // Always check everything when checking for orphaned files Globals::globalConfig.dlConf.iInclude = Util::getOptionValue("all", GlobalConstants::INCLUDE_OPTIONS); Globals::globalConfig.dlConf.iInstallerLanguage = Util::getOptionValue("all", GlobalConstants::LANGUAGES); Globals::globalConfig.dlConf.iInstallerPlatform = Util::getOptionValue("all", GlobalConstants::PLATFORMS); Globals::globalConfig.dlConf.vLanguagePriority.clear(); Globals::globalConfig.dlConf.vPlatformPriority.clear(); Config config = Globals::globalConfig; // Checking orphans after download. // Game details have already been retrieved but possibly filtered. // Therefore we need to clear game details and get them again. if (config.bDownload) { this->gameItems.clear(); this->games.clear(); } if (this->games.empty()) this->getGameDetails(); std::vector orphans; for (unsigned int i = 0; i < games.size(); ++i) { std::cerr << "Checking for orphaned files " << i+1 << " / " << games.size() << "\r" << std::flush; std::vector filepath_vector; try { std::vector paths; std::vector platformIds; platformIds.push_back(0); for (unsigned int j = 0; j < GlobalConstants::PLATFORMS.size(); ++j) { platformIds.push_back(GlobalConstants::PLATFORMS[j].id); } for (unsigned int j = 0; j < platformIds.size(); ++j) { std::string directory = games[i].makeCustomFilepath("", games[i], config.dirConf); boost::filesystem::path path (directory); if (boost::filesystem::exists(path)) { bool bDuplicate = false; for (unsigned int k = 0; k < paths.size(); ++k) { if (path == paths[k]) { bDuplicate = true; break; } } if (!bDuplicate) paths.push_back(path); } } for (unsigned int j = 0; j < paths.size(); ++j) { std::size_t pathlen = config.dirConf.sDirectory.length(); if (boost::filesystem::exists(paths[j])) { if (boost::filesystem::is_directory(paths[j])) { // Recursively iterate over files in directory boost::filesystem::recursive_directory_iterator end_iter; boost::filesystem::recursive_directory_iterator dir_iter(paths[j]); while (dir_iter != end_iter) { if (boost::filesystem::is_regular_file(dir_iter->status())) { std::string filepath = dir_iter->path().string(); if (config.ignorelist.isBlacklisted(filepath.substr(pathlen))) { if (config.iMsgLevel >= MSGLEVEL_VERBOSE) std::cerr << "skipped ignorelisted file " << filepath << std::endl; } else if (config.blacklist.isBlacklisted(filepath.substr(pathlen))) { if (config.iMsgLevel >= MSGLEVEL_VERBOSE) std::cerr << "skipped blacklisted file " << filepath << std::endl; } else { boost::regex expression(config.sOrphanRegex); // Limit to files matching the regex boost::match_results what; if (boost::regex_search(filepath, what, expression)) filepath_vector.push_back(dir_iter->path()); } } dir_iter++; } } } else std::cerr << paths[j] << " does not exist" << std::endl; } } catch (const boost::filesystem::filesystem_error& ex) { std::cout << ex.what() << std::endl; } if (!filepath_vector.empty()) { for (unsigned int j = 0; j < filepath_vector.size(); ++j) { bool bFoundFile = isPresent(games[i].installers, filepath_vector[j], config.blacklist) || isPresent(games[i].extras, filepath_vector[j], config.blacklist) || isPresent(games[i].patches, filepath_vector[j], config.blacklist) || isPresent(games[i].languagepacks, filepath_vector[j], config.blacklist); if (!bFoundFile) { // Check dlcs for (unsigned int k = 0; k < games[i].dlcs.size(); ++k) { bFoundFile = isPresent(games[i].dlcs[k].installers, filepath_vector[j], config.blacklist) || isPresent(games[i].dlcs[k].extras, filepath_vector[j], config.blacklist) || isPresent(games[i].dlcs[k].patches, filepath_vector[j], config.blacklist) || isPresent(games[i].dlcs[k].languagepacks, filepath_vector[j], config.blacklist); if(bFoundFile) break; } } if (!bFoundFile) orphans.push_back(filepath_vector[j].string()); } } } std::cout << std::endl; if (!orphans.empty()) { for (unsigned int i = 0; i < orphans.size(); ++i) { if (Globals::globalConfig.dlConf.bDeleteOrphans) { std::string filepath = orphans[i]; std::cout << "Deleting " << filepath << std::endl; if (boost::filesystem::exists(filepath)) if (!boost::filesystem::remove(filepath)) std::cerr << "Failed to delete " << filepath << std::endl; } else std::cout << orphans[i] << std::endl; } } else { std::cout << "No orphaned files" << std::endl; } return; } // Check status of files void Downloader::checkStatus() { if (this->games.empty()) this->getGameDetails(); // Create a vector containing all game files std::vector vGameFiles; for (unsigned int i = 0; i < games.size(); ++i) { std::vector vec = games[i].getGameFileVector(); vGameFiles.insert(std::end(vGameFiles), std::begin(vec), std::end(vec)); } for (unsigned int i = 0; i < vGameFiles.size(); ++i) { unsigned int type = vGameFiles[i].type; if (!(type & Globals::globalConfig.dlConf.iInclude)) continue; boost::filesystem::path filepath = vGameFiles[i].getFilepath(); if (Globals::globalConfig.blacklist.isBlacklisted(filepath.native())) continue; std::string filePathString = filepath.filename().string(); std::string gamename = vGameFiles[i].gamename; if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) { uintmax_t filesize = boost::filesystem::file_size(filepath); uintmax_t filesize_xml = 0; boost::filesystem::path path = filepath; boost::filesystem::path local_xml_file; if (!gamename.empty()) local_xml_file = Globals::globalConfig.sXMLDirectory + "/" + gamename + "/" + path.filename().string() + ".xml"; else local_xml_file = Globals::globalConfig.sXMLDirectory + "/" + path.filename().string() + ".xml"; if (boost::filesystem::exists(local_xml_file)) { tinyxml2::XMLDocument local_xml; local_xml.LoadFile(local_xml_file.string().c_str()); tinyxml2::XMLElement *fileElemLocal = local_xml.FirstChildElement("file"); if (fileElemLocal) { std::string filesize_xml_str = fileElemLocal->Attribute("total_size"); filesize_xml = std::stoull(filesize_xml_str); } } if (Globals::globalConfig.bSizeOnly) { // Check for incomplete file by comparing the filesizes if (filesize_xml > 0 && filesize_xml != filesize) { addStatusLine("FS", gamename, filePathString, filesize, ""); continue; } else { addStatusLine("OK", gamename, filePathString, filesize, ""); continue; } } else { std::string remoteHash; bool bHashOK = true; // assume hash OK // GOG only provides xml data for installers, patches and language packs if (type & (GlobalConstants::GFTYPE_INSTALLER | GlobalConstants::GFTYPE_PATCH | GlobalConstants::GFTYPE_LANGPACK)) remoteHash = this->getRemoteFileHash(vGameFiles[i]); std::string localHash = Util::getLocalFileHash(Globals::globalConfig.sXMLDirectory, filepath.string(), gamename, Globals::globalConfig.bUseFastCheck); if (!remoteHash.empty()) { if (remoteHash != localHash) bHashOK = false; else { // Check for incomplete file by comparing the filesizes // Remote hash was saved but download was incomplete and therefore getLocalFileHash returned the same as getRemoteFileHash if (filesize_xml > 0 && filesize_xml != filesize) { localHash = Util::getFileHash(path.string(), RHASH_MD5); addStatusLine("FS", gamename, filePathString, filesize, localHash); continue; } } } addStatusLine(bHashOK ? "OK" : "MD5", gamename, filePathString, filesize, localHash); } } else { addStatusLine("ND", gamename, filePathString, 0, ""); } } return; } void Downloader::addStatusLine(const std::string& statusCode, const std::string& gamename, const std::string& filepath, const uintmax_t& filesize, const std::string& localHash) { std::cout << statusCode << " " << gamename << " " << filepath; if (filesize > 0) std::cout << " " << filesize; if (!localHash.empty()) std::cout << " " << localHash; std::cout << std::endl; return; } std::string Downloader::getLocalFileHash(const std::string& filepath, const std::string& gamename) { std::string localHash; boost::filesystem::path path = filepath; boost::filesystem::path local_xml_file; if (!gamename.empty()) local_xml_file = Globals::globalConfig.sXMLDirectory + "/" + gamename + "/" + path.filename().string() + ".xml"; else local_xml_file = Globals::globalConfig.sXMLDirectory + "/" + path.filename().string() + ".xml"; if (Globals::globalConfig.dlConf.bAutomaticXMLCreation && !boost::filesystem::exists(local_xml_file) && boost::filesystem::exists(path)) { std::string xml_directory = Globals::globalConfig.sXMLDirectory + "/" + gamename; Util::createXML(filepath, Globals::globalConfig.iChunkSize, xml_directory); } localHash = Util::getLocalFileHash(Globals::globalConfig.sXMLDirectory, filepath, gamename); return localHash; } std::string Downloader::getRemoteFileHash(const gameFile& gf) { std::string remoteHash; // Refresh Galaxy login if token is expired if (gogGalaxy->isTokenExpired()) { if (!gogGalaxy->refreshLogin()) { std::cerr << "Galaxy API failed to refresh login" << std::endl; return remoteHash; } } // Get downlink JSON from Galaxy API Json::Value downlinkJson = gogGalaxy->getResponseJson(gf.galaxy_downlink_json_url); if (downlinkJson.empty()) { std::cerr << "Empty JSON response" << std::endl; return remoteHash; } std::string xml_url; if (downlinkJson.isMember("checksum")) if (!downlinkJson["checksum"].empty()) xml_url = downlinkJson["checksum"].asString(); // Get XML data std::string xml; if (!xml_url.empty()) xml = gogGalaxy->getResponse(xml_url); if (!xml.empty()) { tinyxml2::XMLDocument remote_xml; remote_xml.Parse(xml.c_str()); tinyxml2::XMLElement *fileElemRemote = remote_xml.FirstChildElement("file"); if (fileElemRemote) { remoteHash = fileElemRemote->Attribute("md5"); } } return remoteHash; } /* Load game details from cache file returns 0 if successful returns 1 if cache file doesn't exist returns 2 if JSON parsing failed returns 3 if cache is too old returns 4 if JSON doesn't contain "games" node returns 5 if cache version doesn't match */ int Downloader::loadGameDetailsCache() { int res = 0; std::string cachepath = Globals::globalConfig.sCacheDirectory + "/gamedetails.json"; // Make sure file exists boost::filesystem::path path = cachepath; if (!boost::filesystem::exists(path)) { return res = 1; } bptime::ptime now = bptime::second_clock::local_time(); bptime::ptime cachedate; Json::Value root = Util::readJsonFile(cachepath); if (root.empty()) { return 2; } if (root.isMember("date")) { cachedate = bptime::from_iso_string(root["date"].asString()); if ((now - cachedate) > bptime::minutes(Globals::globalConfig.iCacheValid)) { // cache is too old return 3; } } int iCacheVersion = 0; if (root.isMember("gamedetails-cache-version")) iCacheVersion = root["gamedetails-cache-version"].asInt(); if (iCacheVersion != GlobalConstants::GAMEDETAILS_CACHE_VERSION) { return 5; } if (root.isMember("games")) { this->games = getGameDetailsFromJsonNode(root["games"]); return 0; } return 4; } /* Save game details to cache file returns 0 if successful returns 1 if fails */ int Downloader::saveGameDetailsCache() { int res = 0; // Don't try to save cache if we don't have any game details if (this->games.empty()) { return 1; } std::string cachepath = Globals::globalConfig.sCacheDirectory + "/gamedetails.json"; Json::Value json; json["gamedetails-cache-version"] = GlobalConstants::GAMEDETAILS_CACHE_VERSION; json["version-string"] = Globals::globalConfig.sVersionString; json["version-number"] = Globals::globalConfig.sVersionNumber; json["date"] = bptime::to_iso_string(bptime::second_clock::local_time()); for (unsigned int i = 0; i < this->games.size(); ++i) json["games"].append(this->games[i].getDetailsAsJson()); std::ofstream ofs(cachepath); if (!ofs) { res = 1; } else { ofs << json << std::endl; ofs.close(); } return res; } std::vector Downloader::getGameDetailsFromJsonNode(Json::Value root, const int& recursion_level) { std::vector details; // If root node is not array and we use root.size() it will return the number of nodes --> limit to 1 "array" node to make sure it is handled properly for (unsigned int i = 0; i < (root.isArray() ? root.size() : 1); ++i) { Json::Value gameDetailsNode = (root.isArray() ? root[i] : root); // This json node can be array or non-array so take that into account gameDetails game; game.gamename = gameDetailsNode["gamename"].asString(); game.gamename_basegame = gameDetailsNode["gamename_basegame"].asString(); // DLCs are handled as part of the game so make sure that filtering is done with base game name if (recursion_level == 0) // recursion level is 0 when handling base game { boost::regex expression(Globals::globalConfig.sGameRegex); boost::match_results what; if (!boost::regex_search(game.gamename, what, expression)) // Check if name matches the specified regex continue; } game.title = gameDetailsNode["title"].asString(); game.title_basegame = gameDetailsNode["title_basegame"].asString(); game.icon = gameDetailsNode["icon"].asString(); game.serials = gameDetailsNode["serials"].asString(); game.changelog = gameDetailsNode["changelog"].asString(); game.product_id = gameDetailsNode["product_id"].asString(); // Make a vector of valid node names to make things easier std::vector nodes; nodes.push_back("extras"); nodes.push_back("installers"); nodes.push_back("patches"); nodes.push_back("languagepacks"); nodes.push_back("dlcs"); gameSpecificConfig conf; conf.dlConf = Globals::globalConfig.dlConf; conf.dirConf = Globals::globalConfig.dirConf; if (Util::getGameSpecificConfig(game.gamename, &conf) > 0) std::cerr << game.gamename << " - Language: " << conf.dlConf.iInstallerLanguage << ", Platform: " << conf.dlConf.iInstallerPlatform << ", Include: " << Util::getOptionNameString(conf.dlConf.iInclude, GlobalConstants::INCLUDE_OPTIONS) << std::endl; for (unsigned int j = 0; j < nodes.size(); ++j) { std::string nodeName = nodes[j]; if (gameDetailsNode.isMember(nodeName)) { Json::Value fileDetailsNodeVector = gameDetailsNode[nodeName]; for (unsigned int index = 0; index < fileDetailsNodeVector.size(); ++index) { Json::Value fileDetailsNode = fileDetailsNodeVector[index]; gameFile fileDetails; if (nodeName != "dlcs") { fileDetails.updated = fileDetailsNode["updated"].asInt(); fileDetails.id = fileDetailsNode["id"].asString(); fileDetails.name = fileDetailsNode["name"].asString(); fileDetails.path = fileDetailsNode["path"].asString(); fileDetails.size = fileDetailsNode["size"].asString(); fileDetails.platform = fileDetailsNode["platform"].asUInt(); fileDetails.language = fileDetailsNode["language"].asUInt(); fileDetails.silent = fileDetailsNode["silent"].asInt(); fileDetails.gamename = fileDetailsNode["gamename"].asString(); fileDetails.title = fileDetailsNode["title"].asString(); fileDetails.gamename_basegame = fileDetailsNode["gamename_basegame"].asString(); fileDetails.title_basegame = fileDetailsNode["title_basegame"].asString(); fileDetails.type = fileDetailsNode["type"].asUInt(); fileDetails.galaxy_downlink_json_url = fileDetailsNode["galaxy_downlink_json_url"].asString(); if (!fileDetailsNode["version"].empty()) fileDetails.version = fileDetailsNode["version"].asString(); if (nodeName != "extras" && !(fileDetails.platform & conf.dlConf.iInstallerPlatform)) continue; if (nodeName != "extras" && !(fileDetails.language & conf.dlConf.iInstallerLanguage)) continue; } if (nodeName == "extras" && (conf.dlConf.iInclude & GlobalConstants::GFTYPE_EXTRA)) { game.extras.push_back(fileDetails); } else if (nodeName == "installers" && (conf.dlConf.iInclude & GlobalConstants::GFTYPE_INSTALLER)) { game.installers.push_back(fileDetails); } else if (nodeName == "patches" && (conf.dlConf.iInclude & GlobalConstants::GFTYPE_PATCH)) { game.patches.push_back(fileDetails); } else if (nodeName == "languagepacks" && (conf.dlConf.iInclude & GlobalConstants::GFTYPE_LANGPACK)) { game.languagepacks.push_back(fileDetails); } else if (nodeName == "dlcs" && (conf.dlConf.iInclude & GlobalConstants::GFTYPE_DLC)) { std::vector dlcs = this->getGameDetailsFromJsonNode(fileDetailsNode, recursion_level + 1); game.dlcs.insert(game.dlcs.end(), dlcs.begin(), dlcs.end()); } } } } if (!game.extras.empty() || !game.installers.empty() || !game.patches.empty() || !game.languagepacks.empty() || !game.dlcs.empty()) { game.filterWithPriorities(conf); game.filterWithType(conf.dlConf.iInclude); details.push_back(game); } } return details; } void Downloader::updateCache() { // Make sure that all details get cached Globals::globalConfig.dlConf.iInclude = Util::getOptionValue("all", GlobalConstants::INCLUDE_OPTIONS); Globals::globalConfig.sGameRegex = ".*"; Globals::globalConfig.dlConf.iInstallerLanguage = Util::getOptionValue("all", GlobalConstants::LANGUAGES); Globals::globalConfig.dlConf.iInstallerPlatform = Util::getOptionValue("all", GlobalConstants::PLATFORMS); Globals::globalConfig.dlConf.vLanguagePriority.clear(); Globals::globalConfig.dlConf.vPlatformPriority.clear(); this->getGameList(); this->getGameDetails(); if (this->saveGameDetailsCache()) std::cout << "Failed to save cache" << std::endl; return; } // Save JSON data to file void Downloader::saveJsonFile(const std::string& json, const std::string& filepath) { // Get directory from filepath boost::filesystem::path pathname = filepath; std::string directory = pathname.parent_path().string(); // Check that directory exists and create subdirectories boost::filesystem::path path = directory; if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { std::cout << path << " is not directory" << std::endl; return; } } else { if (!boost::filesystem::create_directories(path)) { std::cout << "Failed to create directory: " << path << std::endl; return; } } std::ofstream ofs(filepath); if (ofs) { std::cout << "Saving JSON data: " << filepath << std::endl; ofs << json; ofs.close(); } else { std::cout << "Failed to create file: " << filepath << std::endl; } return; } // Save serials to file void Downloader::saveSerials(const std::string& serials, const std::string& filepath) { bool bFileExists = boost::filesystem::exists(filepath); if (bFileExists) return; // Get directory from filepath boost::filesystem::path pathname = filepath; std::string directory = pathname.parent_path().string(); // Check that directory exists and create subdirectories boost::filesystem::path path = directory; if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { std::cout << path << " is not directory" << std::endl; return; } } else { if (!boost::filesystem::create_directories(path)) { std::cout << "Failed to create directory: " << path << std::endl; return; } } std::ofstream ofs(filepath); if (ofs) { std::cout << "Saving serials: " << filepath << std::endl; ofs << serials; ofs.close(); } else { std::cout << "Failed to create file: " << filepath << std::endl; } return; } // Save changelog to file void Downloader::saveChangelog(const std::string& changelog, const std::string& filepath) { // Get directory from filepath boost::filesystem::path pathname = filepath; std::string directory = pathname.parent_path().string(); // Check that directory exists and create subdirectories boost::filesystem::path path = directory; if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { std::cout << path << " is not directory" << std::endl; return; } } else { if (!boost::filesystem::create_directories(path)) { std::cout << "Failed to create directory: " << path << std::endl; return; } } // Check whether the changelog has changed if (boost::filesystem::exists(filepath)) { std::ifstream ifs(filepath); if (ifs) { std::string existing_changelog((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); ifs.close(); if (changelog == existing_changelog) { std::cout << "Changelog unchanged. Skipping: " << filepath << std::endl; return; } } } std::ofstream ofs(filepath); if (ofs) { std::cout << "Saving changelog: " << filepath << std::endl; ofs << changelog; ofs.close(); } else { std::cout << "Failed to create file: " << filepath << std::endl; } return; } int Downloader::downloadFileWithId(const std::string& fileid_string, const std::string& output_filepath) { if (gogGalaxy->isTokenExpired()) { if (!gogGalaxy->refreshLogin()) { std::cerr << "Galaxy API failed to refresh login" << std::endl; return 1; } } DownloadConfig dlConf = Globals::globalConfig.dlConf; dlConf.iInclude = Util::getOptionValue("all", GlobalConstants::INCLUDE_OPTIONS); dlConf.bDuplicateHandler = false; // Disable duplicate handler int res = 1; CURLcode result = CURLE_RECV_ERROR; // assume network error size_t pos = fileid_string.find("/"); if (pos == std::string::npos) { std::cerr << "Invalid file id " << fileid_string << ": could not find separator \"/\"" << std::endl; } else if (!output_filepath.empty() && boost::filesystem::is_directory(output_filepath)) { std::cerr << "Failed to create the file " << output_filepath << ": Is a directory" << std::endl; } else { bool bIsDLC = false; std::string gamename, dlc_gamename, fileid, url; std::vector fileid_vector = Util::tokenize(fileid_string, "/"); if (fileid_vector.size() == 3) bIsDLC = true; gamename = fileid_vector[0]; if (bIsDLC) { dlc_gamename = fileid_vector[1]; fileid = fileid_vector[2]; } else fileid = fileid_vector[1]; std::string product_id; std::string gamename_select = "^" + gamename + "$"; bool bSelectOK = this->galaxySelectProductIdHelper(gamename_select, product_id); if (!bSelectOK || product_id.empty()) { std::cerr << "Failed to get numerical product id" << std::endl; return 1; } Json::Value productInfo = gogGalaxy->getProductInfo(product_id); if (productInfo.empty()) { std::cerr << "Failed to get product info" << std::endl; return 1; } gameDetails gd = gogGalaxy->productInfoJsonToGameDetails(productInfo, dlConf); gd.makeFilepaths(Globals::globalConfig.dirConf); auto vFiles = gd.getGameFileVector(); gameFile gf; bool bFoundMatchingFile = false; for (auto f : vFiles) { if (bIsDLC) { if (f.gamename != dlc_gamename) continue; } if (f.id == fileid) { gf = f; bFoundMatchingFile = true; break; } } if (!bFoundMatchingFile) { std::string error_msg = "Failed to find file info ("; error_msg += "product id: " + product_id; error_msg += (bIsDLC ? " / dlc gamename: " + dlc_gamename : ""); error_msg += " / file id: " + fileid; error_msg += ")"; std::cerr << error_msg << std::endl; return 1; } Json::Value downlinkJson = gogGalaxy->getResponseJson(gf.galaxy_downlink_json_url); if (downlinkJson.empty()) { std::cerr << "Empty JSON response" << std::endl; return 1; } if (downlinkJson.isMember("downlink")) { url = downlinkJson["downlink"].asString(); } else { std::cerr << "Invalid JSON response" << std::endl; return 1; } std::string xml_url; if (downlinkJson.isMember("checksum")) { if (!downlinkJson["checksum"].empty()) xml_url = downlinkJson["checksum"].asString(); } // Get XML data std::string xml_data; if (!xml_url.empty()) { xml_data = gogGalaxy->getResponse(xml_url); if (xml_data.empty()) { std::cerr << "Failed to get XML data" << std::endl; } } std::string filename, filepath; filename = gogGalaxy->getPathFromDownlinkUrl(url, gf.gamename); if (output_filepath.empty()) filepath = gf.getFilepath(); else filepath = output_filepath; std::cout << "Downloading: " << filepath << std::endl; result = this->downloadFile(url, filepath, xml_data, gf.gamename); std::cout << std::endl; } if (result == CURLE_OK) res = 0; return res; } void Downloader::showWishlist() { std::vector wishlistItems = gogWebsite->getWishlistItems(); for (unsigned int i = 0; i < wishlistItems.size(); ++i) { wishlistItem item = wishlistItems[i]; std::string platforms_text = Util::getOptionNameString(item.platform, GlobalConstants::PLATFORMS); std::string tags_text; for (unsigned int j = 0; j < item.tags.size(); ++j) { tags_text += (tags_text.empty() ? "" : ", ")+item.tags[j]; } if (!tags_text.empty()) tags_text = "[" + tags_text + "]"; std::string price_text = item.price; if (item.bIsDiscounted) price_text += " (-" + item.discount_percent + " | -" + item.discount + ")"; std::cout << item.title; if (!tags_text.empty()) std::cout << " " << tags_text; std::cout << std::endl; std::cout << "\t" << item.url << std::endl; if (item.platform != 0) std::cout << "\tPlatforms: " << platforms_text << std::endl; if (item.release_date_time != 0) std::cout << "\tRelease date: " << bptime::to_simple_string(bptime::from_time_t(item.release_date_time)) << std::endl; std::cout << "\tPrice: " << price_text << std::endl; if (item.bIsBonusStoreCreditIncluded) std::cout << "\tStore credit: " << item.store_credit << std::endl; std::cout << std::endl; } return; } void Downloader::processCloudSaveUploadQueue(Config conf, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; std::unique_ptr galaxy { new galaxyAPI(Globals::globalConfig.curlConf) }; if (!galaxy->init()) { if (!galaxy->refreshLogin()) { msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; } } CURL* dlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Util::CurlReadChunkMemoryCallback); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); xferInfo xferinfo; xferinfo.tid = tid; xferinfo.curlhandle = dlhandle; curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); cloudSaveFile csf; std::string access_token; if (!Globals::galaxyConf.isExpired()) { access_token = Globals::galaxyConf.getAccessToken(); } if (access_token.empty()) { return; } std::string bearer = "Authorization: Bearer " + access_token; while(dlCloudSaveQueue.try_pop(csf)) { CURLcode result = CURLE_RECV_ERROR; // assume network error int iRetryCount = 0; iTotalRemainingBytes.fetch_sub(csf.fileSize); vDownloadInfo[tid].setFilename(csf.path); std::string filecontents; { std::ifstream in { csf.location, std::ios_base::in | std::ios_base::binary }; in >> filecontents; in.close(); } ChunkMemoryStruct cms { &filecontents[0], (curl_off_t)filecontents.size() }; auto md5 = Util::getChunkHash((std::uint8_t*)filecontents.data(), filecontents.size(), RHASH_MD5); auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path; curl_slist *header = nullptr; header = curl_slist_append(header, bearer.c_str()); header = curl_slist_append(header, ("X-Object-Meta-LocalLastModified: " + boost::posix_time::to_iso_extended_string(csf.lastModified)).c_str()); header = curl_slist_append(header, ("Etag: " + md5).c_str()); header = curl_slist_append(header, "Content-Type: Octet-Stream"); header = curl_slist_append(header, ("Content-Length: " + std::to_string(filecontents.size())).c_str()); curl_easy_setopt(dlhandle, CURLOPT_UPLOAD, 1L); curl_easy_setopt(dlhandle, CURLOPT_CUSTOMREQUEST, "PUT"); curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header); curl_easy_setopt(dlhandle, CURLOPT_READDATA, &cms); curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); msgQueue.push(Message("Begin upload: " + csf.path, MSGTYPE_INFO, msg_prefix, MSGLEVEL_DEFAULT)); bool bShouldRetry = false; long int response_code = 0; std::string retry_reason; do { if (conf.iWait > 0) usleep(conf.iWait); // Wait before continuing response_code = 0; // Make sure that response code is reset if (iRetryCount != 0) { std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + boost::filesystem::path(csf.location).filename().string(); if (!retry_reason.empty()) retry_msg += " (" + retry_reason + ")"; msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix, MSGLEVEL_DEFAULT)); } retry_reason.clear(); // reset retry reason xferinfo.offset = 0; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); switch (result) { // Retry on these errors case CURLE_PARTIAL_FILE: case CURLE_OPERATION_TIMEDOUT: case CURLE_RECV_ERROR: case CURLE_SSL_CONNECT_ERROR: bShouldRetry = true; break; // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable" case CURLE_HTTP_RETURNED_ERROR: curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (response_code == 416 || response_code == 422 || response_code == 400 || response_code == 422) { msgQueue.push(Message(std::to_string(response_code) + ": " + curl_easy_strerror(result), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); bShouldRetry = false; } else bShouldRetry = true; break; default: bShouldRetry = false; break; } if (bShouldRetry) { iRetryCount++; retry_reason = std::to_string(response_code) + ": " + curl_easy_strerror(result); } } while (bShouldRetry && (iRetryCount <= conf.iRetries)); curl_slist_free_all(header); } curl_easy_cleanup(dlhandle); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix, MSGLEVEL_DEFAULT)); } void Downloader::processCloudSaveDownloadQueue(Config conf, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; std::unique_ptr galaxy { new galaxyAPI(Globals::globalConfig.curlConf) }; if (!galaxy->init()) { if (!galaxy->refreshLogin()) { msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; } } CURL* dlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); curl_slist *header = nullptr; std::string access_token; if (!Globals::galaxyConf.isExpired()) access_token = Globals::galaxyConf.getAccessToken(); if (!access_token.empty()) { std::string bearer = "Authorization: Bearer " + access_token; header = curl_slist_append(header, bearer.c_str()); } curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); xferInfo xferinfo; xferinfo.tid = tid; xferinfo.curlhandle = dlhandle; curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); cloudSaveFile csf; while(dlCloudSaveQueue.try_pop(csf)) { CURLcode result = CURLE_RECV_ERROR; // assume network error int iRetryCount = 0; off_t iResumePosition = 0; bool bResume = false; iTotalRemainingBytes.fetch_sub(csf.fileSize); // Get directory from filepath boost::filesystem::path filepath = csf.location + ".~incomplete"; filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); boost::filesystem::path directory = filepath.parent_path(); vDownloadInfo[tid].setFilename(csf.path); bResume = boost::filesystem::exists(filepath); msgQueue.push(Message("Begin download: " + csf.path, MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); // Check that directory exists and create subdirectories std::unique_lock ul { mtx_create_directories }; // Use mutex to avoid possible race conditions if (boost::filesystem::exists(directory)) { if (!boost::filesystem::is_directory(directory)) { msgQueue.push(Message(directory.string() + " is not directory, skipping file (" + filepath.filename().string() + ")", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_ALWAYS)); continue; } } else if (!boost::filesystem::create_directories(directory)) { msgQueue.push(Message("Failed to create directory (" + directory.string() + "), skipping file (" + filepath.filename().string() + ")", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path; curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header); curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); long int response_code = 0; bool bShouldRetry = false; std::string retry_reason; do { if (conf.iWait > 0) usleep(conf.iWait); // Wait before continuing response_code = 0; // Make sure that response code is reset if (iRetryCount != 0) { std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(); if (!retry_reason.empty()) retry_msg += " (" + retry_reason + ")"; msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); } retry_reason = ""; // reset retry reason FILE* outfile; // If a file was partially downloaded if (bResume) { iResumePosition = boost::filesystem::file_size(filepath); if ((outfile=fopen(filepath.string().c_str(), "r+"))!=NULL) { fseek(outfile, 0, SEEK_END); curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, iResumePosition); curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); } else { msgQueue.push(Message("Failed to open " + filepath.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } else // File doesn't exist, create new file { if ((outfile=fopen(filepath.string().c_str(), "w"))!=NULL) { curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0); // start downloading from the beginning of file curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); } else { msgQueue.push(Message("Failed to create " + filepath.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } xferinfo.offset = iResumePosition; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); fclose(outfile); switch (result) { // Retry on these errors case CURLE_PARTIAL_FILE: case CURLE_OPERATION_TIMEDOUT: case CURLE_RECV_ERROR: case CURLE_SSL_CONNECT_ERROR: bShouldRetry = true; break; // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable" case CURLE_HTTP_RETURNED_ERROR: curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (response_code == 416) bShouldRetry = false; else bShouldRetry = true; break; default: bShouldRetry = false; break; } if (bShouldRetry) { iRetryCount++; retry_reason = std::string(curl_easy_strerror(result)); if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) { bResume = true; } } } while (bShouldRetry && (iRetryCount <= conf.iRetries)); if (result == CURLE_OK || result == CURLE_RANGE_ERROR || (result == CURLE_HTTP_RETURNED_ERROR && response_code == 416)) { // Set timestamp for downloaded file to same value as file on server // and rename "filename.~incomplete" to "filename" long filetime = -1; CURLcode res = curl_easy_getinfo(dlhandle, CURLINFO_FILETIME, &filetime); if (res == CURLE_OK && filetime >= 0) { std::time_t timestamp = (std::time_t)filetime; try { boost::filesystem::rename(filepath, csf.location); boost::filesystem::last_write_time(csf.location, timestamp); } catch(const boost::filesystem::filesystem_error& e) { msgQueue.push(Message(e.what(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); } } // Average download speed std::ostringstream dlrate_avg; std::string rate_unit; progressInfo progress_info = vDownloadInfo[tid].getProgressInfo(); if (progress_info.rate_avg > 1048576) // 1 MB { progress_info.rate_avg /= 1048576; rate_unit = "MB/s"; } else { progress_info.rate_avg /= 1024; rate_unit = "kB/s"; } dlrate_avg << std::setprecision(2) << std::fixed << progress_info.rate_avg << rate_unit; msgQueue.push(Message("Download complete: " + csf.path + " (@ " + dlrate_avg.str() + ")", MSGTYPE_SUCCESS, msg_prefix, MSGLEVEL_DEFAULT)); } else { std::string msg = "Download complete (" + static_cast(curl_easy_strerror(result)); if (response_code > 0) msg += " (" + std::to_string(response_code) + ")"; msg += "): " + filepath.filename().string(); msgQueue.push(Message(msg, MSGTYPE_WARNING, msg_prefix, MSGLEVEL_DEFAULT)); // Delete the file if download failed and was not a resume attempt or the result is zero length file if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) { if ((result != CURLE_PARTIAL_FILE && !bResume && result != CURLE_OPERATION_TIMEDOUT) || boost::filesystem::file_size(filepath) == 0) { if (!boost::filesystem::remove(filepath)) msgQueue.push(Message("Failed to delete " + filepath.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); } } } } curl_slist_free_all(header); curl_easy_cleanup(dlhandle); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix, MSGLEVEL_DEFAULT)); } void Downloader::processDownloadQueue(Config conf, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; galaxyAPI* galaxy = new galaxyAPI(Globals::globalConfig.curlConf); if (!galaxy->init()) { if (!galaxy->refreshLogin()) { delete galaxy; msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; } } CURL* curlheader = curl_easy_init(); Util::CurlHandleSetDefaultOptions(curlheader, conf.curlConf); curl_easy_setopt(curlheader, CURLOPT_NOPROGRESS, 1L); curl_easy_setopt(curlheader, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlheader, CURLOPT_HEADER, 1L); curl_easy_setopt(curlheader, CURLOPT_NOBODY, 1L); CURL* dlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); xferInfo xferinfo; xferinfo.tid = tid; xferinfo.curlhandle = dlhandle; curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); gameFile gf; while (dlQueue.try_pop(gf)) { CURLcode result = CURLE_RECV_ERROR; // assume network error int iRetryCount = 0; off_t iResumePosition = 0; vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); unsigned long long filesize = 0; try { filesize = std::stoll(gf.size); } catch (std::invalid_argument& e) { filesize = 0; } iTotalRemainingBytes.fetch_sub(filesize); // Get directory from filepath boost::filesystem::path filepath = gf.getFilepath(); filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); boost::filesystem::path directory = filepath.parent_path(); // Skip blacklisted files if (conf.blacklist.isBlacklisted(filepath.string())) { msgQueue.push(Message("Blacklisted file: " + filepath.string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); continue; } std::string filenameXML = filepath.filename().string() + ".xml"; std::string xml_directory = conf.sXMLDirectory + "/" + gf.gamename; boost::filesystem::path local_xml_file = xml_directory + "/" + filenameXML; vDownloadInfo[tid].setFilename(filepath.filename().string()); msgQueue.push(Message("Begin download: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); // Check that directory exists and create subdirectories mtx_create_directories.lock(); // Use mutex to avoid possible race conditions if (boost::filesystem::exists(directory)) { if (!boost::filesystem::is_directory(directory)) { mtx_create_directories.unlock(); msgQueue.push(Message(directory.string() + " is not directory, skipping file (" + filepath.filename().string() + ")", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_ALWAYS)); continue; } else { mtx_create_directories.unlock(); } } else { if (!boost::filesystem::create_directories(directory)) { mtx_create_directories.unlock(); msgQueue.push(Message("Failed to create directory (" + directory.string() + "), skipping file (" + filepath.filename().string() + ")", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } else { mtx_create_directories.unlock(); } } bool bSameVersion = true; // assume same version bool bLocalXMLExists = boost::filesystem::exists(local_xml_file); // This is additional check to see if remote xml should be saved to speed up future version checks // Refresh Galaxy login if token is expired if (galaxy->isTokenExpired()) { if (!galaxy->refreshLogin()) { msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); delete galaxy; return; } } // Get downlink JSON from Galaxy API Json::Value downlinkJson = galaxy->getResponseJson(gf.galaxy_downlink_json_url); if (downlinkJson.empty()) { msgQueue.push(Message("Empty JSON response, skipping file", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); continue; } if (!downlinkJson.isMember("downlink")) { msgQueue.push(Message("Invalid JSON response, skipping file", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); continue; } std::string xml; bool bFileAlreadyExists = false; bool bIsComplete = false; off_t filesize_api = 0; try { filesize_api = std::stol(gf.size); } catch (std::invalid_argument& e) { filesize_api = 0; } if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) bFileAlreadyExists = true; if (gf.type & (GlobalConstants::GFTYPE_INSTALLER | GlobalConstants::GFTYPE_PATCH) && conf.dlConf.bRemoteXML) { std::string xml_url; if (downlinkJson.isMember("checksum")) if (!downlinkJson["checksum"].empty()) xml_url = downlinkJson["checksum"].asString(); // Get XML data if (conf.dlConf.bRemoteXML && !xml_url.empty()) xml = galaxy->getResponse(xml_url); if (!xml.empty() && !Globals::globalConfig.bSizeOnly) { std::string localHash = Util::getLocalFileHash(conf.sXMLDirectory, filepath.string(), gf.gamename); // Do version check if local hash exists if (!localHash.empty()) { tinyxml2::XMLDocument remote_xml; remote_xml.Parse(xml.c_str()); tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); if (fileElem) { std::string remoteHash = fileElem->Attribute("md5"); if (remoteHash != localHash) bSameVersion = false; } } } } else if ((gf.type & GlobalConstants::GFTYPE_EXTRA) && bFileAlreadyExists) { off_t filesize_local = boost::filesystem::file_size(filepath); off_t filesize_xml = 0; off_t filesize_compare = 0; if (bLocalXMLExists) { tinyxml2::XMLDocument local_xml; local_xml.LoadFile(local_xml_file.string().c_str()); tinyxml2::XMLElement *fileElem = local_xml.FirstChildElement("file"); if (fileElem) { std::string total_size = fileElem->Attribute("total_size"); if (!total_size.empty()) { filesize_xml = std::stol(total_size); try { filesize_xml = std::stol(total_size); } catch (std::invalid_argument& e) { filesize_xml = 0; } } } } if(Globals::globalConfig.bTrustAPIForExtras) { filesize_compare = filesize_api; } else { // API is not trusted to give correct details for extras // Get size from content-length header and compare to it instead off_t filesize_content_length = 0; std::ostringstream memory; std::string url = downlinkJson["downlink"].asString(); curl_easy_setopt(curlheader, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlheader, CURLOPT_WRITEDATA, &memory); curl_easy_perform(curlheader); curl_easy_getinfo(curlheader, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &filesize_content_length); memory.str(std::string()); filesize_compare = filesize_content_length; msgQueue.push(Message(filepath.filename().string() + ": filesize_local: " + std::to_string(filesize_local) + ", filesize_api: " + std::to_string(filesize_api) + ", filesize_content_length: " + std::to_string(filesize_content_length), MSGTYPE_INFO, msg_prefix, MSGLEVEL_DEBUG)); } bool bLocalAssumedComplete = false; if (filesize_xml > 0) { bLocalAssumedComplete = (filesize_local == filesize_xml); } if (bLocalAssumedComplete) { bSameVersion = (filesize_local == filesize_compare); if (bSameVersion) bIsComplete = true; } else { if (filesize_local == filesize_compare) { bIsComplete = true; bSameVersion = true; } else { bIsComplete = false; // Assume same version if smaller than remote file bSameVersion = (filesize_local < filesize_compare); } } } if (bIsComplete) { msgQueue.push(Message("Skipping complete file: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); } bool bResume = false; if (bFileAlreadyExists && !bIsComplete) { if (bSameVersion) { bResume = true; // Check if file is complete so we can skip it instead of resuming if (!xml.empty()) { off_t filesize_xml; off_t filesize_local = boost::filesystem::file_size(filepath); tinyxml2::XMLDocument remote_xml; remote_xml.Parse(xml.c_str()); tinyxml2::XMLElement *fileElem = remote_xml.FirstChildElement("file"); if (fileElem) { std::string total_size = fileElem->Attribute("total_size"); try { filesize_xml = std::stoull(total_size); } catch (std::invalid_argument& e) { filesize_xml = 0; } if (filesize_local == filesize_xml) { msgQueue.push(Message("Skipping complete file: " + filepath.filename().string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); bIsComplete = true; // Set to true so we can skip after saving xml data } } } } else { msgQueue.push(Message("Remote file is different, renaming local file", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); std::string date_old = "." + bptime::to_iso_string(bptime::second_clock::local_time()) + ".old"; boost::filesystem::path new_name = filepath.string() + date_old; // Rename old file by appending date and ".old" to filename boost::system::error_code ec; boost::filesystem::rename(filepath, new_name, ec); // Rename the file if (ec) { msgQueue.push(Message("Failed to rename " + filepath.string() + " to " + new_name.string() + " - Skipping file", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); continue; } } } // Save remote XML if (!xml.empty()) { if ((bLocalXMLExists && !bSameVersion) || !bLocalXMLExists) { // Check that directory exists and create subdirectories boost::filesystem::path path = xml_directory; mtx_create_directories.lock(); // Use mutex to avoid race conditions if (boost::filesystem::exists(path)) { if (!boost::filesystem::is_directory(path)) { msgQueue.push(Message(path.string() + " is not directory", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_ALWAYS)); } } else { if (!boost::filesystem::create_directories(path)) { msgQueue.push(Message("Failed to create directory: " + path.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); } } mtx_create_directories.unlock(); std::ofstream ofs(local_xml_file.string().c_str()); if (ofs) { ofs << xml; ofs.close(); } else { msgQueue.push(Message("Can't create " + local_xml_file.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); } } } // File was complete and we have saved xml data so we can skip it if (bIsComplete) continue; std::string url = downlinkJson["downlink"].asString(); curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); long int response_code = 0; bool bShouldRetry = false; std::string retry_reason; do { if (conf.iWait > 0) usleep(conf.iWait); // Wait before continuing response_code = 0; // Make sure that response code is reset if (iRetryCount != 0) { std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath.filename().string(); if (!retry_reason.empty()) retry_msg += " (" + retry_reason + ")"; msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); } retry_reason = ""; // reset retry reason FILE* outfile; // File exists, resume if (bResume) { iResumePosition = boost::filesystem::file_size(filepath); if ((outfile=fopen(filepath.string().c_str(), "r+"))!=NULL) { fseek(outfile, 0, SEEK_END); curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, iResumePosition); curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); } else { msgQueue.push(Message("Failed to open " + filepath.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } else // File doesn't exist, create new file { if ((outfile=fopen(filepath.string().c_str(), "w"))!=NULL) { curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0); // start downloading from the beginning of file curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); } else { msgQueue.push(Message("Failed to create " + filepath.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } xferinfo.offset = iResumePosition; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); fclose(outfile); switch (result) { // Retry on these errors case CURLE_PARTIAL_FILE: case CURLE_OPERATION_TIMEDOUT: case CURLE_RECV_ERROR: case CURLE_SSL_CONNECT_ERROR: bShouldRetry = true; break; // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable" case CURLE_HTTP_RETURNED_ERROR: curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (response_code == 416) bShouldRetry = false; else bShouldRetry = true; break; default: bShouldRetry = false; break; } if (bShouldRetry) { iRetryCount++; retry_reason = std::string(curl_easy_strerror(result)); if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) bResume = true; } } while (bShouldRetry && (iRetryCount <= conf.iRetries)); if (result == CURLE_OK || result == CURLE_RANGE_ERROR || (result == CURLE_HTTP_RETURNED_ERROR && response_code == 416)) { // Set timestamp for downloaded file to same value as file on server long filetime = -1; CURLcode res = curl_easy_getinfo(dlhandle, CURLINFO_FILETIME, &filetime); if (res == CURLE_OK && filetime >= 0) { std::time_t timestamp = (std::time_t)filetime; try { boost::filesystem::last_write_time(filepath, timestamp); } catch(const boost::filesystem::filesystem_error& e) { msgQueue.push(Message(e.what(), MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); } } // Average download speed std::ostringstream dlrate_avg; std::string rate_unit; progressInfo progress_info = vDownloadInfo[tid].getProgressInfo(); if (progress_info.rate_avg > 1048576) // 1 MB { progress_info.rate_avg /= 1048576; rate_unit = "MB/s"; } else { progress_info.rate_avg /= 1024; rate_unit = "kB/s"; } dlrate_avg << std::setprecision(2) << std::fixed << progress_info.rate_avg << rate_unit; msgQueue.push(Message("Download complete: " + filepath.filename().string() + " (@ " + dlrate_avg.str() + ")", MSGTYPE_SUCCESS, msg_prefix, MSGLEVEL_DEFAULT)); } else { std::string msg = "Download complete (" + static_cast(curl_easy_strerror(result)); if (response_code > 0) msg += " (" + std::to_string(response_code) + ")"; msg += "): " + filepath.filename().string(); msgQueue.push(Message(msg, MSGTYPE_WARNING, msg_prefix, MSGLEVEL_DEFAULT)); // Delete the file if download failed and was not a resume attempt or the result is zero length file if (boost::filesystem::exists(filepath) && boost::filesystem::is_regular_file(filepath)) { if ((result != CURLE_PARTIAL_FILE && !bResume && result != CURLE_OPERATION_TIMEDOUT) || boost::filesystem::file_size(filepath) == 0) { if (!boost::filesystem::remove(filepath)) msgQueue.push(Message("Failed to delete " + filepath.filename().string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); } } } // Automatic xml creation if (conf.dlConf.bAutomaticXMLCreation) { if (result == CURLE_OK) { if ((gf.type & GlobalConstants::GFTYPE_EXTRA) || (conf.dlConf.bRemoteXML && !bLocalXMLExists && xml.empty())) createXMLQueue.push(gf); } } } curl_easy_cleanup(curlheader); curl_easy_cleanup(dlhandle); delete galaxy; vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); msgQueue.push(Message("Finished all tasks", MSGTYPE_INFO, msg_prefix, MSGLEVEL_DEFAULT)); return; } int Downloader::progressCallbackForThread(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { // unused so lets prevent warnings and be more pedantic (void) ulnow; (void) ultotal; xferInfo* xferinfo = static_cast(clientp); // Update progress info every 100ms if (xferinfo->timer.getTimeBetweenUpdates()>=100 || dlnow == dltotal) { xferinfo->timer.reset(); progressInfo info; info.dlnow = dlnow; info.dltotal = dltotal; curl_off_t curl_rate; if (xferinfo->isChunk) { info.dlnow += xferinfo->chunk_file_offset; info.dltotal = xferinfo->chunk_file_total; } // trying to get rate and setting to NaN if it fails if (CURLE_OK != curl_easy_getinfo(xferinfo->curlhandle, CURLINFO_SPEED_DOWNLOAD_T, &curl_rate)) info.rate_avg = std::numeric_limits::quiet_NaN(); else info.rate_avg = static_cast(curl_rate); // setting full dlwnow and dltotal if (xferinfo->offset > 0) { info.dlnow += xferinfo->offset; info.dltotal += xferinfo->offset; } // 10 second average download speed // Don't use static value of 10 seconds because update interval depends on when and how often progress callback is called xferinfo->TimeAndSize.push_back(std::make_pair(time(NULL), static_cast(info.dlnow))); if (xferinfo->TimeAndSize.size() > 100) // 100 * 100ms = 10s { xferinfo->TimeAndSize.pop_front(); time_t time_first = xferinfo->TimeAndSize.front().first; uintmax_t size_first = xferinfo->TimeAndSize.front().second; time_t time_last = xferinfo->TimeAndSize.back().first; uintmax_t size_last = xferinfo->TimeAndSize.back().second; info.rate = (size_last - size_first) / static_cast((time_last - time_first)); } else { info.rate = info.rate_avg; } vDownloadInfo[xferinfo->tid].setProgressInfo(info); vDownloadInfo[xferinfo->tid].setStatus(DLSTATUS_RUNNING); } return 0; } template void Downloader::printProgress(const ThreadSafeQueue& download_queue) { // Print progress information until all threads have finished their tasks ProgressBar bar(Globals::globalConfig.bUnicode, Globals::globalConfig.bColor); unsigned int dl_status = DLSTATUS_NOTSTARTED; while (dl_status != DLSTATUS_FINISHED) { dl_status = DLSTATUS_NOTSTARTED; // Print progress information once per 100ms std::this_thread::sleep_for(std::chrono::milliseconds(Globals::globalConfig.iProgressInterval)); std::cout << "\033[J\r" << std::flush; // Clear screen from the current line down to the bottom of the screen // Print messages from message queue first Message msg; while (msgQueue.try_pop(msg)) { if (msg.getLevel() <= Globals::globalConfig.iMsgLevel) std::cout << msg.getFormattedString(Globals::globalConfig.bColor, true) << std::endl; if (Globals::globalConfig.bReport) { this->report_ofs << msg.getTimestampString() << ": " << msg.getMessage() << std::endl; } } int iTermWidth = Util::getTerminalWidth(); double total_rate = 0; bptime::time_duration eta_total_seconds; // Create progress info text for all download threads std::vector vProgressText; for (unsigned int i = 0; i < vDownloadInfo.size(); ++i) { std::string progress_text; int bar_length = 26; int min_bar_length = 5; unsigned int status = vDownloadInfo[i].getStatus(); dl_status |= status; if (status == DLSTATUS_FINISHED) { vProgressText.push_back("#" + std::to_string(i) + ": Finished"); continue; } std::string filename = vDownloadInfo[i].getFilename(); progressInfo progress_info = vDownloadInfo[i].getProgressInfo(); total_rate += progress_info.rate; bool starting = ((0 == progress_info.dlnow) && (0 == progress_info.dltotal)); double fraction = starting ? 0.0 : static_cast(progress_info.dlnow) / static_cast(progress_info.dltotal); std::string progress_percentage_text = Util::formattedString("%3.0f%% ", fraction * 100); int progress_percentage_text_length = progress_percentage_text.length() + 1; bptime::time_duration eta(bptime::seconds((long)((progress_info.dltotal - progress_info.dlnow) / progress_info.rate))); eta_total_seconds += eta; std::string etastring = Util::makeEtaString(eta); std::string rate_unit; if (progress_info.rate > 1048576) // 1 MB { progress_info.rate /= 1048576; rate_unit = "MB/s"; } else { progress_info.rate /= 1024; rate_unit = "kB/s"; } std::string progress_status_text = Util::formattedString(" %0.2f/%0.2fMB @ %0.2f%s ETA: %s", static_cast(progress_info.dlnow)/1024/1024, static_cast(progress_info.dltotal)/1024/1024, progress_info.rate, rate_unit.c_str(), etastring.c_str()); int status_text_length = progress_status_text.length() + 1; if ((status_text_length + progress_percentage_text_length + bar_length) > iTermWidth) bar_length -= (status_text_length + progress_percentage_text_length + bar_length) - iTermWidth; // Don't draw progressbar if length is less than min_bar_length std::string progress_bar_text; if (bar_length >= min_bar_length) progress_bar_text = bar.createBarString(bar_length, fraction); progress_text = progress_percentage_text + progress_bar_text + progress_status_text; std::string filename_text = "#" + std::to_string(i) + " " + filename; Util::shortenStringToTerminalWidth(filename_text); vProgressText.push_back(filename_text); vProgressText.push_back(progress_text); } // Total download speed and number of remaining tasks in download queue if (dl_status != DLSTATUS_FINISHED) { unsigned long long total_remaining = iTotalRemainingBytes.load(); std::string total_eta_str; if (total_remaining > 0) { bptime::time_duration eta(bptime::seconds((long)(total_remaining / total_rate))); eta += eta_total_seconds; std::string eta_str = Util::makeEtaString(eta); double total_remaining_double = static_cast(total_remaining)/1048576; std::string total_remaining_unit = "MB"; std::vector units = { "GB", "TB", "PB" }; if (total_remaining_double > 1024) { for (const auto& unit : units) { total_remaining_double /= 1024; total_remaining_unit = unit; if (total_remaining_double < 1024) break; } } total_eta_str = Util::formattedString(" (%0.2f%s) ETA: %s", total_remaining_double, total_remaining_unit.c_str(), eta_str.c_str()); } std::ostringstream ss; if (Globals::globalConfig.iThreads > 1) { std::string rate_unit; if (total_rate > 1048576) // 1 MB { total_rate /= 1048576; rate_unit = "MB/s"; } else { total_rate /= 1024; rate_unit = "kB/s"; } ss << "Total: " << std::setprecision(2) << std::fixed << total_rate << rate_unit << " | "; } ss << "Remaining: " << download_queue.size(); if (!total_eta_str.empty()) ss << total_eta_str; vProgressText.push_back(ss.str()); } // Print progress info for (unsigned int i = 0; i < vProgressText.size(); ++i) { std::cout << vProgressText[i] << std::endl; } // Move cursor up by vProgressText.size() rows if (dl_status != DLSTATUS_FINISHED) { std::cout << "\033[" << vProgressText.size() << "A\r" << std::flush; } } } void Downloader::getGameDetailsThread(Config config, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; galaxyAPI* galaxy = new galaxyAPI(Globals::globalConfig.curlConf); if (!galaxy->init()) { if (!galaxy->refreshLogin()) { delete galaxy; msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; } } // Create new GOG website handle Website* website = new Website(); if (!website->IsLoggedIn()) { delete galaxy; delete website; msgQueue.push(Message("Website not logged in", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; } // Set default game specific directory options to values from config DirectoryConfig dirConfDefault; dirConfDefault = config.dirConf; gameItem game_item; while (gameItemQueue.try_pop(game_item)) { gameDetails game; gameSpecificConfig conf; conf.dlConf = config.dlConf; conf.dirConf = dirConfDefault; conf.dlConf.bIgnoreDLCCount = false; if (!config.bUpdateCache) // Disable game specific config files for cache update { int iOptionsOverridden = Util::getGameSpecificConfig(game_item.name, &conf); if (iOptionsOverridden > 0) { std::ostringstream ss; ss << game_item.name << " - " << iOptionsOverridden << " options overridden with game specific options"; if (config.iMsgLevel >= MSGLEVEL_DEBUG) { if (conf.dlConf.bIgnoreDLCCount) ss << std::endl << "\tIgnore DLC count"; if (conf.dlConf.iInclude != config.dlConf.iInclude) ss << std::endl << "\tInclude: " << Util::getOptionNameString(conf.dlConf.iInclude, GlobalConstants::INCLUDE_OPTIONS); if (conf.dlConf.iInstallerLanguage != config.dlConf.iInstallerLanguage) ss << std::endl << "\tLanguage: " << Util::getOptionNameString(conf.dlConf.iInstallerLanguage, GlobalConstants::LANGUAGES); if (conf.dlConf.vLanguagePriority != config.dlConf.vLanguagePriority) { ss << std::endl << "\tLanguage priority:"; for (unsigned int j = 0; j < conf.dlConf.vLanguagePriority.size(); ++j) { ss << std::endl << "\t " << j << ": " << Util::getOptionNameString(conf.dlConf.vLanguagePriority[j], GlobalConstants::LANGUAGES); } } if (conf.dlConf.iInstallerPlatform != config.dlConf.iInstallerPlatform) ss << std::endl << "\tPlatform: " << Util::getOptionNameString(conf.dlConf.iInstallerPlatform, GlobalConstants::PLATFORMS); if (conf.dlConf.vPlatformPriority != config.dlConf.vPlatformPriority) { ss << std::endl << "\tPlatform priority:"; for (unsigned int j = 0; j < conf.dlConf.vPlatformPriority.size(); ++j) { ss << std::endl << "\t " << j << ": " << Util::getOptionNameString(conf.dlConf.vPlatformPriority[j], GlobalConstants::PLATFORMS); } } } msgQueue.push(Message(ss.str(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); } } // Refresh Galaxy login if token is expired if (galaxy->isTokenExpired()) { if (!galaxy->refreshLogin()) { msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } Json::Value product_info = galaxy->getProductInfo(game_item.id); game = galaxy->productInfoJsonToGameDetails(product_info, conf.dlConf); game.filterWithPriorities(conf); game.filterWithType(conf.dlConf.iInclude); if (conf.dlConf.bSaveProductJson && game.productJson.empty()) game.productJson = product_info.toStyledString(); if (conf.dlConf.bSaveProductJson && game.dlcs.size()) { for (unsigned int i = 0; i < game.dlcs.size(); ++i) { if (game.dlcs[i].productJson.empty()) { Json::Value dlc_info = galaxy->getProductInfo(game.dlcs[i].product_id); game.dlcs[i].productJson = dlc_info.toStyledString(); } } } if ((conf.dlConf.bSaveSerials && game.serials.empty()) || (conf.dlConf.bSaveChangelogs && game.changelog.empty()) || (conf.dlConf.bSaveGameDetailsJson && game.gameDetailsJson.empty()) ) { Json::Value gameDetailsJSON; if (!game_item.gamedetailsjson.empty()) gameDetailsJSON = game_item.gamedetailsjson; if (gameDetailsJSON.empty()) gameDetailsJSON = website->getGameDetailsJSON(game_item.id); if (conf.dlConf.bSaveSerials && game.serials.empty()) game.serials = Downloader::getSerialsFromJSON(gameDetailsJSON); if (conf.dlConf.bSaveChangelogs && game.changelog.empty()) game.changelog = Downloader::getChangelogFromJSON(gameDetailsJSON); if (conf.dlConf.bSaveGameDetailsJson && game.gameDetailsJson.empty()) game.gameDetailsJson = gameDetailsJSON.toStyledString(); } game.makeFilepaths(conf.dirConf); gameDetailsQueue.push(game); } vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); delete galaxy; delete website; return; } void Downloader::saveGalaxyJSON() { if (!Globals::galaxyConf.getJSON().empty()) { std::ofstream ofs(Globals::galaxyConf.getFilepath()); if (!ofs) { std::cerr << "Failed to write " << Globals::galaxyConf.getFilepath() << std::endl; } else { ofs << Globals::galaxyConf.getJSON() << std::endl; ofs.close(); } if (!Globals::globalConfig.bRespectUmask) Util::setFilePermissions(Globals::galaxyConf.getFilepath(), boost::filesystem::owner_read | boost::filesystem::owner_write); } } int Downloader::galaxyGetBuildIndexWithBuildId(Json::Value json, const std::string& build_id) { int build_index = -1; for (unsigned int i = 0; i < json["items"].size(); ++i) { std::string build_id_json = json["items"][i]["build_id"].asString(); if (build_id == build_id_json) { build_index = i; break; } } // If we didn't match any build index with given id // Try to use build id as index if (build_index == -1) { int build_id_as_int = -1; try { build_id_as_int = std::stoi(build_id); } catch (...) { // Failed to cast build id to int } build_index = build_id_as_int; } return build_index; } bool Downloader::galaxySelectProductIdHelper(const std::string& product_id, std::string& selected_product) { selected_product = product_id; // Check to see if product_id is id or gamename boost::regex expression("^[0-9]+$"); boost::match_results what; if (!boost::regex_search(product_id, what, expression)) { Globals::globalConfig.sGameRegex = product_id; this->getGameList(); if (this->gameItems.empty()) { std::cerr << "Didn't match any products" << std::endl; return false; } if (this->gameItems.size() == 1) { selected_product = this->gameItems[0].id; } else { std::cout << "Select product:" << std::endl; for (unsigned int i = 0; i < this->gameItems.size(); ++i) std::cout << i << ": " << this->gameItems[i].name << std::endl; if (!isatty(STDIN_FILENO)) { std::cerr << "Unable to read selection" << std::endl; return false; } int iSelect = -1; int iSelectMax = this->gameItems.size(); while (iSelect < 0 || iSelect >= iSelectMax) { std::cerr << "> "; std::string selection; std::getline(std::cin, selection); try { iSelect = std::stoi(selection); } catch(std::invalid_argument& e) { std::cerr << e.what() << std::endl; } } selected_product = this->gameItems[iSelect].id; } } return true; } std::vector Downloader::galaxyGetDepotItemVectorFromJson(const Json::Value& json, const unsigned int& iGalaxyArch) { std::string product_id = json["baseProductId"].asString(); std::string sLanguageRegex = "en|eng|english|en[_-]US"; unsigned int iLanguage = Globals::globalConfig.dlConf.iGalaxyLanguage; for (unsigned int i = 0; i < GlobalConstants::LANGUAGES.size(); ++i) { if (GlobalConstants::LANGUAGES[i].id == iLanguage) { sLanguageRegex = GlobalConstants::LANGUAGES[i].regexp; break; } } std::string sGalaxyArch = "64"; for (unsigned int i = 0; i < GlobalConstants::GALAXY_ARCHS.size(); ++i) { if (GlobalConstants::GALAXY_ARCHS[i].id == iGalaxyArch) { sGalaxyArch = GlobalConstants::GALAXY_ARCHS[i].code; break; } } std::vector items; for (unsigned int i = 0; i < json["depots"].size(); ++i) { std::vector vec = gogGalaxy->getFilteredDepotItemsVectorFromJson(json["depots"][i], sLanguageRegex, sGalaxyArch); if (!vec.empty()) items.insert(std::end(items), std::begin(vec), std::end(vec)); } if (!(Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC)) { std::vector items_no_dlc; for (auto it : items) { if (it.product_id == product_id) items_no_dlc.push_back(it); } items = items_no_dlc; } // Add dependency ids to vector std::vector dependencies; if (json.isMember("dependencies") && Globals::globalConfig.dlConf.bGalaxyDependencies) { for (unsigned int i = 0; i < json["dependencies"].size(); ++i) { dependencies.push_back(json["dependencies"][i].asString()); } } // Add dependencies to items vector if (!dependencies.empty()) { Json::Value dependenciesJson = gogGalaxy->getDependenciesJson(); if (!dependenciesJson.empty() && dependenciesJson.isMember("depots")) { for (unsigned int i = 0; i < dependenciesJson["depots"].size(); ++i) { std::string dependencyId = dependenciesJson["depots"][i]["dependencyId"].asString(); if (std::any_of(dependencies.begin(), dependencies.end(), [dependencyId](std::string dependency){return dependency == dependencyId;})) { std::vector vec = gogGalaxy->getFilteredDepotItemsVectorFromJson(dependenciesJson["depots"][i], sLanguageRegex, sGalaxyArch, true); if (!vec.empty()) items.insert(std::end(items), std::begin(vec), std::end(vec)); } } } } // Set product id for items and add product id to small files container name for (auto it = items.begin(); it != items.end(); ++it) { if (it->product_id.empty()) { it->product_id = product_id; } if (it->isSmallFilesContainer) { it->path += "_" + it->product_id; } } return items; } void Downloader::galaxyInstallGame(const std::string& product_id, const std::string& build_id, const unsigned int& iGalaxyArch) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->galaxyInstallGameById(id, build_id, iGalaxyArch); } } void Downloader::galaxyInstallGameById(const std::string& product_id, const std::string& build_id, const unsigned int& iGalaxyArch) { std::string sPlatform; unsigned int iPlatform = Globals::globalConfig.dlConf.iGalaxyPlatform; if (iPlatform == GlobalConstants::PLATFORM_LINUX) sPlatform = "linux"; else if (iPlatform == GlobalConstants::PLATFORM_MAC) sPlatform = "osx"; else sPlatform = "windows"; Json::Value json = gogGalaxy->getProductBuilds(product_id, sPlatform); // JSON is empty and platform is Linux. Most likely cause is that Galaxy API doesn't have Linux support if (json.empty() && iPlatform == GlobalConstants::PLATFORM_LINUX) { std::cout << "Galaxy API doesn't have Linux support" << std::endl; // Galaxy install hack for Linux std::cout << "Trying to use installers as repository" << std::endl; this->galaxyInstallGame_MojoSetupHack(product_id); return; } int build_index = this->galaxyGetBuildIndexWithBuildId(json, build_id); build_index = std::max(0, build_index); if (json["items"][build_index]["generation"].asInt() != 2) { std::cout << "Only generation 2 builds are supported currently" << std::endl; return; } std::string link = json["items"][build_index]["link"].asString(); std::string buildHash; buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); // Save builds json to another variable for later use Json::Value json_builds = json; json = gogGalaxy->getManifestV2(buildHash); std::string game_title = json["products"][0]["name"].asString(); std::string install_directory; if (Globals::globalConfig.dirConf.bSubDirectories) { install_directory = this->getGalaxyInstallDirectory(gogGalaxy, json); } std::string install_path = Globals::globalConfig.dirConf.sDirectory + install_directory; std::vector items = this->galaxyGetDepotItemVectorFromJson(json, iGalaxyArch); // Remove blacklisted files from items vector for (std::vector::iterator it = items.begin(); it != items.end();) { std::string item_install_path = install_path + "/" + it->path; if (Globals::globalConfig.blacklist.isBlacklisted(item_install_path)) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cout << "Skipping blacklisted file: " << item_install_path << std::endl; it = items.erase(it); } else { ++it; } } std::vector items_smallfiles; std::vector sfc_vector; bool bUseSmallFilesContainer = true; for (auto item : items) { if (item.isInSFC) { std::string item_install_path = install_path + "/" + item.path; if (boost::filesystem::exists(item_install_path)) { bUseSmallFilesContainer = false; break; } } } if (!bUseSmallFilesContainer) { for (std::vector::iterator it = items.begin(); it != items.end();) { if (it->isSmallFilesContainer) it = items.erase(it); else ++it; } } else { for (std::vector::iterator it = items.begin(); it != items.end();) { if (it->isSmallFilesContainer) sfc_vector.push_back(*it); if (it->isInSFC) { items_smallfiles.push_back(*it); it = items.erase(it); } else ++it; } } // Check for differences between previously installed build and new build std::vector items_old; std::string info_path = install_path + "/goggame-" + product_id + ".info"; std::string old_build_id; int old_build_index = -1; if (boost::filesystem::exists(info_path)) { Json::Value info_json = Util::readJsonFile(info_path); if (!info_json.empty()) old_build_id = info_json["buildId"].asString(); if (!old_build_id.empty()) { old_build_index = this->galaxyGetBuildIndexWithBuildId(json_builds, old_build_id); } } // Check for deleted files between builds if (old_build_index >= 0 && old_build_index != build_index) { std::string link = json_builds["items"][old_build_index]["link"].asString(); std::string buildHash_old; buildHash_old.assign(link.begin()+link.find_last_of("/")+1, link.end()); Json::Value json_old = gogGalaxy->getManifestV2(buildHash_old); items_old = this->galaxyGetDepotItemVectorFromJson(json_old, iGalaxyArch); } std::vector deleted_filepaths; if (!items_old.empty()) { for (auto old_item: items_old) { bool isDeleted = true; for (auto item: items) { if (old_item.path == item.path) { isDeleted = false; break; } } if (isDeleted) deleted_filepaths.push_back(old_item.path); } } // Delete old files if (!deleted_filepaths.empty()) { for (auto path : deleted_filepaths) { std::string filepath = install_path + "/" + path; std::cout << "Deleting " << filepath << std::endl; if (boost::filesystem::exists(filepath)) if (!boost::filesystem::remove(filepath)) std::cerr << "Failed to delete " << filepath << std::endl; } } uintmax_t totalSize = 0; for (unsigned int i = 0; i < items.size(); ++i) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) { std::cout << items[i].path << std::endl; std::cout << "\tChunks: " << items[i].chunks.size() << std::endl; std::cout << "\tmd5: " << items[i].md5 << std::endl; } totalSize += items[i].totalSizeUncompressed; iTotalRemainingBytes.fetch_add(items[i].totalSizeCompressed); dlQueueGalaxy.push(items[i]); } std::cout << game_title << std::endl; std::cout << "Files: " << items.size() << std::endl; std::cout << "Total size installed: " << Util::makeSizeString(totalSize) << std::endl; if (Globals::globalConfig.dlConf.bFreeSpaceCheck) { boost::filesystem::path path = boost::filesystem::absolute(install_path); while(!boost::filesystem::exists(path) && !path.empty()) { path = path.parent_path(); } if(boost::filesystem::exists(path) && !path.empty()) { boost::filesystem::space_info space = boost::filesystem::space(path); if (space.available < totalSize) { std::cerr << "Not enough free space in " << boost::filesystem::canonical(path) << " (" << Util::makeSizeString(space.available) << ")"<< std::endl; exit(1); } } } // Limit thread count to number of items in download queue unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlQueueGalaxy.size())); // Create download threads std::vector vThreads; for (unsigned int i = 0; i < iThreads; ++i) { DownloadInfo dlInfo; dlInfo.setStatus(DLSTATUS_NOTSTARTED); vDownloadInfo.push_back(dlInfo); vThreads.push_back(std::thread(Downloader::processGalaxyDownloadQueue, install_path, Globals::globalConfig, i)); } this->printProgress(dlQueueGalaxy); // Join threads for (unsigned int i = 0; i < vThreads.size(); ++i) vThreads[i].join(); vThreads.clear(); vDownloadInfo.clear(); if (bUseSmallFilesContainer) { for (auto container : sfc_vector) { std::string container_install_path = install_path + "/" + container.path; if (!boost::filesystem::exists(container_install_path)) continue; std::cout << "Extracting small files container " << container_install_path << std::endl; for (auto item : items_smallfiles) { if (item.product_id != container.product_id) continue; std::string item_install_path = install_path + "/" + item.path; std::cout << item_install_path << std::endl; std::ifstream sfc(container_install_path, std::ifstream::binary); if (sfc) { sfc.seekg(item.sfc_offset, sfc.beg); char *filecontents = (char *) malloc(item.sfc_size); sfc.read(filecontents, item.sfc_size); sfc.close(); // Check that directory exists and create it boost::filesystem::path path = item_install_path; boost::filesystem::path directory = path.parent_path(); if (!boost::filesystem::exists(directory)) { if (!boost::filesystem::create_directories(directory)) { std::cout << "Failed to create directory: " << directory << std::endl; free(filecontents); continue; } } std::ofstream output(item_install_path, std::ofstream::binary); if (output) { output.write(filecontents, item.sfc_size); output.close(); } free(filecontents); } } std::cout << "Deleting small files container " << container_install_path << std::endl; if (!boost::filesystem::remove(container_install_path)) std::cerr << "Failed to delete " << container_install_path << std::endl; } } std::cout << "Checking for orphaned files" << std::endl; if (bUseSmallFilesContainer) { // Add small files back to items vector for ophan checking items.insert(std::end(items), std::begin(items_smallfiles), std::end(items_smallfiles)); } std::vector orphans = this->galaxyGetOrphanedFiles(items, install_path); std::cout << "\t" << orphans.size() << " orphaned files" << std::endl; for (unsigned int i = 0; i < orphans.size(); ++i) { if (Globals::globalConfig.dlConf.bDeleteOrphans) { std::string filepath = orphans[i]; std::cout << "Deleting " << filepath << std::endl; if (boost::filesystem::exists(filepath)) if (!boost::filesystem::remove(filepath)) std::cerr << "Failed to delete " << filepath << std::endl; } else std::cout << "\t" << orphans[i] << std::endl; } } void Downloader::galaxyListCDNs(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->galaxyListCDNsById(id, build_id); } } void Downloader::galaxyListCDNsById(const std::string& product_id, const std::string& build_id) { std::string sPlatform; unsigned int iPlatform = Globals::globalConfig.dlConf.iGalaxyPlatform; if (iPlatform == GlobalConstants::PLATFORM_LINUX) sPlatform = "linux"; else if (iPlatform == GlobalConstants::PLATFORM_MAC) sPlatform = "osx"; else sPlatform = "windows"; Json::Value json = gogGalaxy->getProductBuilds(product_id, sPlatform); // JSON is empty and platform is Linux. Most likely cause is that Galaxy API doesn't have Linux support if (json.empty() && iPlatform == GlobalConstants::PLATFORM_LINUX) { std::cout << "Galaxy API doesn't have Linux support" << std::endl; return; } int build_index = this->galaxyGetBuildIndexWithBuildId(json, build_id); build_index = std::max(0, build_index); if (json["items"][build_index]["generation"].asInt() != 2) { std::cout << "Only generation 2 builds are supported currently" << std::endl; return; } std::string link = json["items"][build_index]["link"].asString(); std::string buildHash; buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); json = gogGalaxy->getSecureLink(product_id, "/"); std::vector vEndpointNames; if (!json.empty()) { for (unsigned int i = 0; i < json["urls"].size(); ++i) { std::string endpoint_name = json["urls"][i]["endpoint_name"].asString(); if (!endpoint_name.empty()) vEndpointNames.push_back(endpoint_name); } } for (auto endpoint : vEndpointNames) std::cout << endpoint << std::endl; return; } void Downloader::processGalaxyDownloadQueue(const std::string& install_path, Config conf, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; galaxyAPI* galaxy = new galaxyAPI(Globals::globalConfig.curlConf); if (!galaxy->init()) { if (!galaxy->refreshLogin()) { delete galaxy; msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); return; } } CURL* dlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(dlhandle, Globals::globalConfig.curlConf); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); xferInfo xferinfo; xferinfo.tid = tid; xferinfo.curlhandle = dlhandle; curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); galaxyDepotItem item; std::string prev_product_id = ""; std::vector cdnUrlTemplates; while (dlQueueGalaxy.try_pop(item)) { xferinfo.isChunk = false; xferinfo.chunk_file_offset = 0; xferinfo.chunk_file_total = item.totalSizeCompressed; if (item.product_id != prev_product_id) cdnUrlTemplates.clear(); vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); iTotalRemainingBytes.fetch_sub(item.totalSizeCompressed); boost::filesystem::path path = install_path + "/" + item.path; // Check that directory exists and create it boost::filesystem::path directory = path.parent_path(); mtx_create_directories.lock(); // Use mutex to avoid possible race conditions if (boost::filesystem::exists(directory)) { if (!boost::filesystem::is_directory(directory)) { msgQueue.push(Message(directory.string() + " is not directory", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); delete galaxy; mtx_create_directories.unlock(); return; } } else { if (!boost::filesystem::create_directories(directory)) { msgQueue.push(Message("Failed to create directory: " + directory.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); delete galaxy; mtx_create_directories.unlock(); return; } } mtx_create_directories.unlock(); vDownloadInfo[tid].setFilename(path.string()); unsigned int start_chunk = 0; if (boost::filesystem::exists(path)) { msgQueue.push(Message("File already exists: " + path.string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); unsigned int resume_chunk = 0; uintmax_t filesize = boost::filesystem::file_size(path); if (filesize == item.totalSizeUncompressed) { // File is same size if (item.totalSizeUncompressed == 0 || Util::getFileHash(path.string(), RHASH_MD5) == item.md5) { msgQueue.push(Message(path.string() + ": OK", MSGTYPE_SUCCESS, msg_prefix, MSGLEVEL_DEFAULT)); continue; } else { msgQueue.push(Message(path.string() + ": MD5 mismatch", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_DEFAULT)); if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } } } else if (filesize > item.totalSizeUncompressed) { // File is bigger than on server, delete old file and start from beginning msgQueue.push(Message(path.string() + ": File is bigger than expected. Deleting old file and starting from beginning", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } } else { // File is smaller than on server, resume for (unsigned int j = 0; j < item.chunks.size(); ++j) { if (item.chunks[j].offset_uncompressed == filesize) { resume_chunk = j; break; } } if (resume_chunk > 0) { msgQueue.push(Message(path.string() + ": Resume from chunk " + std::to_string(resume_chunk), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); // Get chunk hash for previous chunk FILE* f = fopen(path.string().c_str(), "r"); if (!f) { msgQueue.push(Message(path.string() + ": Failed to open", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_DEFAULT)); continue; } unsigned int previous_chunk = resume_chunk - 1; uintmax_t chunk_size = item.chunks[previous_chunk].size_uncompressed; // use fseeko to support large files on 32 bit platforms fseeko(f, item.chunks[previous_chunk].offset_uncompressed, SEEK_SET); unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *)); if (chunk == NULL) { msgQueue.push(Message(path.string() + ": Memory error - Chunk " + std::to_string(resume_chunk), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_DEFAULT)); fclose(f); continue; } uintmax_t fread_size = fread(chunk, 1, chunk_size, f); fclose(f); if (fread_size != chunk_size) { msgQueue.push(Message(path.string() + ": Read error - Chunk " + std::to_string(resume_chunk), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_DEFAULT)); free(chunk); continue; } std::string chunk_hash = Util::getChunkHash(chunk, chunk_size, RHASH_MD5); free(chunk); if (chunk_hash == item.chunks[previous_chunk].md5_uncompressed) { // Hash for previous chunk matches, resume at this position start_chunk = resume_chunk; } else { // Hash for previous chunk is different, delete old file and start from beginning msgQueue.push(Message(path.string() + ": Chunk hash is different. Deleting old file and starting from beginning.", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_DEFAULT)); continue; } } } else { msgQueue.push(Message(path.string() + ": Failed to find valid resume position. Deleting old file and starting from beginning.", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); continue; } } } } bool bChunkFailure = false; std::time_t timestamp = -1; // Handle empty files if (item.chunks.empty()) { // Create empty file std::ofstream ofs(path.string(), std::ofstream::out | std::ofstream::binary); if (ofs) ofs.close(); } for (unsigned int j = start_chunk; j < item.chunks.size(); ++j) { xferinfo.isChunk = true; xferinfo.chunk_file_offset = item.chunks[j].offset_compressed; // Set offset for progress info // Refresh Galaxy login if token is expired if (galaxy->isTokenExpired()) { if (!galaxy->refreshLogin()) { msgQueue.push(Message("Galaxy API failed to refresh login", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); delete galaxy; return; } } std::string galaxyPath = galaxy->hashToGalaxyPath(item.chunks[j].md5_compressed); // Get url templates for cdns // Regular files can re-use these // Dependencies require new url everytime if (cdnUrlTemplates.empty() || item.isDependency) { Json::Value json; if (item.isDependency) json = galaxy->getDependencyLink(galaxyPath); else json = galaxy->getSecureLink(item.product_id, "/"); if (json.empty()) { bChunkFailure = true; std::string error_message = path.string() + ": Empty JSON response (product: " + item.product_id + ", chunk #"+ std::to_string(j) + ": " + item.chunks[j].md5_compressed + ")"; msgQueue.push(Message(error_message, MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); break; } cdnUrlTemplates = galaxy->cdnUrlTemplatesFromJson(json, conf.dlConf.vGalaxyCDNPriority); } if (cdnUrlTemplates.empty()) { bChunkFailure = true; msgQueue.push(Message(path.string() + ": Failed to get download url", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_DEFAULT)); break; } std::string url = cdnUrlTemplates[0]; if (item.isDependency) { while(Util::replaceString(url, "{LGOGDOWNLOADER_GALAXY_PATH}", "")); cdnUrlTemplates.clear(); // Clear templates } else { galaxyPath = "/" + galaxyPath; while(Util::replaceString(url, "{LGOGDOWNLOADER_GALAXY_PATH}", galaxyPath)); prev_product_id = item.product_id; } ChunkMemoryStruct chunk; chunk.memory = (char *) malloc(1); chunk.size = 0; curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteChunkMemoryCallback); curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &chunk); curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, 0); std::string filepath_and_chunk = path.string() + " (chunk " + std::to_string(j + 1) + "/" + std::to_string(item.chunks.size()) + ")"; vDownloadInfo[tid].setFilename(filepath_and_chunk); CURLcode result; int iRetryCount = 0; long int response_code = 0; bool bShouldRetry = false; std::string retry_reason; do { if (Globals::globalConfig.iWait > 0) usleep(Globals::globalConfig.iWait); // Delay the request by specified time response_code = 0; // Make sure that response code is reset if (iRetryCount != 0) { std::string retry_msg = "Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + filepath_and_chunk; if (!retry_reason.empty()) retry_msg += " (" + retry_reason + ")"; msgQueue.push(Message(retry_msg, MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); curl_easy_setopt(dlhandle, CURLOPT_RESUME_FROM_LARGE, chunk.size); } retry_reason = ""; // reset retry reason xferinfo.offset = chunk.size; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); switch (result) { // Retry on these errors case CURLE_PARTIAL_FILE: case CURLE_OPERATION_TIMEDOUT: case CURLE_RECV_ERROR: bShouldRetry = true; break; // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "416 Range Not Satisfiable" case CURLE_HTTP_RETURNED_ERROR: curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (response_code == 416) bShouldRetry = false; else bShouldRetry = true; break; default: bShouldRetry = false; break; } if (bShouldRetry) { iRetryCount++; retry_reason = std::string(curl_easy_strerror(result)); } } while (bShouldRetry && (iRetryCount <= conf.iRetries)); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 0L); if (result != CURLE_OK) { msgQueue.push(Message(std::string(curl_easy_strerror(result)), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); if (result == CURLE_HTTP_RETURNED_ERROR) { long int response_code = 0; result = curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (result == CURLE_OK) msgQueue.push(Message("HTTP ERROR: " + std::to_string(response_code) + " (" + url + ")", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); else msgQueue.push(Message("HTTP ERROR: failed to get error code: " + std::string(curl_easy_strerror(result)) + " (" + url + ")", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); } } else { // Get timestamp for downloaded file long filetime = -1; result = curl_easy_getinfo(dlhandle, CURLINFO_FILETIME, &filetime); if (result == CURLE_OK && filetime >= 0) timestamp = (std::time_t)filetime; } std::ofstream ofs(path.string(), std::ofstream::out | std::ofstream::binary | std::ofstream::app); if (ofs) { boost::iostreams::filtering_streambuf output; output.push(boost::iostreams::zlib_decompressor(GlobalConstants::ZLIB_WINDOW_SIZE)); output.push(ofs); boost::iostreams::write(output, chunk.memory, chunk.size); } if (ofs) ofs.close(); free(chunk.memory); } if (bChunkFailure) { msgQueue.push(Message(path.string() + ": Chunk failure, skipping file", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_VERBOSE)); continue; } // Set timestamp for downloaded file to same value as file on server if (boost::filesystem::exists(path) && timestamp >= 0) { try { boost::filesystem::last_write_time(path, timestamp); } catch(const boost::filesystem::filesystem_error& e) { msgQueue.push(Message(e.what(), MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); } } msgQueue.push(Message("Download complete: " + path.string(), MSGTYPE_SUCCESS, msg_prefix, MSGLEVEL_DEFAULT)); } vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); delete galaxy; curl_easy_cleanup(dlhandle); return; } void Downloader::galaxyShowBuilds(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->galaxyShowBuildsById(id, build_id); } } void Downloader::galaxyShowBuildsById(const std::string& product_id, const std::string& build_id) { std::string sPlatform; unsigned int iPlatform = Globals::globalConfig.dlConf.iGalaxyPlatform; if (iPlatform == GlobalConstants::PLATFORM_LINUX) sPlatform = "linux"; else if (iPlatform == GlobalConstants::PLATFORM_MAC) sPlatform = "osx"; else sPlatform = "windows"; Json::Value json = gogGalaxy->getProductBuilds(product_id, sPlatform); // JSON is empty and platform is Linux. Most likely cause is that Galaxy API doesn't have Linux support if (json.empty() && iPlatform == GlobalConstants::PLATFORM_LINUX) { std::cout << "Galaxy API doesn't have Linux support" << std::endl; std::cout << "Checking for installers that can be used as repository" << std::endl; DownloadConfig dlConf = Globals::globalConfig.dlConf; dlConf.iInclude = GlobalConstants::GFTYPE_INSTALLER; dlConf.iInstallerPlatform = dlConf.iGalaxyPlatform; dlConf.iInstallerLanguage = dlConf.iGalaxyLanguage; Json::Value product_info = gogGalaxy->getProductInfo(product_id); gameDetails game = gogGalaxy->productInfoJsonToGameDetails(product_info, dlConf); std::vector vInstallers; if (!game.installers.empty()) { vInstallers.push_back(game.installers[0]); for (unsigned int i = 0; i < game.dlcs.size(); ++i) { if (!game.dlcs[i].installers.empty()) vInstallers.push_back(game.dlcs[i].installers[0]); } } if (vInstallers.empty()) { std::cout << "No installers found" << std::endl; } else { std::cout << "Using these installers" << std::endl; for (auto installer : vInstallers) { std::string str = installer.gamename + "/" + installer.id; if (installer.version.empty()) str += " (no version info available)"; else str += " (Version: " + installer.version + ")"; std::cout << "\t" << str << std::endl; } } return; } int build_index = this->galaxyGetBuildIndexWithBuildId(json, build_id); if (build_index < 0) { for (unsigned int i = 0; i < json["items"].size(); ++i) { std::cout << i << ": " << "Version " << json["items"][i]["version_name"].asString() << " - " << json["items"][i]["date_published"].asString() << " (Gen " << json["items"][i]["generation"].asInt() << ")" << " (Build id: " << json["items"][i]["build_id"].asString() << ")" << std::endl; } return; } std::string link = json["items"][build_index]["link"].asString(); if (json["items"][build_index]["generation"].asInt() == 1) { json = gogGalaxy->getManifestV1(link); } else if (json["items"][build_index]["generation"].asInt() == 2) { std::string buildHash; buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); json = gogGalaxy->getManifestV2(buildHash); } else { std::cout << "Only generation 1 and 2 builds are supported currently" << std::endl; return; } Json::StyledStreamWriter().write(std::cout, json); return; } std::string parseLocationHelper(const std::string &location, const std::map &var) { char search_arg[2] {'?', '>'}; auto it = std::search(std::begin(location), std::end(location), std::begin(search_arg), std::end(search_arg)); if(it == std::end(location)) { return location; } std::string var_name { std::begin(location) + 2, it }; auto relative_path = it + 2; auto var_value = var.find(var_name); if(var_value == std::end(var)) { return location; } std::string parsedLocation; parsedLocation.insert(std::end(parsedLocation), std::begin(var_value->second), std::end(var_value->second)); parsedLocation.insert(std::end(parsedLocation), relative_path, std::end(location)); return parsedLocation; } std::string parseLocation(const std::string &location, const std::map &var) { auto parsedLocation = parseLocationHelper(location, var); Util::replaceAllString(parsedLocation, "\\", "/"); return parsedLocation; } std::pair getline(std::string::const_iterator begin, std::string::const_iterator end) { while(begin != end) { if(*begin == '\r') { return { begin, begin + 2 }; } if(*begin == '\n') { return { begin, begin + 1 }; } ++begin; } return { end, end }; } void Downloader::uploadCloudSaves(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->uploadCloudSavesById(id, build_id); } } void Downloader::deleteCloudSaves(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->deleteCloudSavesById(id, build_id); } } void Downloader::downloadCloudSaves(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->downloadCloudSavesById(id, build_id); } } void Downloader::galaxyShowCloudSaves(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->galaxyShowCloudSavesById(id, build_id); } } void Downloader::galaxyShowLocalCloudSaves(const std::string& product_id, const std::string& build_id) { std::string id; if(this->galaxySelectProductIdHelper(product_id, id)) { if (!id.empty()) this->galaxyShowLocalCloudSavesById(id, build_id); } } std::map Downloader::cloudSaveLocations(const std::string& product_id, const std::string& build_id) { std::string sPlatform; unsigned int iPlatform = Globals::globalConfig.dlConf.iGalaxyPlatform; if (iPlatform == GlobalConstants::PLATFORM_LINUX) { // Linux is not yet supported for cloud saves std::cout << "Cloud saves for Linux builds not yet supported" << std::endl; return {}; } else if (iPlatform == GlobalConstants::PLATFORM_MAC) { std::cout << "Cloud saves for Mac builds not yet supported" << std::endl; return {}; } else sPlatform = "windows"; Json::Value json = gogGalaxy->getProductBuilds(product_id, sPlatform); int build_index = this->galaxyGetBuildIndexWithBuildId(json, build_id); build_index = std::max(0, build_index); std::string link = json["items"][build_index]["link"].asString(); Json::Value manifest; if (json["items"][build_index]["generation"].asInt() != 2) { std::cout << "Only generation 2 builds are supported currently" << std::endl; return {}; } std::string buildHash; buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); manifest = gogGalaxy->getManifestV2(buildHash); std::string clientId = manifest["clientId"].asString(); std::string secret = manifest["clientSecret"].asString(); if(!gogGalaxy->refreshLogin(clientId, secret, Globals::galaxyConf.getRefreshToken(), false)) { std::cout << "Couldn't refresh login" << std::endl; return {}; } std::string install_directory; if (Globals::globalConfig.dirConf.bSubDirectories) { install_directory = this->getGalaxyInstallDirectory(gogGalaxy, json); } std::string platform; switch(iPlatform) { case GlobalConstants::PLATFORM_WINDOWS: platform = "Windows"; break; default: std::cout << "Only Windows supported for now for cloud support" << std::endl; return {}; } std::string install_path = Globals::globalConfig.dirConf.sDirectory + install_directory; std::string document_path = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/Documents/"; std::string appdata_roaming = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/AppData/Roaming/"; std::string appdata_local_path = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/AppData/Local/"; std::string appdata_local_low_path = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/AppData/LocalLow/"; std::string saved_games = Globals::globalConfig.dirConf.sWinePrefix + "drive_c/users/" + username() + "/Save Games/"; auto cloud_saves_json = gogGalaxy->getCloudPathAsJson(manifest["clientId"].asString())["content"][platform]["cloudStorage"]; auto enabled = cloud_saves_json["enabled"].asBool(); if(!enabled) { return {}; } std::map vars { { "INSTALL", std::move(install_path) }, { "DOCUMENTS", std::move(document_path) }, { "APPLICATION_DATA_ROAMING", std::move(appdata_roaming)}, { "APPLICATION_DATA_LOCAL", std::move(appdata_local_path) }, { "APPLICATION_DATA_LOCAL_LOW", std::move(appdata_local_low_path) }, { "SAVED_GAMES", std::move(saved_games) }, }; std::map name_to_location; for(auto &cloud_save : cloud_saves_json["locations"]) { std::string location = parseLocation(cloud_save["location"].asString(), vars); name_to_location.insert({cloud_save["name"].asString(), std::move(location)}); } if(name_to_location.empty()) { std::string location; switch(iPlatform) { case GlobalConstants::PLATFORM_WINDOWS: location = vars["APPLICATION_DATA_LOCAL"] + "/GOG.com/Galaxy/Applications/" + Globals::galaxyConf.getClientId() + "/Storage"; break; default: std::cout << "Only Windows supported for now for cloud support" << std::endl; return {}; } name_to_location.insert({"__default", std::move(location)}); } return name_to_location; } int Downloader::cloudSaveListByIdForEach(const std::string& product_id, const std::string& build_id, const std::function &f) { auto name_to_location = this->cloudSaveLocations(product_id, build_id); if(name_to_location.empty()) { std::cout << "No cloud save locations found" << std::endl; return -1; } std::string url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + "/" + Globals::galaxyConf.getClientId(); auto fileList = gogGalaxy->getResponseJson(url, "application/json"); for(auto &fileJson : fileList) { auto path = fileJson["name"].asString(); if(!whitelisted(path)) { continue; } auto pos = path.find_first_of('/'); auto location = name_to_location[path.substr(0, pos)] + path.substr(pos); auto filesize = fileJson["bytes"].asUInt64(); auto last_modified = boost::posix_time::from_iso_extended_string(fileJson["last_modified"].asString()); cloudSaveFile csf { last_modified, filesize, std::move(path), std::move(location) }; f(csf); } return 0; } void Downloader::uploadCloudSavesById(const std::string& product_id, const std::string& build_id) { auto name_to_locations = cloudSaveLocations(product_id, build_id); if(name_to_locations.empty()) { std::cout << "Cloud saves not supported for this game" << std::endl; } std::map path_to_cloudSaveFile; for(auto &name_to_location : name_to_locations) { auto &name = name_to_location.first; auto &location = name_to_location.second; if(!boost::filesystem::exists(location) || !boost::filesystem::is_directory(location)) { continue; } const char endswith[] = ".~incomplete"; dirForEach(location, [&](boost::filesystem::directory_iterator file) { auto path = file->path(); // If path ends with ".~incomplete", then skip this file if( path.size() >= sizeof(endswith) && strcmp(path.c_str() + (path.size() + 1 - sizeof(endswith)), endswith) == 0 ) { return; } auto remote_path = (name / boost::filesystem::relative(*file, location)).string(); if(!whitelisted(remote_path)) { return; } cloudSaveFile csf { boost::posix_time::from_time_t(boost::filesystem::last_write_time(*file) - 1), boost::filesystem::file_size(*file), std::move(remote_path), file->path().string() }; path_to_cloudSaveFile.insert(std::make_pair(csf.path, std::move(csf))); }); } if(path_to_cloudSaveFile.empty()) { std::cout << "No local cloud saves found" << std::endl; return; } auto res = this->cloudSaveListByIdForEach(product_id, build_id, [&](cloudSaveFile &csf) { auto it = path_to_cloudSaveFile.find(csf.path); //If remote save is not locally stored, skip if(it == std::end(path_to_cloudSaveFile)) { return; } cloudSaveFile local_csf { std::move(it->second) }; path_to_cloudSaveFile.erase(it); if(Globals::globalConfig.bCloudForce || csf.lastModified < local_csf.lastModified) { iTotalRemainingBytes.fetch_add(local_csf.fileSize); dlCloudSaveQueue.push(local_csf); } }); for(auto &path_csf : path_to_cloudSaveFile) { auto &csf = path_csf.second; iTotalRemainingBytes.fetch_add(csf.fileSize); dlCloudSaveQueue.push(csf); } if(res || dlCloudSaveQueue.empty()) { return; } // Limit thread count to number of items in upload queue unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlCloudSaveQueue.size())); // Create download threads std::vector vThreads; for (unsigned int i = 0; i < iThreads; ++i) { DownloadInfo dlInfo; dlInfo.setStatus(DLSTATUS_NOTSTARTED); vDownloadInfo.push_back(dlInfo); vThreads.push_back(std::thread(Downloader::processCloudSaveUploadQueue, Globals::globalConfig, i)); } this->printProgress(dlCloudSaveQueue); // Join threads for (unsigned int i = 0; i < vThreads.size(); ++i) { vThreads[i].join(); } vThreads.clear(); vDownloadInfo.clear(); } void Downloader::deleteCloudSavesById(const std::string& product_id, const std::string& build_id) { if(Globals::globalConfig.cloudWhiteList.empty() && !Globals::globalConfig.bCloudForce) { std::cout << "No files have been whitelisted, either use \'--cloud-whitelist\' or \'--cloud-force\'" << std::endl; return; } curl_slist *header = nullptr; std::string access_token; if (!Globals::galaxyConf.isExpired()) { access_token = Globals::galaxyConf.getAccessToken(); } if (!access_token.empty()) { std::string bearer = "Authorization: Bearer " + access_token; header = curl_slist_append(header, bearer.c_str()); } auto dlhandle = curl_easy_init(); curl_easy_setopt(dlhandle, CURLOPT_HTTPHEADER, header); curl_easy_setopt(dlhandle, CURLOPT_CUSTOMREQUEST, "DELETE"); this->cloudSaveListByIdForEach(product_id, build_id, [dlhandle](cloudSaveFile &csf) { auto url = "https://cloudstorage.gog.com/v1/" + Globals::galaxyConf.getUserId() + '/' + Globals::galaxyConf.getClientId() + '/' + csf.path; curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); auto result = curl_easy_perform(dlhandle); if(result == CURLE_HTTP_RETURNED_ERROR) { long response_code = 0; curl_easy_getinfo(dlhandle, CURLINFO_RESPONSE_CODE, &response_code); std::cout << response_code << ": " << curl_easy_strerror(result); } }); curl_slist_free_all(header); curl_easy_cleanup(dlhandle); } void Downloader::downloadCloudSavesById(const std::string& product_id, const std::string& build_id) { auto res = this->cloudSaveListByIdForEach(product_id, build_id, [](cloudSaveFile &csf) { boost::filesystem::path filepath = csf.location; if(boost::filesystem::exists(filepath)) { // last_write_time minus a single second, since time_t is only accurate to the second unlike boost::posix_time::ptime auto time = boost::posix_time::from_time_t(boost::filesystem::last_write_time(filepath) - 1); if(!Globals::globalConfig.bCloudForce && time <= csf.lastModified) { std::cout << "Already up to date -- skipping: " << csf.path << std::endl; return; // This file is already completed } } if(boost::filesystem::is_directory(filepath)) { std::cout << "is a directory: " << csf.location << std::endl; return; } iTotalRemainingBytes.fetch_add(csf.fileSize); dlCloudSaveQueue.push(std::move(csf)); }); if(res || dlCloudSaveQueue.empty()) { return; } // Limit thread count to number of items in download queue unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlCloudSaveQueue.size())); // Create download threads std::vector vThreads; for (unsigned int i = 0; i < iThreads; ++i) { DownloadInfo dlInfo; dlInfo.setStatus(DLSTATUS_NOTSTARTED); vDownloadInfo.push_back(dlInfo); vThreads.push_back(std::thread(Downloader::processCloudSaveDownloadQueue, Globals::globalConfig, i)); } this->printProgress(dlCloudSaveQueue); // Join threads for (unsigned int i = 0; i < vThreads.size(); ++i) { vThreads[i].join(); } vThreads.clear(); vDownloadInfo.clear(); } void Downloader::galaxyShowCloudSavesById(const std::string& product_id, const std::string& build_id) { this->cloudSaveListByIdForEach(product_id, build_id, [](cloudSaveFile &csf) { boost::filesystem::path filepath = csf.location; filepath = boost::filesystem::absolute(filepath, boost::filesystem::current_path()); if(boost::filesystem::exists(filepath)) { auto size = boost::filesystem::file_size(filepath); // last_write_time minus a single second, since time_t is only accurate to the second unlike boost::posix_time::ptime auto time = boost::filesystem::last_write_time(filepath) - 1; if(csf.fileSize < size) { std::cout << csf.path << " :: not yet completed download" << std::endl; } else if(boost::posix_time::from_time_t(time) <= csf.lastModified) { std::cout << csf.path << " :: Already up to date" << std::endl; } else { std::cout << csf.path << " :: Out of date" << std::endl; } } else { std::cout << csf.path << " :: Isn't downloaded yet" << std::endl; } }); } void Downloader::galaxyShowLocalCloudSavesById(const std::string& product_id, const std::string& build_id) { auto name_to_locations = cloudSaveLocations(product_id, build_id); if(name_to_locations.empty()) { std::cout << "Cloud saves not supported for this game" << std::endl; } std::map path_to_cloudSaveFile; for(auto &name_to_location : name_to_locations) { auto &name = name_to_location.first; auto &location = name_to_location.second; if(!boost::filesystem::exists(location) || !boost::filesystem::is_directory(location)) { continue; } dirForEach(location, [&](boost::filesystem::directory_iterator file) { auto path = (name / boost::filesystem::relative(*file, location)).string(); if(!whitelisted(path)) { return; } cloudSaveFile csf { boost::posix_time::from_time_t(boost::filesystem::last_write_time(*file) - 1), boost::filesystem::file_size(*file), std::move(path), file->path().string() }; path_to_cloudSaveFile.insert(std::make_pair(csf.path, std::move(csf))); }); } if(path_to_cloudSaveFile.empty()) { std::cout << "No local cloud saves found" << std::endl; return; } this->cloudSaveListByIdForEach(product_id, build_id, [&](cloudSaveFile &csf) { auto it = path_to_cloudSaveFile.find(csf.path); //If remote save is not locally stored, skip if(it == std::end(path_to_cloudSaveFile)) { return; } cloudSaveFile local_csf { std::move(it->second) }; path_to_cloudSaveFile.erase(it); std::cout << csf.path << ": "; if(csf.lastModified < local_csf.lastModified) { std::cout << "remote save out of date: it should be synchronized" << std::endl; } else { std::cout << "up to date" << std::endl; } }); for(auto &path_csf : path_to_cloudSaveFile) { auto &csf = path_csf.second; std::cout << csf.path << ": there's only a local copy" << std::endl; } } std::vector Downloader::galaxyGetOrphanedFiles(const std::vector& items, const std::string& install_path) { std::vector orphans; std::vector item_paths; for (unsigned int i = 0; i < items.size(); ++i) item_paths.push_back(install_path + "/" + items[i].path); std::vector filepath_vector; try { std::size_t pathlen = Globals::globalConfig.dirConf.sDirectory.length(); if (boost::filesystem::exists(install_path)) { if (boost::filesystem::is_directory(install_path)) { // Recursively iterate over files in directory boost::filesystem::recursive_directory_iterator end_iter; boost::filesystem::recursive_directory_iterator dir_iter(install_path); while (dir_iter != end_iter) { if (boost::filesystem::is_regular_file(dir_iter->status())) { std::string filepath = dir_iter->path().string(); if (Globals::globalConfig.ignorelist.isBlacklisted(filepath.substr(pathlen))) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cerr << "skipped ignorelisted file " << filepath << std::endl; } else if (Globals::globalConfig.blacklist.isBlacklisted(filepath.substr(pathlen))) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cerr << "skipped blacklisted file " << filepath << std::endl; } else { filepath_vector.push_back(dir_iter->path()); } } dir_iter++; } } } else std::cerr << install_path << " does not exist" << std::endl; } catch (const boost::filesystem::filesystem_error& ex) { std::cout << ex.what() << std::endl; } std::sort(item_paths.begin(), item_paths.end()); std::sort(filepath_vector.begin(), filepath_vector.end()); if (!filepath_vector.empty()) { for (unsigned int i = 0; i < filepath_vector.size(); ++i) { bool bFileIsOrphaned = true; for (std::vector::iterator it = item_paths.begin(); it != item_paths.end(); it++) { boost::filesystem::path item_path = *it; boost::filesystem::path file_path = filepath_vector[i].native(); if (item_path == file_path) { bFileIsOrphaned = false; item_paths.erase(it); break; } } if (bFileIsOrphaned) orphans.push_back(filepath_vector[i].string()); } } return orphans; } void Downloader::galaxyInstallGame_MojoSetupHack(const std::string& product_id) { DownloadConfig dlConf = Globals::globalConfig.dlConf; dlConf.iInclude |= GlobalConstants::GFTYPE_BASE_INSTALLER; dlConf.iInstallerPlatform = dlConf.iGalaxyPlatform; dlConf.iInstallerLanguage = dlConf.iGalaxyLanguage; Json::Value product_info = gogGalaxy->getProductInfo(product_id); gameDetails game = gogGalaxy->productInfoJsonToGameDetails(product_info, dlConf); std::vector vInstallers; if (!game.installers.empty()) { vInstallers.push_back(game.installers[0]); for (unsigned int i = 0; i < game.dlcs.size(); ++i) { if (!game.dlcs[i].installers.empty()) vInstallers.push_back(game.dlcs[i].installers[0]); } } if (!vInstallers.empty()) { std::vector zipFileEntries; for (unsigned int i = 0; i < vInstallers.size(); ++i) { std::vector vFiles; std::cout << "Getting file list for " << vInstallers[i].gamename << "/" << vInstallers[i].id << std::endl; if (this->mojoSetupGetFileVector(vInstallers[i], vFiles)) { std::cerr << "Failed to get file list" << std::endl; return; } else { zipFileEntries.insert(std::end(zipFileEntries), std::begin(vFiles), std::end(vFiles)); } } std::string install_directory; if (Globals::globalConfig.dirConf.bSubDirectories) { Json::Value windows_builds = gogGalaxy->getProductBuilds(product_id, "windows"); if (!windows_builds.empty()) { std::string link = windows_builds["items"][0]["link"].asString(); std::string buildHash; buildHash.assign(link.begin()+link.find_last_of("/")+1, link.end()); Json::Value manifest = gogGalaxy->getManifestV2(buildHash); if (!manifest.empty()) { install_directory = this->getGalaxyInstallDirectory(gogGalaxy, manifest); } } } std::string install_path = Globals::globalConfig.dirConf.sDirectory + "/" + install_directory + "/"; std::vector vZipDirectories; std::vector vZipFiles; std::vector vZipFilesSymlink; std::vector vZipFilesSplit; // Determine if installer contains split files and get list of base file paths std::vector vSplitFileBasePaths; for (const auto& zfe : zipFileEntries) { std::string noarch = "data/noarch/"; std::string split_files = noarch + "support/split_files"; if (zfe.filepath.find(split_files) != std::string::npos) { std::cout << "Getting info about split files" << std::endl; std::string url = zfe.installer_url; std::string dlrange = std::to_string(zfe.start_offset_mojosetup) + "-" + std::to_string(zfe.end_offset); curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); std::stringstream splitfiles_compressed; std::stringstream splitfiles_uncompressed; CURLcode result = CURLE_RECV_ERROR; curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &splitfiles_compressed); curl_easy_setopt(curlhandle, CURLOPT_RANGE, dlrange.c_str()); result = curl_easy_perform(curlhandle); curl_easy_setopt(curlhandle, CURLOPT_RANGE, NULL); if (result == CURLE_OK) { if (ZipUtil::extractStream(&splitfiles_compressed, &splitfiles_uncompressed) == 0) { std::string path; while (std::getline(splitfiles_uncompressed, path)) { // Replace the leading "./" in base file path with install path Util::replaceString(path, "./", install_path); while (Util::replaceString(path, "//", "/")); // Replace any double slashes with single slash vSplitFileBasePaths.push_back(path); } } } } } bool bContainsSplitFiles = !vSplitFileBasePaths.empty(); for (std::uintmax_t i = 0; i < zipFileEntries.size(); ++i) { // Ignore all files and directories that are not in "data/noarch/" directory std::string noarch = "data/noarch/"; if (zipFileEntries[i].filepath.find(noarch) == std::string::npos || zipFileEntries[i].filepath == noarch) continue; zipFileEntry zfe = zipFileEntries[i]; Util::replaceString(zfe.filepath, noarch, install_path); while (Util::replaceString(zfe.filepath, "//", "/")); // Replace any double slashes with single slash if (zfe.filepath.at(zfe.filepath.length()-1) == '/') vZipDirectories.push_back(zfe); else if (ZipUtil::isSymlink(zfe.file_attributes)) vZipFilesSymlink.push_back(zfe); else { // Check for split files if (bContainsSplitFiles) { boost::regex expression("^(.*)(\\.split\\d+)$"); boost::match_results what; if (boost::regex_search(zfe.filepath, what, expression)) { std::string basePath = what[1]; std::string partExt = what[2]; // Check against list of base file paths read from "data/noarch/support/split_files" if ( std::any_of( vSplitFileBasePaths.begin(), vSplitFileBasePaths.end(), [basePath](const std::string& path) { return path == basePath; } ) ) { zfe.isSplitFile = true; zfe.splitFileBasePath = basePath; zfe.splitFilePartExt = partExt; } } if (zfe.isSplitFile) vZipFilesSplit.push_back(zfe); else vZipFiles.push_back(zfe); } else { vZipFiles.push_back(zfe); } } } // Set start and end offsets for split files // Create map of split files for combining them later splitFilesMap mSplitFiles; if (!vZipFilesSplit.empty()) { std::sort(vZipFilesSplit.begin(), vZipFilesSplit.end(), [](const zipFileEntry& i, const zipFileEntry& j) -> bool { return i.filepath < j.filepath; }); std::string prevBasePath = ""; off_t prevEndOffset = 0; for (auto& zfe : vZipFilesSplit) { if (zfe.splitFileBasePath == prevBasePath) zfe.splitFileStartOffset = prevEndOffset; else zfe.splitFileStartOffset = 0; zfe.splitFileEndOffset = zfe.splitFileStartOffset + zfe.uncomp_size; prevBasePath = zfe.splitFileBasePath; prevEndOffset = zfe.splitFileEndOffset; if (mSplitFiles.count(zfe.splitFileBasePath) > 0) { mSplitFiles[zfe.splitFileBasePath].push_back(zfe); } else { std::vector vec; vec.push_back(zfe); mSplitFiles[zfe.splitFileBasePath] = vec; } } vZipFiles.insert(std::end(vZipFiles), std::begin(vZipFilesSplit), std::end(vZipFilesSplit)); } // Add files to download queue uintmax_t totalSize = 0; for (std::uintmax_t i = 0; i < vZipFiles.size(); ++i) { // Don't add blacklisted files if (Globals::globalConfig.blacklist.isBlacklisted(vZipFiles[i].filepath)) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cout << "Skipping blacklisted file: " << vZipFiles[i].filepath << std::endl; continue; } dlQueueGalaxy_MojoSetupHack.push(vZipFiles[i]); iTotalRemainingBytes.fetch_add(vZipFiles[i].comp_size); totalSize += vZipFiles[i].uncomp_size; } // Add symlinks to download queue for (std::uintmax_t i = 0; i < vZipFilesSymlink.size(); ++i) { // Don't add blacklisted files if (Globals::globalConfig.blacklist.isBlacklisted(vZipFilesSymlink[i].filepath)) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cout << "Skipping blacklisted file: " << vZipFilesSymlink[i].filepath << std::endl; continue; } dlQueueGalaxy_MojoSetupHack.push(vZipFilesSymlink[i]); iTotalRemainingBytes.fetch_add(vZipFilesSymlink[i].comp_size); totalSize += vZipFilesSymlink[i].uncomp_size; } std::cout << game.title << std::endl; std::cout << "Files: " << dlQueueGalaxy_MojoSetupHack.size() << std::endl; std::cout << "Total size installed: " << Util::makeSizeString(totalSize) << std::endl; if (Globals::globalConfig.dlConf.bFreeSpaceCheck) { boost::filesystem::path path = boost::filesystem::absolute(install_path); while(!boost::filesystem::exists(path) && !path.empty()) { path = path.parent_path(); } if(boost::filesystem::exists(path) && !path.empty()) { boost::filesystem::space_info space = boost::filesystem::space(path); if (space.available < totalSize) { std::cerr << "Not enough free space in " << boost::filesystem::canonical(path) << " (" << Util::makeSizeString(space.available) << ")"<< std::endl; exit(1); } } } // Create directories for (std::uintmax_t i = 0; i < vZipDirectories.size(); ++i) { if (!boost::filesystem::exists(vZipDirectories[i].filepath)) { if (!boost::filesystem::create_directories(vZipDirectories[i].filepath)) { std::cerr << "Failed to create directory " << vZipDirectories[i].filepath << std::endl; return; } } } // Limit thread count to number of items in download queue unsigned int iThreads = std::min(Globals::globalConfig.iThreads, static_cast(dlQueueGalaxy_MojoSetupHack.size())); // Create download threads std::vector vThreads; for (unsigned int i = 0; i < iThreads; ++i) { DownloadInfo dlInfo; dlInfo.setStatus(DLSTATUS_NOTSTARTED); vDownloadInfo.push_back(dlInfo); vThreads.push_back(std::thread(Downloader::processGalaxyDownloadQueue_MojoSetupHack, Globals::globalConfig, i)); } this->printProgress(dlQueueGalaxy_MojoSetupHack); // Join threads for (unsigned int i = 0; i < vThreads.size(); ++i) vThreads[i].join(); vThreads.clear(); vDownloadInfo.clear(); // Combine split files if (!mSplitFiles.empty()) { this->galaxyInstallGame_MojoSetupHack_CombineSplitFiles(mSplitFiles, true); } } else { std::cout << "No installers found" << std::endl; } } void Downloader::galaxyInstallGame_MojoSetupHack_CombineSplitFiles(const splitFilesMap& mSplitFiles, const bool& bAppendToFirst) { for (const auto& baseFile : mSplitFiles) { // Check that all parts exist bool bAllPartsExist = true; for (const auto& splitFile : baseFile.second) { if (!boost::filesystem::exists(splitFile.filepath)) { bAllPartsExist = false; break; } } bool bBaseFileExists = boost::filesystem::exists(baseFile.first); if (!bAllPartsExist) { if (bBaseFileExists) { // Base file exist and we're missing parts. // This should mean that we already have complete file. // So we can safely skip this file without informing the user continue; } else { // Base file doesn't exist and we're missing parts. Print message about it before skipping file. std::cout << baseFile.first << " is missing parts. Skipping this file." << std::endl; continue; } } // Delete base file if it already exists if (bBaseFileExists) { std::cout << baseFile.first << " already exists. Deleting old file." << std::endl; if (!boost::filesystem::remove(baseFile.first)) { std::cout << baseFile.first << ": Failed to delete" << std::endl; continue; } } std::cout << "Beginning to combine " << baseFile.first << std::endl; std::ofstream ofs; // Create base file for appending if we aren't appending to first part if (!bAppendToFirst) { ofs.open(baseFile.first, std::ios_base::binary | std::ios_base::app); if (!ofs.is_open()) { std::cout << "Failed to create " << baseFile.first << std::endl; continue; } } for (const auto& splitFile : baseFile.second) { std::cout << "\t" << splitFile.filepath << std::endl; // Append to first file is set and current file is first in vector. // Open file for appending and continue to next file if (bAppendToFirst && (&splitFile == &baseFile.second.front())) { ofs.open(splitFile.filepath, std::ios_base::binary | std::ios_base::app); if (!ofs.is_open()) { std::cout << "Failed to open " << splitFile.filepath << std::endl; break; } continue; } std::ifstream ifs(splitFile.filepath, std::ios_base::binary); if (!ifs) { std::cout << "Failed to open " << splitFile.filepath << ". Deleting incomplete file." << std::endl; ofs.close(); if (!boost::filesystem::remove(baseFile.first)) { std::cout << baseFile.first << ": Failed to delete" << std::endl; } break; } ofs << ifs.rdbuf(); ifs.close(); // Delete split file if (!boost::filesystem::remove(splitFile.filepath)) { std::cout << splitFile.filepath << ": Failed to delete" << std::endl; } } if (ofs) ofs.close(); // Appending to first file so we must rename it if (bAppendToFirst) { boost::filesystem::path splitFilePath = baseFile.second.front().filepath; boost::filesystem::path baseFilePath = baseFile.first; boost::system::error_code ec; boost::filesystem::rename(splitFilePath, baseFilePath, ec); if (ec) { std::cout << "Failed to rename " << splitFilePath.string() << "to " << baseFilePath.string(); } } } return; } void Downloader::processGalaxyDownloadQueue_MojoSetupHack(Config conf, const unsigned int& tid) { std::string msg_prefix = "[Thread #" + std::to_string(tid) + "]"; CURL* dlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(dlhandle, conf.curlConf); curl_easy_setopt(dlhandle, CURLOPT_NOPROGRESS, 0); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); curl_easy_setopt(dlhandle, CURLOPT_FILETIME, 1L); xferInfo xferinfo; xferinfo.tid = tid; xferinfo.curlhandle = dlhandle; curl_easy_setopt(dlhandle, CURLOPT_XFERINFOFUNCTION, Downloader::progressCallbackForThread); curl_easy_setopt(dlhandle, CURLOPT_XFERINFODATA, &xferinfo); zipFileEntry zfe; while (dlQueueGalaxy_MojoSetupHack.try_pop(zfe)) { vDownloadInfo[tid].setStatus(DLSTATUS_STARTING); iTotalRemainingBytes.fetch_sub(zfe.comp_size); boost::filesystem::path path = zfe.filepath; boost::filesystem::path path_tmp = zfe.filepath + ".lgogdltmp"; // Check that directory exists and create it boost::filesystem::path directory = path.parent_path(); mtx_create_directories.lock(); // Use mutex to avoid possible race conditions if (boost::filesystem::exists(directory)) { if (!boost::filesystem::is_directory(directory)) { msgQueue.push(Message(directory.string() + " is not directory", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); mtx_create_directories.unlock(); return; } } else { if (!boost::filesystem::create_directories(directory)) { msgQueue.push(Message("Failed to create directory: " + directory.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); mtx_create_directories.unlock(); return; } } mtx_create_directories.unlock(); vDownloadInfo[tid].setFilename(path.string()); if (ZipUtil::isSymlink(zfe.file_attributes)) { if (boost::filesystem::is_symlink(path)) { msgQueue.push(Message("Symlink already exists: " + path.string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); continue; } } else { if (zfe.isSplitFile) { if (boost::filesystem::exists(zfe.splitFileBasePath)) { msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file exists. Checking if it is same version.", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); std::string crc32 = Util::getFileHashRange(zfe.splitFileBasePath, RHASH_CRC32, zfe.splitFileStartOffset, zfe.splitFileEndOffset); if (crc32 == Util::formattedString("%08x", zfe.crc32)) { msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file is same version. Skipping file.", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); continue; } else { msgQueue.push(Message(path.string() + ": Complete file (" + zfe.splitFileBasePath + ") of split file is different version. Continuing to download file.", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); } } } if (boost::filesystem::exists(path)) { msgQueue.push(Message("File already exists: " + path.string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); off_t filesize = static_cast(boost::filesystem::file_size(path)); if (filesize == zfe.uncomp_size) { // File is same size if (Util::getFileHash(path.string(), RHASH_CRC32) == Util::formattedString("%08x", zfe.crc32)) { msgQueue.push(Message(path.string() + ": OK", MSGTYPE_SUCCESS, msg_prefix, MSGLEVEL_VERBOSE)); continue; } else { msgQueue.push(Message(path.string() + ": CRC32 mismatch. Deleting old file.", MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } } } else { // File size mismatch msgQueue.push(Message(path.string() + ": File size mismatch. Deleting old file.", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } } } } off_t resume_from = 0; if (boost::filesystem::exists(path_tmp)) { off_t filesize = static_cast(boost::filesystem::file_size(path_tmp)); if (filesize < zfe.comp_size) { // Continue resume_from = filesize; } else { // Delete old file msgQueue.push(Message(path_tmp.string() + ": Deleting old file.", MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); if (!boost::filesystem::remove(path_tmp)) { msgQueue.push(Message(path_tmp.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } } } std::string url = zfe.installer_url; std::string dlrange = std::to_string(zfe.start_offset_mojosetup) + "-" + std::to_string(zfe.end_offset); curl_easy_setopt(dlhandle, CURLOPT_URL, url.c_str()); if (ZipUtil::isSymlink(zfe.file_attributes)) { // Symlink std::stringstream symlink_compressed; std::stringstream symlink_uncompressed; std::string link_target; CURLcode result = CURLE_RECV_ERROR; curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &symlink_compressed); curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str()); vDownloadInfo[tid].setFilename(path.string()); if (conf.iWait > 0) usleep(conf.iWait); // Delay the request by specified time xferinfo.offset = 0; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); if (result != CURLE_OK) { symlink_compressed.str(std::string()); msgQueue.push(Message(path.string() + ": Failed to download", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } int res = ZipUtil::extractStream(&symlink_compressed, &symlink_uncompressed); symlink_compressed.str(std::string()); if (res != 0) { std::string msg = "Extraction failed ("; switch (res) { case 1: msg += "invalid input stream"; break; case 2: msg += "unsupported compression method"; break; case 3: msg += "invalid output stream"; break; case 4: msg += "zlib error"; break; default: msg += "unknown error"; break; } msg += ")"; msgQueue.push(Message(msg + " " + path.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); symlink_uncompressed.str(std::string()); continue; } link_target = symlink_uncompressed.str(); symlink_uncompressed.str(std::string()); if (!link_target.empty()) { if (!boost::filesystem::exists(path)) { msgQueue.push(Message(path.string() + ": Creating symlink to " + link_target, MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); boost::filesystem::create_symlink(link_target, path); } } } else { // Download file CURLcode result = CURLE_RECV_ERROR; off_t max_size_memory = 5 << 20; // 5MB if (zfe.comp_size < max_size_memory) // Handle small files in memory { std::ofstream ofs(path.string(), std::ofstream::out | std::ofstream::binary); if (!ofs) { msgQueue.push(Message("Failed to create " + path_tmp.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); continue; } std::stringstream data_compressed; vDownloadInfo[tid].setFilename(path.string()); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, &data_compressed); curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str()); xferinfo.offset = 0; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); if (result != CURLE_OK) { data_compressed.str(std::string()); ofs.close(); msgQueue.push(Message(path.string() + ": Failed to download", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); if (boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path)) { if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); } } continue; } int res = ZipUtil::extractStream(&data_compressed, &ofs); data_compressed.str(std::string()); ofs.close(); if (res != 0) { std::string msg = "Extraction failed ("; switch (res) { case 1: msg += "invalid input stream"; break; case 2: msg += "unsupported compression method"; break; case 3: msg += "invalid output stream"; break; case 4: msg += "zlib error"; break; default: msg += "unknown error"; break; } msg += ")"; msgQueue.push(Message(msg + " " + path.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); data_compressed.str(std::string()); if (boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path)) { if (!boost::filesystem::remove(path)) { msgQueue.push(Message(path.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); } } continue; } if (boost::filesystem::exists(path)) { // Set file permission boost::filesystem::perms permissions = ZipUtil::getBoostFilePermission(zfe.file_attributes); Util::setFilePermissions(path, permissions); // Set timestamp if (zfe.timestamp > 0) { try { boost::filesystem::last_write_time(path, zfe.timestamp); } catch(const boost::filesystem::filesystem_error& e) { msgQueue.push(Message(e.what(), MSGTYPE_WARNING, msg_prefix, MSGLEVEL_VERBOSE)); } } } } else // Use temporary file for bigger files { vDownloadInfo[tid].setFilename(path_tmp.string()); curl_easy_setopt(dlhandle, CURLOPT_WRITEFUNCTION, Downloader::writeData); curl_easy_setopt(dlhandle, CURLOPT_READFUNCTION, Downloader::readData); int iRetryCount = 0; do { if (iRetryCount != 0) msgQueue.push(Message("Retry " + std::to_string(iRetryCount) + "/" + std::to_string(conf.iRetries) + ": " + path_tmp.filename().string(), MSGTYPE_INFO, msg_prefix, MSGLEVEL_VERBOSE)); FILE* outfile; // File exists, resume if (resume_from > 0) { if ((outfile=fopen(path_tmp.string().c_str(), "r+"))!=NULL) { fseek(outfile, 0, SEEK_END); dlrange = std::to_string(zfe.start_offset_mojosetup + resume_from) + "-" + std::to_string(zfe.end_offset); curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str()); } else { msgQueue.push(Message("Failed to open " + path_tmp.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } else // File doesn't exist, create new file { if ((outfile=fopen(path_tmp.string().c_str(), "w"))!=NULL) { curl_easy_setopt(dlhandle, CURLOPT_WRITEDATA, outfile); curl_easy_setopt(dlhandle, CURLOPT_RANGE, dlrange.c_str()); } else { msgQueue.push(Message("Failed to create " + path_tmp.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); break; } } if (conf.iWait > 0) usleep(conf.iWait); // Delay the request by specified time xferinfo.offset = 0; xferinfo.timer.reset(); xferinfo.TimeAndSize.clear(); result = curl_easy_perform(dlhandle); fclose(outfile); if (result == CURLE_PARTIAL_FILE || result == CURLE_OPERATION_TIMEDOUT || result == CURLE_RECV_ERROR) { iRetryCount++; if (boost::filesystem::exists(path_tmp) && boost::filesystem::is_regular_file(path_tmp)) resume_from = static_cast(boost::filesystem::file_size(path_tmp)); } } while ((result == CURLE_PARTIAL_FILE || result == CURLE_OPERATION_TIMEDOUT || result == CURLE_RECV_ERROR) && (iRetryCount <= conf.iRetries)); if (result == CURLE_OK) { // Extract file int res = ZipUtil::extractFile(path_tmp.string(), path.string()); bool bFailed = false; if (res != 0) { bFailed = true; std::string msg = "Extraction failed ("; unsigned int msg_type = MSGTYPE_ERROR; switch (res) { case 1: msg += "failed to open input file"; break; case 2: msg += "unsupported compression method"; break; case 3: msg += "failed to create output file"; break; case 4: msg += "zlib error"; break; case 5: msg += "failed to set timestamp"; msg_type = MSGTYPE_WARNING; bFailed = false; break; default: msg += "unknown error"; break; } msg += ")"; msgQueue.push(Message(msg + " " + path_tmp.string(), msg_type, msg_prefix, MSGLEVEL_ALWAYS)); } if (bFailed) continue; if (boost::filesystem::exists(path_tmp) && boost::filesystem::is_regular_file(path_tmp)) { if (!boost::filesystem::remove(path_tmp)) { msgQueue.push(Message(path_tmp.string() + ": Failed to delete", MSGTYPE_ERROR, msg_prefix, MSGLEVEL_ALWAYS)); } } // Set file permission boost::filesystem::perms permissions = ZipUtil::getBoostFilePermission(zfe.file_attributes); if (boost::filesystem::exists(path)) Util::setFilePermissions(path, permissions); } else { msgQueue.push(Message("Download failed " + path_tmp.string(), MSGTYPE_ERROR, msg_prefix, MSGLEVEL_DEFAULT)); continue; } } } msgQueue.push(Message("Download complete: " + path.string(), MSGTYPE_SUCCESS, msg_prefix, MSGLEVEL_DEFAULT)); } vDownloadInfo[tid].setStatus(DLSTATUS_FINISHED); curl_easy_cleanup(dlhandle); return; } int Downloader::mojoSetupGetFileVector(const gameFile& gf, std::vector& vFiles) { Json::Value downlinkJson = gogGalaxy->getResponseJson(gf.galaxy_downlink_json_url); if (downlinkJson.empty()) { std::cerr << "Empty JSON response" << std::endl; return 1; } if (!downlinkJson.isMember("downlink")) { std::cerr << "Invalid JSON response" << std::endl; return 1; } std::string xml_url; if (downlinkJson.isMember("checksum")) { if (!downlinkJson["checksum"].empty()) xml_url = downlinkJson["checksum"].asString(); } else { std::cerr << "Invalid JSON response. Response doesn't contain XML url." << std::endl; return 1; } // Get XML data curl_off_t file_size = 0; bool bMissingXML = false; bool bXMLParsingError = false; std::string xml_data = gogGalaxy->getResponse(xml_url); if (xml_data.empty()) { std::cerr << "Failed to get XML data" << std::endl; bMissingXML = true; } if (!bMissingXML) { tinyxml2::XMLDocument xml; xml.Parse(xml_data.c_str()); tinyxml2::XMLElement *fileElem = xml.FirstChildElement("file"); if (!fileElem) { std::cerr << "Failed to parse XML data" << std::endl; bXMLParsingError = true; } else { std::string total_size = fileElem->Attribute("total_size"); try { file_size = std::stoull(total_size); } catch (std::invalid_argument& e) { file_size = 0; } } } std::string installer_url = downlinkJson["downlink"].asString(); if (installer_url.empty()) { std::cerr << "Failed to get installer url" << std::endl; return 1; } if (bXMLParsingError || bMissingXML || file_size == 0) { std::cerr << "Failed to get file size from XML data, trying to get Content-Length header" << std::endl; std::stringstream header; curl_easy_setopt(curlhandle, CURLOPT_URL, installer_url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_HEADER, 1); curl_easy_setopt(curlhandle, CURLOPT_NOBODY, 1); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &header); curl_easy_perform(curlhandle); curl_easy_setopt(curlhandle, CURLOPT_HEADER, 0); curl_easy_setopt(curlhandle, CURLOPT_NOBODY, 0); curl_easy_getinfo(curlhandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &file_size); } if (file_size <= 0) { std::cerr << "Failed to get file size" << std::endl; return 1; } off_t head_size = 100 << 10; // 100 kB off_t tail_size = 200 << 10; // 200 kB std::string head_range = "0-" + std::to_string(head_size); std::string tail_range = std::to_string(file_size - tail_size) + "-" + std::to_string(file_size); CURLcode result; // Get head std::stringstream head; curl_easy_setopt(curlhandle, CURLOPT_URL, installer_url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &head); curl_easy_setopt(curlhandle, CURLOPT_RANGE, head_range.c_str()); result = curl_easy_perform(curlhandle); if (result != CURLE_OK) { std::cerr << "Failed to download data" << std::endl; return 1; } // Get zip start offset in MojoSetup installer off_t mojosetup_zip_offset = 0; off_t mojosetup_script_size = ZipUtil::getMojoSetupScriptSize(&head); head.seekg(0, head.beg); off_t mojosetup_installer_size = ZipUtil::getMojoSetupInstallerSize(&head); head.str(std::string()); if (mojosetup_script_size == -1 || mojosetup_installer_size == -1) { std::cerr << "Failed to get Zip offset" << std::endl; return 1; } else { mojosetup_zip_offset = mojosetup_script_size + mojosetup_installer_size; } // Get tail std::stringstream tail; curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail); curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str()); result = curl_easy_perform(curlhandle); if (result != CURLE_OK) { std::cerr << "Failed to download data" << std::endl; return 1; } off_t offset_zip_eocd = ZipUtil::getZipEOCDOffset(&tail); off_t offset_zip64_eocd = ZipUtil::getZip64EOCDOffset(&tail); if (offset_zip_eocd < 0) { std::cerr << "Failed to find Zip EOCD offset" << std::endl; return 1; } zipEOCD eocd = ZipUtil::readZipEOCDStruct(&tail, offset_zip_eocd); uint64_t cd_offset = eocd.cd_start_offset; uint64_t cd_total = eocd.total_cd_records; if (offset_zip64_eocd >= 0) { zip64EOCD eocd64 = ZipUtil::readZip64EOCDStruct(&tail, offset_zip64_eocd); if (cd_offset == UINT32_MAX) cd_offset = eocd64.cd_offset; if (cd_total == UINT16_MAX) cd_total = eocd64.cd_total; } off_t cd_offset_in_stream = 0; off_t mojosetup_cd_offset = mojosetup_zip_offset + cd_offset; off_t cd_offset_from_file_end = file_size - mojosetup_cd_offset; if (cd_offset_from_file_end > tail_size) { tail.str(std::string()); tail_range = std::to_string(mojosetup_cd_offset) + "-" + std::to_string(file_size); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &tail); curl_easy_setopt(curlhandle, CURLOPT_RANGE, tail_range.c_str()); result = curl_easy_perform(curlhandle); if (result != CURLE_OK) { std::cerr << "Failed to download data" << std::endl; return 1; } } else { cd_offset_in_stream = tail_size - cd_offset_from_file_end; } tail.seekg(cd_offset_in_stream, tail.beg); uint32_t signature = ZipUtil::readUInt32(&tail); if (signature != ZIP_CD_HEADER_SIGNATURE) { std::cerr << "Failed to find Zip Central Directory" << std::endl; return 1; } // Read file entries from Zip Central Directory tail.seekg(cd_offset_in_stream, tail.beg); for (std::uint64_t i = 0; i < cd_total; ++i) { zipCDEntry cd; cd = ZipUtil::readZipCDEntry(&tail); zipFileEntry zfe; zfe.filepath = cd.filename; zfe.comp_size = cd.comp_size; zfe.uncomp_size = cd.uncomp_size; zfe.start_offset_zip = cd.disk_offset; zfe.start_offset_mojosetup = zfe.start_offset_zip + mojosetup_zip_offset; zfe.file_attributes = cd.external_file_attr >> 16; zfe.crc32 = cd.crc32; zfe.timestamp = cd.timestamp; zfe.installer_url = installer_url; vFiles.push_back(zfe); } tail.str(std::string()); // Set end offset for all entries vFiles[vFiles.size() - 1].end_offset = mojosetup_cd_offset - 1; for (std::uintmax_t i = 0; i < (vFiles.size() - 1); i++) { vFiles[i].end_offset = vFiles[i+1].start_offset_mojosetup - 1; } return 0; } std::string Downloader::getGalaxyInstallDirectory(galaxyAPI *galaxyHandle, const Json::Value& manifest) { std::string install_directory = Globals::globalConfig.dirConf.sGalaxyInstallSubdir; std::string product_id = manifest["baseProductId"].asString(); // Templates for installation subdir std::map templates; templates["%install_dir%"] = manifest["installDirectory"].asString(); templates["%product_id%"] = product_id; std::vector templates_need_info = { "%gamename%", "%title%", "%title_stripped%" }; if (std::any_of(templates_need_info.begin(), templates_need_info.end(), [install_directory](std::string template_dir){return template_dir == install_directory;})) { Json::Value productInfo = galaxyHandle->getProductInfo(product_id); std::string gamename = productInfo["slug"].asString(); std::string title = productInfo["title"].asString(); if (!gamename.empty()) templates["%gamename%"] = productInfo["slug"].asString(); if (!title.empty()) templates["%title%"] = productInfo["title"].asString(); } if (templates.count("%install_dir%")) { templates["%install_dir_stripped%"] = Util::getStrippedString(templates["%install_dir%"]); } if (templates.count("%title%")) { templates["%title_stripped%"] = Util::getStrippedString(templates["%title%"]);; } if (templates.count(install_directory)) { install_directory = templates[install_directory]; } return install_directory; } void Downloader::printGameDetailsAsText(gameDetails& game) { std::cout << "gamename: " << game.gamename << std::endl << "product id: " << game.product_id << std::endl << "title: " << game.title << std::endl << "icon: " << game.icon << std::endl; if (!game.serials.empty()) std::cout << "serials:" << std::endl << game.serials << std::endl; // List installers if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_BASE_INSTALLER) && !game.installers.empty()) { std::cout << "installers: " << std::endl; for (auto gf : game.installers) { this->printGameFileDetailsAsText(gf); } } // List extras if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_BASE_EXTRA) && !game.extras.empty()) { std::cout << "extras: " << std::endl; for (auto gf : game.extras) { this->printGameFileDetailsAsText(gf); } } // List patches if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_BASE_PATCH) && !game.patches.empty()) { std::cout << "patches: " << std::endl; for (auto gf : game.patches) { this->printGameFileDetailsAsText(gf); } } // List language packs if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_BASE_LANGPACK) && !game.languagepacks.empty()) { std::cout << "language packs: " << std::endl; for (auto gf : game.languagepacks) { this->printGameFileDetailsAsText(gf); } } if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC) && !game.dlcs.empty()) { std::cout << "DLCs: " << std::endl; for (auto dlc : game.dlcs) { std::cout << "DLC gamename: " << dlc.gamename << std::endl << "product id: " << dlc.product_id << std::endl; if (!dlc.serials.empty()) std::cout << "serials:" << dlc.serials << std::endl; if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC_INSTALLER) && !dlc.installers.empty()) { for (auto gf : dlc.installers) { this->printGameFileDetailsAsText(gf); } } if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC_PATCH) && !dlc.patches.empty()) { for (auto gf : dlc.patches) { this->printGameFileDetailsAsText(gf); } } if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC_EXTRA) && !dlc.extras.empty()) { for (auto gf : dlc.extras) { this->printGameFileDetailsAsText(gf); } } if ((Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC_LANGPACK) && !dlc.languagepacks.empty()) { for (auto gf : dlc.languagepacks) { this->printGameFileDetailsAsText(gf); } } } } } void Downloader::printGameFileDetailsAsText(gameFile& gf) { std::string filepath = gf.getFilepath(); if (Globals::globalConfig.blacklist.isBlacklisted(filepath)) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cerr << "skipped blacklisted file " << filepath << std::endl; return; } std::cout << "\tid: " << gf.id << std::endl << "\tname: " << gf.name << std::endl << "\tpath: " << gf.path << std::endl << "\tsize: " << gf.size << std::endl; if (gf.type & GlobalConstants::GFTYPE_INSTALLER) std::cout << "\tupdated: " << (gf.updated ? "True" : "False") << std::endl; if (gf.type & (GlobalConstants::GFTYPE_INSTALLER | GlobalConstants::GFTYPE_PATCH)) { std::string languages = Util::getOptionNameString(gf.language, GlobalConstants::LANGUAGES); std::cout << "\tlanguage: " << languages << std::endl; } if (gf.type & GlobalConstants::GFTYPE_INSTALLER) std::cout << "\tversion: " << gf.version << std::endl; std::cout << std::endl; } lgogdownloader-3.17/src/galaxyapi.cpp000066400000000000000000000674161476654310300177540ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "galaxyapi.h" #include "message.h" #include "ziputil.h" #include #include #include #include #include GalaxyConfig Globals::galaxyConf; size_t galaxyAPI::writeMemoryCallback(char *ptr, size_t size, size_t nmemb, void *userp) { std::ostringstream *stream = (std::ostringstream*)userp; std::streamsize count = (std::streamsize) size * nmemb; stream->write(ptr, count); return count; } galaxyAPI::galaxyAPI(CurlConfig& conf) { this->curlConf = conf; curlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(curlhandle, this->curlConf); } galaxyAPI::~galaxyAPI() { curl_easy_cleanup(curlhandle); } /* Initialize the API returns 0 if failed returns 1 if successful */ int galaxyAPI::init() { int res = 0; if (!this->isTokenExpired()) { res = 1; } else res = 0; return res; } bool galaxyAPI::refreshLogin(const std::string &clientId, const std::string &clientSecret, const std::string &refreshToken, bool newSession) { std::string refresh_url = "https://auth.gog.com/token?client_id=" + clientId + "&client_secret=" + clientSecret + "&grant_type=refresh_token" + "&refresh_token=" + refreshToken + (newSession ? "" : "&without_new_session=1"); // std::cout << refresh_url << std::endl; Json::Value token_json = this->getResponseJson(refresh_url); if (token_json.empty()) return false; token_json["client_id"] = clientId; token_json["client_secret"] = clientSecret; Globals::galaxyConf.setJSON(token_json); return true; } bool galaxyAPI::refreshLogin() { return refreshLogin(Globals::galaxyConf.getClientId(), Globals::galaxyConf.getClientSecret(), Globals::galaxyConf.getRefreshToken(), true); } bool galaxyAPI::isTokenExpired() { bool res = false; if (Globals::galaxyConf.isExpired()) res = true; return res; } std::string galaxyAPI::getResponse(const std::string& url, const char *encoding) { struct curl_slist *header = NULL; std::string access_token; if (!Globals::galaxyConf.isExpired()) access_token = Globals::galaxyConf.getAccessToken(); if (!access_token.empty()) { std::string bearer = "Authorization: Bearer " + access_token; header = curl_slist_append(header, bearer.c_str()); } if(encoding) { auto accept = "Accept: " + std::string(encoding); header = curl_slist_append(header, accept.c_str()); } curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, header); curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_ACCEPT_ENCODING, ""); int max_retries = std::min(3, Globals::globalConfig.iRetries); std::string response; auto res = Util::CurlHandleGetResponse(curlhandle, response, max_retries); if(res) { long int response_code = 0; curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_VERBOSE) std::cout << "Response code for " << url << " is [" << response_code << ']' << std::endl; } curl_easy_setopt(curlhandle, CURLOPT_ACCEPT_ENCODING, NULL); curl_easy_setopt(curlhandle, CURLOPT_HTTPHEADER, NULL); curl_slist_free_all(header); return response; } Json::Value galaxyAPI::getResponseJson(const std::string& url, const char *encoding) { std::istringstream response(this->getResponse(url, encoding)); Json::Value json; if (!response.str().empty()) { try { response >> json; } catch(const Json::Exception& exc) { // Failed to parse json response // Check for zlib header and decompress if header found response.seekg(0, response.beg); uint16_t header = ZipUtil::readUInt16(&response); std::vector zlib_headers = { 0x0178, 0x5e78, 0x9c78, 0xda78 }; bool is_zlib_compressed = std::any_of( zlib_headers.begin(), zlib_headers.end(), [header](uint16_t zlib_header) { return header == zlib_header; } ); if (is_zlib_compressed) { std::string response_compressed = response.str(); std::stringstream response_decompressed; boost::iostreams::filtering_streambuf in; in.push(boost::iostreams::zlib_decompressor(GlobalConstants::ZLIB_WINDOW_SIZE)); in.push(boost::make_iterator_range(response_compressed)); boost::iostreams::copy(in, response_decompressed); try { response_decompressed >> json; } catch(const Json::Exception& exc) { // Failed to parse json std::cout << "Failed to parse json: " << exc.what(); } } else { std::cout << "Failed to parse json: " << exc.what(); } } } return json; } Json::Value galaxyAPI::getProductBuilds(const std::string& product_id, const std::string& platform, const std::string& generation) { std::string url = "https://content-system.gog.com/products/" + product_id + "/os/" + platform + "/builds?generation=" + generation; return this->getResponseJson(url); } Json::Value galaxyAPI::getManifestV1(const std::string& product_id, const std::string& build_id, const std::string& manifest_id, const std::string& platform) { std::string url = "https://cdn.gog.com/content-system/v1/manifests/" + product_id + "/" + platform + "/" + build_id + "/" + manifest_id + ".json"; return this->getManifestV1(url); } Json::Value galaxyAPI::getManifestV1(const std::string& manifest_url) { return this->getResponseJson(manifest_url); } Json::Value galaxyAPI::getManifestV2(std::string manifest_hash, const bool& is_dependency) { if (!manifest_hash.empty() && manifest_hash.find("/") == std::string::npos) manifest_hash = this->hashToGalaxyPath(manifest_hash); std::string url; if (is_dependency) url = "https://cdn.gog.com/content-system/v2/dependencies/meta/" + manifest_hash; else url = "https://cdn.gog.com/content-system/v2/meta/" + manifest_hash; return this->getResponseJson(url); } Json::Value galaxyAPI::getCloudPathAsJson(const std::string &clientId) { std::string url = "https://remote-config.gog.com/components/galaxy_client/clients/" + clientId + "?component_version=2.0.51"; return this->getResponseJson(url); } Json::Value galaxyAPI::getSecureLink(const std::string& product_id, const std::string& path) { std::string url = "https://content-system.gog.com/products/" + product_id + "/secure_link?generation=2&path=" + path + "&_version=2"; return this->getResponseJson(url); } Json::Value galaxyAPI::getDependencyLink(const std::string& path) { std::string url = "https://content-system.gog.com/open_link?generation=2&_version=2&path=/dependencies/store/" + path; return this->getResponseJson(url); } std::string galaxyAPI::hashToGalaxyPath(const std::string& hash) { std::string galaxy_path = hash; if (galaxy_path.find("/") == std::string::npos) galaxy_path.assign(hash.begin(), hash.begin()+2).append("/").append(hash.begin()+2, hash.begin()+4).append("/").append(hash); return galaxy_path; } std::vector galaxyAPI::getDepotItemsVector(const std::string& hash, const bool& is_dependency) { Json::Value json = this->getManifestV2(hash, is_dependency); std::vector items; if (json["depot"].isMember("smallFilesContainer")) { if (json["depot"]["smallFilesContainer"]["chunks"].isArray()) { galaxyDepotItem item; item.totalSizeCompressed = 0; item.totalSizeUncompressed = 0; item.path = "galaxy_smallfilescontainer"; item.isDependency = is_dependency; item.isSmallFilesContainer = true; for (unsigned int i = 0; i < json["depot"]["smallFilesContainer"]["chunks"].size(); ++i) { Json::Value json_chunk = json["depot"]["smallFilesContainer"]["chunks"][i]; galaxyDepotItemChunk chunk; chunk.md5_compressed = json_chunk["compressedMd5"].asString(); chunk.md5_uncompressed = json_chunk["md5"].asString(); chunk.size_compressed = json_chunk["compressedSize"].asLargestUInt(); chunk.size_uncompressed = json_chunk["size"].asLargestUInt(); chunk.offset_compressed = item.totalSizeCompressed; chunk.offset_uncompressed = item.totalSizeUncompressed; item.totalSizeCompressed += chunk.size_compressed; item.totalSizeUncompressed += chunk.size_uncompressed; item.chunks.push_back(chunk); } if (json["depot"]["smallFilesContainer"].isMember("md5")) item.md5 = json["depot"]["smallFilesContainer"]["md5"].asString(); else if (json["depot"]["smallFilesContainer"]["chunks"].size() == 1) item.md5 = json["depot"]["smallFilesContainer"]["chunks"][0]["md5"].asString(); else item.md5 = std::string(); items.push_back(item); } } for (unsigned int i = 0; i < json["depot"]["items"].size(); ++i) { if (json["depot"]["items"][i]["chunks"].isArray()) { galaxyDepotItem item; item.totalSizeCompressed = 0; item.totalSizeUncompressed = 0; item.path = json["depot"]["items"][i]["path"].asString(); item.isDependency = is_dependency; if (json["depot"]["items"][i].isMember("sfcRef")) { item.isInSFC = true; item.sfc_offset = json["depot"]["items"][i]["sfcRef"]["offset"].asLargestUInt(); item.sfc_size = json["depot"]["items"][i]["sfcRef"]["size"].asLargestUInt(); } if (Globals::globalConfig.dlConf.bGalaxyLowercasePath && Globals::globalConfig.dlConf.iGalaxyPlatform == GlobalConstants::PLATFORM_WINDOWS) { boost::algorithm::to_lower(item.path); } while (Util::replaceString(item.path, "\\", "/")); for (unsigned int j = 0; j < json["depot"]["items"][i]["chunks"].size(); ++j) { Json::Value json_chunk = json["depot"]["items"][i]["chunks"][j]; galaxyDepotItemChunk chunk; chunk.md5_compressed = json_chunk["compressedMd5"].asString(); chunk.md5_uncompressed = json_chunk["md5"].asString(); chunk.size_compressed = json_chunk["compressedSize"].asLargestUInt(); chunk.size_uncompressed = json_chunk["size"].asLargestUInt(); chunk.offset_compressed = item.totalSizeCompressed; chunk.offset_uncompressed = item.totalSizeUncompressed; item.totalSizeCompressed += chunk.size_compressed; item.totalSizeUncompressed += chunk.size_uncompressed; item.chunks.push_back(chunk); } if (json["depot"]["items"][i].isMember("md5")) item.md5 = json["depot"]["items"][i]["md5"].asString(); else if (json["depot"]["items"][i]["chunks"].size() == 1) item.md5 = json["depot"]["items"][i]["chunks"][0]["md5"].asString(); else item.md5 = std::string(); items.push_back(item); } } return items; } Json::Value galaxyAPI::getProductInfo(const std::string& product_id) { std::string url = "https://api.gog.com/products/" + product_id + "?expand=downloads,expanded_dlcs,description,screenshots,videos,related_products,changelog&locale=en-US"; return this->getResponseJson(url); } gameDetails galaxyAPI::productInfoJsonToGameDetails(const Json::Value& json, const DownloadConfig& dlConf) { gameDetails gamedetails; gamedetails.gamename = json["slug"].asString(); gamedetails.product_id = json["id"].asString(); gamedetails.title = json["title"].asString(); gamedetails.icon = "https:" + json["images"]["icon"].asString(); gamedetails.logo = "https:" + json["images"]["logo"].asString(); Util::replaceString(gamedetails.logo, "_glx_logo.jpg", ".jpg"); if (json.isMember("changelog")) gamedetails.changelog = json["changelog"].asString(); if (dlConf.iInclude & GlobalConstants::GFTYPE_INSTALLER) { gamedetails.installers = this->fileJsonNodeToGameFileVector(gamedetails.gamename, json["downloads"]["installers"], GlobalConstants::GFTYPE_BASE_INSTALLER, dlConf); for (auto &item : gamedetails.installers) item.title = gamedetails.title; } if (dlConf.iInclude & GlobalConstants::GFTYPE_EXTRA) { gamedetails.extras = this->fileJsonNodeToGameFileVector(gamedetails.gamename, json["downloads"]["bonus_content"], GlobalConstants::GFTYPE_BASE_EXTRA, dlConf); for (auto &item : gamedetails.extras) item.title = gamedetails.title; } if (dlConf.iInclude & GlobalConstants::GFTYPE_PATCH) { gamedetails.patches = this->fileJsonNodeToGameFileVector(gamedetails.gamename, json["downloads"]["patches"], GlobalConstants::GFTYPE_BASE_PATCH, dlConf); for (auto &item : gamedetails.patches) item.title = gamedetails.title; } if (dlConf.iInclude & GlobalConstants::GFTYPE_LANGPACK) { gamedetails.languagepacks = this->fileJsonNodeToGameFileVector(gamedetails.gamename, json["downloads"]["language_packs"], GlobalConstants::GFTYPE_BASE_LANGPACK, dlConf); for (auto &item : gamedetails.languagepacks) item.title = gamedetails.title; } if (dlConf.iInclude & GlobalConstants::GFTYPE_DLC) { if (json.isMember("expanded_dlcs")) { for (unsigned int i = 0; i < json["expanded_dlcs"].size(); ++i) { std::string dlc_id = json["expanded_dlcs"][i]["id"].asString(); if (!Globals::vOwnedGamesIds.empty()) { if (std::find(Globals::vOwnedGamesIds.begin(), Globals::vOwnedGamesIds.end(), dlc_id) == Globals::vOwnedGamesIds.end()) continue; } gameDetails dlc_gamedetails = this->productInfoJsonToGameDetails(json["expanded_dlcs"][i], dlConf); dlc_gamedetails.title_basegame = gamedetails.title; dlc_gamedetails.gamename_basegame = gamedetails.gamename; // Add DLC type to all DLC files for (unsigned int j = 0; j < dlc_gamedetails.installers.size(); ++j) { dlc_gamedetails.installers[j].type = GlobalConstants::GFTYPE_DLC_INSTALLER; dlc_gamedetails.installers[j].title_basegame = dlc_gamedetails.title_basegame; dlc_gamedetails.installers[j].gamename_basegame = dlc_gamedetails.gamename_basegame; } for (unsigned int j = 0; j < dlc_gamedetails.extras.size(); ++j) { dlc_gamedetails.extras[j].type = GlobalConstants::GFTYPE_DLC_EXTRA; dlc_gamedetails.extras[j].title_basegame = dlc_gamedetails.title_basegame; dlc_gamedetails.extras[j].gamename_basegame = dlc_gamedetails.gamename_basegame; } for (unsigned int j = 0; j < dlc_gamedetails.patches.size(); ++j) { dlc_gamedetails.patches[j].type = GlobalConstants::GFTYPE_DLC_PATCH; dlc_gamedetails.patches[j].title_basegame = dlc_gamedetails.title_basegame; dlc_gamedetails.patches[j].gamename_basegame = dlc_gamedetails.gamename_basegame; } for (unsigned int j = 0; j < dlc_gamedetails.languagepacks.size(); ++j) { dlc_gamedetails.languagepacks[j].type = GlobalConstants::GFTYPE_DLC_LANGPACK; dlc_gamedetails.languagepacks[j].title_basegame = dlc_gamedetails.title_basegame; dlc_gamedetails.languagepacks[j].gamename_basegame = dlc_gamedetails.gamename_basegame; } // Add DLC only if it has any files if (!dlc_gamedetails.installers.empty() || !dlc_gamedetails.extras.empty() || !dlc_gamedetails.patches.empty() || !dlc_gamedetails.languagepacks.empty()) gamedetails.dlcs.push_back(dlc_gamedetails); } } } return gamedetails; } std::vector galaxyAPI::fileJsonNodeToGameFileVector(const std::string& gamename, const Json::Value& json, const unsigned int& type, const DownloadConfig& dlConf) { std::vector gamefiles; unsigned int iInfoNodes = json.size(); for (unsigned int i = 0; i < iInfoNodes; ++i) { Json::Value infoNode = json[i]; unsigned int iFiles = infoNode["files"].size(); std::string name = infoNode["name"].asString(); std::string version = ""; if (!infoNode["version"].empty()) version = infoNode["version"].asString(); unsigned int iPlatform = GlobalConstants::PLATFORM_WINDOWS; unsigned int iLanguage = GlobalConstants::LANGUAGE_EN; if (!(type & GlobalConstants::GFTYPE_EXTRA)) { iPlatform = Util::getOptionValue(infoNode["os"].asString(), GlobalConstants::PLATFORMS); iLanguage = Util::getOptionValue(infoNode["language"].asString(), GlobalConstants::LANGUAGES); if (!(iPlatform & dlConf.iInstallerPlatform)) continue; if (!(iLanguage & dlConf.iInstallerLanguage)) continue; } // Skip file if count and total_size is zero // https://github.com/Sude-/lgogdownloader/issues/200 unsigned int count = infoNode["count"].asUInt(); uintmax_t total_size = infoNode["total_size"].asLargestUInt(); if (count == 0 && total_size == 0) continue; for (unsigned int j = 0; j < iFiles; ++j) { Json::Value fileNode = infoNode["files"][j]; std::string downlink = fileNode["downlink"].asString(); Json::Value downlinkJson = this->getResponseJson(downlink); if (downlinkJson.empty()) continue; std::string downlink_url = downlinkJson["downlink"].asString(); std::string path = this->getPathFromDownlinkUrl(downlink_url, gamename); // Check to see if path ends in "/secure" or "/securex" which means that we got invalid path for some reason boost::regex path_re("/securex?$", boost::regex::perl | boost::regex::icase); boost::match_results what; if (boost::regex_search(path, what, path_re)) continue; gameFile gf; gf.gamename = gamename; gf.type = type; gf.id = fileNode["id"].asString(); gf.name = name; gf.path = path; gf.size = Util::getJsonUIntValueAsString(fileNode["size"]); gf.updated = 0; // assume not updated gf.galaxy_downlink_json_url = downlink; gf.version = version; if (!(type & GlobalConstants::GFTYPE_EXTRA)) { gf.platform = iPlatform; gf.language = iLanguage; } if (dlConf.bDuplicateHandler) { bool bDuplicate = false; for (unsigned int k = 0; k < gamefiles.size(); ++k) { if (gamefiles[k].path == gf.path) { if (!(type & GlobalConstants::GFTYPE_EXTRA)) gamefiles[k].language |= gf.language; // Add language code to installer bDuplicate = true; break; } } if (bDuplicate) continue; } gamefiles.push_back(gf); } } return gamefiles; } Json::Value galaxyAPI::getUserData() { std::string url = "https://embed.gog.com/userData.json"; return this->getResponseJson(url); } Json::Value galaxyAPI::getDependenciesJson() { std::string url = "https://content-system.gog.com/dependencies/repository?generation=2"; Json::Value dependencies; Json::Value repository = this->getResponseJson(url); if (!repository.empty()) { if (repository.isMember("repository_manifest")) { std::string manifest_url = repository["repository_manifest"].asString(); dependencies = this->getResponseJson(manifest_url); } } return dependencies; } std::vector galaxyAPI::getFilteredDepotItemsVectorFromJson(const Json::Value& depot_json, const std::string& galaxy_language, const std::string& galaxy_arch, const bool& is_dependency) { std::vector items; bool bSelectedLanguage = false; bool bSelectedArch = false; boost::regex language_re("^(" + galaxy_language + ")$", boost::regex::perl | boost::regex::icase); boost::match_results what; for (unsigned int j = 0; j < depot_json["languages"].size(); ++j) { std::string language = depot_json["languages"][j].asString(); if (language == "*" || boost::regex_search(language, what, language_re)) bSelectedLanguage = true; } if (depot_json.isMember("osBitness")) { for (unsigned int j = 0; j < depot_json["osBitness"].size(); ++j) { std::string osBitness = depot_json["osBitness"][j].asString(); if (osBitness == "*" || osBitness == galaxy_arch) bSelectedArch = true; } } else { // No osBitness found, assume that we want this depot bSelectedArch = true; } if (bSelectedLanguage && bSelectedArch) { std::string depotHash = depot_json["manifest"].asString(); std::string depot_product_id = depot_json["productId"].asString(); items = this->getDepotItemsVector(depotHash, is_dependency); // Set product id for items if (!depot_product_id.empty()) { for (auto it = items.begin(); it != items.end(); ++it) { it->product_id = depot_product_id; } } } return items; } std::string galaxyAPI::getPathFromDownlinkUrl(const std::string& downlink_url, const std::string& gamename) { std::string path; std::string downlink_url_unescaped = (std::string)curl_easy_unescape(curlhandle, downlink_url.c_str(), downlink_url.size(), NULL); size_t filename_start_pos = 0; // If url ends in "/" then remove it if (downlink_url_unescaped.back() == '/') downlink_url_unescaped.assign(downlink_url_unescaped.begin(), downlink_url_unescaped.end()-1); // Assume that filename starts after last "/" in url if (downlink_url_unescaped.find_last_of("/") != std::string::npos) filename_start_pos = downlink_url_unescaped.find_last_of("/") + 1; // Url contains "/gamename/" if (downlink_url_unescaped.find("/" + gamename + "/") != std::string::npos) filename_start_pos = downlink_url_unescaped.find("/" + gamename + "/"); // Assume that filename ends at the end of url size_t filename_end_pos = downlink_url_unescaped.length(); // Check to see if url has any query strings if (downlink_url_unescaped.find("?") != std::string::npos) { // Assume that filename ends at first "?" filename_end_pos = downlink_url_unescaped.find_first_of("?"); // Check for "?path=" if (downlink_url_unescaped.find("?path=") != std::string::npos) { size_t token_pos = downlink_url_unescaped.find("&token="); size_t access_token_pos = downlink_url_unescaped.find("&access_token="); if ((token_pos != std::string::npos) && (access_token_pos != std::string::npos)) { filename_end_pos = std::min(token_pos, access_token_pos); } else { if (downlink_url_unescaped.find_first_of("&") != std::string::npos) filename_end_pos = downlink_url_unescaped.find_first_of("&"); } } } path.assign(downlink_url_unescaped.begin()+filename_start_pos, downlink_url_unescaped.begin()+filename_end_pos); // Make sure that path contains "/gamename/" if (path.find("/" + gamename + "/") == std::string::npos) path = "/" + gamename + "/" + path; // Workaround for filename issue caused by different (currently unknown) url formatting scheme // https://github.com/Sude-/lgogdownloader/issues/126 if (path.find("?") != std::string::npos) { if (path.find_last_of("?") > path.find_last_of("/")) { path.assign(path.begin(), path.begin()+path.find_last_of("?")); } } return path; } std::vector galaxyAPI::cdnUrlTemplatesFromJson(const Json::Value& json, const std::vector& cdnPriority) { // Handle priority of CDNs struct urlPriority { std::string url; int priority; }; std::vector cdnUrls; // Build a vector of all urls and their priority score for (unsigned int i = 0; i < json["urls"].size(); ++i) { std::string endpoint_name = json["urls"][i]["endpoint_name"].asString(); unsigned int score = cdnPriority.size(); for (unsigned int idx = 0; idx < score; ++idx) { if (endpoint_name == cdnPriority[idx]) { score = idx; break; } } // Couldn't find a match when assigning score if (score == cdnPriority.size()) { // Add index value to score // This way unknown CDNs have priority based on the order they appear in json score += i; } // Build url according to url_format std::string url = json["urls"][i]["url_format"].asString(); for (auto cdn_url_template_param : json["urls"][i]["parameters"].getMemberNames()) { std::string template_to_replace = "{" + cdn_url_template_param + "}"; std::string replacement = json["urls"][i]["parameters"][cdn_url_template_param].asString(); // Add our own template to path if (template_to_replace == "{path}") { replacement += "{LGOGDOWNLOADER_GALAXY_PATH}"; } while(Util::replaceString(url, template_to_replace, replacement)); } urlPriority cdnurl; cdnurl.url = url; cdnurl.priority = score; cdnUrls.push_back(cdnurl); } if (!cdnUrls.empty()) { // Sort urls by priority (lowest score first) std::sort(cdnUrls.begin(), cdnUrls.end(), [](urlPriority a, urlPriority b) { return (a.priority < b.priority); } ); } std::vector cdnUrlTemplates; for (auto cdnurl : cdnUrls) cdnUrlTemplates.push_back(cdnurl.url); return cdnUrlTemplates; } lgogdownloader-3.17/src/gamedetails.cpp000066400000000000000000000350651476654310300202470ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "gamedetails.h" gameDetails::gameDetails() { //ctor } gameDetails::~gameDetails() { //dtor } void gameDetails::filterWithPriorities(const gameSpecificConfig& config) { if (config.dlConf.vPlatformPriority.empty() && config.dlConf.vLanguagePriority.empty()) return; filterListWithPriorities(installers, config); filterListWithPriorities(patches, config); filterListWithPriorities(languagepacks, config); for (unsigned int i = 0; i < dlcs.size(); ++i) { filterListWithPriorities(dlcs[i].installers, config); filterListWithPriorities(dlcs[i].patches, config); filterListWithPriorities(dlcs[i].languagepacks, config); } } void gameDetails::filterListWithPriorities(std::vector& list, const gameSpecificConfig& config) { /* Compute the score of each item - we use a scoring mechanism and we keep all ties Like if someone asked French then English and Linux then Windows, but there are only Windows French, Windows English and Linux English versions, we'll get the Windows French and Linux English ones. Score is inverted: lower is better. */ int bestscore = -1; for (std::vector::iterator fileDetails = list.begin(); fileDetails != list.end(); fileDetails++) { fileDetails->score = 0; if (!config.dlConf.vPlatformPriority.empty()) { for (size_t i = 0; i != config.dlConf.vPlatformPriority.size(); i++) if (fileDetails->platform & config.dlConf.vPlatformPriority[i]) { fileDetails->score += i; break; } } if (!config.dlConf.vLanguagePriority.empty()) { for (size_t i = 0; i != config.dlConf.vLanguagePriority.size(); i++) if (fileDetails->language & config.dlConf.vLanguagePriority[i]) { fileDetails->score += i; break; } } if ((fileDetails->score < bestscore) or (bestscore < 0)) bestscore = fileDetails->score; } for (std::vector::iterator fileDetails = list.begin(); fileDetails != list.end(); ) { if (fileDetails->score > bestscore) fileDetails = list.erase(fileDetails); else fileDetails++; } } void gameDetails::makeFilepaths(const DirectoryConfig& config) { std::string filepath; std::string logo_ext = ".jpg"; // Assume jpg std::string icon_ext = ".png"; // Assume png if (this->logo.rfind(".") != std::string::npos) logo_ext = this->logo.substr(this->logo.rfind(".")); if (this->icon.rfind(".") != std::string::npos) icon_ext = this->icon.substr(this->icon.rfind(".")); // Add gamename to filenames to make sure we don't overwrite them with files from dlcs std::string serials_filename = "serials_" + this->gamename + ".txt"; std::string logo_filename = "logo_" + this->gamename + logo_ext; std::string icon_filename = "icon_" + this->gamename + icon_ext; std::string product_json_filename = "product_" + this->gamename + ".json"; this->serialsFilepath = this->makeCustomFilepath(std::string("serials.txt"), *this, config); this->logoFilepath = this->makeCustomFilepath(logo_filename, *this, config); this->iconFilepath = this->makeCustomFilepath(icon_filename, *this, config); this->changelogFilepath = this->makeCustomFilepath(std::string("changelog_") + gamename + ".html", *this, config); this->gameDetailsJsonFilepath = this->makeCustomFilepath(std::string("game-details.json"), *this, config); this->productJsonFilepath = this->makeCustomFilepath(product_json_filename, *this, config); for (auto &installer : this->installers) { filepath = this->makeFilepath(installer, config); installer.setFilepath(filepath); } for (auto &extra : this->extras) { filepath = this->makeFilepath(extra, config); extra.setFilepath(filepath); } for (auto &patch : this->patches) { filepath = this->makeFilepath(patch, config); patch.setFilepath(filepath); } for (auto &languagepack : this->languagepacks) { filepath = this->makeFilepath(languagepack, config); languagepack.setFilepath(filepath); } for (auto &dlc : this->dlcs) { // Add gamename to filenames to make sure we don't overwrite basegame files with these std::string dlc_serials_filename = "serials_" + dlc.gamename + ".txt"; std::string dlc_logo_filename = "logo_" + dlc.gamename + logo_ext; std::string dlc_icon_filename = "icon_" + dlc.gamename + icon_ext; std::string dlc_product_json_filename = "product_" + dlc.gamename + ".json"; dlc.serialsFilepath = this->makeCustomFilepath(dlc_serials_filename, dlc, config); dlc.logoFilepath = this->makeCustomFilepath(dlc_logo_filename, dlc, config); dlc.iconFilepath = this->makeCustomFilepath(dlc_icon_filename, dlc, config); dlc.changelogFilepath = this->makeCustomFilepath(std::string("changelog_") + dlc.gamename + ".html", dlc, config); dlc.productJsonFilepath = this->makeCustomFilepath(dlc_product_json_filename, dlc, config); for (auto &installer : dlc.installers) { filepath = this->makeFilepath(installer, config); installer.setFilepath(filepath); } for (auto &extra : dlc.extras) { filepath = this->makeFilepath(extra, config); extra.setFilepath(filepath); } for (auto &patch : dlc.patches) { filepath = this->makeFilepath(patch, config); patch.setFilepath(filepath); } for (auto &languagepack : dlc.languagepacks) { filepath = this->makeFilepath(languagepack, config); languagepack.setFilepath(filepath); } } } Json::Value gameDetails::getDetailsAsJson() { Json::Value json; json["gamename"] = this->gamename; json["gamename_basegame"] = this->gamename_basegame; json["product_id"] = this->product_id; json["title"] = this->title; json["title_basegame"] = this->title_basegame; json["icon"] = this->icon; json["serials"] = this->serials; json["changelog"] = this->changelog; for (unsigned int i = 0; i < this->extras.size(); ++i) json["extras"].append(this->extras[i].getAsJson()); for (unsigned int i = 0; i < this->installers.size(); ++i) json["installers"].append(this->installers[i].getAsJson()); for (unsigned int i = 0; i < this->patches.size(); ++i) json["patches"].append(this->patches[i].getAsJson()); for (unsigned int i = 0; i < this->languagepacks.size(); ++i) json["languagepacks"].append(this->languagepacks[i].getAsJson()); if (!this->dlcs.empty()) { for (unsigned int i = 0; i < this->dlcs.size(); ++i) { json["dlcs"].append(this->dlcs[i].getDetailsAsJson()); } } return json; } std::string gameDetails::getSerialsFilepath() { return this->serialsFilepath; } std::string gameDetails::getLogoFilepath() { return this->logoFilepath; } std::string gameDetails::getIconFilepath() { return this->iconFilepath; } std::string gameDetails::getChangelogFilepath() { return this->changelogFilepath; } std::string gameDetails::getGameDetailsJsonFilepath() { return this->gameDetailsJsonFilepath; } std::string gameDetails::getProductJsonFilepath() { return this->productJsonFilepath; } // Return vector containing all game files std::vector gameDetails::getGameFileVector() { std::vector vGameFiles; vGameFiles.insert(std::end(vGameFiles), std::begin(installers), std::end(installers)); vGameFiles.insert(std::end(vGameFiles), std::begin(patches), std::end(patches)); vGameFiles.insert(std::end(vGameFiles), std::begin(extras), std::end(extras)); vGameFiles.insert(std::end(vGameFiles), std::begin(languagepacks), std::end(languagepacks)); if (!dlcs.empty()) { for (unsigned int i = 0; i < dlcs.size(); ++i) { std::vector vGameFilesDLC = dlcs[i].getGameFileVector(); vGameFiles.insert(std::end(vGameFiles), std::begin(vGameFilesDLC), std::end(vGameFilesDLC)); } } return vGameFiles; } // Return vector containing all game files matching download filters std::vector gameDetails::getGameFileVectorFiltered(const unsigned int& iType) { std::vector vGameFiles; for (auto gf : this->getGameFileVector()) { if (gf.type & iType) vGameFiles.push_back(gf); } return vGameFiles; } void gameDetails::filterWithType(const unsigned int& iType) { filterListWithType(installers, iType); filterListWithType(patches, iType); filterListWithType(extras, iType); filterListWithType(languagepacks, iType); for (unsigned int i = 0; i < dlcs.size(); ++i) { filterListWithType(dlcs[i].installers, iType); filterListWithType(dlcs[i].patches, iType); filterListWithType(dlcs[i].extras, iType); filterListWithType(dlcs[i].languagepacks, iType); } } void gameDetails::filterListWithType(std::vector& list, const unsigned int& iType) { for (std::vector::iterator gf = list.begin(); gf != list.end();) { if (!(gf->type & iType)) gf = list.erase(gf); else gf++; } } std::string gameDetails::makeFilepath(const gameFile& gf, const DirectoryConfig& dirConf) { std::map templates; std::string path = gf.path; std::string filename = path; if (path.find_last_of("/") != std::string::npos) filename = path.substr(path.find_last_of("/")+1, path.length()); std::string subdir; if (dirConf.bSubDirectories) { if (gf.type & GlobalConstants::GFTYPE_INSTALLER) { subdir = dirConf.sInstallersSubdir; } else if (gf.type & GlobalConstants::GFTYPE_EXTRA) { subdir = dirConf.sExtrasSubdir; } else if (gf.type & GlobalConstants::GFTYPE_PATCH) { subdir = dirConf.sPatchesSubdir; } else if (gf.type & GlobalConstants::GFTYPE_LANGPACK) { subdir = dirConf.sLanguagePackSubdir; } if (gf.type & GlobalConstants::GFTYPE_DLC) { subdir = dirConf.sDLCSubdir + "/" + subdir; } } if (!dirConf.sGameSubdir.empty()) { subdir = dirConf.sGameSubdir + "/" + subdir; } std::string gamename = gf.gamename; std::string title = gf.title; std::string dlc_gamename; std::string dlc_title; if (gf.type & GlobalConstants::GFTYPE_DLC) { gamename = gf.gamename_basegame; title = gf.title_basegame; dlc_gamename = gf.gamename; dlc_title = gf.title; } std::string filepath = dirConf.sDirectory + "/" + subdir + "/" + filename; std::string platform; for (unsigned int i = 0; i < GlobalConstants::PLATFORMS.size(); ++i) { if ((gf.platform & GlobalConstants::PLATFORMS[i].id) == GlobalConstants::PLATFORMS[i].id) { platform = boost::algorithm::to_lower_copy(GlobalConstants::PLATFORMS[i].str); break; } } if (platform.empty()) { if (filepath.find("%gamename%/%platform%") != std::string::npos) platform = ""; else platform = "no_platform"; } // Don't save certain files in "no_platform" folder std::string logo_filename = "/logo_" + gf.gamename + ".jpg"; std::string icon_filename = "/icon_" + gf.gamename + ".png"; std::string product_json_filename = "/product_" + gf.gamename + ".json"; if ( filepath.rfind(logo_filename) != std::string::npos || filepath.rfind(icon_filename) != std::string::npos || filepath.rfind(product_json_filename) != std::string::npos ) platform = ""; std::string gamename_firstletter; if (!gamename.empty()) { if (std::isdigit(gamename.front())) gamename_firstletter = "0"; else gamename_firstletter = gamename.front(); } std::string gamename_transformed; std::string gamename_transformed_firstletter; if (filepath.find("%gamename_transformed%") != std::string::npos || filepath.find("%gamename_transformed_firstletter%") != std::string::npos) { gamename_transformed = Util::transformGamename(gamename); if (!gamename_transformed.empty()) { if (std::isdigit(gamename_transformed.front())) gamename_transformed_firstletter = "0"; else gamename_transformed_firstletter = gamename_transformed.front(); } } templates["%gamename%"] = gamename; templates["%gamename_firstletter%"] = gamename_firstletter; templates["%title%"] = title; templates["%title_stripped%"] = Util::getStrippedString(title); templates["%dlcname%"] = dlc_gamename; templates["%dlc_title%"] = dlc_title; templates["%dlc_title_stripped%"] = Util::getStrippedString(dlc_title); templates["%platform%"] = platform; templates["%gamename_transformed%"] = gamename_transformed; templates["%gamename_transformed_firstletter%"] = gamename_transformed_firstletter; for (auto t : templates) Util::replaceAllString(filepath, t.first, t.second); Util::replaceAllString(filepath, "//", "/"); // Replace any double slashes with single slash return filepath; } std::string gameDetails::makeCustomFilepath(const std::string& filename, const gameDetails& gd, const DirectoryConfig& dirConf) { gameFile gf; gf.gamename = gd.gamename; gf.path = "/" + filename; gf.title = gd.title; gf.gamename_basegame = gd.gamename_basegame; gf.title_basegame = gd.title_basegame; if (gf.gamename_basegame.empty()) gf.type = GlobalConstants::GFTYPE_CUSTOM_BASE; else gf.type = GlobalConstants::GFTYPE_CUSTOM_DLC; std::string filepath; filepath = this->makeFilepath(gf, dirConf); return filepath; } lgogdownloader-3.17/src/gamefile.cpp000066400000000000000000000026371476654310300175400ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "gamefile.h" gameFile::gameFile() { this->platform = GlobalConstants::PLATFORM_WINDOWS; this->language = GlobalConstants::LANGUAGE_EN; this->silent = 0; this->type = 0; this->version = ""; } gameFile::~gameFile() { //dtor } void gameFile::setFilepath(const std::string& path) { this->filepath = path; } std::string gameFile::getFilepath() const { return this->filepath; } Json::Value gameFile::getAsJson() { Json::Value json; json["updated"] = this->updated; json["id"] = this->id; json["name"] = this->name; json["path"] = this->path; json["size"] = this->size; json["platform"] = this->platform; json["language"] = this->language; json["silent"] = this->silent; json["gamename"] = this->gamename; json["title"] = this->title; json["gamename_basegame"] = this->gamename_basegame; json["title_basegame"] = this->title_basegame; json["type"] = this->type; json["galaxy_downlink_json_url"] = this->galaxy_downlink_json_url; if (!this->version.empty()) json["version"] = this->version; return json; } lgogdownloader-3.17/src/gui_login.cpp000066400000000000000000000103401476654310300177310ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "gui_login.h" #include #include #include #include #include GuiLogin::GuiLogin() { // constructor } GuiLogin::~GuiLogin() { // destructor } void GuiLogin::loadFinished(bool success) { QWebEngineView *view = qobject_cast(sender()); std::string url = view->page()->url().toString().toUtf8().constData(); // Autofill login form if (success && url.find("https://login.gog.com/auth?client_id=") != std::string::npos) { if (!this->login_username.empty() && !this->login_password.empty()) { std::string js_fill_username = "document.getElementById(\"login_username\").value = \"" + this->login_username + "\";"; std::string js_fill_password = "document.getElementById(\"login_password\").value = \"" + this->login_password + "\";"; std::string js = js_fill_username + js_fill_password; view->page()->runJavaScript(QString::fromStdString(js)); } } // Get auth code else if (success && url.find("https://embed.gog.com/on_login_success") != std::string::npos) { std::string find_str = "code="; auto pos = url.find(find_str); if (pos != std::string::npos) { pos += find_str.length(); std::string code; code.assign(url.begin()+pos, url.end()); if (!code.empty()) { this->auth_code = code; QCoreApplication::exit(); } } } } void GuiLogin::cookieAdded(const QNetworkCookie& cookie) { std::string raw_cookie = cookie.toRawForm().toStdString(); if (!raw_cookie.empty()) { std::string set_cookie = "Set-Cookie: " + raw_cookie; bool duplicate = false; for (auto cookie : this->cookies) { if (set_cookie == cookie) { duplicate = true; break; } } if (!duplicate) this->cookies.push_back(set_cookie); } } void GuiLogin::Login() { this->Login(std::string(), std::string()); } void GuiLogin::Login(const std::string& username, const std::string& password) { QByteArray redirect_uri = QUrl::toPercentEncoding(QString::fromStdString(Globals::galaxyConf.getRedirectUri())); std::string auth_url = "https://auth.gog.com/auth?client_id=" + Globals::galaxyConf.getClientId() + "&redirect_uri=" + redirect_uri.toStdString() + "&response_type=code"; QUrl url = QString::fromStdString(auth_url); this->login_username = username; this->login_password = password; std::vector version_string( Globals::globalConfig.sVersionString.c_str(), Globals::globalConfig.sVersionString.c_str() + Globals::globalConfig.sVersionString.size() + 1 ); int argc = 1; char *argv[] = {&version_string[0]}; QApplication app(argc, argv); QWidget window; QVBoxLayout *layout = new QVBoxLayout; window.resize(440, 540); QWebEngineView *webengine = new QWebEngineView(&window); layout->addWidget(webengine); QWebEngineProfile profile; profile.setHttpUserAgent(QString::fromStdString(Globals::globalConfig.curlConf.sUserAgent)); QWebEnginePage page(&profile); cookiestore = profile.cookieStore(); QObject::connect( webengine, SIGNAL(loadFinished(bool)), this, SLOT(loadFinished(bool)) ); QObject::connect( this->cookiestore, SIGNAL(cookieAdded(const QNetworkCookie&)), this, SLOT(cookieAdded(const QNetworkCookie&)) ); webengine->resize(window.frameSize()); webengine->setPage(&page); webengine->setUrl(url); window.setLayout(layout); window.show(); app.exec(); } std::string GuiLogin::getCode() { return this->auth_code; } std::vector GuiLogin::getCookies() { return this->cookies; } #include "moc_gui_login.cpp" lgogdownloader-3.17/src/progressbar.cpp000066400000000000000000000060701476654310300203130ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "progressbar.h" #include #include ProgressBar::ProgressBar(bool bUnicode, bool bColor) : // Based on block characters. // See https://en.wikipedia.org/wiki/List_of_Unicode_characters#Block_elements // u8"\u2591" - you can try using this ("light shade") instead of space, but it looks worse, // since partial bar has no shade behind it. m_bar_chars { " ", // 0/8 u8"\u258F", // 1/8 u8"\u258E", // 2/8 u8"\u258D", // 3/8 u8"\u258C", // 4/8 u8"\u258B", // 5/8 u8"\u258A", // 6/8 u8"\u2589", // 7/8 u8"\u2588" /* 8/8 */ }, m_left_border(u8"\u2595"), // right 1/8th m_right_border(u8"\u258F"), // left 1/8th m_simple_left_border("["), m_simple_right_border("]"), m_simple_empty_fill(" "), m_simple_bar_char("="), // using vt100 escape sequences for colors... See http://ascii-table.com/ansi-escape-sequences.php m_bar_color("\033[1;34m"), m_border_color("\033[1;37m"), COLOR_RESET("\033[0m"), m_use_unicode(bUnicode), m_use_color(bColor) { } ProgressBar::~ProgressBar() { //dtor } void ProgressBar::draw(unsigned int length, double fraction) { std::cout << createBarString(length, fraction); } std::string ProgressBar::createBarString(unsigned int length, double fraction) { std::ostringstream ss; // validation if (!std::isnormal(fraction) || (fraction < 0.0)) fraction = 0.0; else if (fraction > 1.0) fraction = 1.0; double bar_part = fraction * length; double whole_bar_chars = std::floor(bar_part); unsigned int whole_bar_chars_i = (unsigned int) whole_bar_chars; // The bar uses symbols graded with 1/8 unsigned int partial_bar_char_index = (unsigned int) std::floor((bar_part - whole_bar_chars) * 8.0); // left border if (m_use_color) ss << m_border_color; ss << (m_use_unicode ? m_left_border : m_simple_left_border); // whole completed bars if (m_use_color) ss << m_bar_color; unsigned int i = 0; for (; i < whole_bar_chars_i; i++) { ss << (m_use_unicode ? m_bar_chars[8] : m_simple_bar_char); } // partial completed bar if (i < length) ss << (m_use_unicode ? m_bar_chars[partial_bar_char_index] : m_simple_empty_fill); // whole unfinished bars if (m_use_color) ss << COLOR_RESET; for (i = whole_bar_chars_i + 1; i < length; i++) { // first entry in m_bar_chars is assumed to be the empty bar ss << (m_use_unicode ? m_bar_chars[0] : m_simple_empty_fill); } // right border if (m_use_color) ss << m_border_color; ss << (m_use_unicode ? m_right_border : m_simple_right_border); if (m_use_color) ss << COLOR_RESET; return ss.str(); } lgogdownloader-3.17/src/util.cpp000066400000000000000000000740471476654310300167500ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "util.h" #include #include #include #include #include #include #include #include #include #include std::string Util::getFileHash(const std::string& filename, unsigned hash_id) { unsigned char digest[rhash_get_digest_size(hash_id)]; char result[rhash_get_hash_length(hash_id) + 1]; int i = rhash_file(hash_id, filename.c_str(), digest); if (i < 0) std::cerr << "LibRHash error: " << strerror(errno) << std::endl; else rhash_print_bytes(result, digest, rhash_get_digest_size(hash_id), RHPR_HEX); return result; } std::string Util::getFileHashRange(const std::string& filepath, unsigned hash_id, off_t range_start, off_t range_end) { char result[rhash_get_hash_length(hash_id) + 1]; if (!boost::filesystem::exists(filepath)) return result; off_t filesize = boost::filesystem::file_size(filepath); if (range_end == 0 || range_end > filesize) range_end = filesize; if (range_end < range_start) { off_t tmp = range_start; range_start = range_end; range_end = tmp; } off_t chunk_size = 10 << 20; // 10MB off_t rangesize = range_end - range_start; off_t remaining = rangesize % chunk_size; int chunks = (remaining == 0) ? rangesize/chunk_size : (rangesize/chunk_size)+1; rhash rhash_context; rhash_context = rhash_init(hash_id); FILE *infile = fopen(filepath.c_str(), "r"); for (int i = 0; i < chunks; i++) { off_t chunk_begin = range_start + i*chunk_size; fseek(infile, chunk_begin, SEEK_SET); if ((i == chunks-1) && (remaining != 0)) chunk_size = remaining; unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *)); if (chunk == NULL) { std::cerr << "Memory error" << std::endl; fclose(infile); return result; } off_t size = fread(chunk, 1, chunk_size, infile); if (size != chunk_size) { std::cerr << "Read error" << std::endl; free(chunk); fclose(infile); return result; } rhash_update(rhash_context, chunk, chunk_size); free(chunk); } fclose(infile); rhash_final(rhash_context, NULL); rhash_print(result, rhash_context, hash_id, RHPR_HEX); rhash_free(rhash_context); return result; } std::string Util::getChunkHash(unsigned char *chunk, uintmax_t chunk_size, unsigned hash_id) { unsigned char digest[rhash_get_digest_size(hash_id)]; char result[rhash_get_hash_length(hash_id) + 1]; int i = rhash_msg(hash_id, chunk, chunk_size, digest); if (i < 0) std::cerr << "LibRHash error: " << strerror(errno) << std::endl; else rhash_print_bytes(result, digest, rhash_get_digest_size(hash_id), RHPR_HEX); return result; } // Create GOG XML int Util::createXML(std::string filepath, uintmax_t chunk_size, std::string xml_dir) { int res = 0; FILE *infile; FILE *xmlfile; uintmax_t filesize, size; int chunks, i; if (xml_dir.empty()) { xml_dir = Util::getCacheHome() + "/lgogdownloader/xml"; } // Make sure directory exists boost::filesystem::path path = xml_dir; if (!boost::filesystem::exists(path)) { if (!boost::filesystem::create_directories(path)) { std::cerr << "Failed to create directory: " << path << std::endl; return res; } } if ((infile=fopen(filepath.c_str(), "r"))!=NULL) { //File exists fseek(infile, 0, SEEK_END); filesize = ftell(infile); rewind(infile); } else { std::cerr << filepath << " doesn't exist" << std::endl; return res; } // Get filename boost::filesystem::path pathname = filepath; std::string filename = pathname.filename().string(); std::string filenameXML = xml_dir + "/" + filename + ".xml"; std::cout << filename << std::endl; //Determine number of chunks int remaining = filesize % chunk_size; chunks = (remaining == 0) ? filesize/chunk_size : (filesize/chunk_size)+1; std::cout << "Filesize: " << filesize << " bytes" << std::endl << "Chunks: " << chunks << std::endl << "Chunk size: " << (chunk_size >> 20) << " MB" << std::endl; tinyxml2::XMLDocument xml; tinyxml2::XMLElement *fileElem = xml.NewElement("file"); fileElem->SetAttribute("name", filename.c_str()); fileElem->SetAttribute("chunks", chunks); fileElem->SetAttribute("total_size", std::to_string(filesize).c_str()); std::cout << "Getting MD5 for chunks" << std::endl; rhash rhash_context; rhash_context = rhash_init(RHASH_MD5); if(!rhash_context) { std::cerr << "error: couldn't initialize rhash context" << std::endl; fclose(infile); return res; } char rhash_result[rhash_get_hash_length(RHASH_MD5) + 1]; for (i = 0; i < chunks; i++) { uintmax_t range_begin = i*chunk_size; fseek(infile, range_begin, SEEK_SET); if ((i == chunks-1) && (remaining != 0)) chunk_size = remaining; uintmax_t range_end = range_begin + chunk_size - 1; unsigned char *chunk = (unsigned char *) malloc(chunk_size * sizeof(unsigned char *)); if (chunk == NULL) { std::cerr << "Memory error" << std::endl; fclose(infile); return res; } size = fread(chunk, 1, chunk_size, infile); if (size != chunk_size) { std::cerr << "Read error" << std::endl; free(chunk); fclose(infile); return res; } std::string hash = Util::getChunkHash(chunk, chunk_size, RHASH_MD5); rhash_update(rhash_context, chunk, chunk_size); // Update hash for the whole file free(chunk); tinyxml2::XMLElement *chunkElem = xml.NewElement("chunk"); chunkElem->SetAttribute("id", i); chunkElem->SetAttribute("from", std::to_string(range_begin).c_str()); chunkElem->SetAttribute("to", std::to_string(range_end).c_str()); chunkElem->SetAttribute("method", "md5"); tinyxml2::XMLText *text = xml.NewText(hash.c_str()); chunkElem->LinkEndChild(text); fileElem->LinkEndChild(chunkElem); std::cout << "Chunks hashed " << (i+1) << " / " << chunks << "\r" << std::flush; } fclose(infile); rhash_final(rhash_context, NULL); rhash_print(rhash_result, rhash_context, RHASH_MD5, RHPR_HEX); rhash_free(rhash_context); std::cout << std::endl << "MD5: " << rhash_result << std::endl; fileElem->SetAttribute("md5", rhash_result); xml.LinkEndChild(fileElem); std::cout << "Writing XML: " << filenameXML << std::endl; if ((xmlfile=fopen(filenameXML.c_str(), "w"))!=NULL) { tinyxml2::XMLPrinter printer(xmlfile); xml.Print(&printer); fclose(xmlfile); res = 1; } else { std::cerr << "Can't create " << filenameXML << std::endl; return res; } return res; } /* Overrides global settings with game specific settings returns 0 if fails returns number of changed settings if succesful */ int Util::getGameSpecificConfig(std::string gamename, gameSpecificConfig* conf, std::string directory) { int res = 0; if (directory.empty()) { directory = Util::getConfigHome() + "/lgogdownloader/gamespecific"; } std::string filepath = directory + "/" + gamename + ".conf"; // Make sure file exists boost::filesystem::path path = filepath; if (!boost::filesystem::exists(path)) { return res; } std::ifstream json(filepath, std::ifstream::binary); Json::Value root; try { json >> root; } catch (const Json::Exception& exc) { std::cerr << "Failed to parse game specific config " << filepath << std::endl; std::cerr << exc.what() << std::endl; return res; } if (root.isMember("language")) { if (root["language"].isInt()) conf->dlConf.iInstallerLanguage = root["language"].asUInt(); else { Util::parseOptionString(root["language"].asString(), conf->dlConf.vLanguagePriority, conf->dlConf.iInstallerLanguage, GlobalConstants::LANGUAGES); } res++; } if (root.isMember("platform")) { if (root["platform"].isInt()) conf->dlConf.iInstallerPlatform = root["platform"].asUInt(); else { Util::parseOptionString(root["platform"].asString(), conf->dlConf.vPlatformPriority, conf->dlConf.iInstallerPlatform, GlobalConstants::PLATFORMS); } res++; } // Warn about deprecated option if (root.isMember("dlc")) { std::cerr << filepath << " contains deprecated option \"dlc\" which will be ignored, use \"include\" instead" << std::endl; } if (root.isMember("include")) { conf->dlConf.iInclude = 0; std::vector vInclude = Util::tokenize(root["include"].asString(), ","); for (std::vector::iterator it = vInclude.begin(); it != vInclude.end(); it++) { conf->dlConf.iInclude |= Util::getOptionValue(*it, GlobalConstants::INCLUDE_OPTIONS); } res++; } if (root.isMember("ignore-dlc-count")) { conf->dlConf.bIgnoreDLCCount = root["ignore-dlc-count"].asBool(); res++; } if (root.isMember("subdirectories")) { conf->dirConf.bSubDirectories = root["subdirectories"].asBool(); res++; } if (root.isMember("directory")) { conf->dirConf.sDirectory = root["directory"].asString(); res++; } if (root.isMember("subdir-game")) { conf->dirConf.sGameSubdir = root["subdir-game"].asString(); res++; } if (root.isMember("subdir-installers")) { conf->dirConf.sInstallersSubdir = root["subdir-installers"].asString(); res++; } if (root.isMember("subdir-extras")) { conf->dirConf.sExtrasSubdir = root["subdir-extras"].asString(); res++; } if (root.isMember("subdir-patches")) { conf->dirConf.sPatchesSubdir = root["subdir-patches"].asString(); res++; } if (root.isMember("subdir-language-packs")) { conf->dirConf.sLanguagePackSubdir = root["subdir-language-packs"].asString(); res++; } if (root.isMember("subdir-dlc")) { conf->dirConf.sDLCSubdir = root["subdir-dlc"].asString(); res++; } return res; } int Util::replaceString(std::string& str, const std::string& to_replace, const std::string& replace_with) { size_t pos = str.find(to_replace); if (pos == std::string::npos) { return 0; } str.replace(str.begin()+pos, str.begin()+pos+to_replace.length(), replace_with); return 1; } int Util::replaceAllString(std::string& str, const std::string& to_replace, const std::string& replace_with) { size_t pos = str.find(to_replace); if (pos == std::string::npos) { return 0; } do { str.replace(str.begin()+pos, str.begin()+pos+to_replace.length(), replace_with); pos = str.find(to_replace); } while(pos != std::string::npos); return 1; } void Util::setFilePermissions(const boost::filesystem::path& path, const boost::filesystem::perms& permissions) { if (boost::filesystem::exists(path)) { if (boost::filesystem::is_regular_file(path)) { boost::filesystem::file_status s = boost::filesystem::status(path); if (s.permissions() != permissions) { boost::system::error_code ec; boost::filesystem::permissions(path, permissions, ec); if (ec) { std::cerr << "Failed to set file permissions for " << path.string() << std::endl; } } } } } int Util::getTerminalWidth() { int width; if(isatty(STDOUT_FILENO)) { struct winsize w; ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); width = static_cast(w.ws_col); } else width = 10000;//Something sufficiently big return width; } void Util::getManualUrlsFromJSON(const Json::Value &root, std::vector &urls) { if(root.size() > 0) { for(Json::ValueConstIterator it = root.begin() ; it != root.end() ; ++it) { if (it.key() == "manualUrl") { Json::Value url = *it; urls.push_back(url.asString()); } else getManualUrlsFromJSON(*it, urls); } } return; } std::vector Util::getDLCNamesFromJSON(const Json::Value &root) { std::vector urls, dlcnames; getManualUrlsFromJSON(root, urls); for (unsigned int i = 0; i < urls.size(); ++i) { std::string gamename; std::string url_prefix = "/downloads/"; if (urls[i].find(url_prefix) == std::string::npos) continue; gamename.assign(urls[i].begin()+urls[i].find(url_prefix)+url_prefix.length(), urls[i].begin()+urls[i].find_last_of("/")); bool bDuplicate = false; for (unsigned int j = 0; j < dlcnames.size(); ++j) { if (gamename == dlcnames[j]) { bDuplicate = true; break; } } if (!bDuplicate) dlcnames.push_back(gamename); } return dlcnames; } std::string Util::getHomeDir() { return (std::string)getenv("HOME"); } std::string Util::getConfigHome() { std::string configHome; char *xdgconfig = getenv("XDG_CONFIG_HOME"); if (xdgconfig) configHome = (std::string)xdgconfig; else configHome = Util::getHomeDir() + "/.config"; return configHome; } std::string Util::getCacheHome() { std::string cacheHome; char *xdgcache = getenv("XDG_CACHE_HOME"); if (xdgcache) cacheHome = (std::string)xdgcache; else cacheHome = Util::getHomeDir() + "/.cache"; return cacheHome; } std::vector Util::tokenize(const std::string& str, const std::string& separator) { std::vector tokens; std::string token; size_t idx = 0, found; while ((found = str.find(separator, idx)) != std::string::npos) { token = str.substr(idx, found - idx); if (!token.empty()) tokens.push_back(token); idx = found + separator.length(); } token = str.substr(idx); if (!token.empty()) tokens.push_back(token); return tokens; } unsigned int Util::getOptionValue(const std::string& str, const std::vector& options, const bool& bAllowStringToIntConversion) { unsigned int value = 0; boost::regex expression("^[+-]?\\d+$", boost::regex::perl); boost::match_results what; if (str == "all") { for (unsigned int i = 0; i < options.size(); ++i) value |= options[i].id; } else if (boost::regex_search(str, what, expression) && bAllowStringToIntConversion) { value = std::stoi(str); } else { for (unsigned int i = 0; i < options.size(); ++i) { if (!options[i].regexp.empty()) { boost::regex expr("^(" + options[i].regexp + ")$", boost::regex::perl | boost::regex::icase); if (boost::regex_search(str, what, expr)) { value = options[i].id; break; } } if (str == options[i].code) { value = options[i].id; break; } } } return value; } std::string Util::getOptionNameString(const unsigned int& value, const std::vector& options) { std::string str; for (unsigned int i = 0; i < options.size(); ++i) { if ((value & options[i].id) == options[i].id) str += (str.empty() ? "" : ", ")+options[i].str; } return str; } // Parse the options string void Util::parseOptionString(const std::string &option_string, std::vector &priority, unsigned int &type, const std::vector& options) { type = 0; priority.clear(); std::vector tokens_priority = Util::tokenize(option_string, ","); for (std::vector::iterator it_priority = tokens_priority.begin(); it_priority != tokens_priority.end(); it_priority++) { unsigned int value = 0; std::vector tokens_value = Util::tokenize(*it_priority, "+"); for (std::vector::iterator it_value = tokens_value.begin(); it_value != tokens_value.end(); it_value++) { value |= Util::getOptionValue(*it_value, options); } priority.push_back(value); type |= value; } } std::string Util::getLocalFileHash(const std::string& xml_dir, const std::string& filepath, const std::string& gamename, const bool& useFastCheck) { std::string localHash; boost::filesystem::path path = filepath; boost::filesystem::path local_xml_file; if (!gamename.empty()) local_xml_file = xml_dir + "/" + gamename + "/" + path.filename().string() + ".xml"; else local_xml_file = xml_dir + "/" + path.filename().string() + ".xml"; if (boost::filesystem::exists(local_xml_file) && useFastCheck) { tinyxml2::XMLDocument local_xml; local_xml.LoadFile(local_xml_file.string().c_str()); tinyxml2::XMLElement *fileElem = local_xml.FirstChildElement("file"); if (fileElem) { localHash = fileElem->Attribute("md5"); } } else if (boost::filesystem::exists(path) && boost::filesystem::is_regular_file(path)) { localHash = Util::getFileHash(path.string(), RHASH_MD5); } return localHash; } void Util::shortenStringToTerminalWidth(std::string& str) { int iStrLen = static_cast(str.length()); int iTermWidth = Util::getTerminalWidth(); if (iStrLen >= iTermWidth) { size_t chars_to_remove = (iStrLen - iTermWidth) + 4; size_t middle = iStrLen / 2; size_t pos1 = middle - (chars_to_remove / 2); size_t pos2 = middle + (chars_to_remove / 2); str.replace(str.begin()+pos1, str.begin()+pos2, "..."); } } std::string Util::getJsonUIntValueAsString(const Json::Value& json_value) { std::string value; try { value = json_value.asString(); } catch (...) { try { uintmax_t value_uint = json_value.asLargestUInt(); value = std::to_string(value_uint); } catch (...) { value = ""; } } return value; } std::string Util::getStrippedString(std::string str) { str.erase( std::remove_if(str.begin(), str.end(), [](unsigned char c) { bool bIsValid = false; bIsValid = (std::isspace(c) && std::isprint(c)) || std::isalnum(c); std::vector validChars = { '-', '_', '.', '(', ')', '[', ']', '{', '}' }; if (std::any_of(validChars.begin(), validChars.end(), [c](unsigned char x){return x == c;})) { bIsValid = true; } return !bIsValid; } ), str.end() ); return str; } std::string Util::makeEtaString(const unsigned long long& iBytesRemaining, const double& dlRate) { boost::posix_time::time_duration duration(boost::posix_time::seconds((long)(iBytesRemaining / dlRate))); return Util::makeEtaString(duration); } std::string Util::makeEtaString(const boost::posix_time::time_duration& duration) { std::string etastr; std::stringstream eta_ss; if (duration.hours() > 23) { eta_ss << duration.hours() / 24 << "d " << std::setfill('0') << std::setw(2) << duration.hours() % 24 << "h " << std::setfill('0') << std::setw(2) << duration.minutes() << "m " << std::setfill('0') << std::setw(2) << duration.seconds() << "s"; } else if (duration.hours() > 0) { eta_ss << duration.hours() << "h " << std::setfill('0') << std::setw(2) << duration.minutes() << "m " << std::setfill('0') << std::setw(2) << duration.seconds() << "s"; } else if (duration.minutes() > 0) { eta_ss << duration.minutes() << "m " << std::setfill('0') << std::setw(2) << duration.seconds() << "s"; } else { eta_ss << duration.seconds() << "s"; } etastr = eta_ss.str(); return etastr; } void Util::CurlHandleSetDefaultOptions(CURL* curlhandle, const CurlConfig& conf) { curl_easy_setopt(curlhandle, CURLOPT_USERAGENT, conf.sUserAgent.c_str()); curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(curlhandle, CURLOPT_CONNECTTIMEOUT, conf.iTimeout); curl_easy_setopt(curlhandle, CURLOPT_FAILONERROR, true); curl_easy_setopt(curlhandle, CURLOPT_COOKIEFILE, conf.sCookiePath.c_str()); curl_easy_setopt(curlhandle, CURLOPT_SSL_VERIFYPEER, conf.bVerifyPeer); curl_easy_setopt(curlhandle, CURLOPT_VERBOSE, conf.bVerbose); curl_easy_setopt(curlhandle, CURLOPT_MAX_RECV_SPEED_LARGE, conf.iDownloadRate); // Assume that we have connection error and abort transfer with CURLE_OPERATION_TIMEDOUT if download speed is less than 200 B/s for 30 seconds curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_TIME, conf.iLowSpeedTimeout); curl_easy_setopt(curlhandle, CURLOPT_LOW_SPEED_LIMIT, conf.iLowSpeedTimeoutRate); if (!conf.sCACertPath.empty()) curl_easy_setopt(curlhandle, CURLOPT_CAINFO, conf.sCACertPath.c_str()); if (!conf.sInterface.empty()) { curl_easy_setopt(curlhandle, CURLOPT_DNS_INTERFACE, conf.sInterface.c_str()); curl_easy_setopt(curlhandle, CURLOPT_INTERFACE, conf.sInterface.c_str()); } } std::string Util::CurlHandleGetInfoString(CURL* curlhandle, CURLINFO info) { char* str; return (curl_easy_getinfo(curlhandle, info, &str) == CURLE_OK) ? str : ""; } CURLcode Util::CurlGetResponse(const std::string& url, std::string& response, int max_retries) { CURLcode result; CURL *handle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(handle, Globals::globalConfig.curlConf); curl_easy_setopt(handle, CURLOPT_URL, url.c_str()); result = Util::CurlHandleGetResponse(handle, response, max_retries); curl_easy_cleanup(handle); return result; } CURLcode Util::CurlHandleGetResponse(CURL* curlhandle, std::string& response, int max_retries) { CURLcode result; int retries = 0; std::ostringstream memory; bool bShouldRetry = false; long int response_code = 0; if (max_retries < 0) max_retries = Globals::globalConfig.iRetries; curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory); do { if (bShouldRetry) retries++; if (Globals::globalConfig.iWait > 0) usleep(Globals::globalConfig.iWait); // Delay the request by specified time result = curl_easy_perform(curlhandle); response = memory.str(); memory.str(std::string()); switch (result) { // Retry on these errors case CURLE_PARTIAL_FILE: case CURLE_OPERATION_TIMEDOUT: case CURLE_RECV_ERROR: bShouldRetry = true; break; // Retry on CURLE_HTTP_RETURNED_ERROR if response code is not "404 Not Found" case CURLE_HTTP_RETURNED_ERROR: curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (response_code == 404 || response_code == 403) { bShouldRetry = false; } else bShouldRetry = true; break; default: bShouldRetry = false; break; } if (retries >= max_retries) bShouldRetry = false; } while (bShouldRetry); return result; } curl_off_t Util::CurlWriteMemoryCallback(char *ptr, curl_off_t size, curl_off_t nmemb, void *userp) { std::ostringstream *stream = (std::ostringstream*)userp; std::streamsize count = (std::streamsize) size * nmemb; stream->write(ptr, count); return count; } curl_off_t Util::CurlWriteChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, void *userp) { curl_off_t realsize = size * nmemb; ChunkMemoryStruct *mem = (ChunkMemoryStruct *)userp; mem->memory = (char *) realloc(mem->memory, mem->size + realsize + 1); if(mem->memory == NULL) { std::cout << "Not enough memory (realloc returned NULL)" << std::endl; return 0; } memcpy(&(mem->memory[mem->size]), contents, realsize); mem->size += realsize; mem->memory[mem->size] = 0; return realsize; } curl_off_t Util::CurlReadChunkMemoryCallback(void *contents, curl_off_t size, curl_off_t nmemb, ChunkMemoryStruct *mem) { curl_off_t realsize = std::min(size * nmemb, mem->size); std::copy(mem->memory, mem->memory + realsize, (char*)contents); mem->size -= realsize; mem->memory += realsize; return realsize; } std::string Util::makeSizeString(const unsigned long long& iSizeInBytes) { auto units = { "B", "kB", "MB", "GB", "TB", "PB" }; std::string size_unit = "B"; double iSize = static_cast(iSizeInBytes); for (auto unit : units) { size_unit = unit; if (iSize < 1024) break; iSize /= 1024; } return formattedString("%0.2f %s", iSize, size_unit.c_str()); } Json::Value Util::readJsonFile(const std::string& path) { Json::Value json; std::ifstream ifs(path, std::ifstream::binary); if (ifs) { try { ifs >> json; } catch (const Json::Exception& exc) { std::cerr << "Failed to parse " << path << std::endl; std::cerr << exc.what() << std::endl; } ifs.close(); } else { std::cerr << "Failed to open " << path << std::endl; } return json; } std::string Util::transformGamename(const std::string& gamename) { std::string gamename_transformed = gamename; for (auto transformMatch : Globals::globalConfig.transformationsJSON.getMemberNames()) { boost::regex expression(transformMatch); boost::match_results what; if (boost::regex_search(gamename_transformed, what, expression)) { // Get list of exceptions std::vector vExceptions; if (Globals::globalConfig.transformationsJSON[transformMatch].isMember("exceptions")) { if (Globals::globalConfig.transformationsJSON[transformMatch]["exceptions"].isArray()) { for (auto exception : Globals::globalConfig.transformationsJSON[transformMatch]["exceptions"]) vExceptions.push_back(exception.asString()); } else { vExceptions.push_back(Globals::globalConfig.transformationsJSON[transformMatch]["exceptions"].asString()); } } // Skip if gamename matches exception if (std::any_of(vExceptions.begin(), vExceptions.end(), [gamename](std::string exception){return exception == gamename;})) continue; boost::regex transformRegex(Globals::globalConfig.transformationsJSON[transformMatch]["regex"].asString()); std::string transformReplacement = Globals::globalConfig.transformationsJSON[transformMatch]["replacement"].asString(); gamename_transformed = boost::regex_replace(gamename_transformed, transformRegex, transformReplacement); } } return gamename_transformed; } std::string Util::htmlToXhtml(const std::string& html) { std::string xhtml; TidyBuffer buffer; int rc = -1; TidyDoc doc = tidyCreate(); tidyBufInit(&buffer); tidyOptSetBool(doc, TidyXhtmlOut, yes); tidyOptSetBool(doc, TidyForceOutput, yes); tidyOptSetBool(doc, TidyShowInfo, no); tidyOptSetBool(doc, TidyShowWarnings, no); tidyOptSetInt(doc, TidyWrapLen, 0); rc = tidyParseString(doc, html.c_str()); if ( rc >= 0 ) rc = tidyCleanAndRepair(doc); if ( rc >= 0 ) rc = tidySaveBuffer(doc, &buffer); if (rc >= 0) { if (buffer.size > 0) xhtml = std::string((char*)buffer.bp, buffer.size); } else { std::cerr << "Severe error occured: " << std::string(strerror(rc)) << std::endl; } tidyBufFree(&buffer); tidyRelease(doc); return xhtml; } tinyxml2::XMLNode* Util::nextXMLNode(tinyxml2::XMLNode* node) { if (node->FirstChildElement()) // Has child element, go to first child node = node->FirstChildElement(); else if (node->NextSiblingElement()) // Has sibling element, go to first sibling node = node->NextSiblingElement(); else { // Go to parent node until it has sibling while(node->Parent() && !node->Parent()->NextSiblingElement()) node = node->Parent(); if(node->Parent() && node->Parent()->NextSiblingElement()) node = node->Parent()->NextSiblingElement(); else // Reached the end node = nullptr; } return node; } lgogdownloader-3.17/src/website.cpp000066400000000000000000000740011476654310300174230ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "website.h" #include "globalconstants.h" #include "message.h" #include #include #ifdef USE_QT_GUI_LOGIN #include "gui_login.h" #endif Website::Website() { this->retries = 0; curlhandle = curl_easy_init(); Util::CurlHandleSetDefaultOptions(curlhandle, Globals::globalConfig.curlConf); curl_easy_setopt(curlhandle, CURLOPT_COOKIEJAR, Globals::globalConfig.curlConf.sCookiePath.c_str()); } Website::~Website() { curl_easy_cleanup(curlhandle); } std::string Website::getResponse(const std::string& url) { std::string response; curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); int max_retries = std::min(3, Globals::globalConfig.iRetries); CURLcode result = Util::CurlHandleGetResponse(curlhandle, response, max_retries); if (result != CURLE_OK) { std::cout << curl_easy_strerror(result) << std::endl; if (result == CURLE_HTTP_RETURNED_ERROR) { long int response_code = 0; result = curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); std::cout << "HTTP ERROR: "; if (result == CURLE_OK) std::cout << response_code << " (" << url << ")" << std::endl; else std::cout << "failed to get error code: " << curl_easy_strerror(result) << " (" << url << ")" << std::endl; } else if (result == CURLE_SSL_CACERT) { std::cout << "Try using CA certificate bundle from cURL: https://curl.haxx.se/ca/cacert.pem" << std::endl; std::cout << "Use --cacert to set the path for CA certificate bundle" << std::endl; } } return response; } Json::Value Website::getResponseJson(const std::string& url) { std::istringstream response(this->getResponse(url)); Json::Value json; if (!response.str().empty()) { try { response >> json; } catch(const Json::Exception& exc) { if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_DEBUG) std::cerr << "DEBUG INFO (Website::getResponseJson)" << std::endl << json << std::endl; std::cout << "Failed to parse json: " << exc.what(); } } return json; } Json::Value Website::getGameDetailsJSON(const std::string& gameid) { std::string gameDataUrl = "https://www.gog.com/account/gameDetails/" + gameid + ".json"; Json::Value json = this->getResponseJson(gameDataUrl); return json; } // Get list of games from account page std::vector Website::getGames() { std::vector games; Json::Value root; int iPage = 1; bool bAllPagesParsed = false; int iUpdated = Globals::globalConfig.bUpdated ? 1 : 0; int iHidden = 0; std::string tags; for (auto tag : Globals::globalConfig.dlConf.vTags) { if (tags.empty()) tags = tag; else tags += "," + tag; } Globals::vOwnedGamesIds = this->getOwnedGamesIds(); std::vector jsonProductInfo; do { std::string url = "https://www.gog.com/account/getFilteredProducts?hiddenFlag=" + std::to_string(iHidden) + "&isUpdated=" + std::to_string(iUpdated) + "&mediaType=1&sortBy=title&system=&page=" + std::to_string(iPage); if (!tags.empty()) url += "&tags=" + tags; Json::Value root = this->getResponseJson(url); if (root.empty()) continue; std::cerr << "\033[KGetting product data " << root["page"].asInt() << " / " << root["totalPages"].asInt() << "\r" << std::flush; if (root["page"].asInt() == root["totalPages"].asInt() || root["totalPages"].asInt() == 0) bAllPagesParsed = true; // Make the next loop handle hidden products if (Globals::globalConfig.bIncludeHiddenProducts && bAllPagesParsed && iHidden == 0) { bAllPagesParsed = false; iHidden = 1; iPage = 0; // Set to 0 because we increment it at the end of the loop } if (root["products"].isArray()) { for (auto product : root["products"]) jsonProductInfo.push_back(product); } iPage++; } while (!bAllPagesParsed); std::cerr << std::endl; unsigned int iProduct = 0; unsigned int iProductTotal = jsonProductInfo.size(); for (auto product : jsonProductInfo) { iProduct++; std::cerr << "\033[KGetting game names " << iProduct << " / " << iProductTotal << "\r" << std::flush; gameItem game; game.name = product["slug"].asString(); game.id = product["id"].isInt() ? std::to_string(product["id"].asInt()) : product["id"].asString(); game.isnew = product["isNew"].asBool(); if (product.isMember("updates")) { if (product["updates"].isNull()) { /* In some cases the value can be null. * For example when user owns a dlc but not the base game * https://github.com/Sude-/lgogdownloader/issues/101 * Assume that there are no updates in this case */ game.updates = 0; } else if (product["updates"].isInt()) game.updates = product["updates"].asInt(); else { try { game.updates = std::stoi(product["updates"].asString()); } catch (std::invalid_argument& e) { game.updates = 0; // Assume no updates } } } unsigned int platform = 0; if (product["worksOn"]["Windows"].asBool()) platform |= GlobalConstants::PLATFORM_WINDOWS; if (product["worksOn"]["Mac"].asBool()) platform |= GlobalConstants::PLATFORM_MAC; if (product["worksOn"]["Linux"].asBool()) platform |= GlobalConstants::PLATFORM_LINUX; // Skip if not new and flag is set if (Globals::globalConfig.bNew && !game.isnew) continue; // Skip if platform doesn't match if (Globals::globalConfig.bPlatformDetection && !(platform & Globals::globalConfig.dlConf.iInstallerPlatform)) continue; // Filter the game list if (!Globals::globalConfig.sGameRegex.empty()) { boost::regex expression(Globals::globalConfig.sGameRegex); boost::match_results what; if (!boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex continue; } if (Globals::globalConfig.dlConf.iInclude & GlobalConstants::GFTYPE_DLC) { int dlcCount = product["dlcCount"].asInt(); bool bDownloadDLCInfo = (dlcCount != 0); if (!bDownloadDLCInfo && !Globals::globalConfig.sIgnoreDLCCountRegex.empty()) { boost::regex expression(Globals::globalConfig.sIgnoreDLCCountRegex); boost::match_results what; if (boost::regex_search(game.name, what, expression)) // Check if name matches the specified regex { bDownloadDLCInfo = true; } } // Check game specific config if (!Globals::globalConfig.bUpdateCache) // Disable game specific config files for cache update { gameSpecificConfig conf; conf.dlConf.bIgnoreDLCCount = bDownloadDLCInfo; Util::getGameSpecificConfig(game.name, &conf); bDownloadDLCInfo = conf.dlConf.bIgnoreDLCCount; } if (bDownloadDLCInfo && !Globals::globalConfig.sGameRegex.empty()) { // don't download unnecessary info if user is only interested in a subset of his account boost::regex expression(Globals::globalConfig.sGameRegex); boost::match_results what; if (!boost::regex_search(game.name, what, expression)) { bDownloadDLCInfo = false; } } if (bDownloadDLCInfo) { game.gamedetailsjson = this->getGameDetailsJSON(game.id); if (!game.gamedetailsjson.empty()) game.dlcnames = Util::getDLCNamesFromJSON(game.gamedetailsjson["dlcs"]); } } games.push_back(game); } std::cerr << std::endl; if (Globals::globalConfig.bIncludeHiddenProducts) { std::sort(games.begin(), games.end(), [](const gameItem& i, const gameItem& j) -> bool { return i.name < j.name; }); } return games; } // Login to GOG website int Website::Login(const std::string& email, const std::string& password) { // Reset client id and client secret to ensure we can log-in Globals::galaxyConf.resetClient(); int res = 0; std::string auth_code; auth_code = this->LoginGetAuthCode(email, password); if (!auth_code.empty()) { std::string token_url = "https://auth.gog.com/token?client_id=" + Globals::galaxyConf.getClientId() + "&client_secret=" + Globals::galaxyConf.getClientSecret() + "&grant_type=authorization_code&code=" + auth_code + "&redirect_uri=" + (std::string)curl_easy_escape(curlhandle, Globals::galaxyConf.getRedirectUri().c_str(), Globals::galaxyConf.getRedirectUri().size()); std::string json = this->getResponse(token_url); if (!json.empty()) { Json::Value token_json; std::istringstream json_stream(json); try { json_stream >> token_json; Globals::galaxyConf.setJSON(token_json); res = 1; } catch (const Json::Exception& exc) { std::cerr << "Failed to parse json" << std::endl << json << std::endl; std::cerr << exc.what() << std::endl; } } } else { std::cout << "Failed to get auth code" << std::endl; res = 0; } if (res >= 1) curl_easy_setopt(curlhandle, CURLOPT_COOKIELIST, "FLUSH"); // Write all known cookies to the file specified by CURLOPT_COOKIEJAR return res; } std::string Website::LoginGetAuthCode(const std::string& email, const std::string& password) { std::string auth_code; bool bRecaptcha = false; bool bForceGUI = false; #ifdef USE_QT_GUI_LOGIN bForceGUI = Globals::globalConfig.bForceGUILogin; #endif std::string auth_url = "https://auth.gog.com/auth?client_id=" + Globals::galaxyConf.getClientId() + "&redirect_uri=" + (std::string)curl_easy_escape(curlhandle, Globals::galaxyConf.getRedirectUri().c_str(), Globals::galaxyConf.getRedirectUri().size()) + "&response_type=code&layout=default&brand=gog"; std::string login_form_html = this->getResponse(auth_url); if (Globals::globalConfig.iMsgLevel >= MSGLEVEL_DEBUG) { std::cerr << "DEBUG INFO (Website::LoginGetAuthCode)" << std::endl; std::cerr << login_form_html << std::endl; } if (login_form_html.find("class=\"g-recaptcha form__recaptcha\"") != std::string::npos) { bRecaptcha = true; } // Try normal login if GUI or browser is not forced if (!(bForceGUI || Globals::globalConfig.bForceBrowserLogin)) { auth_code = this->LoginGetAuthCodeCurl(login_form_html, email, password); } #ifdef USE_QT_GUI_LOGIN if ((Globals::globalConfig.bEnableLoginGUI && auth_code.empty()) || bForceGUI) { auth_code = this->LoginGetAuthCodeGUI(email, password); // If GUI is forced then stop here and don't offer browser login if (bForceGUI) return auth_code; } #endif if ((auth_code.empty() && bRecaptcha) || Globals::globalConfig.bForceBrowserLogin) auth_code = this->LoginGetAuthCodeBrowser(auth_url); return auth_code; } std::string Website::LoginGetAuthCodeCurl(const std::string& login_form_html, const std::string& email, const std::string& password) { std::string auth_code; std::string postdata; std::ostringstream memory; std::string token; std::string tagname_username = "login[username]"; std::string tagname_password = "login[password]"; std::string tagname_login = "login[login]"; std::string tagname_token = "login[_token]"; std::string login_form_xhtml = Util::htmlToXhtml(login_form_html); tinyxml2::XMLDocument doc; doc.Parse(login_form_xhtml.c_str()); tinyxml2::XMLNode* node = doc.FirstChildElement("html"); while(node) { tinyxml2::XMLElement *element = node->ToElement(); if (element->Name() && !std::string(element->Name()).compare("input")) { std::string name = element->Attribute("name"); if (name == tagname_token) { token = element->Attribute("value"); break; } } node = Util::nextXMLNode(node); } if (token.empty()) { std::cout << "Failed to get login token" << std::endl; return std::string(); } //Create postdata - escape characters in email/password to support special characters postdata = (std::string)curl_easy_escape(curlhandle, tagname_username.c_str(), tagname_username.size()) + "=" + (std::string)curl_easy_escape(curlhandle, email.c_str(), email.size()) + "&" + (std::string)curl_easy_escape(curlhandle, tagname_password.c_str(), tagname_password.size()) + "=" + (std::string)curl_easy_escape(curlhandle, password.c_str(), password.size()) + "&" + (std::string)curl_easy_escape(curlhandle, tagname_login.c_str(), tagname_login.size()) + "=" + "&" + (std::string)curl_easy_escape(curlhandle, tagname_token.c_str(), tagname_token.size()) + "=" + (std::string)curl_easy_escape(curlhandle, token.c_str(), token.size()); curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login_check"); curl_easy_setopt(curlhandle, CURLOPT_POST, 1); curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str()); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0); curl_easy_setopt(curlhandle, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL); // Don't follow to redirect location because we need to check it for two step authorization. curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 0); CURLcode result = curl_easy_perform(curlhandle); memory.str(std::string()); if (result != CURLE_OK) { // Expected to hit maximum amount of redirects so don't print error on it if (result != CURLE_TOO_MANY_REDIRECTS) std::cout << curl_easy_strerror(result) << std::endl; } // Handle two step authorization std::string redirect_url = Util::CurlHandleGetInfoString(curlhandle, CURLINFO_REDIRECT_URL); if (redirect_url.find("two_step") != std::string::npos) { std::string security_code; std::string tagname_two_step_send = "second_step_authentication[send]"; std::string tagname_two_step_auth_letter_1 = "second_step_authentication[token][letter_1]"; std::string tagname_two_step_auth_letter_2 = "second_step_authentication[token][letter_2]"; std::string tagname_two_step_auth_letter_3 = "second_step_authentication[token][letter_3]"; std::string tagname_two_step_auth_letter_4 = "second_step_authentication[token][letter_4]"; std::string tagname_two_step_token = "second_step_authentication[_token]"; std::string token_two_step; std::string two_step_html = this->getResponse(redirect_url); redirect_url = ""; std::string two_step_xhtml = Util::htmlToXhtml(two_step_html); doc.Parse(two_step_xhtml.c_str()); node = doc.FirstChildElement("html"); while(node) { tinyxml2::XMLElement *element = node->ToElement(); if (element->Name() && !std::string(element->Name()).compare("input")) { std::string name = element->Attribute("name"); if (name == tagname_two_step_token) { token_two_step = element->Attribute("value"); break; } } node = Util::nextXMLNode(node); } std::cerr << "Security code: "; std::getline(std::cin,security_code); if (security_code.size() != 4) { std::cerr << "Security code must be 4 characters long" << std::endl; exit(1); } postdata = (std::string)curl_easy_escape(curlhandle, tagname_two_step_auth_letter_1.c_str(), tagname_two_step_auth_letter_1.size()) + "=" + security_code[0] + "&" + (std::string)curl_easy_escape(curlhandle, tagname_two_step_auth_letter_2.c_str(), tagname_two_step_auth_letter_2.size()) + "=" + security_code[1] + "&" + (std::string)curl_easy_escape(curlhandle, tagname_two_step_auth_letter_3.c_str(), tagname_two_step_auth_letter_3.size()) + "=" + security_code[2] + "&" + (std::string)curl_easy_escape(curlhandle, tagname_two_step_auth_letter_4.c_str(), tagname_two_step_auth_letter_4.size()) + "=" + security_code[3] + "&" + (std::string)curl_easy_escape(curlhandle, tagname_two_step_send.c_str(), tagname_two_step_send.size()) + "=" + "&" + (std::string)curl_easy_escape(curlhandle, tagname_two_step_token.c_str(), tagname_two_step_token.size()) + "=" + (std::string)curl_easy_escape(curlhandle, token_two_step.c_str(), token_two_step.size()); curl_easy_setopt(curlhandle, CURLOPT_URL, "https://login.gog.com/login/two_step"); curl_easy_setopt(curlhandle, CURLOPT_POST, 1); curl_easy_setopt(curlhandle, CURLOPT_POSTFIELDS, postdata.c_str()); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, 0); curl_easy_setopt(curlhandle, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL); // Don't follow to redirect location because it doesn't work properly. Must clean up the redirect url first. curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 0); result = curl_easy_perform(curlhandle); memory.str(std::string()); redirect_url = Util::CurlHandleGetInfoString(curlhandle, CURLINFO_REDIRECT_URL); } if (!redirect_url.empty()) { long response_code; do { curl_easy_setopt(curlhandle, CURLOPT_URL, redirect_url.c_str()); result = curl_easy_perform(curlhandle); memory.str(std::string()); result = curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); if ((response_code / 100) == 3) redirect_url = Util::CurlHandleGetInfoString(curlhandle, CURLINFO_REDIRECT_URL); boost::regex re(".*code=(.*?)([\?&].*|$)", boost::regex_constants::icase); boost::match_results what; if (boost::regex_search(redirect_url, what, re)) { auth_code = what[1]; if (!auth_code.empty()) break; } } while (result == CURLE_OK && (response_code / 100) == 3); } curl_easy_setopt(curlhandle, CURLOPT_URL, redirect_url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_HTTPGET, 1); curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, -1); curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); result = curl_easy_perform(curlhandle); if (result != CURLE_OK) { std::cout << curl_easy_strerror(result) << std::endl; } return auth_code; } std::string Website::LoginGetAuthCodeBrowser(const std::string& auth_url) { std::string auth_code; std::string url; std::cerr << "Login using browser at the following url" << std::endl; std::cerr << auth_url << std::endl << std::endl; std::cerr << "Copy & paste the full url from your browser here after login is complete" << std::endl; std::cerr << "URL: "; std::getline(std::cin, url); boost::regex re(".*code=(.*?)([\?&].*|$)", boost::regex_constants::icase); boost::match_results what; if (boost::regex_search(url, what, re)) { auth_code = what[1]; } std::ostringstream memory; curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_HTTPGET, 1); curl_easy_setopt(curlhandle, CURLOPT_MAXREDIRS, -1); curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); CURLcode result = curl_easy_perform(curlhandle); memory.str(std::string()); if (result != CURLE_OK) { std::cout << curl_easy_strerror(result) << std::endl; } return auth_code; } #ifdef USE_QT_GUI_LOGIN std::string Website::LoginGetAuthCodeGUI(const std::string& email, const std::string& password) { std::string auth_code; GuiLogin gl; gl.Login(email, password); auto cookies = gl.getCookies(); for (auto cookie : cookies) { curl_easy_setopt(curlhandle, CURLOPT_COOKIELIST, cookie.c_str()); } auth_code = gl.getCode(); return auth_code; } #endif bool Website::IsLoggedIn() { return this->IsloggedInSimple(); } /* Simple login check. Check login by trying to get account page. If response code isn't 200 then login failed. returns true if we are logged in returns false if we are not logged in */ bool Website::IsloggedInSimple() { bool bIsLoggedIn = false; std::ostringstream memory; std::string url = "https://www.gog.com/account"; long int response_code = 0; curl_easy_setopt(curlhandle, CURLOPT_URL, url.c_str()); curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 0); curl_easy_setopt(curlhandle, CURLOPT_WRITEFUNCTION, Util::CurlWriteMemoryCallback); curl_easy_setopt(curlhandle, CURLOPT_WRITEDATA, &memory); curl_easy_setopt(curlhandle, CURLOPT_NOPROGRESS, 1); curl_easy_perform(curlhandle); memory.str(std::string()); curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); std::vector response_codes = { 301, 302, 307, 308 }; if (response_code == 200) bIsLoggedIn = true; // We are logged in else if (std::any_of(response_codes.begin(), response_codes.end(), [response_code](long code){return code == response_code;})) { response_code = 0; std::string redir_url = Util::CurlHandleGetInfoString(curlhandle, CURLINFO_REDIRECT_URL);; if (redir_url == "https://embed.gog.com/account") { curl_easy_setopt(curlhandle, CURLOPT_URL, redir_url.c_str()); curl_easy_perform(curlhandle); memory.str(std::string()); curl_easy_getinfo(curlhandle, CURLINFO_RESPONSE_CODE, &response_code); if (response_code == 200) bIsLoggedIn = true; // We are logged in } } curl_easy_setopt(curlhandle, CURLOPT_FOLLOWLOCATION, 1); return bIsLoggedIn; } std::vector Website::getWishlistItems() { Json::Value root; Json::CharReaderBuilder builder; int i = 1; bool bAllPagesParsed = false; std::vector wishlistItems; do { std::string url = "https://www.gog.com/account/wishlist/search?hasHiddenProducts=false&hiddenFlag=0&isUpdated=0&mediaType=0&sortBy=title&system=&page=" + std::to_string(i); Json::Value root = this->getResponseJson(url); if (root.empty()) continue; if (root["page"].asInt() >= root["totalPages"].asInt()) bAllPagesParsed = true; if (root["products"].isArray()) { for (unsigned int i = 0; i < root["products"].size(); ++i) { wishlistItem item; Json::Value product = root["products"][i]; item.platform = 0; std::string platforms_text; bool bIsMovie = product["isMovie"].asBool(); if (!bIsMovie) { if (product["worksOn"]["Windows"].asBool()) item.platform |= GlobalConstants::PLATFORM_WINDOWS; if (product["worksOn"]["Mac"].asBool()) item.platform |= GlobalConstants::PLATFORM_MAC; if (product["worksOn"]["Linux"].asBool()) item.platform |= GlobalConstants::PLATFORM_LINUX; // Skip if platform doesn't match if (Globals::globalConfig.bPlatformDetection && !(item.platform & Globals::globalConfig.dlConf.iInstallerPlatform)) continue; } if (product["isComingSoon"].asBool()) item.tags.push_back("Coming soon"); if (product["isDiscounted"].asBool()) item.tags.push_back("Discount"); if (bIsMovie) item.tags.push_back("Movie"); item.release_date_time = 0; if (product.isMember("releaseDate") && product["isComingSoon"].asBool()) { if (!product["releaseDate"].empty()) { if (product["releaseDate"].isInt()) { item.release_date_time = product["releaseDate"].asInt(); } else { std::string release_date_time_string = product["releaseDate"].asString(); if (!release_date_time_string.empty()) { try { item.release_date_time = std::stoi(release_date_time_string); } catch (std::invalid_argument& e) { item.release_date_time = 0; } } } } } item.currency = product["price"]["symbol"].asString(); item.price = product["price"]["finalAmount"].isDouble() ? std::to_string(product["price"]["finalAmount"].asDouble()) + item.currency : product["price"]["finalAmount"].asString() + item.currency; item.discount_percent = product["price"]["discountPercentage"].isInt() ? std::to_string(product["price"]["discountPercentage"].asInt()) + "%" : product["price"]["discountPercentage"].asString() + "%"; item.discount = product["price"]["discountDifference"].isDouble() ? std::to_string(product["price"]["discountDifference"].asDouble()) + item.currency : product["price"]["discountDifference"].asString() + item.currency; item.store_credit = product["price"]["bonusStoreCreditAmount"].isDouble() ? std::to_string(product["price"]["bonusStoreCreditAmount"].asDouble()) + item.currency : product["price"]["bonusStoreCreditAmount"].asString() + item.currency; item.url = product["url"].asString(); if (!(item.url.find("http") == 0)) { if (item.url.front() == '/') item.url = "https://www.gog.com" + item.url; else item.url = "https://www.gog.com/" + item.url; } item.title = product["title"].asString(); item.bIsBonusStoreCreditIncluded = product["price"]["isBonusStoreCreditIncluded"].asBool(); item.bIsDiscounted = product["isDiscounted"].asBool(); wishlistItems.push_back(item); } } i++; } while (!bAllPagesParsed); return wishlistItems; } std::map Website::getTags() { std::string url = "https://www.gog.com/account/getFilteredProducts?mediaType=1&sortBy=title&system=&page=1"; std::string response = this->getResponse(url); std::istringstream json_stream(response); Json::Value json; std::map tags; try { // Parse JSON json_stream >> json; } catch (const Json::Exception& exc) { std::cout << exc.what(); if (!response.empty()) { if(response[0] != '{') { // Response was not JSON. Assume that cookies have expired. std::cerr << "Response was not JSON. Cookies have most likely expired. Try --login first." << std::endl; } } exit(1); } tags = this->getTagsFromJson(json["tags"]); return tags; } std::map Website::getTagsFromJson(const Json::Value& json) { std::map tags; if (!json.empty()) { for (auto node : json) { tags[node["id"].asString()] = node["name"].asString(); } } return tags; } std::vector Website::getOwnedGamesIds() { std::vector vOwnedGamesIds; Json::Value owned_json = this->getResponseJson("https://www.gog.com/user/data/games"); if (owned_json.isMember("owned")) { for (auto id : owned_json["owned"]) vOwnedGamesIds.push_back(id.asString()); } return vOwnedGamesIds; } lgogdownloader-3.17/src/ziputil.cpp000066400000000000000000000452221476654310300174640ustar00rootroot00000000000000/* This program is free software. It comes without any warranty, to * the extent permitted by applicable law. You can redistribute it * and/or modify it under the terms of the Do What The Fuck You Want * To Public License, Version 2, as published by Sam Hocevar. See * http://www.wtfpl.net/ for more details. */ #include "ziputil.h" #include #include #include #include #include #include #include off_t ZipUtil::getMojoSetupScriptSize(std::stringstream *stream) { off_t mojosetup_script_size = -1; int script_lines = 0; boost::regex re("offset=`head -n (\\d+?) \"\\$0\"", boost::regex::perl | boost::regex::icase); boost::match_results what; if (boost::regex_search(stream->str(), what, re)) { script_lines = std::stoi(what[1]); } std::string script; for (int i = 0; i < script_lines; ++i) { std::string line; std::getline(*stream, line); script += line + "\n"; } mojosetup_script_size = script.size(); return mojosetup_script_size; } off_t ZipUtil::getMojoSetupInstallerSize(std::stringstream *stream) { off_t mojosetup_installer_size = -1; boost::regex re("filesizes=\"(\\d+?)\"", boost::regex::perl | boost::regex::icase); boost::match_results what; if (boost::regex_search(stream->str(), what, re)) { mojosetup_installer_size = std::stoll(what[1]); } return mojosetup_installer_size; } struct tm ZipUtil::date_time_to_tm(uint64_t date, uint64_t time) { /* DOS date time format * Y|Y|Y|Y|Y|Y|Y|M| |M|M|M|D|D|D|D|D| |h|h|h|h|h|m|m|m| |m|m|m|s|s|s|s|s * * second is divided by 2 * month starts at 1 * https://msdn.microsoft.com/en-us/library/windows/desktop/ms724247%28v=vs.85%29.aspx */ uint64_t local_time_base_year = 1900; uint64_t dos_time_base_year = 1980; struct tm timeinfo; timeinfo.tm_year = (uint16_t)(((date & 0xFE00) >> 9) - local_time_base_year + dos_time_base_year); timeinfo.tm_mon = (uint16_t)(((date & 0x1E0) >> 5) - 1); timeinfo.tm_mday = (uint16_t)(date & 0x1F); timeinfo.tm_hour = (uint16_t)((time & 0xF800) >> 11); timeinfo.tm_min = (uint16_t)((time & 0x7E0) >> 5); timeinfo.tm_sec = (uint16_t)(2 * (time & 0x1F)); timeinfo.tm_isdst = -1; return timeinfo; } bool ZipUtil::isValidDate(struct tm timeinfo) { if (!(timeinfo.tm_year >= 0 && timeinfo.tm_year <= 207)) return false; if (!(timeinfo.tm_mon >= 0 && timeinfo.tm_mon <= 11)) return false; if (!(timeinfo.tm_mday >= 1 && timeinfo.tm_mday <= 31)) return false; if (!(timeinfo.tm_hour >= 0 && timeinfo.tm_hour <= 23)) return false; if (!(timeinfo.tm_min >= 0 && timeinfo.tm_min <= 59)) return false; if (!(timeinfo.tm_sec >= 0 && timeinfo.tm_sec <= 59)) return false; return true; } uint64_t ZipUtil::readValue(std::istream *stream, uint32_t len) { uint64_t value = 0; for (uint32_t i = 0; i < len; i++) { value |= ((uint64_t)(stream->get() & 0xFF)) << (i * 8); } return value; } uint64_t ZipUtil::readUInt64(std::istream *stream) { uint64_t value = (uint64_t)readValue(stream, sizeof(uint64_t)); return value; } uint32_t ZipUtil::readUInt32(std::istream *stream) { uint32_t value = (uint32_t)readValue(stream, sizeof(uint32_t)); return value; } uint16_t ZipUtil::readUInt16(std::istream *stream) { uint16_t value = (uint16_t)readValue(stream, sizeof(uint16_t)); return value; } uint8_t ZipUtil::readUInt8(std::istream *stream) { uint8_t value = (uint8_t)readValue(stream, sizeof(uint8_t)); return value; } off_t ZipUtil::getZipEOCDOffsetSignature(std::istream *stream, const uint32_t& signature) { off_t offset = -1; stream->seekg(0, stream->end); off_t stream_length = stream->tellg(); for (off_t i = 4; i <= stream_length; i++) { off_t pos = stream_length - i; stream->seekg(pos, stream->beg); if (readUInt32(stream) == signature) { offset = stream->tellg(); offset -= 4; break; } } return offset; } off_t ZipUtil::getZipEOCDOffset(std::istream *stream) { return getZipEOCDOffsetSignature(stream, ZIP_EOCD_HEADER_SIGNATURE); } off_t ZipUtil::getZip64EOCDOffset(std::istream *stream) { return getZipEOCDOffsetSignature(stream, ZIP_EOCD_HEADER_SIGNATURE64); } zipEOCD ZipUtil::readZipEOCDStruct(std::istream *stream, const off_t& eocd_start_pos) { zipEOCD eocd; stream->seekg(eocd_start_pos, stream->beg); // end of central dir signature <4 bytes> eocd.header = readUInt32(stream); // number of this disk <2 bytes> eocd.disk = readUInt16(stream); // Number of this disk // number of the disk with the start of the central directory <2 bytes> eocd.cd_start_disk = readUInt16(stream); // total number of entries in the central directory on this disk <2 bytes> eocd.cd_records = readUInt16(stream); // total number of entries in the central directory <2 bytes> eocd.total_cd_records = readUInt16(stream); // size of the central directory <4 bytes> eocd.cd_size = readUInt32(stream); // offset of start of central directory with respect to the starting disk number <4 bytes> eocd.cd_start_offset = readUInt32(stream); // .ZIP file comment length <2 bytes> eocd.comment_length = readUInt16(stream); // .ZIP file comment if (eocd.comment_length > 0) { char *buf = new char[eocd.comment_length + 1]; stream->read(buf, eocd.comment_length); eocd.comment = std::string(buf, eocd.comment_length); delete[] buf; } return eocd; } zip64EOCD ZipUtil::readZip64EOCDStruct(std::istream *stream, const off_t& eocd_start_pos) { zip64EOCD eocd; stream->seekg(eocd_start_pos, stream->beg); // zip64 end of central dir signature <4 bytes> eocd.header = readUInt32(stream); // size of zip64 end of central directory record <8 bytes> eocd.directory_record_size = readUInt64(stream); /* The value stored into the "size of zip64 end of central * directory record" should be the size of the remaining * record and should not include the leading 12 bytes. * * Size = SizeOfFixedFields + SizeOfVariableData - 12 */ // version made by <2 bytes> eocd.version_made_by = readUInt16(stream); // version needed to extract <2 bytes> eocd.version_needed = readUInt16(stream); // number of this disk <4 bytes> eocd.cd = readUInt32(stream); // number of the disk with the start of the central directory <8 bytes> eocd.cd_start = readUInt32(stream); // total number of entries in the central directory on this disk <8 bytes> eocd.cd_total_disk = readUInt64(stream); // total number of entries in the central directory <8 bytes> eocd.cd_total = readUInt64(stream); // size of the central directory <8 bytes> eocd.cd_size = readUInt64(stream); // offset of start of central directory with respect to the starting disk number <8 bytes> eocd.cd_offset = readUInt64(stream); // zip64 extensible data sector // This is data is not needed for our purposes so we just ignore this data return eocd; } zipCDEntry ZipUtil::readZipCDEntry(std::istream *stream) { zipCDEntry cd; char *buf; // file header signature <4 bytes> cd.header = readUInt32(stream); cd.isLocalCDEntry = (cd.header == ZIP_LOCAL_HEADER_SIGNATURE); if (!cd.isLocalCDEntry) { // version made by <2 bytes> cd.version_made_by = readUInt16(stream); } // version needed to extract <2 bytes> cd.version_needed = readUInt16(stream); // general purpose bit flag <2 bytes> cd.flag = readUInt16(stream); // compression method <2 bytes> cd.compression_method = readUInt16(stream); // last mod file time <2 bytes> cd.mod_time = readUInt16(stream); // last mod file date <2 bytes> cd.mod_date = readUInt16(stream); // crc-32 <4 bytes> cd.crc32 = readUInt32(stream); // compressed size <4 bytes> cd.comp_size = readUInt32(stream); // uncompressed size <4 bytes> cd.uncomp_size = readUInt32(stream); // file name length <2 bytes> cd.filename_length = readUInt16(stream); // extra field length <2 bytes> cd.extra_length = readUInt16(stream); if (!cd.isLocalCDEntry) { // file comment length <2 bytes> cd.comment_length = readUInt16(stream); // disk number start <2 bytes> cd.disk_num = readUInt16(stream); // internal file attributes <2 bytes> cd.internal_file_attr = readUInt16(stream); // external file attributes <4 bytes> cd.external_file_attr = readUInt32(stream); // relative offset of local header <4 bytes> cd.disk_offset = readUInt32(stream); } // file name buf = new char[cd.filename_length + 1]; stream->read(buf, cd.filename_length); cd.filename = std::string(buf, cd.filename_length); delete[] buf; // extra field buf = new char[cd.extra_length + 1]; stream->read(buf, cd.extra_length); cd.extra = std::string(buf, cd.extra_length); delete[] buf; std::stringstream extra_stream(cd.extra); cd.timestamp = 0; struct tm timeinfo = date_time_to_tm(cd.mod_date, cd.mod_time); if (isValidDate(timeinfo)) cd.timestamp = mktime(&timeinfo); // Read extra fields off_t i = 0; while (i < cd.extra_length) { /* Extra field * <2 bytes> signature * <2 bytes> size of extra field data * extra field data */ uint16_t header_id = readUInt16(&extra_stream); uint16_t extra_data_size = readUInt16(&extra_stream); if (header_id == ZIP_EXTENSION_ZIP64) { /* Zip64 Extended Information Extra Field * <8 bytes> size of uncompressed file * <8 bytes> size of compressed data * <8 bytes> offset of local header record * <4 bytes> number of the disk on which this file starts * * The fields only appear if the corresponding Local or Central * directory record field is set to UINT16_MAX or UINT32_MAX */ if (cd.uncomp_size == UINT32_MAX) cd.uncomp_size = readUInt64(&extra_stream); if (cd.comp_size == UINT32_MAX) cd.comp_size = readUInt64(&extra_stream); if (cd.disk_offset == UINT32_MAX) cd.disk_offset = readUInt64(&extra_stream); if (cd.disk_num == UINT16_MAX) cd.disk_num = readUInt32(&extra_stream); } else if (header_id == ZIP_EXTENDED_TIMESTAMP) { /* Extended Timestamp Extra Field * * Local header version * <1 byte> info bits * <4 bytes> modification time * <4 bytes> access time * <4 bytes> creation time * * Central header version * <1 byte> info bits * <4 bytes> modification time * * The lower three info bits in both headers indicate * which timestamps are present in the local extra field * bit 0 if set, modification time is present * bit 1 if set, access time is present * bit 2 if set, creation time is present * bits 3-7 reserved for additional timestamps; not set * * If info bits indicate that modification time is present * in the local header field, it must be present in the * central header field. * Those times that are present will appear in the order * indicated, but any combination of times may be omitted. */ uint32_t modification_time = 0; uint32_t access_time = 0; uint32_t creation_time = 0; uint8_t flags = readUInt8(&extra_stream); if (flags & 0x1) // modification time is present { modification_time = readUInt32(&extra_stream); cd.timestamp = modification_time; } if (cd.isLocalCDEntry) { if (flags & 0x2) // access time is present { access_time = readUInt32(&extra_stream); } if (flags & 0x4) // creation time is present { creation_time = readUInt32(&extra_stream); } } // access time and creation time are unused currently // suppress -Wunused-but-set-variable messages by casting these variables to void (void) access_time; (void) creation_time; } else if (header_id == ZIP_INFOZIP_UNIX_NEW) { /* Info-ZIP New Unix Extra Field * <1 byte> version * <1 byte> size of uid * uid * <1 byte> size of gid * gid * * Currently Version is set to the number 1. If there is a need * to change this field, the version will be incremented. * UID and GID entries are stored in standard little endian format */ uint8_t version = readUInt8(&extra_stream); if (version == 1) { uint64_t uid = 0; uint64_t gid = 0; uint8_t uid_size = readUInt8(&extra_stream); for (uint8_t i = 0; i < uid_size; i++) { uid |= ((uint64_t)extra_stream.get()) << (i * 8); } uint8_t gid_size = readUInt8(&extra_stream); for (uint8_t i = 0; i < gid_size; i++) { gid |= ((uint64_t)extra_stream.get()) << (i * 8); } } else { // Unknown version // Skip the rest of this field extra_stream.seekg(extra_data_size - 1, extra_stream.cur); } } else { // Skip over unknown/unimplemented extra field extra_stream.seekg(extra_data_size, extra_stream.cur); } i += 4 + extra_data_size; } // file comment buf = new char[cd.comment_length + 1]; stream->read(buf, cd.comment_length); cd.comment = std::string(buf, cd.comment_length); delete[] buf; return cd; } /* Extract file returns 0 if successful returns 1 if input file could not be opened returns 2 if compression method is unsupported returns 3 if output file could not be created returns 4 if zlib error returns 5 if failed to set timestamp */ int ZipUtil::extractFile(const std::string& input_file_path, const std::string& output_file_path) { std::ifstream input_file(input_file_path, std::ifstream::in | std::ifstream::binary); if (!input_file) { // Could not open input file return 1; } // Read header zipCDEntry cd = readZipCDEntry(&input_file); if (!(cd.compression_method == boost::iostreams::zlib::deflated || cd.compression_method == boost::iostreams::zlib::no_compression)) { // Unsupported compression method return 2; } boost::iostreams::zlib_params p; p.window_bits = 15; p.noheader = true; // zlib header and trailing adler-32 checksum is omitted std::ofstream output_file(output_file_path, std::ofstream::out | std::ofstream::binary); if (!output_file) { // Failed to create output file return 3; } // Uncompress boost::iostreams::filtering_streambuf in; if (cd.compression_method == boost::iostreams::zlib::deflated) in.push(boost::iostreams::zlib_decompressor(p)); in.push(input_file); try { boost::iostreams::copy(in, output_file); } catch(boost::iostreams::zlib_error & e) { // zlib error return 4; } input_file.close(); output_file.close(); if (cd.timestamp > 0) { boost::system::error_code ec; boost::filesystem::last_write_time(output_file_path, cd.timestamp, ec); if (ec) { // Failed to set timestamp return 5; } } return 0; } /* Extract stream to stream returns 0 if successful returns 1 if input stream is not valid returns 2 if compression method is unsupported returns 3 if output stream is not valid returns 4 if zlib error */ int ZipUtil::extractStream(std::istream* input_stream, std::ostream* output_stream) { if (!input_stream) { // Input stream not valid return 1; } // Read header zipCDEntry cd = readZipCDEntry(input_stream); if (!(cd.compression_method == boost::iostreams::zlib::deflated || cd.compression_method == boost::iostreams::zlib::no_compression)) { // Unsupported compression method return 2; } boost::iostreams::zlib_params p; p.window_bits = 15; p.noheader = true; // zlib header and trailing adler-32 checksum is omitted if (!output_stream) { // Output stream not valid return 3; } // Uncompress boost::iostreams::filtering_streambuf in; if (cd.compression_method == boost::iostreams::zlib::deflated) in.push(boost::iostreams::zlib_decompressor(p)); in.push(*input_stream); try { boost::iostreams::copy(in, *output_stream); } catch(boost::iostreams::zlib_error & e) { // zlib error return 4; } return 0; } boost::filesystem::perms ZipUtil::getBoostFilePermission(const uint16_t& attributes) { boost::filesystem::perms perms = boost::filesystem::no_perms; if (attributes & S_IRUSR) perms |= boost::filesystem::owner_read; if (attributes & S_IWUSR) perms |= boost::filesystem::owner_write; if (attributes & S_IXUSR) perms |= boost::filesystem::owner_exe; if (attributes & S_IRGRP) perms |= boost::filesystem::group_read; if (attributes & S_IWGRP) perms |= boost::filesystem::group_write; if (attributes & S_IXGRP) perms |= boost::filesystem::group_exe; if (attributes & S_IROTH) perms |= boost::filesystem::others_read; if (attributes & S_IWOTH) perms |= boost::filesystem::others_write; if (attributes & S_IXOTH) perms |= boost::filesystem::others_exe; return perms; } bool ZipUtil::isSymlink(const uint16_t& attributes) { bool bSymlink = ((attributes & S_IFMT) == S_IFLNK); return bSymlink; }