diff --git a/.gitattributes b/.gitattributes index 18f17680c..9e24d84bb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,7 @@ nzbget-setup.nsi eol=crlf windows/package-info.json eol=crlf windows/resources/resource.h eol=crlf windows/resources/nzbget.rc eol=crlf +tests/testdata/**/* binary # Configure GitHub's language detector lib/* linguist-vendored linguist-language=C++ diff --git a/.github/ISSUE_TEMPLATE/01_bug_report.yml b/.github/ISSUE_TEMPLATE/01_bug_report.yml index 26dd81e64..d6bfbe495 100644 --- a/.github/ISSUE_TEMPLATE/01_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/01_bug_report.yml @@ -14,7 +14,8 @@ body: label: NZBGet Version description: Which version of NZBGet has this bug? options: - - v24.4-testing + - v24.5-testing + - v24.4-stable - v24.3-stable - v24.2-stable - v24.1-stable diff --git a/.github/ISSUE_TEMPLATE/02_feature_request.yml b/.github/ISSUE_TEMPLATE/02_feature_request.yml index 903c175ca..b34646233 100644 --- a/.github/ISSUE_TEMPLATE/02_feature_request.yml +++ b/.github/ISSUE_TEMPLATE/02_feature_request.yml @@ -19,6 +19,8 @@ body: - macOS - NAS/Synology/QNAP - Linux/Docker + - FreeBSD + - Android validations: required: true - type: textarea diff --git a/.github/ISSUE_TEMPLATE/03_build.yml b/.github/ISSUE_TEMPLATE/03_build.yml index 5a6b4800b..2d530b27f 100644 --- a/.github/ISSUE_TEMPLATE/03_build.yml +++ b/.github/ISSUE_TEMPLATE/03_build.yml @@ -7,7 +7,8 @@ body: label: NZBGet Version description: Version of NZBGet for the scope of this issue options: - - v24.4-testing + - v24.5-testing + - v24.4-stable - v24.3-stable - v24.2-stable - v24.1-stable @@ -29,6 +30,8 @@ body: - macOS - NAS/Synology/QNAP - Linux/Docker + - FreeBSD + - Android validations: required: true - type: textarea diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 4de0d1bd7..335259318 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -1,4 +1,4 @@ -name: android build +name: android build on: workflow_call: @@ -9,7 +9,7 @@ jobs: runs-on: [self-hosted, nzbget-android] steps: - + - name: Checkout uses: actions/checkout@v4 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3c45a07c..5ad28809e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,10 +17,14 @@ jobs: build-android: uses: ./.github/workflows/android.yml + build-freebsd: + uses: ./.github/workflows/freebsd.yml + build-osx: uses: ./.github/workflows/osx.yml permissions: actions: write + secrets: inherit build-linux-pkg: uses: ./.github/workflows/linux-pkg.yml @@ -42,7 +46,7 @@ jobs: env: PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }} runs-on: ubuntu-latest - needs: [build-windows, build-linux, build-osx, build-android, repack-qnap, build-linux-pkg] + needs: [build-windows, build-linux, build-osx, build-android, build-freebsd, repack-qnap, build-linux-pkg] permissions: actions: write steps: @@ -56,6 +60,7 @@ jobs: mv nzbget-windows-installers/* builds || true mv nzbget-linux-installers/* builds || true mv nzbget-android-installers/* builds || true + mv nzbget-freebsd-installers/* builds || true mv nzbget-osx-installers/* builds || true mv nzbget-qnap-packages/* builds || true mv nzbget-deb-packages/* builds || true @@ -71,7 +76,7 @@ jobs: echo "nzbget_signatures({" | tee $SIGS_FILE echo | tee -a $SIGS_FILE - for FILE in *.exe *.run *.zip *.spk *.qpkg *.deb *.rpm; do + for FILE in *.exe *.run *.zip *.spk *.qpkg *.deb *.rpm *.dmg; do [ -f $FILE ] || continue MD5=$(openssl dgst -md5 $FILE | cut -d ' ' -f 2) @@ -107,6 +112,7 @@ jobs: nzbget-windows-installers nzbget-linux-installers nzbget-android-installers + nzbget-freebsd-installers nzbget-osx-installers nzbget-qnap-packages diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml new file mode 100644 index 000000000..c86de611d --- /dev/null +++ b/.github/workflows/freebsd.yml @@ -0,0 +1,47 @@ +name: freebsd build + +on: + workflow_call: + workflow_dispatch: + +jobs: + build: + runs-on: [self-hosted, nzbget-freebsd] + + steps: + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build + run: | + BUILD_PARAMS="freebsd bin installer" + if [ "$GITHUB_REF_NAME" == "develop" ] || [ "$GITHUB_REF_NAME" == "main" ]; then + BUILD_PARAMS="$BUILD_PARAMS debug release" + else + BUILD_PARAMS="$BUILD_PARAMS release" + fi + if [ "$GITHUB_REF_NAME" != "main" ]; then + BUILD_PARAMS="$BUILD_PARAMS testing" + fi + bash linux/build-nzbget.sh $BUILD_PARAMS + + - name: Rename build artifacts + if: github.ref_name != 'main' && github.ref_name != 'develop' + run: | + cd build + SUFFIX="-${GITHUB_REF_NAME/\//-}-bin-freebsd.run" + for FILE in *.run; do + [ -f $FILE ] || continue + NEW_FILE=${FILE/-bin-freebsd.run/$SUFFIX} + mv $FILE $NEW_FILE + done + + - name: Upload build artifacts + uses: actions/upload-artifact@v4 + with: + name: nzbget-freebsd-installers + path: build/*.run + retention-days: 5 diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml index 53a52c45e..d24d8d9c0 100644 --- a/.github/workflows/osx.yml +++ b/.github/workflows/osx.yml @@ -9,12 +9,12 @@ jobs: runs-on: [self-hosted, macos, x64] steps: - + - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Build run: | if [ "$GITHUB_REF_NAME" != "main" ]; then @@ -45,12 +45,12 @@ jobs: runs-on: [self-hosted, macos, arm64] steps: - + - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - + - name: Build run: | if [ "$GITHUB_REF_NAME" != "main" ]; then @@ -77,23 +77,58 @@ jobs: path: build/*-universal.zip retention-days: 5 + sign-universal: + runs-on: [self-hosted, macos, arm64] + needs: [build-universal] + + steps: + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download build artifacts + uses: actions/download-artifact@v4 + + - name: Sign and notarize + env: + NOTARIZE: ${{ vars.OSX_NOTARIZE }} + KEYCHAIN_PASSWORD: ${{ secrets.OSX_KEYCHAIN_PASSWORD }} + DEVELOPER_IDENTITY: ${{ secrets.OSX_DEVELOPER_IDENTITY }} + NOTARY_KEY_ID: ${{ secrets.OSX_NOTARY_KEY_ID }} + NOTARY_KEY_ISSUER: ${{ secrets.OSX_NOTARY_KEY_ISSUER }} + run: | + mkdir -p build + cp osx/sign/* build + cp nzbget-osx-installers-universal/*.zip build/ + cd build + bash nzbget-sign.sh *.zip + + - name: Upload signed build artifacts + uses: actions/upload-artifact@v4 + with: + name: nzbget-osx-installers-universal-signed + path: build/*.dmg + retention-days: 5 + combine-osx-artifacts: runs-on: ubuntu-latest - needs: [build-x64, build-universal] + needs: [build-x64, sign-universal] permissions: actions: write steps: - + - name: Download build artifacts uses: actions/download-artifact@v4 - + - name: Combine artifacts run: | mkdir -p nzbget-osx-installers mv nzbget-osx-installers-x64/* nzbget-osx-installers - mv nzbget-osx-installers-universal/* nzbget-osx-installers + mv nzbget-osx-installers-universal-signed/* nzbget-osx-installers - - name: Upload build artifacts with signatures + - name: Upload combined build artifacts uses: actions/upload-artifact@v4 with: name: nzbget-osx-installers @@ -106,3 +141,4 @@ jobs: name: | nzbget-osx-installers-x64 nzbget-osx-installers-universal + nzbget-osx-installers-universal-signed diff --git a/.github/workflows/windows-tests.yml b/.github/workflows/windows-tests.yml index 8fed2240d..06847dadb 100644 --- a/.github/workflows/windows-tests.yml +++ b/.github/workflows/windows-tests.yml @@ -3,7 +3,6 @@ name: windows tests on: push: branches: - - feature/* - develop - main pull_request: diff --git a/CMakeLists.txt b/CMakeLists.txt index d3f818488..ebe8ca946 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,12 +4,11 @@ if (PROJECT_SOURCE_DIR STREQUAL PROJECT_BINARY_DIR) message(FATAL_ERROR "In-source builds are not allowed. You should create separate directory for build files.") endif() -set_property(GLOBAL PROPERTY PACKAGE) -set_property(GLOBAL PROPERTY LIBS) -set_property(GLOBAL PROPERTY INCLUDES) - -set(VERSION "24.3") +set(VERSION "24.4") set(PACKAGE "nzbget") +set(LIBS "") +set(INCLUDES ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) + set(PACKAGE_BUGREPORT "https://github.com/nzbgetcom/nzbget/issues") set(CMAKE_CXX_STANDARD 17) set(CMAKE_C_STANDARD 17) @@ -17,16 +16,13 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_C_EXTENSIONS OFF) -set(CMAKE_CONFIGURATION_TYPES "Release" "Debug") string(REGEX MATCH "^([0-9]+)\\.([0-9]+)" VERSION_MATCH ${VERSION}) set(VERSION_MAJOR ${CMAKE_MATCH_1}) set(VERSION_MINOR ${CMAKE_MATCH_2}) -add_compile_definitions(HAVE_CONFIG_H=1) - if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + set(CMAKE_BUILD_TYPE "Release") endif() option(BUILD_ONLY_TESTS "Build only tests (for CI)") @@ -38,60 +34,12 @@ project( LANGUAGES C CXX ) -if(APPLE) - # On macOS Cmake, when cross-compiling, sometimes CMAKE_SYSTEM_PROCESSOR wrongfully stays - # the same as CMAKE_HOST_SYSTEM_PROCESSOR regardless the target CPU. - # The manual call to set(CMAKE_SYSTEM_PROCESSOR) has to be set after the project() call. - # because project() might reset CMAKE_SYSTEM_PROCESSOR back to the value of CMAKE_HOST_SYSTEM_PROCESSOR. - # Check if CMAKE_SYSTEM_PROCESSOR is not equal to CMAKE_OSX_ARCHITECTURES - if(NOT CMAKE_OSX_ARCHITECTURES STREQUAL "") - if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_OSX_ARCHITECTURES) - # Split CMAKE_OSX_ARCHITECTURES into a list - string(REPLACE ";" " " ARCH_LIST ${CMAKE_OSX_ARCHITECTURES}) - separate_arguments(ARCH_LIST UNIX_COMMAND ${ARCH_LIST}) - # Count the number of architectures - list(LENGTH ARCH_LIST ARCH_COUNT) - # Ensure that exactly one architecture is specified - if(NOT ARCH_COUNT EQUAL 1) - message(FATAL_ERROR "CMAKE_OSX_ARCHITECTURES must have exactly one value. Current value: ${CMAKE_OSX_ARCHITECTURES}") - endif() - set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_OSX_ARCHITECTURES}) - message(STATUS "CMAKE_SYSTEM_PROCESSOR is manually set to ${CMAKE_SYSTEM_PROCESSOR}") - endif() - endif() -endif() - -if(CMAKE_BUILD_TYPE STREQUAL "Debug") - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") - set(CMAKE_CXX_FLAGS "-O0 -pthread -g -DDEBUG -Weverything -Wno-c++98-compat" CACHE STRING "" FORCE) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "-O0 -pthread -g -DDEBUG -Wall -Wextra" CACHE STRING "" FORCE) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_CXX_FLAGS "/Od /Zi /MTd /MP /W4 /EHs /DDEBUG /D_DEBUG /DWIN32 /wd4800 /wd4267" CACHE STRING "" FORCE) - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} winmm.lib Dbghelp.lib libcpmtd.lib" CACHE STRING "" FORCE) - endif() -elseif(CMAKE_BUILD_TYPE STREQUAL "Release") - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") - set(CMAKE_CXX_FLAGS "-O2 -g0 -pthread -DNDEBUG -Weverything -Wno-c++98-compat" CACHE STRING "" FORCE) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(CMAKE_CXX_FLAGS "-O2 -g0 -pthread -DNDEBUG -Wall -Wextra" CACHE STRING "" FORCE) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - set(CMAKE_CXX_FLAGS "/O2 /MT /MP /W4 /EHs /DNDEBUG /DWIN32 /wd4800 /wd4267" CACHE STRING "" FORCE) - set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} winmm.lib" CACHE STRING "" FORCE) - endif() -endif() - -set(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS} CACHE STRING "" FORCE) -set(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS} CACHE STRING "" FORCE) -set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS} CACHE STRING "" FORCE) -set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS} CACHE STRING "" FORCE) - -include_directories(${CMAKE_BINARY_DIR}) -include_directories(${CMAKE_SOURCE_DIR}) +include(cmake/common.cmake) +include(daemon/sources.cmake) if(NOT BUILD_ONLY_TESTS) - include(daemon/sources.cmake) add_executable(${PACKAGE} ${SRC}) + target_precompile_headers(${PACKAGE} PRIVATE ${CMAKE_SOURCE_DIR}/daemon/main/nzbget.h) if(CMAKE_BUILD_TYPE STREQUAL "Release") add_custom_command( @@ -105,6 +53,7 @@ endif() if(WIN32) include(cmake/windows.cmake) + if(NOT BUILD_ONLY_TESTS) target_sources(${PACKAGE} PRIVATE ${CMAKE_SOURCE_DIR}/windows/resources/nzbget.rc) configure_file( @@ -125,11 +74,14 @@ if(WIN32) endif() else() include(cmake/posix.cmake) + if(NOT BUILD_ONLY_TESTS) include(${CMAKE_SOURCE_DIR}/cmake/install.cmake) endif() endif() +include(lib/sources.cmake) + configure_file( ${CMAKE_SOURCE_DIR}/cmake/config.h.in ${CMAKE_BINARY_DIR}/config.h @@ -137,21 +89,7 @@ configure_file( if(NOT BUILD_ONLY_TESTS) target_link_libraries(${PACKAGE} PRIVATE ${LIBS}) - target_include_directories(${PACKAGE} PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/connect - ${CMAKE_SOURCE_DIR}/daemon/extension - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/frontend - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/daemon/nserv - ${CMAKE_SOURCE_DIR}/daemon/postprocess - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/system - ${INCLUDES} - ) + target_include_directories(${PACKAGE} PRIVATE ${INCLUDES}) endif() if(ENABLE_TESTS OR BUILD_ONLY_TESTS) diff --git a/ChangeLog.md b/ChangeLog.md index 48d552abf..387b07df9 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,31 @@ +nzbget-24.4 + - Features: + - Replaced the `Par2` library with `Par2-turbo` which provides faster file recovery on x86/ARM platforms + [#420](https://github.com/nzbgetcom/nzbget/pull/420) + - Better UTF-8 support + [#412](https://github.com/nzbgetcom/nzbget/pull/412) + - FreeBSD packaging support + [#417](https://github.com/nzbgetcom/nzbget/pull/416) + + - Bug fixes: + - Fixed excessive CPU usage by decreasing `cert.pem` certificate loading for TLS connections + [#400](https://github.com/nzbgetcom/nzbget/pull/400) + - Fixed password-protected unpacking when the nzb file was added to the queue via command line + [#398](https://github.com/nzbgetcom/nzbget/pull/398) + - STATUS page now shows the correct Windows version Windows 7 instead of incorrectly showing Windows XP + [#419](https://github.com/nzbgetcom/nzbget/pull/419) + - Fixed duplications on the STATUS page + [#407](https://github.com/nzbgetcom/nzbget/pull/407) + - Fixed `nzbget.conf.template` installation path on POSIX + [#405](https://github.com/nzbgetcom/nzbget/pull/405) + - Fixed `DaemonUsername` check on QNAP + [#391](https://github.com/nzbgetcom/nzbget/pull/391) + + - For developers: + - Fixed `postprocess` tests + [#413](https://github.com/nzbgetcom/nzbget/pull/413) + - Removed `Par2` source files from the repository. `CMake` now pulls and builds `Par2-turbo` directly from its repository + nzbget-24.3 - Features: - Disk performance tests for better analysis of user environment performance diff --git a/README.md b/README.md index 6b5e0ee1c..b6621b958 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ ![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.1/total?label=v24.1) ![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.2/total?label=v24.2) ![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.3/total?label=v24.3) +![GitHub release (by tag)](https://img.shields.io/github/downloads/nzbgetcom/nzbget/v24.4/total?label=v24.4) ![docker pulls](https://img.shields.io/docker/pulls/nzbgetcom/nzbget.svg) [![linux build](https://github.com/nzbgetcom/nzbget/actions/workflows/linux.yml/badge.svg?branch=main)](https://github.com/nzbgetcom/nzbget/actions/workflows/linux.yml) @@ -23,6 +24,9 @@ ![GitHub repo size](https://img.shields.io/github/repo-size/nzbgetcom/nzbget) +[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?&logo=discord&logoColor=white)](https://discord.gg/mV9Vn9sM7C) + + NZBGet is a binary downloader, which downloads files from Usenet based-on information given in nzb files. NZBGet is written in C++ and is known for its performance and efficiency. @@ -63,6 +67,8 @@ Android packages are available for Android 5.0+. [Android readme](docs/ANDROID.m `Linux`: Linux kernel 2.6 and later, x86 (32 or 64 Bit), ARM 32-bit (armel armhf), ARM 64-bit (aarch64), MIPS (mipseb mipsel), PowerPC (ppc6xx ppc500), RISC-V 64-bit (riscv64) +`FreeBSD`: FreeBSD 13.0+ x86_64 + `macOS`: X64 binary: macOS Mojave 10.14+, Universal (Intel / Apple Silicon) binary: macOS Monterey 12+ ## Building from sources @@ -80,7 +86,7 @@ Android packages are available for Android 5.0+. [Android readme](docs/ANDROID.m ## Contribution -Contributions are very welcome - not only from developers, but from our users too - please don't hesitate to participate in [discussions](https://github.com/nzbgetcom/nzbget/discussions) or [create a new discussion](https://github.com/nzbgetcom/nzbget/discussions/new/choose) +Contributions are very welcome - not only from developers, but from our users too - please don't hesitate to participate in [discussions](https://github.com/nzbgetcom/nzbget/discussions) or [create a new discussion](https://github.com/nzbgetcom/nzbget/discussions/new/choose) or [join our Discord server](https://discord.gg/mV9Vn9sM7C). For more information - see [Contributing](docs/CONTRIBUTING.md). diff --git a/cmake/common.cmake b/cmake/common.cmake new file mode 100644 index 000000000..88fec5815 --- /dev/null +++ b/cmake/common.cmake @@ -0,0 +1,24 @@ +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") + add_compile_options(-Weverything) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + add_compile_options(-Wall -Wextra) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_compile_options(/Zi /MTd /MP /EHs /W4 /utf-8) + endif() +elseif(CMAKE_BUILD_TYPE STREQUAL "Release") + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + add_compile_options(/MT /Oi /MP /EHs /GR- /W0 /utf-8) + else() + add_compile_options(-fno-rtti -Wno-unused-function -Wno-unused-variable -Wno-unused-parameter) + if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "powerpc") + add_compile_options(-fstack-protector) + endif() + endif() + + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang") + add_compile_options(-Wno-c++98-compat) + endif() +endif() + +include(FetchContent) diff --git a/cmake/install.cmake b/cmake/install.cmake index 91c48b31a..090fd4c85 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -1,15 +1,19 @@ -set(DOC_FILES - ${CMAKE_SOURCE_DIR}/ChangeLog.md - ${CMAKE_SOURCE_DIR}/COPYING -) -set(SHARE_DIR ${CMAKE_INSTALL_PREFIX}/share/${PACKAGE}) -set(CONF_FILE ${CMAKE_SOURCE_DIR}/nzbget.conf) -set(WEBUI_DIR ${CMAKE_SOURCE_DIR}/webui) -set(DOC_FILES_DEST ${SHARE_DIR}/doc) -set(CONF_TEMPLATE_FILE_DEST ${SHARE_DIR}) -set(WEBUI_DIR_DEST ${SHARE_DIR}) +set(CONF_FILE nzbget.conf) +set(SHARE_DIR_DEST ${CMAKE_INSTALL_PREFIX}/share/${PACKAGE}) +set(DOC_FILES_SRC ${CMAKE_SOURCE_DIR}/ChangeLog.md ${CMAKE_SOURCE_DIR}/COPYING) +set(CONF_FILE_SRC ${CMAKE_SOURCE_DIR}/${CONF_FILE}) +set(WEBUI_DIR_SRC ${CMAKE_SOURCE_DIR}/webui) +set(DOC_FILES_DEST ${SHARE_DIR_DEST}/doc) +set(WEBUI_DIR_DEST ${SHARE_DIR_DEST}) +set(TEMPLATE_CONF_FILE_DEST ${SHARE_DIR_DEST}) +set(CONF_FILE_DEST ${CMAKE_INSTALL_PREFIX}/etc) set(BIN_FILE_DEST ${CMAKE_INSTALL_PREFIX}/bin) +file(READ ${CONF_FILE_SRC} CONFIG_CONTENT) +string(REPLACE "WebDir=" "WebDir=${WEBUI_DIR_DEST}/webui" MODIFIED_CONFIG_CONTENT "${CONFIG_CONTENT}") +string(REPLACE "ConfigTemplate=" "ConfigTemplate=${TEMPLATE_CONF_FILE_DEST}/${CONF_FILE}" MODIFIED_CONFIG_CONTENT "${MODIFIED_CONFIG_CONTENT}") +file(WRITE ${CMAKE_BINARY_DIR}/${CONF_FILE} ${MODIFIED_CONFIG_CONTENT}) + install(TARGETS ${PACKAGE} PERMISSIONS OWNER_EXECUTE OWNER_WRITE @@ -18,32 +22,26 @@ install(TARGETS ${PACKAGE} PERMISSIONS GROUP_EXECUTE WORLD_READ WORLD_EXECUTE - DESTINATION ${BIN_FILE_DEST}) -install(FILES ${DOC_FILES} DESTINATION ${DOC_FILES_DEST}) -install(DIRECTORY ${WEBUI_DIR} DESTINATION ${WEBUI_DIR_DEST}) - -file(READ ${CONF_FILE} CONFIG_CONTENT) -string(REPLACE "WebDir=" "WebDir=${WEBUI_DIR_DEST}/webui" MODIFIED_CONFIG_CONTENT "${CONFIG_CONTENT}") -string(REPLACE "ConfigTemplate=" "ConfigTemplate=${CONF_TEMPLATE_FILE_DEST}/nzbget.conf" MODIFIED_CONFIG_CONTENT "${MODIFIED_CONFIG_CONTENT}") -file(WRITE ${CMAKE_BINARY_DIR}/nzbget.conf "${MODIFIED_CONFIG_CONTENT}") + DESTINATION ${BIN_FILE_DEST} +) +install(DIRECTORY ${WEBUI_DIR_SRC} DESTINATION ${WEBUI_DIR_DEST}) +install(FILES ${DOC_FILES_SRC} DESTINATION ${DOC_FILES_DEST}) +install(FILES ${CMAKE_BINARY_DIR}/${CONF_FILE} DESTINATION ${TEMPLATE_CONF_FILE_DEST}) add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -E remove_directory ${DOC_FILES_DEST} - COMMAND ${CMAKE_COMMAND} -E remove_directory ${SHARE_DIR} + COMMAND ${CMAKE_COMMAND} -E remove_directory ${SHARE_DIR_DEST} COMMAND ${CMAKE_COMMAND} -E remove ${BIN_FILE_DEST}/${PACKAGE} COMMENT "Uninstalling" ${PACKAGE} ) add_custom_target(install-conf - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/etc - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/nzbget.conf ${CMAKE_INSTALL_PREFIX}/etc/nzbget.conf - COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/nzbget.conf ${CONF_TEMPLATE_FILE_DEST}/nzbget.conf + COMMAND ${CMAKE_COMMAND} -E make_directory ${CONF_FILE_DEST} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_BINARY_DIR}/${CONF_FILE} ${CONF_FILE_DEST}/${CONF_FILE} COMMENT "Installing nzbget.conf" ) add_custom_target(uninstall-conf - COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/etc - COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_INSTALL_PREFIX}/etc/nzbget.conf - COMMAND ${CMAKE_COMMAND} -E remove ${CONF_TEMPLATE_FILE_DEST}/nzbget.conf + COMMAND ${CMAKE_COMMAND} -E remove ${CONF_FILE_DEST}/${CONF_FILE} COMMENT "Unstalling nzbget.conf" ) diff --git a/cmake/par2-turbo.cmake b/cmake/par2-turbo.cmake new file mode 100644 index 000000000..8b92c122e --- /dev/null +++ b/cmake/par2-turbo.cmake @@ -0,0 +1,17 @@ +set(FETCHCONTENT_QUIET FALSE) +FetchContent_Declare( + par2-turbo + GIT_REPOSITORY https://github.com/nzbgetcom/par2cmdline-turbo.git + GIT_TAG v1.1.1-nzbget + TLS_VERIFY TRUE + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE +) + +add_compile_definitions(HAVE_CONFIG_H PARPAR_ENABLE_HASHER_MD5CRC) +set(BUILD_TOOL OFF CACHE BOOL "") +set(BUILD_LIB ON CACHE BOOL "") +FetchContent_MakeAvailable(par2-turbo) + +set(LIBS ${LIBS} par2-turbo gf16 hasher) +set(INCLUDES ${INCLUDES} ${par2_SOURCE_DIR}/include) diff --git a/cmake/posix.cmake b/cmake/posix.cmake index cac7665fe..bc69e70c6 100644 --- a/cmake/posix.cmake +++ b/cmake/posix.cmake @@ -31,6 +31,29 @@ message(STATUS " DISABLE CURSES: ${DISABLE_CURSES}") message(STATUS " DISABLE GZIP: ${DISABLE_GZIP}") message(STATUS " DISABLE PARCHECK: ${DISABLE_PARCHECK}") +if(APPLE) + # On macOS Cmake, when cross-compiling, sometimes CMAKE_SYSTEM_PROCESSOR wrongfully stays + # the same as CMAKE_HOST_SYSTEM_PROCESSOR regardless the target CPU. + # The manual call to set(CMAKE_SYSTEM_PROCESSOR) has to be set after the project() call. + # because project() might reset CMAKE_SYSTEM_PROCESSOR back to the value of CMAKE_HOST_SYSTEM_PROCESSOR. + # Check if CMAKE_SYSTEM_PROCESSOR is not equal to CMAKE_OSX_ARCHITECTURES + if(NOT CMAKE_OSX_ARCHITECTURES STREQUAL "") + if(NOT CMAKE_SYSTEM_PROCESSOR STREQUAL CMAKE_OSX_ARCHITECTURES) + # Split CMAKE_OSX_ARCHITECTURES into a list + string(REPLACE ";" " " ARCH_LIST ${CMAKE_OSX_ARCHITECTURES}) + separate_arguments(ARCH_LIST UNIX_COMMAND ${ARCH_LIST}) + # Count the number of architectures + list(LENGTH ARCH_LIST ARCH_COUNT) + # Ensure that exactly one architecture is specified + if(NOT ARCH_COUNT EQUAL 1) + message(FATAL_ERROR "CMAKE_OSX_ARCHITECTURES must have exactly one value. Current value: ${CMAKE_OSX_ARCHITECTURES}") + endif() + set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_OSX_ARCHITECTURES}) + message(STATUS "CMAKE_SYSTEM_PROCESSOR is manually set to ${CMAKE_SYSTEM_PROCESSOR}") + endif() + endif() +endif() + if(ENABLE_CLANG_TIDY) set(CMAKE_CXX_CLANG_TIDY clang-tidy -checks=-*,readability-*) endif() @@ -41,10 +64,13 @@ if(ENABLE_STATIC) set(CMAKE_EXE_LINKER_FLAGS_DEBUG "-static" CACHE STRING "" FORCE) set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} -static" CACHE STRING "" FORCE) endif() + set(BUILD_SHARED_LIBS OFF) - set(LIBS $ENV{LIBS}) - set(INCLUDES $ENV{INCLUDES}) - include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) + set(LIBS ${LIBS} $ENV{LIBS}) + set(INCLUDES ${INCLUDES} $ENV{INCLUDES}) + + # for the sub-projects + include_directories($ENV{INCLUDES}) else() find_package(Threads REQUIRED) find_package(LibXml2 REQUIRED) @@ -67,6 +93,7 @@ else() if(NOT DISABLE_CURSES) set(CURSES_NEED_NCURSES TRUE) + set(CURSES_NEED_WIDE TRUE) find_package(Curses REQUIRED) set(INCLUDES ${INCLUDES} ${CURSES_INCLUDE_DIRS}) set(LIBS ${LIBS} ${CURSES_LIBRARIES}) @@ -82,24 +109,22 @@ else() if(NOT Boost_JSON_FOUND) message(STATUS "The Boost library will be installed from github") - include(ExternalProject) include(${CMAKE_SOURCE_DIR}/cmake/boost.cmake) - include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) - add_dependencies(${PACKAGE} boost) add_dependencies(yencode boost) - add_dependencies(par2 boost) add_dependencies(regex boost) else() set(LIBS ${LIBS} Boost::json) set(INCLUDES ${INCLUDES} ${Boost_INCLUDE_DIR}) - - include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) endif() endif() +if(NOT DISABLE_PARCHECK) + include(${CMAKE_SOURCE_DIR}/cmake/par2-turbo.cmake) +endif() + include(CheckIncludeFiles) include(CheckLibraryExists) include(CheckSymbolExists) diff --git a/cmake/toolchain.cmake b/cmake/toolchain.cmake index b20989268..62790d6e4 100644 --- a/cmake/toolchain.cmake +++ b/cmake/toolchain.cmake @@ -1,29 +1,44 @@ -set(CMAKE_SYSTEM_NAME Linux) +if(DEFINED ENV{CMAKE_SYSTEM_NAME}) + if (NOT CMAKE_SYSTEM_NAME) + set(CMAKE_SYSTEM_NAME $ENV{CMAKE_SYSTEM_NAME}) + endif() +else() + if (CMAKE_SYSTEM_NAME) + set(ENV{CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_NAME}) + else() + set(CMAKE_SYSTEM_NAME Linux) + endif() +endif() set(CMAKE_SYSTEM_PROCESSOR ${ARCH}) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) -if(NOT COMPILER) - set(COMPILER "gcc" CACHE STRING "") -endif() - -if("${COMPILER}" STREQUAL "gcc") - set(C_COMPILER "gcc") - set(CXX_COMPILER "g++") +if("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") + set(CMAKE_C_COMPILER clang) + set(CMAKE_CXX_COMPILER clang++) + SET(CMAKE_CXX_COMPILER_TARGET x86_64-pc-freebsd) else() - set(C_COMPILER ${COMPILER}) - set(CXX_COMPILER ${COMPILER}++) -endif() + if(NOT COMPILER) + set(COMPILER "gcc" CACHE STRING "") + endif() -set(CMAKE_AR ${TOOLCHAIN_PREFIX}-ar) -set(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}-as) -set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-${C_COMPILER}) -set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-${CXX_COMPILER}) -set(CMAKE_LINKER ${TOOLCHAIN_PREFIX}-ld) -set(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}-objcopy) -set(CMAKE_RANLIB ${TOOLCHAIN_PREFIX}-ranlib) -set(CMAKE_SIZE ${TOOLCHAIN_PREFIX}-size) -set(CMAKE_STRIP ${TOOLCHAIN_PREFIX}-strip) + if("${COMPILER}" STREQUAL "gcc") + set(C_COMPILER "gcc") + set(CXX_COMPILER "g++") + else() + set(C_COMPILER ${COMPILER}) + set(CXX_COMPILER ${COMPILER}++) + endif() + set(CMAKE_AR ${TOOLCHAIN_PREFIX}-ar) + set(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}-as) + set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-${C_COMPILER}) + set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-${CXX_COMPILER}) + set(CMAKE_LINKER ${TOOLCHAIN_PREFIX}-ld) + set(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}-objcopy) + set(CMAKE_RANLIB ${TOOLCHAIN_PREFIX}-ranlib) + set(CMAKE_SIZE ${TOOLCHAIN_PREFIX}-size) + set(CMAKE_STRIP ${TOOLCHAIN_PREFIX}-strip) +endif() set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) diff --git a/cmake/windows.cmake b/cmake/windows.cmake index bfc265bd7..d833cd01e 100644 --- a/cmake/windows.cmake +++ b/cmake/windows.cmake @@ -16,8 +16,10 @@ find_package(Threads REQUIRED) find_package(LibXml2 REQUIRED) find_package(Boost REQUIRED COMPONENTS json) -set(LIBS Threads::Threads Boost::json LibXml2::LibXml2) -set(INCLUDES ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR}) +include(${CMAKE_SOURCE_DIR}/cmake/par2-turbo.cmake) + +set(LIBS ${LIBS} Threads::Threads Boost::json LibXml2::LibXml2 winmm.lib) +set(INCLUDES ${INCLUDES} ${Boost_INCLUDE_DIR} ${LIBXML2_INCLUDE_DIR}) if(NOT DISABLE_TLS) find_package(OpenSSL REQUIRED) @@ -30,8 +32,6 @@ endif() find_package(ZLIB REQUIRED) set(LIBS ${LIBS} ZLIB::ZLIB) set(INCLUDES ${INCLUDES} ${ZLIB_INCLUDE_DIRS}) - -include(${CMAKE_SOURCE_DIR}/lib/sources.cmake) set(INCLUDES ${INCLUDES} ${CMAKE_SOURCE_DIR}/daemon/windows ${CMAKE_SOURCE_DIR}/windows/resources diff --git a/daemon/connect/Connection.cpp b/daemon/connect/Connection.cpp index 86e5bae49..342840f55 100644 --- a/daemon/connect/Connection.cpp +++ b/daemon/connect/Connection.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2019 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -715,7 +716,7 @@ bool Connection::DoConnect() } #ifndef DISABLE_TLS - if (m_tls && !StartTls(true, nullptr, nullptr)) + if (m_tls && !StartTls(true, "", "")) { return false; } diff --git a/daemon/connect/TlsSocket.cpp b/daemon/connect/TlsSocket.cpp index f23adf9ae..4c0856e83 100644 --- a/daemon/connect/TlsSocket.cpp +++ b/daemon/connect/TlsSocket.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2008-2017 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -29,7 +30,11 @@ #include "FileSystem.h" #include "Options.h" -CString TlsSocket::m_certStore; +std::string TlsSocket::m_certStore; + +#ifdef HAVE_OPENSSL +X509_STORE* TlsSocket::m_X509Store = nullptr; +#endif #ifdef HAVE_LIBGNUTLS #ifdef NEED_GCRYPT_LOCKING @@ -182,6 +187,45 @@ void TlsSocket::Init() #endif /* HAVE_OPENSSL */ } +void TlsSocket::InitOptions(const char* certStore) +{ + m_certStore = certStore; + +#ifdef HAVE_OPENSSL + InitX509Store(m_certStore); +#endif +} + +#ifdef HAVE_OPENSSL +void TlsSocket::InitX509Store(const std::string& certStore) +{ + if (m_X509Store) + { + FreeX509Store(m_X509Store); + } + + m_X509Store = X509_STORE_new(); + if (!m_X509Store) + { + error("Could not create certificate store"); + return; + } + + if (!X509_STORE_load_locations(m_X509Store, certStore.c_str(), nullptr)) + { + FreeX509Store(m_X509Store); + error("Could not load certificate store location"); + return; + } +} + +void TlsSocket::FreeX509Store(X509_STORE* store) +{ + X509_STORE_free(m_X509Store); + m_X509Store = nullptr; +} +#endif + void TlsSocket::Final() { #ifdef HAVE_LIBGNUTLS @@ -189,6 +233,8 @@ void TlsSocket::Final() #endif /* HAVE_LIBGNUTLS */ #ifdef HAVE_OPENSSL + X509_STORE_free(m_X509Store); + #ifndef LIBRESSL_VERSION_NUMBER #if OPENSSL_VERSION_NUMBER < 0x30000000L FIPS_mode_set(0); @@ -287,10 +333,10 @@ bool TlsSocket::Start() m_context = cred; - if (m_certFile && m_keyFile) + if (!m_certFile.empty() && !m_keyFile.empty()) { m_retCode = gnutls_certificate_set_x509_key_file((gnutls_certificate_credentials_t)m_context, - m_certFile, m_keyFile, GNUTLS_X509_FMT_PEM); + m_certFile.c_str(), m_keyFile.c_str(), GNUTLS_X509_FMT_PEM); if (m_retCode != 0) { ReportError("Could not load certificate or key file", false); @@ -312,8 +358,8 @@ bool TlsSocket::Start() m_initialized = true; - const char* priority = !m_cipher.Empty() ? m_cipher.Str() : - (m_certFile && m_keyFile ? "NORMAL:!VERS-SSL3.0" : "NORMAL"); + const char* priority = !m_cipher.empty() ? m_cipher.c_str() : + (!m_certFile.empty() && !m_keyFile.empty() ? "NORMAL:!VERS-SSL3.0" : "NORMAL"); m_retCode = gnutls_priority_set_direct((gnutls_session_t)m_session, priority, nullptr); if (m_retCode != 0) @@ -323,9 +369,9 @@ bool TlsSocket::Start() return false; } - if (m_host) + if (!m_host.empty()) { - m_retCode = gnutls_server_name_set((gnutls_session_t)m_session, GNUTLS_NAME_DNS, m_host, m_host.Length()); + m_retCode = gnutls_server_name_set((gnutls_session_t)m_session, GNUTLS_NAME_DNS, m_host.c_str(), m_host.size()); if (m_retCode != 0) { ReportError("Could not set hostname for TLS"); @@ -348,12 +394,12 @@ bool TlsSocket::Start() m_retCode = gnutls_handshake((gnutls_session_t)m_session); if (m_retCode != 0) { - ReportError(BString<1024>("TLS handshake failed for %s", *m_host)); + ReportError(BString<1024>("TLS handshake failed for %s", m_host.c_str())); Close(); return false; } - if (m_isClient && !m_certStore.Empty() && !ValidateCert()) + if (m_isClient && !m_certStore.empty() && !ValidateCert()) { Close(); return false; @@ -372,15 +418,15 @@ bool TlsSocket::Start() return false; } - if (m_certFile && m_keyFile) + if (!m_certFile.empty() && !m_keyFile.empty()) { - if (SSL_CTX_use_certificate_chain_file((SSL_CTX*)m_context, m_certFile) != 1) + if (SSL_CTX_use_certificate_chain_file((SSL_CTX*)m_context, m_certFile.c_str()) != 1) { ReportError("Could not load certificate file", false); Close(); return false; } - if (SSL_CTX_use_PrivateKey_file((SSL_CTX*)m_context, m_keyFile, SSL_FILETYPE_PEM) != 1) + if (SSL_CTX_use_PrivateKey_file((SSL_CTX*)m_context, m_keyFile.c_str(), SSL_FILETYPE_PEM) != 1) { ReportError("Could not load key file", false); Close(); @@ -411,15 +457,17 @@ bool TlsSocket::Start() EC_KEY_free(ecdh); } - if (m_isClient && !m_certStore.Empty()) + if (m_isClient && !m_certStore.empty()) { - // Enable certificate validation - if (SSL_CTX_load_verify_locations((SSL_CTX*)m_context, m_certStore, nullptr) != 1) + if (!m_X509Store) { - ReportError("Could not set certificate store location", false); + error("Could not set certificate store location. Make sure the CertPath option is correct."); Close(); return false; } + + SSL_CTX_set1_cert_store((SSL_CTX*)m_context, m_X509Store); + if (m_certVerifLevel > Options::ECertVerifLevel::cvNone) { SSL_CTX_set_verify((SSL_CTX*)m_context, SSL_VERIFY_PEER, nullptr); @@ -438,14 +486,14 @@ bool TlsSocket::Start() return false; } - if (!m_cipher.Empty() && !SSL_set_cipher_list((SSL*)m_session, m_cipher)) + if (!m_cipher.empty() && !SSL_set_cipher_list((SSL*)m_session, m_cipher.c_str())) { ReportError("Could not select cipher for TLS", false); Close(); return false; } - if (m_isClient && m_host && !SSL_set_tlsext_host_name((SSL*)m_session, m_host)) + if (m_isClient && !m_host.empty() && !SSL_set_tlsext_host_name((SSL*)m_session, m_host.c_str())) { ReportError("Could not set host name for TLS"); Close(); @@ -467,17 +515,17 @@ bool TlsSocket::Start() { PrintError(BString<1024>("TLS certificate verification failed for %s: %s." " For more info visit https://nzbget.com/documentation/certificate-verification/", - *m_host, X509_verify_cert_error_string(verifyRes))); + m_host.c_str(), X509_verify_cert_error_string(verifyRes))); } else { - ReportError(BString<1024>("TLS handshake failed for %s", *m_host)); + ReportError(BString<1024>("TLS handshake failed for %s", m_host.c_str())); } Close(); return false; } - if (m_isClient && !m_certStore.Empty() && !ValidateCert()) + if (m_isClient && !m_certStore.empty() && !ValidateCert()) { Close(); return false; @@ -493,9 +541,9 @@ bool TlsSocket::ValidateCert() #ifdef HAVE_LIBGNUTLS #if GNUTLS_VERSION_NUMBER >= 0x030104 #if GNUTLS_VERSION_NUMBER >= 0x030306 - if (FileSystem::DirectoryExists(m_certStore)) + if (FileSystem::DirectoryExists(m_certStore.c_str())) { - if (gnutls_certificate_set_x509_trust_dir((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0) + if (gnutls_certificate_set_x509_trust_dir((gnutls_certificate_credentials_t)m_context, m_certStore.c_str(), GNUTLS_X509_FMT_PEM) < 0) { ReportError("Could not set certificate store location"); return false; @@ -504,7 +552,7 @@ bool TlsSocket::ValidateCert() else #endif { - if (gnutls_certificate_set_x509_trust_file((gnutls_certificate_credentials_t)m_context, m_certStore, GNUTLS_X509_FMT_PEM) < 0) + if (gnutls_certificate_set_x509_trust_file((gnutls_certificate_credentials_t)m_context, m_certStore.c_str(), GNUTLS_X509_FMT_PEM) < 0) { ReportError("Could not set certificate store location"); return false; @@ -512,7 +560,7 @@ bool TlsSocket::ValidateCert() } unsigned int status = 0; - if (gnutls_certificate_verify_peers3((gnutls_session_t)m_session, m_host, &status) != 0 || + if (gnutls_certificate_verify_peers3((gnutls_session_t)m_session, m_host.c_str(), &status) != 0 || gnutls_certificate_type_get((gnutls_session_t)m_session) != GNUTLS_CRT_X509) { ReportError("Could not verify TLS certificate"); @@ -536,7 +584,7 @@ bool TlsSocket::ValidateCert() if (gnutls_x509_crt_get_dn_by_oid(cert, GNUTLS_OID_X520_COMMON_NAME, 0, 0, dn, &size) == 0) { PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)." - " For more info visit https://nzbget.com/documentation/certificate-verification/", *m_host, dn)); + " For more info visit https://nzbget.com/documentation/certificate-verification/", m_host.c_str(), dn)); gnutls_x509_crt_deinit(cert); return false; } @@ -548,13 +596,13 @@ bool TlsSocket::ValidateCert() if (gnutls_certificate_verification_status_print(status, GNUTLS_CRT_X509, &msgdata, 0) == 0) { PrintError(BString<1024>("TLS certificate verification failed for %s: %s." - " For more info visit https://nzbget.com/documentation/certificate-verification/", *m_host, msgdata.data)); + " For more info visit https://nzbget.com/documentation/certificate-verification/", m_host.c_str(), msgdata.data)); gnutls_free(&msgdata); } else { ReportError(BString<1024>("TLS certificate verification failed for %s." - " For more info visit https://nzbget.com/documentation/certificate-verification/", *m_host)); + " For more info visit https://nzbget.com/documentation/certificate-verification/", m_host.c_str())); } return false; } @@ -568,13 +616,13 @@ bool TlsSocket::ValidateCert() if (!cert) { PrintError(BString<1024>("TLS certificate verification failed for %s: no certificate provided by server." - " For more info visit https://nzbget.com/documentation/certificate-verification/", *m_host)); + " For more info visit https://nzbget.com/documentation/certificate-verification/", m_host.c_str())); return false; } #ifdef HAVE_X509_CHECK_HOST // hostname verification - if (m_certVerifLevel > Options::ECertVerifLevel::cvMinimal && !m_host.Empty() && X509_check_host(cert, m_host, m_host.Length(), 0, nullptr) != 1) + if (m_certVerifLevel > Options::ECertVerifLevel::cvMinimal && !m_host.empty() && X509_check_host(cert, m_host.c_str(), m_host.size(), 0, nullptr) != 1) { const unsigned char* certHost = nullptr; // Find the position of the CN field in the Subject field of the certificate @@ -599,7 +647,7 @@ bool TlsSocket::ValidateCert() } PrintError(BString<1024>("TLS certificate verification failed for %s: certificate hostname mismatch (%s)." - " For more info visit https://nzbget.com/documentation/certificate-verification/", *m_host, certHost)); + " For more info visit https://nzbget.com/documentation/certificate-verification/", m_host.c_str(), certHost)); X509_free(cert); return false; } diff --git a/daemon/connect/TlsSocket.h b/daemon/connect/TlsSocket.h index 5a1c30d25..cf690aa9a 100644 --- a/daemon/connect/TlsSocket.h +++ b/daemon/connect/TlsSocket.h @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2008-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ #ifndef TLSSOCKET_H @@ -22,19 +23,31 @@ #ifndef DISABLE_TLS -#include "NString.h" - class TlsSocket { public: - TlsSocket(SOCKET socket, bool isClient, const char* host, - const char* certFile, const char* keyFile, const char* cipher, int certVerifLevel) : - m_socket(socket), m_isClient(isClient), m_host(host), - m_certFile(certFile), m_keyFile(keyFile), m_cipher(cipher), m_certVerifLevel(certVerifLevel) {} + TlsSocket( + SOCKET socket, + bool isClient, + const char* host, + const char* certFile, + const char* keyFile, + const char* cipher, + int certVerifLevel) + : m_socket(socket) + , m_isClient(isClient) + , m_host(host ? host : "") + , m_certFile(certFile ? certFile : "") + , m_keyFile(keyFile ? keyFile : "") + , m_cipher(cipher ? cipher : "") + , m_certVerifLevel(certVerifLevel) {} + virtual ~TlsSocket(); + static void Init(); - static void InitOptions(const char* certStore) { m_certStore = certStore; } + static void InitOptions(const char* certStore); static void Final(); + bool Start(); void Close(); int Send(const char* buffer, int size); @@ -45,17 +58,27 @@ class TlsSocket virtual void PrintError(const char* errMsg); private: +#ifdef HAVE_OPENSSL + static void InitX509Store(const std::string& certStore); + static void FreeX509Store(X509_STORE* store); + + static X509_STORE* m_X509Store; +#endif + static std::string m_certStore; + SOCKET m_socket; + + std::string m_host; + std::string m_certFile; + std::string m_keyFile; + std::string m_cipher; + bool m_isClient; - CString m_host; - CString m_certFile; - CString m_keyFile; - CString m_cipher; bool m_suppressErrors = false; bool m_initialized = false; bool m_connected = false; + int m_retCode; - static CString m_certStore; int m_certVerifLevel; // using "void*" to prevent the including of GnuTLS/OpenSSL header files into TlsSocket.h diff --git a/daemon/frontend/NCursesFrontend.cpp b/daemon/frontend/NCursesFrontend.cpp index 6dd4fb5db..8668a0a60 100644 --- a/daemon/frontend/NCursesFrontend.cpp +++ b/daemon/frontend/NCursesFrontend.cpp @@ -53,22 +53,25 @@ void curses_clear() extern void ExitProc(); -static const int NCURSES_COLORPAIR_TEXT = 1; -static const int NCURSES_COLORPAIR_INFO = 2; -static const int NCURSES_COLORPAIR_WARNING = 3; -static const int NCURSES_COLORPAIR_ERROR = 4; -static const int NCURSES_COLORPAIR_DEBUG = 5; -static const int NCURSES_COLORPAIR_DETAIL = 6; -static const int NCURSES_COLORPAIR_STATUS = 7; -static const int NCURSES_COLORPAIR_KEYBAR = 8; -static const int NCURSES_COLORPAIR_INFOLINE = 9; -static const int NCURSES_COLORPAIR_TEXTHIGHL = 10; -static const int NCURSES_COLORPAIR_CURSOR = 11; -static const int NCURSES_COLORPAIR_HINT = 12; - -static const int MAX_SCREEN_WIDTH = 512; +const int NCURSES_COLORPAIR_TEXT = 1; +const int NCURSES_COLORPAIR_INFO = 2; +const int NCURSES_COLORPAIR_WARNING = 3; +const int NCURSES_COLORPAIR_ERROR = 4; +const int NCURSES_COLORPAIR_DEBUG = 5; +const int NCURSES_COLORPAIR_DETAIL = 6; +const int NCURSES_COLORPAIR_STATUS = 7; +const int NCURSES_COLORPAIR_KEYBAR = 8; +const int NCURSES_COLORPAIR_INFOLINE = 9; +const int NCURSES_COLORPAIR_TEXTHIGHL = 10; +const int NCURSES_COLORPAIR_CURSOR = 11; +const int NCURSES_COLORPAIR_HINT = 12; + +const int MAX_SCREEN_WIDTH = 512; #ifdef WIN32 + +#include "Utf8.h" + static const int COLOR_BLACK = 0; static const int COLOR_BLUE = FOREGROUND_BLUE; static const int COLOR_RED = FOREGROUND_RED; @@ -350,31 +353,23 @@ int NCursesFrontend::CalcQueueSize() return queueSize; } +#ifndef WIN32 + void NCursesFrontend::PlotLine(const char * string, int row, int pos, int colorPair) { - BString<1024> buffer("%-*s", m_screenWidth, string); - int len = buffer.Length(); - if (len > m_screenWidth - pos && m_screenWidth - pos < MAX_SCREEN_WIDTH) + std::string buffer(m_screenWidth + 1, '\0'); + snprintf(buffer.data(), buffer.size(), "%-*s", m_screenWidth, string); + + if (Util::CmpGreater(buffer.size(), m_screenWidth - pos) && m_screenWidth - pos < MAX_SCREEN_WIDTH) { buffer[m_screenWidth - pos] = '\0'; } - PlotText(buffer, row, pos, colorPair, false); + PlotText(buffer.data(), row, pos, colorPair, false); } void NCursesFrontend::PlotText(const char * string, int row, int pos, int colorPair, bool blink) { -#ifdef WIN32 - int bufPos = row * m_screenWidth + pos; - int len = strlen(string); - for (int i = 0; i < len; i++) - { - char c = string[i]; - CharToOemBuff(&c, &c, 1); - m_screenBuffer[bufPos + i].Char.AsciiChar = c; - m_screenBuffer[bufPos + i].Attributes = m_colorAttr[colorPair]; - } -#else if( m_useColor ) { attron(COLOR_PAIR(colorPair)); @@ -392,16 +387,51 @@ void NCursesFrontend::PlotText(const char * string, int row, int pos, int colorP attroff(A_BLINK); } } -#endif } +#else + +void NCursesFrontend::PlotLine(const char * str, int row, int pos, int colorPair) +{ + auto res = Utf8::Utf8ToWide(str); + if (!res.has_value()) + { + warn("Failed to convert %s to wide string", str); + return; + } + + std::wstring wstr = std::move(res.value()); + std::wstring buffer(m_screenWidth + 1, '\0'); + swprintf(buffer.data(), buffer.size(), L"%-*s", m_screenWidth, wstr.c_str()); + + if (Util::CmpGreater(buffer.size(), m_screenWidth - pos) && m_screenWidth - pos < MAX_SCREEN_WIDTH) + { + buffer[m_screenWidth - pos] = '\0'; + } + + PlotText(buffer.data(), row, pos, colorPair, false); +} + +void NCursesFrontend::PlotText(const wchar_t* wstr, int row, int pos, int colorPair, bool blink) +{ + int bufPos = row * m_screenWidth + pos; + size_t len = wcslen(wstr); + for (size_t i = 0; i < len; ++i) + { + m_screenBuffer[bufPos + i].Char.UnicodeChar = wstr[i]; + m_screenBuffer[bufPos + i].Attributes = m_colorAttr[colorPair]; + } +} + +#endif + void NCursesFrontend::RefreshScreen() { #ifdef WIN32 bool bufChanged = !std::equal(m_screenBuffer.begin(), m_screenBuffer.end(), m_oldScreenBuffer.begin(), m_oldScreenBuffer.end(), [](CHAR_INFO& a, CHAR_INFO& b) { - return a.Char.AsciiChar == b.Char.AsciiChar && a.Attributes == b.Attributes; + return a.Char.UnicodeChar == b.Char.UnicodeChar && a.Attributes == b.Attributes; }); if (bufChanged) @@ -417,7 +447,7 @@ void NCursesFrontend::RefreshScreen() HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO BufInfo; GetConsoleScreenBufferInfo(hConsole, &BufInfo); - WriteConsoleOutput(hConsole, m_screenBuffer.data(), BufSize, BufCoord, &BufInfo.srWindow); + WriteConsoleOutputW(hConsole, m_screenBuffer.data(), BufSize, BufCoord, &BufInfo.srWindow); BufInfo.dwCursorPosition.X = BufInfo.srWindow.Right; BufInfo.dwCursorPosition.Y = BufInfo.srWindow.Bottom; @@ -446,8 +476,8 @@ void NCursesFrontend::PrintMessages() { int lineNr = m_messagesWinTop; - BString<1024> buffer("%s Messages", m_useColor ? "" : "*** "); - PlotLine(buffer, lineNr++, 0, NCURSES_COLORPAIR_INFOLINE); + std::string buffer = (m_useColor ? "" : "*** ") + std::string(" Messages"); + PlotLine(buffer.c_str(), lineNr++, 0, NCURSES_COLORPAIR_INFOLINE); int line = lineNr + m_messagesWinClientHeight - 1; int linesToPrint = m_messagesWinClientHeight; @@ -482,10 +512,6 @@ void NCursesFrontend::PrintMessages() int NCursesFrontend::PrintMessage(Message& msg, int row, int maxLines) { - const char* messageType[] = { "INFO ", "WARNING ", "ERROR ", "DEBUG ", "DETAIL "}; - const int messageTypeColor[] = { NCURSES_COLORPAIR_INFO, NCURSES_COLORPAIR_WARNING, - NCURSES_COLORPAIR_ERROR, NCURSES_COLORPAIR_DEBUG, NCURSES_COLORPAIR_DETAIL }; - CString text; if (m_showTimestamp) @@ -522,11 +548,15 @@ int NCursesFrontend::PrintMessage(Message& msg, int row, int maxLines) PlotLine(text + winWidth * i, r, 8, NCURSES_COLORPAIR_TEXT); if (i == 0) { - PlotText(messageType[msg.GetKind()], r, 0, messageTypeColor[msg.GetKind()], false); + PlotText(m_messageTypes[msg.GetKind()], r, 0, m_messageColorTypes[msg.GetKind()], false); } else { - PlotText(" ", r, 0, messageTypeColor[msg.GetKind()], false); +#ifdef WIN32 + PlotText(L" ", r, 0, m_messageColorTypes[msg.GetKind()], false); +#else + PlotText(" ", r, 0, m_messageColorTypes[msg.GetKind()], false); +#endif } lines++; } @@ -577,12 +607,12 @@ void NCursesFrontend::PrintKeyInputBar() int queueSize = CalcQueueSize(); int inputBarRow = m_screenHeight - 1; - if (!m_hint.Empty()) + if (!m_hint.empty()) { time_t time = Util::CurrentTime(); if (time - m_startHint < 5) { - PlotLine(m_hint, inputBarRow, 0, NCURSES_COLORPAIR_HINT); + PlotLine(m_hint.c_str(), inputBarRow, 0, NCURSES_COLORPAIR_HINT); return; } else @@ -627,21 +657,24 @@ void NCursesFrontend::PrintKeyInputBar() break; } case downloadRate: - BString<100> hint("Download rate: %i", m_inputValue); - PlotLine(hint, inputBarRow, 0, NCURSES_COLORPAIR_KEYBAR); + std::string hint = "Download rate: " + std::to_string(m_inputValue); + PlotLine(hint.c_str(), inputBarRow, 0, NCURSES_COLORPAIR_KEYBAR); + // Print the cursor +#ifdef WIN32 + PlotText(L" ", inputBarRow, 15 + m_inputNumberIndex, NCURSES_COLORPAIR_CURSOR, true); +#else + PlotText(" ", inputBarRow, 15 + m_inputNumberIndex, NCURSES_COLORPAIR_CURSOR, true); +#endif break; } } void NCursesFrontend::SetHint(const char* hint) { - m_hint = hint; - if (!m_hint.Empty()) - { - m_startHint = Util::CurrentTime(); - } + if (hint) m_hint = hint; + if (!m_hint.empty()) m_startHint = Util::CurrentTime(); } void NCursesFrontend::PrintQueue() diff --git a/daemon/frontend/NCursesFrontend.h b/daemon/frontend/NCursesFrontend.h index 67eb0b730..c283e1050 100644 --- a/daemon/frontend/NCursesFrontend.h +++ b/daemon/frontend/NCursesFrontend.h @@ -24,11 +24,26 @@ #ifndef DISABLE_CURSES -#include "NString.h" +#include #include "Frontend.h" #include "Log.h" #include "DownloadInfo.h" +extern const int NCURSES_COLORPAIR_TEXT; +extern const int NCURSES_COLORPAIR_INFO; +extern const int NCURSES_COLORPAIR_WARNING; +extern const int NCURSES_COLORPAIR_ERROR; +extern const int NCURSES_COLORPAIR_DEBUG; +extern const int NCURSES_COLORPAIR_DETAIL; +extern const int NCURSES_COLORPAIR_STATUS; +extern const int NCURSES_COLORPAIR_KEYBAR; +extern const int NCURSES_COLORPAIR_INFOLINE; +extern const int NCURSES_COLORPAIR_TEXTHIGHL; +extern const int NCURSES_COLORPAIR_CURSOR; +extern const int NCURSES_COLORPAIR_HINT; + +extern const int MAX_SCREEN_WIDTH; + class NCursesFrontend : public Frontend { public: @@ -61,7 +76,7 @@ class NCursesFrontend : public Frontend int m_lastEditEntry = -1; bool m_lastPausePars = false; int m_queueScrollOffset = 0; - CString m_hint; + std::string m_hint; time_t m_startHint; int m_colWidthFiles; int m_colWidthTotal; @@ -70,6 +85,30 @@ class NCursesFrontend : public Frontend // Inputting numbers int m_inputNumberIndex = 0; int m_inputValue; +#ifdef WIN32 + const wchar_t* m_messageTypes[5] = { + L"INFO ", + L"WARNING ", + L"ERROR ", + L"DEBUG ", + L"DETAIL " + }; +#else + const char* m_messageTypes[5] = { + "INFO ", + "WARNING ", + "ERROR ", + "DEBUG ", + "DETAIL " + }; +#endif + const int m_messageColorTypes[5] = { + NCURSES_COLORPAIR_INFO, + NCURSES_COLORPAIR_WARNING, + NCURSES_COLORPAIR_ERROR, + NCURSES_COLORPAIR_DEBUG, + NCURSES_COLORPAIR_DETAIL + }; #ifdef WIN32 std::vector m_screenBuffer; @@ -87,9 +126,11 @@ class NCursesFrontend : public Frontend #ifdef WIN32 void init_pair(int colorNumber, WORD wForeColor, WORD wBackColor); + void PlotText(const wchar_t* string, int row, int pos, int colorPair, bool blink); +#else + void PlotText(const char* string, int row, int pos, int colorPair, bool blink); #endif - void PlotLine(const char * string, int row, int pos, int colorPair); - void PlotText(const char * string, int row, int pos, int colorPair, bool blink); + void PlotLine(const char* string, int row, int pos, int colorPair); void PrintMessages(); void PrintQueue(); void PrintFileQueue(); diff --git a/daemon/main/Options.cpp b/daemon/main/Options.cpp index eb8902c5f..20c2e9934 100644 --- a/daemon/main/Options.cpp +++ b/daemon/main/Options.cpp @@ -482,7 +482,7 @@ void Options::InitDefaults() SetOption(OPTION_FILENAMING, "article"); SetOption(OPTION_PARRENAME, "yes"); SetOption(OPTION_PARBUFFER, "16"); - SetOption(OPTION_PARTHREADS, "1"); + SetOption(OPTION_PARTHREADS, "0"); SetOption(OPTION_RARRENAME, "yes"); SetOption(OPTION_HEALTHCHECK, "none"); SetOption(OPTION_DIRECTRENAME, "no"); diff --git a/daemon/main/nzbget.cpp b/daemon/main/nzbget.cpp index 04ad097f3..367921481 100644 --- a/daemon/main/nzbget.cpp +++ b/daemon/main/nzbget.cpp @@ -62,6 +62,7 @@ #include "WinService.h" #include "WinConsole.h" #include "WebDownloader.h" +#include "Utf8.h" #endif #ifndef DISABLE_NSERV #include "NServMain.h" @@ -72,6 +73,8 @@ #include #endif +#include + // Prototypes void RunMain(); @@ -106,7 +109,6 @@ int g_ArgumentCount; char* (*g_EnvironmentVariables)[] = nullptr; char* (*g_Arguments)[] = nullptr; - /* * Main entry point */ @@ -122,8 +124,12 @@ int main(int argc, char *argv[], char *argp[]) #endif ); #endif + + SetConsoleOutputCP(CP_UTF8); #endif + setlocale(LC_CTYPE, ""); + Util::Init(); YEncode::init(); @@ -303,7 +309,7 @@ void NZBGet::Init() m_scanner->InitOptions(); m_queueScriptCoordinator->InitOptions(); #ifndef DISABLE_TLS - TlsSocket::InitOptions(g_Options->GetCertCheck() ? g_Options->GetCertStore() : nullptr); + TlsSocket::InitOptions(g_Options->GetCertCheck() ? g_Options->GetCertStore() : ""); #endif if (m_commandLineParser->GetDaemonMode()) @@ -675,6 +681,12 @@ void NZBGet::ProcessStandalone() return; } std::unique_ptr nzbInfo = nzbFile.DetachNzbInfo(); + + if (!nzbFile.GetPassword().empty()) + { + nzbInfo->GetParameters()->SetParameter("*Unpack:Password", nzbFile.GetPassword().c_str()); + } + m_scanner->InitPPParameters(category, nzbInfo->GetParameters(), false); m_queueCoordinator->AddNzbFileToQueue(std::move(nzbInfo), nullptr, false); } @@ -1006,8 +1018,13 @@ void NZBGet::Daemonize() } } + /* Backward compatibility with QNAP which doesn't have a "root" user, + but for historical reasons in nzbget.conf we use "root" as the default value in DaemonUsername + which causes problems when running nzbget as a daemon on QNAP. */ + bool backwardCompRoot = strcmp(m_options->GetDaemonUsername(), "root") == 0; + /* Drop user if there is one, and we were run as root */ - if (getuid() == 0 || geteuid() == 0) + if (!backwardCompRoot && (getuid() == 0 || geteuid() == 0)) { struct passwd *pw = getpwnam(m_options->GetDaemonUsername()); if (pw == nullptr) diff --git a/daemon/main/nzbget.h b/daemon/main/nzbget.h index 2eaca0853..3c674004f 100644 --- a/daemon/main/nzbget.h +++ b/daemon/main/nzbget.h @@ -22,66 +22,11 @@ #ifndef NZBGET_H #define NZBGET_H -#ifdef HAVE_CONFIG_H #include "config.h" -#endif /***************** DEFINES FOR WINDOWS *****************/ #ifdef WIN32 -/* Define to 1 to not use curses */ -//#define DISABLE_CURSES - -/* Define to 1 to disable smart par-verification and restoration */ -//#define DISABLE_PARCHECK - -/* Define to 1 to disable TLS/SSL-support. */ -//#define DISABLE_TLS - -#ifndef DISABLE_TLS -/* Define to 1 to use OpenSSL library for TLS/SSL-support */ -#define HAVE_OPENSSL -/* Define to 1 to use GnuTLS library for TLS/SSL-support */ -//#define HAVE_LIBGNUTLS -#endif - -/* Define to 1 if OpenSSL supports function "X509_check_host". */ -#define HAVE_X509_CHECK_HOST 1 - -/* Define to the name of macro which returns the name of function being -compiled */ -#define FUNCTION_MACRO_NAME __FUNCTION__ - -/* Define to 1 if ctime_r takes 2 arguments */ -#undef HAVE_CTIME_R_2 - -/* Define to 1 if ctime_r takes 3 arguments */ -#define HAVE_CTIME_R_3 - -/* Define to 1 if getopt_long is supported */ -#undef HAVE_GETOPT_LONG - -/* Define to 1 if variadic macros are supported */ -#define HAVE_VARIADIC_MACROS - -/* Define to 1 if function GetAddrInfo is supported */ -#define HAVE_GETADDRINFO - -/* Determine what socket length (socklen_t) data type is */ -#define SOCKLEN_T socklen_t - -/* Define to 1 if you have the header file. */ -#ifndef DISABLE_REGEX -#define HAVE_REGEX_H 1 -// Static linking to regex library -#define REGEX_STATIC -#endif - -#ifndef DISABLE_GZIP -// Static linking to zlib library -//#define ZLIB_WINAPI -#endif - /* Suppress warnings */ #define _CRT_SECURE_NO_DEPRECATE @@ -204,6 +149,7 @@ compiled */ #include #include +#include #include #include #include @@ -216,6 +162,9 @@ compiled */ #include #include #include +#include +#include +#include #include #include #include @@ -225,6 +174,7 @@ compiled */ #include #include #include +#include #include #include @@ -232,6 +182,8 @@ compiled */ #include #include +#include + // NOTE: do not include in "nzbget.h". contains objects requiring // intialization, causing every unit in nzbget to have initialization routine. This in particular // is causing fatal problems in SIMD units which must not have static initialization because diff --git a/daemon/postprocess/ParChecker.cpp b/daemon/postprocess/ParChecker.cpp index 7e40ed5f5..9e48f8842 100644 --- a/daemon/postprocess/ParChecker.cpp +++ b/daemon/postprocess/ParChecker.cpp @@ -23,8 +23,11 @@ #ifndef DISABLE_PARCHECK -#include "par2cmdline.h" -#include "par2repairer.h" +#include +#include + +#include +#include #include "ParChecker.h" #include "ParParser.h" @@ -33,71 +36,71 @@ #include "Util.h" #include "FileSystem.h" -const char* Par2CmdLineErrStr[] = { "OK", +const char* Par2CmdLineErrStr[] = { + "OK", "data files are damaged and there is enough recovery data available to repair them", "data files are damaged and there is insufficient recovery data available to be able to repair them", "there was something wrong with the command line arguments", "the PAR2 files did not contain sufficient information about the data files to be able to verify them", "repair completed but the data files still appear to be damaged", - "an error occured when accessing files", + "an error occurred when accessing files", "internal error occurred", - "out of memory" }; + "out of memory" +}; -class RepairThread; +const char* Par2StageMessage[] = { + "Loading file", + "Verifying file", + "Repairing file", + "Verifying repaired file" +}; -class Repairer : public Par2::Par2Repairer, public ParChecker::AbstractRepairer +class Repairer final : public Par2::Par2Repairer, public ParChecker::AbstractRepairer { public: - Repairer(ParChecker* owner): - Par2::Par2Repairer(owner->m_parCout, owner->m_parCerr), - m_owner(owner), commandLine(owner->m_parCout, owner->m_parCerr) {} + Repairer(ParChecker* owner) + : Par2::Par2Repairer(owner->m_parCout, owner->m_parCerr, Par2::nlNormal) + , m_owner(owner) + , m_commandLine() + { + m_threadsToUse = g_Options->GetParThreads() > 0 + ? g_Options->GetParThreads() + : Util::NumberOfCpuCores(); + m_threadsToUse = m_threadsToUse > 0 ? m_threadsToUse : 1; + m_memToUse = g_Options->GetParBuffer() > 0 + ? g_Options->GetParBuffer() + : 0; + } Par2::Result PreProcess(const char *parFilename); Par2::Result Process(bool dorepair); virtual Repairer* GetRepairer() { return this; } protected: - virtual void sig_filename(std::string filename) { m_owner->signal_filename(filename); } - virtual void sig_progress(int progress) { m_owner->signal_progress(progress); } - virtual void sig_done(std::string filename, int available, int total) { m_owner->signal_done(filename, available, total); } - - virtual bool ScanDataFile(Par2::DiskFile *diskfile, Par2::Par2RepairerSourceFile* &sourcefile, - Par2::MatchType &matchtype, Par2::MD5Hash &hashfull, Par2::MD5Hash &hash16k, Par2::u32 &count); - virtual bool RepairData(Par2::u32 inputindex, size_t blocklength); + void SigFilename(std::string filename) override { m_owner->SignalFilename(std::move(filename)); } + void SigProgress(int progress) override { m_owner->SignalProgress(progress); } + void SigDone(std::string filename, int available, int total) override { m_owner->SignalDone(std::move(filename), available, total); } + + bool ScanDataFile( + Par2::DiskFile *diskfile, + std::string basepath, + Par2::Par2RepairerSourceFile* &sourcefile, + Par2::MatchType &matchtype, + Par2::MD5Hash &hashfull, + Par2::MD5Hash &hash16k, + Par2::u32 &count, + std::mutex& mtx) override; private: - typedef vector Threads; + typedef std::vector Threads; ParChecker* m_owner; - Par2::CommandLine commandLine; - Threads m_threads; - bool m_parallel; - Mutex progresslock; + Par2::CommandLine m_commandLine; + int m_threadsToUse; + int m_memToUse; - virtual void BeginRepair(); - virtual void EndRepair(); - void RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength); - static void SyncSleep(); + void BeginRepair() override; friend class ParChecker; - friend class RepairThread; -}; - -class RepairThread : public Thread -{ -public: - RepairThread(Repairer* owner) : m_owner(owner) {} - void RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength); - bool IsWorking() { return m_working; } - -protected: - virtual void Run(); - -private: - Repairer* m_owner; - Par2::u32 m_inputindex; - Par2::u32 m_outputindex; - size_t m_blocklength; - std::atomic m_working{false}; }; class RepairCreatorPacket : public Par2::CreatorPacket @@ -107,7 +110,8 @@ class RepairCreatorPacket : public Par2::CreatorPacket Par2::Result Repairer::PreProcess(const char *parFilename) { - BString<100> memParam("-m%i", g_Options->GetParBuffer()); + std::string memParam = "-m" + std::to_string(m_memToUse); + std::string threadsParam = "-t" + std::to_string(m_threadsToUse); if (g_Options->GetParScan() == Options::psFull) { @@ -119,51 +123,67 @@ Par2::Result Repairer::PreProcess(const char *parFilename) basename[1] = '\0'; } - const char* argv[] = { "par2", "r", "-v", memParam, parFilename, wildcardParam }; - if (!commandLine.Parse(6, (char**)argv)) + const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename, wildcardParam }; + if (!m_commandLine.Parse(7, (char**)argv)) { return Par2::eInvalidCommandLineArguments; } } else { - const char* argv[] = { "par2", "r", "-v", memParam, parFilename }; - if (!commandLine.Parse(5, (char**)argv)) + const char* argv[] = { "par2", "r", "-v", memParam.c_str(), threadsParam.c_str(), parFilename }; + if (!m_commandLine.Parse(6, (char**)argv)) { return Par2::eInvalidCommandLineArguments; } } - return Par2Repairer::PreProcess(commandLine); + return Par2Repairer::PreProcess(m_commandLine); } Par2::Result Repairer::Process(bool dorepair) { - Par2::Result res = Par2Repairer::Process(commandLine, dorepair); + Par2::Result res = Par2Repairer::Process( + m_commandLine.GetMemoryLimit(), + m_commandLine.GetBasePath(), + m_commandLine.GetNumThreads(), + m_commandLine.GetFileThreads(), + m_commandLine.GetParFilename(), + m_commandLine.GetExtraFiles(), + dorepair, + m_commandLine.GetPurgeFiles(), + m_commandLine.GetSkipData(), + m_commandLine.GetSkipLeaway()); debug("ParChecker: Process-result=%i", res); return res; } -bool Repairer::ScanDataFile(Par2::DiskFile *diskfile, Par2::Par2RepairerSourceFile* &sourcefile, - Par2::MatchType &matchtype, Par2::MD5Hash &hashfull, Par2::MD5Hash &hash16k, Par2::u32 &count) +bool Repairer::ScanDataFile( + Par2::DiskFile *diskfile, + std::string basepath, + Par2::Par2RepairerSourceFile* &sourcefile, + Par2::MatchType &matchtype, + Par2::MD5Hash &hashfull, + Par2::MD5Hash &hash16k, + Par2::u32 &count, + std::mutex& mtx) { - if (m_owner->GetParQuick() && sourcefile) + if (m_owner->GetParQuick() && sourcefile && diskfile) { - string path; - string name; - Par2::DiskFile::SplitFilename(diskfile->FileName(), path, name); + std::string name; + Par2::DiskFile::SplitFilename(diskfile->FileName(), basepath, name); - sig_filename(name); + SigFilename(name); if (!(m_owner->GetStage() == ParChecker::ptVerifyingRepaired && m_owner->GetParFull())) { int availableBlocks = sourcefile->BlockCount(); - ParChecker::EFileStatus fileStatus = m_owner->VerifyDataFile(diskfile, sourcefile, &availableBlocks); + ParChecker::EFileStatus fileStatus = m_owner->VerifyDataFile(*diskfile, *sourcefile, availableBlocks); if (fileStatus != ParChecker::fsUnknown) { - sig_done(name, availableBlocks, sourcefile->BlockCount()); - sig_progress(1000); + SigDone(name, availableBlocks, sourcefile->BlockCount()); + SigProgress(1000); matchtype = fileStatus == ParChecker::fsSuccess ? Par2::eFullMatch : fileStatus == ParChecker::fsPartial ? Par2::ePartialMatch : Par2::eNoMatch; m_owner->SetParFull(false); @@ -172,174 +192,28 @@ bool Repairer::ScanDataFile(Par2::DiskFile *diskfile, Par2::Par2RepairerSourceFi } } - return Par2Repairer::ScanDataFile(diskfile, sourcefile, matchtype, hashfull, hash16k, count); + return Par2Repairer::ScanDataFile( + diskfile, + basepath, + sourcefile, + matchtype, + hashfull, + hash16k, + count, + mtx); } void Repairer::BeginRepair() { - int maxThreads = g_Options->GetParThreads() > 0 ? g_Options->GetParThreads() : Util::NumberOfCpuCores(); - maxThreads = maxThreads > 0 ? maxThreads : 1; - - int threads = maxThreads > (int)missingblockcount ? (int)missingblockcount : maxThreads; - - m_owner->PrintMessage(Message::mkInfo, "Using %i of max %i thread(s) to repair %i block(s) for %s", - threads, maxThreads, (int)missingblockcount, *m_owner->m_nzbName); - - m_parallel = threads > 1; - - if (m_parallel) - { - for (int i = 0; i < threads; i++) - { - RepairThread* repairThread = new RepairThread(this); - m_threads.push_back(repairThread); - repairThread->SetAutoDestroy(true); - repairThread->Start(); - } - -#ifdef WIN32 - timeBeginPeriod(1); -#endif - } + m_owner->PrintMessage(Message::mkInfo, "Using %i thread(s) and %i MB to repair %i block(s) for %s", + m_threadsToUse, m_memToUse, (int)missingblockcount, m_owner->m_nzbName.c_str()); } -void Repairer::EndRepair() -{ - if (m_parallel) - { - for (Thread* thread : m_threads) - { - thread->Stop(); - } - -#ifdef WIN32 - timeEndPeriod(1); -#endif - } -} - -bool Repairer::RepairData(Par2::u32 inputindex, size_t blocklength) -{ - if (!m_parallel) - { - return false; - } - - for (Par2::u32 outputindex = 0; outputindex < missingblockcount; ) - { - bool jobAdded = false; - for (Thread* thread : m_threads) - { - RepairThread* repairThread = (RepairThread*)thread; - if (!repairThread->IsWorking()) - { - repairThread->RepairBlock(inputindex, outputindex, blocklength); - outputindex++; - jobAdded = true; - break; - } - } - - if (cancelled) - { - break; - } - - if (!jobAdded) - { - SyncSleep(); - } - } - - // Wait until all m_Threads complete their jobs - bool working = true; - while (working) - { - working = false; - for (Thread* thread : m_threads) - { - RepairThread* repairThread = (RepairThread*)thread; - if (repairThread->IsWorking()) - { - working = true; - SyncSleep(); - break; - } - } - } - - return true; -} - -void Repairer::RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength) -{ - // Select the appropriate part of the output buffer - void *outbuf = &((Par2::u8*)outputbuffer)[chunksize * outputindex]; - - // Process the data - rs.Process(blocklength, inputindex, inputbuffer, outputindex, outbuf); - - if (noiselevel > Par2::CommandLine::nlQuiet) - { - // Update a progress indicator - - Par2::u32 oldfraction; - Par2::u32 newfraction; - { - Guard guard(progresslock); - oldfraction = (Par2::u32)(1000 * progress / totaldata); - progress += blocklength; - newfraction = (Par2::u32)(1000 * progress / totaldata); - } - - if (oldfraction != newfraction) - { - sig_progress(newfraction); - } - } -} - -// Sleep for synchronisation -void Repairer::SyncSleep() -{ -#ifdef WIN32 - // Windows doesn't allow sleep intervals less than one millisecond - Sleep(1); -#else - usleep(100); -#endif -} - -void RepairThread::Run() -{ - while (!IsStopped()) - { - if (m_working) - { - m_owner->RepairBlock(m_inputindex, m_outputindex, m_blocklength); - m_working = false; - } - else - { - Repairer::SyncSleep(); - } - } -} - -void RepairThread::RepairBlock(Par2::u32 inputindex, Par2::u32 outputindex, size_t blocklength) -{ - m_inputindex = inputindex; - m_outputindex = outputindex; - m_blocklength = blocklength; - m_working = true; -} - - int ParChecker::StreamBuf::overflow(int ch) { if (ch == '\n' || ch == '\r') { - char* msg = (char*)*m_buffer; + char* msg = m_buffer.data(); // make par2-logging less verbose bool extraDebug = !msg || strchr(msg, '%') || @@ -363,14 +237,14 @@ int ParChecker::StreamBuf::overflow(int ch) } } - m_buffer.Clear(); + m_buffer.clear(); } else { char bf[2]; bf[0] = (char)ch; bf[1] = '\0'; - m_buffer.Append(bf); + m_buffer.append(std::string(bf)); } return (int)ch; } @@ -378,13 +252,14 @@ int ParChecker::StreamBuf::overflow(int ch) void ParChecker::Cleanup() { - Guard guard(m_repairerMutex); + std::lock_guard guard{ m_repairerMtx }; + m_repairer.reset(); m_queuedParFiles.clear(); m_processedFiles.clear(); m_sourceFiles.clear(); m_dupeSources.clear(); - m_errMsg = nullptr; + m_errMsg = ""; } void ParChecker::Execute() @@ -393,7 +268,7 @@ void ParChecker::Execute() if (m_status == psRepairNotNeeded && m_parQuick && m_forceRepair && !IsStopped()) { - PrintMessage(Message::mkInfo, "Performing full par-check for %s", *m_nzbName); + PrintMessage(Message::mkInfo, "Performing full par-check for %s", m_nzbName.c_str()); m_parQuick = false; m_status = RunParCheckAll(); } @@ -404,29 +279,29 @@ void ParChecker::Execute() ParChecker::EStatus ParChecker::RunParCheckAll() { ParParser::ParFileList fileList; - if (!ParParser::FindMainPars(m_destDir, &fileList)) + if (!ParParser::FindMainPars(m_destDir.c_str(), &fileList)) { - PrintMessage(Message::mkError, "Could not start par-check for %s. Could not find any par-files", *m_nzbName); + PrintMessage(Message::mkError, "Could not start par-check for %s. Could not find any par-files", m_nzbName.c_str()); return psFailed; } EStatus allStatus = psRepairNotNeeded; m_parFull = true; - for (CString& parFilename : fileList) + for (std::string& parFilename : fileList) { - debug("Found par: %s", *parFilename); + debug("Found par: %s", parFilename.c_str()); if (!IsStopped()) { - BString<1024> fullParFilename( "%s%c%s", *m_destDir, PATH_SEPARATOR, *parFilename); + BString<1024> fullParFilename( "%s%c%s", m_destDir.c_str(), PATH_SEPARATOR, parFilename.c_str()); int baseLen = 0; - ParParser::ParseParFilename(parFilename, true, &baseLen, nullptr); + ParParser::ParseParFilename(parFilename.c_str(), true, &baseLen, nullptr); BString<1024> infoName; - infoName.Set(parFilename, baseLen); + infoName.Set(parFilename.c_str(), baseLen); - BString<1024> parInfoName("%s%c%s", *m_nzbName, PATH_SEPARATOR, *infoName); + BString<1024> parInfoName("%s%c%s", m_nzbName.c_str(), PATH_SEPARATOR, *infoName); SetInfoName(parInfoName); EStatus status = RunParCheck(fullParFilename); @@ -445,7 +320,7 @@ ParChecker::EStatus ParChecker::RunParCheckAll() ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) { Cleanup(); - m_parFilename = parFilename; + m_parFilename = parFilename ? parFilename : ""; m_stage = ptLoadingPars; m_processedCount = 0; m_extraFiles = 0; @@ -454,11 +329,11 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) m_hasDamagedFiles = false; EStatus status = psFailed; - PrintMessage(Message::mkInfo, "Verifying %s", *m_infoName); + PrintMessage(Message::mkInfo, "Verifying %s", m_infoName.c_str()); - debug("par: %s", m_parFilename); + debug("par: %s", m_parFilename.c_str()); - m_progressLabel.Format("Verifying %s", *m_infoName); + m_progressLabel = "Verifying " + m_infoName; m_fileProgress = 0; m_stageProgress = 0; UpdateProgress(); @@ -470,8 +345,8 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) return psFailed; } - CString creator = GetPacketCreator(); - info("Recovery files created by: %s", creator.Empty() ? "" : *creator); + std::string creator = GetPacketCreator(); + info("Recovery files created by: %s", creator.empty() ? "" : creator.c_str()); m_stage = ptVerifyingSources; res = GetRepairer()->Process(false); @@ -530,7 +405,7 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) if (res == Par2::eSuccess || !m_hasDamagedFiles) { - PrintMessage(Message::mkInfo, "Repair not needed for %s", *m_infoName); + PrintMessage(Message::mkInfo, "Repair not needed for %s", m_infoName.c_str()); status = psRepairNotNeeded; } else if (res == Par2::eRepairPossible) @@ -538,10 +413,10 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) status = psRepairPossible; if (g_Options->GetParRepair()) { - PrintMessage(Message::mkInfo, "Repairing %s", *m_infoName); + PrintMessage(Message::mkInfo, "Repairing %s", m_infoName.c_str()); SaveSourceList(); - m_progressLabel.Format("Repairing %s", *m_infoName); + m_progressLabel = std::string("Repairing ") + m_infoName.c_str(); m_fileProgress = 0; m_stageProgress = 0; m_processedCount = 0; @@ -552,7 +427,7 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) res = GetRepairer()->Process(true); if (res == Par2::eSuccess) { - PrintMessage(Message::mkInfo, "Successfully repaired %s", *m_infoName); + PrintMessage(Message::mkInfo, "Successfully repaired %s", m_infoName.c_str()); status = psRepaired; StatDupeSources(&m_dupeSources); DeleteLeftovers(); @@ -564,7 +439,7 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) } else { - PrintMessage(Message::mkInfo, "Repair possible for %s", *m_infoName); + PrintMessage(Message::mkInfo, "Repair possible for %s", m_infoName.c_str()); } } @@ -572,25 +447,25 @@ ParChecker::EStatus ParChecker::RunParCheck(const char* parFilename) { if (m_stage >= ptRepairing) { - PrintMessage(Message::mkWarning, "Repair cancelled for %s", *m_infoName); + PrintMessage(Message::mkWarning, "Repair cancelled for %s", m_infoName.c_str()); m_errMsg = "repair cancelled"; status = psRepairPossible; } else { - PrintMessage(Message::mkWarning, "Par-check cancelled for %s", *m_infoName); + PrintMessage(Message::mkWarning, "Par-check cancelled for %s", m_infoName.c_str()); m_errMsg = "par-check cancelled"; status = psFailed; } } else if (status == psFailed) { - if (!m_errMsg && (int)res >= 0 && (int)res <= 8) + if (m_errMsg.empty() && (int)res >= 0 && (int)res <= 8) { m_errMsg = Par2CmdLineErrStr[res]; } PrintMessage(Message::mkError, "Repair failed for %s: %s. Recovery files created by: %s", - *m_infoName, *m_errMsg, creator.Empty() ? "" : *creator); + m_infoName.c_str(), m_errMsg.c_str(), creator.empty() ? "" : creator.c_str()); } Cleanup(); @@ -605,31 +480,31 @@ int ParChecker::PreProcessPar() Cleanup(); { - Guard guard(m_repairerMutex); + std::lock_guard guard{ m_repairerMtx }; m_repairer = std::make_unique(this); } - res = GetRepairer()->PreProcess(m_parFilename); + res = GetRepairer()->PreProcess(m_parFilename.c_str()); debug("ParChecker: PreProcess-result=%i", res); if (IsStopped()) { - PrintMessage(Message::mkError, "Could not verify %s: stopping", *m_infoName); + PrintMessage(Message::mkError, "Could not verify %s: stopping", m_infoName.c_str()); m_errMsg = "par-check was stopped"; return Par2::eRepairFailed; } if (res == Par2::eInvalidCommandLineArguments) { - PrintMessage(Message::mkError, "Could not start par-check for %s. Par-file: %s", *m_infoName, m_parFilename); + PrintMessage(Message::mkError, "Could not start par-check for %s. Par-file: %s", m_infoName.c_str(), m_parFilename.c_str()); m_errMsg = "Command line could not be parsed"; return res; } if (res != Par2::eSuccess) { - PrintMessage(Message::mkWarning, "Could not verify %s: par2-file could not be processed", *m_infoName); - PrintMessage(Message::mkInfo, "Requesting more par2-files for %s", *m_infoName); + PrintMessage(Message::mkWarning, "Could not verify %s: par2-file could not be processed", m_infoName.c_str()); + PrintMessage(Message::mkInfo, "Requesting more par2-files for %s", m_infoName.c_str()); bool hasMorePars = LoadMainParBak(); if (!hasMorePars) { @@ -641,7 +516,7 @@ int ParChecker::PreProcessPar() if (res != Par2::eSuccess) { - PrintMessage(Message::mkError, "Could not verify %s: par2-file could not be processed", *m_infoName); + PrintMessage(Message::mkError, "Could not verify %s: par2-file could not be processed", m_infoName.c_str()); m_errMsg = "par2-file could not be processed"; return res; } @@ -655,7 +530,7 @@ bool ParChecker::LoadMainParBak() { bool hasMorePars = false; { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; hasMorePars = !m_queuedParFiles.empty(); m_queuedParFiles.clear(); } @@ -675,7 +550,7 @@ bool ParChecker::LoadMainParBak() } { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; hasMorePars = !m_queuedParFiles.empty(); m_queuedParFilesChanged = false; } @@ -692,7 +567,7 @@ bool ParChecker::LoadMainParBak() while (!queuedParFilesChanged && !IsStopped()) { { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; queuedParFilesChanged = m_queuedParFilesChanged; } Util::Sleep(100); @@ -719,12 +594,12 @@ int ParChecker::ProcessMorePars() if (moreFilesLoaded) { - PrintMessage(Message::mkInfo, "Need more %i par-block(s) for %s", missingblockcount, *m_infoName); + PrintMessage(Message::mkInfo, "Need more %i par-block(s) for %s", missingblockcount, m_infoName.c_str()); } bool hasMorePars; { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; hasMorePars = !m_queuedParFiles.empty(); } @@ -740,14 +615,18 @@ int ParChecker::ProcessMorePars() } { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; hasMorePars = !m_queuedParFiles.empty(); m_queuedParFilesChanged = false; } if (!requested && !hasMorePars) { - m_errMsg.Format("not enough par-blocks, %i block(s) needed, but %i block(s) available", missingblockcount, blockFound); + m_errMsg = std::string("not enough par-blocks, ") + + std::to_string(missingblockcount) + + " block(s) needed, but " + + std::to_string(blockFound) + + " block(s) available"; break; } @@ -758,7 +637,7 @@ int ParChecker::ProcessMorePars() while (!queuedParFilesChanged && !IsStopped()) { { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; queuedParFilesChanged = m_queuedParFilesChanged; } Util::Sleep(100); @@ -786,21 +665,21 @@ bool ParChecker::LoadMorePars() { FileList moreFiles; { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; moreFiles = std::move(m_queuedParFiles); m_queuedParFiles.clear(); } - for (CString& parFilename : moreFiles) + for (std::string& parFilename : moreFiles) { - bool loadedOK = GetRepairer()->LoadPacketsFromFile(*parFilename); + bool loadedOK = GetRepairer()->LoadPacketsFromFile(parFilename.c_str()); if (loadedOK) { - PrintMessage(Message::mkInfo, "File %s successfully loaded for par-check", FileSystem::BaseFileName(parFilename)); + PrintMessage(Message::mkInfo, "File %s successfully loaded for par-check", FileSystem::BaseFileName(parFilename.c_str())); } else { - PrintMessage(Message::mkInfo, "Could not load file %s for par-check", FileSystem::BaseFileName(parFilename)); + PrintMessage(Message::mkInfo, "Could not load file %s for par-check", FileSystem::BaseFileName(parFilename.c_str())); } } @@ -809,22 +688,22 @@ bool ParChecker::LoadMorePars() void ParChecker::AddParFile(const char * parFilename) { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; m_queuedParFiles.push_back(parFilename); m_queuedParFilesChanged = true; } void ParChecker::QueueChanged() { - Guard guard(m_queuedParFilesMutex); + std::lock_guard guard{ m_queuedParFilesMtx }; m_queuedParFilesChanged = true; } bool ParChecker::AddSplittedFragments() { - std::list extrafiles; + std::vector extrafiles; - DirBrowser dir(m_destDir); + DirBrowser dir(m_destDir.c_str()); while (const char* filename = dir.Next()) { if (!IsParredFile(filename) && !IsProcessedFile(filename)) @@ -842,9 +721,7 @@ bool ParChecker::AddSplittedFragments() MaybeSplittedFragement(filename, original))) { detail("Found splitted fragment %s", filename); - BString<1024> fullfilename("%s%c%s", *m_destDir, PATH_SEPARATOR, filename); - Par2::CommandLine::ExtraFile extrafile(*fullfilename, FileSystem::FileSize(fullfilename)); - extrafiles.push_back(extrafile); + extrafiles.push_back(m_destDir + PATH_SEPARATOR + filename); break; } } @@ -857,8 +734,8 @@ bool ParChecker::AddSplittedFragments() { m_extraFiles += extrafiles.size(); m_verifyingExtraFiles = true; - PrintMessage(Message::mkInfo, "Found %i splitted fragments for %s", (int)extrafiles.size(), *m_infoName); - fragmentsAdded = GetRepairer()->VerifyExtraFiles(extrafiles); + PrintMessage(Message::mkInfo, "Found %i splitted fragments for %s", (int)extrafiles.size(), m_infoName.c_str()); + fragmentsAdded = GetRepairer()->VerifyExtraFiles(extrafiles, m_destDir); GetRepairer()->UpdateVerificationResults(); m_verifyingExtraFiles = false; } @@ -902,12 +779,12 @@ bool ParChecker::MaybeSplittedFragement(const char* filename1, const char* filen bool ParChecker::AddMissingFiles() { - return AddExtraFiles(true, false, m_destDir); + return AddExtraFiles(true, false, m_destDir.c_str()); } bool ParChecker::AddDupeFiles() { - BString<1024> directory = m_parFilename; + BString<1024> directory = m_parFilename.c_str(); bool added = AddExtraFiles(false, false, directory); @@ -951,48 +828,50 @@ bool ParChecker::AddDupeFiles() * Files with the same name as in par-file (and a differnt extension) are * placed at the top of the list to be scanned first. */ -void ParChecker::SortExtraFiles(void* extrafiles) +void ParChecker::SortExtraFiles(std::vector& extrafiles) { - CString baseParFilename = FileSystem::BaseFileName(m_parFilename); - if (char* ext = strrchr(baseParFilename, '.')) *ext = '\0'; // trim extension + std::string baseParFilename = FileSystem::BaseFileName(m_parFilename.c_str()); + if (char* ext = strrchr(baseParFilename.data(), '.')) *ext = '\0'; // trim extension - ((std::list*)extrafiles)->sort( - [&baseParFilename](Par2::CommandLine::ExtraFile& file1, Par2::CommandLine::ExtraFile& file2) + std::sort( + begin(extrafiles), + end(extrafiles), + [&baseParFilename](const std::string& a, const std::string& b) { - BString<1024> name1 = FileSystem::BaseFileName(file1.FileName().c_str()); - if (char* ext = strrchr(name1, '.')) *ext = '\0'; // trim extension + std::string name1 = FileSystem::BaseFileName(a.c_str()); + if (char* ext = strrchr(name1.data(), '.')) *ext = '\0'; // trim extension - BString<1024> name2 = FileSystem::BaseFileName(file2.FileName().c_str()); - if (char* ext = strrchr(name2, '.')) *ext = '\0'; // trim extension + std::string name2 = FileSystem::BaseFileName(b.c_str()); + if (char* ext = strrchr(name2.data(), '.')) *ext = '\0'; // trim extension - return strcmp(name1, baseParFilename) == 0 && strcmp(name1, name2) != 0; - }); + return strcmp(name1.c_str(), baseParFilename.c_str()) == 0 && strcmp(name1.c_str(), name2.c_str()) != 0; + } + ); } bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* directory) { if (externalDir) { - PrintMessage(Message::mkInfo, "Performing dupe par-scan for %s in %s", *m_infoName, FileSystem::BaseFileName(directory)); + PrintMessage(Message::mkInfo, "Performing dupe par-scan for %s in %s", m_infoName.c_str(), FileSystem::BaseFileName(directory)); } else { - PrintMessage(Message::mkInfo, "Performing extra par-scan for %s", *m_infoName); + PrintMessage(Message::mkInfo, "Performing extra par-scan for %s", m_infoName.c_str()); } - std::list extrafiles; + std::vector extrafiles; DirBrowser dir(directory); while (const char* filename = dir.Next()) { if (externalDir || (!IsParredFile(filename) && !IsProcessedFile(filename))) { - BString<1024> fullfilename("%s%c%s", directory, PATH_SEPARATOR, filename); - extrafiles.emplace_back(*fullfilename, FileSystem::FileSize(fullfilename)); + extrafiles.push_back(std::string(directory) + PATH_SEPARATOR + filename); } } - SortExtraFiles(&extrafiles); + SortExtraFiles(extrafiles); // Scan files bool filesAdded = false; @@ -1002,18 +881,16 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d m_verifyingExtraFiles = true; // adding files one by one until all missing files are found - - while (!IsStopped() && extrafiles.size() > 0) + size_t idx = 0; + while (!IsStopped() && idx < extrafiles.size()) { - std::list extrafiles1; - extrafiles1.splice(extrafiles1.end(), extrafiles, extrafiles.begin()); - - Par2::CommandLine::ExtraFile& extraFile = extrafiles1.front(); + const std::string& extraFile = extrafiles[idx]; + ++idx; int wasFilesMissing = GetRepairer()->missingfilecount; int wasBlocksMissing = GetRepairer()->missingblockcount; - GetRepairer()->VerifyExtraFiles(extrafiles1); + GetRepairer()->VerifyExtraFiles({ extraFile }, m_destDir); GetRepairer()->UpdateVerificationResults(); bool fileAdded = wasFilesMissing > (int)GetRepairer()->missingfilecount; @@ -1021,8 +898,8 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d if (fileAdded && !externalDir) { - PrintMessage(Message::mkInfo, "Found missing file %s", FileSystem::BaseFileName(extraFile.FileName().c_str())); - RegisterParredFile(FileSystem::BaseFileName(extraFile.FileName().c_str())); + PrintMessage(Message::mkInfo, "Found missing file %s", FileSystem::BaseFileName(extraFile.c_str())); + RegisterParredFile(FileSystem::BaseFileName(extraFile.c_str())); } else if (blockAdded) { @@ -1052,9 +929,9 @@ bool ParChecker::AddExtraFiles(bool onlyMissing, bool externalDir, const char* d bool ParChecker::IsProcessedFile(const char* filename) { - for (CString& processedFilename : m_processedFiles) + for (std::string& processedFilename : m_processedFiles) { - if (!strcasecmp(FileSystem::BaseFileName(processedFilename), filename)) + if (!strcasecmp(FileSystem::BaseFileName(processedFilename.c_str()), filename)) { return true; } @@ -1063,16 +940,9 @@ bool ParChecker::IsProcessedFile(const char* filename) return false; } -void ParChecker::signal_filename(std::string str) +void ParChecker::SignalFilename(std::string str) { - if (!m_lastFilename.compare(str)) - { - return; - } - - m_lastFilename = str; - - const char* stageMessage[] = { "Loading file", "Verifying file", "Repairing file", "Verifying repaired file" }; + std::lock_guard guard{ m_sigFileNameMtx }; if (m_stage == ptRepairing) { @@ -1083,22 +953,25 @@ void ParChecker::signal_filename(std::string str) // because repaired files are not verified in this mode if (!(m_stage == ptVerifyingRepaired && m_parQuick)) { - PrintMessage(Message::mkInfo, "%s %s", stageMessage[m_stage], str.c_str()); + PrintMessage(Message::mkInfo, "%s %s", Par2StageMessage[m_stage], str.c_str()); } + m_progressLabel = Par2StageMessage[m_stage] + (" " + str); + if (m_stage == ptLoadingPars || m_stage == ptVerifyingSources) { - m_processedFiles.push_back(str.c_str()); + m_processedFiles.push_back(std::move(str)); } - m_progressLabel.Format("%s %s", stageMessage[m_stage], str.c_str()); m_fileProgress = 0; UpdateProgress(); } -void ParChecker::signal_progress(int progress) +void ParChecker::SignalProgress(int progress) { - m_fileProgress = (int)progress; + std::lock_guard guard{ m_sigProgressMtx }; + + m_fileProgress = progress; if (m_stage == ptRepairing) { @@ -1152,20 +1025,21 @@ void ParChecker::signal_progress(int progress) UpdateProgress(); } -void ParChecker::signal_done(std::string str, int available, int total) +void ParChecker::SignalDone(std::string fileName, int available, int total) { - m_processedCount++; + std::lock_guard guard{ m_sigDoneMtx }; + + ++m_processedCount; if (m_stage == ptVerifyingSources) { if (available < total && !m_verifyingExtraFiles) { - const char* filename = str.c_str(); - bool fileExists = true; for (Par2::Par2RepairerSourceFile* sourcefile : GetRepairer()->sourcefiles) { - if (sourcefile && !strcmp(filename, FileSystem::BaseFileName(sourcefile->TargetFileName().c_str())) && + std::string targetFileName = sourcefile->TargetFileName(); + if (sourcefile && !strcmp(fileName.c_str(), FileSystem::BaseFileName(targetFileName.c_str())) && !sourcefile->GetTargetExists()) { fileExists = false; @@ -1173,23 +1047,23 @@ void ParChecker::signal_done(std::string str, int available, int total) } } - bool ignore = Util::MatchFileExt(filename, g_Options->GetParIgnoreExt(), ",;"); + bool ignore = Util::MatchFileExt(fileName.c_str(), g_Options->GetParIgnoreExt(), ",;"); m_hasDamagedFiles |= !ignore; if (fileExists) { PrintMessage(Message::mkWarning, "File %s has %i bad block(s) of total %i block(s)%s", - filename, total - available, total, ignore ? ", ignoring" : ""); + fileName.c_str(), total - available, total, ignore ? ", ignoring" : ""); } else { PrintMessage(Message::mkWarning, "File %s with %i block(s) is missing%s", - filename, total, ignore ? ", ignoring" : ""); + fileName.c_str(), total, ignore ? ", ignoring" : ""); } - if (!IsProcessedFile(filename)) + if (!IsProcessedFile(fileName.c_str())) { - m_processedFiles.push_back(filename); + m_processedFiles.push_back(std::move(fileName)); } } } @@ -1207,16 +1081,15 @@ void ParChecker::CheckEmptyFiles() if (sourcefile && sourcefile->GetDescriptionPacket()) { // GetDescriptionPacket()->FileName() returns a temp string object, which we need to hold for a while - std::string filenameObj = Par2::DiskFile::TranslateFilename(sourcefile->GetDescriptionPacket()->FileName()); - const char* filename = filenameObj.c_str(); - if (!Util::EmptyStr(filename) && !IsProcessedFile(filename)) + std::string filenameObj = sourcefile->GetDescriptionPacket()->FileName(); + if (!filenameObj.empty() && !IsProcessedFile(filenameObj.c_str())) { - bool ignore = Util::MatchFileExt(filename, g_Options->GetParIgnoreExt(), ",;"); + bool ignore = Util::MatchFileExt(filenameObj.c_str(), g_Options->GetParIgnoreExt(), ",;"); m_hasDamagedFiles |= !ignore; int total = sourcefile->GetVerificationPacket() ? sourcefile->GetVerificationPacket()->BlockCount() : 0; PrintMessage(Message::mkWarning, "File %s has %i bad block(s) of total %i block(s)%s", - filename, total, total, ignore ? ", ignoring" : ""); + filenameObj.c_str(), total, total, ignore ? ", ignoring" : ""); } } else @@ -1229,7 +1102,7 @@ void ParChecker::CheckEmptyFiles() void ParChecker::Cancel() { { - Guard guard(m_repairerMutex); + std::lock_guard guard{ m_repairerMtx }; if (m_repairer) { m_repairer->GetRepairer()->cancelled = true; @@ -1244,7 +1117,7 @@ void ParChecker::SaveSourceList() for (Par2::Par2RepairerSourceFile* sourcefile : GetRepairer()->sourcefiles) { - vector::iterator it2 = sourcefile->SourceBlocks(); + std::vector::iterator it2 = sourcefile->SourceBlocks(); for (int i = 0; i < (int)sourcefile->BlockCount(); i++, it2++) { Par2::DataBlock block = *it2; @@ -1280,8 +1153,9 @@ void ParChecker::DeleteLeftovers() if (!found) { - PrintMessage(Message::mkInfo, "Deleting file %s", FileSystem::BaseFileName(sourceFile->FileName().c_str())); - FileSystem::DeleteFile(sourceFile->FileName().c_str()); + std::string fileName = sourceFile->FileName().c_str(); + PrintMessage(Message::mkInfo, "Deleting file %s", FileSystem::BaseFileName(fileName.c_str())); + FileSystem::DeleteFile(fileName.c_str()); } } } @@ -1301,7 +1175,7 @@ void ParChecker::DeleteLeftovers() * The limitation can be avoided by using something more smart than "verificationhashtable.Lookup" * but in the real life all blocks have unique CRCs and the simple "Lookup" works good enough. */ -ParChecker::EFileStatus ParChecker::VerifyDataFile(void* diskfile, void* sourcefile, int* availableBlocks) +ParChecker::EFileStatus ParChecker::VerifyDataFile(Par2::DiskFile& diskFile, Par2::Par2RepairerSourceFile& sourceFile, int& availableBlocks) { if (m_stage != ptVerifyingSources) { @@ -1311,48 +1185,44 @@ ParChecker::EFileStatus ParChecker::VerifyDataFile(void* diskfile, void* sourcef return fsSuccess; } - Par2::DiskFile* diskFile = (Par2::DiskFile*)diskfile; - Par2::Par2RepairerSourceFile* sourceFile = (Par2::Par2RepairerSourceFile*)sourcefile; - if (!sourcefile || !sourceFile->GetTargetExists()) + if (!sourceFile.GetTargetExists()) { return fsUnknown; } - Par2::VerificationPacket* packet = sourceFile->GetVerificationPacket(); + Par2::VerificationPacket* packet = sourceFile.GetVerificationPacket(); if (!packet) { return fsUnknown; } - std::string filenameObj = sourceFile->GetTargetFile()->FileName(); - const char* filename = filenameObj.c_str(); - - if (FileSystem::FileSize(filename) == 0 && sourceFile->BlockCount() > 0) + std::string filename = sourceFile.GetTargetFile()->FileName(); + if (FileSystem::FileSize(filename.c_str()) == 0 && sourceFile.BlockCount() > 0) { - *availableBlocks = 0; + availableBlocks = 0; return fsFailure; } // find file status and CRC computed during download uint32 downloadCrc; SegmentList segments; - EFileStatus fileStatus = FindFileCrc(FileSystem::BaseFileName(filename), &downloadCrc, &segments); + EFileStatus fileStatus = FindFileCrc(FileSystem::BaseFileName(filename.c_str()), &downloadCrc, &segments); ValidBlocks validBlocks; if (fileStatus == fsFailure || fileStatus == fsUnknown) { return fileStatus; } - else if ((fileStatus == fsSuccess && !VerifySuccessDataFile(diskfile, sourcefile, downloadCrc)) || - (fileStatus == fsPartial && !VerifyPartialDataFile(diskfile, sourcefile, &segments, &validBlocks))) + else if ((fileStatus == fsSuccess && !VerifySuccessDataFile(diskFile, sourceFile, downloadCrc)) || + (fileStatus == fsPartial && !VerifyPartialDataFile(diskFile, sourceFile, segments, validBlocks))) { PrintMessage(Message::mkWarning, "Quick verification failed for %s file %s, performing full verification instead", - fileStatus == fsSuccess ? "good" : "damaged", FileSystem::BaseFileName(filename)); + fileStatus == fsSuccess ? "good" : "damaged", FileSystem::BaseFileName(filename.c_str())); return fsUnknown; // let libpar2 do the full verification of the file } // attach verification blocks to the file - *availableBlocks = 0; + availableBlocks = 0; Par2::u64 blocksize = GetRepairer()->mainpacket->BlockSize(); std::deque undoList; for (uint32 i = 0; i < packet->BlockCount(); i++) @@ -1364,7 +1234,7 @@ ParChecker::EFileStatus ParChecker::VerifyDataFile(void* diskfile, void* sourcef // Look for a match const Par2::VerificationHashEntry* hashEntry = GetRepairer()->verificationhashtable.Lookup(blockCrc); - if (!hashEntry || hashEntry->SourceFile() != sourceFile || hashEntry->IsSet()) + if (!hashEntry || hashEntry->SourceFile() != &sourceFile || hashEntry->IsSet()) { // no match found, revert back the changes made by "pHashEntry->SetBlock" for (const Par2::VerificationHashEntry* undoEntry : undoList) @@ -1375,28 +1245,27 @@ ParChecker::EFileStatus ParChecker::VerifyDataFile(void* diskfile, void* sourcef } undoList.push_back(hashEntry); - hashEntry->SetBlock(diskFile, i*blocksize); - (*availableBlocks)++; + hashEntry->SetBlock(&diskFile, i * blocksize); + ++availableBlocks; } } m_quickFiles++; PrintMessage(Message::mkDetail, "Quickly verified %s file %s", - fileStatus == fsSuccess ? "good" : "damaged", FileSystem::BaseFileName(filename)); + fileStatus == fsSuccess ? "good" : "damaged", FileSystem::BaseFileName(filename.c_str())); return fileStatus; } -bool ParChecker::VerifySuccessDataFile(void* diskfile, void* sourcefile, uint32 downloadCrc) +bool ParChecker::VerifySuccessDataFile(Par2::DiskFile& diskFile, Par2::Par2RepairerSourceFile& sourceFile, uint32 downloadCrc) { - Par2::Par2RepairerSourceFile* sourceFile = (Par2::Par2RepairerSourceFile*)sourcefile; Par2::u64 blocksize = GetRepairer()->mainpacket->BlockSize(); - Par2::VerificationPacket* packet = sourceFile->GetVerificationPacket(); + Par2::VerificationPacket* packet = sourceFile.GetVerificationPacket(); // extend lDownloadCrc to block size downloadCrc = Par2::CRCUpdateBlock(downloadCrc ^ 0xFFFFFFFF, - (size_t)(blocksize * packet->BlockCount() > sourceFile->GetTargetFile()->FileSize() ? - blocksize * packet->BlockCount() - sourceFile->GetTargetFile()->FileSize() : 0) + (size_t)(blocksize * packet->BlockCount() > sourceFile.GetTargetFile()->FileSize() ? + blocksize * packet->BlockCount() - sourceFile.GetTargetFile()->FileSize() : 0) ) ^ 0xFFFFFFFF; debug("Download-CRC: %.8x", downloadCrc); @@ -1408,23 +1277,23 @@ bool ParChecker::VerifySuccessDataFile(void* diskfile, void* sourcefile, uint32 Par2::u32 blockCrc = entry->crc; parCrc = i == 0 ? blockCrc : Crc32::Combine(parCrc, blockCrc, (uint32)blocksize); } - debug("Block-CRC: %x, filename: %s", parCrc, FileSystem::BaseFileName(sourceFile->GetTargetFile()->FileName().c_str())); + std::string fileName = sourceFile.GetTargetFile()->FileName(); + debug("Block-CRC: %x, filename: %s", parCrc, FileSystem::BaseFileName(fileName.c_str())); return parCrc == downloadCrc; } -bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, SegmentList* segments, ValidBlocks* validBlocks) +bool ParChecker::VerifyPartialDataFile(Par2::DiskFile& diskFile, Par2::Par2RepairerSourceFile& sourceFile, SegmentList& segments, ValidBlocks& validBlocks) { - Par2::Par2RepairerSourceFile* sourceFile = (Par2::Par2RepairerSourceFile*)sourcefile; - Par2::VerificationPacket* packet = sourceFile->GetVerificationPacket(); + Par2::VerificationPacket* packet = sourceFile.GetVerificationPacket(); int64 blocksize = GetRepairer()->mainpacket->BlockSize(); - std::string filenameObj = sourceFile->GetTargetFile()->FileName(); + std::string filenameObj = sourceFile.GetTargetFile()->FileName(); const char* filename = filenameObj.c_str(); - int64 fileSize = sourceFile->GetTargetFile()->FileSize(); + int64 fileSize = sourceFile.GetTargetFile()->FileSize(); // determine presumably valid and bad blocks based on article download status - validBlocks->resize(packet->BlockCount(), false); - for (int i = 0; i < (int)validBlocks->size(); i++) + validBlocks.resize(packet->BlockCount(), false); + for (size_t i = 0; i < validBlocks.size(); ++i) { int64 blockStart = i * blocksize; int64 blockEnd = blockStart + blocksize < fileSize - 1 ? blockStart + blocksize : fileSize - 1; @@ -1454,7 +1323,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment curOffset = segment.GetOffset() + segment.GetSize(); } } - validBlocks->at(i) = blockOK && blockEndFound; + validBlocks[i] = blockOK && blockEndFound; } DiskFile infile; @@ -1472,10 +1341,10 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment // let libpar2 do the full verification of the file in this case. uint32 parCrc = 0; int blockStart = -1; - validBlocks->push_back(false); // end marker - for (int i = 0; i < (int)validBlocks->size(); i++) + validBlocks.push_back(false); // end marker + for (size_t i = 0; i < validBlocks.size(); ++i) { - bool validBlock = validBlocks->at(i); + bool validBlock = validBlocks[i]; if (validBlock) { if (blockStart == -1) @@ -1495,7 +1364,7 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment int64 bytesEnd = blockEnd * blocksize + blocksize - 1; uint32 downloadCrc = 0; bool ok = SmartCalcFileRangeCrc(infile, bytesStart, - bytesEnd < fileSize - 1 ? bytesEnd : fileSize - 1, segments, &downloadCrc); + bytesEnd < fileSize - 1 ? bytesEnd : fileSize - 1, segments, downloadCrc); if (ok && bytesEnd > fileSize - 1) { // for the last block: extend lDownloadCrc to block size @@ -1521,8 +1390,8 @@ bool ParChecker::VerifyPartialDataFile(void* diskfile, void* sourcefile, Segment * Compute CRC of bytes range of file using CRCs of segments and reading some data directly * from file if necessary */ -bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, SegmentList* segments, - uint32* downloadCrcOut) +bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, SegmentList& segments, + uint32& downloadCrcOut) { uint32 downloadCrc = 0; bool started = false; @@ -1531,7 +1400,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S if (!started && segment.GetOffset() > start) { // read start of range from file - if (!DumbCalcFileRangeCrc(file, start, segment.GetOffset() - 1, &downloadCrc)) + if (!DumbCalcFileRangeCrc(file, start, segment.GetOffset() - 1, downloadCrc)) { return false; } @@ -1557,7 +1426,7 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S { // read end of range from file uint32 partialCrc = 0; - if (!DumbCalcFileRangeCrc(file, segment.GetOffset(), end, &partialCrc)) + if (!DumbCalcFileRangeCrc(file, segment.GetOffset(), end, partialCrc)) { return false; } @@ -1568,14 +1437,14 @@ bool ParChecker::SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, S } } - *downloadCrcOut = downloadCrc; + downloadCrcOut = downloadCrc; return true; } /* * Compute CRC of bytes range of file reading the data directly from file */ -bool ParChecker::DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32* downloadCrcOut) +bool ParChecker::DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32& downloadCrcOut) { if (!file.Seek(start)) { @@ -1594,26 +1463,24 @@ bool ParChecker::DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, ui start += cnt; } - *downloadCrcOut = downloadCrc.Finish(); + downloadCrcOut = downloadCrc.Finish(); return true; } -CString ParChecker::GetPacketCreator() +std::string ParChecker::GetPacketCreator() { Par2::CREATORPACKET* creatorpacket; if (GetRepairer()->creatorpacket && (creatorpacket = (Par2::CREATORPACKET*)(((RepairCreatorPacket*)GetRepairer()->creatorpacket)->packetdata))) { - int len = (int)(creatorpacket->header.length - sizeof(Par2::PACKET_HEADER)); - BString<1024> creator; + size_t len = (creatorpacket->header.length - sizeof(Par2::PACKET_HEADER)); if (len > 0) { - creator.Set((const char*)creatorpacket->client, len); + return std::string((const char*)creatorpacket->client); } - return *creator; } - return nullptr; + return ""; } #endif diff --git a/daemon/postprocess/ParChecker.h b/daemon/postprocess/ParChecker.h index fbc82adee..7f340b1ce 100644 --- a/daemon/postprocess/ParChecker.h +++ b/daemon/postprocess/ParChecker.h @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2007-2019 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -23,6 +24,11 @@ #ifndef DISABLE_PARCHECK +#include +#include + +#include + #include "NString.h" #include "Container.h" #include "FileSystem.h" @@ -33,6 +39,8 @@ class Repairer; class ParChecker { public: + virtual ~ParChecker() = default; + enum EStatus { psFailed, @@ -57,11 +65,11 @@ class ParChecker }; void Execute(); - void SetDestDir(const char* destDir) { m_destDir = destDir; } - const char* GetParFilename() { return m_parFilename; } - const char* GetInfoName() { return m_infoName; } - void SetInfoName(const char* infoName) { m_infoName = infoName; } - void SetNzbName(const char* nzbName) { m_nzbName = nzbName; } + void SetDestDir(const char* destDir) { m_destDir = destDir ? destDir : ""; } + const char* GetParFilename() { return m_parFilename.c_str(); } + const char* GetInfoName() { return m_infoName.c_str(); } + void SetInfoName(const char* infoName) { m_infoName = infoName ? infoName : ""; } + void SetNzbName(const char* nzbName) { m_nzbName = nzbName ? nzbName : ""; } void SetParQuick(bool parQuick) { m_parQuick = parQuick; } bool GetParQuick() { return m_parQuick; } void SetForceRepair(bool forceRepair) { m_forceRepair = forceRepair; } @@ -77,8 +85,12 @@ class ParChecker class Segment { public: - Segment(bool success, int64 offset, int size, uint32 crc) : - m_success(success), m_offset(offset), m_size(size), m_crc(crc) {} + Segment(bool success, int64 offset, int size, uint32 crc) + : m_success(success) + , m_offset(offset) + , m_size(size) + , m_crc(crc) + {} bool GetSuccess() { return m_success; } int64 GetOffset() { return m_offset; } int GetSize() { return m_size; } @@ -96,16 +108,18 @@ class ParChecker class DupeSource { public: - DupeSource(int id, const char* directory) : - m_id(id), m_directory(directory) {} + DupeSource(int id, const char* directory) + : m_id(id) + , m_directory(directory ? directory : "") + {} int GetId() { return m_id; } - const char* GetDirectory() { return m_directory; } + const char* GetDirectory() { return m_directory.c_str(); } int GetUsedBlocks() { return m_usedBlocks; } void SetUsedBlocks(int usedBlocks) { m_usedBlocks = usedBlocks; } private: int m_id; - CString m_directory; + std::string m_directory; int m_usedBlocks = 0; }; @@ -136,7 +150,7 @@ class ParChecker virtual void RequestDupeSources(DupeSourceList* dupeSourceList) {} virtual void StatDupeSources(DupeSourceList* dupeSourceList) {} EStage GetStage() { return m_stage; } - const char* GetProgressLabel() { return m_progressLabel; } + const char* GetProgressLabel() { return m_progressLabel.c_str(); } int GetFileProgress() { return m_fileProgress; } int GetStageProgress() { return m_stageProgress; } @@ -149,45 +163,47 @@ class ParChecker private: ParChecker* m_owner; Message::EKind m_kind; - StringBuilder m_buffer; + std::string m_buffer; }; - typedef std::deque FileList; + typedef std::deque FileList; typedef std::deque SourceList; typedef std::vector ValidBlocks; - CString m_infoName; - CString m_destDir; - CString m_nzbName; - const char* m_parFilename = nullptr; - EStatus m_status = psFailed; - EStage m_stage; - CString m_errMsg; - FileList m_queuedParFiles; - Mutex m_queuedParFilesMutex; bool m_queuedParFilesChanged; - FileList m_processedFiles; + bool m_verifyingExtraFiles; + bool m_cancelled; + bool m_hasDamagedFiles; + bool m_parQuick = false; + bool m_forceRepair = false; + bool m_parFull = false; int m_processedCount; int m_filesToRepair; int m_extraFiles; int m_quickFiles; - bool m_verifyingExtraFiles; - CString m_progressLabel; int m_fileProgress; int m_stageProgress; - bool m_cancelled; + std::string m_infoName; + std::string m_destDir; + std::string m_nzbName; + std::string m_parFilename; + std::string m_progressLabel; + std::string m_errMsg; + EStatus m_status = psFailed; + EStage m_stage; + FileList m_queuedParFiles; + FileList m_processedFiles; SourceList m_sourceFiles; - std::string m_lastFilename; - bool m_hasDamagedFiles; - bool m_parQuick = false; - bool m_forceRepair = false; - bool m_parFull = false; DupeSourceList m_dupeSources; StreamBuf m_parOutStream{this, Message::mkDetail}; StreamBuf m_parErrStream{this, Message::mkError}; std::ostream m_parCout{&m_parOutStream}; std::ostream m_parCerr{&m_parErrStream}; - Mutex m_repairerMutex; + std::mutex m_queuedParFilesMtx; + std::mutex m_repairerMtx; + std::mutex m_sigFileNameMtx; + std::mutex m_sigProgressMtx; + std::mutex m_sigDoneMtx; // "m_repairer" should be of type "Par2::Par2Repairer", however to prevent the // including of libpar2-headers into this header-file we use an empty abstract class. @@ -208,20 +224,17 @@ class ParChecker bool IsProcessedFile(const char* filename); void SaveSourceList(); void DeleteLeftovers(); - void signal_filename(std::string str); - void signal_progress(int progress); - void signal_done(std::string str, int available, int total); - // declared as void* to prevent the including of libpar2-headers into this header-file - // Par2::DiskFile* pDiskfile, Par2::Par2RepairerSourceFile* pSourcefile - EFileStatus VerifyDataFile(void* diskfile, void* sourcefile, int* availableBlocks); - bool VerifySuccessDataFile(void* diskfile, void* sourcefile, uint32 downloadCrc); - bool VerifyPartialDataFile(void* diskfile, void* sourcefile, SegmentList* segments, ValidBlocks* validBlocks); - void SortExtraFiles(void* extrafiles); - bool SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, SegmentList* segments, - uint32* downloadCrc); - bool DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32* downloadCrc); + void SignalFilename(std::string str); + void SignalProgress(int progress); + void SignalDone(std::string str, int available, int total); + EFileStatus VerifyDataFile(Par2::DiskFile& diskFile, Par2::Par2RepairerSourceFile& sourceFile, int& availableBlocks); + bool VerifySuccessDataFile(Par2::DiskFile& diskFile, Par2::Par2RepairerSourceFile& sourceFile, uint32 downloadCrc); + bool VerifyPartialDataFile(Par2::DiskFile& diskFile, Par2::Par2RepairerSourceFile& sourceFile, SegmentList& segments, ValidBlocks& validBlocks); + void SortExtraFiles(std::vector& extrafiles); + bool SmartCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, SegmentList& segments, uint32& downloadCrc); + bool DumbCalcFileRangeCrc(DiskFile& file, int64 start, int64 end, uint32& downloadCrc); void CheckEmptyFiles(); - CString GetPacketCreator(); + std::string GetPacketCreator(); bool MaybeSplittedFragement(const char* filename1, const char* filename2); friend class Repairer; diff --git a/daemon/postprocess/ParParser.cpp b/daemon/postprocess/ParParser.cpp index 6145b91fe..69535f070 100644 --- a/daemon/postprocess/ParParser.cpp +++ b/daemon/postprocess/ParParser.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -42,9 +43,9 @@ bool ParParser::FindMainPars(const char* path, ParFileList* fileList) // check if the base file already added to list bool exists = false; - for (CString& filename2 : fileList) + for (std::string& filename2 : *fileList) { - exists = SameParCollection(filename, filename2, true); + exists = SameParCollection(filename, filename2.c_str(), true); if (exists) { break; diff --git a/daemon/postprocess/ParParser.h b/daemon/postprocess/ParParser.h index b58b9d983..7d3ece94a 100644 --- a/daemon/postprocess/ParParser.h +++ b/daemon/postprocess/ParParser.h @@ -21,13 +21,13 @@ #ifndef PARPARSER_H #define PARPARSER_H -#include "NString.h" +#include #include "Container.h" class ParParser { public: - typedef std::vector ParFileList; + typedef std::vector ParFileList; static bool FindMainPars(const char* path, ParFileList* fileList); static bool ParseParFilename(const char* parFilename, bool confirmedFilename, int* baseNameLen, int* blocks); diff --git a/daemon/postprocess/ParRenamer.cpp b/daemon/postprocess/ParRenamer.cpp index 41b615362..fdb7030d4 100644 --- a/daemon/postprocess/ParRenamer.cpp +++ b/daemon/postprocess/ParRenamer.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2013-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -22,9 +23,9 @@ #ifndef DISABLE_PARCHECK -#include "par2cmdline.h" -#include "par2repairer.h" -#include "md5.h" +#include +#include +#include #include "ParRenamer.h" #include "ParParser.h" @@ -33,10 +34,10 @@ #include "Util.h" #include "FileSystem.h" -class ParRenamerRepairer : public Par2::Par2Repairer +class ParRenamerRepairer final : public Par2::Par2Repairer { public: - ParRenamerRepairer() : Par2::Par2Repairer(m_nout, m_nout) {}; + ParRenamerRepairer() : Par2::Par2Repairer(m_nout, m_nout, Par2::nlNormal) {}; friend class ParRenamer; private: class NullStreamBuf : public std::streambuf {}; @@ -112,9 +113,9 @@ void ParRenamer::LoadMainParFiles(const char* destDir) ParParser::ParFileList parFileList; ParParser::FindMainPars(destDir, &parFileList); - for (CString& parFilename : parFileList) + for (std::string& parFilename : parFileList) { - BString<1024> fullParFilename("%s%c%s", destDir, PATH_SEPARATOR, *parFilename); + BString<1024> fullParFilename("%s%c%s", destDir, PATH_SEPARATOR, parFilename.c_str()); LoadParFile(fullParFilename); } } @@ -177,7 +178,7 @@ void ParRenamer::LoadParFile(const char* parFilename) m_hasDamagedParFiles = true; continue; } - std::string filename = Par2::DiskFile::TranslateFilename(sourceFile->GetDescriptionPacket()->FileName()); + std::string filename = sourceFile->GetDescriptionPacket()->FileName(); std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print(); bool exists = std::find_if(m_fileHashList.begin(), m_fileHashList.end(), diff --git a/daemon/postprocess/Repair.h b/daemon/postprocess/Repair.h index 5ef1c30de..783bd0a3f 100644 --- a/daemon/postprocess/Repair.h +++ b/daemon/postprocess/Repair.h @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2007-2019 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -48,7 +49,7 @@ class RepairController : public Thread, public ScriptController bool RequestMorePars(NzbInfo* nzbInfo, const char* parFilename, int blockNeeded, int* blockFound); private: - class PostParChecker: public ParChecker + class PostParChecker final : public ParChecker { public: PostInfo* GetPostInfo() { return m_postInfo; } @@ -67,7 +68,7 @@ class RepairController : public Thread, public ScriptController virtual void PrintMessage(Message::EKind kind, const char* format, ...) PRINTF_SYNTAX(3); virtual void RegisterParredFile(const char* filename); virtual bool IsParredFile(const char* filename); - virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments); + EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments) override; virtual const char* FindFileOrigname(const char* filename); virtual void RequestDupeSources(DupeSourceList* dupeSourceList); virtual void StatDupeSources(DupeSourceList* dupeSourceList); diff --git a/daemon/queue/DirectRenamer.cpp b/daemon/queue/DirectRenamer.cpp index 0d178c030..714a5c02a 100644 --- a/daemon/queue/DirectRenamer.cpp +++ b/daemon/queue/DirectRenamer.cpp @@ -19,15 +19,17 @@ #include "nzbget.h" + #include "DirectRenamer.h" #include "Options.h" #include "FileSystem.h" #include "ParParser.h" #ifndef DISABLE_PARCHECK -#include "par2cmdline.h" -#include "par2fileformat.h" -#include "md5.h" +#include +#include +#include +#include #endif class RenameContentAnalyzer : public ArticleContentAnalyzer @@ -55,7 +57,7 @@ class RenameContentAnalyzer : public ArticleContentAnalyzer class DirectParRepairer : public Par2::Par2Repairer { public: - DirectParRepairer() : Par2::Par2Repairer(m_nout, m_nout) {}; + DirectParRepairer() : Par2::Par2Repairer(m_nout, m_nout, Par2::nlNormal) {}; friend class DirectParLoader; private: @@ -155,7 +157,7 @@ void DirectParLoader::LoadParFile(const char* parFile) nzbInfo->PrintMessage(Message::mkWarning, "Damaged par2-file detected: %s", FileSystem::BaseFileName(parFile)); return; } - std::string filename = Par2::DiskFile::TranslateFilename(sourceFile->GetDescriptionPacket()->FileName()); + std::string filename = sourceFile->GetDescriptionPacket()->FileName(); std::string hash = sourceFile->GetDescriptionPacket()->Hash16k().print(); debug("file: %s, hash-16k: %s", filename.c_str(), hash.c_str()); diff --git a/daemon/queue/NzbFile.cpp b/daemon/queue/NzbFile.cpp index 6c87e2614..18ec3db90 100644 --- a/daemon/queue/NzbFile.cpp +++ b/daemon/queue/NzbFile.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -41,7 +42,7 @@ NzbFile::NzbFile(const char* fileName, const char* category) : void NzbFile::LogDebugInfo() { - info(" NZBFile %s", *m_fileName); + info(" NZBFile %s", m_fileName.c_str()); } void NzbFile::AddArticle(FileInfo* fileInfo, std::unique_ptr articleInfo) @@ -389,11 +390,7 @@ void NzbFile::ProcessFiles() } } - if (m_password) - { - ReadPasswordFromNzb(); - } - else + if (m_password.empty()) { ReadPasswordFromFilename(); } @@ -403,62 +400,18 @@ void NzbFile::ProcessFiles() */ void NzbFile::ReadPasswordFromFilename() { - int startDelimiter = m_fileName.Find("{{") + 2; - int stopDelimiter = m_fileName.Find("}}"); - int lengthDelimiter = stopDelimiter - startDelimiter; - if (lengthDelimiter > 0) - { - CString namepassword(m_fileName, m_fileName.Length()); - namepassword.Replace(stopDelimiter, m_fileName.Length() - stopDelimiter, "", 0); - namepassword.Replace(0, startDelimiter, "", 0); - m_password = namepassword.Str(); - } -} -/** - * Password read using XML-parser may have special characters (such as TAB) stripped. - * This function rereads password directly from file to keep all characters intact. - */ -void NzbFile::ReadPasswordFromNzb() -{ - DiskFile file; - if (!file.Open(m_fileName, DiskFile::omRead)) - { - return; - } + size_t start = m_fileName.find("{{"); + if (start == std::string::npos) return; + + start += 2; - // obtain file size. - file.Seek(0, DiskFile::soEnd); - int size = (int)file.Position(); - file.Seek(0, DiskFile::soSet); + size_t end = m_fileName.find("}}", start); + if (end == std::string::npos) return; - // reading first 4KB of the file - - CharBuffer buf(4096); - - size = size < 4096 ? size : 4096; - - // copy the file into the buffer. - file.Read(buf, size); - - file.Close(); - - buf[size-1] = '\0'; - - char* metaPassword = strstr(buf, ""); - if (metaPassword) - { - metaPassword += 22; // length of '' - char* end = strstr(metaPassword, ""); - if (end) - { - *end = '\0'; - WebUtil::XmlDecode(metaPassword); - m_password = metaPassword; - } - } + if (start < end) + m_password = m_fileName.substr(start, end - start); } - bool NzbFile::Parse() { xmlSAXHandler SAX_handler = {0}; @@ -470,19 +423,19 @@ bool NzbFile::Parse() m_ignoreNextError = false; - int ret = xmlSAXUserParseFile(&SAX_handler, this, m_fileName); + int ret = xmlSAXUserParseFile(&SAX_handler, this, m_fileName.c_str()); if (ret != 0) { m_nzbInfo->AddMessage(Message::mkError, BString<1024>( - "Error parsing nzb-file %s", FileSystem::BaseFileName(m_fileName))); + "Error parsing nzb-file %s", FileSystem::BaseFileName(m_fileName.c_str()))); return false; } if (m_nzbInfo->GetFileList()->empty()) { m_nzbInfo->AddMessage(Message::mkError, BString<1024>( - "Error parsing nzb-file %s: file has no content", FileSystem::BaseFileName(m_fileName))); + "Error parsing nzb-file %s: file has no content", FileSystem::BaseFileName(m_fileName.c_str()))); return false; } @@ -500,7 +453,7 @@ void NzbFile::Parse_StartElement(const char *name, const char **atts) if (!strcmp("file", name)) { m_fileInfo = std::make_unique(); - m_fileInfo->SetFilename(m_fileName); + m_fileInfo->SetFilename(m_fileName.c_str()); if (!atts) { diff --git a/daemon/queue/NzbFile.h b/daemon/queue/NzbFile.h index 069f972ba..759354b49 100644 --- a/daemon/queue/NzbFile.h +++ b/daemon/queue/NzbFile.h @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -15,7 +16,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -30,16 +31,16 @@ class NzbFile public: NzbFile(const char* fileName, const char* category); bool Parse(); - const char* GetFileName() const { return m_fileName; } + const char* GetFileName() const { return m_fileName.c_str(); } std::unique_ptr DetachNzbInfo() { return std::move(m_nzbInfo); } - const char* GetPassword() { return m_password; } + const std::string& GetPassword() const { return m_password; } void LogDebugInfo(); private: std::unique_ptr m_nzbInfo; - CString m_fileName; - CString m_password; + std::string m_fileName; + std::string m_password; void AddArticle(FileInfo* fileInfo, std::unique_ptr articleInfo); void AddFileInfo(std::unique_ptr fileInfo); @@ -48,7 +49,6 @@ class NzbFile void ProcessFiles(); void CalcHashes(); bool HasDuplicateFilenames(); - void ReadPasswordFromNzb(); void ReadPasswordFromFilename(); diff --git a/daemon/queue/Scanner.cpp b/daemon/queue/Scanner.cpp index b34789ed9..909dd7562 100644 --- a/daemon/queue/Scanner.cpp +++ b/daemon/queue/Scanner.cpp @@ -458,9 +458,9 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch nzbInfo->SetSkipDiskWrite(urlInfo->GetSkipDiskWrite()); } - if (nzbFile.GetPassword()) + if (!nzbFile.GetPassword().empty()) { - nzbInfo->GetParameters()->SetParameter("*Unpack:Password", nzbFile.GetPassword()); + nzbInfo->GetParameters()->SetParameter("*Unpack:Password", nzbFile.GetPassword().c_str()); } nzbInfo->GetParameters()->CopyFrom(parameters); diff --git a/daemon/remote/WebServer.cpp b/daemon/remote/WebServer.cpp index 121633fe7..38fab8576 100644 --- a/daemon/remote/WebServer.cpp +++ b/daemon/remote/WebServer.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2012-2017 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -26,11 +27,6 @@ #include "Util.h" #include "FileSystem.h" -#ifndef DISABLE_PARCHECK -#include "par2cmdline.h" -#include "md5.h" -#endif - static const char* ERR_HTTP_OK = "200 OK"; static const char* ERR_HTTP_NOT_MODIFIED = "304 Not Modified"; static const char* ERR_HTTP_BAD_REQUEST = "400 Bad Request"; @@ -475,16 +471,8 @@ void WebProcessor::SendBodyResponse(const char* body, int bodyLen, const char* c { BString<1024> newETag; -#ifndef DISABLE_PARCHECK - Par2::MD5Hash hash; - Par2::MD5Context md5; - md5.Update(body, bodyLen); - md5.Final(hash); - newETag.Format("\"%s\"", hash.print().c_str()); -#else - uint32 hash = Util::HashBJ96(body, bodyLen, 0); + size_t hash = m_hasher(body); newETag.Format("\"%x\"", hash); -#endif unchanged = m_oldETag && !strcmp(newETag, m_oldETag); if (unchanged) diff --git a/daemon/remote/WebServer.h b/daemon/remote/WebServer.h index a88c47da4..81b510509 100644 --- a/daemon/remote/WebServer.h +++ b/daemon/remote/WebServer.h @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2012-2017 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -65,6 +66,7 @@ class WebProcessor CString m_forwardedFor; CString m_oldETag; bool m_keepAlive = false; + std::hash m_hasher; void Dispatch(); void SendAuthResponse(); diff --git a/daemon/sources.cmake b/daemon/sources.cmake index f799d86d0..d29c767ff 100644 --- a/daemon/sources.cmake +++ b/daemon/sources.cmake @@ -105,8 +105,25 @@ set(WIN32_SRC ${CMAKE_SOURCE_DIR}/daemon/windows/StdAfx.cpp ${CMAKE_SOURCE_DIR}/daemon/windows/WinConsole.cpp ${CMAKE_SOURCE_DIR}/daemon/windows/WinService.cpp + ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp ) if(WIN32) set(SRC ${SRC} ${WIN32_SRC}) + set(INCLUDES ${INCLUDES} ${CMAKE_SOURCE_DIR}/windows) endif() + +set(INCLUDES ${INCLUDES} + ${CMAKE_SOURCE_DIR}/daemon/connect + ${CMAKE_SOURCE_DIR}/daemon/extension + ${CMAKE_SOURCE_DIR}/daemon/feed + ${CMAKE_SOURCE_DIR}/daemon/frontend + ${CMAKE_SOURCE_DIR}/daemon/main + ${CMAKE_SOURCE_DIR}/daemon/nntp + ${CMAKE_SOURCE_DIR}/daemon/nserv + ${CMAKE_SOURCE_DIR}/daemon/postprocess + ${CMAKE_SOURCE_DIR}/daemon/queue + ${CMAKE_SOURCE_DIR}/daemon/remote + ${CMAKE_SOURCE_DIR}/daemon/system + ${CMAKE_SOURCE_DIR}/daemon/util +) diff --git a/daemon/system/OS.cpp b/daemon/system/OS.cpp index 6cfe882d3..068f95283 100644 --- a/daemon/system/OS.cpp +++ b/daemon/system/OS.cpp @@ -74,6 +74,7 @@ namespace System if (buildNum >= m_win11BuildVersion) m_version = "11"; else if (buildNum >= m_win10BuildVersion) m_version = "10"; else if (buildNum >= m_win8BuildVersion) m_version = "8"; + else if (buildNum >= m_win7BuildVersion) m_version = "7"; else if (buildNum >= m_winXPBuildVersion) m_version = "XP"; else { diff --git a/daemon/system/SystemInfo.cpp b/daemon/system/SystemInfo.cpp index 308d953e7..89942b6d9 100644 --- a/daemon/system/SystemInfo.cpp +++ b/daemon/system/SystemInfo.cpp @@ -35,10 +35,13 @@ #ifdef HAVE_NCURSES_NCURSES_H #include #endif +#ifndef DISABLE_PARCHECK +#include +#endif namespace System { - static const size_t BUFFER_SIZE = 512; + constexpr size_t BUFFER_SIZE = 512; SystemInfo::SystemInfo() { @@ -79,6 +82,10 @@ namespace System m_libraries.push_back({ "GnuTLS", GNUTLS_VERSION }); #endif +#ifndef DISABLE_PARCHECK + m_libraries.push_back({ "Par2-turbo", PAR2_TURBO_LIB_VERSION }); +#endif + #ifdef BOOST_LIB_VERSION m_libraries.push_back({ "Boost", BOOST_LIB_VERSION }); #endif @@ -162,7 +169,7 @@ namespace System Tool tool; tool.name = "UnRAR"; - tool.path = GetUnpackerPath(g_Options->GetUnrarCmd()); + tool.path = GetToolPath(g_Options->GetUnrarCmd()); tool.version = GetUnpackerVersion(tool.path, "UNRAR"); return tool; @@ -173,20 +180,20 @@ namespace System Tool tool; tool.name = "7-Zip"; - tool.path = GetUnpackerPath(g_Options->GetSevenZipCmd()); + tool.path = GetToolPath(g_Options->GetSevenZipCmd()); tool.version = GetUnpackerVersion(tool.path, tool.name.c_str()); return tool; } - std::string SystemInfo::GetUnpackerPath(const char* unpackerCmd) const + std::string SystemInfo::GetToolPath(const char* cmd) const { - if (Util::EmptyStr(unpackerCmd)) + if (Util::EmptyStr(cmd)) { return ""; } - std::string path = FileSystem::ExtractFilePathFromCmd(unpackerCmd); + std::string path = FileSystem::ExtractFilePathFromCmd(cmd); auto result = FileSystem::GetFileRealPath(path); @@ -195,11 +202,16 @@ namespace System return result.value(); } - std::string cmd = Util::FIND_CMD + std::string(unpackerCmd); + std::string fullCmd = Util::FIND_CMD + std::string(cmd); + + auto pipe = Util::MakePipe(fullCmd); + if (!pipe) + { + return ""; + } char buffer[BUFFER_SIZE]; - auto pipe = Util::MakePipe(cmd); - if (pipe && fgets(buffer, BUFFER_SIZE, pipe.get())) + if (fgets(buffer, BUFFER_SIZE, pipe.get())) { std::string resultPath{ buffer }; Util::Trim(resultPath); @@ -251,9 +263,14 @@ namespace System std::string cmd = Util::FIND_CMD + result.value(); - char buffer[BUFFER_SIZE]; auto pipe = Util::MakePipe(cmd); - if (pipe && fgets(buffer, BUFFER_SIZE, pipe.get())) + if (!pipe) + { + return std::nullopt; + } + + char buffer[BUFFER_SIZE]; + if (fgets(buffer, BUFFER_SIZE, pipe.get())) { std::string path{ buffer }; Util::Trim(path); @@ -268,8 +285,13 @@ namespace System std::string cmd = FileSystem::EscapePathForShell(path) + " --version 2>&1"; { auto pipe = Util::MakePipe(cmd); + if (!pipe) + { + return ""; + } + char buffer[BUFFER_SIZE]; - if (pipe && fgets(buffer, BUFFER_SIZE, pipe.get())) + if (fgets(buffer, BUFFER_SIZE, pipe.get())) { // e.g. Python 3.12.3 std::string version{ buffer }; @@ -298,7 +320,18 @@ namespace System return match[0].str(); } - return std::string(""); + return ""; + }; + + std::string SystemInfo::ParseParVersion(const std::string& line) const + { + // e.g. par2cmdline-turbo version 1.1.1 + if (size_t pos = line.find_last_of(" "); pos != std::string::npos) + { + return line.substr(pos); + } + + return ""; }; std::ostream& operator<<(std::ostream& os, const SystemInfo& sysinfo) diff --git a/daemon/system/SystemInfo.h b/daemon/system/SystemInfo.h index 78fb007f9..3792e75e5 100644 --- a/daemon/system/SystemInfo.h +++ b/daemon/system/SystemInfo.h @@ -58,8 +58,9 @@ namespace System Tool GetUnrar() const; void InitLibsInfo(); std::string ParseUnpackerVersion(const std::string& line) const; - std::string GetUnpackerPath(const char* unpackerCmd) const; - std::string GetUnpackerVersion(const std::string& path, const char* marker) const; + std::string ParseParVersion(const std::string& line) const; + std::string GetToolPath(const char* cmd) const; + std::string GetUnpackerVersion(const std::string& path, const char* marker) const;\ std::optional FindPython() const; std::optional GetPythonVersion(const std::string path) const; diff --git a/daemon/util/Utf8.cpp b/daemon/util/Utf8.cpp new file mode 100644 index 000000000..267884f8b --- /dev/null +++ b/daemon/util/Utf8.cpp @@ -0,0 +1,60 @@ + +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include "nzbget.h" + +#include + +#include "Utf8.h" +#include "Log.h" + +namespace Utf8 +{ + std::optional Utf8ToWide(const std::string& str) noexcept + { + if (str.empty()) return L""; + + int requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, nullptr, 0); + if (requiredSize <= 0) return std::nullopt; + + std::wstring wstr(requiredSize, '\0'); + + requiredSize = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), -1, wstr.data(), requiredSize); + if (requiredSize <= 0) return std::nullopt; + + return wstr; + } + + std::optional WideToUtf8(const std::wstring& wstr) noexcept + { + if (wstr.empty()) return ""; + + int requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, nullptr, 0, nullptr, nullptr); + if (requiredSize <= 0) return std::nullopt; + + std::string str(requiredSize, '\0'); + + requiredSize = WideCharToMultiByte(CP_UTF8, 0, wstr.c_str(), -1, str.data(), requiredSize, nullptr, nullptr); + if (requiredSize <= 0) return std::nullopt; + + return str; + } +} diff --git a/daemon/util/Utf8.h b/daemon/util/Utf8.h new file mode 100644 index 000000000..0e4003eef --- /dev/null +++ b/daemon/util/Utf8.h @@ -0,0 +1,33 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef UTF8_H +#define UTF8_H + +#include +#include + +namespace Utf8 +{ + std::optional Utf8ToWide(const std::string& str) noexcept; + std::optional WideToUtf8(const std::wstring& wstr) noexcept; +} + +#endif diff --git a/daemon/util/Util.cpp b/daemon/util/Util.cpp index a371b77e8..a17d590a9 100644 --- a/daemon/util/Util.cpp +++ b/daemon/util/Util.cpp @@ -855,14 +855,7 @@ void Util::SetStandByMode(bool standBy) int Util::NumberOfCpuCores() { -#ifdef WIN32 - SYSTEM_INFO sysinfo; - GetSystemInfo(&sysinfo); - return sysinfo.dwNumberOfProcessors; -#elif HAVE_SC_NPROCESSORS_ONLN - return sysconf(_SC_NPROCESSORS_ONLN); -#endif - return -1; + return std::thread::hardware_concurrency(); } int64 Util::CurrentTicks() @@ -884,13 +877,9 @@ int64 Util::CurrentTicks() #endif } -void Util::Sleep(int milliseconds) +void Util::Sleep(int ms) { -#ifdef WIN32 - ::Sleep(milliseconds); -#else - usleep(milliseconds * 1000); -#endif + std::this_thread::sleep_for(std::chrono::milliseconds(ms)); } uint32 WebUtil::DecodeBase64(char* inputBuffer, int inputBufferLength, char* outputBuffer) diff --git a/docker/Dockerfile b/docker/Dockerfile index b8ddad00a..ffa5c7c8b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -101,4 +101,5 @@ RUN chmod +x /entrypoint.sh && \ mkdir -p /config && \ mkdir -p /downloads && \ chown -R user:users /app /config /downloads +EXPOSE 6789 ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docs/HOW_TO_USE.md b/docs/HOW_TO_USE.md index e4d4d9b85..ff522dc6b 100644 --- a/docs/HOW_TO_USE.md +++ b/docs/HOW_TO_USE.md @@ -80,12 +80,6 @@ To stop server use: ``` nzbget -Q ``` -TIP for POSIX users: with included script "nzbgetd" you can use standard -commands to control daemon: -``` -nzbgetd start -nzbgetd stop -``` When NZBGet is started in console server mode it displays a message that it is ready to receive download requests. In daemon mode it doesn't print any @@ -154,6 +148,54 @@ or: nzbget -o createlog=no -C ``` +### Running nzbget as systemd service (Linux only) + +It is possible to run nzbget as a systemd service under modern Linux distributions. +To do that, create `nzbget.service` file in `/etc/systemd/system/`. Example service files (replace `/path/to/nzbget` with real path, and `nzbget_user`/`nzbget_group` with real user/group): + +- nzbget runs as a daemon and logs to own log file: +``` +[Unit] +Description=NZBGet Service +After=network.target + +[Service] +Type=forking +ExecStart=/path/to/nzbget -D +ExecStop=/path/to/nzbget -Q +User=nzbget_user +Group=nzbget_group +Restart=on-failure + +[Install] +WantedBy=multi-user.target +``` + +- nzbget runs as a server with logging to stdout, logs are forwarded to journald +``` +[Unit] +Description=NZBGet Service +After=network.target + +[Service] +Type=simple +ExecStart=/path/to/nzbget -s -o OutputMode=log +ExecStop=/path/to/nzbget -Q +User=nzbget_user +Group=nzbget_group +Restart=on-failure + +[Install] +WantedBy=multi-user.target +``` + +Then run (start nzbget and enable autostart): +``` +sudo systemctl daemon-reload +sudo systemctl enable nzbget +sudo systemctl start nzbget +``` + ### Running client & server on seperate machines: ------------------------------------------------- diff --git a/lib/par2/AUTHORS b/lib/par2/AUTHORS deleted file mode 100644 index cdab0d512..000000000 --- a/lib/par2/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -par2cmdline 0.4: Peter Brian Clements -library API: Francois Lesueur -changes for NZBGet: Andrey Prygunkov \ No newline at end of file diff --git a/lib/par2/README b/lib/par2/README deleted file mode 100644 index f43869f4a..000000000 --- a/lib/par2/README +++ /dev/null @@ -1,326 +0,0 @@ -Based on libpar2 v 0.4 (http://launchpad.net/libpar2), which -is a Debian/Ubuntu fork of original libpar2 (http://parchive.sourceforge.net), -which is a library for par2 based on par2cmdline 0.4. - -Changes made to libpar2-0.4: - - removed files not required for NZBGet; - - eliminated dependency from libsigc++ by changing of library - API to use virtual functions instead of sigc++ signals. - -par2cmdline README follows : - -------------------------------------------------------------- - -par2cmdline is a PAR 2.0 compatible file verification and repair tool. - -See http://parchive.sourceforge.net for details of PAR 2.0 specification -and discussion of all things PAR. - -WHAT EXACTLY IS PAR2CMDLINE? - -par2cmdline is a program for creating and using PAR2 files to detect -damage in data files and repair them if necessary. It can be used with -any kind of file. - -WHY IS PAR 2.0 better than PAR 1.0? - - * It is not necessary to split a single large file into many equally - size small files (although you can still do so if you wish). - - * There is no loss of efficiency when operating on multiple files - of different sizes. - - * It is possible to repair damaged files (using exactly the amount of - recovery data that corresponds to the amount of damage), rather than - requiring the complete reconstruction of the damaged file. - - * Recovery files may be of different sizes making it possible to - obtain exactly the amount of recovery data required to carry out - a repair. - - * Because damaged data files are still useable during the recovery - process, less recovery data is required to achieve a successfull - repair. It is not therefore necessary to create as much recovery - data in the first place to achieve the same level of protection. - - * You can protect up to 32768 files rather than the 256 that PAR 1.0 - is limited to. - - * Damaged or incomplete recovery files can also be used during the - recovery process in the same way that damaged data files can. - - * You require less recovery data to provide the same level of protection - from damage compared with PAR 1.0. - -DOES PAR 2.0 HAVE ANY DISSADVANTAGES? - -Yes, there is one dissadvantage: - - * All PAR 2.0 program will take somewhat longer to create recovery - files than a PAR 1.0 program does. - -This dissadvantage is considerably mitigated by the fact that you don't -need to create as much recovery data in the first place to provide the -same level of protection against loss and damage. - -COMPILING PAR2CMDLINE - -You should have received par2cmdline in the form of source code which -you can compile on your computer. You may optionally have received a -pre-compiled version of the program for your operating system. - -If you have only downloaded a precompiled executable, then the source -code should be available from the same location that you downloaded the -executable from. - -If you have MS Visual Studio .NET, then just open the par2cmdline.sln -file and compile. You should then copy par2cmdline.exe to an appropriate -location that is on your path. - -To compile on Linux and other unix variants use the following commands: - - ./configure - make - make check - make install - -See INSTALL for full details of how to use the "configure" script. - -USING PAR2CMDLINE - -The command line parameters for par2cmdline are as follow: - - par2 c(reate) [options] [files] - par2 v(erify) [options] [files] - par2 r(epair) [options] [files] - - Also: - - par2create [options] [files] - par2verify [options] [files] - par2repair [options] [files] - - Options: - - -b : Set the Block-Count - -s : Set the Block-Size (Don't use both -b and -s) - -r : Level of Redundancy (%) - -c : Recovery block count (don't use both -r and -c) - -f : First Recovery-Block-Number - -u : Uniform recovery file sizes - -l : Limit size of recovery files (Don't use both -u and -l) - -n : Number of recovery files (Don't use both -n and -l) - -m : Memory (in MB) to use - -v [-v]: Be more verbose - -q [-q]: Be more quiet (-qq gives silence) - -- : Treat all remaining CommandLine as filenames - -If you wish to create par2 files for a single source file, you may leave -out the name of the par2 file from the command line. par2cmdline will then -assume that you wish to base the filenames for the par2 files on the name -of the source file. - -You may also leave off the .par2 file extension when verifying and repairing. - -CREATING PAR2 FILES - -With PAR 2.0 you can create PAR2 recovery files for as few as 1 or as many as -32768 files. If you wanted to create PAR1 recovery files for a single file -you are forced to split the file into muliple parts and RAR is frequently -used for this purpose. You do NOT need to split files with PAR 2.0. - -To create PAR 2 recovery files for a single data file (e.g. one called -test.mpg), you can use the following command: - - par2 create test.mpg - -If test.mpg is an 800 MB file, then this will create a total of 8 PAR2 files -with the following filenames (taking roughly 6 minutes on a PC with a -1500MHz CPU): - - test.mpg.par2 - This is an index file for verification only - test.mpg.vol00+01.par2 - Recovery file with 1 recovery block - test.mpg.vol01+02.par2 - Recovery file with 2 recovery blocks - test.mpg.vol03+04.par2 - Recovery file with 4 recovery blocks - test.mpg.vol07+08.par2 - Recovery file with 8 recovery blocks - test.mpg.vol15+16.par2 - Recovery file with 16 recovery blocks - test.mpg.vol31+32.par2 - Recovery file with 32 recovery blocks - test.mpg.vol63+37.par2 - Recovery file with 37 recovery blocks - -The test.mpg.par2 file is 39 KB in size and the other files vary in size from -443 KB to 15 MB. - -These par2 files will enable the recovery of up to 100 errors totalling 40 MB -of lost or damaged data from the original test.mpg file when it and the par2 -files are posted on UseNet. - -When posting on UseNet it is recommended that you use the "-s" option to set -a blocksize that is equal to the Article size that you will use to post the -data file. If you wanted to post the test.mpg file using an article size -of 300 KB then the command you would type is: - - par2 create -s307200 test.mpg - -This will create 9 PAR2 files instead of 8, and they will be capable of -correcting up to 134 errors totalling 40 MB. It will take roughly 8 minutes -to create the recovery files this time. - -In both of these two examples, the total quantity of recovery data created -was 40 MB (which is 5% of 800 MB). If you wish to create a greater or lesser -quantity of recovery data, you can use the "-r" option. - -To create 10% recovery data instead of the default of 5% and also to use a -block size of 300 KB, you would use the following command: - - par2 create -s307200 -r10 test.mpg - -This would also create 9 PAR2 files, but they would be able to correct up to -269 errors totalling 80 MB. Since twice as much recovery data is created, it -will take about 16 minutes to do so with a 1500MHz CPU. - -The "-u" and "-n" options can be used to control exactly how many recovery -files are created and how the recovery blocks are distributed amoungst them. -They do not affect the total quantity of recovery data created. - -The "-f" option is used when you create additional recovery data. - -e.g. If you have already created 10% and want another 5% then you migh use -the following command: - - par2 create -s307200 -r5 -f300 test.mpg - -This specifies the same block size (which is a requirement for additional -recovery files), 5% recovery data, and a first block number of 300. - -The "-m" option controls how much memory par2cmdline uses. It defaults to -16 MB unless you override it. - -CREATING PAR2 FILES FOR MULTIPLE DATA FILES - -When creating PAR2 recovery files form multiple data files, you must specify -the base filename to use for the par2 files and the names of all of the data -files. - -If test.mpg had been split into multiple RAR files, then you could use: - - par2 create test.mpg.rar.par2 test.mpg.part*.rar - -The files filename "test.mpg.rar.par2" says what you want the par2 files to -be called and "test.mpg.part*.rar" should select all of the RAR files. - -VERIFYING AND REPAIRING - -When using par2 recovery files to verify or repair the data files from -which they were created, you only need to specify the filename of one -of the par2 files to par2cmdline. - -e.g.: - - par2 verify test.mpg.par2 - -This tells par2cmdline to use the information in test.mpg.par2 to verify the -data files. - -par2cmdline will automatically search for the other par2 files that were -created and use the information they contain to determine the filenames -of the original data files and then to verify them. - -If all of the data files are ok, then par2cmdline will report that repair -will not be required. - -If any of the data files are missing or damaged, par2cmdline will report -the details of what it has found. If the recovery files contain enough -recovery blocks to repair the damage, you will be told that repair is -possible. Otherwise you will be told exactly how many recovery blocks -will be required in order to repair. - -To carry out a repair use the following command: - - par2 repair test.mpg.par2 - -This tells par2cmdline to verify and if possible repair any damaged or -missing files. If a repair is carried out, then each file which is -repaired will be re-verified to confirm that the repair was successful. - -MISSNAMED AND INCOMPLETE DATA FILES - -If any of the recovery files or data files have the wrong filename, then -par2cmdline will not automatically find and scan them. - -To have par2cmdline scan such files, you must include them on the command -line when attempting to verify or repair. - -e.g.: - - par2 r test.mpg.par2 other.mpg - -This tells par2cmdline to scan the file called other.mpg to see if it -contains any data belonging to the original data files. - -If one of the extra files specified in this way is an exact match -for a data file, then the repair process will rename the file so that -it has the correct filename. - -Because par2cmdline is designed to be able to find good data within a -damaged file, it can do the same with incomplete files downloaded from -UseNet. If some of the articles for a file are missing, you should still -download the file and save it to disk for par2cmdline to scan. If you -do this then you may find that you can carry out a repair in a situation -where you would not otherwise have sufficient recovery data. - -You can have par2cmdline scan all files that are in the current directory -using a command such as: - - par2 r test.mpg.par2 * - -WHAT TO DO WHEN YOU ARE TOLD YOU NEED MORE RECOVERY BLOCKS - -If par2cmdline determines that any of the data files are damaged or -missing and finds that there is insufficient recovery data to effect -a repair, you will be told that you need a certain number of recovery -blocks. You can obtain these by downloading additional recovery files. - -In order to make things easy, par2 files have filenames that tell you -exactly how many recovery blocks each one contains. - -Assuming that the following command was used to create recovery data: - - par2 c -b1000 -r5 test.mpg - -Then the recovery files that are created would be called: - - test.mpg.par2 - test.mpg.vol00+01.par2 - test.mpg.vol01+02.par2 - test.mpg.vol03+04.par2 - test.mpg.vol07+08.par2 - test.mpg.vol15+16.par2 - test.mpg.vol31+19.par2 - -The first file in this list does not contain any recovery data, it only -contains information sufficient to verify the data files. - -Each of the other files contains a different number of recovery blocks. -The number after the '+' sign is the number of recovery blocks and the -number preceding the '+' sign is the block number of the first recovery -block in that file. - -If par2cmdline told you that you needed 10 recovery blocks, then you would -need "test.mpg.vol01+02.par2" and "test.mpg.vol07+08.par". You might of course -choose to fetch "test.mpg.vol15+16.par2" instead (in which case you would have -an extra 6 recovery blocks which would not be used for the repair). - -NOTES - -This version of par2cmdline does not support recording path information for -files. Whilst you can create recovery files for files from multiple locations, -it will expect all files to be in the current directory when verifying and -repairing. This limitation will be corrected in an update. - -REED SOLOMON CODING - -PAR2 uses Reed Solomon Coding to perform its calculations. For details of this -coding technique try the following link: - -``A Tutorial on Reed-Solomon Coding for Fault-Tolerance in RAID-like Systems'' - diff --git a/lib/par2/commandline.cpp b/lib/par2/commandline.cpp deleted file mode 100644 index 43c326ad4..000000000 --- a/lib/par2/commandline.cpp +++ /dev/null @@ -1,836 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -CommandLine::ExtraFile::ExtraFile(void) -: filename() -, filesize(0) -{ -} - -CommandLine::ExtraFile::ExtraFile(const CommandLine::ExtraFile &other) -: filename(other.filename) -, filesize(other.filesize) -{ -} - -CommandLine::ExtraFile& CommandLine::ExtraFile::operator=(const CommandLine::ExtraFile &other) -{ - filename = other.filename; - filesize = other.filesize; - - return *this; -} - -CommandLine::ExtraFile::ExtraFile(const string &name, u64 size) -: filename(name) -, filesize(size) -{ -} - - -CommandLine::CommandLine(std::ostream& cout, std::ostream& cerr) -: operation(opNone) -, version(verUnknown) -, noiselevel(nlUnknown) -, blockcount(0) -, blocksize(0) -, firstblock(0) -, recoveryfilescheme(scUnknown) -, recoveryfilecount(0) -, recoveryblockcount(0) -, recoveryblockcountset(false) -, redundancy(0) -, redundancyset(false) -, parfilename() -, extrafiles() -, totalsourcesize(0) -, largestsourcesize(0) -, memorylimit(0) -, cout(cout) -, cerr(cerr) -{ -} - -void CommandLine::usage(void) -{ - std::cout << - "\n" - "Usage:\n" - "\n" - " par2 c(reate) [options] [files] : Create PAR2 files\n" - " par2 v(erify) [options] [files] : Verify files using PAR2 file\n" - " par2 r(epair) [options] [files] : Repair files using PAR2 files\n" - "\n" - "You may also leave out the \"c\", \"v\", and \"r\" commands by using \"parcreate\",\n" - "\"par2verify\", or \"par2repair\" instead.\n" - "\n" - "Options:\n" - "\n" - " -b : Set the Block-Count\n" - " -s : Set the Block-Size (Don't use both -b and -s)\n" - " -r : Level of Redundancy (%%)\n" - " -c : Recovery block count (Don't use both -r and -c)\n" - " -f : First Recovery-Block-Number\n" - " -u : Uniform recovery file sizes\n" - " -l : Limit size of recovery files (Don't use both -u and -l)\n" - " -n : Number of recovery files (Don't use both -n and -l)\n" - " -m : Memory (in MB) to use\n" - " -v [-v]: Be more verbose\n" - " -q [-q]: Be more quiet (-q -q gives silence)\n" - " -- : Treat all remaining CommandLine as filenames\n" - "\n" - "If you wish to create par2 files for a single source file, you may leave\n" - "out the name of the par2 file from the command line.\n"; -} - -bool CommandLine::Parse(int argc, char *argv[]) -{ - if (argc<1) - { - return false; - } - - // Split the program name into path and filename - string path, name; - DiskFile::SplitFilename(argv[0], path, name); - argc--; - argv++; - - // Strip ".exe" from the end - if (name.size() > 4 && 0 == stricmp(".exe", name.substr(name.length()-4).c_str())) - { - name = name.substr(0, name.length()-4); - } - - // Check the resulting program name - if (0 == stricmp("par2create", name.c_str())) - { - operation = opCreate; - } - else if (0 == stricmp("par2verify", name.c_str())) - { - operation = opVerify; - } - else if (0 == stricmp("par2repair", name.c_str())) - { - operation = opRepair; - } - - // Have we determined what operation we want? - if (operation == opNone) - { - if (argc<2) - { - cerr << "Not enough command line arguments." << endl; - return false; - } - - switch (tolower(argv[0][0])) - { - case 'c': - if (argv[0][1] == 0 || 0 == stricmp(argv[0], "create")) - operation = opCreate; - break; - case 'v': - if (argv[0][1] == 0 || 0 == stricmp(argv[0], "verify")) - operation = opVerify; - break; - case 'r': - if (argv[0][1] == 0 || 0 == stricmp(argv[0], "repair")) - operation = opRepair; - break; - } - if (operation == opNone) - { - cerr << "Invalid operation specified: " << argv[0] << endl; - return false; - } - argc--; - argv++; - } - - bool options = true; - - while (argc>0) - { - if (argv[0][0]) - { - if (options && argv[0][0] != '-') - options = false; - - if (options) - { - switch (tolower(argv[0][1])) - { - case 'b': // Set the block count - { - if (operation != opCreate) - { - cerr << "Cannot specify block count unless creating." << endl; - return false; - } - if (blockcount > 0) - { - cerr << "Cannot specify block count twice." << endl; - return false; - } - else if (blocksize > 0) - { - cerr << "Cannot specify both block count and block size." << endl; - return false; - } - - char *p = &argv[0][2]; - while (blockcount <= 3276 && *p && isdigit(*p)) - { - blockcount = blockcount * 10 + (*p - '0'); - p++; - } - if (0 == blockcount || blockcount > 32768 || *p) - { - cerr << "Invalid block count option: " << argv[0] << endl; - return false; - } - } - break; - - case 's': // Set the block size - { - if (operation != opCreate) - { - cerr << "Cannot specify block size unless creating." << endl; - return false; - } - if (blocksize > 0) - { - cerr << "Cannot specify block size twice." << endl; - return false; - } - else if (blockcount > 0) - { - cerr << "Cannot specify both block count and block size." << endl; - return false; - } - - char *p = &argv[0][2]; - while (blocksize <= 429496729 && *p && isdigit(*p)) - { - blocksize = blocksize * 10 + (*p - '0'); - p++; - } - if (*p || blocksize == 0) - { - cerr << "Invalid block size option: " << argv[0] << endl; - return false; - } - if (blocksize & 3) - { - cerr << "Block size must be a multiple of 4." << endl; - return false; - } - } - break; - - case 'r': // Set the amount of redundancy required - { - if (operation != opCreate) - { - cerr << "Cannot specify redundancy unless creating." << endl; - return false; - } - if (redundancyset) - { - cerr << "Cannot specify redundancy twice." << endl; - return false; - } - else if (recoveryblockcountset) - { - cerr << "Cannot specify both redundancy and recovery block count." << endl; - return false; - } - - char *p = &argv[0][2]; - while (redundancy <= 10 && *p && isdigit(*p)) - { - redundancy = redundancy * 10 + (*p - '0'); - p++; - } - if (redundancy > 100 || *p) - { - cerr << "Invalid redundancy option: " << argv[0] << endl; - return false; - } - if (redundancy == 0 && recoveryfilecount > 0) - { - cerr << "Cannot set redundancy to 0 and file count > 0" << endl; - return false; - } - redundancyset = true; - } - break; - - case 'c': // Set the number of recovery blocks to create - { - if (operation != opCreate) - { - cerr << "Cannot specify recovery block count unless creating." << endl; - return false; - } - if (recoveryblockcountset) - { - cerr << "Cannot specify recovery block count twice." << endl; - return false; - } - else if (redundancyset) - { - cerr << "Cannot specify both recovery block count and redundancy." << endl; - return false; - } - - char *p = &argv[0][2]; - while (recoveryblockcount <= 32768 && *p && isdigit(*p)) - { - recoveryblockcount = recoveryblockcount * 10 + (*p - '0'); - p++; - } - if (recoveryblockcount > 32768 || *p) - { - cerr << "Invalid recoveryblockcount option: " << argv[0] << endl; - return false; - } - if (recoveryblockcount == 0 && recoveryfilecount > 0) - { - cerr << "Cannot set recoveryblockcount to 0 and file count > 0" << endl; - return false; - } - recoveryblockcountset = true; - } - break; - - case 'f': // Specify the First block recovery number - { - if (operation != opCreate) - { - cerr << "Cannot specify first block number unless creating." << endl; - return false; - } - if (firstblock > 0) - { - cerr << "Cannot specify first block twice." << endl; - return false; - } - - char *p = &argv[0][2]; - while (firstblock <= 3276 && *p && isdigit(*p)) - { - firstblock = firstblock * 10 + (*p - '0'); - p++; - } - if (firstblock > 32768 || *p) - { - cerr << "Invalid first block option: " << argv[0] << endl; - return false; - } - } - break; - - case 'u': // Specify uniformly sized recovery files - { - if (operation != opCreate) - { - cerr << "Cannot specify uniform files unless creating." << endl; - return false; - } - if (argv[0][2]) - { - cerr << "Invalid option: " << argv[0] << endl; - return false; - } - if (recoveryfilescheme != scUnknown) - { - cerr << "Cannot specify two recovery file size schemes." << endl; - return false; - } - - recoveryfilescheme = scUniform; - } - break; - - case 'l': // Limit the size of the recovery files - { - if (operation != opCreate) - { - cerr << "Cannot specify limit files unless creating." << endl; - return false; - } - if (argv[0][2]) - { - cerr << "Invalid option: " << argv[0] << endl; - return false; - } - if (recoveryfilescheme != scUnknown) - { - cerr << "Cannot specify two recovery file size schemes." << endl; - return false; - } - if (recoveryfilecount > 0) - { - cerr << "Cannot specify limited size and number of files at the same time." << endl; - return false; - } - - recoveryfilescheme = scLimited; - } - break; - - case 'n': // Specify the number of recovery files - { - if (operation != opCreate) - { - cerr << "Cannot specify recovery file count unless creating." << endl; - return false; - } - if (recoveryfilecount > 0) - { - cerr << "Cannot specify recovery file count twice." << endl; - return false; - } - if (redundancyset && redundancy == 0) - { - cerr << "Cannot set file count when redundancy is set to 0." << endl; - return false; - } - if (recoveryblockcountset && recoveryblockcount == 0) - { - cerr << "Cannot set file count when recovery block count is set to 0." << endl; - return false; - } - if (recoveryfilescheme == scLimited) - { - cerr << "Cannot specify limited size and number of files at the same time." << endl; - return false; - } - - char *p = &argv[0][2]; - while (*p && isdigit(*p)) - { - recoveryfilecount = recoveryfilecount * 10 + (*p - '0'); - p++; - } - if (recoveryfilecount == 0 || *p) - { - cerr << "Invalid recovery file count option: " << argv[0] << endl; - return false; - } - } - break; - - case 'm': // Specify how much memory to use for output buffers - { - if (memorylimit > 0) - { - cerr << "Cannot specify memory limit twice." << endl; - return false; - } - - char *p = &argv[0][2]; - while (*p && isdigit(*p)) - { - memorylimit = memorylimit * 10 + (*p - '0'); - p++; - } - if (memorylimit == 0 || *p) - { - cerr << "Invalid memory limit option: " << argv[0] << endl; - return false; - } - } - break; - - case 'v': - { - switch (noiselevel) - { - case nlUnknown: - { - if (argv[0][2] == 'v') - noiselevel = nlDebug; - else - noiselevel = nlNoisy; - } - break; - case nlNoisy: - case nlDebug: - noiselevel = nlDebug; - break; - default: - cerr << "Cannot use both -v and -q." << endl; - return false; - break; - } - } - break; - - case 'q': - { - switch (noiselevel) - { - case nlUnknown: - { - if (argv[0][2] == 'q') - noiselevel = nlSilent; - else - noiselevel = nlQuiet; - } - break; - case nlQuiet: - case nlSilent: - noiselevel = nlSilent; - break; - default: - cerr << "Cannot use both -v and -q." << endl; - return false; - break; - } - } - break; - - case '-': - { - argc--; - argv++; - options = false; - continue; - } - break; - default: - { - cerr << "Invalid option specified: " << argv[0] << endl; - return false; - } - } - } - else - { - list *filenames; - - // If the argument includes wildcard characters, - // search the disk for matching files - if (strchr(argv[0], '*') || strchr(argv[0], '?')) - { - string path; - string name; - DiskFile::SplitFilename(argv[0], path, name); - - filenames = DiskFile::FindFiles(path, name); - } - else - { - //start of shell expanded * patch. -- Michael Evans - //The shell might expaned * so, if we have our name and we're creating, then filter for files... - if ((parfilename.length() != 0) && (operation == opCreate)) - { -#ifdef WIN32 - if (GetFileAttributes(argv[0]) & FILE_ATTRIBUTE_DIRECTORY) // != 0, but no need... -#else //Not WIN32, probably *nix - struct stat st; - if (!(stat(argv[0], &st) == 0 && S_ISREG(st.st_mode))) -#endif - { - cerr << "Skipping non-regular file: " << argv[0] << endl; - argc--; - argv++; - options = false; - continue; - } - }//end of shell expanded * patch. -- Michael Evans - filenames = new list; - filenames->push_back(argv[0]); - } - - list::iterator fn = filenames->begin(); - while (fn != filenames->end()) - { - // Convert filename from command line into a full path + filename - string filename = DiskFile::GetCanonicalPathname(*fn); - - // If this is the first file on the command line, then it - // is the main PAR2 file. - if (parfilename.length() == 0) - { - // If we are verifying or repairing, the PAR2 file must - // already exist - if (operation != opCreate) - { - // Find the last '.' in the filename - string::size_type where = filename.find_last_of('.'); - if (where != string::npos) - { - // Get what follows the last '.' - string tail = filename.substr(where+1); - - if (0 == stricmp(tail.c_str(), "par2")) - { - parfilename = filename; - version = verPar2; - } - else if (0 == stricmp(tail.c_str(), "par") || - (tail.size() == 3 && - tolower(tail[0]) == 'p' && - isdigit(tail[1]) && - isdigit(tail[2]))) - { - parfilename = filename; - version = verPar1; - } - } - - // If we haven't figured out which version of PAR file we - // are using from the file extension, then presumable the - // files filename was actually the name of a data file. - if (version == verUnknown) - { - // Check for the existence of a PAR2 of PAR file. - if (DiskFile::FileExists(filename + ".par2")) - { - version = verPar2; - parfilename = filename + ".par2"; - } - else if (DiskFile::FileExists(filename + ".PAR2")) - { - version = verPar2; - parfilename = filename + ".PAR2"; - } - else if (DiskFile::FileExists(filename + ".par")) - { - version = verPar1; - parfilename = filename + ".par"; - } - else if (DiskFile::FileExists(filename + ".PAR")) - { - version = verPar1; - parfilename = filename + ".PAR"; - } - } - else - { - // Does the specified PAR or PAR2 file exist - if (!DiskFile::FileExists(filename)) - { - version = verUnknown; - } - } - - if (version == verUnknown) - { - cerr << "The recovery file does not exist: " << filename << endl; - return false; - } - } - else - { - // We are creating a new file - version = verPar2; - parfilename = filename; - } - } - else - { - // All other files must exist - if (!DiskFile::FileExists(filename)) - { - cerr << "The source file does not exist: " << filename << endl; - return false; - } - - u64 filesize = DiskFile::GetFileSize(filename); - - // Ignore all 0 byte files - if (filesize > 0) - { - extrafiles.push_back(ExtraFile(filename, filesize)); - - // track the total size of the source files and how - // big the largest one is. - totalsourcesize += filesize; - if (largestsourcesize < filesize) - largestsourcesize = filesize; - } - else - { - cout << "Skipping 0 byte file: " << filename << endl; - } - } - - ++fn; - } - delete filenames; - } - } - - argc--; - argv++; - } - - if (parfilename.length() == 0) - { - cerr << "You must specify a Recovery file." << endl; - return false; - } - - // Default noise level - if (noiselevel == nlUnknown) - { - noiselevel = nlNormal; - } - - // If we a creating, check the other parameters - if (operation == opCreate) - { - // If no recovery file size scheme is specified then use Variable - if (recoveryfilescheme == scUnknown) - { - recoveryfilescheme = scVariable; - } - - // If neither block count not block size is specified - if (blockcount == 0 && blocksize == 0) - { - // Use a block count of 2000 - blockcount = 2000; - } - - // If we are creating, the source files must be given. - if (extrafiles.size() == 0) - { - // Does the par filename include the ".par2" on the end? - if (parfilename.length() > 5 && 0 == stricmp(parfilename.substr(parfilename.length()-5, 5).c_str(), ".par2")) - { - // Yes it does. - cerr << "You must specify a list of files when creating." << endl; - return false; - } - else - { - // No it does not. - - // In that case check to see if the file exists, and if it does - // assume that you wish to create par2 files for it. - - u64 filesize = 0; - if (DiskFile::FileExists(parfilename) && - (filesize = DiskFile::GetFileSize(parfilename)) > 0) - { - extrafiles.push_back(ExtraFile(parfilename, filesize)); - - // track the total size of the source files and how - // big the largest one is. - totalsourcesize += filesize; - if (largestsourcesize < filesize) - largestsourcesize = filesize; - } - else - { - // The file does not exist or it is empty. - - cerr << "You must specify a list of files when creating." << endl; - return false; - } - } - } - - // Strip the ".par2" from the end of the filename of the main PAR2 file. - if (parfilename.length() > 5 && 0 == stricmp(parfilename.substr(parfilename.length()-5, 5).c_str(), ".par2")) - { - parfilename = parfilename.substr(0, parfilename.length()-5); - } - - // Assume a redundancy of 5% if neither redundancy or recoveryblockcount were set. - if (!redundancyset && !recoveryblockcountset) - { - redundancy = 5; - } - } - - // Assume a memory limit of 16MB if not specified. - if (memorylimit == 0) - { -#ifdef WIN32 - u64 TotalPhysicalMemory = 0; - - HMODULE hLib = ::LoadLibraryA("kernel32.dll"); - if (NULL != hLib) - { - BOOL (WINAPI *pfn)(LPMEMORYSTATUSEX) = (BOOL (WINAPI*)(LPMEMORYSTATUSEX))::GetProcAddress(hLib, "GlobalMemoryStatusEx"); - - if (NULL != pfn) - { - MEMORYSTATUSEX mse; - mse.dwLength = sizeof(mse); - if (pfn(&mse)) - { - TotalPhysicalMemory = mse.ullTotalPhys; - } - } - - ::FreeLibrary(hLib); - } - - if (TotalPhysicalMemory == 0) - { - MEMORYSTATUS ms; - ::ZeroMemory(&ms, sizeof(ms)); - ::GlobalMemoryStatus(&ms); - - TotalPhysicalMemory = ms.dwTotalPhys; - } - - if (TotalPhysicalMemory == 0) - { - // Assume 128MB - TotalPhysicalMemory = 128 * 1048576; - } - - // Half of total physical memory - memorylimit = (size_t)(TotalPhysicalMemory / 1048576 / 2); -#else - memorylimit = 16; -#endif - } - memorylimit *= 1048576; - - return true; -} - -} // end namespace Par2 diff --git a/lib/par2/commandline.h b/lib/par2/commandline.h deleted file mode 100644 index e2e8ce371..000000000 --- a/lib/par2/commandline.h +++ /dev/null @@ -1,165 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __COMMANDLINE_H__ -#define __COMMANDLINE_H__ - -namespace Par2 -{ - -// The CommandLine object is responsible for understanding the format -// of the command line parameters are parsing the command line to -// extract details as to what the user wants to do. - -class CommandLine -{ -public: - CommandLine(std::ostream& cout, std::ostream& cerr); - - // Parse the supplied command line arguments. - bool Parse(int argc, char *argv[]); - - // Display details of the correct format for command line parameters. - static void usage(void); - - // What operation will we be carrying out - typedef enum - { - opNone = 0, - opCreate, // Create new PAR2 recovery volumes - opVerify, // Verify but don't repair damaged data files - opRepair // Verify and if possible repair damaged data files - } Operation; - - typedef enum - { - verUnknown = 0, - verPar1, // Processing PAR 1.0 files - verPar2 // Processing PAR 2.0 files - } Version; - - typedef enum - { - scUnknown = 0, - scVariable, // Each PAR2 file will have 2x as many blocks as previous - scLimited, // Limit PAR2 file size - scUniform // All PAR2 files the same size - } Scheme; - - typedef enum - { - nlUnknown = 0, - nlSilent, // Absolutely no output (other than errors) - nlQuiet, // Bare minimum of output - nlNormal, // Normal level of output - nlNoisy, // Lots of output - nlDebug // Extra debugging information - } NoiseLevel; - - // Any extra files listed on the command line - class ExtraFile - { - public: - ExtraFile(void); - ExtraFile(const ExtraFile&); - ExtraFile& operator=(const ExtraFile&); - - ExtraFile(const string &name, u64 size); - - string FileName(void) const {return filename;} - u64 FileSize(void) const {return filesize;} - - protected: - string filename; - u64 filesize; - }; - -public: - // Accessor functions for the command line parameters - - CommandLine::Operation GetOperation(void) const {return operation;} - CommandLine::Version GetVersion(void) const {return version;} - u64 GetBlockSize(void) const {return blocksize;} - u32 GetBlockCount(void) const {return blockcount;} - u32 GetRedundancy(void) const {return redundancy;} - u32 GetFirstRecoveryBlock(void) const {return firstblock;} - u32 GetRecoveryFileCount(void) const {return recoveryfilecount;} - u32 GetRecoveryBlockCount(void) const {return recoveryblockcount;} - CommandLine::Scheme GetRecoveryFileScheme(void) const {return recoveryfilescheme;} - size_t GetMemoryLimit(void) const {return memorylimit;} - u64 GetLargestSourceSize(void) const {return largestsourcesize;} - u64 GetTotalSourceSize(void) const {return totalsourcesize;} - CommandLine::NoiseLevel GetNoiseLevel(void) const {return noiselevel;} - - string GetParFilename(void) const {return parfilename;} - const list& GetExtraFiles(void) const {return extrafiles;} - -protected: - Operation operation; // The operation to be carried out. - Version version; // What version files will be processed. - - NoiseLevel noiselevel; // How much display output should there be. - - u32 blockcount; // How many blocks the source files should - // be virtually split into. - - u64 blocksize; // What virtual block size to use. - - u32 firstblock; // What the exponent value for the first - // recovery block will be. - - Scheme recoveryfilescheme; // How the the size of the recovery files should - // be calculated. - - u32 recoveryfilecount; // How many recovery files should be created. - - u32 recoveryblockcount; // How many recovery blocks should be created. - bool recoveryblockcountset; // Set if the recoveryblockcount as been specified - - u32 redundancy; // What percentage of recovery data should - // be created. - bool redundancyset; // Set if the redundancy has been specified - - string parfilename; // The name of the PAR2 file to create, or - // the name of the first PAR2 file to read - // when verifying or repairing. - - list extrafiles; // The list of other files specified on the - // command line. When creating, this will be - // the source files, and when verifying or - // repairing, this will be additional PAR2 - // files or data files to be examined. - - u64 totalsourcesize; // Total size of the source files. - - u64 largestsourcesize; // Size of the largest source file. - - size_t memorylimit; // How much memory is permitted to be used - // for the output buffer when creating - // or repairing. - - std::ostream& cout; - std::ostream& cerr; -}; - -typedef list::const_iterator ExtraFileIterator; - -} // end namespace Par2 - -#endif // __COMMANDLINE_H__ diff --git a/lib/par2/crc.cpp b/lib/par2/crc.cpp deleted file mode 100644 index d6fa265fd..000000000 --- a/lib/par2/crc.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// The one and only CCITT CRC32 lookup table -crc32table ccitttable(0xEDB88320L); - -// Construct the CRC32 lookup table from the specified polynomial -void GenerateCRC32Table(u32 polynomial, u32 (&table)[256]) -{ - for (u32 i = 0; i <= 255 ; i++) - { - u32 crc = i; - - for (u32 j = 0; j < 8; j++) - { - crc = (crc >> 1) ^ ((crc & 1) ? polynomial : 0); - } - - table[i] = crc; - } -} - -// Construct a CRC32 lookup table for windowing -void GenerateWindowTable(u64 window, u32 (&target)[256]) -{ - for (u32 i=0; i<=255; i++) - { - u32 crc = ccitttable.table[i]; - - for (u64 j=0; j> 8) & 0x00ffffffL) ^ ccitttable.table[(u8)crc]; - } - - target[i] = crc; - } -} - -// Construct the mask value to apply to the CRC when windowing -u32 ComputeWindowMask(u64 window) -{ - u32 result = ~0; - while (window > 0) - { - result = CRCUpdateChar(result, (char)0); - - window--; - } - result ^= ~0; - - return result; -} - -} // end namespace Par2 diff --git a/lib/par2/crc.h b/lib/par2/crc.h deleted file mode 100644 index 0644c8b5a..000000000 --- a/lib/par2/crc.h +++ /dev/null @@ -1,117 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __CRC_H__ -#define __CRC_H__ - -namespace Par2 -{ - -// These global functions are used to compute the CCITT CRC32 checksum of -// blocks of data. - -// The CRC for a block of data may be computed piecemeal be repeatedly -// calling CRCUpdateChar, and CRCUpdateBlock. - -// Given the CRC for a block of data in a buffer, CRCSlideChar may be used -// to quickly compute the CRC for the block of data in the buffer that is the -// same size but offset one character later in the buffer. - - -// Construct the CRC32 lookup table from the specified polynomial -void GenerateCRC32Table(u32 polynomial, u32 (&table)[256]); - -// A CRC32 lookup table -struct crc32table -{ - crc32table(u32 polynomial) - { - GenerateCRC32Table(polynomial, table); - } - - u32 table[256]; -}; - -// The one and only CCITT CRC32 lookup table -extern crc32table ccitttable; - -// Update the CRC using one character -inline u32 CRCUpdateChar(u32 crc, u8 ch) -{ - return ((crc >> 8) & 0x00ffffffL) ^ ccitttable.table[(u8)crc ^ ch]; -} - -// Update the CRC using a block of characters in a buffer -inline u32 CRCUpdateBlock(u32 crc, size_t length, const void *buffer) -{ - const unsigned char *current = (const unsigned char *)buffer; - - while (length-- > 0) - { - crc = ((crc >> 8) & 0x00ffffffL) ^ ccitttable.table[(u8)crc ^ (*current++)]; - } - - return crc; -} - -// Update the CRC using a block of 0s. -inline u32 CRCUpdateBlock(u32 crc, size_t length) -{ - while (length-- > 0) - { - crc = ((crc >> 8) & 0x00ffffffL) ^ ccitttable.table[(u8)crc]; - } - - return crc; -} - -// Construct a CRC32 lookup table for windowing -void GenerateWindowTable(u64 window, u32 (&windowtable)[256]); -// Construct the mask value to apply to the CRC when windowing -u32 ComputeWindowMask(u64 window); - -// Slide the CRC along a buffer by one character (removing the old and adding the new). -// The new character is added using the main CCITT CRC32 table, and the old character -// is removed using the windowtable. -inline u32 CRCSlideChar(u32 crc, u8 chNew, u8 chOld, const u32 (&windowtable)[256]) -{ - return ((crc >> 8) & 0x00ffffffL) ^ ccitttable.table[(u8)crc ^ chNew] ^ windowtable[chOld]; -} - -/* - - char *buffer; - u64 window; - - //... - - u32 windowtable[256]; - GenerateWindowTable(window, windowtable); - u32 windowmask = ComputeWindowMask(window); - - u32 crc = ~0 ^ CRCUpdateBlock(~0, window, buffer); - crc = windowmask ^ CRCSlideChar(windowmask ^ crc, buffer[window], buffer[0], windowtable); - - assert(crc == ~0 ^ CRCUpdateBlock(~0, window, buffer+1)); - -*/ - -} // end namespace Par2 - -#endif // __CRC_H__ diff --git a/lib/par2/creatorpacket.cpp b/lib/par2/creatorpacket.cpp deleted file mode 100644 index 46da03aa7..000000000 --- a/lib/par2/creatorpacket.cpp +++ /dev/null @@ -1,91 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Construct the creator packet. - -// The only external information required to complete construction is -// the set_id_hash (which is normally computed from information in the -// main packet). - -bool CreatorPacket::Create(const MD5Hash &setid) -{ - string creator = "Created by " PACKAGE " version " VERSION "."; - - // Allocate a packet just large enough for creator name - CREATORPACKET *packet = (CREATORPACKET *)AllocatePacket(sizeof(*packet) + (~3 & (3+(u32)creator.size()))); - - // Fill in the details the we know - packet->header.magic = packet_magic; - packet->header.length = packetlength; - //packet->header.hash; // Compute shortly - packet->header.setid = setid; - packet->header.type = creatorpacket_type; - - // Copy the creator description into the packet - memcpy(packet->client, creator.c_str(), creator.size()); - - // Compute the packet hash - MD5Context packetcontext; - packetcontext.Update(&packet->header.setid, packetlength - offsetof(PACKET_HEADER, setid)); - packetcontext.Final(packet->header.hash); - - return true; -} - -// Load the packet from disk. - -bool CreatorPacket::Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - // Is the packet long enough - if (header.length <= sizeof(CREATORPACKET)) - { - return false; - } - - // Is the packet too large (what is the longest reasonable creator description) - if (header.length - sizeof(CREATORPACKET) > 100000) - { - return false; - } - - // Allocate the packet (with a little extra so we will have NULLs after the description) - CREATORPACKET *packet = (CREATORPACKET *)AllocatePacket((size_t)header.length, 4); - packet->header = header; - - // Load the rest of the packet from disk - return diskfile->Read(offset + sizeof(PACKET_HEADER), - packet->client, - (size_t)packet->header.length - sizeof(PACKET_HEADER)); -} - -} // end namespace Par2 diff --git a/lib/par2/creatorpacket.h b/lib/par2/creatorpacket.h deleted file mode 100644 index b466d2277..000000000 --- a/lib/par2/creatorpacket.h +++ /dev/null @@ -1,49 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __CREATORPACKET_H__ -#define __CREATORPACKET_H__ - -namespace Par2 -{ - -// The creator packet records details as to which PAR2 client -// created a particular recovery file. - -// The PAR 2.0 specification requires the presence of a -// creator packet, but it is not actually needed for the -// verification or recovery of damaged files. - -class CreatorPacket : public CriticalPacket -{ -public: - // Construct the packet - CreatorPacket(void) {}; - ~CreatorPacket(void) {}; - - // Create a creator packet for a specified set id hash value - bool Create(const MD5Hash &set_id_hash); - - // Load a creator packet from a specified file - bool Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); -}; - -} // end namespace Par2 - -#endif // __CREATORPACKET_H__ diff --git a/lib/par2/criticalpacket.cpp b/lib/par2/criticalpacket.cpp deleted file mode 100644 index f1355695e..000000000 --- a/lib/par2/criticalpacket.cpp +++ /dev/null @@ -1,53 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -bool CriticalPacket::WritePacket(DiskFile &diskfile, u64 fileoffset) const -{ - assert(packetdata != 0 && packetlength != 0); - - return diskfile.Write(fileoffset, packetdata, packetlength); -} - -void CriticalPacket::FinishPacket(const MD5Hash &setid) -{ - assert(packetdata != 0 && packetlength >= sizeof(PACKET_HEADER)); - - PACKET_HEADER *header = (PACKET_HEADER*)packetdata; - header->setid = setid; - - MD5Context packetcontext; - packetcontext.Update(&header->setid, packetlength - offsetof(PACKET_HEADER, setid)); - packetcontext.Final(header->hash); -} - -} // end namespace Par2 diff --git a/lib/par2/criticalpacket.h b/lib/par2/criticalpacket.h deleted file mode 100644 index 3799364db..000000000 --- a/lib/par2/criticalpacket.h +++ /dev/null @@ -1,150 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __CRITICALPACKET_H__ -#define __CRITICALPACKET_H__ - -namespace Par2 -{ - -// Base class for main packet, file verification packet, file description packet -// and creator packet. - -// These packets are all small and are held in memory in their entirity - -class CriticalPacket -{ -public: - CriticalPacket(void); - ~CriticalPacket(void); - -public: - // Write a copy of the packet to the specified file at the specified offset - bool WritePacket(DiskFile &diskfile, u64 fileoffset) const; - - // Obtain the lenght of the packet. - size_t PacketLength(void) const; - - // Allocate some memory for the packet (plus some extra padding). - void* AllocatePacket(size_t length, size_t extra = 0); - - // Finish a packet (by storing the set_id_hash and then computing the packet_hash). - void FinishPacket(const MD5Hash &set_id_hash); - -protected: - u8 *packetdata; - size_t packetlength; -}; - -inline CriticalPacket::CriticalPacket(void) -{ - // There is no data initially - packetdata = 0; - packetlength = 0; -} - -inline CriticalPacket::~CriticalPacket(void) -{ - // Delete the data for the packet - delete [] packetdata; -} - -inline size_t CriticalPacket::PacketLength(void) const -{ - return packetlength; -} - -inline void* CriticalPacket::AllocatePacket(size_t length, size_t extra) -{ - // Hey! We can't allocate the packet twice - assert(packetlength == 0 && packetdata == 0); - - // Remember the requested packet length - packetlength = length; - - // Allocate and clear the requested packet length plus the extra. - packetdata = new u8[length+extra]; - memset(packetdata, 0, length+extra); - - return packetdata; -} - -// Class used to record the fact that a copy of a particular critical packet -// will be written to a particular file at a specific offset. - -class CriticalPacketEntry -{ -public: - CriticalPacketEntry(DiskFile *_diskfile, - u64 _offset, - const CriticalPacket *_packet) - : diskfile(_diskfile) - , offset(_offset) - , packet(_packet) - {} - CriticalPacketEntry(void) - : diskfile(0) - , offset(0) - , packet(0) - {} - CriticalPacketEntry(const CriticalPacketEntry &other) - : diskfile(other.diskfile) - , offset(other.offset) - , packet(other.packet) - {} - CriticalPacketEntry& operator=(const CriticalPacketEntry &other) - { - diskfile = other.diskfile; - offset = other.offset; - packet = other.packet; - return *this; - } - -public: - // Write the packet to disk. - bool WritePacket(void) const; - - // Obtain the length of the packet. - u64 PacketLength(void) const; - -protected: - DiskFile *diskfile; - u64 offset; - const CriticalPacket *packet; -}; - -inline bool CriticalPacketEntry::WritePacket(void) const -{ - assert(packet != 0 && diskfile != 0); - - // Tell the packet to write itself to disk - return packet->WritePacket(*diskfile, offset); -} - -inline u64 CriticalPacketEntry::PacketLength(void) const -{ - assert(packet != 0); - - // Ask the packet how big it is. - return packet->PacketLength(); -} - -} // end namespace Par2 - -#endif // __CRITICALPACKET_H__ diff --git a/lib/par2/datablock.cpp b/lib/par2/datablock.cpp deleted file mode 100644 index 5871723ef..000000000 --- a/lib/par2/datablock.cpp +++ /dev/null @@ -1,113 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Open the file associated with the data block if is not already open -bool DataBlock::Open(void) -{ - if (diskfile == 0) - return false; - - if (diskfile->IsOpen()) - return true; - - return diskfile->Open(); -} - -// Read some data at a specified position within a data block -// into a buffer in memory - -bool DataBlock::ReadData(u64 position, // Position within the block - size_t size, // Size of the memory buffer - void *buffer) // Pointer to memory buffer -{ - assert(diskfile != 0); - - // Check to see if the position from which data is to be read - // is within the bounds of the data block - if (length > position) - { - // Compute the file offset and how much data to physically read from disk - u64 fileoffset = offset + position; - size_t want = (size_t)min((u64)size, length - position); - - // Read the data from the file into the buffer - if (!diskfile->Read(fileoffset, buffer, want)) - return false; - - // If the read extends beyond the end of the data block, - // then the rest of the buffer is zeroed. - if (want < size) - { - memset(&((u8*)buffer)[want], 0, size-want); - } - } - else - { - // Zero the whole buffer - memset(buffer, 0, size); - } - - return true; -} - -// Write some data at a specified position within a datablock -// from memory to disk - -bool DataBlock::WriteData(u64 position, // Position within the block - size_t size, // Size of the memory buffer - const void *buffer, // Pointer to memory buffer - size_t &wrote) // Amount actually written -{ - assert(diskfile != 0); - - wrote = 0; - - // Check to see if the position from which data is to be written - // is within the bounds of the data block - if (length > position) - { - // Compute the file offset and how much data to physically write to disk - u64 fileoffset = offset + position; - size_t have = (size_t)min((u64)size, length - position); - - // Write the data from the buffer to disk - if (!diskfile->Write(fileoffset, buffer, have)) - return false; - - wrote = have; - } - - return true; -} - -} // end namespace Par2 diff --git a/lib/par2/datablock.h b/lib/par2/datablock.h deleted file mode 100644 index 8c8e9cb89..000000000 --- a/lib/par2/datablock.h +++ /dev/null @@ -1,138 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __DATABLOCK_H__ -#define __DATABLOCK_H__ - -namespace Par2 -{ - -class DiskFile; - -// A Data Block is a block of data of a specific length at a specific -// offset in a specific file. - -// It may be either a block of data in a source file from which recovery -// data is being computed, a block of recovery data in a recovery file, or -// a block in a target file that is being reconstructed. - -class DataBlock -{ -public: - DataBlock(void); - ~DataBlock(void); - -public: - // Set the length of the block - void SetLength(u64 length); - - // Set the location of the block - void SetLocation(DiskFile *diskfile, u64 offset); - void ClearLocation(void); - -public: - // Check to see if the location of the block has been set - bool IsSet(void) const; - - // Which disk file is this data block in - DiskFile* GetDiskFile(void) const; - - // What offset is the block located at - u64 GetOffset(void) const; - - // What is the length of this block - u64 GetLength(void) const; - -public: - // Open the disk file if it is not already open (so that it can be read) - bool Open(void); - - // Read some of the data from disk into memory. - bool ReadData(u64 position, size_t size, void *buffer); - - // Write some of the data from memory to disk - bool WriteData(u64 position, size_t size, const void *buffer, size_t &wrote); - -protected: - DiskFile *diskfile; // Which disk file is the block associated with - u64 offset; // What is the file offset - u64 length; // How large is the block -}; - - -// Construct the data block -inline DataBlock::DataBlock(void) -{ - diskfile = 0; - offset = 0; - length = 0; -} - -// Destroy the data block -inline DataBlock::~DataBlock(void) -{ -} - -// Set the length of the block -inline void DataBlock::SetLength(u64 _length) -{ - length = _length; -} - -// Set the location of the block -inline void DataBlock::SetLocation(DiskFile *_diskfile, u64 _offset) -{ - diskfile = _diskfile; - offset = _offset; -} - -// Clear the location of the block -inline void DataBlock::ClearLocation(void) -{ - diskfile = 0; - offset = 0; -} - -// Check to see of the location is known -inline bool DataBlock::IsSet(void) const -{ - return (diskfile != 0); -} - -// Which disk file is this data block in -inline DiskFile* DataBlock::GetDiskFile(void) const -{ - return diskfile; -} - -// What offset is the block located at -inline u64 DataBlock::GetOffset(void) const -{ - return offset; -} - -// What is the length of this block -inline u64 DataBlock::GetLength(void) const -{ - return length; -} - -} // end namespace Par2 - -#endif // __DATABLOCK_H__ diff --git a/lib/par2/descriptionpacket.cpp b/lib/par2/descriptionpacket.cpp deleted file mode 100644 index 7409a9bcc..000000000 --- a/lib/par2/descriptionpacket.cpp +++ /dev/null @@ -1,122 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" -#include "Util.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Construct the packet and store the filename and size. - -bool DescriptionPacket::Create(string filename, u64 filesize) -{ - // Allocate some extra bytes for the packet in memory so that strlen() can - // be used on the filename. The extra bytes do not get written to disk. - FILEDESCRIPTIONPACKET *packet = (FILEDESCRIPTIONPACKET *)AllocatePacket(sizeof(*packet) + (~3 & (3 + (u32)filename.size())), 4); - - // Store everything that is currently known in the packet. - - packet->header.magic = packet_magic; - packet->header.length = packetlength; - //packet->header.hash; // Not known yet - //packet->header.setid; // Not known yet - packet->header.type = filedescriptionpacket_type; - - //packet->fileid; // Not known yet - //packet->hashfull; // Not known yet - //packet->hash16k; // Not known yet - packet->length = filesize; - - memcpy(packet->name, filename.c_str(), filename.size()); - - return true; -} - - -void DescriptionPacket::Hash16k(const MD5Hash &hash) -{ - ((FILEDESCRIPTIONPACKET *)packetdata)->hash16k = hash; -} - -void DescriptionPacket::HashFull(const MD5Hash &hash) -{ - ((FILEDESCRIPTIONPACKET *)packetdata)->hashfull = hash; -} - -void DescriptionPacket::ComputeFileId(void) -{ - FILEDESCRIPTIONPACKET *packet = ((FILEDESCRIPTIONPACKET *)packetdata); - - // Compute the fileid from the hash, length, and name fields in the packet. - - MD5Context context; - context.Update(&packet->hash16k, - sizeof(FILEDESCRIPTIONPACKET)-offsetof(FILEDESCRIPTIONPACKET,hash16k) - +strlen((const char*)packet->name)); - context.Final(packet->fileid); -} - -// Load a description packet from a specified file -bool DescriptionPacket::Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - // Is the packet big enough - if (header.length <= sizeof(FILEDESCRIPTIONPACKET)) - { - return false; - } - - // Is the packet too large (what is the longest permissible filename) - if (header.length - sizeof(FILEDESCRIPTIONPACKET) > 100000) - { - return false; - } - - // Allocate the packet (with a little extra so we will have NULLs after the filename) - FILEDESCRIPTIONPACKET *packet = (FILEDESCRIPTIONPACKET *)AllocatePacket((size_t)header.length, 4); - - packet->header = header; - - // Read the rest of the packet from disk - if (!diskfile->Read(offset + sizeof(PACKET_HEADER), - &packet->fileid, - (size_t)packet->header.length - sizeof(PACKET_HEADER))) - return false; - - filename = *WebUtil::Latin1ToUtf8((char*)((FILEDESCRIPTIONPACKET*)packetdata)->name); - - // Are the file and 16k hashes consistent - if (packet->length <= 16384 && packet->hash16k != packet->hashfull) - { - return false; - } - - return true; -} - -} // end namespace Par2 diff --git a/lib/par2/descriptionpacket.h b/lib/par2/descriptionpacket.h deleted file mode 100644 index 1bd7af5e7..000000000 --- a/lib/par2/descriptionpacket.h +++ /dev/null @@ -1,109 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __DESCRIPTIONPACKET_H__ -#define __DESCRIPTIONPACKET_H__ - -namespace Par2 -{ - -// The description packet records details about a file (including its name, -// size, and the Hash of both the whole file and the first 16k of the file). - -class DescriptionPacket : public CriticalPacket -{ -public: - // Construct the packet - DescriptionPacket(void) {}; - ~DescriptionPacket(void) {}; - -public: - // Construct the packet and store the filename and size. - bool Create(string _filename, u64 _filesize); - - // Store the computed Hash values in the packet. - void Hash16k(const MD5Hash &hash); - void HashFull(const MD5Hash &hash); - - // Compute and return the file id hash from information in the packet - void ComputeFileId(void); - const MD5Hash& FileId(void) const; - - // Return the size of the file - u64 FileSize(void) const; - -public: - // Load a description packet from a specified file - bool Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - - // Return the name of the file - string FileName(void) const; - - // Get the Hash values from the packet - const MD5Hash& HashFull(void) const; - const MD5Hash& Hash16k(void) const; -protected: - string filename; -}; - -// Get the file id from the packet -inline const MD5Hash& DescriptionPacket::FileId(void) const -{ - assert(packetdata != 0); - - return ((const FILEDESCRIPTIONPACKET*)packetdata)->fileid; -} - -// Get the size of the file from the packet -inline u64 DescriptionPacket::FileSize(void) const -{ - assert(packetdata != 0); - - return ((const FILEDESCRIPTIONPACKET*)packetdata)->length; -} - -// Get the name of the file from the packet -// NB whilst the file format does not guarantee that the name will have a NULL -// termination character, par2cmdline always allocates a little extra data -// and fills it with NULLs to allow the filename to be directly read out of -// the packet. -inline string DescriptionPacket::FileName(void) const -{ - return filename.c_str(); -} - -// Get the full file hash value from the packet -inline const MD5Hash& DescriptionPacket::HashFull(void) const -{ - assert(packetdata != 0); - - return ((const FILEDESCRIPTIONPACKET*)packetdata)->hashfull; -} - -// The the hash of the first 16k of the file from the packet -inline const MD5Hash& DescriptionPacket::Hash16k(void) const -{ - assert(packetdata != 0); - - return ((const FILEDESCRIPTIONPACKET*)packetdata)->hash16k; -} - -} // end namespace Par2 - -#endif // __DESCRIPTIONPACKET_H__ diff --git a/lib/par2/diskfile.cpp b/lib/par2/diskfile.cpp deleted file mode 100644 index b457d245c..000000000 --- a/lib/par2/diskfile.cpp +++ /dev/null @@ -1,933 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" -#include "FileSystem.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -#ifdef WIN32 -///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#define OffsetType __int64 -#define MaxOffset 0x7fffffffffffffffI64 -#define LengthType unsigned int -#define MaxLength 0xffffffffUL - -DiskFile::DiskFile(std::ostream& cerr) : - cerr(cerr) -{ - filename; - filesize = 0; - offset = 0; - - hFile = INVALID_HANDLE_VALUE; - - exists = false; -} - -DiskFile::~DiskFile(void) -{ - if (hFile != INVALID_HANDLE_VALUE) - ::CloseHandle(hFile); -} - -// Create new file on disk and make sure that there is enough -// space on disk for it. -bool DiskFile::Create(string _filename, u64 _filesize) -{ - assert(hFile == INVALID_HANDLE_VALUE); - - filename = _filename; - filesize = _filesize; - - // Create the file - hFile = ::CreateFileW(FileSystem::UtfPathToWidePath(_filename.c_str()), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_NEW, 0, NULL); - if (hFile == INVALID_HANDLE_VALUE) - { - DWORD error = ::GetLastError(); - - cerr << "Could not create \"" << _filename << "\": " << ErrorMessage(error) << endl; - - return false; - } - - if (filesize > 0) - { - // Seek to the end of the file - LONG lowoffset = ((LONG*)&filesize)[0]; - LONG highoffset = ((LONG*)&filesize)[1]; - - if (INVALID_SET_FILE_POINTER == SetFilePointer(hFile, lowoffset, &highoffset, FILE_BEGIN)) - { - DWORD error = ::GetLastError(); - - cerr << "Could not set size of \"" << _filename << "\": " << ErrorMessage(error) << endl; - - ::CloseHandle(hFile); - hFile = INVALID_HANDLE_VALUE; - ::DeleteFile(_filename.c_str()); - - return false; - } - - // Set the end of the file - if (!::SetEndOfFile(hFile)) - { - DWORD error = ::GetLastError(); - - cerr << "Could not set size of \"" << _filename << "\": " << ErrorMessage(error) << endl; - - ::CloseHandle(hFile); - hFile = INVALID_HANDLE_VALUE; - ::DeleteFile(_filename.c_str()); - - return false; - } - } - - offset = filesize; - - exists = true; - return true; -} - -// Write some data to disk - -bool DiskFile::Write(u64 _offset, const void *buffer, size_t length) -{ - assert(hFile != INVALID_HANDLE_VALUE); - - if (offset != _offset) - { - LONG lowoffset = ((LONG*)&_offset)[0]; - LONG highoffset = ((LONG*)&_offset)[1]; - - // Seek to the required offset - if (INVALID_SET_FILE_POINTER == SetFilePointer(hFile, lowoffset, &highoffset, FILE_BEGIN)) - { - DWORD error = ::GetLastError(); - - cerr << "Could not write " << (u64)length << " bytes to \"" << filename << "\" at offset " << _offset << ": " << ErrorMessage(error) << endl; - - return false; - } - offset = _offset; - } - - if (length > MaxLength) - { - cerr << "Could not write " << (u64)length << " bytes to \"" << filename << "\" at offset " << _offset << ": " << "Write too long" << endl; - - return false; - } - - DWORD write = (LengthType)length; - DWORD wrote; - - // Write the data - if (!::WriteFile(hFile, buffer, write, &wrote, NULL)) - { - DWORD error = ::GetLastError(); - - cerr << "Could not write " << (u64)length << " bytes to \"" << filename << "\" at offset " << _offset << ": " << ErrorMessage(error) << endl; - - return false; - } - - offset += length; - - if (filesize < offset) - { - filesize = offset; - } - - return true; -} - -// Open the file - -bool DiskFile::Open(string _filename, u64 _filesize) -{ - assert(hFile == INVALID_HANDLE_VALUE); - - filename = _filename; - filesize = _filesize; - - hFile = ::CreateFileW(FileSystem::UtfPathToWidePath(_filename.c_str()), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL); - if (hFile == INVALID_HANDLE_VALUE) - { - DWORD error = ::GetLastError(); - - switch (error) - { - case ERROR_FILE_NOT_FOUND: - case ERROR_PATH_NOT_FOUND: - break; - default: - cerr << "Could not open \"" << _filename << "\": " << ErrorMessage(error) << endl; - } - - return false; - } - - offset = 0; - exists = true; - - return true; -} - -// Read some data from disk - -bool DiskFile::Read(u64 _offset, void *buffer, size_t length) -{ - assert(hFile != INVALID_HANDLE_VALUE); - - if (offset != _offset) - { - LONG lowoffset = ((LONG*)&_offset)[0]; - LONG highoffset = ((LONG*)&_offset)[1]; - - // Seek to the required offset - if (INVALID_SET_FILE_POINTER == SetFilePointer(hFile, lowoffset, &highoffset, FILE_BEGIN)) - { - DWORD error = ::GetLastError(); - - cerr << "Could not read " << (u64)length << " bytes from \"" << filename << "\" at offset " << _offset << ": " << ErrorMessage(error) << endl; - - return false; - } - offset = _offset; - } - - if (length > MaxLength) - { - cerr << "Could not read " << (u64)length << " bytes from \"" << filename << "\" at offset " << _offset << ": " << "Read too long" << endl; - - return false; - } - - DWORD want = (LengthType)length; - DWORD got; - - // Read the data - if (!::ReadFile(hFile, buffer, want, &got, NULL)) - { - DWORD error = ::GetLastError(); - - cerr << "Could not read " << (u64)length << " bytes from \"" << filename << "\" at offset " << _offset << ": " << ErrorMessage(error) << endl; - - return false; - } - - offset += length; - - return true; -} - -void DiskFile::Close(void) -{ - if (hFile != INVALID_HANDLE_VALUE) - { - ::CloseHandle(hFile); - hFile = INVALID_HANDLE_VALUE; - } -} - -string DiskFile::GetCanonicalPathname(string filename) -{ - char fullname[MAX_PATH]; - char *filepart; - - // Resolve a relative path to a full path - int length = ::GetFullPathName(filename.c_str(), sizeof(fullname), fullname, &filepart); - if (length <= 0 || sizeof(fullname) < length) - return filename; - - // Make sure the drive letter is upper case. - fullname[0] = toupper(fullname[0]); - - // Translate all /'s to \'s - char *current = strchr(fullname, '/'); - while (current) - { - *current++ = '\\'; - current = strchr(current, '/'); - } - - // Copy the root directory to the output string - string longname(fullname, 3); - - // Start processing at the first path component - current = &fullname[3]; - char *limit = &fullname[length]; - - // Process until we reach the end of the full name - while (current < limit) - { - char *tail; - - // Find the next \, or the end of the string - (tail = strchr(current, '\\')) || (tail = limit); - *tail = 0; - - // Create a wildcard to search for the path - string wild = longname + current; - WIN32_FIND_DATA finddata; - HANDLE hFind = ::FindFirstFile(wild.c_str(), &finddata); - if (hFind != INVALID_HANDLE_VALUE) - // Copy the component found to the output - longname += finddata.cFileName; - else - // If the component was not found then just copy the component to the - // output buffer verbatim. - longname += current; - ::FindClose(hFind); - - current = tail + 1; - - // If we have not reached the end of the name, add a "\" - if (current < limit) - longname += '\\'; - } - - return longname; -} - -list* DiskFile::FindFiles(string path, string wildcard) -{ - list *matches = new list; - - wildcard = path + wildcard; - WIN32_FIND_DATAW fd; - HANDLE h = ::FindFirstFileW(FileSystem::UtfPathToWidePath(wildcard.c_str()), &fd); - if (h != INVALID_HANDLE_VALUE) - { - do - { - if (0 == (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) - { - matches->push_back(path + *FileSystem::WidePathToUtfPath(fd.cFileName)); - } - } while (::FindNextFileW(h, &fd)); - ::FindClose(h); - } - - return matches; -} - - - - -///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#else // !WIN32 -///////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -#ifdef HAVE_FSEEKO -# define OffsetType off_t -# define MaxOffset ((off_t)0x7fffffffffffffffULL) -# define fseek fseeko -#else -# if _FILE_OFFSET_BITS == 64 -# define OffsetType unsigned long long -# define MaxOffset 0x7fffffffffffffffULL -# else -# define OffsetType long -# define MaxOffset 0x7fffffffUL -# endif -#endif - -#define LengthType unsigned int -#define MaxLength 0xffffffffUL - -DiskFile::DiskFile(std::ostream& cerr) : - cerr(cerr) -{ - //filename; - filesize = 0; - offset = 0; - - file = 0; - - exists = false; -} - -DiskFile::~DiskFile(void) -{ - if (file != 0) - fclose(file); -} - -// Create new file on disk and make sure that there is enough -// space on disk for it. -bool DiskFile::Create(string _filename, u64 _filesize) -{ - assert(file == 0); - - filename = _filename; - filesize = _filesize; - - file = fopen(_filename.c_str(), "wb"); - if (file == 0) - { - cerr << "Could not create: " << _filename << endl; - - return false; - } - - if (_filesize > (u64)MaxOffset) - { - cerr << "Requested file size for " << _filename << " is too large." << endl; - return false; - } - - if (_filesize > 0) - { - if (fseek(file, (OffsetType)_filesize-1, SEEK_SET)) - { - fclose(file); - file = 0; - ::remove(filename.c_str()); - - cerr << "Could not set end of file: " << _filename << endl; - return false; - } - - if (1 != fwrite(&_filesize, 1, 1, file)) - { - fclose(file); - file = 0; - ::remove(filename.c_str()); - - cerr << "Could not set end of file: " << _filename << endl; - return false; - } - } - - offset = filesize; - - exists = true; - return true; -} - -// Write some data to disk - -bool DiskFile::Write(u64 _offset, const void *buffer, size_t length) -{ - assert(file != 0); - - if (offset != _offset) - { - if (_offset > (u64)MaxOffset) - { - cerr << "Could not write " << (u64)length << " bytes to " << filename << " at offset " << _offset << endl; - return false; - } - - - if (fseek(file, (OffsetType)_offset, SEEK_SET)) - { - cerr << "Could not write " << (u64)length << " bytes to " << filename << " at offset " << _offset << endl; - return false; - } - offset = _offset; - } - - if (length > MaxLength) - { - cerr << "Could not write " << (u64)length << " bytes to " << filename << " at offset " << _offset << endl; - return false; - } - - if (1 != fwrite(buffer, (LengthType)length, 1, file)) - { - cerr << "Could not write " << (u64)length << " bytes to " << filename << " at offset " << _offset << endl; - return false; - } - - offset += length; - - if (filesize < offset) - { - filesize = offset; - } - - return true; -} - -// Open the file - -bool DiskFile::Open(string _filename, u64 _filesize) -{ - assert(file == 0); - - filename = _filename; - filesize = _filesize; - - if (_filesize > (u64)MaxOffset) - { - cerr << "File size for " << _filename << " is too large." << endl; - return false; - } - - file = fopen(filename.c_str(), "rb"); - if (file == 0) - { - return false; - } - - offset = 0; - exists = true; - - return true; -} - -// Read some data from disk - -bool DiskFile::Read(u64 _offset, void *buffer, size_t length) -{ - assert(file != 0); - - if (offset != _offset) - { - if (_offset > (u64)MaxOffset) - { - cerr << "Could not read " << (u64)length << " bytes from " << filename << " at offset " << _offset << endl; - return false; - } - - - if (fseek(file, (OffsetType)_offset, SEEK_SET)) - { - cerr << "Could not read " << (u64)length << " bytes from " << filename << " at offset " << _offset << endl; - return false; - } - offset = _offset; - } - - if (length > MaxLength) - { - cerr << "Could not read " << (u64)length << " bytes from " << filename << " at offset " << _offset << endl; - return false; - } - - if (1 != fread(buffer, (LengthType)length, 1, file)) - { - cerr << "Could not read " << (u64)length << " bytes from " << filename << " at offset " << _offset << endl; - return false; - } - - offset += length; - - return true; -} - -void DiskFile::Close(void) -{ - if (file != 0) - { - fclose(file); - file = 0; - } -} - -// Attempt to get the full pathname of the file -string DiskFile::GetCanonicalPathname(string filename) -{ - // Is the supplied path already an absolute one - if (filename.size() == 0 || filename[0] == '/') - return filename; - - // Get the current directory - char curdir[1000]; - if (0 == getcwd(curdir, sizeof(curdir))) - { - return filename; - } - - - // Allocate a work buffer and copy the resulting full path into it. - char *work = new char[strlen(curdir) + filename.size() + 2]; - strcpy(work, curdir); - if (work[strlen(work)-1] != '/') - strcat(work, "/"); - strcat(work, filename.c_str()); - - char *in = work; - char *out = work; - - while (*in) - { - if (*in == '/') - { - if (in[1] == '.' && in[2] == '/') - { - // skip the input past /./ - in += 2; - } - else if (in[1] == '.' && in[2] == '.' && in[3] == '/') - { - // backtrack the output if /../ was found on the input - in += 3; - if (out > work) - { - do - { - out--; - } while (out > work && *out != '/'); - } - } - else - { - *out++ = *in++; - } - } - else - { - *out++ = *in++; - } - } - *out = 0; - - string result = work; - delete [] work; - - return result; -} - -list* DiskFile::FindFiles(string path, string wildcard) -{ - list *matches = new list; - - string::size_type where; - - if ((where = wildcard.find_first_of('*')) != string::npos || - (where = wildcard.find_first_of('?')) != string::npos) - { - string front = wildcard.substr(0, where); - bool multiple = wildcard[where] == '*'; - string back = wildcard.substr(where+1); - - DIR *dirp = opendir(path.c_str()); - if (dirp != 0) - { - struct dirent *d; - while ((d = readdir(dirp)) != 0) - { - string name = d->d_name; - - if (name == "." || name == "..") - continue; - - if (multiple) - { - if (name.size() >= wildcard.size() && - name.substr(0, where) == front && - name.substr(name.size()-back.size()) == back) - { - matches->push_back(path + name); - } - } - else - { - if (name.size() == wildcard.size()) - { - string::const_iterator pw = wildcard.begin(); - string::const_iterator pn = name.begin(); - while (pw != wildcard.end()) - { - if (*pw != '?' && *pw != *pn) - break; - ++pw; - ++pn; - } - - if (pw == wildcard.end()) - { - matches->push_back(path + name); - } - } - } - - } - closedir(dirp); - } - } - else - { - struct stat st; - string fn = path + wildcard; - if (stat(fn.c_str(), &st) == 0) - { - matches->push_back(path + wildcard); - } - } - - return matches; -} - -///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -#endif - - - - - - - - - - - - - - - - - - - - - - - -bool DiskFile::Open(void) -{ - string _filename = filename; - - return Open(_filename); -} - -bool DiskFile::Open(string _filename) -{ - return Open(_filename, GetFileSize(_filename)); -} - - - - - - - - - -// Delete the file - -bool DiskFile::Delete(void) -{ -#ifdef WIN32 - assert(hFile == INVALID_HANDLE_VALUE); -#else - assert(file == 0); -#endif - - if (filename.size() > 0 && 0 == unlink(filename.c_str())) - { - return true; - } - else - { - cerr << "Cannot delete " << filename << endl; - - return false; - } -} - - - - - - - - - -//string DiskFile::GetPathFromFilename(string filename) -//{ -// string::size_type where; -// -// if (string::npos != (where = filename.find_last_of('/')) || -// string::npos != (where = filename.find_last_of('\\'))) -// { -// return filename.substr(0, where+1); -// } -// else -// { -// return "." PATHSEP; -// } -//} - -void DiskFile::SplitFilename(string filename, string &path, string &name) -{ - string::size_type where; - - if (string::npos != (where = filename.find_last_of('/')) || - string::npos != (where = filename.find_last_of('\\'))) - { - path = filename.substr(0, where+1); - name = filename.substr(where+1); - } - else - { - path = "." PATHSEP; - name = filename; - } -} - -bool DiskFile::FileExists(string filename) -{ - return FileSystem::FileExists(filename.c_str()); -} - -u64 DiskFile::GetFileSize(string filename) -{ - int64 size = FileSystem::FileSize(filename.c_str()); - return size > 0 ? size : 0; -} - - - -// Take a filename from a PAR2 file and replace any characters -// which would be illegal for a file on disk -string DiskFile::TranslateFilename(string filename) -{ - return *FileSystem::MakeValidFilename(filename.c_str()); -} - -bool DiskFile::Rename(void) -{ - char newname[1024+1]; - u32 index = 0; - - do - { - int length = snprintf(newname, 1024, "%s.%d", filename.c_str(), ++index); - if (length < 0 || length >= 1024) - { - cerr << filename << " cannot be renamed." << endl; - return false; - } - newname[length] = 0; - } while (FileSystem::FileExists(newname)); - - return Rename(newname); -} - -bool DiskFile::Rename(string _filename) -{ -#ifdef WIN32 - assert(hFile == INVALID_HANDLE_VALUE); -#else - assert(file == 0); -#endif - - if (FileSystem::MoveFile(filename.c_str(), _filename.c_str())) - { - filename = _filename; - - return true; - } - else - { - cerr << filename << " cannot be renamed to " << _filename << endl; - - return false; - } -} - -#ifdef WIN32 -string DiskFile::ErrorMessage(DWORD error) -{ - string result; - - LPVOID lpMsgBuf; - if (::FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - error, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&lpMsgBuf, - 0, - NULL)) - { - result = (char*)lpMsgBuf; - LocalFree(lpMsgBuf); - } - else - { - char message[40]; - snprintf(message, sizeof(message), "Unknown error code (%d)", error); - result = message; - } - - return result; -} -#endif - -DiskFileMap::DiskFileMap(void) -{ -} - -DiskFileMap::~DiskFileMap(void) -{ - map::iterator fi = diskfilemap.begin(); - while (fi != diskfilemap.end()) - { - delete (*fi).second; - - ++fi; - } -} - -bool DiskFileMap::Insert(DiskFile *diskfile) -{ - string filename = diskfile->FileName(); - assert(filename.length() != 0); - - pair::const_iterator,bool> location = diskfilemap.insert(pair(filename, diskfile)); - - return location.second; -} - -void DiskFileMap::Remove(DiskFile *diskfile) -{ - string filename = diskfile->FileName(); - assert(filename.length() != 0); - - diskfilemap.erase(filename); -} - -DiskFile* DiskFileMap::Find(string filename) const -{ - assert(filename.length() != 0); - - map::const_iterator f = diskfilemap.find(filename); - - return (f != diskfilemap.end()) ? f->second : 0; -} - -} // end namespace Par2 diff --git a/lib/par2/diskfile.h b/lib/par2/diskfile.h deleted file mode 100644 index 2967a496e..000000000 --- a/lib/par2/diskfile.h +++ /dev/null @@ -1,132 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __DISKFILE_H__ -#define __DISKFILE_H__ - -namespace Par2 -{ - -// A disk file can be any type of file that par2cmdline needs -// to read or write data from or to. - -class DiskFile -{ -public: - DiskFile(std::ostream& cerr); - ~DiskFile(void); - - // Create a file and set its length - bool Create(string filename, u64 filesize); - - // Write some data to the file - bool Write(u64 offset, const void *buffer, size_t length); - - // Open the file - bool Open(void); - bool Open(string filename); - bool Open(string filename, u64 filesize); - - // Check to see if the file is open -#ifdef WIN32 - bool IsOpen(void) const {return hFile != INVALID_HANDLE_VALUE;} -#else - bool IsOpen(void) const {return file != 0;} -#endif - - // Read some data from the file - bool Read(u64 offset, void *buffer, size_t length); - - // Close the file - void Close(void); - - // Get the size of the file - u64 FileSize(void) const {return filesize;} - - // Get the name of the file - string FileName(void) const {return filename;} - - // Does the file exist - bool Exists(void) const {return exists;} - - // Rename the file - bool Rename(void); // Pick a filename automatically - bool Rename(string filename); - - // Delete the file - bool Delete(void); - -public: - static string GetCanonicalPathname(string filename); - - static void SplitFilename(string filename, string &path, string &name); - static string TranslateFilename(string filename); - - static bool FileExists(string filename); - static u64 GetFileSize(string filename); - - // Search the specified path for files which match the specified wildcard - // and return their names in a list. - static list* FindFiles(string path, string wildcard); - -protected: - string filename; - u64 filesize; - - // OS file handle -#ifdef WIN32 - HANDLE hFile; -#else - FILE *file; -#endif - - // Current offset within the file - u64 offset; - - // Does the file exist - bool exists; - -protected: -#ifdef WIN32 - static string ErrorMessage(DWORD error); -#endif - - std::ostream& cerr; -}; - -// This class keeps track of which DiskFile objects exist -// and which file on disk they are associated with. -// It is used to avoid a file being processed twice. -class DiskFileMap -{ -public: - DiskFileMap(void); - ~DiskFileMap(void); - - bool Insert(DiskFile *diskfile); - void Remove(DiskFile *diskfile); - DiskFile* Find(string filename) const; - -protected: - map diskfilemap; // Map from filename to DiskFile -}; - -} // end namespace Par2 - -#endif // __DISKFILE_H__ diff --git a/lib/par2/filechecksummer.cpp b/lib/par2/filechecksummer.cpp deleted file mode 100644 index efbcccb09..000000000 --- a/lib/par2/filechecksummer.cpp +++ /dev/null @@ -1,261 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Construct the checksummer and allocate buffers - -FileCheckSummer::FileCheckSummer(DiskFile *_diskfile, - u64 _blocksize, - const u32 (&_windowtable)[256], - u32 _windowmask) -: diskfile(_diskfile) -, blocksize(_blocksize) -, windowtable(_windowtable) -, windowmask(_windowmask) -{ - buffer = new char[(size_t)blocksize*2]; - - filesize = diskfile->FileSize(); - - currentoffset = 0; -} - -FileCheckSummer::~FileCheckSummer(void) -{ - delete [] buffer; -} - -// Start reading the file at the beginning -bool FileCheckSummer::Start(void) -{ - currentoffset = readoffset = 0; - - tailpointer = outpointer = buffer; - inpointer = &buffer[blocksize]; - - // Fill the buffer with new data - if (!Fill()) - return false; - - // Compute the checksum for the block - checksum = ~0 ^ CRCUpdateBlock(~0, (size_t)blocksize, buffer); - - return true; -} - -// Jump ahead -bool FileCheckSummer::Jump(u64 distance) -{ - // Are we already at the end of the file - if (currentoffset >= filesize) - return false; - - // Special distances - if (distance == 0) - return false; - if (distance == 1) - return Step(); - - // Not allowed to jump more than one block - assert(distance <= blocksize); - if (distance > blocksize) - distance = blocksize; - - // Advance the current offset and check if we have reached the end of the file - currentoffset += distance; - if (currentoffset >= filesize) - { - currentoffset = filesize; - tailpointer = outpointer = buffer; - memset(buffer, 0, (size_t)blocksize); - checksum = 0; - - return true; - } - - // Move past the data being discarded - outpointer += distance; - assert(outpointer <= tailpointer); - - // Is there any data left in the buffer that we are keeping - size_t keep = tailpointer - outpointer; - if (keep > 0) - { - // Move it back to the start of the buffer - memmove(buffer, outpointer, keep); - tailpointer = &buffer[keep]; - } - else - { - tailpointer = buffer; - } - - outpointer = buffer; - inpointer = &buffer[blocksize]; - - if (!Fill()) - return false; - - // Compute the checksum for the block - checksum = ~0 ^ CRCUpdateBlock(~0, (size_t)blocksize, buffer); - - return true; -} - -// Fill the buffer from disk - -bool FileCheckSummer::Fill(void) -{ - // Have we already reached the end of the file - if (readoffset >= filesize) - return true; - - // How much data can we read into the buffer - size_t want = (size_t)min(filesize-readoffset, (u64)(&buffer[2*blocksize]-tailpointer)); - - if (want > 0) - { - // Read data - if (!diskfile->Read(readoffset, tailpointer, want)) - return false; - - UpdateHashes(readoffset, tailpointer, want); - readoffset += want; - tailpointer += want; - } - - // Did we fill the buffer - want = &buffer[2*blocksize] - tailpointer; - if (want > 0) - { - // Blank the rest of the buffer - memset(tailpointer, 0, want); - } - - return true; -} - -// Update the full file hash and the 16k hash using the new data -void FileCheckSummer::UpdateHashes(u64 offset, const void *buffer, size_t length) -{ - // Are we already beyond the first 16k - if (offset >= 16384) - { - contextfull.Update(buffer, length); - } - // Would we reach the 16k mark - else if (offset+length >= 16384) - { - // Finish the 16k hash - size_t first = (size_t)(16384-offset); - context16k.Update(buffer, first); - - // Continue with the full hash - contextfull = context16k; - - // Do we go beyond the 16k mark - if (offset+length > 16384) - { - contextfull.Update(&((const char*)buffer)[first], length-first); - } - } - else - { - context16k.Update(buffer, length); - } -} - -// Return the full file hash and the 16k file hash -void FileCheckSummer::GetFileHashes(MD5Hash &hashfull, MD5Hash &hash16k) const -{ - // Compute the hash of the first 16k - MD5Context context = context16k; - context.Final(hash16k); - - // Is the file smaller than 16k - if (filesize < 16384) - { - // The hashes are the same - hashfull = hash16k; - } - else - { - // Compute the hash of the full file - context = contextfull; - context.Final(hashfull); - } -} - -// Compute and return the current hash -MD5Hash FileCheckSummer::Hash(void) -{ - MD5Context context; - context.Update(outpointer, (size_t)blocksize); - - MD5Hash hash; - context.Final(hash); - - return hash; -} - -u32 FileCheckSummer::ShortChecksum(u64 blocklength) -{ - u32 crc = CRCUpdateBlock(~0, (size_t)blocklength, outpointer); - - if (blocksize > blocklength) - { - crc = CRCUpdateBlock(crc, (size_t)(blocksize-blocklength)); - } - - crc ^= ~0; - - return crc; -} - -MD5Hash FileCheckSummer::ShortHash(u64 blocklength) -{ - MD5Context context; - context.Update(outpointer, (size_t)blocklength); - - if (blocksize > blocklength) - { - context.Update((size_t)(blocksize-blocklength)); - } - - // Get the hash value - MD5Hash hash; - context.Final(hash); - - return hash; -} - -} // end namespace Par2 diff --git a/lib/par2/filechecksummer.h b/lib/par2/filechecksummer.h deleted file mode 100644 index 18e31b906..000000000 --- a/lib/par2/filechecksummer.h +++ /dev/null @@ -1,182 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __FILECHECKSUMMER_H__ -#define __FILECHECKSUMMER_H__ - -namespace Par2 -{ - -// This source file defines the FileCheckSummer object which is used -// when scanning a data file to find blocks of undamaged data. -// -// The object uses a "window" into the data file and slides that window -// along the file computing the CRC of the data in that window as it -// goes. If the computed CRC matches the value for a block of data -// from a target data file, then the MD5 Hash value is also computed -// and compared with the value for that block of data. When a match -// has been confirmed, the object jumps forward to where the next -// block of data is expected to start. Whilst the file is being scanned -// the object also computes the MD5 Hash of the whole file and of -// the first 16k of the file for later tests. - -class FileCheckSummer -{ -public: - FileCheckSummer(DiskFile *diskfile, - u64 blocksize, - const u32 (&windowtable)[256], - u32 windowmask); - ~FileCheckSummer(void); - - // Start reading the file at the beginning - bool Start(void); - - // Jump ahead the specified distance - bool Jump(u64 distance); - - // Step forward one byte - bool Step(void); - - // Return the current checksum - u32 Checksum(void) const; - - // Compute and return the current hash - MD5Hash Hash(void); - - // Compute short values of checksum and hash - u32 ShortChecksum(u64 blocklength); - MD5Hash ShortHash(u64 blocklength); - - // Do we have less than a full block of data - bool ShortBlock(void) const; - u64 BlockLength(void) const; - - // Return the current file offset - u64 Offset(void) const; - - // Return the full file hash and the 16k file hash - void GetFileHashes(MD5Hash &hashfull, MD5Hash &hash16k) const; - - // Which disk file is this - const DiskFile* GetDiskFile(void) const {return diskfile;} - -protected: - DiskFile *diskfile; - u64 blocksize; - const u32 (&windowtable)[256]; - u32 windowmask; - - u64 filesize; - - u64 currentoffset; // file offset for current window position - char *buffer; // buffer for reading from the file - char *outpointer; // position in buffer of scan window - char *inpointer; // &outpointer[blocksize]; - char *tailpointer; // after last valid data in buffer - - // File offset for next read - u64 readoffset; - - // The current checksum - u32 checksum; - - // MD5 hash of whole file and of first 16k - MD5Context contextfull; - MD5Context context16k; - -protected: - //void ComputeCurrentCRC(void); - void UpdateHashes(u64 offset, const void *buffer, size_t length); - - //// Fill the buffers with more data from disk - bool Fill(void); -}; - -// Return the current checksum - -inline u32 FileCheckSummer::Checksum(void) const -{ - return checksum; -} - -// Return the current block length - -inline u64 FileCheckSummer::BlockLength(void) const -{ - return min(blocksize, filesize-currentoffset); -} - -// Return whether or not the current block is a short one. -inline bool FileCheckSummer::ShortBlock(void) const -{ - return BlockLength() < blocksize; -} - -// Return the current file offset -inline u64 FileCheckSummer::Offset(void) const -{ - return currentoffset; -} - -// Step forward one byte -inline bool FileCheckSummer::Step(void) -{ - // Are we already at the end of the file - if (currentoffset >= filesize) - return false; - - // Advance the file offset and check to see if - // we have reached the end of the file - if (++currentoffset >= filesize) - { - currentoffset = filesize; - tailpointer = outpointer = buffer; - memset(buffer, 0, (size_t)blocksize); - checksum = 0; - - return true; - } - - // Get the incoming and outgoing characters - char inch = *inpointer++; - char outch = *outpointer++; - - // Update the checksum - checksum = windowmask ^ CRCSlideChar(windowmask ^ checksum, inch, outch, windowtable); - - // Can the window slide further - if (outpointer < &buffer[blocksize]) - return true; - - assert(outpointer == &buffer[blocksize]); - - // Copy the data back to the beginning of the buffer - memmove(buffer, outpointer, (size_t)blocksize); - inpointer = outpointer; - outpointer = buffer; - tailpointer -= blocksize; - - // Fill the rest of the buffer - return Fill(); -} - -} // end namespace Par2 - -#endif // __FILECHECKSUMMER_H__ diff --git a/lib/par2/galois.cpp b/lib/par2/galois.cpp deleted file mode 100644 index 523525e5f..000000000 --- a/lib/par2/galois.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - diff --git a/lib/par2/galois.h b/lib/par2/galois.h deleted file mode 100644 index a5ca31426..000000000 --- a/lib/par2/galois.h +++ /dev/null @@ -1,343 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __GALOIS_H__ -#define __GALOIS_H__ - -namespace Par2 -{ - -template class GaloisTable; -template class Galois; - -template class GaloisLongMultiplyTable; - -// This source file defines the Galois object for carrying out -// arithmetic in GF(2^16) using the generator 0x1100B. - -// Also defined are the GaloisTable object (which contains log and -// anti log tables for use in multiplication and division), and -// the GaloisLongMultiplyTable object (which contains tables for -// carrying out multiplation of 16-bit galois numbers 8 bits at a time). - -template -class GaloisTable -{ -public: - typedef valuetype ValueType; - - GaloisTable(void); - - enum - { - Bits = bits, - Count = 1< -class Galois -{ -public: - typedef valuetype ValueType; - - // Basic constructors - Galois(void) {}; - Galois(ValueType v); - - // Copy and assignment - Galois(const Galois &right) {value = right.value;} - Galois& operator = (const Galois &right) { value = right.value; return *this;} - - // Addition - Galois operator + (const Galois &right) const { return (value ^ right.value); } - Galois& operator += (const Galois &right) { value ^= right.value; return *this;} - - // Subtraction - Galois operator - (const Galois &right) const { return (value ^ right.value); } - Galois& operator -= (const Galois &right) { value ^= right.value; return *this;} - - // Multiplication - Galois operator * (const Galois &right) const; - Galois& operator *= (const Galois &right); - - // Division - Galois operator / (const Galois &right) const; - Galois& operator /= (const Galois &right); - - // Power - Galois pow(unsigned int right) const; - Galois operator ^ (unsigned int right) const; - Galois& operator ^= (unsigned int right); - - // Cast to value and value access - operator ValueType(void) const {return value;} - ValueType Value(void) const {return value;} - - // Direct log and antilog - ValueType Log(void) const; - ValueType ALog(void) const; - - enum - { - Bits = GaloisTable::Bits, - Count = GaloisTable::Count, - Limit = GaloisTable::Limit, - }; - -protected: - ValueType value; - - static GaloisTable table; -}; - -#ifdef LONGMULTIPLY -template -class GaloisLongMultiplyTable -{ -public: - GaloisLongMultiplyTable(void); - - typedef g G; - - enum - { - Bytes = ((G::Bits + 7) >> 3), - Count = ((Bytes * (Bytes+1)) / 2), - }; - - G tables[Count * 256 * 256]; -}; -#endif - -// Construct the log and antilog tables from the generator - -template -inline GaloisTable::GaloisTable(void) -{ - u32 b = 1; - - for (u32 l=0; l -GaloisTable Galois::table; - - -template -inline Galois::Galois(typename Galois::ValueType v) -{ - value = v; -} - -template -inline Galois Galois::operator * (const Galois &right) const -{ - if (value == 0 || right.value == 0) return 0; - unsigned int sum = table.log[value] + table.log[right.value]; - if (sum >= Limit) - { - return table.antilog[sum-Limit]; - } - else - { - return table.antilog[sum]; - } -} - -template -inline Galois& Galois::operator *= (const Galois &right) -{ - if (value == 0 || right.value == 0) - { - value = 0; - } - else - { - unsigned int sum = table.log[value] + table.log[right.value]; - if (sum >= Limit) - { - value = table.antilog[sum-Limit]; - } - else - { - value = table.antilog[sum]; - } - } - - return *this; -} - -template -inline Galois Galois::operator / (const Galois &right) const -{ - if (value == 0) return 0; - - assert(right.value != 0); - if (right.value == 0) {return 0;} // Division by 0! - - int sum = table.log[value] - table.log[right.value]; - if (sum < 0) - { - return table.antilog[sum+Limit]; - } - else - { - return table.antilog[sum]; - } -} - -template -inline Galois& Galois::operator /= (const Galois &right) -{ - if (value == 0) return *this; - - assert(right.value != 0); - if (right.value == 0) {return *this;} // Division by 0! - - int sum = table.log[value] - table.log[right.value]; - if (sum < 0) - { - value = table.antilog[sum+Limit]; - } - else - { - value = table.antilog[sum]; - } - - return *this; -} - -template -inline Galois Galois::pow(unsigned int right) const -{ - if (right == 0) return 1; - if (value == 0) return 0; - - unsigned int sum = table.log[value] * right; - - sum = (sum >> Bits) + (sum & Limit); - if (sum >= Limit) - { - return table.antilog[sum-Limit]; - } - else - { - return table.antilog[sum]; - } -} - -template -inline Galois Galois::operator ^ (unsigned int right) const -{ - if (right == 0) return 1; - if (value == 0) return 0; - - unsigned int sum = table.log[value] * right; - - sum = (sum >> Bits) + (sum & Limit); - if (sum >= Limit) - { - return table.antilog[sum-Limit]; - } - else - { - return table.antilog[sum]; - } -} - -template -inline Galois& Galois::operator ^= (unsigned int right) -{ - if (right == 1) {value = 1; return *this;} - if (value == 0) return *this; - - unsigned int sum = table.log[value] * right; - - sum = (sum >> Bits) + (sum & Limit); - if (sum >= Limit) - { - value = table.antilog[sum-Limit]; - } - else - { - value = table.antilog[sum]; - } - - return *this; -} - -template -inline valuetype Galois::Log(void) const -{ - return table.log[value]; -} - -template -inline valuetype Galois::ALog(void) const -{ - return table.antilog[value]; -} - -#ifdef LONGMULTIPLY -template -inline GaloisLongMultiplyTable::GaloisLongMultiplyTable(void) -{ - G *table = tables; - - for (unsigned int i=0; i Galois8; -typedef Galois<16,0x1100B,u16> Galois16; - -} // end namespace Par2 - -#endif // __GALOIS_H__ diff --git a/lib/par2/letype.h b/lib/par2/letype.h deleted file mode 100644 index d4a50c683..000000000 --- a/lib/par2/letype.h +++ /dev/null @@ -1,125 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __LETYPE_H__ -#define __LETYPE_H__ - -namespace Par2 -{ - -#if __BYTE_ORDER == __LITTLE_ENDIAN - -typedef u16 leu16; -typedef u32 leu32; -typedef u64 leu64; - -#else - -struct leu16 -{ - leu16& operator=(const u16 &other); - - operator u16(void) const; - - u16 value; -}; - -inline leu16& leu16::operator=(const u16 &other) -{ - ((unsigned char*)&value)[0] = (unsigned char)((other >> 0) & 0xff); - ((unsigned char*)&value)[1] = (unsigned char)((other >> 8) & 0xff); - - return *this; -} - -inline leu16::operator u16(void) const -{ - return ((unsigned char*)&value)[0] << 0 | - ((unsigned char*)&value)[1] << 8; -} - - -struct leu32 -{ - leu32& operator=(const u32 &other); - - operator u32(void) const; - - u32 value; -}; - -inline leu32& leu32::operator=(const u32 &other) -{ - ((unsigned char*)&value)[0] = (unsigned char)((other >> 0) & 0xff); - ((unsigned char*)&value)[1] = (unsigned char)((other >> 8) & 0xff); - ((unsigned char*)&value)[2] = (unsigned char)((other >> 16) & 0xff); - ((unsigned char*)&value)[3] = (unsigned char)((other >> 24) & 0xff); - - return *this; -} - -inline leu32::operator u32(void) const -{ - return ((unsigned char*)&value)[0] << 0 | - ((unsigned char*)&value)[1] << 8 | - ((unsigned char*)&value)[2] << 16 | - ((unsigned char*)&value)[3] << 24; -} - - -struct leu64 -{ - leu64& operator=(const u64 &other); - - operator u64(void) const; - - u64 value; -}; - -inline leu64& leu64::operator=(const u64 &other) -{ - ((unsigned char*)&value)[0] = (unsigned char)((other >> 0) & 0xff); - ((unsigned char*)&value)[1] = (unsigned char)((other >> 8) & 0xff); - ((unsigned char*)&value)[2] = (unsigned char)((other >> 16) & 0xff); - ((unsigned char*)&value)[3] = (unsigned char)((other >> 24) & 0xff); - ((unsigned char*)&value)[4] = (unsigned char)((other >> 32) & 0xff); - ((unsigned char*)&value)[5] = (unsigned char)((other >> 40) & 0xff); - ((unsigned char*)&value)[6] = (unsigned char)((other >> 48) & 0xff); - ((unsigned char*)&value)[7] = (unsigned char)((other >> 56) & 0xff); - - return *this; -} - -inline leu64::operator u64(void) const -{ - return (u64)(((unsigned char*)&value)[0]) << 0 | - (u64)(((unsigned char*)&value)[1]) << 8 | - (u64)(((unsigned char*)&value)[2]) << 16 | - (u64)(((unsigned char*)&value)[3]) << 24 | - (u64)(((unsigned char*)&value)[4]) << 32 | - (u64)(((unsigned char*)&value)[5]) << 40 | - (u64)(((unsigned char*)&value)[6]) << 48 | - (u64)(((unsigned char*)&value)[7]) << 56; -} - -#endif - -} // end namespace Par2 - -#endif // __LETYPE_H__ diff --git a/lib/par2/mainpacket.cpp b/lib/par2/mainpacket.cpp deleted file mode 100644 index ced831390..000000000 --- a/lib/par2/mainpacket.cpp +++ /dev/null @@ -1,137 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Construct the main packet from the source files and the block size -/* -bool MainPacket::Create(vector &sourcefiles, u64 _blocksize) -{ - recoverablefilecount = totalfilecount =(u32)sourcefiles.size(); - blocksize = _blocksize; - - // Allocate memory for the main packet with enough fileid entries - MAINPACKET *packet = (MAINPACKET *)AllocatePacket(sizeof(MAINPACKET) + totalfilecount * sizeof(MD5Hash)); - - // Record the details we already know in the packet - packet->header.magic = packet_magic; - packet->header.length = packetlength; - //packet->header.hash; // Compute shortly - //packet->header.setid; // Compute shortly - packet->header.type = mainpacket_type; - - packet->blocksize = _blocksize; - packet->recoverablefilecount = totalfilecount; - //packet->fileid; // Compute shortly - - // Sort the source files according to their fileid values - if (totalfilecount > 1) - { - sort(sourcefiles.begin(), sourcefiles.end(), Par2CreatorSourceFile::CompareLess); - } - - // Store the fileid values in the main packet - vector::const_iterator sourcefile; - MD5Hash *hash; - for ((sourcefile=sourcefiles.begin()),(hash=packet->fileid); - sourcefile!=sourcefiles.end(); - ++sourcefile, ++hash) - { - *hash = (*sourcefile)->FileId(); - } - - // Compute the set_id_hash - MD5Context setidcontext; - setidcontext.Update(&packet->blocksize, packetlength - offsetof(MAINPACKET, blocksize)); - setidcontext.Final(packet->header.setid); - - // Compute the packet_hash - MD5Context packetcontext; - packetcontext.Update(&packet->header.setid, packetlength - offsetof(MAINPACKET, header.setid)); - packetcontext.Final(packet->header.hash); - - return true; -} -*/ - -// Load a main packet from a specified file - -bool MainPacket::Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - // Is the packet large enough - if (header.length < sizeof(MAINPACKET)) - { - return false; - } - - // Is there a whole number of fileid values - if (0 < (header.length - sizeof(MAINPACKET)) % sizeof(MD5Hash)) - { - return false; - } - - // Is the packet too large - if (header.length > sizeof(MAINPACKET) + 32768 * sizeof(MD5Hash)) - { - return false; - } - - // Compute the total number of entries in the fileid array - totalfilecount = (u32)(((size_t)header.length - sizeof(MAINPACKET)) / sizeof(MD5Hash)); - - MAINPACKET *packet = (MAINPACKET *)AllocatePacket((size_t)header.length); - - packet->header = header; - - // Read the rest of the packet from disk - if (!diskfile->Read(offset + sizeof(PACKET_HEADER), - &packet->blocksize, - (size_t)packet->header.length - sizeof(PACKET_HEADER))) - return false; - - // Does the packet have enough fileid values - recoverablefilecount = packet->recoverablefilecount; - if (recoverablefilecount > totalfilecount) - { - return false; - } - - // Is the block size valid - blocksize = packet->blocksize; - if (blocksize == 0 || (blocksize & 3) != 0) - { - return false; - } - - return true; -} - -} // end namespace Par2 diff --git a/lib/par2/mainpacket.h b/lib/par2/mainpacket.h deleted file mode 100644 index 29e21be9f..000000000 --- a/lib/par2/mainpacket.h +++ /dev/null @@ -1,109 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __MAINPACKET_H__ -#define __MAINPACKET_H__ - -namespace Par2 -{ - -// The main packet ties all other critical packets together. -// It specifies the block size to use for both verification of -// files and for the Reed Solomon computation. -// It also specifies how many of the source files are repairable -// and in what order they should be processed. - -class MainPacket : public CriticalPacket -{ -public: - // Construct the packet - MainPacket(void) {}; - ~MainPacket(void) {}; - -public: - // Construct the main packet from the source file list and block size. - // "sourcefiles" will be sorted base on their FileId value. - /*bool Create(vector &sourcefiles, - u64 _blocksize);*/ - - // Load a main packet from a specified file - bool Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - -public: - // Get the set id. - const MD5Hash& SetId(void) const; - - // Get the block size. - u64 BlockSize(void) const; - - // Get the file counts. - u32 RecoverableFileCount(void) const; - u32 TotalFileCount(void) const; - - // Get the fileid of one file - const MD5Hash& FileId(u32 filenumber) const; - -protected: - u64 blocksize; - u32 totalfilecount; - u32 recoverablefilecount; -}; - -// Get the data block size -inline u64 MainPacket::BlockSize(void) const -{ - assert(packetdata != 0); - - return blocksize; -} - -// Get the number of recoverable files -inline u32 MainPacket::RecoverableFileCount(void) const -{ - assert(packetdata != 0); - - return recoverablefilecount; -} - -// Get the total number of files -inline u32 MainPacket::TotalFileCount(void) const -{ - assert(packetdata != 0); - - return totalfilecount; -} - -// Get the file id hash of one of the files -inline const MD5Hash& MainPacket::FileId(u32 filenumber) const -{ - assert(packetdata != 0); - assert(filenumberfileid()[filenumber]; - return ((const MAINPACKET*)packetdata)->fileid[filenumber]; -} - -inline const MD5Hash& MainPacket::SetId(void) const -{ - return ((const MAINPACKET*)packetdata)->header.setid; -} - -} // end namespace Par2 - -#endif // __MAINPACKET_H__ diff --git a/lib/par2/md5.cpp b/lib/par2/md5.cpp deleted file mode 100644 index 5029c0269..000000000 --- a/lib/par2/md5.cpp +++ /dev/null @@ -1,356 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Convert hash values to hex - -ostream& operator<<(ostream &result, const MD5Hash &h) -{ - char buffer[33]; - - sprintf(buffer, - "%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X", - h.hash[15], h.hash[14], h.hash[13], h.hash[12], - h.hash[11], h.hash[10], h.hash[9], h.hash[8], - h.hash[7], h.hash[6], h.hash[5], h.hash[4], - h.hash[3], h.hash[2], h.hash[1], h.hash[0]); - - return result << buffer; -} - -string MD5Hash::print(void) const -{ - char buffer[33]; - - sprintf(buffer, - "%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X%02X", - hash[15], hash[14], hash[13], hash[12], - hash[11], hash[10], hash[9], hash[8], - hash[7], hash[6], hash[5], hash[4], - hash[3], hash[2], hash[1], hash[0]); - - return buffer; -} - -MD5State::MD5State(void) -{ - Reset(); -} - -// Initialise the 16 byte state -void MD5State::Reset(void) -{ - state[0] = 0x67452301; - state[1] = 0xefcdab89; - state[2] = 0x98badcfe; - state[3] = 0x10325476; -} - -// Update the state using 64 bytes of new data -void MD5State::UpdateState(const u32 (&block)[16]) -{ - // Primitive operations -#define F1(x,y,z) ( ((x) & (y)) | ((~(x)) & (z)) ) -#define F2(x,y,z) ( ((x) & (z)) | ((~(z)) & (y)) ) -#define F3(x,y,z) ( (x) ^ (y) ^ (z) ) -#define F4(x,y,z) ( (y) ^ ( (x) | ~(z) ) ) - -// The first version of ROL does not work on an Alpha CPU! -//#define ROL(x,y) ( ((x) << (y)) | (((unsigned int)x) >> (32-y)) ) -#define ROL(x,y) ( ((x) << (y)) | (((x) >> (32-y)) & ((1< 0) - { - size_t size = min(buffersize-used, length); - Update(wordblock, size); - length -= size; - } - - // Update as many whole buffers as possible - while (length >= buffersize) - { - Update(wordblock, buffersize); - - length -= buffersize; - } - - // Update any remainder - if (length > 0) - { - Update(wordblock, length); - } -} - -// Update using data from a buffer -void MD5Context::Update(const void *buffer, size_t length) -{ - const unsigned char *current = (const unsigned char *)buffer; - - // Update the total amount of data processed. - bytes += length; - - // Process any whole blocks - while (used + length >= buffersize) - { - size_t have = buffersize - used; - - memcpy(&block[used], current, have); - - current += have; - length -= have; - - u32 wordblock[16]; - for (int i=0; i<16; i++) - { - // Convert source data from little endian format to internal format if different - wordblock[i] = ( ((u32)block[i*4+3]) << 24 ) | - ( ((u32)block[i*4+2]) << 16 ) | - ( ((u32)block[i*4+1]) << 8 ) | - ( ((u32)block[i*4+0]) << 0 ); - } - - MD5State::UpdateState(wordblock); - - used = 0; - } - - // Store any remainder - if (length > 0) - { - memcpy(&block[used], current, length); - used += length; - } -} - -// Finalise the computation and extract the Hash value -void MD5Context::Final(MD5Hash &output) -{ - // Temporary work buffer - u8 buffer[64]; - - // How many bits were processed - u64 bits = bytes << 3; - - // Pad as much as needed so that there are exactly 8 bytes needed to fill the buffer - size_t padding; - if (used >= buffersize-8) - { - padding = buffersize-8 + buffersize - used; - } - else - { - padding = buffersize-8 - used; - } - memset(buffer, 0, padding); - buffer[0] = 0x80; - Update(buffer, padding); - - // Pad with an additional 8 bytes containing the bit count in little endian format - buffer[7] = (unsigned char)((bits >> 56) & 0xFF); - buffer[6] = (unsigned char)((bits >> 48) & 0xFF); - buffer[5] = (unsigned char)((bits >> 40) & 0xFF); - buffer[4] = (unsigned char)((bits >> 32) & 0xFF); - buffer[3] = (unsigned char)((bits >> 24) & 0xFF); - buffer[2] = (unsigned char)((bits >> 16) & 0xFF); - buffer[1] = (unsigned char)((bits >> 8) & 0xFF); - buffer[0] = (unsigned char)((bits >> 0) & 0xFF); - Update(buffer, 8); - - for (int i = 0; i < 4; i++) - { - // Read out the state and convert it from internal format to little endian format - output.hash[4*i+3] = (u8)((MD5State::state[i] >> 24) & 0xFF); - output.hash[4*i+2] = (u8)((MD5State::state[i] >> 16) & 0xFF); - output.hash[4*i+1] = (u8)((MD5State::state[i] >> 8) & 0xFF); - output.hash[4*i+0] = (u8)((MD5State::state[i] >> 0) & 0xFF); - } -} - -// Return the Hash value -MD5Hash MD5Context::Hash(void) const -{ - MD5Hash output; - - for (unsigned int i = 0; i < 4; i++) - { - // Read out the state and convert it from internal format to little endian format - output.hash[4*i+3] = (unsigned char)((MD5State::state[i] >> 24) & 0xFF); - output.hash[4*i+2] = (unsigned char)((MD5State::state[i] >> 16) & 0xFF); - output.hash[4*i+1] = (unsigned char)((MD5State::state[i] >> 8) & 0xFF); - output.hash[4*i+0] = (unsigned char)((MD5State::state[i] >> 0) & 0xFF); - } - - return output; -} - -ostream& operator<<(ostream &result, const MD5Context &c) -{ - char buffer[50]; - - sprintf(buffer, - "%08X%08X%08X%08X:%08X%08X", - c.state[3],c.state[2],c.state[1],c.state[0], - (u32)((c.bytes >> 32) & 0xffffffff), - (u32)(c.bytes & 0xffffffff)); - - return result << buffer; -} - -string MD5Context::print(void) const -{ - char buffer[50]; - - sprintf(buffer, - "%08X%08X%08X%08X:%08X%08X", - state[3],state[2],state[1],state[0], - (u32)((bytes >> 32) & 0xffffffff), - (u32)(bytes & 0xffffffff)); - - return buffer; -} - -} // end namespace Par2 diff --git a/lib/par2/md5.h b/lib/par2/md5.h deleted file mode 100644 index e0164c04a..000000000 --- a/lib/par2/md5.h +++ /dev/null @@ -1,158 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __MD5_H__ -#define __MD5_H__ - -#ifdef WIN32 -#pragma pack(push, 1) -#define PACKED -#else -#define PACKED __attribute__ ((packed)) -#endif - -namespace Par2 -{ - -// This file defines the MD5Hash and MD5Context objects which are used -// to compute and manipulate the MD5 Hash values for a block of data. - -// Usage: -// -// MD5Context context; -// context.Update(buffer, length); -// -// MD5Hash hash; -// context.Final(hash); - - - -// MD5 Hash value - -struct MD5Hash; -ostream& operator<<(ostream &s, const MD5Hash &hash); - -struct MD5Hash -{ - // Comparison operators - bool operator==(const MD5Hash &other) const; - bool operator!=(const MD5Hash &other) const; - - bool operator<(const MD5Hash &other) const; - bool operator>=(const MD5Hash &other) const; - bool operator>(const MD5Hash &other) const; - bool operator<=(const MD5Hash &other) const; - - // Convert value to hex - friend ostream& operator<<(ostream &s, const MD5Hash &hash); - string print(void) const; - - u8 hash[16]; // 16 byte MD5 Hash value -} PACKED; - -// Intermediate computation state - -class MD5State -{ -public: - MD5State(void); - void Reset(void); - -public: - void UpdateState(const u32 (&block)[16]); - -protected: - u32 state[4]; // 16 byte MD5 computation state -}; - -// MD5 computation context with 64 byte buffer - -class MD5Context : public MD5State -{ -public: - MD5Context(void); - ~MD5Context(void) {}; - void Reset(void); - - // Process data from a buffer - void Update(const void *buffer, size_t length); - - // Process 0 bytes - void Update(size_t length); - - // Compute the final hash value - void Final(MD5Hash &output); - - // Get the Hash value and the total number of bytes processed. - MD5Hash Hash(void) const; - u64 Bytes(void) const {return bytes;} - - friend ostream& operator<<(ostream &s, const MD5Context &context); - string print(void) const; - -protected: - enum {buffersize = 64}; - unsigned char block[buffersize]; - size_t used; - - u64 bytes; -}; - -// Compare hash values - -inline bool MD5Hash::operator==(const MD5Hash &other) const -{ - return (0==memcmp(&hash, &other.hash, sizeof(hash))); -} -inline bool MD5Hash::operator!=(const MD5Hash &other) const -{ - return !operator==(other); -} - -inline bool MD5Hash::operator<(const MD5Hash &other) const -{ - size_t index = 15; - while (index > 0 && hash[index] == other.hash[index]) - { - index--; - } - - return hash[index] < other.hash[index]; -} -inline bool MD5Hash::operator>=(const MD5Hash &other) const -{ - return !operator<(other); -} -inline bool MD5Hash::operator>(const MD5Hash &other) const -{ - return other.operator<(*this); -} -inline bool MD5Hash::operator<=(const MD5Hash &other) const -{ - return !other.operator<(*this); -} - -} // end namespace Par2 - -#ifdef WIN32 -#pragma pack(pop) -#endif -#undef PACKED - -#endif // __MD5_H__ diff --git a/lib/par2/par2cmdline.h b/lib/par2/par2cmdline.h deleted file mode 100644 index 5240f094f..000000000 --- a/lib/par2/par2cmdline.h +++ /dev/null @@ -1,166 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __PARCMDLINE_H__ -#define __PARCMDLINE_H__ - -#ifdef WIN32 - -#define stat _stat - -namespace Par2 -{ - -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned long u32; -typedef unsigned __int64 u64; - -#ifndef _SIZE_T_DEFINED -# ifdef _WIN64 -typedef unsigned __int64 size_t; -# else -typedef unsigned int size_t; -# endif -# define _SIZE_T_DEFINED -#endif - -} // end namespace Par2 - -#else // WIN32 - -namespace Par2 -{ - -#if HAVE_STDINT_H -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; -#else -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; -typedef unsigned long long u64; -#endif - -} // end namespace Par2 - -#if !HAVE_STRICMP -# define stricmp strcasecmp -#endif - -#define _MAX_PATH 255 - -#endif - -#ifdef WIN32 -#define PATHSEP "\\" -#define ALTPATHSEP "/" -#else -#define PATHSEP "/" -#define ALTPATHSEP "\\" -#endif - -namespace Par2 -{ - -// Return type of par2cmdline -typedef enum Result -{ - eSuccess = 0, - - eRepairPossible = 1, // Data files are damaged and there is - // enough recovery data available to - // repair them. - - eRepairNotPossible = 2, // Data files are damaged and there is - // insufficient recovery data available - // to be able to repair them. - - eInvalidCommandLineArguments = 3, // There was something wrong with the - // command line arguments - - eInsufficientCriticalData = 4, // The PAR2 files did not contain sufficient - // information about the data files to be able - // to verify them. - - eRepairFailed = 5, // Repair completed but the data files - // still appear to be damaged. - - - eFileIOError = 6, // An error occured when accessing files - eLogicError = 7, // In internal error occurred - eMemoryError = 8, // Out of memory - -} Result; - -} // end namespace Par2 - -#define LONGMULTIPLY - -using namespace std; - -#ifdef offsetof -#undef offsetof -#endif -#define offsetof(TYPE, MEMBER) ((size_t) ((char*)(&((TYPE *)1)->MEMBER) - (char*)1)) - -#include "letype.h" -// par2cmdline includes - -#include "galois.h" -#include "crc.h" -#include "md5.h" -#include "par2fileformat.h" -#include "commandline.h" -#include "reedsolomon.h" - -#include "diskfile.h" -#include "datablock.h" - -#include "criticalpacket.h" -//#include "par2creatorsourcefile.h" - -#include "mainpacket.h" -#include "creatorpacket.h" -#include "descriptionpacket.h" -#include "verificationpacket.h" -#include "recoverypacket.h" - -#include "par2repairersourcefile.h" - -#include "filechecksummer.h" -#include "verificationhashtable.h" - -//#include "par2creator.h" -#include "par2repairer.h" - -//#include "par1fileformat.h" -//#include "par1repairersourcefile.h" -//#include "par1repairer.h" - -// Heap checking -#ifdef _MSC_VER -#define _CRTDBG_MAP_ALLOC -#define DEBUG_NEW new(_NORMAL_BLOCK, THIS_FILE, __LINE__) -#endif - -#endif // __PARCMDLINE_H__ - diff --git a/lib/par2/par2fileformat.cpp b/lib/par2/par2fileformat.cpp deleted file mode 100644 index c5678584e..000000000 --- a/lib/par2/par2fileformat.cpp +++ /dev/null @@ -1,33 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -namespace Par2 -{ - -MAGIC packet_magic = {{'P', 'A', 'R', '2', '\0','P', 'K', 'T'}}; -PACKETTYPE fileverificationpacket_type = {{'P', 'A', 'R', ' ', '2', '.', '0', '\0', 'I', 'F', 'S', 'C', '\0','\0','\0','\0'}}; -PACKETTYPE filedescriptionpacket_type = {{'P', 'A', 'R', ' ', '2', '.', '0', '\0', 'F', 'i', 'l', 'e', 'D', 'e', 's', 'c' }}; -PACKETTYPE mainpacket_type = {{'P', 'A', 'R', ' ', '2', '.', '0', '\0', 'M', 'a', 'i', 'n', '\0','\0','\0','\0'}}; -PACKETTYPE recoveryblockpacket_type = {{'P', 'A', 'R', ' ', '2', '.', '0', '\0', 'R', 'e', 'c', 'v', 'S', 'l', 'i', 'c' }}; -PACKETTYPE creatorpacket_type = {{'P', 'A', 'R', ' ', '2', '.', '0', '\0', 'C', 'r', 'e', 'a', 't', 'o', 'r', '\0'}}; - -} // end namespace Par2 diff --git a/lib/par2/par2fileformat.h b/lib/par2/par2fileformat.h deleted file mode 100644 index 6ddecd117..000000000 --- a/lib/par2/par2fileformat.h +++ /dev/null @@ -1,203 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __PAR2FILEFORMAT_H__ -#define __PAR2FILEFORMAT_H__ - -namespace Par2 -{ - -// This file defines the format of a PAR2 file. - -// PAR2 files consist of one or more "packets" that contain information -// that is required to be able to verify and repair damaged data files. - -// All packets start with a short "header" which contains information -// used to describe what sort of data is stored in the rest of the packet -// and also to allow that data to be verified. - -// This file details the format for the following packet types described -// in the PAR 2.0 specification: - -// Main Packet struct MAINPACKET -// File Description Packet struct FILEDESCRIPTIONPACKET -// Input File Slice Checksum Packet struct FILEVERIFICATIONPACKET -// Recovery Slice Packet struct RECOVERYBLOCKPACKET -// Creator Packet struct CREATORPACKET - - -#ifdef WIN32 -#pragma pack(push, 1) -#define PACKED -#else -#define PACKED __attribute__ ((packed)) -#endif - -#ifdef _MSC_VER -#pragma warning(disable:4200) -#endif - -// All numeric fields in the file format are in LITTLE ENDIAN format. - -// The types leu32 and leu64 are defined in letype.h - -// Two simple types used in the packet header. -struct MAGIC {u8 magic[8];} PACKED; -struct PACKETTYPE {u8 type[16];} PACKED; - -// Every packet starts with a packet header. -struct PACKET_HEADER -{ - // Header - MAGIC magic; // = {'P', 'A', 'R', '2', '\0', 'P', 'K', 'T'} - leu64 length; // Length of entire packet including header - MD5Hash hash; // Hash of entire packet excepting the first 3 fields - MD5Hash setid; // Normally computed as the Hash of body of "Main Packet" - PACKETTYPE type; // Used to specify the meaning of the rest of the packet -} PACKED; - -// The file verification packet is used to determine whether or not any -// parts of a damaged file are useable. -// It contains a FileId used to pair it with a corresponding file description -// packet, followed by an array of hash and crc values. The number of entries in -// the array can be determined from the packet_length. -struct FILEVERIFICATIONENTRY -{ - MD5Hash hash; - leu32 crc; -} PACKED; -struct FILEVERIFICATIONPACKET -{ - PACKET_HEADER header; - // Body - MD5Hash fileid; // MD5hash of file_hash_16k, file_length, file_name - FILEVERIFICATIONENTRY entries[]; -} PACKED; - -// The file description packet is used to record the name of the file, -// its size, and the Hash of both the whole file and the first 16k of -// the file. -// If the name of the file is an exact multiple of 4 characters in length -// then it may not have a NULL termination. If the name of the file is not -// an exact multiple of 4, then it will be padded with 0 bytes at the -// end to make it up to a multiple of 4. -struct FILEDESCRIPTIONPACKET -{ - PACKET_HEADER header; - // Body - MD5Hash fileid; // MD5hash of [hash16k, length, name] - MD5Hash hashfull; // MD5 Hash of the whole file - MD5Hash hash16k; // MD5 Hash of the first 16k of the file - leu64 length; // Length of the file - u8 name[]; // Name of the file, padded with 1 to 3 zero bytes to reach - // a multiple of 4 bytes. - // Actual length can be determined from overall packet - // length and then working backwards to find the first non - // zero character. - - //u8* name(void) {return (u8*)&this[1];} - //const u8* name(void) const {return (const u8*)&this[1];} -} PACKED; - -// The main packet is used to tie together the other packets in a recovery file. -// It specifies the block size used to virtually slice the source files, a count -// of the number of source files, and an array of Hash values used to specify -// in what order the source files are processed. -// Each entry in the fileid array corresponds with the fileid value -// in a file description packet and a file verification packet. -// The fileid array may contain more entries than the count of the number -// of recoverable files. The extra entries correspond to files that were not -// used during the creation of the recovery files and which may not therefore -// be repaired if they are found to be damaged. -struct MAINPACKET -{ - PACKET_HEADER header; - // Body - leu64 blocksize; - leu32 recoverablefilecount; - MD5Hash fileid[0]; - //MD5Hash* fileid(void) {return (MD5Hash*)&this[1];} - //const MD5Hash* fileid(void) const {return (const MD5Hash*)&this[1];} -} PACKED; - -// The creator packet is used to identify which program created a particular -// recovery file. It is not required for verification or recovery of damaged -// files. -struct CREATORPACKET -{ - PACKET_HEADER header; - // Body - u8 client[]; - //u8* client(void) {return (u8*)&this[1];} -} PACKED; - -// The recovery block packet contains a single block of recovery data along -// with the exponent value used during the computation of that block. -struct RECOVERYBLOCKPACKET -{ - PACKET_HEADER header; - // Body - leu32 exponent; -// unsigned long data[]; -// unsigned long* data(void) {return (unsigned long*)&this[1];} -} PACKED; - -#ifdef _MSC_VER -#pragma warning(default:4200) -#endif - -#ifdef WIN32 -#pragma pack(pop) -#endif -#undef PACKED - - -// Operators for comparing the MAGIC and PACKETTYPE values - -inline bool operator == (const MAGIC &left, const MAGIC &right) -{ - return (0==memcmp(&left, &right, sizeof(left))); -} - -inline bool operator != (const MAGIC &left, const MAGIC &right) -{ - return !operator==(left, right); -} - -inline bool operator == (const PACKETTYPE &left, const PACKETTYPE &right) -{ - return (0==memcmp(&left, &right, sizeof(left))); -} - -inline bool operator != (const PACKETTYPE &left, const PACKETTYPE &right) -{ - return !operator==(left, right); -} - -extern MAGIC packet_magic; - -extern PACKETTYPE fileverificationpacket_type; -extern PACKETTYPE filedescriptionpacket_type; -extern PACKETTYPE mainpacket_type; -extern PACKETTYPE recoveryblockpacket_type; -extern PACKETTYPE creatorpacket_type; - -} // end namespace Par2 - -#endif //__PAR2FILEFORMAT_H__ diff --git a/lib/par2/par2repairer.cpp b/lib/par2/par2repairer.cpp deleted file mode 100644 index eb7257f65..000000000 --- a/lib/par2/par2repairer.cpp +++ /dev/null @@ -1,2561 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -Par2Repairer::Par2Repairer(std::ostream& cout, std::ostream& cerr): - cout(cout), cerr(cerr), rs(cout, cerr) -{ - firstpacket = true; - mainpacket = 0; - creatorpacket = 0; - - blocksize = 0; - sourceblockcount = 0; - - blocksallocated = false; - - availableblockcount = 0; - missingblockcount = 0; - - completefilecount = 0; - renamedfilecount = 0; - damagedfilecount = 0; - missingfilecount = 0; - - inputbuffer = 0; - outputbuffer = 0; - - noiselevel = CommandLine::nlNormal; - headers = new ParHeaders; - alreadyloaded = false; - - cancelled = false; -} - -Par2Repairer::~Par2Repairer(void) -{ - delete [] (u8*)inputbuffer; - delete [] (u8*)outputbuffer; - - map::iterator rp = recoverypacketmap.begin(); - while (rp != recoverypacketmap.end()) - { - delete (*rp).second; - - ++rp; - } - - map::iterator sf = sourcefilemap.begin(); - while (sf != sourcefilemap.end()) - { - Par2RepairerSourceFile *sourcefile = (*sf).second; - delete sourcefile; - - ++sf; - } - - delete mainpacket; - delete creatorpacket; - delete headers; -} - - -Result Par2Repairer::PreProcess(const CommandLine &commandline) -{ - //sig_filename.emit("coucou cmoa"); - // What noiselevel are we using - noiselevel = commandline.GetNoiseLevel(); - - // Get filesnames from the command line - string par2filename = commandline.GetParFilename(); - const list &extrafiles = commandline.GetExtraFiles(); - - // Determine the searchpath from the location of the main PAR2 file - string name; - DiskFile::SplitFilename(par2filename, searchpath, name); - - // Load packets from the main PAR2 file - if (!LoadPacketsFromFile(searchpath + name)) - return eLogicError; - - // Load packets from other PAR2 files with names based on the original PAR2 file - if (!LoadPacketsFromOtherFiles(par2filename)) - return eLogicError; - - // Load packets from any other PAR2 files whose names are given on the command line - if (!LoadPacketsFromExtraFiles(extrafiles)) - return eLogicError; - - if (noiselevel > CommandLine::nlQuiet) - cout << endl; - - // Check that the packets are consistent and discard any that are not - if (!CheckPacketConsistency()) - return eInsufficientCriticalData; - - // Use the information in the main packet to get the source files - // into the correct order and determine their filenames - if (!CreateSourceFileList()) - return eLogicError; - - // Determine the total number of DataBlocks for the recoverable source files - // The allocate the DataBlocks and assign them to each source file - if (!AllocateSourceBlocks()) - return eLogicError; - - headers->setid = setid.print(); - headers->block_size = blocksize; - headers->chunk_size = chunksize; - headers->data_blocks = sourceblockcount; - headers->data_size = totalsize; - headers->recoverable_files = mainpacket->RecoverableFileCount(); - headers->other_files = - mainpacket->TotalFileCount() - mainpacket->RecoverableFileCount(); - sig_headers(headers); - /* - cout << "Values:" << endl << - "setid: " << setid << endl << - "blocksize: " << blocksize << endl << - "chunksize: " << chunksize << endl << - "sourceblockcount: " << sourceblockcount << endl << - "availableblockcount: " << availableblockcount << endl << - "missingblockcount: " << missingblockcount << endl << - "completefilecount: " << completefilecount << endl << - "renamedfilecount: " << renamedfilecount << endl << - "damagedfilecount: " << damagedfilecount << endl << - "missingfilecount: " << missingfilecount << endl << - "progress: " << progress << endl << - "totaldata: " << totaldata << endl << - "totalsize: " << totalsize << endl << - "RecoverableFileCount: " << mainpacket->RecoverableFileCount() << endl << - "TotalFileCount: " << mainpacket->TotalFileCount() << endl; - */ - - return eSuccess; -} - - -Result Par2Repairer::Process(const CommandLine &commandline, bool dorepair) -{ - //sig_filename.emit("coucou cmoa"); - // What noiselevel are we using - noiselevel = commandline.GetNoiseLevel(); - - // Get filesnames from the command line - string par2filename = commandline.GetParFilename(); - const list &extrafiles = commandline.GetExtraFiles(); - - // Determine the searchpath from the location of the main PAR2 file - string name; - DiskFile::SplitFilename(par2filename, searchpath, name); - - // Load packets from the main PAR2 file - /* if (!LoadPacketsFromFile(searchpath + name)) - return eLogicError; - - // Load packets from other PAR2 files with names based on the original PAR2 file - if (!LoadPacketsFromOtherFiles(par2filename)) - return eLogicError; - - // Load packets from any other PAR2 files whose names are given on the command line - if (!LoadPacketsFromExtraFiles(extrafiles)) - return eLogicError; - - if (noiselevel > CommandLine::nlQuiet) - cout << endl; - - // Check that the packets are consistent and discard any that are not - if (!CheckPacketConsistency()) - return eInsufficientCriticalData; - - // Use the information in the main packet to get the source files - // into the correct order and determine their filenames - if (!CreateSourceFileList()) - return eLogicError; - - // Determine the total number of DataBlocks for the recoverable source files - // The allocate the DataBlocks and assign them to each source file - if (!AllocateSourceBlocks()) - return eLogicError; - */ - // ---------- /Header ------------- - - // Create a verification hash table for all files for which we have not - // found a complete version of the file and for which we have - // a verification packet - if (!alreadyloaded) { - if (!PrepareVerificationHashTable()) - return eLogicError; - - // Compute the table for the sliding CRC computation - if (!ComputeWindowTable()) - return eLogicError; - - if (noiselevel > CommandLine::nlQuiet) - cout << endl << "Verifying source files:" << endl << endl; - - // Attempt to verify all of the source files - if (!VerifySourceFiles()) - return eFileIOError; - - if (completefilecountRecoverableFileCount()) - { - if (noiselevel > CommandLine::nlQuiet) - cout << endl << "Scanning extra files:" << endl << endl; - - // Scan any extra files specified on the command line - if (!VerifyExtraFiles(extrafiles)) - return eLogicError; - } - - // Find out how much data we have found - UpdateVerificationResults(); - alreadyloaded = true; - } - if (noiselevel > CommandLine::nlSilent) - cout << endl; - - // Check the verification results and report the results - if (!CheckVerificationResults()) - return eRepairNotPossible; - - // Are any of the files incomplete - if (completefilecountRecoverableFileCount()) - { - // Do we want to carry out a repair - if (dorepair) - { - if (noiselevel > CommandLine::nlSilent) - cout << endl; - - // Rename any damaged or missnamed target files. - if (!RenameTargetFiles()) - return eFileIOError; - - // Are we still missing any files - if (completefilecountRecoverableFileCount()) - { - // Work out which files are being repaired, create them, and allocate - // target DataBlocks to them, and remember them for later verification. - if (!CreateTargetFiles()) - return eFileIOError; - - // Work out which data blocks are available, which need to be copied - // directly to the output, and which need to be recreated, and compute - // the appropriate Reed Solomon matrix. - if (!ComputeRSmatrix()) - { - // Delete all of the partly reconstructed files - DeleteIncompleteTargetFiles(); - return eFileIOError; - } - - if (noiselevel > CommandLine::nlSilent) - cout << endl; - - // Allocate memory buffers for reading and writing data to disk. - if (!AllocateBuffers(commandline.GetMemoryLimit())) - { - // Delete all of the partly reconstructed files - DeleteIncompleteTargetFiles(); - return eMemoryError; - } - - // Set the total amount of data to be processed. - progress = 0; - totaldata = blocksize * sourceblockcount * (missingblockcount > 0 ? missingblockcount : 1); - - BeginRepair(); - - // Start at an offset of 0 within a block. - u64 blockoffset = 0; - while (blockoffset < blocksize) // Continue until the end of the block. - { - // Work out how much data to process this time. - size_t blocklength = (size_t)min((u64)chunksize, blocksize-blockoffset); - - // Read source data, process it through the RS matrix and write it to disk. - - if (!ProcessData(blockoffset, blocklength)) - { - // Delete all of the partly reconstructed files - DeleteIncompleteTargetFiles(); - EndRepair(); - return eFileIOError; - } - - // Advance to the need offset within each block - blockoffset += blocklength; - } - - EndRepair(); - - if (noiselevel > CommandLine::nlSilent) - cout << endl << "Verifying repaired files:" << endl << endl; - - // Verify that all of the reconstructed target files are now correct - if (!VerifyTargetFiles()) - { - // Delete all of the partly reconstructed files - DeleteIncompleteTargetFiles(); - return eFileIOError; - } - } - - // Are all of the target files now complete? - if (completefilecountRecoverableFileCount()) - { - cerr << "Repair Failed." << endl; - return eRepairFailed; - } - else - { - if (noiselevel > CommandLine::nlSilent) - cout << endl << "Repair complete." << endl; - } - } - else - { - return eRepairPossible; - } - } - - return eSuccess; -} - -// Load the packets from the specified file -bool Par2Repairer::LoadPacketsFromFile(string filename) -{ - // Skip the file if it has already been processed - if (diskFileMap.Find(filename) != 0) - { - return true; - } - - DiskFile *diskfile = new DiskFile(cerr); - - // Open the file - if (!diskfile->Open(filename)) - { - // If we could not open the file, ignore the error and - // proceed to the next file - delete diskfile; - return true; - } - - if (noiselevel > CommandLine::nlSilent) - { - string path; - string name; - DiskFile::SplitFilename(filename, path, name); - cout << "Loading \"" << name << "\"." << endl; - sig_filename(name); - } - - // How many useable packets have we found - u32 packets = 0; - - // How many recovery packets were there - u32 recoverypackets = 0; - - // How big is the file - u64 filesize = diskfile->FileSize(); - if (filesize > 0) - { - // Allocate a buffer to read data into - // The buffer should be large enough to hold a whole - // critical packet (i.e. file verification, file description, main, - // and creator), but not necessarily a whole recovery packet. - size_t buffersize = (size_t)min((u64)1048576, filesize); - u8 *buffer = new u8[buffersize]; - - // Progress indicator - u64 progress = 0; - - // Start at the beginning of the file - u64 offset = 0; - - // Continue as long as there is at least enough for the packet header - while (offset + sizeof(PACKET_HEADER) <= filesize) - { - if (noiselevel > CommandLine::nlQuiet) - { - // Update a progress indicator - u32 oldfraction = (u32)(1000 * progress / filesize); - u32 newfraction = (u32)(1000 * offset / filesize); - if (oldfraction != newfraction) - { - cout << "Loading: " << newfraction/10 << '.' << newfraction%10 << "%\r" << flush; - progress = offset; - sig_progress(newfraction); - - if (cancelled) - { - break; - } - } - } - - // Attempt to read the next packet header - PACKET_HEADER header; - if (!diskfile->Read(offset, &header, sizeof(header))) - break; - - // Does this look like it might be a packet - if (packet_magic != header.magic) - { - offset++; - - // Is there still enough for at least a whole packet header - while (offset + sizeof(PACKET_HEADER) <= filesize) - { - // How much can we read into the buffer - size_t want = (size_t)min((u64)buffersize, filesize-offset); - - // Fill the buffer - if (!diskfile->Read(offset, buffer, want)) - { - offset = filesize; - break; - } - - // Scan the buffer for the magic value - u8 *current = buffer; - u8 *limit = &buffer[want-sizeof(PACKET_HEADER)]; - while (current <= limit && packet_magic != ((PACKET_HEADER*)current)->magic) - { - current++; - } - - // What file offset did we reach - offset += current-buffer; - - // Did we find the magic - if (current <= limit) - { - memcpy(&header, current, sizeof(header)); - break; - } - } - - // Did we reach the end of the file - if (offset + sizeof(PACKET_HEADER) > filesize) - { - break; - } - } - - // We have found the magic - - // Check the packet length - if (sizeof(PACKET_HEADER) > header.length || // packet length is too small - 0 != (header.length & 3) || // packet length is not a multiple of 4 - filesize < offset + header.length) // packet would extend beyond the end of the file - { - offset++; - continue; - } - - // Compute the MD5 Hash of the packet - MD5Context context; - context.Update(&header.setid, sizeof(header)-offsetof(PACKET_HEADER, setid)); - - // How much more do I need to read to get the whole packet - u64 current = offset+sizeof(PACKET_HEADER); - u64 limit = offset+header.length; - while (current < limit) - { - size_t want = (size_t)min((u64)buffersize, limit-current); - - if (!diskfile->Read(current, buffer, want)) - break; - - context.Update(buffer, want); - - current += want; - } - - // Did the whole packet get processed - if (currentClose(); - - // Did we actually find any interesting packets - if (packets > 0) - { - if (noiselevel > CommandLine::nlQuiet) - { - cout << "Loaded " << packets << " new packets"; - if (recoverypackets > 0) cout << " including " << recoverypackets << " recovery blocks"; - cout << endl; - } - - // Remember that the file was processed - bool success = diskFileMap.Insert(diskfile); - assert(success); (void)success; - } - else - { - if (noiselevel > CommandLine::nlQuiet) - cout << "No new packets found" << endl; - delete diskfile; - } - - if (cancelled) - { - return false; - } - - return true; -} - -// Finish loading a recovery packet -bool Par2Repairer::LoadRecoveryPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - RecoveryPacket *packet = new RecoveryPacket; - - // Load the packet from disk - if (!packet->Load(diskfile, offset, header)) - { - delete packet; - return false; - } - - // What is the exponent value of this recovery packet - u32 exponent = packet->Exponent(); - - // Try to insert the new packet into the recovery packet map - pair::const_iterator, bool> location = recoverypacketmap.insert(pair(exponent, packet)); - - // Did the insert fail - if (!location.second) - { - // The packet must be a duplicate of one we already have - delete packet; - return false; - } - - return true; -} - -// Finish loading a file description packet -bool Par2Repairer::LoadDescriptionPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - DescriptionPacket *packet = new DescriptionPacket; - - // Load the packet from disk - if (!packet->Load(diskfile, offset, header)) - { - delete packet; - return false; - } - - // What is the fileid - const MD5Hash &fileid = packet->FileId(); - - // Look up the fileid in the source file map for an existing source file entry - map::iterator sfmi = sourcefilemap.find(fileid); - Par2RepairerSourceFile *sourcefile = (sfmi == sourcefilemap.end()) ? 0 :sfmi->second; - - // Was there an existing source file - if (sourcefile) - { - // Does the source file already have a description packet - if (sourcefile->GetDescriptionPacket()) - { - // Yes. We don't need another copy - delete packet; - return false; - } - else - { - // No. Store the packet in the source file - sourcefile->SetDescriptionPacket(packet); - return true; - } - } - else - { - // Create a new source file for the packet - sourcefile = new Par2RepairerSourceFile(packet, NULL); - - // Record the source file in the source file map - sourcefilemap.insert(pair(fileid, sourcefile)); - - return true; - } -} - -// Finish loading a file verification packet -bool Par2Repairer::LoadVerificationPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - VerificationPacket *packet = new VerificationPacket; - - // Load the packet from disk - if (!packet->Load(diskfile, offset, header)) - { - delete packet; - return false; - } - - // What is the fileid - const MD5Hash &fileid = packet->FileId(); - - // Look up the fileid in the source file map for an existing source file entry - map::iterator sfmi = sourcefilemap.find(fileid); - Par2RepairerSourceFile *sourcefile = (sfmi == sourcefilemap.end()) ? 0 :sfmi->second; - - // Was there an existing source file - if (sourcefile) - { - // Does the source file already have a verification packet - if (sourcefile->GetVerificationPacket()) - { - // Yes. We don't need another copy. - delete packet; - return false; - } - else - { - // No. Store the packet in the source file - sourcefile->SetVerificationPacket(packet); - - return true; - } - } - else - { - // Create a new source file for the packet - sourcefile = new Par2RepairerSourceFile(NULL, packet); - - // Record the source file in the source file map - sourcefilemap.insert(pair(fileid, sourcefile)); - - return true; - } -} - -// Finish loading the main packet -bool Par2Repairer::LoadMainPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - // Do we already have a main packet - if (0 != mainpacket) - return false; - - MainPacket *packet = new MainPacket; - - // Load the packet from disk; - if (!packet->Load(diskfile, offset, header)) - { - delete packet; - return false; - } - - mainpacket = packet; - - return true; -} - -// Finish loading the creator packet -bool Par2Repairer::LoadCreatorPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - // Do we already have a creator packet - if (0 != creatorpacket) - return false; - - CreatorPacket *packet = new CreatorPacket; - - // Load the packet from disk; - if (!packet->Load(diskfile, offset, header)) - { - delete packet; - return false; - } - - creatorpacket = packet; - - return true; -} - -// Load packets from other PAR2 files with names based on the original PAR2 file -bool Par2Repairer::LoadPacketsFromOtherFiles(string filename) -{ - // Split the original PAR2 filename into path and name parts - string path; - string name; - DiskFile::SplitFilename(filename, path, name); - - string::size_type where; - - // Trim ".par2" off of the end original name - - // Look for the last "." in the filename - while (string::npos != (where = name.find_last_of('.'))) - { - // Trim what follows the last . - string tail = name.substr(where+1); - name = name.substr(0,where); - - // Was what followed the last "." "par2" - if (0 == stricmp(tail.c_str(), "par2")) - break; - } - - // If what is left ends in ".volNNN-NNN" or ".volNNN+NNN" strip that as well - - // Is there another "." - if (string::npos != (where = name.find_last_of('.'))) - { - // What follows the "." - string tail = name.substr(where+1); - - // Scan what follows the last "." to see of it matches vol123-456 or vol123+456 - int n = 0; - string::const_iterator p; - for (p=tail.begin(); p!=tail.end(); ++p) - { - char ch = *p; - - if (0 == n) - { - if (tolower(ch) == 'v') { n++; } else { break; } - } - else if (1 == n) - { - if (tolower(ch) == 'o') { n++; } else { break; } - } - else if (2 == n) - { - if (tolower(ch) == 'l') { n++; } else { break; } - } - else if (3 == n) - { - if (isdigit(ch)) {} else if (ch == '-' || ch == '+') { n++; } else { break; } - } - else if (4 == n) - { - if (isdigit(ch)) {} else { break; } - } - } - - // If we matched then retain only what preceeds the "." - if (p == tail.end()) - { - name = name.substr(0,where); - } - } - - // Find files called "*.par2" or "name.*.par2" - - { - string wildcard = name.empty() ? "*.par2" : name + ".*.par2"; - list *files = DiskFile::FindFiles(path, wildcard); - - // Load packets from each file that was found - for (list::const_iterator s=files->begin(); s!=files->end(); ++s) - { - LoadPacketsFromFile(*s); - if (cancelled) - { - break; - } - } - - delete files; - if (cancelled) - { - return false; - } - } - - { - string wildcard = name.empty() ? "*.PAR2" : name + ".*.PAR2"; - list *files = DiskFile::FindFiles(path, wildcard); - - // Load packets from each file that was found - for (list::const_iterator s=files->begin(); s!=files->end(); ++s) - { - LoadPacketsFromFile(*s); - if (cancelled) - { - break; - } - } - - delete files; - if (cancelled) - { - return false; - } - } - - return true; -} - -// Load packets from any other PAR2 files whose names are given on the command line -bool Par2Repairer::LoadPacketsFromExtraFiles(const list &extrafiles) -{ - for (ExtraFileIterator i=extrafiles.begin(); i!=extrafiles.end(); i++) - { - string filename = i->FileName(); - - // If the filename contains ".par2" anywhere - if (string::npos != filename.find(".par2") || - string::npos != filename.find(".PAR2")) - { - LoadPacketsFromFile(filename); - if (cancelled) - { - break; - } - } - } - - if (cancelled) - { - return false; - } - - return true; -} - -// Check that the packets are consistent and discard any that are not -bool Par2Repairer::CheckPacketConsistency(void) -{ - // Do we have a main packet - if (0 == mainpacket) - { - // If we don't have a main packet, then there is nothing more that we can do. - // We cannot verify or repair any files. - - cerr << "Main packet not found." << endl; - return false; - } - - // Remember the block size from the main packet - blocksize = mainpacket->BlockSize(); - - // Check that the recovery blocks have the correct amount of data - // and discard any that don't - { - map::iterator rp = recoverypacketmap.begin(); - while (rp != recoverypacketmap.end()) - { - if (rp->second->BlockSize() == blocksize) - { - ++rp; - } - else - { - cerr << "Incorrect sized recovery block for exponent " << rp->second->Exponent() << " discarded" << endl; - - delete rp->second; - map::iterator x = rp++; - recoverypacketmap.erase(x); - } - } - } - - // Check for source files that have no description packet or where the - // verification packet has the wrong number of entries and discard them. - { - map::iterator sf = sourcefilemap.begin(); - while (sf != sourcefilemap.end()) - { - // Do we have a description packet - DescriptionPacket *descriptionpacket = sf->second->GetDescriptionPacket(); - if (descriptionpacket == 0) - { - // No description packet - - // Discard the source file - delete sf->second; - map::iterator x = sf++; - sourcefilemap.erase(x); - - continue; - } - - // Compute and store the block count from the filesize and blocksize - sf->second->SetBlockCount(blocksize); - - // Do we have a verification packet - VerificationPacket *verificationpacket = sf->second->GetVerificationPacket(); - if (verificationpacket == 0) - { - // No verification packet - - // That is ok, but we won't be able to use block verification. - - // Proceed to the next file. - ++sf; - - continue; - } - - // Work out the block count for the file from the file size - // and compare that with the verification packet - u64 filesize = descriptionpacket->FileSize(); - u32 blockcount = verificationpacket->BlockCount(); - - if ((filesize + blocksize-1) / blocksize != (u64)blockcount) - { - // The block counts are different! - - cerr << "Incorrectly sized verification packet for \"" << descriptionpacket->FileName() << "\" discarded" << endl; - - // Discard the source file - - delete sf->second; - map::iterator x = sf++; - sourcefilemap.erase(x); - - continue; - } - - // Everything is ok. - - // Proceed to the next file - ++sf; - } - } - - if (noiselevel > CommandLine::nlQuiet) - { - cout << "There are " - << mainpacket->RecoverableFileCount() - << " recoverable files and " - << mainpacket->TotalFileCount() - mainpacket->RecoverableFileCount() - << " other files." - << endl; - - cout << "The block size used was " - << blocksize - << " bytes." - << endl; - } - - return true; -} - -// Use the information in the main packet to get the source files -// into the correct order and determine their filenames -bool Par2Repairer::CreateSourceFileList(void) -{ - // For each FileId entry in the main packet - for (u32 filenumber=0; filenumberTotalFileCount(); filenumber++) - { - const MD5Hash &fileid = mainpacket->FileId(filenumber); - - // Look up the fileid in the source file map - map::iterator sfmi = sourcefilemap.find(fileid); - Par2RepairerSourceFile *sourcefile = (sfmi == sourcefilemap.end()) ? 0 :sfmi->second; - - if (sourcefile) - { - sourcefile->ComputeTargetFileName(searchpath); - } - - sourcefiles.push_back(sourcefile); - } - - return true; -} - -// Determine the total number of DataBlocks for the recoverable source files -// The allocate the DataBlocks and assign them to each source file -bool Par2Repairer::AllocateSourceBlocks(void) -{ - sourceblockcount = 0; - - u32 filenumber = 0; - vector::iterator sf = sourcefiles.begin(); - - // For each recoverable source file - while (filenumber < mainpacket->RecoverableFileCount() && sf != sourcefiles.end()) - { - // Do we have a source file - Par2RepairerSourceFile *sourcefile = *sf; - if (sourcefile) - { - sourceblockcount += sourcefile->BlockCount(); - } - else - { - // No details for this source file so we don't know what the - // total number of source blocks is -// sourceblockcount = 0; -// break; - } - - ++sf; - ++filenumber; - } - - // Did we determine the total number of source blocks - if (sourceblockcount > 0) - { - // Yes. - - // Allocate all of the Source and Target DataBlocks (which will be used - // to read and write data to disk). - - sourceblocks.resize(sourceblockcount); - targetblocks.resize(sourceblockcount); - - // Which DataBlocks will be allocated first - vector::iterator sourceblock = sourceblocks.begin(); - vector::iterator targetblock = targetblocks.begin(); - - u32 blocknumber = 0; - totalsize = 0; - - filenumber = 0; - sf = sourcefiles.begin(); - - while (filenumber < mainpacket->RecoverableFileCount() && sf != sourcefiles.end()) - { - Par2RepairerSourceFile *sourcefile = *sf; - - if (sourcefile) - { - totalsize += sourcefile->GetDescriptionPacket()->FileSize(); - u32 blockcount = sourcefile->BlockCount(); - - // Allocate the source and target DataBlocks to the sourcefile - sourcefile->SetBlocks(blocknumber, blockcount, sourceblock, targetblock, blocksize); - - blocknumber++; - - sourceblock += blockcount; - targetblock += blockcount; - } - - ++sf; - ++filenumber; - } - - blocksallocated = true; - - if (noiselevel > CommandLine::nlQuiet) - { - cout << "There are a total of " - << sourceblockcount - << " data blocks." - << endl; - - cout << "The total size of the data files is " - << totalsize - << " bytes." - << endl; - } - } - - return true; -} - -// Create a verification hash table for all files for which we have not -// found a complete version of the file and for which we have -// a verification packet -bool Par2Repairer::PrepareVerificationHashTable(void) -{ - // Choose a size for the hash table - verificationhashtable.SetLimit(sourceblockcount); - - // Will any files be block verifiable - blockverifiable = false; - - // For each source file - vector::iterator sf = sourcefiles.begin(); - while (sf != sourcefiles.end()) - { - // Get the source file - Par2RepairerSourceFile *sourcefile = *sf; - - if (sourcefile) - { - // Do we have a verification packet - if (0 != sourcefile->GetVerificationPacket()) - { - // Yes. Load the verification entries into the hash table - verificationhashtable.Load(sourcefile, blocksize); - - blockverifiable = true; - } - else - { - // No. We can only check the whole file - unverifiablesourcefiles.push_back(sourcefile); - } - } - - ++sf; - } - - return true; -} - -// Compute the table for the sliding CRC computation -bool Par2Repairer::ComputeWindowTable(void) -{ - if (blockverifiable) - { - GenerateWindowTable(blocksize, windowtable); - windowmask = ComputeWindowMask(blocksize); - } - - return true; -} - -static bool SortSourceFilesByFileName(Par2RepairerSourceFile *low, - Par2RepairerSourceFile *high) -{ - return low->TargetFileName() < high->TargetFileName(); -} - -// Attempt to verify all of the source files -bool Par2Repairer::VerifySourceFiles(void) -{ - bool finalresult = true; - - // Created a sorted list of the source files and verify them in that - // order rather than the order they are in the main packet. - vector sortedfiles; - - u32 filenumber = 0; - vector::iterator sf = sourcefiles.begin(); - while (sf != sourcefiles.end()) - { - // Do we have a source file - Par2RepairerSourceFile *sourcefile = *sf; - if (sourcefile) - { - sortedfiles.push_back(sourcefile); - } - else - { - // Was this one of the recoverable files - if (filenumber < mainpacket->RecoverableFileCount()) - { - cerr << "No details available for recoverable file number " << filenumber+1 << "." << endl << "Recovery will not be possible." << endl; - - // Set error but let verification of other files continue - finalresult = false; - } - else - { - cerr << "No details available for non-recoverable file number " << filenumber - mainpacket->RecoverableFileCount() + 1 << endl; - } - } - - ++sf; - } - - sort(sortedfiles.begin(), sortedfiles.end(), SortSourceFilesByFileName); - - // Start verifying the files - sf = sortedfiles.begin(); - while (sf != sortedfiles.end()) - { - if (cancelled) - { - return false; - } - - // Do we have a source file - Par2RepairerSourceFile *sourcefile = *sf; - - // What filename does the file use - string filename = sourcefile->TargetFileName(); - - // Check to see if we have already used this file - if (diskFileMap.Find(filename) != 0) - { - // The file has already been used! - - cerr << "Source file " << filenumber+1 << " is a duplicate." << endl; - - return false; - } - - DiskFile *diskfile = new DiskFile(cerr); - - // Does the target file exist - if (diskfile->Open(filename)) - { - // Yes. Record that fact. - sourcefile->SetTargetExists(true); - - // Remember that the DiskFile is the target file - sourcefile->SetTargetFile(diskfile); - - // Remember that we have processed this file - bool success = diskFileMap.Insert(diskfile); - assert(success); (void)success; - // Do the actual verification - if (!VerifyDataFile(diskfile, sourcefile)) - finalresult = false; - - // We have finished with the file for now - diskfile->Close(); - - // Find out how much data we have found - UpdateVerificationResults(); - } - else - { - // The file does not exist. - delete diskfile; - - if (noiselevel > CommandLine::nlSilent) - { - string path; - string name; - DiskFile::SplitFilename(filename, path, name); - - cout << "Target: \"" << name << "\" - missing." << endl; - sig_done(name, 0, sourcefile && sourcefile->GetVerificationPacket() ? sourcefile->GetVerificationPacket()->BlockCount() : 0); - } - } - - ++sf; - } - - return finalresult; -} - -// Scan any extra files specified on the command line -bool Par2Repairer::VerifyExtraFiles(const list &extrafiles) -{ - for (ExtraFileIterator i=extrafiles.begin(); - i!=extrafiles.end() && completefilecountRecoverableFileCount(); - ++i) - { - string filename = i->FileName(); - - // If the filename does not include ".par2" we are interested in it. - if (string::npos == filename.find(".par2") && - string::npos == filename.find(".PAR2")) - { - filename = DiskFile::GetCanonicalPathname(filename); - - // Has this file already been dealt with - if (diskFileMap.Find(filename) == 0) - { - DiskFile *diskfile = new DiskFile(cerr); - - // Does the file exist - if (!diskfile->Open(filename)) - { - delete diskfile; - continue; - } - - // Remember that we have processed this file - bool success = diskFileMap.Insert(diskfile); - assert(success); (void)success; - - // Do the actual verification - VerifyDataFile(diskfile, 0); - // Ignore errors - - // We have finished with the file for now - diskfile->Close(); - - // Find out how much data we have found - UpdateVerificationResults(); - } - } - } - - return true; -} - -// Attempt to match the data in the DiskFile with the source file -bool Par2Repairer::VerifyDataFile(DiskFile *diskfile, Par2RepairerSourceFile *sourcefile) -{ - MatchType matchtype; // What type of match was made - MD5Hash hashfull; // The MD5 Hash of the whole file - MD5Hash hash16k; // The MD5 Hash of the files 16k of the file - - // Are there any files that can be verified at the block level - if (blockverifiable) - { - u32 count; - - // Scan the file at the block level. - - if (!ScanDataFile(diskfile, // [in] The file to scan - sourcefile, // [in/out] Modified in the match is for another source file - matchtype, // [out] - hashfull, // [out] - hash16k, // [out] - count)) // [out] - return false; - - switch (matchtype) - { - case eNoMatch: - // No data was found at all. - - // Continue to next test. - break; - case ePartialMatch: - { - // We found some data. - - // Return them. - return true; - } - break; - case eFullMatch: - { - // We found a perfect match. - - sourcefile->SetCompleteFile(diskfile); - - // Return the match - return true; - } - break; - } - } - - // We did not find a match for any blocks of data within the file, but if - // there are any files for which we did not have a verification packet - // we can try a simple match of the hash for the whole file. - - // Are there any files that cannot be verified at the block level - if (unverifiablesourcefiles.size() > 0) - { - // Would we have already computed the file hashes - if (!blockverifiable) - { - u64 filesize = diskfile->FileSize(); - - size_t buffersize = 1024*1024; - if (buffersize > min(blocksize, filesize)) - buffersize = (size_t)min(blocksize, filesize); - - char *buffer = new char[buffersize]; - - u64 offset = 0; - - MD5Context context; - - while (offset < filesize) - { - size_t want = (size_t)min((u64)buffersize, filesize-offset); - - if (!diskfile->Read(offset, buffer, want)) - { - delete [] buffer; - return false; - } - - // Will the newly read data reach the 16k boundary - if (offset < 16384 && offset + want >= 16384) - { - context.Update(buffer, (size_t)(16384-offset)); - - // Compute the 16k hash - MD5Context temp = context; - temp.Final(hash16k); - - // Is there more data - if (offset + want > 16384) - { - context.Update(&buffer[16384-offset], (size_t)(offset+want)-16384); - } - } - else - { - context.Update(buffer, want); - } - - offset += want; - } - - // Compute the file hash - MD5Hash hashfull; - context.Final(hashfull); - - // If we did not have 16k of data, then the 16k hash - // is the same as the full hash - if (filesize < 16384) - { - hash16k = hashfull; - } - } - - list::iterator sf = unverifiablesourcefiles.begin(); - - // Compare the hash values of each source file for a match - while (sf != unverifiablesourcefiles.end()) - { - sourcefile = *sf; - - // Does the file match - if (sourcefile->GetCompleteFile() == 0 && - diskfile->FileSize() == sourcefile->GetDescriptionPacket()->FileSize() && - hash16k == sourcefile->GetDescriptionPacket()->Hash16k() && - hashfull == sourcefile->GetDescriptionPacket()->HashFull()) - { - if (noiselevel > CommandLine::nlSilent) - cout << diskfile->FileName() << " is a perfect match for " << sourcefile->GetDescriptionPacket()->FileName() << endl; - - // Record that we have a perfect match for this source file - sourcefile->SetCompleteFile(diskfile); - - if (blocksallocated) - { - // Allocate all of the DataBlocks for the source file to the DiskFile - - u64 offset = 0; - u64 filesize = sourcefile->GetDescriptionPacket()->FileSize(); - - vector::iterator sb = sourcefile->SourceBlocks(); - - while (offset < filesize) - { - DataBlock &datablock = *sb; - - datablock.SetLocation(diskfile, offset); - datablock.SetLength(min(blocksize, filesize-offset)); - - offset += blocksize; - ++sb; - } - } - - // Return the match - return true; - } - - ++sf; - } - } - - return true; -} - -// Perform a sliding window scan of the DiskFile looking for blocks of data that -// might belong to any of the source files (for which a verification packet was -// available). If a block of data might be from more than one source file, prefer -// the one specified by the "sourcefile" parameter. If the first data block -// found is for a different source file then "sourcefile" is changed accordingly. -bool Par2Repairer::ScanDataFile(DiskFile *diskfile, // [in] - Par2RepairerSourceFile* &sourcefile, // [in/out] - MatchType &matchtype, // [out] - MD5Hash &hashfull, // [out] - MD5Hash &hash16k, // [out] - u32 &count) // [out] -{ - // Remember which file we wanted to match - Par2RepairerSourceFile *originalsourcefile = sourcefile; - - matchtype = eNoMatch; - - // Is the file empty - if (diskfile->FileSize() == 0) - { - // If the file is empty, then just return - return true; - } - - string path; - string name; - DiskFile::SplitFilename(diskfile->FileName(), path, name); - - sig_filename(name); - - string shortname; - if (name.size() > 56) - { - shortname = name.substr(0, 28) + "..." + name.substr(name.size()-28); - } - else - { - shortname = name; - } - - // Create the checksummer for the file and start reading from it - FileCheckSummer filechecksummer(diskfile, blocksize, windowtable, windowmask); - if (!filechecksummer.Start()) - return false; - - // Assume we will make a perfect match for the file - matchtype = eFullMatch; - - // How many matches have we had - count = 0; - - // How many blocks have already been found - u32 duplicatecount = 0; - - // Have we found data blocks in this file that belong to more than one target file - bool multipletargets = false; - - // Which block do we expect to find first - const VerificationHashEntry *nextentry = 0; - - u64 progress = 0; - - // Whilst we have not reached the end of the file - while (filechecksummer.Offset() < diskfile->FileSize()) - { - if (noiselevel > CommandLine::nlQuiet) - { - // Update a progress indicator - u32 oldfraction = (u32)(1000 * progress / diskfile->FileSize()); - u32 newfraction = (u32)(1000 * (progress = filechecksummer.Offset()) / diskfile->FileSize()); - if (oldfraction != newfraction) - { - cout << "Scanning: \"" << shortname << "\": " << newfraction/10 << '.' << newfraction%10 << "%\r" << flush; - sig_progress(newfraction); - - if (cancelled) - { - break; - } - } - } - - // If we fail to find a match, it might be because it was a duplicate of a block - // that we have already found. - bool duplicate; - - // Look for a match - const VerificationHashEntry *currententry = verificationhashtable.FindMatch(nextentry, sourcefile, filechecksummer, duplicate); - - // Did we find a match - if (currententry != 0) - { - // Is this the first match - if (count == 0) - { - // Which source file was it - sourcefile = currententry->SourceFile(); - - // If the first match found was not actually the first block - // for the source file, or it was not at the start of the - // data file: then this is a partial match. - if (!currententry->FirstBlock() || filechecksummer.Offset() != 0) - { - matchtype = ePartialMatch; - } - } - else - { - // If the match found is not the one which was expected - // then this is a partial match - - if (currententry != nextentry) - { - matchtype = ePartialMatch; - } - - // Is the match from a different source file - if (sourcefile != currententry->SourceFile()) - { - multipletargets = true; - } - } - - if (blocksallocated) - { - // Record the match - currententry->SetBlock(diskfile, filechecksummer.Offset()); - } - - // Update the number of matches found - count++; - - // What entry do we expect next - nextentry = currententry->Next(); - - // Advance to the next block - if (!filechecksummer.Jump(currententry->GetDataBlock()->GetLength())) - return false; - } - else - { - // This cannot be a perfect match - matchtype = ePartialMatch; - - // Was this a duplicate match - if (duplicate) - { - duplicatecount++; - - // What entry would we expect next - nextentry = 0; - - // Advance one whole block - if (!filechecksummer.Jump(blocksize)) - return false; - } - else - { - // What entry do we expect next - nextentry = 0; - - // Advance 1 byte - if (!filechecksummer.Step()) - return false; - } - } - } - - if (cancelled) - { - return false; - } - - // Get the Full and 16k hash values of the file - filechecksummer.GetFileHashes(hashfull, hash16k); - - // Did we make any matches at all - if (count > 0) - { - // If this still might be a perfect match, check the - // hashes, file size, and number of blocks to confirm. - if (matchtype != eFullMatch || - count != sourcefile->GetVerificationPacket()->BlockCount() || - diskfile->FileSize() != sourcefile->GetDescriptionPacket()->FileSize() || - hashfull != sourcefile->GetDescriptionPacket()->HashFull() || - hash16k != sourcefile->GetDescriptionPacket()->Hash16k()) - { - matchtype = ePartialMatch; - - if (noiselevel > CommandLine::nlSilent) - { - // Did we find data from multiple target files - if (multipletargets) - { - // Were we scanning the target file or an extra file - if (originalsourcefile != 0) - { - cout << "Target: \"" - << name - << "\" - damaged, found " - << count - << " data blocks from several target files." - << endl; - } - else - { - cout << "File: \"" - << name - << "\" - found " - << count - << " data blocks from several target files." - << endl; - } - } - else - { - // Did we find data blocks that belong to the target file - if (originalsourcefile == sourcefile) - { - cout << "Target: \"" - << name - << "\" - damaged. Found " - << count - << " of " - << sourcefile->GetVerificationPacket()->BlockCount() - << " data blocks." - << endl; - } - // Were we scanning the target file or an extra file - else if (originalsourcefile != 0) - { - string targetname; - DiskFile::SplitFilename(sourcefile->TargetFileName(), path, targetname); - - cout << "Target: \"" - << name - << "\" - damaged. Found " - << count - << " of " - << sourcefile->GetVerificationPacket()->BlockCount() - << " data blocks from \"" - << targetname - << "\"." - << endl; - } - else - { - string targetname; - DiskFile::SplitFilename(sourcefile->TargetFileName(), path, targetname); - - cout << "File: \"" - << name - << "\" - found " - << count - << " of " - << sourcefile->GetVerificationPacket()->BlockCount() - << " data blocks from \"" - << targetname - << "\"." - << endl; - } - } - } - } - else - { - if (noiselevel > CommandLine::nlSilent) - { - // Did we match the target file - if (originalsourcefile == sourcefile) - { - cout << "Target: \"" << name << "\" - found." << endl; - } - // Were we scanning the target file or an extra file - else if (originalsourcefile != 0) - { - string targetname; - DiskFile::SplitFilename(sourcefile->TargetFileName(), path, targetname); - - cout << "Target: \"" - << name - << "\" - is a match for \"" - << targetname - << "\"." - << endl; - } - else - { - string targetname; - DiskFile::SplitFilename(sourcefile->TargetFileName(), path, targetname); - - cout << "File: \"" - << name - << "\" - is a match for \"" - << targetname - << "\"." - << endl; - } - } - } - } - else - { - matchtype = eNoMatch; - - if (noiselevel > CommandLine::nlSilent) - { - // We found not data, but did the file actually contain blocks we - // had already found in other files. - if (duplicatecount > 0) - { - cout << "File: \"" - << name - << "\" - found " - << duplicatecount - << " duplicate data blocks." - << endl; - } - else - { - cout << "File: \"" - << name - << "\" - no data found." - << endl; - } - } - } - sig_done(name,count, sourcefile && sourcefile->GetVerificationPacket() ? sourcefile->GetVerificationPacket()->BlockCount() : 0); - sig_progress(1000); - return true; -} - -// Find out how much data we have found -void Par2Repairer::UpdateVerificationResults(void) -{ - availableblockcount = 0; - missingblockcount = 0; - - completefilecount = 0; - renamedfilecount = 0; - damagedfilecount = 0; - missingfilecount = 0; - - u32 filenumber = 0; - vector::iterator sf = sourcefiles.begin(); - - // Check the recoverable files - while (sf != sourcefiles.end() && filenumber < mainpacket->TotalFileCount()) - { - Par2RepairerSourceFile *sourcefile = *sf; - - if (sourcefile) - { - // Was a perfect match for the file found - if (sourcefile->GetCompleteFile() != 0) - { - // Is it the target file or a different one - if (sourcefile->GetCompleteFile() == sourcefile->GetTargetFile()) - { - completefilecount++; - } - else - { - renamedfilecount++; - } - - availableblockcount += sourcefile->BlockCount(); - } - else - { - // Count the number of blocks that have been found - vector::iterator sb = sourcefile->SourceBlocks(); - for (u32 blocknumber=0; blocknumberBlockCount(); ++blocknumber, ++sb) - { - DataBlock &datablock = *sb; - - if (datablock.IsSet()) - availableblockcount++; - } - - // Does the target file exist - if (sourcefile->GetTargetExists()) - { - damagedfilecount++; - } - else - { - missingfilecount++; - } - } - } - else - { - missingfilecount++; - } - - ++filenumber; - ++sf; - } - - missingblockcount = sourceblockcount - availableblockcount; -} - -// Check the verification results and report the results -bool Par2Repairer::CheckVerificationResults(void) -{ - // Is repair needed - if (completefilecount < mainpacket->RecoverableFileCount() || - renamedfilecount > 0 || - damagedfilecount > 0 || - missingfilecount > 0) - { - if (noiselevel > CommandLine::nlSilent) - cout << "Repair is required." << endl; - if (noiselevel > CommandLine::nlQuiet) - { - if (renamedfilecount > 0) cout << renamedfilecount << " file(s) have the wrong name." << endl; - if (missingfilecount > 0) cout << missingfilecount << " file(s) are missing." << endl; - if (damagedfilecount > 0) cout << damagedfilecount << " file(s) exist but are damaged." << endl; - if (completefilecount > 0) cout << completefilecount << " file(s) are ok." << endl; - - cout << "You have " << availableblockcount - << " out of " << sourceblockcount - << " data blocks available." << endl; - if (recoverypacketmap.size() > 0) - cout << "You have " << (u32)recoverypacketmap.size() - << " recovery blocks available." << endl; - } - - // Is repair possible - if (recoverypacketmap.size() >= missingblockcount) - { - if (noiselevel > CommandLine::nlSilent) - cout << "Repair is possible." << endl; - - if (noiselevel > CommandLine::nlQuiet) - { - if (recoverypacketmap.size() > missingblockcount) - cout << "You have an excess of " - << (u32)recoverypacketmap.size() - missingblockcount - << " recovery blocks." << endl; - - if (missingblockcount > 0) - cout << missingblockcount - << " recovery blocks will be used to repair." << endl; - else if (recoverypacketmap.size()) - cout << "None of the recovery blocks will be used for the repair." << endl; - } - - return true; - } - else - { - if (noiselevel > CommandLine::nlSilent) - { - cout << "Repair is not possible." << endl; - cout << "You need " << missingblockcount - recoverypacketmap.size() - << " more recovery blocks to be able to repair." << endl; - } - - return false; - } - } - else - { - if (noiselevel > CommandLine::nlSilent) - cout << "All files are correct, repair is not required." << endl; - - return true; - } - - return true; -} - -// Rename any damaged or missnamed target files. -bool Par2Repairer::RenameTargetFiles(void) -{ - u32 filenumber = 0; - vector::iterator sf = sourcefiles.begin(); - - // Rename any damaged target files - while (sf != sourcefiles.end() && filenumber < mainpacket->TotalFileCount()) - { - Par2RepairerSourceFile *sourcefile = *sf; - - // If the target file exists but is not a complete version of the file - if (sourcefile->GetTargetExists() && - sourcefile->GetTargetFile() != sourcefile->GetCompleteFile()) - { - DiskFile *targetfile = sourcefile->GetTargetFile(); - - // Rename it - diskFileMap.Remove(targetfile); - - if (!targetfile->Rename()) - return false; - - bool success = diskFileMap.Insert(targetfile); - assert(success); (void)success; - - // We no longer have a target file - sourcefile->SetTargetExists(false); - sourcefile->SetTargetFile(0); - } - - ++sf; - ++filenumber; - } - - filenumber = 0; - sf = sourcefiles.begin(); - - // Rename any missnamed but complete versions of the files - while (sf != sourcefiles.end() && filenumber < mainpacket->TotalFileCount()) - { - Par2RepairerSourceFile *sourcefile = *sf; - - // If there is no targetfile and there is a complete version - if (sourcefile->GetTargetFile() == 0 && - sourcefile->GetCompleteFile() != 0) - { - DiskFile *targetfile = sourcefile->GetCompleteFile(); - - // Rename it - diskFileMap.Remove(targetfile); - - if (!targetfile->Rename(sourcefile->TargetFileName())) - return false; - - bool success = diskFileMap.Insert(targetfile); - assert(success); (void)success; - - // This file is now the target file - sourcefile->SetTargetExists(true); - sourcefile->SetTargetFile(targetfile); - - // We have one more complete file - completefilecount++; - } - - ++sf; - ++filenumber; - } - - return true; -} - -// Work out which files are being repaired, create them, and allocate -// target DataBlocks to them, and remember them for later verification. -bool Par2Repairer::CreateTargetFiles(void) -{ - u32 filenumber = 0; - vector::iterator sf = sourcefiles.begin(); - - // Create any missing target files - while (sf != sourcefiles.end() && filenumber < mainpacket->TotalFileCount()) - { - Par2RepairerSourceFile *sourcefile = *sf; - - // If the file does not exist - if (!sourcefile->GetTargetExists()) - { - DiskFile *targetfile = new DiskFile(cerr); - string filename = sourcefile->TargetFileName(); - u64 filesize = sourcefile->GetDescriptionPacket()->FileSize(); - - // Create the target file - if (!targetfile->Create(filename, filesize)) - { - delete targetfile; - return false; - } - - // This file is now the target file - sourcefile->SetTargetExists(true); - sourcefile->SetTargetFile(targetfile); - - // Remember this file - bool success = diskFileMap.Insert(targetfile); - assert(success); (void)success; - - u64 offset = 0; - vector::iterator tb = sourcefile->TargetBlocks(); - - // Allocate all of the target data blocks - while (offset < filesize) - { - DataBlock &datablock = *tb; - - datablock.SetLocation(targetfile, offset); - datablock.SetLength(min(blocksize, filesize-offset)); - - offset += blocksize; - ++tb; - } - - // Add the file to the list of those that will need to be verified - // once the repair has completed. - verifylist.push_back(sourcefile); - } - - ++sf; - ++filenumber; - } - - return true; -} - -// Work out which data blocks are available, which need to be copied -// directly to the output, and which need to be recreated, and compute -// the appropriate Reed Solomon matrix. -bool Par2Repairer::ComputeRSmatrix(void) -{ - inputblocks.resize(sourceblockcount); // The DataBlocks that will read from disk - copyblocks.resize(availableblockcount); // Those DataBlocks which need to be copied - outputblocks.resize(missingblockcount); // Those DataBlocks that will re recalculated - - vector::iterator inputblock = inputblocks.begin(); - vector::iterator copyblock = copyblocks.begin(); - vector::iterator outputblock = outputblocks.begin(); - - // Build an array listing which source data blocks are present and which are missing - vector present; - present.resize(sourceblockcount); - - vector::iterator sourceblock = sourceblocks.begin(); - vector::iterator targetblock = targetblocks.begin(); - vector::iterator pres = present.begin(); - - // Iterate through all source blocks for all files - while (sourceblock != sourceblocks.end()) - { - // Was this block found - if (sourceblock->IsSet()) - { -// // Open the file the block was found in. -// if (!sourceblock->Open()) -// return false; - - // Record that the block was found - *pres = true; - - // Add the block to the list of those which will be read - // as input (and which might also need to be copied). - *inputblock = &*sourceblock; - *copyblock = &*targetblock; - - ++inputblock; - ++copyblock; - } - else - { - // Record that the block was missing - *pres = false; - - // Add the block to the list of those to be written - *outputblock = &*targetblock; - ++outputblock; - } - - ++sourceblock; - ++targetblock; - ++pres; - } - - // Set the number of source blocks and which of them are present - if (!rs.SetInput(present)) - return false; - - // Start iterating through the available recovery packets - map::iterator rp = recoverypacketmap.begin(); - - // Continue to fill the remaining list of data blocks to be read - while (inputblock != inputblocks.end()) - { - // Get the next available recovery packet - u32 exponent = rp->first; - RecoveryPacket* recoverypacket = rp->second; - - // Get the DataBlock from the recovery packet - DataBlock *recoveryblock = recoverypacket->GetDataBlock(); - -// // Make sure the file is open -// if (!recoveryblock->Open()) -// return false; - - // Add the recovery block to the list of blocks that will be read - *inputblock = recoveryblock; - - // Record that the corresponding exponent value is the next one - // to use in the RS matrix - if (!rs.SetOutput(true, (u16)exponent)) - return false; - - ++inputblock; - ++rp; - } - - // If we need to, compute and solve the RS matrix - if (missingblockcount == 0) - return true; - - bool success = rs.Compute(noiselevel); - - return success; -} - -// Allocate memory buffers for reading and writing data to disk. -bool Par2Repairer::AllocateBuffers(size_t memorylimit) -{ - // Would single pass processing use too much memory - if (blocksize * missingblockcount > memorylimit) - { - // Pick a size that is small enough - chunksize = ~3 & (memorylimit / missingblockcount); - } - else - { - chunksize = (size_t)blocksize; - } - - // Allocate the two buffers - inputbuffer = new u8[(size_t)chunksize]; - outputbuffer = new u8[(size_t)chunksize * missingblockcount]; - - if (inputbuffer == NULL || outputbuffer == NULL) - { - cerr << "Could not allocate buffer memory." << endl; - return false; - } - - return true; -} - -// Read source data, process it through the RS matrix and write it to disk. -bool Par2Repairer::ProcessData(u64 blockoffset, size_t blocklength) -{ - u64 totalwritten = 0; - - // Clear the output buffer - memset(outputbuffer, 0, (size_t)chunksize * missingblockcount); - - vector::iterator inputblock = inputblocks.begin(); - vector::iterator copyblock = copyblocks.begin(); - u32 inputindex = 0; - - DiskFile *lastopenfile = NULL; - - // Are there any blocks which need to be reconstructed - if (missingblockcount > 0) - { - // For each input block - while (inputblock != inputblocks.end()) - { - // Are we reading from a new file? - if (lastopenfile != (*inputblock)->GetDiskFile()) - { - // Close the last file - if (lastopenfile != NULL) - { - lastopenfile->Close(); - } - - // Open the new file - lastopenfile = (*inputblock)->GetDiskFile(); - if (!lastopenfile->Open()) - { - return false; - } - } - - // Read data from the current input block - if (!(*inputblock)->ReadData(blockoffset, blocklength, inputbuffer)) - return false; - - // Have we reached the last source data block - if (copyblock != copyblocks.end()) - { - // Does this block need to be copied to the target file - if ((*copyblock)->IsSet()) - { - size_t wrote; - - // Write the block back to disk in the new target file - if (!(*copyblock)->WriteData(blockoffset, blocklength, inputbuffer, wrote)) - return false; - - totalwritten += wrote; - } - ++copyblock; - } - - if (!RepairData(inputindex, blocklength)) - { - // For each output block - for (u32 outputindex=0; outputindex CommandLine::nlQuiet) - { - // Update a progress indicator - u32 oldfraction = (u32)(1000 * progress / totaldata); - progress += blocklength; - u32 newfraction = (u32)(1000 * progress / totaldata); - - if (oldfraction != newfraction) - { - cout << "Repairing: " << newfraction/10 << '.' << newfraction%10 << "%\r" << flush; - sig_progress(newfraction); - - if (cancelled) - { - break; - } - } - } - } - } - - if (cancelled) - { - break; - } - - ++inputblock; - ++inputindex; - } - } - else - { - // Reconstruction is not required, we are just copying blocks between files - - // For each block that might need to be copied - while (copyblock != copyblocks.end()) - { - // Does this block need to be copied - if ((*copyblock)->IsSet()) - { - // Are we reading from a new file? - if (lastopenfile != (*inputblock)->GetDiskFile()) - { - // Close the last file - if (lastopenfile != NULL) - { - lastopenfile->Close(); - } - - // Open the new file - lastopenfile = (*inputblock)->GetDiskFile(); - if (!lastopenfile->Open()) - { - return false; - } - } - - // Read data from the current input block - if (!(*inputblock)->ReadData(blockoffset, blocklength, inputbuffer)) - return false; - - size_t wrote; - if (!(*copyblock)->WriteData(blockoffset, blocklength, inputbuffer, wrote)) - return false; - totalwritten += wrote; - } - - if (noiselevel > CommandLine::nlQuiet) - { - // Update a progress indicator - u32 oldfraction = (u32)(1000 * progress / totaldata); - progress += blocklength; - u32 newfraction = (u32)(1000 * progress / totaldata); - - if (oldfraction != newfraction) - { - cout << "Processing: " << newfraction/10 << '.' << newfraction%10 << "%\r" << flush; - sig_progress(newfraction); - - if (cancelled) - { - break; - } - } - } - - if (cancelled) - { - break; - } - - ++copyblock; - ++inputblock; - } - } - - // Close the last file - if (lastopenfile != NULL) - { - lastopenfile->Close(); - } - - if (cancelled) - { - return false; - } - - if (noiselevel > CommandLine::nlQuiet) - cout << "Writing recovered data\r"; - - // For each output block that has been recomputed - vector::iterator outputblock = outputblocks.begin(); - for (u32 outputindex=0; outputindexWriteData(blockoffset, blocklength, outbuf, wrote)) - return false; - totalwritten += wrote; - - ++outputblock; - } - - if (noiselevel > CommandLine::nlQuiet) - cout << "Wrote " << totalwritten << " bytes to disk" << endl; - - return true; -} - -// Verify that all of the reconstructed target files are now correct -bool Par2Repairer::VerifyTargetFiles(void) -{ - bool finalresult = true; - - // Verify the target files in alphabetical order - sort(verifylist.begin(), verifylist.end(), SortSourceFilesByFileName); - - // Iterate through each file in the verification list - for (vector::iterator sf = verifylist.begin(); - sf != verifylist.end(); - ++sf) - { - Par2RepairerSourceFile *sourcefile = *sf; - DiskFile *targetfile = sourcefile->GetTargetFile(); - - // Close the file - if (targetfile->IsOpen()) - targetfile->Close(); - - // Mark all data blocks for the file as unknown - vector::iterator sb = sourcefile->SourceBlocks(); - for (u32 blocknumber=0; blocknumberBlockCount(); blocknumber++) - { - sb->ClearLocation(); - ++sb; - } - - // Say we don't have a complete version of the file - sourcefile->SetCompleteFile(0); - - // Re-open the target file - if (!targetfile->Open()) - { - finalresult = false; - continue; - } - - // Verify the file again - if (!VerifyDataFile(targetfile, sourcefile)) - finalresult = false; - - // Close the file again - targetfile->Close(); - - // Find out how much data we have found - UpdateVerificationResults(); - } - - return finalresult; -} - -// Delete all of the partly reconstructed files -bool Par2Repairer::DeleteIncompleteTargetFiles(void) -{ - vector::iterator sf = verifylist.begin(); - - // Iterate through each file in the verification list - while (sf != verifylist.end()) - { - Par2RepairerSourceFile *sourcefile = *sf; - if (sourcefile->GetTargetExists()) - { - DiskFile *targetfile = sourcefile->GetTargetFile(); - - // Close and delete the file - if (targetfile->IsOpen()) - targetfile->Close(); - targetfile->Delete(); - - // Forget the file - diskFileMap.Remove(targetfile); - delete targetfile; - - // There is no target file - sourcefile->SetTargetExists(false); - sourcefile->SetTargetFile(0); - } - - ++sf; - } - - return true; -} - -} // end namespace Par2 diff --git a/lib/par2/par2repairer.h b/lib/par2/par2repairer.h deleted file mode 100644 index 1c819e8da..000000000 --- a/lib/par2/par2repairer.h +++ /dev/null @@ -1,206 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __PAR2REPAIRER_H__ -#define __PAR2REPAIRER_H__ - -#include "parheaders.h" - -namespace Par2 { - -class Par2Repairer -{ -public: - Par2Repairer(std::ostream& cout, std::ostream& cerr); - ~Par2Repairer(void); - - Result PreProcess(const CommandLine &commandline); - Result Process(const CommandLine &commandline, bool dorepair); - -protected: - // Steps in verifying and repairing files: - - // Load packets from the specified file - bool LoadPacketsFromFile(string filename); - // Finish loading a recovery packet - bool LoadRecoveryPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - // Finish loading a file description packet - bool LoadDescriptionPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - // Finish loading a file verification packet - bool LoadVerificationPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - // Finish loading the main packet - bool LoadMainPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - // Finish loading the creator packet - bool LoadCreatorPacket(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - - // Load packets from other PAR2 files with names based on the original PAR2 file - bool LoadPacketsFromOtherFiles(string filename); - - // Load packets from any other PAR2 files whose names are given on the command line - bool LoadPacketsFromExtraFiles(const list &extrafiles); - - // Check that the packets are consistent and discard any that are not - bool CheckPacketConsistency(void); - - // Use the information in the main packet to get the source files - // into the correct order and determine their filenames - bool CreateSourceFileList(void); - - // Determine the total number of DataBlocks for the recoverable source files - // The allocate the DataBlocks and assign them to each source file - bool AllocateSourceBlocks(void); - - // Create a verification hash table for all files for which we have not - // found a complete version of the file and for which we have - // a verification packet - bool PrepareVerificationHashTable(void); - - // Compute the table for the sliding CRC computation - bool ComputeWindowTable(void); - - // Attempt to verify all of the source files - bool VerifySourceFiles(void); - - // Scan any extra files specified on the command line - bool VerifyExtraFiles(const list &extrafiles); - - // Attempt to match the data in the DiskFile with the source file - bool VerifyDataFile(DiskFile *diskfile, Par2RepairerSourceFile *sourcefile); - - // Perform a sliding window scan of the DiskFile looking for blocks of data that - // might belong to any of the source files (for which a verification packet was - // available). If a block of data might be from more than one source file, prefer - // the one specified by the "sourcefile" parameter. If the first data block - // found is for a different source file then "sourcefile" is changed accordingly. - virtual bool ScanDataFile(DiskFile *diskfile, // [in] The file being scanned - Par2RepairerSourceFile* &sourcefile, // [in/out] The source file matched - MatchType &matchtype, // [out] The type of match - MD5Hash &hashfull, // [out] The full hash of the file - MD5Hash &hash16k, // [out] The hash of the first 16k - u32 &count); // [out] The number of blocks found - - // Find out how much data we have found - void UpdateVerificationResults(void); - - // Check the verification results and report the results - bool CheckVerificationResults(void); - - // Rename any damaged or missnamed target files. - bool RenameTargetFiles(void); - - // Work out which files are being repaired, create them, and allocate - // target DataBlocks to them, and remember them for later verification. - bool CreateTargetFiles(void); - - // Work out which data blocks are available, which need to be copied - // directly to the output, and which need to be recreated, and compute - // the appropriate Reed Solomon matrix. - bool ComputeRSmatrix(void); - - // Allocate memory buffers for reading and writing data to disk. - bool AllocateBuffers(size_t memorylimit); - - // Read source data, process it through the RS matrix and write it to disk. - bool ProcessData(u64 blockoffset, size_t blocklength); - - // Verify that all of the reconstructed target files are now correct - bool VerifyTargetFiles(void); - - // Delete all of the partly reconstructed files - bool DeleteIncompleteTargetFiles(void); - - // Signals - virtual void sig_filename(std::string filename) {} - virtual void sig_progress(int progress) {} - virtual void sig_headers(ParHeaders* headers) {} - virtual void sig_done(std::string filename, int available, int total) {} - - // Repair started - virtual void BeginRepair() {} - - // Repair ended - virtual void EndRepair() {} - - // Repair chunk of data (returns "true" if repaired or "false" if default repair-routine should be used) - virtual bool RepairData(u32 inputindex, size_t blocklength) { return false; } - -protected: - std::ostream& cout; - std::ostream& cerr; - - ParHeaders* headers; // Headers - bool alreadyloaded; // Already loaded ? - CommandLine::NoiseLevel noiselevel; // OnScreen display - - string searchpath; // Where to find files on disk - - bool firstpacket; // Whether or not a valid packet has been found. - MD5Hash setid; // The SetId extracted from the first packet. - - map recoverypacketmap; // One recovery packet for each exponent value. - MainPacket *mainpacket; // One copy of the main packet. - CreatorPacket *creatorpacket; // One copy of the creator packet. - - DiskFileMap diskFileMap; - - map sourcefilemap;// Map from FileId to SourceFile - vector sourcefiles; // The source files - vector verifylist; // Those source files that are being repaired - - u64 blocksize; // The block size. - u64 chunksize; // How much of a block can be processed. - u32 sourceblockcount; // The total number of blocks - u32 availableblockcount; // How many undamaged blocks have been found - u32 missingblockcount; // How many blocks are missing - - bool blocksallocated; // Whether or not the DataBlocks have been allocated - vector sourceblocks; // The DataBlocks that will be read from disk - vector targetblocks; // The DataBlocks that will be written to disk - - u32 windowtable[256]; // Table for sliding CRCs - u32 windowmask; // Maks for sliding CRCs - - bool blockverifiable; // Whether and files can be verified at the block level - VerificationHashTable verificationhashtable; // Hash table for block verification - list unverifiablesourcefiles; // Files that are not block verifiable - - u32 completefilecount; // How many files are fully verified - u32 renamedfilecount; // How many files are verified but have the wrong name - u32 damagedfilecount; // How many files exist but are damaged - u32 missingfilecount; // How many files are completely missing - - vector inputblocks; // Which DataBlocks will be read from disk - vector copyblocks; // Which DataBlocks will copied back to disk - vector outputblocks; // Which DataBlocks have to calculated using RS - - ReedSolomon rs; // The Reed Solomon matrix. - - void *inputbuffer; // Buffer for reading DataBlocks (chunksize) - void *outputbuffer; // Buffer for writing DataBlocks (chunksize * missingblockcount) - - u64 progress; // How much data has been processed. - u64 totaldata; // Total amount of data to be processed. - u64 totalsize; // Total data size - - bool cancelled; // repair cancelled -}; - -} // end namespace Par2 - -#endif // __PAR2REPAIRER_H__ diff --git a/lib/par2/par2repairersourcefile.cpp b/lib/par2/par2repairersourcefile.cpp deleted file mode 100644 index 02b9596f6..000000000 --- a/lib/par2/par2repairersourcefile.cpp +++ /dev/null @@ -1,164 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 { - -Par2RepairerSourceFile::Par2RepairerSourceFile(DescriptionPacket *_descriptionpacket, - VerificationPacket *_verificationpacket) -{ - firstblocknumber = 0; - - descriptionpacket = _descriptionpacket; - verificationpacket = _verificationpacket; - -// verificationhashtable = 0; - - targetexists = false; - targetfile = 0; - completefile = 0; -} - -Par2RepairerSourceFile::~Par2RepairerSourceFile(void) -{ - delete descriptionpacket; - delete verificationpacket; - -// delete verificationhashtable; -} - - -void Par2RepairerSourceFile::SetDescriptionPacket(DescriptionPacket *_descriptionpacket) -{ - descriptionpacket = _descriptionpacket; -} - -void Par2RepairerSourceFile::SetVerificationPacket(VerificationPacket *_verificationpacket) -{ - verificationpacket = _verificationpacket; -} - -void Par2RepairerSourceFile::ComputeTargetFileName(string path) -{ - // Get a version of the filename compatible with the OS - string filename = DiskFile::TranslateFilename(descriptionpacket->FileName()); - - // Strip the path from the filename - string::size_type where; - if (string::npos != (where = filename.find_last_of('\\')) - || string::npos != (where = filename.find_last_of('/')) -#ifdef WIN32 - || string::npos != (where = filename.find_last_of(':')) -#endif - ) - { - filename = filename.substr(where+1); - } - - targetfilename = path + filename; -} - -string Par2RepairerSourceFile::TargetFileName(void) const -{ - return targetfilename; -} - -void Par2RepairerSourceFile::SetTargetFile(DiskFile *diskfile) -{ - targetfile = diskfile; -} - -DiskFile* Par2RepairerSourceFile::GetTargetFile(void) const -{ - return targetfile; -} - -void Par2RepairerSourceFile::SetTargetExists(bool exists) -{ - targetexists = exists; -} - -bool Par2RepairerSourceFile::GetTargetExists(void) const -{ - return targetexists; -} - -void Par2RepairerSourceFile::SetCompleteFile(DiskFile *diskfile) -{ - completefile = diskfile; -} - -DiskFile* Par2RepairerSourceFile::GetCompleteFile(void) const -{ - return completefile; -} - -// Remember which source and target blocks will be used by this file -// and set their lengths appropriately -void Par2RepairerSourceFile::SetBlocks(u32 _blocknumber, - u32 _blockcount, - vector::iterator _sourceblocks, - vector::iterator _targetblocks, - u64 blocksize) -{ - firstblocknumber = _blocknumber; - blockcount = _blockcount; - sourceblocks = _sourceblocks; - targetblocks = _targetblocks; - - if (blockcount > 0) - { - u64 filesize = descriptionpacket->FileSize(); - - vector::iterator sb = sourceblocks; - for (u32 blocknumber=0; blocknumberFileSize() + blocksize-1) / blocksize); - } - else - { - blockcount = 0; - } -} - -} // end namespace Par2 diff --git a/lib/par2/par2repairersourcefile.h b/lib/par2/par2repairersourcefile.h deleted file mode 100644 index e1c6a48f8..000000000 --- a/lib/par2/par2repairersourcefile.h +++ /dev/null @@ -1,111 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __PAR2REPAIRERSOURCEFILE_H__ -#define __PAR2REPAIRERSOURCEFILE_H__ - -namespace Par2 -{ - -enum MatchType -{ - eNoMatch = 0, - ePartialMatch, - eFullMatch -}; - -// The Par2RepairerSourceFile object is used during verification and repair -// to record details about a particular source file and the data blocks -// for that file. - -class Par2RepairerSourceFile -{ -public: - // Construct the object and set the description and verification packets - Par2RepairerSourceFile(DescriptionPacket *descriptionpacket, - VerificationPacket *verificationpacket); - ~Par2RepairerSourceFile(void); - - // Get/Set the description packet - DescriptionPacket* GetDescriptionPacket(void) const {return descriptionpacket;} - void SetDescriptionPacket(DescriptionPacket *descriptionpacket); - - // Get/Set the verification packet - VerificationPacket* GetVerificationPacket(void) const {return verificationpacket;} - void SetVerificationPacket(VerificationPacket *verificationpacket); - - // Record the details as to which data blocks belong to this source - // file and set the length of each allocated block correctly. - void SetBlocks(u32 _blocknumber, - u32 _blockcount, - vector::iterator _sourceblocks, - vector::iterator _targetblocks, - u64 blocksize); - - // Determine the block count from the file size and block size. - void SetBlockCount(u64 blocksize); - - // Set/Get which DiskFile will contain the final repaired version of the file - void SetTargetFile(DiskFile *diskfile); - DiskFile* GetTargetFile(void) const; - - // Set/Get whether or not the target file actually exists - void SetTargetExists(bool exists); - bool GetTargetExists(void) const; - - // Set/Get which DiskFile contains a full undamaged version of the source file - void SetCompleteFile(DiskFile *diskfile); - DiskFile* GetCompleteFile(void) const; - - // Compute/Get the filename for the final repaired version of the file - void ComputeTargetFileName(string path); - string TargetFileName(void) const; - - // Get the number of blocks that the file uses - u32 BlockCount(void) const {return blockcount;} - - // Get the relative block number of the first block in the file - u32 FirstBlockNumber(void) const {return firstblocknumber;} - - // Get the first source DataBlock for the file - vector::iterator SourceBlocks(void) const {return sourceblocks;} - - // Get the first target DataBlock for the file - vector::iterator TargetBlocks(void) const {return targetblocks;} - -protected: - DescriptionPacket *descriptionpacket; // The file description packet - VerificationPacket *verificationpacket; // The file verification packet - - u32 blockcount; // The number of DataBlocks in the file - u32 firstblocknumber; // The block number of the first DataBlock - - vector::iterator sourceblocks; // The first source DataBlock - vector::iterator targetblocks; // The first target DataBlock - - bool targetexists; // Whether the target file exists - DiskFile *targetfile; // The final version of the file - DiskFile *completefile; // A complete version of the file - - string targetfilename; // The filename of the target file -}; - -} // end namespace Par2 - -#endif // __PAR2REPAIRERSOURCEFILE_H__ diff --git a/lib/par2/parheaders.cpp b/lib/par2/parheaders.cpp deleted file mode 100644 index 452d0bb6b..000000000 --- a/lib/par2/parheaders.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "parheaders.h" - -namespace Par2 -{ - -ParHeaders::ParHeaders(void) { - packets = -1; - recovery_block = -1; - recoverable_files = -1; - other_files = -1; - block_size = -1; - data_blocks = -1; - data_size = -1; - setid = ""; - chunk_size = -1; -} - -} // end namespace Par2 diff --git a/lib/par2/parheaders.h b/lib/par2/parheaders.h deleted file mode 100644 index 81e2806cb..000000000 --- a/lib/par2/parheaders.h +++ /dev/null @@ -1,39 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -namespace Par2 -{ - -class ParHeaders { - public: - std::string setid; - int packets; - int recovery_block; - int recoverable_files; - int other_files; - long long block_size; - int data_blocks; - long long data_size; - long long chunk_size; - - ParHeaders(void); - -}; - -} // end namespace Par2 diff --git a/lib/par2/recoverypacket.cpp b/lib/par2/recoverypacket.cpp deleted file mode 100644 index a7a805e8f..000000000 --- a/lib/par2/recoverypacket.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -RecoveryPacket::RecoveryPacket(void) -{ - diskfile = NULL; - offset = 0; - packetcontext = NULL; -} - -RecoveryPacket::~RecoveryPacket(void) -{ - delete packetcontext; -} - -// Create a recovery packet. - -// The packet header can be almost completely filled in using the supplied -// information and computation of the hash of the packet started immediately. -// The hash will be updated as new data is written to the packet. - -void RecoveryPacket::Create(DiskFile *_diskfile, - u64 _offset, - u64 _blocksize, - u32 _exponent, - const MD5Hash &_setid) -{ - diskfile = _diskfile; - offset = _offset; - - // Record everything we know in the packet. - packet.header.magic = packet_magic; - packet.header.length = sizeof(packet) + _blocksize; - //packet.header.hash; // Not known yet. - packet.header.setid = _setid; - packet.header.type = recoveryblockpacket_type; - packet.exponent = _exponent; - - // Start computation of the packet hash - packetcontext = new MD5Context; - packetcontext->Update(&packet.header.setid, - sizeof(RECOVERYBLOCKPACKET)-offsetof(RECOVERYBLOCKPACKET, header.setid)); - - // Set the data block to immediatly follow the header on disk - datablock.SetLocation(_diskfile, _offset + sizeof(packet)); - datablock.SetLength(_blocksize); -} - -// Write data from the buffer to the data block on disk -bool RecoveryPacket::WriteData(u64 position, - size_t size, - const void *buffer) -{ - // Update the packet hash - packetcontext->Update(buffer, size); - - // Write the data to the data block - size_t wrote; - return datablock.WriteData(position, size, buffer, wrote); -} - -// Write the header of the packet to disk -bool RecoveryPacket::WriteHeader(void) -{ - // Finish computing the packet hash - packetcontext->Final(packet.header.hash); - - // Write the header to disk - return diskfile->Write(offset, &packet, sizeof(packet)); -} - -// Load the recovery packet from disk. -// -// The header of the packet will already have been read from disk. The only -// thing that actually needs to be read is the exponent value. -// The recovery data is not read from disk at this point. Its location -// is recovered in the DataBlock object. - -bool RecoveryPacket::Load(DiskFile *_diskfile, - u64 _offset, - PACKET_HEADER &_header) -{ - diskfile = _diskfile; - offset = _offset; - - // Is the packet actually large enough - if (_header.length <= sizeof(packet)) - { - return false; - } - - // Save the fixed header - packet.header = _header; - - // Set the data block to immediatly follow the header on disk - datablock.SetLocation(diskfile, offset + sizeof(packet)); - datablock.SetLength(packet.header.length - sizeof(packet)); - - // Read the rest of the packet header - return diskfile->Read(offset + sizeof(packet.header), &packet.exponent, sizeof(packet)-sizeof(packet.header)); -} - -} // end namespace Par2 diff --git a/lib/par2/recoverypacket.h b/lib/par2/recoverypacket.h deleted file mode 100644 index 876b459a6..000000000 --- a/lib/par2/recoverypacket.h +++ /dev/null @@ -1,105 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __RECOVERYPACKET_H__ -#define __RECOVERYPACKET_H__ - -namespace Par2 -{ - -// The RecoveryPacket object is used to access a specific recovery -// packet within a recovery file (for when creating them, or using -// them during a repair operation). - -// Because the actual recovery data for the packet may be large, the -// RecoveryPacket object only contains a copy of packet header and -// exponent value for the block and uses a DataBlock object for -// all accesses to the actual recovery data in the packet. - -class RecoveryPacket -{ -public: - RecoveryPacket(void); - ~RecoveryPacket(void); - -public: - // Create a recovery packet for a specified file. - void Create(DiskFile *diskfile, // Which file will the packet be stored in - u64 offset, // At what offset will the packet be stored - u64 blocksize, // How much recovery data will it contain - u32 exponent, // What exponent value will be used - const MD5Hash &setid); // What is the SetId - // Write some data to the recovery data block and update the recovery packet. - bool WriteData(u64 position, // Relative position within the data block - size_t size, // Size of data to write to block - const void *buffer); // Buffer containing the data to write - // Finish computing the hash of the recovery packet and write the header to disk. - bool WriteHeader(void); - -public: - // Load a recovery packet from a specified file - bool Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - -public: - // Get the lenght of the packet. - u64 PacketLength(void) const; - - // The the exponent of the packet. - u32 Exponent(void) const; - - // The length of the recovery data - u64 BlockSize(void) const; - - // The data block - DataBlock* GetDataBlock(void); - -protected: - DiskFile *diskfile; // The specific file that this packet is stored in - u64 offset; // The offset at which the packet is stored - - RECOVERYBLOCKPACKET packet; // The packet (excluding the actual recovery data) - - MD5Context *packetcontext; // MD5 Context used to compute the packet hash - - DataBlock datablock; // The recovery data block. -}; - -inline u64 RecoveryPacket::PacketLength(void) const -{ - return packet.header.length; -} - -inline u32 RecoveryPacket::Exponent(void) const -{ - return packet.exponent; -} - -inline u64 RecoveryPacket::BlockSize(void) const -{ - return packet.header.length - sizeof(packet); -} - -inline DataBlock* RecoveryPacket::GetDataBlock(void) -{ - return &datablock; -} - -} // end namespace Par2 - -#endif // __RECOVERYPACKET_H__ diff --git a/lib/par2/reedsolomon.cpp b/lib/par2/reedsolomon.cpp deleted file mode 100644 index 48ac00f6c..000000000 --- a/lib/par2/reedsolomon.cpp +++ /dev/null @@ -1,376 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -u32 gcd(u32 a, u32 b) -{ - if (a && b) - { - while (a && b) - { - if (a>b) - { - a = a%b; - } - else - { - b = b%a; - } - } - - return a+b; - } - else - { - return 0; - } -} - -template <> bool ReedSolomon::SetInput(const vector &present) -{ - inputcount = (u32)present.size(); - - datapresentindex = new u32[inputcount]; - datamissingindex = new u32[inputcount]; - database = new G::ValueType[inputcount]; - - G::ValueType base = 1; - - for (unsigned int index=0; index bool ReedSolomon::SetInput(u32 count) -{ - inputcount = count; - - datapresentindex = new u32[inputcount]; - datamissingindex = new u32[inputcount]; - database = new G::ValueType[inputcount]; - - G::ValueType base = 1; - - for (unsigned int index=0; index bool ReedSolomon::Process(size_t size, u32 inputindex, const void *inputbuffer, u32 outputindex, void *outputbuffer) -{ - // Look up the appropriate element in the RS matrix - Galois8 factor = leftmatrix[outputindex * (datapresent + datamissing) + inputindex]; - - // Do nothing if the factor happens to be 0 - if (factor == 0) - return eSuccess; - -#ifdef LONGMULTIPLY - // The 8-bit long multiplication tables - Galois8 *table = glmt->tables; - - // Split the factor into Low and High bytes - unsigned int fl = (factor >> 0) & 0xff; - - // Get the four separate multiplication tables - Galois8 *LL = &table[(0*256 + fl) * 256 + 0]; // factor.low * source.low - - // Combine the four multiplication tables into two - unsigned int L[256]; - - unsigned int *pL = &L[0]; - - for (unsigned int i=0; i<256; i++) - { - *pL = *LL; - - pL++; - LL++; - } - - // Treat the buffers as arrays of 32-bit unsigned ints. - u32 *src4 = (u32 *)inputbuffer; - u32 *end4 = (u32 *)&((u8*)inputbuffer)[size & ~3]; - u32 *dst4 = (u32 *)outputbuffer; - - // Process the data - while (src4 < end4) - { - u32 s = *src4++; - - // Use the two lookup tables computed earlier - *dst4++ ^= (L[(s >> 0) & 0xff] ) - ^ (L[(s >> 8) & 0xff] << 8 ) - ^ (L[(s >> 16)& 0xff] << 16) - ^ (L[(s >> 24)& 0xff] << 24); - } - - // Process any left over bytes at the end of the buffer - if (size & 3) - { - u8 *src1 = &((u8*)inputbuffer)[size & ~3]; - u8 *end1 = &((u8*)inputbuffer)[size]; - u8 *dst1 = &((u8*)outputbuffer)[size & ~3]; - - // Process the data - while (src1 < end1) - { - u8 s = *src1++; - *dst1++ ^= L[s]; - } - } -#else - // Treat the buffers as arrays of 16-bit Galois values. - - Galois8 *src = (Galois8 *)inputbuffer; - Galois8 *end = (Galois8 *)&((u8*)inputbuffer)[size]; - Galois8 *dst = (Galois8 *)outputbuffer; - - // Process the data - while (src < end) - { - *dst++ += *src++ * factor; - } -#endif - - return eSuccess; -} - - - -//////////////////////////////////////////////////////////////////////////////////////////// - - - -// Set which of the source files are present and which are missing -// and compute the base values to use for the vandermonde matrix. -template <> bool ReedSolomon::SetInput(const vector &present) -{ - inputcount = (u32)present.size(); - - datapresentindex = new u32[inputcount]; - datamissingindex = new u32[inputcount]; - database = new G::ValueType[inputcount]; - - unsigned int logbase = 0; - - for (unsigned int index=0; index= G::Limit) - { - cerr << "Too many input blocks for Reed Solomon matrix." << endl; - return false; - } - G::ValueType base = G(logbase++).ALog(); - - database[index] = base; - } - - return true; -} - -// Record that the specified number of source files are all present -// and compute the base values to use for the vandermonde matrix. -template <> bool ReedSolomon::SetInput(u32 count) -{ - inputcount = count; - - datapresentindex = new u32[inputcount]; - datamissingindex = new u32[inputcount]; - database = new G::ValueType[inputcount]; - - unsigned int logbase = 0; - - for (unsigned int index=0; index= G::Limit) - { - cerr << "Too many input blocks for Reed Solomon matrix." << endl; - return false; - } - G::ValueType base = G(logbase++).ALog(); - - database[index] = base; - } - - return true; -} - -template <> bool ReedSolomon::Process(size_t size, u32 inputindex, const void *inputbuffer, u32 outputindex, void *outputbuffer) -{ - // Look up the appropriate element in the RS matrix - - Galois16 factor = leftmatrix[outputindex * (datapresent + datamissing) + inputindex]; - // Do nothing if the factor happens to be 0 - if (factor == 0) - return eSuccess; - -#ifdef LONGMULTIPLY - // The 8-bit long multiplication tables - Galois16 *table = glmt->tables; - - // Split the factor into Low and High bytes - unsigned int fl = (factor >> 0) & 0xff; - unsigned int fh = (factor >> 8) & 0xff; - - // Get the four separate multiplication tables - Galois16 *LL = &table[(0*256 + fl) * 256 + 0]; // factor.low * source.low - Galois16 *LH = &table[(1*256 + fl) * 256 + 0]; // factor.low * source.high - Galois16 *HL = &table[(1*256 + 0) * 256 + fh]; // factor.high * source.low - Galois16 *HH = &table[(2*256 + fh) * 256 + 0]; // factor.high * source.high - - // Combine the four multiplication tables into two - unsigned int L[256]; - unsigned int H[256]; - -#if __BYTE_ORDER == __LITTLE_ENDIAN - unsigned int *pL = &L[0]; - unsigned int *pH = &H[0]; -#else - unsigned int *pL = &H[0]; - unsigned int *pH = &L[0]; -#endif - - for (unsigned int i=0; i<256; i++) - { -#if __BYTE_ORDER == __LITTLE_ENDIAN - *pL = *LL + *HL; -#else - unsigned int temp = *LL + *HL; - *pL = (temp >> 8) & 0xff | (temp << 8) & 0xff00; -#endif - - pL++; - LL++; - HL+=256; - -#if __BYTE_ORDER == __LITTLE_ENDIAN - *pH = *LH + *HH; -#else - temp = *LH + *HH; - *pH = (temp >> 8) & 0xff | (temp << 8) & 0xff00; -#endif - - pH++; - LH++; - HH++; - } - - // Treat the buffers as arrays of 32-bit unsigned ints. - u32 *src = (u32 *)inputbuffer; - u32 *end = (u32 *)&((u8*)inputbuffer)[size]; - u32 *dst = (u32 *)outputbuffer; - - // Process the data - while (src < end) - { - u32 s = *src++; - - // Use the two lookup tables computed earlier -//#if __BYTE_ORDER == __LITTLE_ENDIAN - u32 d = *dst ^ (L[(s >> 0) & 0xff] ) - ^ (H[(s >> 8) & 0xff] ) - ^ (L[(s >> 16)& 0xff] << 16) - ^ (H[(s >> 24)& 0xff] << 16); - *dst++ = d; -//#else -// *dst++ ^= (L[(s >> 8) & 0xff] ) -// ^ (H[(s >> 0) & 0xff] ) -// ^ (L[(s >> 24)& 0xff] << 16) -// ^ (H[(s >> 16)& 0xff] << 16); -//#endif - } -#else - // Treat the buffers as arrays of 16-bit Galois values. - - // BUG: This only works for __LITTLE_ENDIAN - Galois16 *src = (Galois16 *)inputbuffer; - Galois16 *end = (Galois16 *)&((u8*)inputbuffer)[size]; - Galois16 *dst = (Galois16 *)outputbuffer; - - // Process the data - while (src < end) - { - *dst++ += *src++ * factor; - } -#endif - - return eSuccess; -} - -} // end namespace Par2 diff --git a/lib/par2/reedsolomon.h b/lib/par2/reedsolomon.h deleted file mode 100644 index 591d3c866..000000000 --- a/lib/par2/reedsolomon.h +++ /dev/null @@ -1,516 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __REEDSOLOMON_H__ -#define __REEDSOLOMON_H__ - -namespace Par2 -{ - -// The ReedSolomon object is used to calculate and store the matrix -// used during recovery block creation or data block reconstruction. -// -// During initialisation, one RSOutputRow object is created for each -// recovery block that either needs to be created or is available for -// use. - -class RSOutputRow -{ -public: - RSOutputRow(void) {}; - RSOutputRow(bool _present, u16 _exponent) : present(_present), exponent(_exponent) {} - -public: - bool present; - u16 exponent; -}; - -template -class ReedSolomon -{ -public: - typedef g G; - - ReedSolomon(std::ostream& cout, std::ostream& cerr); - ~ReedSolomon(void); - - // Set which input blocks are present or missing - bool SetInput(const vector &present); // Some input blocks are present - bool SetInput(u32 count); // All input blocks are present - - // Set which output block are available or need to be computed - bool SetOutput(bool present, u16 exponent); - bool SetOutput(bool present, u16 lowexponent, u16 highexponent); - - // Compute the RS Matrix - bool Compute(CommandLine::NoiseLevel noiselevel); - - // Process a block of data - bool Process(size_t size, // The size of the block of data - u32 inputindex, // The column in the RS matrix - const void *inputbuffer, // Buffer containing input data - u32 outputindex, // The row in the RS matrix - void *outputbuffer); // Buffer containing output data - -protected: - // Perform Gaussian Elimination - bool GaussElim(CommandLine::NoiseLevel noiselevel, - unsigned int rows, - unsigned int leftcols, - G *leftmatrix, - G *rightmatrix, - unsigned int datamissing); - -protected: - u32 inputcount; // Total number of input blocks - - u32 datapresent; // Number of input blocks that are present - u32 datamissing; // Number of input blocks that are missing - u32 *datapresentindex; // The index numbers of the data blocks that are present - u32 *datamissingindex; // The index numbers of the data blocks that are missing - - typename G::ValueType *database;// The "base" value to use for each input block - - u32 outputcount; // Total number of output blocks - - u32 parpresent; // Number of output blocks that are present - u32 parmissing; // Number of output blocks that are missing - u32 *parpresentindex; // The index numbers of the output blocks that are present - u32 *parmissingindex; // The index numbers of the output blocks that are missing - - vector outputrows; // Details of the output blocks - - G *leftmatrix; // The main matrix - - // When the matrices are initialised: values of the form base ^ exponent are - // stored (where the base values are obtained from database[] and the exponent - // values are obtained from outputrows[]). - -#ifdef LONGMULTIPLY - GaloisLongMultiplyTable *glmt; // A multiplication table used by Process() -#endif - - std::ostream& cout; - std::ostream& cerr; -}; - -template -inline ReedSolomon::ReedSolomon(std::ostream& cout, std::ostream& cerr) : - cout(cout), cerr(cerr) -{ - inputcount = 0; - - datapresent = 0; - datamissing = 0; - datapresentindex = 0; - datamissingindex = 0; - database = 0; - - outputrows.clear(); - - outputcount = 0; - - parpresent = 0; - parmissing = 0; - parpresentindex = 0; - parmissingindex = 0; - - leftmatrix = 0; - -#ifdef LONGMULTIPLY - glmt = new GaloisLongMultiplyTable; -#endif -} - -template -inline ReedSolomon::~ReedSolomon(void) -{ - delete [] datapresentindex; - delete [] datamissingindex; - delete [] database; - delete [] parpresentindex; - delete [] parmissingindex; - delete [] leftmatrix; - -#ifdef LONGMULTIPLY - delete glmt; -#endif -} - -u32 gcd(u32 a, u32 b); - -// Record whether the recovery block with the specified -// exponent values is present or missing. -template -inline bool ReedSolomon::SetOutput(bool present, u16 exponent) -{ - // Store the exponent and whether or not the recovery block is present or missing - outputrows.push_back(RSOutputRow(present, exponent)); - - outputcount++; - - // Update the counts. - if (present) - { - parpresent++; - } - else - { - parmissing++; - } - - return true; -} - -// Record whether the recovery blocks with the specified -// range of exponent values are present or missing. -template -inline bool ReedSolomon::SetOutput(bool present, u16 lowexponent, u16 highexponent) -{ - for (unsigned int exponent=lowexponent; exponent<=highexponent; exponent++) - { - if (!SetOutput(present, exponent)) - return false; - } - - return true; -} - -// Construct the Vandermonde matrix and solve it if necessary -template -inline bool ReedSolomon::Compute(CommandLine::NoiseLevel noiselevel) -{ - u32 outcount = datamissing + parmissing; - u32 incount = datapresent + datamissing; - - if (datamissing > parpresent) - { - cerr << "Not enough recovery blocks." << endl; - return false; - } - else if (outcount == 0) - { - cerr << "No output blocks." << endl; - return false; - } - - if (noiselevel > CommandLine::nlQuiet) - cout << "Computing Reed Solomon matrix." << endl; - - /* Layout of RS Matrix: - - parpresent - datapresent datamissing datamissing parmissing - / | \ / | \ - parpresent | (ppi[row])| | | (ppi[row])| | - datamissing | ^ | I | | ^ | 0 | - |(dpi[col]) | | |(dmi[col]) | | - +---------------------+-------------+ +---------------------+-----------+ - | (pmi[row])| | | (pmi[row])| | - parmissing | ^ | 0 | | ^ | I | - |(dpi[col]) | | |(dmi[col]) | | - \ | / \ | / - */ - - // Allocate the left hand matrix - - leftmatrix = new G[outcount * incount](); - - // Allocate the right hand matrix only if we are recovering - - G *rightmatrix = 0; - if (datamissing > 0) - { - rightmatrix = new G[outcount * outcount](); - } - - // Fill in the two matrices: - - vector::const_iterator outputrow = outputrows.begin(); - - // One row for each present recovery block that will be used for a missing data block - for (unsigned int row=0; row CommandLine::nlQuiet) - { - int progress = row * 1000 / (datamissing+parmissing); - cout << "Constructing: " << progress/10 << '.' << progress%10 << "%\r" << flush; - } - - // Get the exponent of the next present recovery block - while (!outputrow->present) - { - outputrow++; - } - u16 exponent = outputrow->exponent; - - // One column for each present data block - for (unsigned int col=0; col 0) - { - // One column for each missing data block - for (unsigned int col=0; col CommandLine::nlQuiet) - { - int progress = (row+datamissing) * 1000 / (datamissing+parmissing); - cout << "Constructing: " << progress/10 << '.' << progress%10 << "%\r" << flush; - } - - // Get the exponent of the next missing recovery block - while (outputrow->present) - { - outputrow++; - } - u16 exponent = outputrow->exponent; - - // One column for each present data block - for (unsigned int col=0; col 0) - { - // One column for each missing data block - for (unsigned int col=0; col CommandLine::nlQuiet) - cout << "Constructing: done." << endl; - - // Solve the matrices only if recovering data - if (datamissing > 0) - { - // Perform Gaussian Elimination and then delete the right matrix (which - // will no longer be required). - bool success = GaussElim(noiselevel, outcount, incount, leftmatrix, rightmatrix, datamissing); - delete [] rightmatrix; - return success; - } - - return true; -} - -// Use Gaussian Elimination to solve the matrices -template -inline bool ReedSolomon::GaussElim(CommandLine::NoiseLevel noiselevel, unsigned int rows, unsigned int leftcols, G *leftmatrix, G *rightmatrix, unsigned int datamissing) -{ - if (noiselevel == CommandLine::nlDebug) - { - for (unsigned int row=0; row8?4:2) << setfill('0') - << (unsigned int)leftmatrix[row*leftcols+col]; - } - cout << ((row==0) ? " \\ /" : (row==rows-1) ? " / \\" : " | |"); - for (unsigned int col=0; col8?4:2) << setfill('0') - << (unsigned int)rightmatrix[row*rows+col]; - } - cout << ((row==0) ? " \\" : (row==rows-1) ? " /" : " | |"); - cout << endl; - - cout << dec << setw(0) << setfill(' '); - } - } - - // Because the matrices being operated on are Vandermonde matrices - // they are guaranteed not to be singular. - - // Additionally, because Galois arithmetic is being used, all calulations - // involve exact values with no loss of precision. It is therefore - // not necessary to carry out any row or column swapping. - - // Solve one row at a time - - int progress = 0; - - // For each row in the matrix - for (unsigned int row=0; row CommandLine::nlQuiet) - { - int newprogress = (row*rows+row2) * 1000 / (datamissing*rows); - if (progress != newprogress) - { - progress = newprogress; - cout << "Solving: " << progress/10 << '.' << progress%10 << "%\r" << flush; - } - } - - if (row != row2) - { - // Get the scaling factor for this row. - G scalevalue = rightmatrix[row2 * rows + row]; - - if (scalevalue == 1) - { - // If the scaling factor happens to be 1, just subtract rows - for (unsigned int col=0; col CommandLine::nlQuiet) - cout << "Solving: done." << endl; - if (noiselevel == CommandLine::nlDebug) - { - for (unsigned int row=0; row8?4:2) << setfill('0') - << (unsigned int)leftmatrix[row*leftcols+col]; - } - cout << ((row==0) ? " \\ /" : (row==rows-1) ? " / \\" : " | |"); - for (unsigned int col=0; col8?4:2) << setfill('0') - << (unsigned int)rightmatrix[row*rows+col]; - } - cout << ((row==0) ? " \\" : (row==rows-1) ? " /" : " | |"); - cout << endl; - - cout << dec << setw(0) << setfill(' '); - } - } - - return true; -} - -} // end namespace Par2 - -#endif // __REEDSOLOMON_H__ diff --git a/lib/par2/verificationhashtable.cpp b/lib/par2/verificationhashtable.cpp deleted file mode 100644 index 5e80c3df5..000000000 --- a/lib/par2/verificationhashtable.cpp +++ /dev/null @@ -1,113 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -VerificationHashTable::VerificationHashTable(void) -{ - hashmask = 0; - hashtable = 0; -} - -VerificationHashTable::~VerificationHashTable(void) -{ - // Destroy the hash table - if (hashtable) - { - for (unsigned int entry=0; entry<=hashmask; entry++) - { - delete hashtable[entry]; - } - } - - delete [] hashtable; -} - -// Allocate the hash table with a reasonable size -void VerificationHashTable::SetLimit(u32 limit) -{ - // Pick a good size for the hash table - hashmask = 256; - while (hashmask < limit && hashmask < 65536) - { - hashmask <<= 1; - } - - // Allocate and clear the hash table - hashtable = new VerificationHashEntry*[hashmask]; - memset(hashtable, 0, hashmask * sizeof(hashtable[0])); - - hashmask--; -} - -// Load data from a verification packet -void VerificationHashTable::Load(Par2RepairerSourceFile *sourcefile, u64 blocksize) -{ - VerificationHashEntry *preventry = 0; - - // Get information from the sourcefile - VerificationPacket *verificationpacket = sourcefile->GetVerificationPacket(); - u32 blockcount = verificationpacket->BlockCount(); - - // Iterate throught the data blocks for the source file and the verification - // entries in the verification packet. - vector::iterator sourceblocks = sourcefile->SourceBlocks(); - const FILEVERIFICATIONENTRY *verificationentry = verificationpacket->VerificationEntry(0); - u32 blocknumber = 0; - - while (blocknumberInsert(&hashtable[entry->Checksum() & hashmask]); - - // Make the previous entry point forwards to this one - if (preventry) - { - preventry->Next(entry); - } - preventry = entry; - - ++blocknumber; - ++sourceblocks; - ++verificationentry; - } -} - -} // end namespace Par2 diff --git a/lib/par2/verificationhashtable.h b/lib/par2/verificationhashtable.h deleted file mode 100644 index 126b90685..000000000 --- a/lib/par2/verificationhashtable.h +++ /dev/null @@ -1,450 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __VERIFICATIONHASHTABLE_H__ -#define __VERIFICATIONHASHTABLE_H__ - -namespace Par2 -{ - -class Par2RepairerSourceFile; -class VerificationHashTable; - -// The VerificationHashEntry objects form the nodes of a binary trees stored -// in a VerificationHashTable object. - -// There is one VerificationHashEntry object for each data block in the original -// source files. - -class VerificationHashEntry -{ -public: - VerificationHashEntry(Par2RepairerSourceFile *_sourcefile, - DataBlock *_datablock, - bool _firstblock, - const FILEVERIFICATIONENTRY *_verificationentry) - { - sourcefile = _sourcefile; - datablock = _datablock; - firstblock = _firstblock; - - crc = _verificationentry->crc; - hash = _verificationentry->hash; - - left = right = same = next = 0; - } - - ~VerificationHashEntry(void) - { - delete left; - delete right; - delete same; - } - - // Insert the current object is a child of the specified parent - void Insert(VerificationHashEntry **parent); - - // Search (starting at the specified parent) for an object with a matching crc - static const VerificationHashEntry* Search(const VerificationHashEntry *entry, u32 crc); - - // Search (starting at the specified parent) for an object with a matching hash - static const VerificationHashEntry* Search(const VerificationHashEntry *entry, const MD5Hash &hash); - - // Comparison operators for searching - bool operator <(const VerificationHashEntry &r) const - { - return (crc < r.crc) || ((crc == r.crc) && (hash < r.hash)); - } - bool operator >(const VerificationHashEntry &r) const - { - return (crc > r.crc) || ((crc == r.crc) && (hash > r.hash)); - } - bool operator ==(const VerificationHashEntry &r) const - { - return (crc == r.crc) && (hash == r.hash); - } - bool operator <=(const VerificationHashEntry &r) const {return !operator>(r);} - bool operator >=(const VerificationHashEntry &r) const {return !operator<(r);} - bool operator !=(const VerificationHashEntry &r) const {return !operator==(r);} - - // Data - Par2RepairerSourceFile* SourceFile(void) const {return sourcefile;} - const DataBlock* GetDataBlock(void) const {return datablock;} - bool FirstBlock(void) const {return firstblock;} - - // Set/Check the associated datablock - void SetBlock(DiskFile *diskfile, u64 offset) const; - bool IsSet(void) const; - - u32 Checksum(void) const {return crc;} - const MD5Hash& Hash(void) const {return hash;} - - VerificationHashEntry* Same(void) const {return same;} - VerificationHashEntry* Next(void) const {return next;} - void Next(VerificationHashEntry *_next) {next = _next;} - -protected: - // Data - Par2RepairerSourceFile *sourcefile; - DataBlock *datablock; - bool firstblock; - - u32 crc; - MD5Hash hash; - -protected: - // Binary tree - VerificationHashEntry *left; - VerificationHashEntry *right; - - // Linked list of entries with the same crc and hash - VerificationHashEntry *same; - - // Linked list of entries in sequence for same file - VerificationHashEntry *next; -}; - -inline void VerificationHashEntry::SetBlock(DiskFile *diskfile, u64 offset) const -{ - datablock->SetLocation(diskfile, offset); -} - -inline bool VerificationHashEntry::IsSet(void) const -{ - return datablock->IsSet(); -} - -// Insert a new entry in the tree -inline void VerificationHashEntry::Insert(VerificationHashEntry **parent) -{ - while (*parent) - { - if (**parent < *this) - { - parent = &(*parent)->right; - } - else if (**parent > *this) - { - parent = &(*parent)->left; - } - else - { - break; - } - } - - while (*parent) - { - parent = &(*parent)->same; - } - - *parent = this; -} - -// Search the tree for an entry with the correct crc -inline const VerificationHashEntry* VerificationHashEntry::Search(const VerificationHashEntry *entry, u32 crc) -{ - while (entry) - { - if (entry->crc < crc) - { - entry = entry->right; - } - else if (entry->crc > crc) - { - entry = entry->left; - } - else - { - break; - } - } - - return entry; -} - -// Search the tree for an entry with the correct hash -inline const VerificationHashEntry* VerificationHashEntry::Search(const VerificationHashEntry *entry, const MD5Hash &hash) -{ - u32 crc = entry->crc; - - while (entry) - { - if ((entry->crc < crc) || ((entry->crc == crc) && (entry->hash < hash))) - { - entry = entry->right; - } - else if ((entry->crc > crc) || ((entry->crc == crc) && (entry->hash > hash))) - { - entry = entry->left; - } - else - { - break; - } - } - - return entry; -} - -// The VerificationHashTable object contains all of the VerificationHashEntry objects -// and is used to find matches for blocks of data in a target file that is being -// scanned. - -// It is initialised by loading data from all available verification packets for the -// source files. - -class VerificationHashTable -{ -public: - VerificationHashTable(void); - ~VerificationHashTable(void); - - void SetLimit(u32 limit); - - // Load the data from the verification packet - void Load(Par2RepairerSourceFile *sourcefile, u64 blocksize); - - // Try to find a match. - // nextentry - The entry which we expect to find next. This is used - // when a sequence of matches are found. - // sourcefile - Which source file we would prefer to find a match for - // if there are more than one possible match (with the - // same crc and hash). - // checksummer - Provides the crc and hash values being tested. - // duplicate - Set on return if the match would have been valid except - // for the fact that the block has already been found. - const VerificationHashEntry* FindMatch(const VerificationHashEntry *nextentry, - const Par2RepairerSourceFile *sourcefile, - FileCheckSummer &checksummer, - bool &duplicate) const; - - // Look up based on the block crc - const VerificationHashEntry* Lookup(u32 crc) const; - - // Continue lookup based on the block hash - const VerificationHashEntry* Lookup(const VerificationHashEntry *entry, - const MD5Hash &hash); - -protected: - VerificationHashEntry **hashtable; - unsigned int hashmask; -}; - -// Search for an entry with the specified crc -inline const VerificationHashEntry* VerificationHashTable::Lookup(u32 crc) const -{ - if (hashmask) - { - return VerificationHashEntry::Search(hashtable[crc & hashmask], crc); - } - - return 0; -} - -// Search for an entry with the specified hash -inline const VerificationHashEntry* VerificationHashTable::Lookup(const VerificationHashEntry *entry, - const MD5Hash &hash) -{ - return VerificationHashEntry::Search(entry, hash); -} - -inline const VerificationHashEntry* VerificationHashTable::FindMatch(const VerificationHashEntry *suggestedentry, - const Par2RepairerSourceFile *sourcefile, - FileCheckSummer &checksummer, - bool &duplicate) const -{ - duplicate = false; - - // Get the current checksum from the checksummer - u32 crc = checksummer.Checksum(); - - MD5Hash hash; - bool havehash = false; - - // Do we know what the next entry should be - if (0 != suggestedentry) - { - // Is the suggested match supposed to be the last one in the file - if (suggestedentry->Next() == 0) - { - // How long should the last block be - u64 length = suggestedentry->GetDataBlock()->GetLength(); - - // Get a short checksum from the checksummer - u32 checksum = checksummer.ShortChecksum(length); - - // Is the checksum correct - if (checksum == suggestedentry->Checksum()) - { - // Get a short hash from the checksummer - hash = checksummer.ShortHash(length); - - // If the hash matches as well, then return it - if (hash == suggestedentry->Hash()) - { - return suggestedentry; - } - } - } - // If the suggested entry has not already been found, compare the checksum - else if (!suggestedentry->IsSet() && suggestedentry->Checksum() == crc) - { - // Get the hash value from the checksummer - havehash = true; - hash = checksummer.Hash(); - - // If the hash value matches, then return it. - if (hash == suggestedentry->Hash()) - { - return suggestedentry; - } - } - } - - // Look for other possible matches for the checksum - const VerificationHashEntry *nextentry = VerificationHashEntry::Search(hashtable[crc & hashmask], crc); - if (0 == nextentry) - return 0; - - // If we don't have the hash yet, get it - if (!havehash) - { - hash = checksummer.Hash(); - } - - // Look for an entry with a matching hash - nextentry = VerificationHashEntry::Search(nextentry, hash); - if (0 == nextentry) - return 0; - - // Is there one match with the same checksum and hash, or many - if (nextentry->Same() == 0) - { - // If the match is for a block that is part of a target file - // for which we already have a complete match, then don't - // return it. - if (nextentry->SourceFile()->GetCompleteFile() != 0) - { - duplicate = true; - return 0; - } - - // If we are close to the end of the file and the block - // length is wrong, don't return it because it is an - // invalid match - if (checksummer.ShortBlock() && checksummer.BlockLength() != nextentry->GetDataBlock()->GetLength()) - { - return 0; - } - - // If the match was at the start of the file and it is the first - // block for a target file, then return it. - if (nextentry->FirstBlock() && checksummer.Offset() == 0) - { - return nextentry; - } - - // Was this match actually the one which had originally been suggested - // but which has presumably already been found - if (nextentry == suggestedentry) - { - // Was the existing match in the same file as the new match - if (nextentry->IsSet() && - nextentry->GetDataBlock()->GetDiskFile() == checksummer.GetDiskFile()) - { - // Yes. Don't return it - duplicate = true; - return 0; - } - else - { - // No, it was in a different file. Return it. - // This ensures that we can find a perfect match for a target - // file even if some of the blocks had already been found - // in a different file. - return nextentry; - } - } - else - { - // return it only if it has not already been used - if (nextentry->IsSet()) - { - duplicate = true; - return 0; - } - - return nextentry; - } - } - - // Do we prefer to match entries for a particular source file - if (0 != sourcefile) - { - const VerificationHashEntry *currententry = nextentry; - nextentry = 0; - - // We don't want entries for the wrong source file, ones that - // have already been matched, or ones that are the wrong length - while (currententry && (currententry->SourceFile() != sourcefile || - currententry->IsSet() || - (checksummer.ShortBlock() && checksummer.BlockLength() != (currententry->GetDataBlock()->GetLength())) - ) - ) - { - // If we found an unused entry (which was presumably for the wrong - // source file) remember it (providing it is the correct length). - if (0 == nextentry && !(currententry->IsSet() || - (checksummer.ShortBlock() && checksummer.BlockLength() != (currententry->GetDataBlock()->GetLength())) - ) - ) - { - nextentry = currententry; - } - - currententry = currententry->Same(); - } - - // If we found an unused entry for the source file we want, return it - if (0 != currententry) - return currententry; - } - - // Check for an unused entry which is the correct length - while (nextentry && (nextentry->IsSet() || - (checksummer.ShortBlock() && checksummer.BlockLength() != nextentry->GetDataBlock()->GetLength()) - ) - ) - { - nextentry = nextentry->Same(); - } - - // Return what we have found - if (nextentry == 0) - { - duplicate = true; - } - - return nextentry; -} - -} // end namespace Par2 - -#endif // __VERIFICATIONHASHTABLE_H__ diff --git a/lib/par2/verificationpacket.cpp b/lib/par2/verificationpacket.cpp deleted file mode 100644 index c7b71c70a..000000000 --- a/lib/par2/verificationpacket.cpp +++ /dev/null @@ -1,109 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#include "nzbget.h" -#include "par2cmdline.h" - -#ifdef _MSC_VER -#ifdef _DEBUG -#undef THIS_FILE -static char THIS_FILE[]=__FILE__; -#define new DEBUG_NEW -#endif -#endif - -namespace Par2 -{ - -// Create a packet large enough for the specified number of blocks - -bool VerificationPacket::Create(u32 _blockcount) -{ - blockcount = _blockcount; - - // Allocate a packet large enough to hold the required number of verification entries. - FILEVERIFICATIONPACKET *packet = (FILEVERIFICATIONPACKET*)AllocatePacket(sizeof(FILEVERIFICATIONPACKET) + blockcount * sizeof(FILEVERIFICATIONENTRY)); - - // Record everything we know in the packet. - packet->header.magic = packet_magic; - packet->header.length = packetlength; - //packet->header.hash; // Not known yet - //packet->header.setid; // Not known yet - packet->header.type = fileverificationpacket_type; - - //packet->fileid; // Not known yet - //packet->entries; // Not known yet - - return true; -} - -void VerificationPacket::FileId(const MD5Hash &fileid) -{ - assert(packetdata != 0); - - // Store the fileid in the packet. - ((FILEVERIFICATIONPACKET*)packetdata)->fileid = fileid; -} - -void VerificationPacket::SetBlockHashAndCRC(u32 blocknumber, const MD5Hash &hash, u32 crc) -{ - assert(packetdata != 0); - assert(blocknumber < blockcount); - - // Store the block hash and block crc in the packet. - FILEVERIFICATIONENTRY &entry = ((FILEVERIFICATIONPACKET*)packetdata)->entries[blocknumber]; - - entry.hash = hash; - entry.crc = crc; -} - -bool VerificationPacket::Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header) -{ - // Is the packet large enough - if (header.length <= sizeof(FILEVERIFICATIONPACKET)) - { - return false; - } - - // Does the packet have a whole number of verification records - if (0 < ((header.length - sizeof(FILEVERIFICATIONPACKET)) % sizeof(FILEVERIFICATIONENTRY))) - { - return false; - } - - // Is the packet too large - if (header.length > sizeof(FILEVERIFICATIONPACKET) + 32768 * sizeof(FILEVERIFICATIONENTRY)) - { - return false; - } - - // Allocate the packet - FILEVERIFICATIONPACKET *packet = (FILEVERIFICATIONPACKET*)AllocatePacket((size_t)header.length); - packet->header = header; - - // How many blocks are there - blockcount = (u32)((((FILEVERIFICATIONPACKET*)packetdata)->header.length - sizeof(FILEVERIFICATIONPACKET)) / sizeof(FILEVERIFICATIONENTRY)); - - // Read the rest of the packet - return diskfile->Read(offset + sizeof(PACKET_HEADER), - &packet->fileid, - (size_t)packet->header.length - sizeof(PACKET_HEADER)); -} - -} // end namespace Par2 diff --git a/lib/par2/verificationpacket.h b/lib/par2/verificationpacket.h deleted file mode 100644 index a10a65ec3..000000000 --- a/lib/par2/verificationpacket.h +++ /dev/null @@ -1,85 +0,0 @@ -// This file is part of par2cmdline (a PAR 2.0 compatible file verification and -// repair tool). See http://parchive.sourceforge.net for details of PAR 2.0. -// -// Copyright (c) 2003 Peter Brian Clements -// -// par2cmdline is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// par2cmdline is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - -#ifndef __VERIFICATIONPACKET_H__ -#define __VERIFICATIONPACKET_H__ - -namespace Par2 -{ - -// The file verification packet stores details that allow individual blocks -// of valid data within a damaged file to be identified. - -class VerificationPacket : public CriticalPacket -{ -public: - // Construct the packet - VerificationPacket(void) {}; - ~VerificationPacket(void) {}; - - // Create a packet large enough for the specified number of blocks - bool Create(u32 blockcount); - - // Set the fileid (computed from the file description packet). - void FileId(const MD5Hash &fileid); - - // Set the block hash and block crc for a specific data block. - void SetBlockHashAndCRC(u32 blocknumber, const MD5Hash &hash, u32 crc); - - // Load a verification packet from a specified file - bool Load(DiskFile *diskfile, u64 offset, PACKET_HEADER &header); - - // Get the FileId - const MD5Hash& FileId(void) const; - - // Get the block count - u32 BlockCount(void) const; - - // Get a specific verification entry - const FILEVERIFICATIONENTRY* VerificationEntry(u32 blocknumber) const; - -protected: - u32 blockcount; -}; - -inline const MD5Hash& VerificationPacket::FileId(void) const -{ - assert(packetdata != 0); - - return ((FILEVERIFICATIONPACKET*)packetdata)->fileid; -} - -inline u32 VerificationPacket::BlockCount(void) const -{ - assert(packetdata != 0); - - return blockcount; -} - -inline const FILEVERIFICATIONENTRY* VerificationPacket::VerificationEntry(u32 blocknumber) const -{ - assert(packetdata != 0); - -// return &((FILEVERIFICATIONPACKET*)packetdata)->entries()[blocknumber]; - return &((FILEVERIFICATIONPACKET*)packetdata)->entries[blocknumber]; -} - -} // end namespace Par2 - -#endif // __VERIFICATIONPACKET_H__ diff --git a/lib/regex/regex.c b/lib/regex/regex.c index 0242bea30..c60ddbfd0 100644 --- a/lib/regex/regex.c +++ b/lib/regex/regex.c @@ -18,9 +18,7 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ -#ifdef HAVE_CONFIG_H #include "config.h" -#endif /* Make sure noone compiles this code with a C++ compiler. */ #ifdef __cplusplus diff --git a/lib/sources.cmake b/lib/sources.cmake index 45cc45fa7..1a660e2c2 100644 --- a/lib/sources.cmake +++ b/lib/sources.cmake @@ -13,86 +13,52 @@ add_library(regex STATIC ${CMAKE_SOURCE_DIR}/lib/regex/regex.c ) target_include_directories(regex PUBLIC - ${CMAKE_SOURCE_DIR}/lib/regex -) - -add_library(yencode STATIC - ${CMAKE_SOURCE_DIR}/lib/yencode/SimdInit.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/SimdDecoder.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/ScalarDecoder.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/Sse2Decoder.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/Ssse3Decoder.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/PclmulCrc.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/NeonDecoder.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/AcleCrc.cpp - ${CMAKE_SOURCE_DIR}/lib/yencode/SliceCrc.cpp -) -target_include_directories(yencode PUBLIC - ${CMAKE_SOURCE_DIR}/lib/yencode - ${CMAKE_SOURCE_DIR}/lib/regex - ${CMAKE_SOURCE_DIR}/daemon/main ${INCLUDES} + ${CMAKE_SOURCE_DIR}/lib/regex ) set_source_files_properties( ${CMAKE_SOURCE_DIR}/lib/yencode/Sse2Decoder.cpp - PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${SSE2_CXXFLAGS}" + PROPERTIES COMPILE_FLAGS "${SSE2_CXXFLAGS}" ) set_source_files_properties( ${CMAKE_SOURCE_DIR}/lib/yencode/Ssse3Decoder.cpp - PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${SSSE3_CXXFLAGS}" + PROPERTIES COMPILE_FLAGS "${SSSE3_CXXFLAGS}" ) set_source_files_properties( ${CMAKE_SOURCE_DIR}/lib/yencode/PclmulCrc.cpp - PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${PCLMUL_CXXFLAGS}" + PROPERTIES COMPILE_FLAGS "${PCLMUL_CXXFLAGS}" ) set_source_files_properties( ${CMAKE_SOURCE_DIR}/lib/yencode/NeonDecoder.cpp - PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS_RELEASE} ${NEON_CXXFLAGS}" + PROPERTIES COMPILE_FLAGS "${NEON_CXXFLAGS}" ) set_source_files_properties( ${CMAKE_SOURCE_DIR}/lib/yencode/AcleCrc.cpp - PROPERTIES COMPILE_FLAGS "${CMAKE_CXX_FLAGS} ${ACLECRC_CXXFLAGS}" + PROPERTIES COMPILE_FLAGS "${ACLECRC_CXXFLAGS}" ) -if(NOT DISABLE_PARCHECK) - add_library(par2 STATIC - ${CMAKE_SOURCE_DIR}/lib/par2/commandline.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/crc.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/creatorpacket.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/criticalpacket.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/datablock.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/descriptionpacket.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/diskfile.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/filechecksummer.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/galois.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/mainpacket.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/md5.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/par2fileformat.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/par2repairer.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/par2repairersourcefile.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/parheaders.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/recoverypacket.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/reedsolomon.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/verificationhashtable.cpp - ${CMAKE_SOURCE_DIR}/lib/par2/verificationpacket.cpp - ) - target_include_directories(par2 PUBLIC - ${CMAKE_SOURCE_DIR}/lib/par2 - ${CMAKE_SOURCE_DIR}/lib/regex - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/util - ${INCLUDES} - ) -endif() +add_library(yencode STATIC + ${CMAKE_SOURCE_DIR}/lib/yencode/SimdInit.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/SimdDecoder.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/ScalarDecoder.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/Sse2Decoder.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/Ssse3Decoder.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/PclmulCrc.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/NeonDecoder.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/AcleCrc.cpp + ${CMAKE_SOURCE_DIR}/lib/yencode/SliceCrc.cpp +) +target_include_directories(yencode PUBLIC + ${CMAKE_SOURCE_DIR}/lib/yencode + ${CMAKE_SOURCE_DIR}/lib/regex + ${CMAKE_SOURCE_DIR}/daemon/main + ${INCLUDES} +) set(LIBS ${LIBS} regex yencode) set(INCLUDES ${INCLUDES} ${CMAKE_SOURCE_DIR}/lib/regex ${CMAKE_SOURCE_DIR}/lib/yencode) -if(NOT DISABLE_PARCHECK) - set(LIBS ${LIBS} par2) - set(INCLUDES ${INCLUDES} ${CMAKE_SOURCE_DIR}/lib/par2) -endif() diff --git a/linux/build-info.md b/linux/build-info.md index db9585f47..c6ff0684d 100644 --- a/linux/build-info.md +++ b/linux/build-info.md @@ -1,20 +1,23 @@ # About -`build-nzbget.sh` is a bash script which is used to build linux and android nzbget packages. +`build-nzbget.sh` is a bash script which is used to build linux, android and freebsd nzbget packages. Supported linux architectures: `armel` `armhf` `aarch64` `i686` `x86_64` `riscv64` `mipseb` `mipsel` `ppc500` `ppc6xx` Supported android architectures: `i686-ndk` `x86_64-ndk` `armhf-ndk` `aarch64-ndk` +Supported freebsd architectures: `x86_64-bsd` + # Prerequisites 1. Linux x86_64 host (Ubuntu 22.04 LTS for example) 2. Installed build dependencies (Ubuntu/Debian example): ``` -sudo apt install autoconf automake bc build-essential cmake cpio curl file git libtool pkg-config rsync unzip wget libtinfo5 +sudo apt install autoconf automake bc build-essential cmake cpio curl file git libtool pkg-config rsync unzip wget libtinfo5 clang ldd ``` 3. Installed buildroot - one per architecture (see [Buildroot setup](#buildroot-setup) below) 4. Installed Android NDK and standalone Android toolkits - one per architecture (see [NDK setup](#ndk-setup) below) +5. Installed FreeBSD x86_64 sysroot (see [FreeBSD sysroot setup](#freebsd-sysroot-setup) below) # Building NZBGet @@ -24,9 +27,10 @@ bash linux/build-nzbget.sh [platforms] [architectures] [output] [configs] [testi ``` Build options: -- platforms: default value `linux android` +- platforms: default value `linux android freebsd` - linux: build linux packages - android: build android packages + - freebsd: build freebsd packages - architectures: default value `all` - linux: - armel @@ -44,6 +48,8 @@ Build options: - x86_64-ndk - armhf-ndk - aarch64-ndk + - freebsd: + - x86_64-bsd - output: default value `bin installer` - bin: build binary package - installer: build installer package @@ -135,7 +141,7 @@ for ARCH in aarch64 armel armhf i686 x86_64 riscv64 mipseb mipsel ppc500 ppc6xx; # NDK setup -Script assumes that andriod toolchains is installed in `/build/android/` - one folder per architecture. +Script assumes that android toolchains is installed in `/build/android/` - one folder per architecture. To install Android toolchain and NDK: @@ -154,3 +160,20 @@ If you want to build all supported toolchains, run ``` for ARCH in i686 x86_64 armhf aarch64; do bash linux/android/build-toolchain.sh $ARCH; done ``` + +# FreeBSD sysroot setup + +Script assumes that FreeBSD sysroot is installed in `/build/freebsd/sysroot` + +To install FreeBSD sysroot: + +Make the /build directory and add the necessary permissions. +``` +sudo mkdir -p /build +sudo chmod 777 /build +``` + +From the cloned repository, run: +``` +bash linux/freebsd/build-toolchain.sh +``` diff --git a/linux/build-nzbget.sh b/linux/build-nzbget.sh index 0a2b99cc3..af8ee77a5 100755 --- a/linux/build-nzbget.sh +++ b/linux/build-nzbget.sh @@ -29,12 +29,15 @@ COREX=4 TESTING="no" # build variables -ALL_ARCHS="armel armhf aarch64 i686 x86_64 riscv64 mipsel mipseb ppc500 ppc6xx i686-ndk x86_64-ndk armhf-ndk aarch64-ndk" -ALL_PLATFORMS="linux android" +ALL_ARCHS="armel armhf aarch64 i686 x86_64 riscv64 mipsel mipseb ppc500 ppc6xx i686-ndk x86_64-ndk armhf-ndk aarch64-ndk x86_64-bsd" +ALL_PLATFORMS="linux android freebsd" OUTPUTDIR=build BUILDROOT_HOME=/build LIB_SRC_PATH=$BUILDROOT_HOME/source LIB_PATH=$BUILDROOT_HOME/lib +# freebsd variables +FREEBSD_SYSROOT=/build/freebsd/sysroot +FREEBSD_CLANG_VER=14 # unpackers versions UNRAR6_VERSION=6.2.12 @@ -66,7 +69,7 @@ parse_args() for PARAM in "$@" do case $PARAM in - android|linux) + android|linux|freebsd) PLATFORMS=`echo "$PLATFORMS $PARAM" | xargs` ;; release|debug) @@ -134,7 +137,10 @@ parse_args() if [[ $ARCH == *-ndk ]] && [ "$PLATFORM" == "android" ]; then ARCH_NEED=1 fi - if [[ $ARCH != *-ndk ]] && [ "$PLATFORM" == "linux" ]; then + if [[ $ARCH == *-bsd ]] && [ "$PLATFORM" == "freebsd" ]; then + ARCH_NEED=1 + fi + if [[ $ARCH != *-ndk ]] && [[ $ARCH != *-bsd ]] && [ "$PLATFORM" == "linux" ]; then ARCH_NEED=1 fi done @@ -174,7 +180,10 @@ filter_archs() if [[ $ARCH == *-ndk ]] && [ "$PLATFORM" == "android" ]; then PLATFORM_ARCHS=`echo "$PLATFORM_ARCHS $ARCH" | xargs` fi - if [[ $ARCH != *-ndk ]] && [ "$PLATFORM" == "linux" ]; then + if [[ $ARCH == *-bsd ]] && [ "$PLATFORM" == "freebsd" ]; then + PLATFORM_ARCHS=`echo "$PLATFORM_ARCHS $ARCH" | xargs` + fi + if [[ $ARCH != *-ndk ]] && [[ $ARCH != *-bsd ]] && [ "$PLATFORM" == "linux" ]; then PLATFORM_ARCHS=`echo "$PLATFORM_ARCHS $ARCH" | xargs` fi done @@ -183,6 +192,7 @@ filter_archs() get_short_arch() { ARCH_SHORT="${ARCH/-ndk/}" + ARCH_SHORT="${ARCH_SHORT/-bsd/}" } download_lib_source() @@ -238,7 +248,11 @@ build_lib() --prefix="$PWD/../$LIB" ;; zlib) - ./configure --static --prefix="$PWD/../$LIB" + if [ "$PLATFORM" == "freebsd" ]; then + LIBS="$FREEBSD_SYSROOT/libs" ./configure --static --prefix="$PWD/../$LIB" + else + ./configure --static --prefix="$PWD/../$LIB" + fi ;; libxml2) ./autogen.sh --host=$HOST \ @@ -280,6 +294,9 @@ build_lib() OPENSSL_ARCH=linux-ppc OPENSSL_OPTS=no-async ;; + x86_64-bsd) + OPENSSL_ARCH=BSD-x86_64 + ;; *-ndk) _CC=$CC unset CC @@ -334,8 +351,12 @@ build_lib() ;; boost) ./bootstrap.sh --with-libraries=json --prefix="$PWD/../$LIB" - echo "using gcc : buildroot : $CXX ; " >> project-config.jam - ./b2 --toolset=gcc-buildroot cxxstd=14 link=static runtime-link=static install + if [ "$PLATFORM" == "freebsd" ]; then + ./b2 --toolset=clang cxxflags="--target=x86_64-pc-freebsd --sysroot=$FREEBSD_SYSROOT -I$FREEBSD_SYSROOT/usr/include/c++/v1" cxxstd=14 link=static runtime-link=static install + else + echo "using gcc : buildroot : $CXX ; " >> project-config.jam + ./b2 --toolset=gcc-buildroot cxxstd=14 link=static runtime-link=static install + fi ;; esac if [ "$LIB" != "boost" ]; then @@ -400,6 +421,9 @@ build_7zip() sed "s|^LIB2 =.*|LIB2 = |g" -i 7zip_gcc.mak sed "s|^CFLAGS_WARN_WALL =.*|CFLAGS_WARN_WALL = -Wall -Wextra|" -i 7zip_gcc.mak fi + if [ "$PLATFORM" == "freebsd" ]; then + sed "s|^MY_ARCH_2 = .*|MY_ARCH_2 = \$(MY_ARCH) --target=x86_64-pc-freebsd --sysroot=$FREEBSD_SYSROOT -I$FREEBSD_SYSROOT/usr/include/c++/v1|" -i 7zip_gcc.mak + fi cd Bundles/Alone make -j $COREX -f makefile.gcc mkdir -p $LIB_PATH/$ARCH/7zip @@ -458,6 +482,10 @@ build_unrar_version() sed "s|^LDFLAGS=.*|LDFLAGS=-static|" -i makefile sed "s|^CXXFLAGS=.*|CXXFLAGS=-std=c++11 -O2|" -i makefile fi + if [ "$PLATFORM" == "freebsd" ]; then + sed "s|^CXXFLAGS=.*|CXXFLAGS=-std=c++11 -O2 -nostdlib --target=x86_64-pc-freebsd --sysroot=$FREEBSD_SYSROOT -I$FREEBSD_SYSROOT/usr/include/c++/v1|" -i makefile + sed "s|^LDFLAGS=.*|LDFLAGS=-static --target=x86_64-pc-freebsd --sysroot=$FREEBSD_SYSROOT -pthread -lc++ -lm -fuse-ld=lld|" -i makefile + fi make clean make -j $COREX mkdir -p $LIB_PATH/$ARCH/unrar @@ -546,6 +574,10 @@ build_bin() export HOST="aarch64-linux-android" CMAKE_SYSTEM_PROCESSOR="aarch64" ;; + x86_64-bsd) + export HOST="x86_64-bsd" + CMAKE_SYSTEM_PROCESSOR="x86_64" + ;; esac @@ -558,6 +590,13 @@ build_bin() export AR="$TOOLCHAIN_PATH/$ARCH/output/host/usr/bin/$HOST-ar" export STRIP="$TOOLCHAIN_PATH/$ARCH/output/host/usr/bin/$HOST-strip" ;; + freebsd) + export CC="clang-$FREEBSD_CLANG_VER" + export CPP="clang-cpp-$FREEBSD_CLANG_VER" + export CXX="clang++-$FREEBSD_CLANG_VER" + unset AR + unset STRIP + ;; *) TOOLCHAIN_PATH=$BUILDROOT_HOME/buildroot export CC="$TOOLCHAIN_PATH/$ARCH/output/host/usr/bin/$HOST-gcc" @@ -574,6 +613,12 @@ build_bin() export LDFLAGS="" export NZBGET_INCLUDES="$TOOLCHAIN_PATH/$ARCH/output/staging/usr/include/;" + if [ $PLATFORM == "freebsd" ]; then + export CXXFLAGS="-Os --sysroot=$FREEBSD_SYSROOT -I$FREEBSD_SYSROOT/usr/include/c++/v1" + export CFLAGS=$CXXFLAGS + export CPPFLAGS=$CXXFLAGS + fi + build_lib "https://ftp.gnu.org/pub/gnu/ncurses/ncurses-$NCURSES_VERSION.tar.gz" build_lib "https://zlib.net/zlib-$ZLIB_VERSION.tar.gz" build_lib "https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.12.4/libxml2-v$LIBXML2_VERSION.tar.gz" @@ -583,17 +628,26 @@ build_bin() build_7zip build_unrar + export INCLUDES="$NZBGET_INCLUDES" + CMAKE_SYSTEM_NAME="Linux" + CMAKE_EXTRA_ARGS="" + TOOLCHAIN_PREFIX="$TOOLCHAIN_PATH/$ARCH/output/host/usr/bin/$HOST" case $PLATFORM in android) export LIBS="$LDFLAGS -lxml2 -lboost_json -lz -lssl -lcrypto -lncursesw -latomic" - CMAKE_EXTRA_ARGS="-DCOMPILER=clang" + CMAKE_EXTRA_ARGS="-DCOMPILER=clang -DTOOLCHAIN_PREFIX=$TOOLCHAIN_PREFIX" + ;; + freebsd) + export LIBS="$LDFLAGS -lxml2 -lboost_json -lz -lssl -lcrypto -lncursesw -lc++ -Wl,--whole-archive -lpthread -Wl,--no-whole-archive" + export INCLUDES="$NZBGET_INCLUDES;$FREEBSD_SYSROOT/usr/include/c++/v1" + CMAKE_SYSTEM_NAME="FreeBSD" + CMAKE_EXTRA_ARGS="-DCMAKE_SYSROOT=$FREEBSD_SYSROOT" ;; *) export LIBS="$LDFLAGS -lxml2 -lrt -lboost_json -lz -lssl -lcrypto -lncursesw -latomic -Wl,--whole-archive -lpthread -Wl,--no-whole-archive" - CMAKE_EXTRA_ARGS="" + CMAKE_EXTRA_ARGS="-DTOOLCHAIN_PREFIX=$TOOLCHAIN_PREFIX" ;; esac - export INCLUDES="$NZBGET_INCLUDES" unset CXXFLAGS unset CPPFLAGS @@ -607,10 +661,9 @@ build_bin() mkdir -p $OUTPUTDIR/$ARCH cmake -S . -B $OUTPUTDIR/$ARCH \ - -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_NAME=$CMAKE_SYSTEM_NAME \ -DCMAKE_SYSTEM_PROCESSOR=$CMAKE_SYSTEM_PROCESSOR \ -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain.cmake \ - -DTOOLCHAIN_PREFIX=$TOOLCHAIN_PATH/$ARCH/output/host/usr/bin/$HOST \ -DENABLE_STATIC=ON \ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ -DVERSION_SUFFIX=$VERSION_SUFFIX \ diff --git a/linux/freebsd/build-toolchain.sh b/linux/freebsd/build-toolchain.sh new file mode 100755 index 000000000..4b7c318fb --- /dev/null +++ b/linux/freebsd/build-toolchain.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# This file is part of nzbget. See . +# +# Copyright (C) 2015-2017 Andrey Prygunkov +# Copyright (C) 2024 phnzb +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +# FreeBSD sysroot parameters +FREEBSD_VERSION="13.0" +FREEBSDIMAGE_URL="http://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/amd64/$FREEBSD_VERSION-RELEASE/base.txz" + +### START OF THE SCRIPT + +ROOTDIR="/build/freebsd" +mkdir -p $ROOTDIR + +# Download all required tools and libraries +SRCDIR=/build/source +FREEBSD_BASE="base-$FREEBSD_VERSION.txz" +mkdir -p $SRCDIR +cd $SRCDIR +if [ ! -f $FREEBSD_BASE ]; then + curl -L -o "$SRCDIR/$FREEBSD_BASE" $FREEBSDIMAGE_URL +fi + +# Creating sysroot for FreeBSD from official FreeBSD installation image. +# Our sysroot contains only libs and includes +SYSROOT="$ROOTDIR/sysroot" +rm -rf $SYSROOT +mkdir -p $SYSROOT +cd $SYSROOT +tar -xf $SRCDIR/$FREEBSD_BASE ./lib/ ./usr/lib/ ./usr/include/ + +cd $SYSROOT/usr/lib +# Fix broken symlinks +find . -xtype l | xargs ls -l | grep ' /lib/' | awk -v SYSROOT="$SYSROOT" '{print "ln -sf "SYSROOT$11" "$9}' | /bin/sh +ln -s libc++.a $SYSROOT/usr/lib/libstdc++.a +ln -s libc++.so $SYSROOT/usr/lib/libstdc++.so diff --git a/nzbget.conf b/nzbget.conf index ba826c1b3..73bcf0243 100644 --- a/nzbget.conf +++ b/nzbget.conf @@ -453,9 +453,9 @@ UpdateCheck=stable # This allows NZBGet daemon to be launched in rc.local (at boot), and # download items as a specific user id. # -# NOTE: This option has effect only if the program was started from -# root-account, otherwise it is ignored and the daemon runs under -# current user id. +# NOTE: If DaemonUsername is not "root" and doesn't exist the daemon will not start. +# NOTE: If the daemon is running as root (superuser) and DaemonUsername exists, +# nzbget will drop root privileges to the user specified by DaemonUsername. DaemonUsername=root # Specify default umask, POSIX only (000-1000). diff --git a/osx/NZBGet-Info.plist b/osx/NZBGet-Info.plist index 4bc8e12a6..303174cd1 100644 --- a/osx/NZBGet-Info.plist +++ b/osx/NZBGet-Info.plist @@ -17,7 +17,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 24.3 + 24.4 CFBundleDocumentTypes diff --git a/osx/sign/backgound.png b/osx/sign/backgound.png new file mode 100644 index 000000000..03bedb8e6 Binary files /dev/null and b/osx/sign/backgound.png differ diff --git a/osx/sign/nzbget-sign.sh b/osx/sign/nzbget-sign.sh new file mode 100755 index 000000000..1b5bd90c5 --- /dev/null +++ b/osx/sign/nzbget-sign.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# +# This file is part of nzbget. See . +# +# Copyright (C) 2024 phnzb +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +set -e + +# This script is used for signing and optionally notarizing nzbget mac os packages +# Script takes nzbget macos package from *-bin-macos-*.zip file, signs binaries in package and package itself. +# Then creates dmg with package, signs and optionally notarizes it + +# pre-requisites +# 1. create-dmg: https://github.com/create-dmg/create-dmg +# 2. Apple Developer ID Application certificate. DEVELOPER_IDENTITY == Certificate User ID +# 3. Notarization: AppStore Connect API key with Developer role, placed in $HOME/.private_keys/. NOTARY_KEY_ID == Key ID, NOTARY_KEY_ISSUER == Key Issuer ID + +# check command-line parameters +APP_ARCHIVE=$1 +if [ -z $APP_ARCHIVE ]; then + echo "No NZBGet archive provided. Exiting." + exit 1 +fi + +# test needed vars +if [ -z "$KEYCHAIN_PASSWORD" ] && [ -z "$DEVELOPER_IDENTITY" ]; then + echo "No config vars found. Exiting" + exit 1 +fi + +if [ "$NOTARIZE" == "true" ]; then + if [ -z "$NOTARY_KEY_ID" ] && [ -z "$NOTARY_KEY_ISSUER" ]; then + echo "No config vars found for notarization. Exiting" + exit 1 + fi +fi + +DMG_NAME=${APP_ARCHIVE/-bin-macos/} +DMG_NAME=${DMG_NAME/.zip/.dmg} + +# unzip archive +unzip "$APP_ARCHIVE" + +SIGN_FILES="\ +MacOS/NZBGet \ +Resources/daemon/usr/local/bin/7za \ +Resources/daemon/usr/local/bin/unrar \ +Resources/daemon/usr/local/bin/nzbget" + +# sign binary files in package +security unlock-keychain -p $KEYCHAIN_PASSWORD ~/Library/Keychains/login.keychain-db +for SIGN_FILE in $SIGN_FILES; do + codesign -s "$DEVELOPER_IDENTITY" -f --timestamp -o runtime -i "com.nzbget" "NZBGet.app/Contents/$SIGN_FILE" +done + +# sign package itself +codesign -s "$DEVELOPER_IDENTITY" -f --timestamp -o runtime -i "com.nzbget" "NZBGet.app" + +# create dmg +create-dmg \ + --volname "NZBGet" \ + --window-pos 100 100 \ + --window-size 435 240 \ + --icon-size 80 \ + --icon "NZBGet.app" 100 95 \ + --hide-extension "NZBGet.app" \ + --app-drop-link 300 95 \ + --background backgound.png \ + "$DMG_NAME" \ + "NZBGet.app/" + +# sign dmg and check +codesign -s "$DEVELOPER_IDENTITY" -f --timestamp -o runtime -i "com.nzbget" "$DMG_NAME" +codesign -vvv --deep --strict "$DMG_NAME" + +# cleanup +rm -rf NZBGet.app + +# notarize dmg - optional +# useful commands: +# notarization history: +# > xcrun notarytool history --key "$HOME/.private_keys/AuthKey_$NOTARY_KEY_ID.p8" --key-id "$NOTARY_KEY_ID" --issuer "$NOTARY_KEY_ISSUER" +# notarization log: +# > xcrun notarytool log "$SUBMISSION_ID" --key "$HOME/.private_keys/AuthKey_$NOTARY_KEY_ID.p8" --key-id "$NOTARY_KEY_ID" --issuer "$NOTARY_KEY_ISSUER" "$SUBMISSION_ID.log" +# notarization check: +# > spctl -a -vvv -t install "$DMG_NAME" +if [ "$NOTARIZE" == "true" ]; then + xcrun notarytool submit "$DMG_NAME" --key "$HOME/.private_keys/AuthKey_$NOTARY_KEY_ID.p8" --key-id "$NOTARY_KEY_ID" --issuer "$NOTARY_KEY_ISSUER" --wait + xcrun stapler staple "$DMG_NAME" +fi diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f18ef513a..355dcb525 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,4 +7,4 @@ add_subdirectory(main) add_subdirectory(extension) add_subdirectory(nntp) add_subdirectory(system) -#add_subdirectory(postprocess) +add_subdirectory(postprocess) diff --git a/tests/extension/CMakeLists.txt b/tests/extension/CMakeLists.txt index aa06baf81..5c78f0dbd 100644 --- a/tests/extension/CMakeLists.txt +++ b/tests/extension/CMakeLists.txt @@ -33,22 +33,7 @@ file(GLOB ExtensionSrc add_executable(ExtensionTests ${ExtensionSrc}) target_link_libraries(ExtensionTests PRIVATE ${LIBS}) - -target_include_directories(ExtensionTests PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/extension - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/daemon/postprocess - ${CMAKE_SOURCE_DIR}/daemon/connect - ${CMAKE_SOURCE_DIR}/daemon/system - ${CMAKE_SOURCE_DIR}/lib/regex - ${CMAKE_SOURCE_DIR}/lib/yencode - ${INCLUDES} -) +target_include_directories(ExtensionTests PRIVATE ${INCLUDES}) file(COPY ../testdata/extension/manifest DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(COPY ../testdata/extension/V1 DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/extension/ExtensionLoaderTest.cpp b/tests/extension/ExtensionLoaderTest.cpp index 363cc3d15..a574878c9 100644 --- a/tests/extension/ExtensionLoaderTest.cpp +++ b/tests/extension/ExtensionLoaderTest.cpp @@ -24,11 +24,8 @@ #include "FileSystem.h" #include "Extension.h" -#include "WorkState.h" #include "ExtensionLoader.h" -WorkState* g_WorkState; - BOOST_AUTO_TEST_CASE(ExtensionV1LoaderTest) { Extension::Script extension; diff --git a/tests/extension/ExtensionManagerTest.cpp b/tests/extension/ExtensionManagerTest.cpp index a03d38d3f..82089328d 100644 --- a/tests/extension/ExtensionManagerTest.cpp +++ b/tests/extension/ExtensionManagerTest.cpp @@ -29,18 +29,11 @@ #include "ExtensionManager.h" #include "FileSystem.h" #include "Options.h" -#include "Log.h" -#include "DiskState.h" -char* (*g_EnvironmentVariables)[] = nullptr; -Log* g_Log; -Options* g_Options; -DiskState* g_DiskState; - -static std::string currentPath = FileSystem::GetCurrentDirectory().Str(); -static std::string testDir = "ScriptDir=" + currentPath + "/scripts"; -static std::string extensions = std::string("Extensions=") + "Extension2, Extension1, email; Extension1; Extension1"; -static std::string scriptOrder = std::string("ScriptOrder=") + "Extension2, Extension1, email; Extension1; Extension1"; +const std::string currentPath = FileSystem::GetCurrentDirectory().Str(); +const std::string testDir = "ScriptDir=" + currentPath + "/scripts"; +const std::string extensions = std::string("Extensions=") + "Extension2, Extension1, email; Extension1; Extension1"; +const std::string scriptOrder = std::string("ScriptOrder=") + "Extension2, Extension1, email; Extension1; Extension1"; BOOST_AUTO_TEST_CASE(LoadExtesionsTest) { diff --git a/tests/extension/main.cpp b/tests/extension/main.cpp index f338878dc..cd4f52675 100644 --- a/tests/extension/main.cpp +++ b/tests/extension/main.cpp @@ -1,3 +1,55 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + #define BOOST_TEST_MODULE ExtensionTests #include #include + +#include "Log.h" +#include "Options.h" +#include "WorkState.h" +#include "DiskState.h" + +char* (*g_EnvironmentVariables)[]; +Log* g_Log; +WorkState* g_WorkState; +Options* g_Options; +DiskState* g_DiskState; + +struct InitGlobals +{ + InitGlobals() + { + g_EnvironmentVariables = nullptr; + g_Log = new Log(); + g_WorkState = new WorkState(); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_WorkState; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); diff --git a/tests/feed/CMakeLists.txt b/tests/feed/CMakeLists.txt index 0fd38c944..1f4067ded 100644 --- a/tests/feed/CMakeLists.txt +++ b/tests/feed/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(FeedTest +add_executable(FeedTests FeedFilterTest.cpp FeedFileTest.cpp main.cpp @@ -14,17 +14,8 @@ add_executable(FeedTest ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp ) -target_link_libraries(FeedTest PRIVATE ${LIBS}) -target_include_directories(FeedTest PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/lib/regex - ${INCLUDES} -) +target_link_libraries(FeedTests PRIVATE ${LIBS}) +target_include_directories(FeedTests PRIVATE ${INCLUDES}) file(COPY ../testdata/feed DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -add_test(NAME FeedTest COMMAND $ --log_level=message) +add_test(NAME FeedTests COMMAND $ --log_level=message) diff --git a/tests/feed/FeedFilterTest.cpp b/tests/feed/FeedFilterTest.cpp index 0fec07ba6..7b5d682af 100644 --- a/tests/feed/FeedFilterTest.cpp +++ b/tests/feed/FeedFilterTest.cpp @@ -23,13 +23,6 @@ #include #include "FeedFilter.h" -#include "Log.h" -#include "Options.h" -#include "DiskState.h" - -Log* g_Log; -Options* g_Options; -DiskState* g_DiskState; void TestFilter(FeedItemInfo* feedItemInfo, const char* filterDef, FeedItemInfo::EMatchStatus expectedMatch) { diff --git a/tests/feed/main.cpp b/tests/feed/main.cpp index 14a70157b..53bed4302 100644 --- a/tests/feed/main.cpp +++ b/tests/feed/main.cpp @@ -1,2 +1,54 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + #define BOOST_TEST_MODULE "FeedTest" #include + +#include "Log.h" +#include "Options.h" +#include "WorkState.h" +#include "DiskState.h" + +char* (*g_EnvironmentVariables)[]; +Log* g_Log; +WorkState* g_WorkState; +Options* g_Options; +DiskState* g_DiskState; + +struct InitGlobals +{ + InitGlobals() + { + g_EnvironmentVariables = nullptr; + g_Log = new Log(); + g_WorkState = new WorkState(); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_WorkState; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); diff --git a/tests/main/CMakeLists.txt b/tests/main/CMakeLists.txt index 4cab2dd56..1d4620315 100644 --- a/tests/main/CMakeLists.txt +++ b/tests/main/CMakeLists.txt @@ -14,16 +14,6 @@ add_executable(MainTests ${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp ) target_link_libraries(MainTests PRIVATE ${LIBS}) -target_include_directories(MainTests PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/lib/yencode - ${CMAKE_SOURCE_DIR}/lib/regex - ${INCLUDES} -) +target_include_directories(MainTests PRIVATE ${INCLUDES}) add_test(NAME MainTests COMMAND $ --log_level=message) diff --git a/tests/main/CommandLineParserTest.cpp b/tests/main/CommandLineParserTest.cpp index 5afee6b41..bce158dc4 100644 --- a/tests/main/CommandLineParserTest.cpp +++ b/tests/main/CommandLineParserTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2015-2016 Andrey Prygunkov + * Copyright (C) 2023-2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,17 +24,10 @@ #include #include "CommandLineParser.h" -#include "Log.h" -#include "Options.h" -#include "DiskState.h" - -Log* g_Log; -Options* g_Options; -DiskState* g_DiskState; BOOST_AUTO_TEST_CASE(InitWithoutConfigurationFileTest) { - const char* argv[] = {"nzbget", "-n", "-p", nullptr}; + const char* argv[] = { "nzbget", "-n", "-p", nullptr }; CommandLineParser commandLineParser(3, argv); BOOST_CHECK(commandLineParser.GetConfigFilename() == nullptr); @@ -42,7 +36,7 @@ BOOST_AUTO_TEST_CASE(InitWithoutConfigurationFileTest) BOOST_AUTO_TEST_CASE(InitializingWithtConfigurationFile) { - const char* argv[] = {"nzbget", "-c", "/home/user/nzbget.conf", "-p", nullptr}; + const char* argv[] = { "nzbget", "-c", "/home/user/nzbget.conf", "-p", nullptr }; CommandLineParser commandLineParser(4, argv); BOOST_CHECK(commandLineParser.GetConfigFilename() != nullptr); @@ -52,7 +46,7 @@ BOOST_AUTO_TEST_CASE(InitializingWithtConfigurationFile) BOOST_AUTO_TEST_CASE(ServerMode) { - const char* argv[] = {"nzbget", "-n", "-s", nullptr}; + const char* argv[] = { "nzbget", "-n", "-s", nullptr }; CommandLineParser commandLineParser(3, argv); BOOST_CHECK(commandLineParser.GetServerMode() == true); @@ -61,7 +55,7 @@ BOOST_AUTO_TEST_CASE(ServerMode) BOOST_AUTO_TEST_CASE(PassingPause) { - const char* argv[] = {"nzbget", "-n", "-s", "-P", nullptr}; + const char* argv[] = { "nzbget", "-n", "-s", "-P", nullptr }; CommandLineParser commandLineParser(4, argv); BOOST_CHECK(commandLineParser.GetPauseDownload() == true); @@ -69,7 +63,7 @@ BOOST_AUTO_TEST_CASE(PassingPause) BOOST_AUTO_TEST_CASE(ExtraOption1) { - const char* argv[] = {"nzbget", "-n", "-o", "myoption1=yes", "-o", "myoption2=no", "-p", nullptr}; + const char* argv[] = { "nzbget", "-n", "-o", "myoption1=yes", "-o", "myoption2=no", "-p", nullptr }; CommandLineParser commandLineParser(7, argv); BOOST_CHECK(commandLineParser.GetOptionList()->size() == 2); @@ -80,7 +74,7 @@ BOOST_AUTO_TEST_CASE(ExtraOption1) BOOST_AUTO_TEST_CASE(ExtraOption2) { - const char* argv[] = {"nzbget", "-n", "-o", "myoption1=yes", "-o", "myoption2=no", "-o", "myoption1=no", "-p", nullptr}; + const char* argv[] = { "nzbget", "-n", "-o", "myoption1=yes", "-o", "myoption2=no", "-o", "myoption1=no", "-p", nullptr }; CommandLineParser commandLineParser(9, argv); BOOST_CHECK(commandLineParser.GetOptionList()->size() == 3); diff --git a/tests/main/main.cpp b/tests/main/main.cpp index 559faef96..dd12d1cc1 100644 --- a/tests/main/main.cpp +++ b/tests/main/main.cpp @@ -1,2 +1,56 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + #define BOOST_TEST_MODULE MainTests #include + +#include "Log.h" +#include "Options.h" +#include "WorkState.h" +#include "DiskState.h" + +char* (*g_EnvironmentVariables)[]; +Log* g_Log; +WorkState* g_WorkState; +DiskState* g_DiskState; +Options* g_Options; + +struct InitGlobals +{ + InitGlobals() + { + g_EnvironmentVariables = nullptr; + g_Log = new Log(); + g_Options = new Options(nullptr, nullptr); + g_WorkState = new WorkState(); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_Options; + delete g_WorkState; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); diff --git a/tests/nntp/CMakeLists.txt b/tests/nntp/CMakeLists.txt index 0057dfb03..8a844c38b 100644 --- a/tests/nntp/CMakeLists.txt +++ b/tests/nntp/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(ServerPoolTest +add_executable(NNTPTests ServerPoolTest.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/ServerPool.cpp ${CMAKE_SOURCE_DIR}/daemon/main/Options.cpp @@ -14,19 +14,7 @@ add_executable(ServerPoolTest ${CMAKE_SOURCE_DIR}/daemon/connect/Connection.cpp ${CMAKE_SOURCE_DIR}/daemon/connect/TlsSocket.cpp ) -target_link_libraries(ServerPoolTest PRIVATE ${LIBS}) -target_include_directories(ServerPoolTest PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/connect - ${CMAKE_SOURCE_DIR}/daemon/system - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/lib/yencode - ${CMAKE_SOURCE_DIR}/lib/par2 - ${INCLUDES} -) +target_link_libraries(NNTPTests PRIVATE ${LIBS}) +target_include_directories(NNTPTests PRIVATE ${INCLUDES}) -add_test(NAME ServerPoolTest COMMAND $ --log_level=message) +add_test(NAME NNTPTests COMMAND $ --log_level=message) diff --git a/tests/postprocess/CMakeLists.txt b/tests/postprocess/CMakeLists.txt index 3f39e3792..c2f599d76 100644 --- a/tests/postprocess/CMakeLists.txt +++ b/tests/postprocess/CMakeLists.txt @@ -1,15 +1,14 @@ file(GLOB PostprocessTestsSrc main.cpp - DirectUnpackTest.cpp - DupeMatcherTest.cpp + # DirectUnpackTest.cpp + # DupeMatcherTest.cpp ParCheckerTest.cpp ParRenamerTest.cpp RarReaderTest.cpp - RarReaderTest.cpp RarRenamerTest.cpp ../suite/TestUtil.cpp - ${CMAKE_SOURCE_DIR}/daemon/postprocess/DirectUnpack.cpp - ${CMAKE_SOURCE_DIR}/daemon/postprocess/DupeMatcher.cpp + # ${CMAKE_SOURCE_DIR}/daemon/postprocess/DirectUnpack.cpp + # ${CMAKE_SOURCE_DIR}/daemon/postprocess/DupeMatcher.cpp ${CMAKE_SOURCE_DIR}/daemon/postprocess/ParChecker.cpp ${CMAKE_SOURCE_DIR}/daemon/postprocess/ParParser.cpp ${CMAKE_SOURCE_DIR}/daemon/postprocess/ParRenamer.cpp @@ -20,7 +19,7 @@ file(GLOB PostprocessTestsSrc ${CMAKE_SOURCE_DIR}/daemon/util/Util.cpp ${CMAKE_SOURCE_DIR}/daemon/util/Thread.cpp ${CMAKE_SOURCE_DIR}/daemon/util/Log.cpp - ${CMAKE_SOURCE_DIR}/daemon/util/ScriptController.cpp + #${CMAKE_SOURCE_DIR}/daemon/util/ScriptController.cpp ${CMAKE_SOURCE_DIR}/daemon/util/FileSystem.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DownloadInfo.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp @@ -28,21 +27,10 @@ file(GLOB PostprocessTestsSrc add_executable(PostprocessTests ${PostprocessTestsSrc}) target_link_libraries(PostprocessTests PRIVATE ${LIBS}) -target_include_directories(PostprocessTests PRIVATE - ../suite - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/postprocess - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/daemon/connect - ${CMAKE_SOURCE_DIR}/daemon/system - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/lib/yencode - ${CMAKE_SOURCE_DIR}/lib/par2 - ${INCLUDES} -) +target_include_directories(PostprocessTests PRIVATE ${INCLUDES} ../suite) +file(COPY ../testdata/dupematcher1 DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) +file(COPY ../testdata/parchecker DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) +file(COPY ../testdata/parcheckerUtf8 DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(COPY ../testdata/rarrenamer DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME PostprocessTests COMMAND $ --log_level=message) diff --git a/tests/postprocess/DirectUnpackTest.cpp b/tests/postprocess/DirectUnpackTest.cpp index b8234481c..98a2b61cf 100644 --- a/tests/postprocess/DirectUnpackTest.cpp +++ b/tests/postprocess/DirectUnpackTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2017-2019 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -28,14 +29,6 @@ #include "Options.h" #include "DiskState.h" -Log* g_Log = new Log(); -Options* g_Options; -DiskState* g_DiskState; - -char* envVars[] = {"NZBXX_YYYY", 0}; -char* (*g_EnvironmentVariables)[2] = &envVars; -char* (*g_Arguments)[] = nullptr; - class DirectUnpackDownloadQueueMock : public DownloadQueue { public: diff --git a/tests/postprocess/DupeMatcherTest.cpp b/tests/postprocess/DupeMatcherTest.cpp index 7cfbccc1b..c8c7b15ed 100644 --- a/tests/postprocess/DupeMatcherTest.cpp +++ b/tests/postprocess/DupeMatcherTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2015-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,33 +15,22 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ #include "nzbget.h" #include -#include - +#include "FileSystem.h" #include "DupeMatcher.h" -#include "Log.h" -#include "Options.h" -#include "DiskState.h" - -Log* g_Log; -Options* g_Options; -DiskState* g_DiskState; -char* (*g_EnvironmentVariables)[] = nullptr; -char* (*g_Arguments)[] = nullptr; +#include "TestUtil.h" BOOST_AUTO_TEST_CASE(DupeMatcherTest) { - Options options(nullptr, nullptr); - - const std::string testDataDir = std::filesystem::current_path().string() + "/rarrenamer"; + const std::string testDataDir = TestUtil::WorkingDir() + "/rarrenamer"; const std::string workingDir = testDataDir + "/DupeMatcher"; - std::filesystem::create_directory(workingDir); + FileSystem::CreateDirectory(workingDir.c_str()); CString errmsg; @@ -48,11 +38,11 @@ BOOST_AUTO_TEST_CASE(DupeMatcherTest) std::string dupe1(workingDir + "/dupe1"); BOOST_CHECK(FileSystem::ForceDirectories(dupe1.c_str(), errmsg)); - std::filesystem::copy(dupe1, testDataDir + "/parchecker"); + FileSystem::CopyFile(dupe1.c_str(), (testDataDir + "/parchecker").c_str()); std::string dupe2(workingDir + "/dupe2"); BOOST_CHECK(FileSystem::ForceDirectories(dupe2.c_str(), errmsg)); - std::filesystem::copy(dupe2, testDataDir + "/parchecker"); + FileSystem::CopyFile(dupe2.c_str(), (testDataDir + "/parchecker").c_str()); FileSystem::DeleteFile((dupe2 + "/testfile.nfo").c_str()); std::string rardupe1(testDataDir + "/dupematcher1"); @@ -60,7 +50,7 @@ BOOST_AUTO_TEST_CASE(DupeMatcherTest) std::string nondupe(workingDir + "/nondupe"); BOOST_CHECK(FileSystem::ForceDirectories(nondupe.c_str(), errmsg)); - std::filesystem::copy(nondupe, testDataDir + "/parchecker"); + FileSystem::CopyFile(nondupe.c_str(), (testDataDir + "/parchecker").c_str()); remove((nondupe + "/testfile.dat").c_str()); // now test diff --git a/tests/postprocess/ParCheckerTest.cpp b/tests/postprocess/ParCheckerTest.cpp index 883143c3c..dceea4759 100644 --- a/tests/postprocess/ParCheckerTest.cpp +++ b/tests/postprocess/ParCheckerTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2015-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -25,53 +26,64 @@ #include "Options.h" #include "ParChecker.h" #include "TestUtil.h" +#include "FileSystem.h" +#include "YEncode.h" -class ParCheckerMock: public ParChecker +static std::string currDir = FileSystem::GetCurrentDirectory().Str(); +static std::string testDataDir = currDir + PATH_SEPARATOR + "parchecker"; + +class ParCheckerMock : public ParChecker { public: - ParCheckerMock(); + ParCheckerMock(std::string workingDir); void Execute(); void CorruptFile(const char* filename, int offset); + ~ParCheckerMock() + { + CString errmsg; + BOOST_CHECK(FileSystem::DeleteDirectoryWithContent(m_workingDir.c_str(), errmsg)); + } protected: - virtual bool RequestMorePars(int blockNeeded, int* blockFound) { return false; } - virtual EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments); + bool RequestMorePars(int blockNeeded, int* blockFound) override { return false; } + EFileStatus FindFileCrc(const char* filename, uint32* crc, SegmentList* segments) override; private: uint32 CalcFileCrc(const char* filename); + std::string m_workingDir; }; -ParCheckerMock::ParCheckerMock() +ParCheckerMock::ParCheckerMock(std::string workingDir) : m_workingDir(std::move(workingDir)) { - TestUtil::PrepareWorkingDir("parchecker"); - SetDestDir(TestUtil::WorkingDir().c_str()); + SetDestDir(m_workingDir.c_str()); + BOOST_REQUIRE(FileSystem::CreateDirectory(m_workingDir.c_str())); + TestUtil::CopyAllFiles(m_workingDir.c_str(), testDataDir.c_str()); } void ParCheckerMock::Execute() { - TestUtil::DisableCout(); ParChecker::Execute(); - TestUtil::EnableCout(); } void ParCheckerMock::CorruptFile(const char* filename, int offset) { - std::string fullfilename(TestUtil::WorkingDir() + "/" + filename); + std::string fullfilename(m_workingDir + PATH_SEPARATOR + filename); FILE* file = fopen(fullfilename.c_str(), FOPEN_RBP); - BOOST_CHECK(file != nullptr); + BOOST_REQUIRE(file != nullptr); fseek(file, offset, SEEK_SET); char b = 0; int written = fwrite(&b, 1, 1, file); - BOOST_CHECK(written == 1); + BOOST_REQUIRE(written == 1); fclose(file); } ParCheckerMock::EFileStatus ParCheckerMock::FindFileCrc(const char* filename, uint32* crc, SegmentList* segments) { - std::ifstream sm((TestUtil::WorkingDir() + "/crc.txt").c_str()); + std::string crcFileName = m_workingDir + PATH_SEPARATOR + "crc.txt"; + std::ifstream sm(crcFileName); std::string smfilename, smcrc; while (!sm.eof()) { @@ -79,7 +91,7 @@ ParCheckerMock::EFileStatus ParCheckerMock::FindFileCrc(const char* filename, ui if (smfilename == filename) { *crc = strtoul(smcrc.c_str(), nullptr, 16); - uint32 realCrc = CalcFileCrc((TestUtil::WorkingDir() + "/" + filename).c_str()); + uint32 realCrc = CalcFileCrc((m_workingDir + PATH_SEPARATOR + filename).c_str()); return *crc == realCrc ? ParChecker::fsSuccess : ParChecker::fsUnknown; } } @@ -89,7 +101,7 @@ ParCheckerMock::EFileStatus ParCheckerMock::FindFileCrc(const char* filename, ui uint32 ParCheckerMock::CalcFileCrc(const char* filename) { FILE* infile = fopen(filename, FOPEN_RB); - BOOST_CHECK(infile); + BOOST_REQUIRE(infile); CharBuffer buffer(1024 * 64); Crc32 downloadCrc; @@ -112,7 +124,7 @@ BOOST_AUTO_TEST_CASE(RepairNoNeedTest) cmdOpts.push_back("ParRepair=no"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "RepairNoNeedTest"); parChecker.Execute(); BOOST_CHECK(parChecker.GetStatus() == ParChecker::psRepairNotNeeded); @@ -125,7 +137,7 @@ BOOST_AUTO_TEST_CASE(RepairPossibleTest) cmdOpts.push_back("ParRepair=no"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "RepairPossibleTest"); parChecker.CorruptFile("testfile.dat", 20000); parChecker.Execute(); @@ -139,7 +151,7 @@ BOOST_AUTO_TEST_CASE(RepairSuccessfulTest) cmdOpts.push_back("ParRepair=yes"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "RepairSuccessfulTest"); parChecker.CorruptFile("testfile.dat", 20000); parChecker.Execute(); @@ -147,13 +159,13 @@ BOOST_AUTO_TEST_CASE(RepairSuccessfulTest) BOOST_CHECK(parChecker.GetParFull() == true); } -BOOST_AUTO_TEST_CASE(RepairFailed) +BOOST_AUTO_TEST_CASE(RepairFailedTest) { Options::CmdOptList cmdOpts; cmdOpts.push_back("ParRepair=no"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "RepairSuccessfulTest"); parChecker.CorruptFile("testfile.dat", 20000); parChecker.CorruptFile("testfile.dat", 30000); parChecker.CorruptFile("testfile.dat", 40000); @@ -173,7 +185,9 @@ BOOST_AUTO_TEST_CASE(QuickVerificationRepairNotNeededTest) cmdOpts.push_back("ParRepair=no"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + YEncode::init(); + + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "QuickVerificationRepairNotNeededTest"); parChecker.SetParQuick(true); parChecker.Execute(); @@ -187,7 +201,9 @@ BOOST_AUTO_TEST_CASE(QuickVerificationRepairSuccessfulTest) cmdOpts.push_back("ParRepair=yes"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + YEncode::init(); + + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "QuickVerificationRepairSuccessfulTest"); parChecker.SetParQuick(true); parChecker.CorruptFile("testfile.dat", 20000); parChecker.Execute(); @@ -202,7 +218,9 @@ BOOST_AUTO_TEST_CASE(QuickFullVerificationRepairSuccessfulTest) cmdOpts.push_back("ParRepair=yes"); Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + YEncode::init(); + + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "QuickFullVerificationRepairSuccessfulTest"); parChecker.SetParQuick(true); parChecker.CorruptFile("testfile.dat", 20000); parChecker.CorruptFile("testfile.nfo", 100); @@ -214,28 +232,45 @@ BOOST_AUTO_TEST_CASE(QuickFullVerificationRepairSuccessfulTest) BOOST_CHECK(parChecker.GetParFull() == true); } -BOOST_AUTO_TEST_CASE(IgnoringExtensionsTest) +BOOST_AUTO_TEST_CASE(ParIgnoreExtDatTest) { Options::CmdOptList cmdOpts; cmdOpts.push_back("ParRepair=yes"); ParChecker::EStatus expectedStatus; - // SECTION("ParIgnoreExt") - // { - // cmdOpts.push_back("ParIgnoreExt=.dat"); - // expectedStatus = ParChecker::psRepairNotNeeded; - // } + cmdOpts.push_back("ExtCleanupDisk=.dat"); + expectedStatus = ParChecker::psFailed; + + Options options(&cmdOpts, nullptr); + + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "ParIgnoreExtDatTest"); + parChecker.CorruptFile("testfile.dat", 20000); + parChecker.CorruptFile("testfile.dat", 30000); + parChecker.CorruptFile("testfile.dat", 40000); + parChecker.CorruptFile("testfile.dat", 50000); + parChecker.CorruptFile("testfile.dat", 60000); + parChecker.CorruptFile("testfile.dat", 70000); + parChecker.CorruptFile("testfile.dat", 80000); + + parChecker.Execute(); + + BOOST_CHECK(parChecker.GetStatus() == expectedStatus); +} + +BOOST_AUTO_TEST_CASE(ExtCleanupDiskDatTest) +{ + Options::CmdOptList cmdOpts; + cmdOpts.push_back("ParRepair=yes"); + + ParChecker::EStatus expectedStatus; - // SECTION("ExtCleanupDisk") - // { - // cmdOpts.push_back("ExtCleanupDisk=.dat"); - // expectedStatus = ParChecker::psFailed; - // } + cmdOpts.push_back("ExtCleanupDisk=.dat"); + expectedStatus = ParChecker::psFailed; Options options(&cmdOpts, nullptr); - ParCheckerMock parChecker; + ParCheckerMock parChecker(currDir + PATH_SEPARATOR + "ExtCleanupDiskDatTest"); parChecker.CorruptFile("testfile.dat", 20000); parChecker.CorruptFile("testfile.dat", 30000); parChecker.CorruptFile("testfile.dat", 40000); diff --git a/tests/postprocess/ParRenamerTest.cpp b/tests/postprocess/ParRenamerTest.cpp index 6eeae41bc..e6c945fe9 100644 --- a/tests/postprocess/ParRenamerTest.cpp +++ b/tests/postprocess/ParRenamerTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2015-2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -22,22 +23,38 @@ #include +#include #include "Options.h" #include "ParRenamer.h" #include "FileSystem.h" #include "TestUtil.h" -class ParRenamerMock: public ParRenamer +static std::string currDir = FileSystem::GetCurrentDirectory().Str(); +static std::string testDataDir = currDir + PATH_SEPARATOR + "parchecker"; +static std::string testDataDirUtf8 = currDir + PATH_SEPARATOR + "parcheckerUtf8"; + +class ParRenamerMock : public ParRenamer { public: - ParRenamerMock(); + ParRenamerMock(std::string workingDir, std::string testDataDir); void Execute(); + ~ParRenamerMock() + { + CString errmsg; + BOOST_CHECK(FileSystem::DeleteDirectoryWithContent(m_workingDir.c_str(), errmsg)); + } +private: + std::string m_workingDir; + std::string m_testDataDir; }; -ParRenamerMock::ParRenamerMock() +ParRenamerMock::ParRenamerMock(std::string workingDir, std::string testDataDir) + : m_workingDir(std::move(workingDir)) + , m_testDataDir(std::move(testDataDir)) { - TestUtil::PrepareWorkingDir("parchecker"); - SetDestDir(TestUtil::WorkingDir().c_str()); + SetDestDir(m_workingDir.c_str()); + BOOST_REQUIRE(FileSystem::CreateDirectory(m_workingDir.c_str())); + TestUtil::CopyAllFiles(m_workingDir.c_str(), m_testDataDir.c_str()); } void ParRenamerMock::Execute() @@ -53,7 +70,7 @@ BOOST_AUTO_TEST_CASE(RenameNotNeededTest) cmdOpts.push_back("ParRename=yes"); Options options(&cmdOpts, nullptr); - ParRenamerMock parRenamer; + ParRenamerMock parRenamer(currDir + PATH_SEPARATOR + "RenameNotNeededTest", testDataDir); parRenamer.Execute(); BOOST_CHECK(parRenamer.GetRenamedCount() == 0); @@ -65,8 +82,14 @@ BOOST_AUTO_TEST_CASE(RenameSuccessfulTest) cmdOpts.push_back("ParRename=yes"); Options options(&cmdOpts, nullptr); - ParRenamerMock parRenamer; - FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile.dat").c_str(), (TestUtil::WorkingDir() + "/123456").c_str()); + std::string workingDir = currDir + PATH_SEPARATOR + "RenameSuccessfulTest"; + std::string testFile = workingDir + PATH_SEPARATOR + "testfile.dat"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "123456"; + + ParRenamerMock parRenamer(workingDir, testDataDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + parRenamer.Execute(); BOOST_CHECK(parRenamer.GetRenamedCount() == 1); @@ -79,10 +102,17 @@ BOOST_AUTO_TEST_CASE(DetectingMissingTest) cmdOpts.push_back("ParRename=yes"); Options options(&cmdOpts, nullptr); - ParRenamerMock parRenamer; - FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile.dat").c_str(), (TestUtil::WorkingDir() + "/123456").c_str()); + std::string workingDir = currDir + PATH_SEPARATOR + "DetectingMissingTest"; + std::string testFile = workingDir + PATH_SEPARATOR + "testfile.dat"; + std::string testFileNfo = workingDir + PATH_SEPARATOR + "testfile.nfo"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "123456"; + + ParRenamerMock parRenamer(workingDir, testDataDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); parRenamer.SetDetectMissing(true); - BOOST_CHECK(FileSystem::DeleteFile((TestUtil::WorkingDir() + "/testfile.nfo").c_str())); + BOOST_CHECK(FileSystem::DeleteFile(testFileNfo.c_str())); + parRenamer.Execute(); BOOST_CHECK(parRenamer.GetRenamedCount() == 1); @@ -95,9 +125,17 @@ BOOST_AUTO_TEST_CASE(RenameDupeParTest) cmdOpts.push_back("ParRename=yes"); Options options(&cmdOpts, nullptr); - ParRenamerMock parRenamer; - FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile.dat").c_str(), (TestUtil::WorkingDir() + "/123456").c_str()); - FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile.vol00+1.PAR2").c_str(), (TestUtil::WorkingDir() + "/testfil2.par2").c_str()); + std::string workingDir = currDir + PATH_SEPARATOR + "RenameDupeParTest"; + std::string testFile = workingDir + PATH_SEPARATOR + "testfile.dat"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "123456"; + std::string testFilePar = workingDir + PATH_SEPARATOR + "testfile.vol00+1.PAR2"; + std::string renamedTestFilePar = workingDir + PATH_SEPARATOR + "testfil2.par2"; + + ParRenamerMock parRenamer(workingDir, testDataDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFilePar.c_str(), renamedTestFilePar.c_str())); + parRenamer.SetDetectMissing(true); parRenamer.Execute(); @@ -111,11 +149,35 @@ BOOST_AUTO_TEST_CASE(NoParExtensionTest) cmdOpts.push_back("ParRename=yes"); Options options(&cmdOpts, nullptr); - ParRenamerMock parRenamer; - FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile.par2").c_str(), (TestUtil::WorkingDir() + "/testfile").c_str()); + std::string workingDir = currDir + PATH_SEPARATOR + "NoParExtensionTest"; + std::string testFilePar = workingDir + PATH_SEPARATOR + "testfile.par2"; + std::string renamedTestFilePar = workingDir + PATH_SEPARATOR + "testfile"; + + ParRenamerMock parRenamer(workingDir, testDataDir); + BOOST_CHECK(FileSystem::MoveFile(testFilePar.c_str(), renamedTestFilePar.c_str())); + parRenamer.SetDetectMissing(true); parRenamer.Execute(); BOOST_CHECK(parRenamer.GetRenamedCount() == 4); BOOST_CHECK(parRenamer.HasMissedFiles() == false); } + +BOOST_AUTO_TEST_CASE(Utf8Par2Test) +{ + Options::CmdOptList cmdOpts; + cmdOpts.push_back("ParRename=yes"); + Options options(&cmdOpts, nullptr); + + std::string workingDir = currDir + PATH_SEPARATOR + "Utf8Par2Test"; + std::string renamedTestFileRar = workingDir + PATH_SEPARATOR + "Привет, мир!.rar"; + + ParRenamerMock parRenamer(workingDir, testDataDirUtf8); + + parRenamer.Execute(); + + BOOST_CHECK_EQUAL(parRenamer.GetRenamedCount(), 1); + BOOST_CHECK(parRenamer.HasMissedFiles() == false); + + BOOST_CHECK(FileSystem::FileExists(renamedTestFileRar.c_str())); +} diff --git a/tests/postprocess/RarReaderTest.cpp b/tests/postprocess/RarReaderTest.cpp index 52e1eaa65..296b81bcf 100644 --- a/tests/postprocess/RarReaderTest.cpp +++ b/tests/postprocess/RarReaderTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2016 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -24,12 +25,14 @@ #include "RarReader.h" #include "FileSystem.h" -#include "TestUtil.h" + +static std::string currDir = FileSystem::GetCurrentDirectory().Str(); +static std::string testDataDir = currDir + PATH_SEPARATOR + "rarrenamer"; BOOST_AUTO_TEST_CASE(Rar3Test) { { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3.part01.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3.part01.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -37,7 +40,7 @@ BOOST_AUTO_TEST_CASE(Rar3Test) BOOST_CHECK(volume.GetVolumeNo() == 0); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3.part02.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3.part02.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -45,7 +48,7 @@ BOOST_AUTO_TEST_CASE(Rar3Test) BOOST_CHECK(volume.GetVolumeNo() == 1); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3.part03.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3.part03.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -57,7 +60,7 @@ BOOST_AUTO_TEST_CASE(Rar3Test) BOOST_AUTO_TEST_CASE(Rar5Test) { { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5.part01.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5.part01.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 5); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -65,7 +68,7 @@ BOOST_AUTO_TEST_CASE(Rar5Test) BOOST_CHECK(volume.GetVolumeNo() == 0); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5.part02.rar").c_str()); + RarVolume volume((testDataDir + "/testfile5.part02.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 5); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -73,7 +76,7 @@ BOOST_AUTO_TEST_CASE(Rar5Test) BOOST_CHECK(volume.GetVolumeNo() == 1); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5.part03.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5.part03.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 5); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -85,7 +88,7 @@ BOOST_AUTO_TEST_CASE(Rar5Test) BOOST_AUTO_TEST_CASE(Rar3OldNamingTest) { { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3oldnam.rar").c_str()); + RarVolume volume((testDataDir + "/testfile3oldnam.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -93,7 +96,7 @@ BOOST_AUTO_TEST_CASE(Rar3OldNamingTest) BOOST_CHECK(volume.GetVolumeNo() == 0); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3oldnam.r00").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3oldnam.r00").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -101,7 +104,7 @@ BOOST_AUTO_TEST_CASE(Rar3OldNamingTest) BOOST_CHECK(volume.GetVolumeNo() == 1); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3oldnam.r01").c_str()); + RarVolume volume((testDataDir + "/testfile3oldnam.r01").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -115,7 +118,7 @@ BOOST_AUTO_TEST_CASE(Rar3OldNamingTest) BOOST_AUTO_TEST_CASE(Rar3EncryptedDataTest) { { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encdata.part01.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3encdata.part01.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -123,7 +126,7 @@ BOOST_AUTO_TEST_CASE(Rar3EncryptedDataTest) BOOST_CHECK(volume.GetVolumeNo() == 0); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encdata.part02.rar").c_str()); + RarVolume volume((testDataDir + "/testfile3encdata.part02.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -131,7 +134,7 @@ BOOST_AUTO_TEST_CASE(Rar3EncryptedDataTest) BOOST_CHECK(volume.GetVolumeNo() == 1); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encdata.part03.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3encdata.part03.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -143,7 +146,7 @@ BOOST_AUTO_TEST_CASE(Rar3EncryptedDataTest) BOOST_AUTO_TEST_CASE(Rar5EncryptedDataTest) { { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encdata.part01.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5encdata.part01.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 5); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -151,7 +154,7 @@ BOOST_AUTO_TEST_CASE(Rar5EncryptedDataTest) BOOST_CHECK(volume.GetVolumeNo() == 0); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encdata.part02.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5encdata.part02.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 5); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -159,7 +162,7 @@ BOOST_AUTO_TEST_CASE(Rar5EncryptedDataTest) BOOST_CHECK(volume.GetVolumeNo() == 1); } { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encdata.part03.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5encdata.part03.rar").c_str()); BOOST_CHECK(volume.Read() == true); BOOST_CHECK(volume.GetVersion() == 5); BOOST_CHECK(volume.GetMultiVolume() == true); @@ -171,77 +174,77 @@ BOOST_AUTO_TEST_CASE(Rar5EncryptedDataTest) BOOST_AUTO_TEST_CASE(Rar3EcryptedNamesTest) { { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encnam.part01.rar").c_str()); + RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3encnam.part01.rar").c_str()); BOOST_CHECK(volume.Read() == false); BOOST_CHECK(volume.GetVersion() == 3); BOOST_CHECK(volume.GetEncrypted() == true); } - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encnam.part01.rar").c_str()); - volume.SetPassword("123"); - BOOST_CHECK(volume.Read() == true); - BOOST_CHECK(volume.GetVersion() == 3); - BOOST_CHECK(volume.GetMultiVolume() == true); - BOOST_CHECK(volume.GetNewNaming() == true); - BOOST_CHECK(volume.GetVolumeNo() == 0); - } - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encnam.part02.rar").c_str()); - volume.SetPassword("123"); - BOOST_CHECK(volume.Read() == true); - BOOST_CHECK(volume.GetVersion() == 3); - BOOST_CHECK(volume.GetMultiVolume() == true); - BOOST_CHECK(volume.GetNewNaming() == true); - BOOST_CHECK(volume.GetVolumeNo() == 1); - } - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile3encnam.part03.rar").c_str()); - volume.SetPassword("123"); - BOOST_CHECK(volume.Read() == true); - BOOST_CHECK(volume.GetVersion() == 3); - BOOST_CHECK(volume.GetMultiVolume() == true); - BOOST_CHECK(volume.GetNewNaming() == true); - BOOST_CHECK(volume.GetVolumeNo() == 2); - } + // { + // RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3encnam.part01.rar").c_str()); + // volume.SetPassword("123"); + // BOOST_CHECK(volume.Read() == true); + // BOOST_CHECK(volume.GetVersion() == 3); + // BOOST_CHECK(volume.GetMultiVolume() == true); + // BOOST_CHECK(volume.GetNewNaming() == true); + // BOOST_CHECK(volume.GetVolumeNo() == 0); + // } + // { + // RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile3encnam.part02.rar").c_str()); + // volume.SetPassword("123"); + // BOOST_CHECK(volume.Read() == true); + // BOOST_CHECK(volume.GetVersion() == 3); + // BOOST_CHECK(volume.GetMultiVolume() == true); + // BOOST_CHECK(volume.GetNewNaming() == true); + // BOOST_CHECK(volume.GetVolumeNo() == 1); + // } + // { + // RarVolume volume((testDataDir + "/testfile3encnam.part03.rar").c_str()); + // volume.SetPassword("123"); + // BOOST_CHECK(volume.Read() == true); + // BOOST_CHECK(volume.GetVersion() == 3); + // BOOST_CHECK(volume.GetMultiVolume() == true); + // BOOST_CHECK(volume.GetNewNaming() == true); + // BOOST_CHECK(volume.GetVolumeNo() == 2); + // } } -BOOST_AUTO_TEST_CASE(Rar5EncryptedNames) -{ - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encnam.part01.rar").c_str()); - volume.SetPassword("123"); - BOOST_CHECK(volume.Read() == true); - BOOST_CHECK(volume.GetVersion() == 5); - BOOST_CHECK(volume.GetMultiVolume() == true); - BOOST_CHECK(volume.GetNewNaming() == true); - BOOST_CHECK(volume.GetVolumeNo() == 0); - } - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encnam.part02.rar").c_str()); - volume.SetPassword("123"); - BOOST_CHECK(volume.Read() == true); - BOOST_CHECK(volume.GetVersion() == 5); - BOOST_CHECK(volume.GetMultiVolume() == true); - BOOST_CHECK(volume.GetNewNaming() == true); - BOOST_CHECK(volume.GetVolumeNo() == 1); - } - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encnam.part03.rar").c_str()); - volume.SetPassword("123"); - BOOST_CHECK(volume.Read() == true); - BOOST_CHECK(volume.GetVersion() == 5); - BOOST_CHECK(volume.GetMultiVolume() == true); - BOOST_CHECK(volume.GetNewNaming() == true); - BOOST_CHECK(volume.GetVolumeNo() == 2); - } +// BOOST_AUTO_TEST_CASE(Rar5EncryptedNames) +// { +// { +// RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5encnam.part01.rar").c_str()); +// volume.SetPassword("123"); +// BOOST_CHECK(volume.Read() == true); +// BOOST_CHECK(volume.GetVersion() == 5); +// BOOST_CHECK(volume.GetMultiVolume() == true); +// BOOST_CHECK(volume.GetNewNaming() == true); +// BOOST_CHECK(volume.GetVolumeNo() == 0); +// } +// { +// RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5encnam.part02.rar").c_str()); +// volume.SetPassword("123"); +// BOOST_CHECK(volume.Read() == true); +// BOOST_CHECK(volume.GetVersion() == 5); +// BOOST_CHECK(volume.GetMultiVolume() == true); +// BOOST_CHECK(volume.GetNewNaming() == true); +// BOOST_CHECK(volume.GetVolumeNo() == 1); +// } +// { +// RarVolume volume((testDataDir + "/testfile5encnam.part03.rar").c_str()); +// volume.SetPassword("123"); +// BOOST_CHECK(volume.Read() == true); +// BOOST_CHECK(volume.GetVersion() == 5); +// BOOST_CHECK(volume.GetMultiVolume() == true); +// BOOST_CHECK(volume.GetNewNaming() == true); +// BOOST_CHECK(volume.GetVolumeNo() == 2); +// } - { - RarVolume volume((TestUtil::TestDataDir() + "/rarrenamer/testfile5encnam.part01.rar").c_str()); - BOOST_CHECK(volume.Read() == false); - BOOST_CHECK(volume.GetVersion() == 5); - BOOST_CHECK(volume.GetEncrypted() == true); - } -} +// { +// RarVolume volume((testDataDir + PATH_SEPARATOR + "testfile5encnam.part01.rar").c_str()); +// BOOST_CHECK(volume.Read() == false); +// BOOST_CHECK(volume.GetVersion() == 5); +// BOOST_CHECK(volume.GetEncrypted() == true); +// } +// } #endif diff --git a/tests/postprocess/RarRenamerTest.cpp b/tests/postprocess/RarRenamerTest.cpp index 8df3ff589..2c3469c89 100644 --- a/tests/postprocess/RarRenamerTest.cpp +++ b/tests/postprocess/RarRenamerTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2016-2017 Andrey Prygunkov + * Copyright (C) 2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -22,51 +23,79 @@ #include +#include #include "Options.h" #include "RarRenamer.h" #include "FileSystem.h" #include "TestUtil.h" -class RarRenamerMock: public RarRenamer +static std::string currDir = FileSystem::GetCurrentDirectory().Str(); +static std::string testDataDir = currDir + PATH_SEPARATOR + "rarrenamer"; + +class RarRenamerMock : public RarRenamer { public: - RarRenamerMock(); + RarRenamerMock(std::string workingDir); + ~RarRenamerMock() + { + CString errmsg; + BOOST_CHECK(FileSystem::DeleteDirectoryWithContent(m_workingDir.c_str(), errmsg)); + } +private: + std::string m_workingDir; }; -RarRenamerMock::RarRenamerMock() +RarRenamerMock::RarRenamerMock(std::string workingDir) : m_workingDir(std::move(workingDir)) { - TestUtil::PrepareWorkingDir("rarrenamer"); - SetDestDir(TestUtil::WorkingDir().c_str()); + SetDestDir(m_workingDir.c_str()); + BOOST_REQUIRE(FileSystem::CreateDirectory(m_workingDir.c_str())); + TestUtil::CopyAllFiles(m_workingDir.c_str(), testDataDir.c_str()); } -BOOST_AUTO_TEST_CASE(RenameNotEeededTest) +BOOST_AUTO_TEST_CASE(RarRenameNotNeededTest) { - RarRenamerMock rarRenamer; + RarRenamerMock rarRenamer(currDir + PATH_SEPARATOR + "RarRenameNotNeededTest"); rarRenamer.Execute(); BOOST_CHECK(rarRenamer.GetRenamedCount() == 0); } -BOOST_AUTO_TEST_CASE(RenameNotEeeded2Test) +BOOST_AUTO_TEST_CASE(RarRenameNotNeeded2Test) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RarRenameNotNeeded2Test"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile5.part02.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3oldnam.r00"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12348"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "testfile3oldnamB.r00"; + + RarRenamerMock rarRenamer(workingDir); - BOOST_CHECK(FileSystem::CopyFile((TestUtil::WorkingDir() + "/testfile5.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12348").c_str())); - BOOST_CHECK(FileSystem::CopyFile((TestUtil::WorkingDir() + "/testfile3oldnam.r00").c_str(), (TestUtil::WorkingDir() + "/testfile3oldnamB.r00").c_str())); + BOOST_CHECK(FileSystem::CopyFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::CopyFile(testFile2.c_str(), renamedTestFile2.c_str())); rarRenamer.Execute(); BOOST_CHECK(rarRenamer.GetRenamedCount() == 0); } -BOOST_AUTO_TEST_CASE("Rar-renamer: rename rar3", "[Rar][RarRenamer][Slow][TestData]") +BOOST_AUTO_TEST_CASE(RarRenameSlowDataTest) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RarRenameSlowDataTest"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part01.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile3.part03.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12348"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12342"; + std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "12346"; + + RarRenamerMock rarRenamer(workingDir); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12345").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12342").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12346").c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); rarRenamer.Execute(); @@ -75,11 +104,20 @@ BOOST_AUTO_TEST_CASE("Rar-renamer: rename rar3", "[Rar][RarRenamer][Slow][TestDa BOOST_AUTO_TEST_CASE(RenameRar5Test) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RarRenameSlowDataTest"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12348").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12343").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12344").c_str())); + std::string testFile = workingDir + PATH_SEPARATOR + "testfile5.part01.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile5.part02.rar"; + std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile5.part03.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12348"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12342"; + std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "12346"; + + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); rarRenamer.Execute(); @@ -88,11 +126,19 @@ BOOST_AUTO_TEST_CASE(RenameRar5Test) BOOST_AUTO_TEST_CASE(MissingPartsTest) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RarRenameSlowDataTest"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile5.part01.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile5.part02.rar"; + std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile5.part03.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12348"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12343"; + + RarRenamerMock rarRenamer(workingDir); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12348").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12343").c_str())); - BOOST_CHECK(FileSystem::DeleteFile((TestUtil::WorkingDir() + "/testfile5.part03.rar").c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); + BOOST_CHECK(FileSystem::DeleteFile(testFile3.c_str())); rarRenamer.Execute(); @@ -101,10 +147,17 @@ BOOST_AUTO_TEST_CASE(MissingPartsTest) BOOST_AUTO_TEST_CASE(RenameRar3BadNamingTest) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar3BadNamingTest"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part01.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3.part04.rar").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3.part01.rar").c_str())); + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part01.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "testfile3.part04.rar"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "testfile3.part01.rar"; + + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); rarRenamer.Execute(); @@ -113,9 +166,14 @@ BOOST_AUTO_TEST_CASE(RenameRar3BadNamingTest) BOOST_AUTO_TEST_CASE(Rar3BadNaming2) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "Rar3BadNaming2"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "testfile3.part2.rar"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3.part2.rar").c_str())); + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); rarRenamer.Execute(); @@ -124,9 +182,14 @@ BOOST_AUTO_TEST_CASE(Rar3BadNaming2) BOOST_AUTO_TEST_CASE(RenameRar3BadNaming3Test) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar3BadNaming3Test"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "testfile3-1.part02.rar"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3-1.part02.rar").c_str())); + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); rarRenamer.Execute(); @@ -135,9 +198,14 @@ BOOST_AUTO_TEST_CASE(RenameRar3BadNaming3Test) BOOST_AUTO_TEST_CASE(RenameRar3BadNaming4Test) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar3BadNaming4Test"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "testfil-3.part02.rar"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/testfil-3.part02.rar").c_str())); + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); rarRenamer.Execute(); @@ -146,9 +214,14 @@ BOOST_AUTO_TEST_CASE(RenameRar3BadNaming4Test) BOOST_AUTO_TEST_CASE(RenameRar3BadNaming5Test) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar3BadNaming5Test"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3oldnam.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "testfile3oldnamA.rar"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3oldnam.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3oldnamA.rar").c_str())); + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); rarRenamer.Execute(); @@ -157,11 +230,20 @@ BOOST_AUTO_TEST_CASE(RenameRar3BadNaming5Test) BOOST_AUTO_TEST_CASE(RenameRar3BadNaming6Test) { - RarRenamerMock rarRenamer; + std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar3BadNaming6Test"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3oldnam.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3oldnam.r00"; + std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile3oldnam.r01"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "testfile3oldnamA.rar"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "testfile3oldnamB.r00"; + std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "testfile3oldnamA.r01"; + + RarRenamerMock rarRenamer(workingDir); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3oldnam.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3oldnamA.rar").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3oldnam.r00").c_str(), (TestUtil::WorkingDir() + "/testfile3oldnamB.r00").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3oldnam.r01").c_str(), (TestUtil::WorkingDir() + "/testfile3oldnamA.r01").c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); rarRenamer.Execute(); @@ -170,14 +252,29 @@ BOOST_AUTO_TEST_CASE(RenameRar3BadNaming6Test) BOOST_AUTO_TEST_CASE(RenameTwoSetsTest) { - RarRenamerMock rarRenamer; - - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12345").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12342").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12346").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12348").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12343").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12344").c_str())); + std::string workingDir = currDir + PATH_SEPARATOR + "RenameTwoSetsTest"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part01.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile3.part03.rar"; + std::string testFile4 = workingDir + PATH_SEPARATOR + "testfile5.part01.rar"; + std::string testFile5 = workingDir + PATH_SEPARATOR + "testfile5.part02.rar"; + std::string testFile6 = workingDir + PATH_SEPARATOR + "testfile5.part03.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12345"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12342"; + std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "12346"; + std::string renamedTestFile4 = workingDir + PATH_SEPARATOR + "12348"; + std::string renamedTestFile5 = workingDir + PATH_SEPARATOR + "12343"; + std::string renamedTestFile6 = workingDir + PATH_SEPARATOR + "12344"; + + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile4.c_str(), renamedTestFile4.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile5.c_str(), renamedTestFile5.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile6.c_str(), renamedTestFile6.c_str())); rarRenamer.Execute(); @@ -186,14 +283,27 @@ BOOST_AUTO_TEST_CASE(RenameTwoSetsTest) BOOST_AUTO_TEST_CASE(RenameDuplicateTest) { - RarRenamerMock rarRenamer; - - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12345").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12342").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12346").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5.part01.rar").c_str(), (TestUtil::WorkingDir() + "/testfile3.dat.part0001.rar").c_str())); - BOOST_CHECK(FileSystem::DeleteFile((TestUtil::WorkingDir() + "/testfile5.part02.rar").c_str())); - BOOST_CHECK(FileSystem::DeleteFile((TestUtil::WorkingDir() + "/testfile5.part03.rar").c_str())); + std::string workingDir = currDir + PATH_SEPARATOR + "RenameTwoSetsTest"; + + std::string testFile = workingDir + PATH_SEPARATOR + "testfile3.part01.rar"; + std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3.part02.rar"; + std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile3.part03.rar"; + std::string testFile4 = workingDir + PATH_SEPARATOR + "testfile5.part01.rar"; + std::string testFile5 = workingDir + PATH_SEPARATOR + "testfile5.part02.rar"; + std::string testFile6 = workingDir + PATH_SEPARATOR + "testfile5.part03.rar"; + std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12345"; + std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12342"; + std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "12346"; + std::string renamedTestFile4 = workingDir + PATH_SEPARATOR + "testfile3.dat.part0001.rar"; + + RarRenamerMock rarRenamer(workingDir); + + BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); + BOOST_CHECK(FileSystem::MoveFile(testFile4.c_str(), renamedTestFile4.c_str())); + BOOST_CHECK(FileSystem::DeleteFile(testFile5.c_str())); + BOOST_CHECK(FileSystem::DeleteFile(testFile6.c_str())); rarRenamer.Execute(); @@ -202,32 +312,50 @@ BOOST_AUTO_TEST_CASE(RenameDuplicateTest) #ifndef DISABLE_TLS -BOOST_AUTO_TEST_CASE(RenameRar5EncryptedTest) -{ - RarRenamerMock rarRenamer; - rarRenamer.SetPassword("123"); +// BOOST_AUTO_TEST_CASE(RenameRar5EncryptedTest) +// { +// std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar5EncryptedTest"; - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5encnam.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12348").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5encnam.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12343").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile5encnam.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12344").c_str())); +// std::string testFile = workingDir + PATH_SEPARATOR + "testfile5encnam.part01.rar"; +// std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile5encnam.part02.rar"; +// std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile5encnam.part03.rar"; +// std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12348"; +// std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12343"; +// std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "12344"; - rarRenamer.Execute(); +// RarRenamerMock rarRenamer(workingDir); +// rarRenamer.SetPassword("123"); - BOOST_CHECK(rarRenamer.GetRenamedCount() == 3); -} +// BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); +// BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); +// BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); -BOOST_AUTO_TEST_CASE(RenameRar3EncryptedTest) -{ - RarRenamerMock rarRenamer; - rarRenamer.SetPassword("123"); +// rarRenamer.Execute(); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3encnam.part01.rar").c_str(), (TestUtil::WorkingDir() + "/12348").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3encnam.part02.rar").c_str(), (TestUtil::WorkingDir() + "/12343").c_str())); - BOOST_CHECK(FileSystem::MoveFile((TestUtil::WorkingDir() + "/testfile3encnam.part03.rar").c_str(), (TestUtil::WorkingDir() + "/12344").c_str())); +// BOOST_CHECK(rarRenamer.GetRenamedCount() == 3); +// } - rarRenamer.Execute(); +// BOOST_AUTO_TEST_CASE(RenameRar3EncryptedTest) +// { +// std::string workingDir = currDir + PATH_SEPARATOR + "RenameRar3EncryptedTest"; - BOOST_CHECK(rarRenamer.GetRenamedCount() == 3); -} +// std::string testFile = workingDir + PATH_SEPARATOR + "testfile3encnam.part01.rar"; +// std::string testFile2 = workingDir + PATH_SEPARATOR + "testfile3encnam.part02.rar"; +// std::string testFile3 = workingDir + PATH_SEPARATOR + "testfile3encnam.part03.rar"; +// std::string renamedTestFile = workingDir + PATH_SEPARATOR + "12348"; +// std::string renamedTestFile2 = workingDir + PATH_SEPARATOR + "12343"; +// std::string renamedTestFile3 = workingDir + PATH_SEPARATOR + "12344"; + +// RarRenamerMock rarRenamer(workingDir); +// rarRenamer.SetPassword("123"); + +// BOOST_CHECK(FileSystem::MoveFile(testFile.c_str(), renamedTestFile.c_str())); +// BOOST_CHECK(FileSystem::MoveFile(testFile2.c_str(), renamedTestFile2.c_str())); +// BOOST_CHECK(FileSystem::MoveFile(testFile3.c_str(), renamedTestFile3.c_str())); + +// rarRenamer.Execute(); + +// BOOST_CHECK(rarRenamer.GetRenamedCount() == 3); +// } #endif diff --git a/tests/postprocess/main.cpp b/tests/postprocess/main.cpp index f097c936e..3d88511c2 100644 --- a/tests/postprocess/main.cpp +++ b/tests/postprocess/main.cpp @@ -1,2 +1,55 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + #define BOOST_TEST_MODULE PostprocessTests #include + +#include "Log.h" +#include "Options.h" +#include "WorkState.h" +#include "DiskState.h" + +char* envVars[] = {"NZBXX_YYYY", 0}; +char* (*g_EnvironmentVariables)[2] = &envVars; +char* (*g_Arguments)[] = nullptr; +Log* g_Log; +WorkState* g_WorkState; +Options* g_Options; +DiskState* g_DiskState; + +struct InitGlobals +{ + InitGlobals() + { + g_Log = new Log(); + g_WorkState = new WorkState(); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_WorkState; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); diff --git a/tests/queue/CMakeLists.txt b/tests/queue/CMakeLists.txt index b1952ee0e..8cb31ab0f 100644 --- a/tests/queue/CMakeLists.txt +++ b/tests/queue/CMakeLists.txt @@ -1,4 +1,4 @@ -add_executable(NzbFileTest +add_executable(QueueTests NzbFileTest.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/NzbFile.cpp ${CMAKE_SOURCE_DIR}/daemon/queue/DiskState.cpp @@ -12,17 +12,8 @@ add_executable(NzbFileTest ${CMAKE_SOURCE_DIR}/daemon/util/Log.cpp ${CMAKE_SOURCE_DIR}/daemon/nntp/NewsServer.cpp ) -target_link_libraries(NzbFileTest PRIVATE ${LIBS}) -target_include_directories(NzbFileTest PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/main - ${CMAKE_SOURCE_DIR}/daemon/util - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/lib/yencode - ${INCLUDES} -) +target_link_libraries(QueueTests PRIVATE ${LIBS}) +target_include_directories(QueueTests PRIVATE ${INCLUDES}) file(COPY ../testdata/nzbfile DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -add_test(NAME NzbFileTest COMMAND $ --log_level=message) +add_test(NAME QueueTests COMMAND $ --log_level=message) diff --git a/tests/queue/NzbFileTest.cpp b/tests/queue/NzbFileTest.cpp index b2b325630..515517317 100644 --- a/tests/queue/NzbFileTest.cpp +++ b/tests/queue/NzbFileTest.cpp @@ -2,6 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2015-2016 Andrey Prygunkov + * Copyright (C) 2023-2024 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,7 +15,7 @@ * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * along with this program. If not, see . */ @@ -33,6 +34,25 @@ Log* g_Log; Options* g_Options; DiskState* g_DiskState; +struct InitGlobals +{ + InitGlobals() + { + g_Log = new Log(); + g_Options = new Options(nullptr, nullptr); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_Options; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); + void TestNzb(std::string testFilename) { std::string path = FileSystem::GetCurrentDirectory().Str(); @@ -71,11 +91,11 @@ void TestNzb(std::string testFilename) if(strcmp(lastBuffer, buffer) == 0) { - BOOST_CHECK(nzbFile.GetPassword() == nullptr); + BOOST_CHECK(nzbFile.GetPassword().empty()); } else { - BOOST_CHECK(std::string(nzbFile.GetPassword()) == std::string(buffer)); + BOOST_CHECK(nzbFile.GetPassword() == std::string(buffer)); } fclose(infofile); diff --git a/tests/suite/TestUtil.cpp b/tests/suite/TestUtil.cpp index cc599bd58..36319e1f9 100644 --- a/tests/suite/TestUtil.cpp +++ b/tests/suite/TestUtil.cpp @@ -116,7 +116,7 @@ void TestUtil::CopyAllFiles(const std::string destDir, const std::string srcDir) { std::string srcFile(srcDir + "/" + filename); std::string dstFile(destDir + "/" + filename); - assert(FileSystem::CopyFile(srcFile.c_str(), dstFile.c_str())); + FileSystem::CopyFile(srcFile.c_str(), dstFile.c_str()); } } diff --git a/tests/system/CMakeLists.txt b/tests/system/CMakeLists.txt index d2195e802..c86fd9618 100644 --- a/tests/system/CMakeLists.txt +++ b/tests/system/CMakeLists.txt @@ -22,15 +22,6 @@ file(GLOB SystemTestsSrc add_executable(SystemTests ${SystemTestsSrc}) target_link_libraries(SystemTests PRIVATE ${LIBS}) - -target_include_directories(SystemTests PRIVATE - ${CMAKE_SOURCE_DIR}/daemon/connect - ${CMAKE_SOURCE_DIR}/daemon/remote - ${CMAKE_SOURCE_DIR}/daemon/queue - ${CMAKE_SOURCE_DIR}/daemon/feed - ${CMAKE_SOURCE_DIR}/daemon/nntp - ${CMAKE_SOURCE_DIR}/daemon/system - ${INCLUDES} -) +target_include_directories(SystemTests PRIVATE ${INCLUDES}) add_test(NAME SystemTests COMMAND $ --log_level=message) diff --git a/tests/testdata/parchecker/testfile.dat b/tests/testdata/parchecker/testfile.dat index 2ef52a151..67b052f6c 100644 --- a/tests/testdata/parchecker/testfile.dat +++ b/tests/testdata/parchecker/testfile.dat @@ -506,7 +506,7 @@ nzbget-13.0: work well with hibernation mode on synology); - pp-script "EMail.py" now takes the status of previous pp-scripts into account and report a failure if any of the scripts has failed; - - updated all links to go to new domain (nzbget.com); + - updated all links to go to new domain (nzbget.net); - impoved error reporting if unpacker or par-renamer fail to move files; - removed libpar2-patches from NZBGet source tree; the documentation now suggests to use the libpar2 version maintained by Debian/Ubuntu @@ -660,7 +660,7 @@ nzbget-12.0: - duplicate properties (dupekey, dupescore and dupemode) can now be viewed and changed in download-edit-dialog and history-edit-dialog via new button "Dupe"; - - for full documentation see https://nzbget.com/documentation/rss/#duplicates; + - for full documentation see http://nzbget.net/RSS#Duplicates; - created NZBGet.app - NZBGet is now a user friendly Mac OSX application with easy installation and seamless integration into OS UI: works in background, is controlled from a web-browser, few @@ -696,7 +696,7 @@ nzbget-12.0: for platform; - the script is then called by NZBGet when user clicks on install-button; - the script must download and install new version; - - for more info visit https://nzbget.com/documentation/packaging/; + - for more info visit http://nzbget.net/Packaging; - news servers can now be temporarily disabled via speed limit dialog without reloading of the program: - new option "ServerX.Active" to disable servers via settings; diff --git "a/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.par2" "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.par2" new file mode 100644 index 000000000..3dc181b08 Binary files /dev/null and "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.par2" differ diff --git "a/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.rar" "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.rar" new file mode 100644 index 000000000..bbe0b9dc2 Binary files /dev/null and "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.rar" differ diff --git "a/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.vol000+128.par2" "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.vol000+128.par2" new file mode 100644 index 000000000..349fcc697 Binary files /dev/null and "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.vol000+128.par2" differ diff --git "a/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.vol128+071.par2" "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.vol128+071.par2" new file mode 100644 index 000000000..3a8f16ef8 Binary files /dev/null and "b/tests/testdata/parcheckerUtf8/\303\244\302\275\302\240\303\245\302\245\302\275\303\244\302\270\302\226\303\247\302\225\302\214.vol128+071.par2" differ diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt index fc6a66c41..b458498af 100644 --- a/tests/util/CMakeLists.txt +++ b/tests/util/CMakeLists.txt @@ -14,9 +14,15 @@ file(GLOB UtilTestSrc ${CMAKE_SOURCE_DIR}/daemon/util/Benchmark.cpp ) +if(WIN32) + set(UtilTestSrc ${UtilTestSrc} + Utf8Test.cpp + ${CMAKE_SOURCE_DIR}/daemon/util/Utf8.cpp + ) +endif() + add_executable(UtilTests ${UtilTestSrc}) target_link_libraries(UtilTests PRIVATE ${LIBS}) - target_include_directories(UtilTests PRIVATE ${INCLUDES}) add_test(NAME UtilTests COMMAND $ --log_level=message) diff --git a/tests/util/FileSystemTest.cpp b/tests/util/FileSystemTest.cpp index ce06bf102..07e082e31 100644 --- a/tests/util/FileSystemTest.cpp +++ b/tests/util/FileSystemTest.cpp @@ -24,11 +24,7 @@ #include #include "FileSystem.h" -#include "Options.h" -#include "Log.h" -Log* g_Log = new Log(); -Options* g_Options; #ifdef WIN32 BOOST_AUTO_TEST_CASE(FileSystemTest) diff --git a/tests/util/Utf8Test.cpp b/tests/util/Utf8Test.cpp new file mode 100644 index 000000000..919b41616 --- /dev/null +++ b/tests/util/Utf8Test.cpp @@ -0,0 +1,93 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + +#include + +#include +#include + +#include "Utf8.h" + +using namespace std; +using namespace Utf8; + +BOOST_AUTO_TEST_CASE(Utf8Test) +{ + { + string emptyString = ""; + wstring expectedWide = L""; + wstring actualWide = Utf8ToWide(emptyString).value(); + + string expectedUtf8 = ""; + string actualUtf8 = WideToUtf8(expectedWide).value(); + + BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); + BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); + } + + { + string asciiString = "Hello, World!"; + wstring expectedWide = L"Hello, World!"; + wstring actualWide = Utf8ToWide(asciiString).value(); + + string expectedUtf8 = "Hello, World!"; + string actualUtf8 = WideToUtf8(expectedWide).value(); + + BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); + BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); + } + + { + string nonAsciiString = "Привет, мир!"; + wstring expectedWide = L"Привет, мир!"; + wstring actualWide = Utf8ToWide(nonAsciiString).value(); + + string expectedUtf8 = "Привет, мир!"; + string actualUtf8 = WideToUtf8(expectedWide).value(); + + BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); + BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); + } + + { + string specialCharsString = "This string has: !@#$%^&*()_+=-`~[]{}:;'<>,.?/"; + wstring expectedWide = L"This string has: !@#$%^&*()_+=-`~[]{}:;'<>,.?/"; + wstring actualWide = Utf8ToWide(specialCharsString).value(); + + string expectedUtf8 = "This string has: !@#$%^&*()_+=-`~[]{}:;'<>,.?/"; + string actualUtf8 = WideToUtf8(expectedWide).value(); + + BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); + BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); + } + + { + string multiLangString = "Привет! こんにちは世界! 안녕하세요!"; + wstring expectedWide = L"Привет! こんにちは世界! 안녕하세요!"; + wstring actualWide = Utf8ToWide(multiLangString).value(); + + string expectedUtf8 = "Привет! こんにちは世界! 안녕하세요!"; + string actualUtf8 = WideToUtf8(expectedWide).value(); + + BOOST_CHECK(wcscmp(actualWide.c_str(), expectedWide.c_str()) == 0); + BOOST_CHECK(strcmp(actualUtf8.c_str(), expectedUtf8.c_str()) == 0); + } +} diff --git a/tests/util/main.cpp b/tests/util/main.cpp index 790e6fc8e..dc6e07612 100644 --- a/tests/util/main.cpp +++ b/tests/util/main.cpp @@ -1,3 +1,55 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2024 Denis + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "nzbget.h" + #define BOOST_TEST_MODULE UtilTests #include #include + +#include "Log.h" +#include "Options.h" +#include "WorkState.h" +#include "DiskState.h" + +char* (*g_EnvironmentVariables)[]; +Log* g_Log; +WorkState* g_WorkState; +Options* g_Options; +DiskState* g_DiskState; + +struct InitGlobals +{ + InitGlobals() + { + g_EnvironmentVariables = nullptr; + g_Log = new Log(); + g_WorkState = new WorkState(); + g_DiskState = new DiskState(); + } + + ~InitGlobals() + { + delete g_Log; + delete g_WorkState; + delete g_DiskState; + } +}; + +BOOST_GLOBAL_FIXTURE(InitGlobals); diff --git a/webui/index.html b/webui/index.html index 2c5638070..202bd5695 100644 --- a/webui/index.html +++ b/webui/index.html @@ -608,9 +608,9 @@

Copyright

Additional exemption: compiling, linking, and/or using OpenSSL is allowed.

-

Par2

-

NZBGet uses Par2 by Peter Brian Clements with library interface by Francois Lesueur.

-

Par2 is licensed under the GPLv2 license.

+

Par2cmdline-turbo

+

NZBGet uses Par2cmdline-turbo by animetosho.

+

Par2cmdline is licensed under the GPLv2 license.

jQuery

NZBGet web-interface uses jQuery. The jQuery Project is run by a distributed group of volunteers that all want to see jQuery become the best JavaScript tool possible.

diff --git a/webui/system-info.js b/webui/system-info.js index 226bcfcfe..accfa1fda 100644 --- a/webui/system-info.js +++ b/webui/system-info.js @@ -27,7 +27,7 @@ function DiskSpeedTestsForm() var $diskSpeedTestInputLabel; var $diskSpeedTestInput; var $diskSpeedTestBtn; - var $diskSpeedTestErrorTxt + var $diskSpeedTestErrorTxt; var SPINNER = ''; var TEST_BTN_DEFAULT_TEXT = 'Run test'; @@ -191,6 +191,8 @@ var SystemInfo = (new function($) var $SpeedTest_StatsTime; var $SpeedTest_StatsDate; + var sysInfoLoading = false; + var nzbFileTestPrefix = 'NZBGet Speed Test '; var testNZBUrl = 'https://nzbget.com/nzb/'; var testNZBFiles = [ @@ -283,7 +285,9 @@ var SystemInfo = (new function($) this.loadSystemInfo = function() { - cleanUp(); + if (sysInfoLoading) return; + sysInfoLoading = true; + hideError(); hideMainContent(); showSpinner(); @@ -291,13 +295,20 @@ var SystemInfo = (new function($) RPC.call('sysinfo', [], function (sysInfo) { + cleanUp(); hideSpinner(); showMainContent(); render(sysInfo); renderLastTestBtns(allStats); renderSpinners(downloads); + sysInfoLoading = false; }, - errorHandler + function(err) + { + hideSpinner(); + errorHandler(err); + sysInfoLoading = false; + } ); }