trezor: support v2.5.2+, add more trezor tests, fix chaingen and tests

- passphrase logic: remove backward compatibility for 2.4.3, code cleanup.
- fix LibUSB cmake for static builds on OSX
- tests: all tests now work with passphrase logic enabled. Passphrase test added with different passphrase. no_passphrase test added, Trezor pin test added. Testing wallet opening with correct and incorrect passphrase. Trezor test chain revamp, cleanup. Smaller chain, chain file versioning added.
- tests: Trezor tests support TEST_MINING_ENABLED, TEST_MINING_TIMEOUT env vars to change mining-related tests behaviour.
- requires protobuf@21 on osx for now (c++14), building with unlinked protobuf: `CMAKE_PREFIX_PATH=$(find /opt/homebrew/Cellar/protobuf@21 -maxdepth 1 -type d -name "21.*" -print -quit) \
make debug-test-trezor -j8`

.github/workflows/build.yml

@@ -23,6 +23,7 @@ env:
   CCACHE_SETTINGS: |
         ccache --max-size=150M
         ccache --set-config=compression=true
+  USE_DEVICE_TREZOR_MANDATORY: ON
 
 jobs:
   build-macos:
@@ -39,7 +40,9 @@ jobs:
         key: ccache-${{ runner.os }}-build-${{ github.sha }}
         restore-keys: ccache-${{ runner.os }}-build-
     - name: install dependencies
-      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache
+      run: |
+        HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf@21 ccache
+        brew link protobuf@21
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}
@@ -65,7 +68,12 @@ jobs:
     - uses: msys2/setup-msys2@v2
       with:
         update: true
-        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
+        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
+    - shell: msys2 {0}
+      run: |
+           curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst
+           curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst
+           pacman --noconfirm -U mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst
     - name: build
       run: |
         ${{env.CCACHE_SETTINGS}}

.github/workflows/depends.yml

@@ -18,6 +18,7 @@ env:
   CCACHE_SETTINGS: |
         ccache --max-size=150M
         ccache --set-config=compression=true
+  USE_DEVICE_TREZOR_MANDATORY: ON
 
 jobs:
   build-cross:

README.md

@@ -590,6 +590,16 @@ Using `depends` might also be easier to compile Monero on Windows than using MSY
 
 The produced binaries still link libc dynamically. If the binary is compiled on a current distribution, it might not run on an older distribution with an older installation of libc. Passing `-DBACKCOMPAT=ON` to cmake will make sure that the binary will run on systems having at least libc version 2.17.
 
+### Trezor hardware wallet support
+
+If you have an issue with building Monero with Trezor support, you can disable it by setting `USE_DEVICE_TREZOR=OFF`, e.g., 
+
+```bash
+USE_DEVICE_TREZOR=OFF make release
+```
+
+For more information, please check out Trezor [src/device_trezor/README.md](src/device_trezor/README.md).
+
 ### Gitian builds
 
 See [contrib/gitian/README.md](contrib/gitian/README.md).

cmake/CheckTrezor.cmake

@@ -1,8 +1,27 @@
-OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON)
+# Function for setting default options default values via env vars
-OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON)
+function(_trezor_default_val val_name val_default)
-OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF)
+    if(NOT DEFINED ENV{${val_name}})
-OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" OFF)
+        set(ENV{${val_name}} ${val_default})
-OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF)
+    endif()
+endfunction()
+
+# Define default options via env vars
+_trezor_default_val(USE_DEVICE_TREZOR ON)
+_trezor_default_val(USE_DEVICE_TREZOR_MANDATORY OFF)
+_trezor_default_val(USE_DEVICE_TREZOR_PROTOBUF_TEST ON)
+_trezor_default_val(USE_DEVICE_TREZOR_LIBUSB ON)
+_trezor_default_val(USE_DEVICE_TREZOR_UDP_RELEASE OFF)
+_trezor_default_val(USE_DEVICE_TREZOR_DEBUG OFF)
+_trezor_default_val(TREZOR_DEBUG OFF)
+
+# Main options
+OPTION(USE_DEVICE_TREZOR "Trezor support compilation" $ENV{USE_DEVICE_TREZOR})
+OPTION(USE_DEVICE_TREZOR_MANDATORY "Trezor compilation is mandatory, fail build if Trezor support cannot be compiled" $ENV{USE_DEVICE_TREZOR_MANDATORY})
+OPTION(USE_DEVICE_TREZOR_PROTOBUF_TEST "Trezor Protobuf test" $ENV{USE_DEVICE_TREZOR_PROTOBUF_TEST})
+OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" $ENV{USE_DEVICE_TREZOR_LIBUSB})
+OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" $ENV{USE_DEVICE_TREZOR_UDP_RELEASE})
+OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" $ENV{USE_DEVICE_TREZOR_DEBUG})
+OPTION(TREZOR_DEBUG "Main Trezor debugging switch" $ENV{TREZOR_DEBUG})
 
 # Helper function to fix cmake < 3.6.0 FindProtobuf variables
 function(_trezor_protobuf_fix_vars)
@@ -30,33 +49,62 @@ function(_trezor_protobuf_fix_vars)
     endif()
 endfunction()
 
+macro(trezor_fatal_msg msg)
+    if ($ENV{USE_DEVICE_TREZOR_MANDATORY})
+        message(FATAL_ERROR
+                "${msg}\n"
+                "==========================================================================\n"
+                "[ERROR] To compile without Trezor support, set USE_DEVICE_TREZOR=OFF. "
+                "It is possible both via cmake variable and environment variable, e.g., "
+                "`USE_DEVICE_TREZOR=OFF make release`\n"
+                "For more information, please check src/device_trezor/README.md\n"
+        )
+    else()
+        message(WARNING
+                "${msg}\n"
+                "==========================================================================\n"
+                "[WARNING] Trezor support cannot be compiled! Skipping Trezor compilation. \n"
+                "For more information, please check src/device_trezor/README.md\n")
+        set(USE_DEVICE_TREZOR OFF)
+        return()  # finish this cmake file processing (as this is macro).
+    endif()
+endmacro()
+
 # Use Trezor master switch
 if (USE_DEVICE_TREZOR)
     # Protobuf is required to build protobuf messages for Trezor
     include(FindProtobuf OPTIONAL)
-    find_package(Protobuf)
+
+    FIND_PACKAGE(Protobuf CONFIG)
+    if (NOT Protobuf_FOUND)
+        FIND_PACKAGE(Protobuf)
+    endif()
+
     _trezor_protobuf_fix_vars()
 
-    # Protobuf handling the cache variables set in docker.
+    # Early fail for optional Trezor support
-    if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR)
+    if(${Protobuf_VERSION} GREATER 21)
-        message(STATUS "Could not find Protobuf")
+        trezor_fatal_msg("Trezor: Unsupported Protobuf version ${Protobuf_VERSION}. Please, use Protobuf v21.")
+    elseif(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR)
+        trezor_fatal_msg("Trezor: Could not find Protobuf")
     elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}")
-        message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}")
+        trezor_fatal_msg("Trezor: Protobuf library not found: ${Protobuf_LIBRARY}")
         unset(Protobuf_FOUND)
     elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
-        message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}")
+        trezor_fatal_msg("Trezor: Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}")
         unset(Protobuf_FOUND)
     elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}")
-        message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}")
+        trezor_fatal_msg("Trezor: Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}")
         unset(Protobuf_FOUND)
     else()
-        message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}")
+        message(STATUS "Trezor: Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}")
         set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR})
-        set(Protobuf_FOUND 1)  # override found if all rquired info was provided by variables
+        set(Protobuf_FOUND 1)  # override found if all required info was provided by variables
     endif()
 
     if(TREZOR_DEBUG)
         set(USE_DEVICE_TREZOR_DEBUG 1)
+        message(STATUS "Trezor: debug build enabled")
     endif()
 
     # Compile debugging support (for tests)
@@ -64,7 +112,7 @@ if (USE_DEVICE_TREZOR)
         add_definitions(-DWITH_TREZOR_DEBUGGING=1)
     endif()
 else()
-    message(STATUS "Trezor support disabled by USE_DEVICE_TREZOR")
+    message(STATUS "Trezor: support disabled by USE_DEVICE_TREZOR")
 endif()
 
 if(Protobuf_FOUND AND USE_DEVICE_TREZOR)
@@ -85,7 +133,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR)
     endif()
 
     if(NOT TREZOR_PYTHON)
-        message(STATUS "Trezor: Python not found")
+        trezor_fatal_msg("Trezor: Python not found")
     endif()
 endif()
 
@@ -93,7 +141,18 @@ endif()
 if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
     execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_CURRENT_LIST_DIR}" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
     if(RET)
-        message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}")
+        trezor_fatal_msg("Trezor: Protobuf test generation failed: ${OUT} ${ERR}")
+    endif()
+
+    if(ANDROID)
+        set(CMAKE_TRY_COMPILE_LINKER_FLAGS "${CMAKE_TRY_COMPILE_LINKER_FLAGS} -llog")
+        set(CMAKE_TRY_COMPILE_LINK_LIBRARIES "${CMAKE_TRY_COMPILE_LINK_LIBRARIES} log")
+    endif()
+
+    if(USE_DEVICE_TREZOR_PROTOBUF_TEST)
+        # For now, Protobuf v21 is the maximum supported version as v23 requires C++17. TODO: Remove once we move to C++17
+        if(${Protobuf_VERSION} GREATER 21)
+            trezor_fatal_msg("Trezor: Unsupported Protobuf version ${Protobuf_VERSION}. Please, use Protobuf v21.")
         endif()
         
         try_compile(Protobuf_COMPILE_TEST_PASSED
@@ -102,18 +161,22 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
             "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc"
             "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.cpp"
             CMAKE_FLAGS
+            CMAKE_EXE_LINKER_FLAGS ${CMAKE_TRY_COMPILE_LINKER_FLAGS}
             "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}"
-        "-DCMAKE_CXX_STANDARD=11"
+            "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}"
-        LINK_LIBRARIES ${Protobuf_LIBRARY}
+            LINK_LIBRARIES ${Protobuf_LIBRARY} ${CMAKE_TRY_COMPILE_LINK_LIBRARIES}
             OUTPUT_VARIABLE OUTPUT
         )
         if(NOT Protobuf_COMPILE_TEST_PASSED)
-        message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.")
+            trezor_fatal_msg("Trezor: Protobuf Compilation test failed: ${OUTPUT}.")
+        endif()
+    else ()
+        message(STATUS "Trezor: Protobuf Compilation test skipped, build may fail later")
     endif()
 endif()
 
 # Try to build protobuf messages
-if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED)
+if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
     set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}")
     set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}")
     set(TREZOR_PROTOBUF_PARAMS "")
@@ -123,11 +186,12 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T
     
     execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py ${TREZOR_PROTOBUF_PARAMS} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
     if(RET)
-        message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})."
+        trezor_fatal_msg("Trezor: protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})."
                 "OUT: ${OUT}, ERR: ${ERR}."
                 "Please read src/device_trezor/trezor/tools/README.md")
-    else()
+    endif()
-        message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"")
+
+    message(STATUS "Trezor: protobuf messages regenerated out: \"${OUT}.\"")
     set(DEVICE_TREZOR_READY 1)
     add_definitions(-DDEVICE_TREZOR_READY=1)
     add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0)
@@ -137,6 +201,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T
     endif()
 
     if(USE_DEVICE_TREZOR_UDP_RELEASE)
+        message(STATUS "Trezor: UDP transport enabled (emulator)")
         add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1)
     endif()
 
@@ -160,7 +225,9 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T
     set(TREZOR_LIBUSB_LIBRARIES "")
     if(LibUSB_COMPILE_TEST_PASSED)
         list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES} ${LIBUSB_DEP_LINKER})
-            message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
+        message(STATUS "Trezor: compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
+    elseif(USE_DEVICE_TREZOR_LIBUSB AND NOT ANDROID)
+        trezor_fatal_msg("Trezor: LibUSB not found or test failed, please install libusb-1.0.26")
     endif()
 
     if (BUILD_GUI_DEPS)
@@ -177,5 +244,4 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T
             string(APPEND TREZOR_DEP_LINKER " -lusb-1.0 ${LIBUSB_DEP_LINKER}")
         endif()
     endif()
-    endif()
 endif()

cmake/FindLibUSB.cmake

@@ -95,30 +95,16 @@ if ( LibUSB_FOUND )
 endif ( LibUSB_FOUND )
 
 if ( LibUSB_FOUND )
-    check_library_exists ( "${LibUSB_LIBRARIES}" usb_open "" LibUSB_FOUND )
-    check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_device_list "" LibUSB_VERSION_1.0 )
-    check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_port_numbers "" LibUSB_VERSION_1.0.16 )
-
-    if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux") OR ANDROID)
-        find_library(LIBUDEV_LIBRARY udev)
-        if(LIBUDEV_LIBRARY)
-            set(LibUSB_LIBRARIES "${LibUSB_LIBRARIES};${LIBUDEV_LIBRARY}")
-        else()
-            message(WARNING "libudev library not found, binaries may fail to link.")
-        endif()
-    endif()
-
-    # Library 1.0.16+ compilation test.
-    # The check_library_exists does not work well on Apple with shared libs.
-    if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC)
     if (APPLE)
         if(DEPENDS)
             list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security")
         else()
             find_library(COREFOUNDATION CoreFoundation)
             find_library(IOKIT IOKit)
+            find_library(SECURITY_FRAMEWORK Security)
             list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${IOKIT})
             list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${COREFOUNDATION})
+            list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${SECURITY_FRAMEWORK})
 
             if(STATIC)
                 find_library(OBJC objc.a)
@@ -131,7 +117,24 @@ if ( LibUSB_FOUND )
         list(APPEND TEST_COMPILE_EXTRA_LIBRARIES setupapi)
     endif()
     list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LibUSB_LIBRARIES})
+    set(CMAKE_REQUIRED_LIBRARIES ${TEST_COMPILE_EXTRA_LIBRARIES})
 
+    check_library_exists ( "${LibUSB_LIBRARIES}" usb_open "" LibUSB_FOUND )
+    check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_device_list "" LibUSB_VERSION_1.0 )
+    check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_port_numbers "" LibUSB_VERSION_1.0.16 )
+
+    if((STATIC AND UNIX AND NOT APPLE AND NOT FREEBSD) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux") OR ANDROID)
+        find_library(LIBUDEV_LIBRARY udev)
+        if(LIBUDEV_LIBRARY)
+            set(LibUSB_LIBRARIES "${LibUSB_LIBRARIES};${LIBUDEV_LIBRARY}")
+        else()
+            message(WARNING "libudev library not found, binaries may fail to link.")
+        endif()
+    endif()
+
+    # Library 1.0.16+ compilation test.
+    # The check_library_exists does not work well on Apple with shared libs.
+    if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC)
         try_compile(LibUSB_COMPILE_TEST_PASSED
                 ${CMAKE_BINARY_DIR}
                 "${CMAKE_CURRENT_LIST_DIR}/test-libusb-version.c"

contrib/brew/Brewfile

@@ -31,4 +31,5 @@ brew "doxygen"
 brew "graphviz"
 brew "libunwind-headers"
 brew "xz"
-brew "protobuf"
+brew "protobuf@21", link: true
+brew "libusb"

contrib/depends/packages/libusb.mk

@@ -3,8 +3,10 @@ $(package)_version=1.0.26
 $(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version)
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5
+$(package)_patches=fix_osx_avail.patch
 
 define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/fix_osx_avail.patch &&\
   autoreconf -i
 endef
 
@@ -13,6 +15,7 @@ define $(package)_set_vars
   $(package)_config_opts_linux=--with-pic --disable-udev
   $(package)_config_opts_mingw32=--disable-udev
   $(package)_config_opts_darwin=--disable-udev
+  $(package)_config_opts_freebsd=--with-pic --disable-udev
 endef
 
 ifneq ($(host_os),darwin)

contrib/depends/packages/native_protobuf.mk

@@ -1,8 +1,9 @@
 package=protobuf3
-$(package)_version=3.6.1
+$(package)_version=21.12
+$(package)_version_protobuf_cpp=3.21.12
 $(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/
-$(package)_file_name=protobuf-cpp-$($(package)_version).tar.gz
+$(package)_file_name=protobuf-cpp-$($(package)_version_protobuf_cpp).tar.gz
-$(package)_sha256_hash=b3732e471a9bb7950f090fd0457ebd2536a9ba0891b7f3785919c654fe2a2529
+$(package)_sha256_hash=4eab9b524aa5913c6fffb20b2a8abf5ef7f95a80bc0701f3a6dbb4c607f73460
 $(package)_cxxflags=-std=c++11
 
 define $(package)_set_vars

contrib/depends/packages/packages.mk

@@ -8,15 +8,15 @@ endif
 hardware_packages := hidapi protobuf libusb
 hardware_native_packages := native_protobuf
 
-android_native_packages = android_ndk
+android_native_packages = android_ndk $(hardware_native_packages)
-android_packages = ncurses readline sodium
+android_packages = ncurses readline sodium protobuf
 
 darwin_native_packages = $(hardware_native_packages)
 darwin_packages = ncurses readline sodium $(hardware_packages)
 
 # not really native...
-freebsd_native_packages = freebsd_base
+freebsd_native_packages = freebsd_base $(hardware_native_packages)
-freebsd_packages = ncurses readline sodium
+freebsd_packages = ncurses readline sodium protobuf libusb
 
 linux_packages = eudev ncurses readline sodium $(hardware_packages)
 linux_native_packages = $(hardware_native_packages)

contrib/depends/packages/protobuf.mk

@@ -1,21 +1,17 @@
 package=protobuf
 $(package)_version=$(native_$(package)_version)
+$(package)_version_protobuf_cpp=$(native_$(package)_version_protobuf_cpp)
 $(package)_download_path=$(native_$(package)_download_path)
 $(package)_file_name=$(native_$(package)_file_name)
 $(package)_sha256_hash=$(native_$(package)_sha256_hash)
 $(package)_dependencies=native_$(package)
 $(package)_cxxflags=-std=c++11
-$(package)_patches=visibility.patch
 
 define $(package)_set_vars
   $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc
   $(package)_config_opts_linux=--with-pic
 endef
 
-define $(package)_preprocess_cmds
-  patch -p0 < $($(package)_patch_dir)/visibility.patch
-endef
-
 define $(package)_config_cmds
   $($(package)_autoconf) AR_FLAGS=$($(package)_arflags)
 endef

contrib/depends/patches/libusb/fix_osx_avail.patch

@@ -0,0 +1,12 @@
+--- a/libusb/os/darwin_usb.h	2023-03-19 12:07:53
++++ b/libusb/os/darwin_usb.h	2023-03-19 12:07:47
+@@ -165,7 +165,8 @@
+   #define __has_builtin(x) 0  // Compatibility with non-clang compilers.
+ #endif
+ #if __has_builtin(__builtin_available)
+-  #define HAS_CAPTURE_DEVICE() __builtin_available(macOS 10.10, *)
++//  #define HAS_CAPTURE_DEVICE() __builtin_available(macOS 10.10, *)
++  #define HAS_CAPTURE_DEVICE() 0
+ #else
+   #define HAS_CAPTURE_DEVICE() 0
+ #endif

contrib/depends/patches/protobuf/visibility.patch

@@ -1,159 +0,0 @@
---- src/google/protobuf/descriptor.cc.O	2018-07-30 22:16:10.000000000 +0000
-+++ src/google/protobuf/descriptor.cc	2022-05-06 13:38:14.827309092 +0000
-@@ -32,6 +32,9 @@
- //  Based on original Protocol Buffers design by
- //  Sanjay Ghemawat, Jeff Dean, and others.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
- #include <algorithm>
- #include <functional>
- #include <google/protobuf/stubs/hash.h>
-@@ -7274,3 +7277,6 @@
- 
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/extension_set.cc.O	2018-07-23 20:56:42.000000000 +0000
-+++ src/google/protobuf/extension_set.cc	2022-05-06 14:48:55.369877050 +0000
-@@ -32,6 +32,9 @@
- //  Based on original Protocol Buffers design by
- //  Sanjay Ghemawat, Jeff Dean, and others.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
- #include <google/protobuf/stubs/hash.h>
- #include <tuple>
- #include <utility>
-@@ -1914,3 +1917,6 @@
- }  // namespace internal
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/extension_set_heavy.cc.O	2018-07-30 22:16:10.000000000 +0000
-+++ src/google/protobuf/extension_set_heavy.cc	2022-05-06 14:14:27.847320946 +0000
-@@ -35,6 +35,10 @@
- // Contains methods defined in extension_set.h which cannot be part of the
- // lite library because they use descriptors or reflection.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
-+
- #include <google/protobuf/stubs/casts.h>
- #include <google/protobuf/descriptor.pb.h>
- #include <google/protobuf/io/coded_stream.h>
-@@ -814,3 +818,6 @@
- }  // namespace internal
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/generated_message_reflection.cc.O	2018-07-23 20:56:42.000000000 +0000
-+++ src/google/protobuf/generated_message_reflection.cc	2022-05-06 13:38:49.655540772 +0000
-@@ -32,6 +32,9 @@
- //  Based on original Protocol Buffers design by
- //  Sanjay Ghemawat, Jeff Dean, and others.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
- #include <algorithm>
- #include <set>
- 
-@@ -2420,3 +2423,6 @@
- }  // namespace internal
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/map_field.cc.O	2018-07-23 20:56:42.000000000 +0000
-+++ src/google/protobuf/map_field.cc	2022-05-06 13:34:44.913905697 +0000
-@@ -28,6 +28,10 @@
- // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
-+
- #include <google/protobuf/map_field.h>
- #include <google/protobuf/map_field_inl.h>
- 
-@@ -462,3 +466,6 @@
- }  // namespace internal
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/text_format.cc.O	2018-07-30 22:16:11.000000000 +0000
-+++ src/google/protobuf/text_format.cc	2022-05-06 13:34:58.881999517 +0000
-@@ -32,6 +32,10 @@
- //  Based on original Protocol Buffers design by
- //  Sanjay Ghemawat, Jeff Dean, and others.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
-+
- #include <algorithm>
- #include <float.h>
- #include <math.h>
-@@ -2258,3 +2262,6 @@
- 
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/wire_format.cc.O	2018-07-23 20:56:42.000000000 +0000
-+++ src/google/protobuf/wire_format.cc	2022-05-06 13:06:23.294219228 +0000
-@@ -32,6 +32,10 @@
- //  Based on original Protocol Buffers design by
- //  Sanjay Ghemawat, Jeff Dean, and others.
- 
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
-+
- #include <stack>
- #include <string>
- #include <vector>
-@@ -1445,3 +1449,7 @@
- }  // namespace internal
- }  // namespace protobuf
- }  // namespace google
-+
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif
---- src/google/protobuf/stubs/status.cc.O	2018-07-23 20:56:42.000000000 +0000
-+++ src/google/protobuf/stubs/status.cc	2022-05-06 15:18:53.393208814 +0000
-@@ -27,6 +27,11 @@
- // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-+
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility push(hidden)
-+#endif
-+
- #include <google/protobuf/stubs/status.h>
- 
- #include <ostream>
-@@ -132,3 +137,6 @@
- }  // namespace util
- }  // namespace protobuf
- }  // namespace google
-+#if defined(__APPLE__) && defined(__arm64__)
-+#pragma GCC visibility pop
-+#endif

contrib/depends/toolchain.cmake.in

@@ -27,12 +27,6 @@ SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a)
 SET(UNBOUND_INCLUDE_DIR @prefix@/include)
 SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a)
 
-if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android")
-SET(LIBUNWIND_INCLUDE_DIR @prefix@/include)
-SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a)
-SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib)
-
-if(NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
 SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a)
 SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a)
 
@@ -41,8 +35,11 @@ SET(Protobuf_PROTOC_EXECUTABLE @prefix@/native/bin/protoc CACHE FILEPATH "Path t
 SET(Protobuf_INCLUDE_DIR @prefix@/include CACHE PATH "Protobuf include dir")
 SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir")
 SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library")
-endif()
 
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android")
+SET(LIBUNWIND_INCLUDE_DIR @prefix@/include)
+SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a)
+SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib)
 endif()
 
 SET(ZMQ_INCLUDE_PATH @prefix@/include)

external/trezor-common

@@ -1 +1 @@
-Subproject commit bff7fdfe436c727982cc553bdfb29a9021b423b0
+Subproject commit bc28c316d05bf1e9ebfe3d7df1ab25831d98d168

src/device_trezor/CMakeLists.txt

@@ -65,12 +65,16 @@ set(trezor_private_headers)
 
 # Protobuf and LibUSB processed by CheckTrezor
 if(DEVICE_TREZOR_READY)
-    message(STATUS "Trezor support enabled")
+    message(STATUS "Trezor: support enabled")
 
     if(USE_DEVICE_TREZOR_DEBUG)
         list(APPEND trezor_headers trezor/debug_link.hpp trezor/messages/messages-debug.pb.h)
         list(APPEND trezor_sources trezor/debug_link.cpp trezor/messages/messages-debug.pb.cc)
-        message(STATUS "Trezor debugging enabled")
+        message(STATUS "Trezor: debugging enabled")
+    endif()
+
+    if(ANDROID)
+        set(TREZOR_EXTRA_LIBRARIES "log")
     endif()
 
     monero_private_headers(device_trezor
@@ -93,10 +97,11 @@ if(DEVICE_TREZOR_READY)
             ${Protobuf_LIBRARY}
             ${TREZOR_LIBUSB_LIBRARIES}
             PRIVATE
-            ${EXTRA_LIBRARIES})
+            ${EXTRA_LIBRARIES}
+            ${TREZOR_EXTRA_LIBRARIES})
 
 else()
-    message(STATUS "Trezor support disabled")
+    message(STATUS "Trezor: support disabled")
     monero_private_headers(device_trezor)
     monero_add_library(device_trezor device_trezor.cpp)
     target_link_libraries(device_trezor PUBLIC cncrypto)

src/device_trezor/README.md

@@ -0,0 +1,74 @@
+# Trezor hardware wallet support
+
+This module adds [Trezor] hardware support to Monero.
+
+
+## Basic information
+
+Trezor integration is based on the following original proposal: https://github.com/ph4r05/monero-trezor-doc
+
+A custom high-level transaction signing protocol uses Trezor in a similar way a cold wallet is used. 
+Transaction is build incrementally on the device. 
+
+Trezor implements the signing protocol in [trezor-firmware] repository, in the [monero](https://github.com/trezor/trezor-firmware/tree/master/core/src/apps/monero) application.
+Please, refer to [monero readme](https://github.com/trezor/trezor-firmware/blob/master/core/src/apps/monero/README.md) for more information.
+
+## Dependencies
+
+Trezor uses [Protobuf](https://protobuf.dev/) library. As Monero is compiled with C++14, the newest Protobuf library version cannot be compiled because it requires C++17 (through its dependency Abseil library).
+This can result in a compilation failure.
+
+Protobuf v21 is the latest compatible protobuf version.
+
+If you want to compile Monero with Trezor support, please make sure the Protobuf v21 is installed.
+
+More about this limitation: [PR #8752](https://github.com/monero-project/monero/pull/8752), 
+[1](https://github.com/monero-project/monero/pull/8752#discussion_r1246174755), [2](https://github.com/monero-project/monero/pull/8752#discussion_r1246480393)
+
+### OSX
+
+To build with installed, but not linked protobuf:
+
+```bash
+CMAKE_PREFIX_PATH=$(find /opt/homebrew/Cellar/protobuf@21 -maxdepth 1 -type d -name "21.*" -print -quit) \
+make release
+```
+
+or to install and link as a default protobuf version:
+```bash
+# Either install all requirements as
+brew update && brew bundle --file=contrib/brew/Brewfile
+
+# or install protobufv21 specifically
+brew install protobuf@21 && brew link protobuf@21
+```
+
+### MSYS32
+
+```bash
+curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst
+curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst
+pacman --noconfirm -U mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst
+```
+
+### Other systems
+
+- install protobufv21
+- point `CMAKE_PREFIX_PATH` environment variable to Protobuf v21 installation.
+
+## Troubleshooting
+
+To disable Trezor support, set `USE_DEVICE_TREZOR=OFF`, e.g.:
+
+```shell
+USE_DEVICE_TREZOR=OFF make release
+```
+
+## Resources:
+
+- First pull request https://github.com/monero-project/monero/pull/4241
+- Integration proposal https://github.com/ph4r05/monero-trezor-doc
+- Integration readme in trezor-firmware https://github.com/trezor/trezor-firmware/blob/master/core/src/apps/monero/README.md
+
+[Trezor]: https://trezor.io/
+[trezor-firmware]: https://github.com/trezor/trezor-firmware/

src/device_trezor/device_trezor.cpp

@@ -165,7 +165,7 @@ namespace trezor {
         auto res = get_address();
 
         cryptonote::address_parse_info info{};
-        bool r = cryptonote::get_account_address_from_str(info, this->network_type, res->address());
+        bool r = cryptonote::get_account_address_from_str(info, this->m_network_type, res->address());
         CHECK_AND_ASSERT_MES(r, false, "Could not parse returned address. Address parse failed: " + res->address());
         CHECK_AND_ASSERT_MES(!info.is_subaddress, false, "Trezor returned a sub address");
 
@@ -693,14 +693,11 @@ namespace trezor {
     unsigned device_trezor::client_version()
     {
       auto trezor_version = get_version();
-      if (trezor_version < pack_version(2, 4, 3)){
+      if (trezor_version < pack_version(2, 5, 2)){
-        throw exc::TrezorException("Minimal Trezor firmware version is 2.4.3. Please update.");
+        throw exc::TrezorException("Minimal Trezor firmware version is 2.5.2. Please update.");
       }
 
-      unsigned client_version = 3;
+      unsigned client_version = 4;  // since 2.5.2
-      if (trezor_version >= pack_version(2, 5, 2)){
-        client_version = 4;
-      }
 
 #ifdef WITH_TREZOR_DEBUGGING
       // Override client version for tests

src/device_trezor/device_trezor.hpp

@@ -102,7 +102,7 @@ namespace trezor {
 
       bool  has_ki_cold_sync() const override { return true; }
       bool  has_tx_cold_sign() const override { return true; }
-      void  set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; }
+      void  set_network_type(cryptonote::network_type network_type) override { this->m_network_type = network_type; }
       void  set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; }
       bool  live_refresh_enabled() const { return m_live_refresh_enabled; }
 

src/device_trezor/device_trezor_base.cpp

@@ -300,9 +300,6 @@ namespace trezor {
         case messages::MessageType_PassphraseRequest:
           on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get()));
           return true;
-        case messages::MessageType_Deprecated_PassphraseStateRequest:
-          on_passphrase_state_request(input, dynamic_cast<const messages::common::Deprecated_PassphraseStateRequest*>(input.m_msg.get()));
-          return true;
         case messages::MessageType_PinMatrixRequest:
           on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get()));
           return true;
@@ -475,21 +472,9 @@ namespace trezor {
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
       MDEBUG("on_passhprase_request");
 
-      // Backward compatibility, migration clause.
-      if (msg->has__on_device() && msg->_on_device()){
-        messages::common::PassphraseAck m;
-        resp = call_raw(&m);
-        return;
-      }
-
       m_seen_passphrase_entry_message = true;
-      bool on_device = true;
+      bool on_device = false;
-      if (msg->has__on_device() && !msg->_on_device()){
+      if (m_features){
-        on_device = false;  // do not enter on device, old devices.
-      }
-
-      if (on_device && m_features && m_features->capabilities_size() > 0){
-        on_device = false;
         for (auto it = m_features->capabilities().begin(); it != m_features->capabilities().end(); it++) {
           if (*it == messages::management::Features::Capability_PassphraseEntry){
             on_device = true;
@@ -526,18 +511,6 @@ namespace trezor {
       resp = call_raw(&m);
     }
 
-    void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg)
-    {
-      MDEBUG("on_passhprase_state_request");
-      CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
-
-      if (msg->has_state()) {
-        m_device_session_id = msg->state();
-      }
-      messages::common::Deprecated_PassphraseStateAck m;
-      resp = call_raw(&m);
-    }
-
 #ifdef WITH_TREZOR_DEBUGGING
     void device_trezor_base::wipe_device()
     {

src/device_trezor/device_trezor_base.hpp

@@ -100,7 +100,7 @@ namespace trezor {
       boost::optional<epee::wipeable_string> m_passphrase;
       messages::MessageType m_last_msg_type;
 
-      cryptonote::network_type network_type;
+      cryptonote::network_type m_network_type;
       bool m_reply_with_empty_passphrase;
       bool m_always_use_empty_passphrase;
       bool m_seen_passphrase_entry_message;
@@ -227,9 +227,9 @@ namespace trezor {
         }
 
         if (network_type){
-          msg->set_network_type(static_cast<uint32_t>(network_type.get()));
+          msg->set_network_type(static_cast<messages::monero::MoneroNetworkType>(network_type.get()));
         } else {
-          msg->set_network_type(static_cast<uint32_t>(this->network_type));
+          msg->set_network_type(static_cast<messages::monero::MoneroNetworkType>(this->m_network_type));
         }
       }
 
@@ -318,7 +318,6 @@ namespace trezor {
     void on_button_pressed();
     void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg);
     void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg);
-    void on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg);
 
 #ifdef WITH_TREZOR_DEBUGGING
     void set_debug(bool debug){

src/device_trezor/trezor/debug_link.cpp

@@ -67,7 +67,7 @@ namespace trezor{
 
   void DebugLink::input_button(bool button){
     messages::debug::DebugLinkDecision decision;
-    decision.set_yes_no(button);
+    decision.set_button(button ? messages::debug::DebugLinkDecision_DebugButton_YES : messages::debug::DebugLinkDecision_DebugButton_NO);
     call(decision, boost::none, true);
   }
 

src/device_trezor/trezor/transport.cpp

@@ -154,6 +154,7 @@ namespace trezor{
   // Helpers
   //
 
+#define PROTO_MAGIC_SIZE 3
 #define PROTO_HEADER_SIZE 6
 
   static size_t message_size(const google::protobuf::Message &req){
@@ -193,7 +194,7 @@ namespace trezor{
     }
 
     serialize_message_header(buff, msg_wire_num, msg_size);
-    if (!req.SerializeToArray(buff + 6, msg_size)){
+    if (!req.SerializeToArray(buff + PROTO_HEADER_SIZE, msg_size)){
       throw exc::EncodingException("Message serialization error");
     }
   }
@@ -252,16 +253,16 @@ namespace trezor{
       throw exc::CommunicationException("Read chunk has invalid size");
     }
 
-    if (memcmp(chunk_buff_raw, "?##", 3) != 0){
+    if (memcmp(chunk_buff_raw, "?##", PROTO_MAGIC_SIZE) != 0){
       throw exc::CommunicationException("Malformed chunk");
     }
 
     uint16_t tag;
     uint32_t len;
-    nread -= 3 + 6;
+    nread -= PROTO_MAGIC_SIZE + PROTO_HEADER_SIZE;
-    deserialize_message_header(chunk_buff_raw + 3, tag, len);
+    deserialize_message_header(chunk_buff_raw + PROTO_MAGIC_SIZE, tag, len);
 
-    epee::wipeable_string data_acc(chunk_buff_raw + 3 + 6, nread);
+    epee::wipeable_string data_acc(chunk_buff_raw + PROTO_MAGIC_SIZE + PROTO_HEADER_SIZE, nread);
     data_acc.reserve(len);
 
     while(nread < len){
@@ -482,7 +483,7 @@ namespace trezor{
     uint16_t msg_tag;
     uint32_t msg_len;
     deserialize_message_header(bin_data->data(), msg_tag, msg_len);
-    if (bin_data->size() != msg_len + 6){
+    if (bin_data->size() != msg_len + PROTO_HEADER_SIZE){
       throw exc::CommunicationException("Response is not well hexcoded");
     }
 
@@ -491,7 +492,7 @@ namespace trezor{
     }
 
     std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag));
-    if (!msg_wrap->ParseFromArray(bin_data->data() + 6, msg_len)){
+    if (!msg_wrap->ParseFromArray(bin_data->data() + PROTO_HEADER_SIZE, msg_len)){
       throw exc::EncodingException("Response is not well hexcoded");
     }
     msg = msg_wrap;

tests/core_tests/chaingen.h

@@ -913,15 +913,6 @@ inline bool do_replay_file(const std::string& filename)
   }                                                                                   \
   VEC_EVENTS.push_back(BLK_NAME);
 
-#define MAKE_NEXT_BLOCK_TX1_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TX1, HF)         \
-  cryptonote::block BLK_NAME;                                                           \
-  {                                                                                   \
-    std::list<cryptonote::transaction> tx_list;                                         \
-    tx_list.push_back(TX1);                                                           \
-    generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, tx_list, HF);              \
-  }                                                                                   \
-  VEC_EVENTS.push_back(BLK_NAME);
-
 #define MAKE_NEXT_BLOCK_TX_LIST(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST)  \
   cryptonote::block BLK_NAME;                                                           \
   generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST);                 \
@@ -946,18 +937,12 @@ inline bool do_replay_file(const std::string& filename)
 
 #define REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT) REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT, boost::none)
 #define REWIND_BLOCKS(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC) REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW)
-#define REWIND_BLOCKS_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, HF) REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, HF)
 
 #define MAKE_TX_MIX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, NMIX, HEAD)                       \
   cryptonote::transaction TX_NAME;                                                             \
   construct_tx_to_key(VEC_EVENTS, TX_NAME, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX); \
   VEC_EVENTS.push_back(TX_NAME);
 
-#define MAKE_TX_MIX_RCT(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, NMIX, HEAD)                       \
-  cryptonote::transaction TX_NAME;                                                             \
-  construct_tx_to_key(VEC_EVENTS, TX_NAME, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX, true, rct::RangeProofPaddedBulletproof); \
-  VEC_EVENTS.push_back(TX_NAME);
-
 #define MAKE_TX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, HEAD) MAKE_TX_MIX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, 0, HEAD)
 
 #define MAKE_TX_MIX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD)             \
@@ -968,36 +953,12 @@ inline bool do_replay_file(const std::string& filename)
     VEC_EVENTS.push_back(t);                                                             \
   }
 
-#define MAKE_TX_MIX_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \
-        MAKE_TX_MIX_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD, rct::RangeProofPaddedBulletproof, 1)
-#define MAKE_TX_MIX_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD, RCT_TYPE, BP_VER)  \
-  {                                                                                      \
-    cryptonote::transaction t;                                                           \
-    construct_tx_to_key(VEC_EVENTS, t, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX, true, RCT_TYPE, BP_VER); \
-    SET_NAME.push_back(t);                                                               \
-    VEC_EVENTS.push_back(t);                                                             \
-  }
-
-#define MAKE_TX_MIX_DEST_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD)            \
-        MAKE_TX_MIX_DEST_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD, rct::RangeProofPaddedBulletproof, 1)
-#define MAKE_TX_MIX_DEST_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD, RCT_TYPE, BP_VER)  \
-  {                                                                                      \
-    cryptonote::transaction t;                                                           \
-    construct_tx_to_key(VEC_EVENTS, t, HEAD, FROM, TO, TESTS_DEFAULT_FEE, NMIX, true, RCT_TYPE, BP_VER); \
-    SET_NAME.push_back(t);                                                               \
-    VEC_EVENTS.push_back(t);                                                             \
-  }
-
 #define MAKE_TX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD) MAKE_TX_MIX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, 0, HEAD)
 
 #define MAKE_TX_LIST_START(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD) \
     std::list<cryptonote::transaction> SET_NAME; \
     MAKE_TX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD);
 
-#define MAKE_TX_LIST_START_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \
-    std::list<cryptonote::transaction> SET_NAME; \
-    MAKE_TX_MIX_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD);
-
 #define MAKE_MINER_TX_AND_KEY_AT_HF_MANUALLY(TX, BLK, HF_VERSION, KEY)                                    \
   transaction TX;                                                                                         \
   if (!construct_miner_tx_manually(get_block_height(BLK) + 1, generator.get_already_generated_coins(BLK), \

tests/trezor/daemon.cpp

@@ -309,11 +309,10 @@ void mock_daemon::stop_p2p()
     m_server.send_stop_signal();
 }
 
-void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address)
+void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address, std::chrono::seconds timeout)
 {
   bool blocks_mined = false;
   const uint64_t start_height = get_height();
-  const auto mining_timeout = std::chrono::seconds(360);
   MDEBUG("Current height before mining: " << start_height);
 
   start_mining(miner_address);
@@ -331,14 +330,14 @@ void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_addres
     }
 
     auto current_time = std::chrono::system_clock::now();
-    if (mining_timeout < current_time - mining_started)
+    if (timeout < current_time - mining_started)
     {
       break;
     }
   }
 
   stop_mining();
-  CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit");
+  CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit: " << timeout.count());
 }
 
 constexpr const std::chrono::seconds mock_daemon::rpc_timeout;

tests/trezor/daemon.h

@@ -139,7 +139,7 @@ public:
   void stop_and_deinit();
   void try_init_and_run(boost::optional<unsigned> initial_port=boost::none);
 
-  void mine_blocks(size_t num_blocks, const std::string &miner_address);
+  void mine_blocks(size_t num_blocks, const std::string &miner_address, std::chrono::seconds timeout = std::chrono::seconds(360));
   void start_mining(const std::string &miner_address, uint64_t threads_count=1, bool do_background_mining=false, bool ignore_battery=true);
   void stop_mining();
   uint64_t get_height();

tests/trezor/trezor_tests.h

@@ -36,16 +36,24 @@
 #include "../core_tests/chaingen.h"
 #include "../core_tests/wallet_tools.h"
 
-#define TREZOR_TEST_FEE 90000000000
 #define TREZOR_TEST_CLSAG_MIXIN 11
 #define TREZOR_TEST_HF15_MIXIN 16
 #define TREZOR_TEST_MIXIN TREZOR_TEST_CLSAG_MIXIN
+#define TREZOR_TEST_MIN_HF_DEFAULT HF_VERSION_BULLETPROOF_PLUS
+#define TREZOR_TEST_MAX_HF_DEFAULT HF_VERSION_BULLETPROOF_PLUS
+#define TREZOR_TEST_KI_SYNC_DEFAULT 1
+#define TREZOR_TEST_MINING_ENABLED_DEFAULT false
+#define TREZOR_TEST_MINING_TIMEOUT_DEFAULT 360
+
+#define MK_TCOINS(amount) ((unsigned long long) ((amount) * (COIN)))
+#define TREZOR_TEST_FEE_FRAC 0.1
+#define TREZOR_TEST_FEE MK_TCOINS(TREZOR_TEST_FEE_FRAC)
 
 /************************************************************************/
 /*                                                                      */
 /************************************************************************/
 class tsx_builder;
-class gen_trezor_base : public test_chain_unit_base
+class gen_trezor_base : public test_chain_unit_base, public hw::i_device_callback
 {
 public:
   friend class tsx_builder;
@@ -54,7 +62,7 @@ public:
   gen_trezor_base(const gen_trezor_base &other);
   virtual ~gen_trezor_base() {};
 
-  virtual void setup_args(const std::string & trezor_path, bool heavy_tests=false);
+  virtual void setup_args(const std::string & trezor_path, bool heavy_tests, bool mining_enabled, long mining_timeout);
   virtual bool generate(std::vector<test_event_entry>& events);
   virtual void load(std::vector<test_event_entry>& events);       // load events, init test obj
   virtual void fix_hf(std::vector<test_event_entry>& events);
@@ -76,42 +84,58 @@ public:
       std::vector<cryptonote::address_parse_info>& dsts_info,
       test_generator &generator,
       std::vector<tools::wallet2*> wallets,
-      bool is_sweep=false);
+      bool is_sweep=false,
+      size_t sender_idx=0);
 
   virtual void test_get_tx(
       std::vector<test_event_entry>& events,
       std::vector<tools::wallet2*> wallets,
+      const cryptonote::account_base * account,
       const std::vector<tools::wallet2::pending_tx> &ptxs,
       const std::vector<std::string> &aux_tx_info);
 
   virtual void mine_and_test(std::vector<test_event_entry>& events);
 
   virtual void rewind_blocks(std::vector<test_event_entry>& events, size_t rewind_n, uint8_t hf);
+  virtual cryptonote::block rewind_blocks_with_decoys(std::vector<test_event_entry>& events, cryptonote::block & head, size_t rewind_n, uint8_t hf, size_t num_decoys_per_block);
 
   virtual void set_hard_fork(uint8_t hf);
 
   crypto::hash head_hash() const { return get_block_hash(m_head); }
   cryptonote::block head_block() const { return m_head; }
   bool heavy_tests() const { return m_heavy_tests; }
+  bool heavy_test_set() const { return m_gen_heavy_test_set; }
+  void heavy_test_set(bool set) { m_gen_heavy_test_set = set; }
   void rct_config(rct::RCTConfig rct_config) { m_rct_config = rct_config; }
-  uint8_t cur_hf() const { return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; }
+  uint8_t cur_hf() const { return !m_hard_forks.empty() ? m_hard_forks.back().first : 0; }
   size_t num_mixin() const { return  m_top_hard_fork >= HF_VERSION_BULLETPROOF_PLUS ? TREZOR_TEST_HF15_MIXIN : TREZOR_TEST_CLSAG_MIXIN; }
   cryptonote::network_type nettype() const { return m_network_type; }
   std::shared_ptr<mock_daemon> daemon() const { return m_daemon; }
   void daemon(std::shared_ptr<mock_daemon> daemon){ m_daemon = std::move(daemon); }
 
+  boost::optional<epee::wipeable_string> on_pin_request() override;
+  boost::optional<epee::wipeable_string> on_passphrase_request(bool &on_device) override;
+
   // Static configuration
   static const uint64_t m_ts_start;
   static const uint64_t m_wallet_ts;
   static const std::string  m_device_name;
+  static const std::string  m_miner_master_seed_str;
   static const std::string  m_master_seed_str;
   static const std::string  m_device_seed;
   static const std::string  m_alice_spend_private;
   static const std::string  m_alice_view_private;
+  static const std::string  m_alice2_passphrase;
+  static const std::string  m_alice2_master_seed;
+  static const std::string  m_bob_master_seed;
+  static const std::string  m_eve_master_seed;
 
 protected:
-  virtual void setup_trezor();
+  virtual void setup_trezor(bool use_passphrase, const std::string & pin);
-  virtual void init_fields();
+  virtual void init_accounts();
+  virtual void init_wallets();
+  virtual void init_trezor_account();
+  virtual void log_wallets_desc();
   virtual void update_client_settings();
   virtual bool verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs);
 
@@ -126,17 +150,27 @@ protected:
   std::vector<test_event_entry> m_events;
 
   std::string m_trezor_path;
+  std::string m_trezor_pin;
+  std::string m_trezor_passphrase;
+  bool m_trezor_use_passphrase = true;
+  bool m_trezor_use_alice2 = false;
   bool m_heavy_tests;
+  bool m_gen_heavy_test_set;
   bool m_test_get_tx_key;
   rct::RCTConfig m_rct_config;
   bool m_live_refresh_enabled;
+  bool m_mining_enabled = TREZOR_TEST_MINING_ENABLED_DEFAULT;
+  long m_mining_timeout = TREZOR_TEST_MINING_TIMEOUT_DEFAULT;
+  bool m_no_change_in_tested_tx = false;
 
   cryptonote::account_base m_miner_account;
   cryptonote::account_base m_bob_account;
   cryptonote::account_base m_alice_account;
   cryptonote::account_base m_eve_account;
+  cryptonote::account_base m_alice2_account;
   hw::trezor::device_trezor * m_trezor;
   std::unique_ptr<tools::wallet2> m_wl_alice;
+  std::unique_ptr<tools::wallet2> m_wl_alice2;
   std::unique_ptr<tools::wallet2> m_wl_bob;
   std::unique_ptr<tools::wallet2> m_wl_eve;
 
@@ -165,9 +199,9 @@ public:
   tsx_builder * payment_id(const std::string & payment_id) { m_payment_id = payment_id; return this; }
   tsx_builder * from(tools::wallet2 *from, uint32_t account=0) { m_from = from; m_account=account; return this; }
   tsx_builder * sources(std::vector<cryptonote::tx_source_entry> & sources, std::vector<size_t> & selected_transfers);
-  tsx_builder * compute_sources(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
+  tsx_builder * compute_sources(boost::optional<size_t> num_utxo=boost::none, uint64_t extra_amount=0, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
-  tsx_builder * compute_sources_to_sub(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
+  tsx_builder * compute_sources_to_sub(boost::optional<size_t> num_utxo=boost::none, uint64_t extra_amount=0, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
-  tsx_builder * compute_sources_to_sub_acc(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
+  tsx_builder * compute_sources_to_sub_acc(boost::optional<size_t> num_utxo=boost::none, uint64_t extra_amount=0, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none);
 
   tsx_builder * destinations(std::vector<cryptonote::tx_destination_entry> &dsts);
   tsx_builder * add_destination(const cryptonote::tx_destination_entry &dst);
@@ -208,11 +242,12 @@ class device_trezor_test : public hw::trezor::device_trezor {
 public:
   size_t m_tx_sign_ctr;
   size_t m_compute_key_image_ctr;
+  std::unordered_set<crypto::key_image> m_live_synced_ki;
 
   device_trezor_test();
 
   void clear_test_counters();
-  void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type);
+  void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type, bool use_passphrase=false, const std::string & pin = "");
 
   bool compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key,
                          const ::crypto::key_derivation &recv_derivation, size_t real_output_index,
@@ -315,6 +350,12 @@ public:
   bool generate(std::vector<test_event_entry>& events) override;
 };
 
+class gen_trezor_16utxo_to_sub : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
 class gen_trezor_many_utxo : public gen_trezor_base
 {
 public:
@@ -327,14 +368,52 @@ public:
   bool generate(std::vector<test_event_entry>& events) override;
 };
 
-// Wallet::API tests
+class gen_trezor_no_passphrase : public gen_trezor_base
-class wallet_api_tests : public gen_trezor_base
 {
 public:
-  virtual ~wallet_api_tests();
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_wallet_passphrase : public gen_trezor_base, public tools::i_wallet2_callback
+{
+public:
+  ~gen_trezor_wallet_passphrase() override;
+  bool generate(std::vector<test_event_entry>& events) override;
+  boost::optional<epee::wipeable_string> on_device_pin_request() override;
+  boost::optional<epee::wipeable_string> on_device_passphrase_request(bool &on_device) override;
+protected:
+  boost::filesystem::path m_wallet_dir;
+};
+
+class gen_trezor_passphrase : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+class gen_trezor_pin : public gen_trezor_base
+{
+public:
+  bool generate(std::vector<test_event_entry>& events) override;
+};
+
+// Wallet::API tests
+class wallet_api_tests : public gen_trezor_base, public Monero::WalletListener
+{
+public:
+  ~wallet_api_tests() override;
   void init();
   bool generate(std::vector<test_event_entry>& events) override;
 
+  Monero::optional<std::string> onDevicePinRequest() override;
+  Monero::optional<std::string> onDevicePassphraseRequest(bool &on_device) override;
+  void moneySpent(const std::string &txId, uint64_t amount) override {};
+  void moneyReceived(const std::string &txId, uint64_t amount) override {};
+  void unconfirmedMoneyReceived(const std::string &txId, uint64_t amount) override {};
+  void newBlock(uint64_t height) override {};
+  void updated() override {};
+  void refreshed() override {};
+
 protected:
   boost::filesystem::path m_wallet_dir;
 };