Merge pull request #9434

7cb69fa epee: string_tools: keep full path in cut_off_extension (tobtoht) c7d4bf4 epee: string_tools: remove dot from get_extension (tobtoht)
epee: string_tools: keep full path in cut_off_extension
2024-08-14 14:19:55 -04:00 · 2024-08-14 19:50:06 +02:00 · 2024-08-14 16:57:49 +02:00 · 2024-08-13 12:33:21 -04:00 · 2024-08-13 12:30:11 -04:00 · 2024-08-13 12:29:10 -04:00
184 changed files with 6703 additions and 2584 deletions
@@ -27,16 +27,16 @@ jobs:
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: /Users/runner/Library/Caches/ccache
        key: ccache-${{ runner.os }}-build-${{ github.sha }}
        restore-keys: ccache-${{ runner.os }}-build-
    - name: install dependencies
-      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc ldns expat libunwind-headers protobuf ccache
+      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache
    - name: build
      run: |
        ${{env.CCACHE_SETTINGS}}
@@ -51,15 +51,15 @@ jobs:
      run:
        shell: msys2 {0}
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: C:\Users\runneradmin\.ccache
        key: ccache-${{ runner.os }}-build-${{ github.sha }}
        restore-keys: ccache-${{ runner.os }}-build-
-    - uses: eine/setup-msys2@v2
+    - uses: msys2/setup-msys2@v2
      with:
        update: true
        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
@@ -77,12 +77,12 @@ jobs:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    strategy:
      matrix:
-        os: [ubuntu-latest, ubuntu-18.04]
+        os: [ubuntu-22.04, ubuntu-20.04]
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
@@ -101,14 +101,14 @@ jobs:
        ${{env.BUILD_DEFAULT_LINUX}}

  libwallet-ubuntu:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
@@ -129,15 +129,15 @@ jobs:

  test-ubuntu:
    needs: build-ubuntu
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
    - name: ccache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
@@ -151,10 +151,11 @@ jobs:
    - name: install monero dependencies
      run: ${{env.APT_INSTALL_LINUX}}
    - name: install Python dependencies
-      run: pip install requests psutil monotonic zmq
+      run: pip install requests psutil monotonic zmq deepdiff
    - name: tests
      env:
        CTEST_OUTPUT_ON_FAILURE: ON
+        DNS_PUBLIC: tcp://9.9.9.9
      run: |
        ${{env.CCACHE_SETTINGS}}
        ${{env.BUILD_DEFAULT_LINUX}}
@@ -166,8 +167,9 @@ jobs:
  source-archive:
    runs-on: ubuntu-20.04
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
+        fetch-depth: 0
        submodules: recursive
    - name: archive
      run: |
@@ -176,7 +178,7 @@ jobs:
        export OUTPUT="$VERSION.tar"
        echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV
        /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT"
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
      with:
        name: ${{ env.OUTPUT }}
        path: /home/runner/work/monero/monero/${{ env.OUTPUT }}
@@ -18,7 +18,7 @@ env:

 jobs:
  build-cross:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    strategy:
@@ -36,13 +36,13 @@ jobs:
            packages: "python3 gperf g++-aarch64-linux-gnu"
          - name: "i686 Win"
            host: "i686-w64-mingw32"
-            packages: "python3 g++-mingw-w64-i686 qttools5-dev-tools"
+            packages: "python3 g++-mingw-w64-i686"
          - name: "i686 Linux"
            host: "i686-pc-linux-gnu"
            packages: "gperf cmake g++-multilib python3-zmq"
          - name: "Win64"
            host: "x86_64-w64-mingw32"
-            packages: "cmake python3 g++-mingw-w64-x86-64 qttools5-dev-tools"
+            packages: "cmake python3 g++-mingw-w64-x86-64"
          - name: "x86_64 Linux"
            host: "x86_64-unknown-linux-gnu"
            packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
@@ -57,19 +57,20 @@ jobs:
            packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
    name: ${{ matrix.toolchain.name }}
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
+        fetch-depth: 0
        submodules: recursive
 # Most volatile cache
    - name: ccache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }}
        restore-keys: ccache-${{ matrix.toolchain.host }}-
 # Less volatile cache
    - name: depends cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: contrib/depends/built
        key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
@@ -78,7 +79,7 @@ jobs:
          depends-${{ matrix.toolchain.host }}-
 # Static cache
    - name: OSX SDK cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: contrib/depends/sdk-sources
        key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }}
@@ -96,7 +97,7 @@ jobs:
      run: |
        ${{env.CCACHE_SETTINGS}}
        make depends target=${{ matrix.toolchain.host }} -j2
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
      if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
      with:
        name: ${{ matrix.toolchain.name }}
@@ -0,0 +1,49 @@
+name: ci/gh-actions/gitian
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-gitian:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        operating-system:
+          - name: "Linux"
+            option: "l"
+          - name: "Windows"
+            option: "w"
+          - name: "Android"
+            option: "a"
+          - name: "FreeBSD"
+            option: "f"
+          - name: "macOS"
+            option: "m"
+    name: ${{ matrix.operating-system.name }}
+    steps:
+    - name: prepare
+      run: |
+        sudo apt update
+        curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py
+        chmod +x gitian-build.py
+    - name: setup
+      run: |
+        ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }}
+    - name: build
+      run: |
+        ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }}
+    - name: post build
+      run: |
+        cd out/${{ github.ref_name }}
+        shasum -a256 *
+        echo \`\`\` >> $GITHUB_STEP_SUMMARY
+        shasum -a256 * >> $GITHUB_STEP_SUMMARY
+        echo \`\`\` >> $GITHUB_STEP_SUMMARY
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.operating-system.name }}
+        path: |
+          out/${{ github.ref_name }}/*
@@ -794,7 +794,7 @@ else()
    set(USE_LTO_DEFAULT false)
    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack,10485760")
    if(NOT BUILD_64)
-      add_definitions(-DWINVER=0x0501 -D_WIN32_WINNT=0x0501)
+      add_definitions(-DWINVER=0x0600 -D_WIN32_WINNT=0x0600)
    endif()
  endif()
  set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes")
@@ -1178,6 +1178,9 @@ find_library(NORM_LIBRARY norm)
 find_library(GSSAPI_LIBRARY gssapi_krb5)
 find_library(PROTOLIB_LIBRARY protolib)
 find_library(SODIUM_LIBRARY sodium)
+find_library(BSD_LIBRARY bsd)
+find_library(MD_LIBRARY md)
+find_library(PROTOKIT_LIBRARY protokit)

 if(NOT ZMQ_INCLUDE_PATH)
  message(FATAL_ERROR "Could not find required header zmq.h")
@@ -1200,6 +1203,15 @@ endif()
 if(SODIUM_LIBRARY)
  set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}")
 endif()
+if(BSD_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${BSD_LIBRARY}")
+endif()
+if(MD_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${MD_LIBRARY}")
+endif()
+if(PROTOKIT_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${PROTOKIT_LIBRARY}")
+endif()

 include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion
 add_subdirectory(contrib)
@@ -48,7 +48,7 @@ all: release-all

 depends:
 	cd contrib/depends && $(MAKE) HOST=$(target) && cd ../.. && mkdir -p build/$(target)/release
-	cd build/$(target)/release && cmake -DCMAKE_TOOLCHAIN_FILE=$(CURDIR)/contrib/depends/$(target)/share/toolchain.cmake ../../.. && $(MAKE)
+	cd build/$(target)/release && USE_DEVICE_TREZOR_MANDATORY=1 cmake -DCMAKE_TOOLCHAIN_FILE=$(CURDIR)/contrib/depends/$(target)/share/toolchain.cmake ../../.. && $(MAKE)

 cmake-debug:
 	mkdir -p $(builddir)/debug
@@ -104,7 +104,7 @@ release-all:

 release-static:
 	mkdir -p $(builddir)/release
-	cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
+	cd $(builddir)/release && cmake -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)

 coverage:
 	mkdir -p $(builddir)/debug
@@ -138,8 +138,8 @@ Dates are provided in the format YYYY-MM-DD.
 | 1978433                        | 2019-11-30 | v12               | v0.15.0.0              | v0.16.0.0                  | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs
 | 2210000                        | 2020-10-17 | v13               | v0.17.0.0              | v0.17.3.2                  | New CLSAG transaction format
 | 2210720                        | 2020-10-18 | v14               | v0.17.1.1              | v0.17.3.2                  | forbid old MLSAG transaction format
-| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.1.0                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
-| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.1.0                  | forbid old v14 transaction format
+| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.3.4                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
+| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.3.4                  | forbid old v14 transaction format
 | XXXXXXX                        | XXX-XX-XX | XXX                | vX.XX.X.X              | vX.XX.X.X                  | XXX |

 X's indicate that these details have not been determined as of commit date.
@@ -178,7 +178,6 @@ library archives (`.a`).
 | libunwind    | any           | NO       | `libunwind8-dev`     | `libunwind`  | `libunwind-devel`  | `libunwind-devel`   | YES      | Stack traces    |
 | liblzma      | any           | NO       | `liblzma-dev`        | `xz`         | `liblzma-devel`    | `xz-devel`          | YES      | For libunwind   |
 | libreadline  | 6.3.0         | NO       | `libreadline6-dev`   | `readline`   | `readline-devel`   | `readline-devel`    | YES      | Input editing   |
-| ldns         | 1.6.17        | NO       | `libldns-dev`        | `ldns`       | `libldns-devel`    | `ldns-devel`        | YES      | SSL toolkit     |
 | expat        | 1.1           | NO       | `libexpat1-dev`      | `expat`      | `expat-devel`      | `expat-devel`       | YES      | XML parsing     |
 | GTest        | 1.5           | YES      | `libgtest-dev`[1]    | `gtest`      | `gtest-devel`      | `gtest-devel`       | YES      | Test suite      |
 | ccache       | any           | NO       | `ccache`             | `ccache`     | `ccache`           | `ccache`            | YES      | Compil. cache   |
@@ -205,23 +204,23 @@ then:
 Install all dependencies at once on Debian/Ubuntu:

 ```
-sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz
+sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz
 ```

 Install all dependencies at once on Arch:
 ```
-sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline ldns expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd
+sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd
 ```

 Install all dependencies at once on Fedora:
 ```
-sudo dnf install gcc gcc-c++ cmake pkgconf boost-devel openssl-devel zeromq-devel openpgm-devel unbound-devel libsodium-devel libunwind-devel xz-devel readline-devel ldns-devel expat-devel gtest-devel ccache doxygen graphviz qt5-linguist hidapi-devel libusbx-devel protobuf-devel protobuf-compiler systemd-devel
+sudo dnf install gcc gcc-c++ cmake pkgconf boost-devel openssl-devel zeromq-devel openpgm-devel unbound-devel libsodium-devel libunwind-devel xz-devel readline-devel expat-devel gtest-devel ccache doxygen graphviz qt5-linguist hidapi-devel libusbx-devel protobuf-devel protobuf-compiler systemd-devel
 ```

 Install all dependencies at once on openSUSE:

 ```
-sudo zypper ref && sudo zypper in cppzmq-devel ldns-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++
+sudo zypper ref && sudo zypper in cppzmq-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++
 ```

 Install all dependencies at once on macOS with the provided Brewfile:
@@ -345,7 +344,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (
    ```bash
    git clone https://github.com/monero-project/monero.git
    cd monero
-    git checkout v0.18.1.0
+    git checkout v0.18.3.4
    ```

 * Build:
@@ -464,10 +463,10 @@ application.
    cd monero
    ```

-* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.1.0'. If you don't care about the version and just want binaries from master, skip this step:
+* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.3.4'. If you don't care about the version and just want binaries from master, skip this step:

    ```bash
-    git checkout v0.18.1.0
+    git checkout v0.18.3.4
    ```

 * If you are on a 64-bit system, run:
@@ -113,7 +113,7 @@ if ( LibUSB_FOUND )
    if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC)
        if (APPLE)
            if(DEPENDS)
-                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security")
            else()
                find_library(COREFOUNDATION CoreFoundation)
                find_library(IOKIT IOKit)
@@ -25,7 +25,6 @@ brew "unbound"
 brew "libsodium"
 brew "miniupnpc"
 brew "readline"
-brew "ldns"
 brew "expat"
 brew "ccache"
 brew "doxygen"
@@ -110,8 +110,7 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)

-qt_packages_$(NO_QT) = $(qt_packages) 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)

 all_packages = $(packages) $(native_packages)
@@ -7,27 +7,12 @@ ac_tool_prefix=${host_alias}-
 if test -z $with_boost; then
  with_boost=$depends_prefix
 fi
-if test -z $with_qt_plugindir; then
-  with_qt_plugindir=$depends_prefix/plugins
-fi
-if test -z $with_qt_translationdir; then
-  with_qt_translationdir=$depends_prefix/translations
-fi

 if test x@host_os@ = xdarwin; then
  BREW=no
  PORT=no
 fi

-if test x@host_os@ = xmingw32; then
-  if test -z $with_qt_incdir; then
-    with_qt_incdir=$depends_prefix/include
-  fi
-  if test -z $with_qt_libdir; then
-    with_qt_libdir=$depends_prefix/lib
-  fi
-fi
-
 PATH=$depends_prefix/native/bin:$PATH
 PKG_CONFIG="`which pkg-config` --static"

@@ -143,8 +143,11 @@ $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig
 $(1)_config_env+=PATH="$(build_prefix)/bin:$(PATH)"
 $(1)_build_env+=PATH="$(build_prefix)/bin:$(PATH)"
 $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)"
-$(1)_autoconf=./configure --host=$($($(1)_type)_host) --disable-dependency-tracking --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
+$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"

+ifeq ($(filter $(1),libusb unbound),)
+$(1)_autoconf += --disable-dependency-tracking
+endif
 ifneq ($($(1)_nm),)
 $(1)_autoconf += NM="$$($(1)_nm)"
 endif
@@ -1,12 +1,12 @@
 package=expat
-$(package)_version=2.4.1
-$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_4_1
+$(package)_version=2.6.0
+$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=2f9b6a580b94577b150a7d5617ad4643a4301a6616ff459307df3e225bcfbf40
+$(package)_sha256_hash=ff60e6a6b6ce570ae012dc7b73169c7fdf4b6bf08c12ed0ec6f55736b78d85ba

 define $(package)_set_vars
-$(package)_config_opts=--enable-static
-$(package)_config_opts=--disable-shared
+$(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples
+$(package)_config_opts+=--enable-option-checking --without-xmlwf --with-pic
 $(package)_config_opts+=--prefix=$(host_prefix)
 endef

@@ -23,6 +23,6 @@ define $(package)_stage_cmds
 endef

 define $(package)_postprocess_cmds
-  rm lib/*.la
+  rm -rf share lib/cmake lib/*.la
 endef

@@ -1,8 +1,8 @@
 package=hidapi
-$(package)_version=0.11.0
-$(package)_download_path=https://github.com/libusb/hidapi/archive
+$(package)_version=0.13.1
+$(package)_download_path=https://github.com/libusb/hidapi/archive/refs/tags
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=391d8e52f2d6a5cf76e2b0c079cfefe25497ba1d4659131297081fc0cd744632
+$(package)_sha256_hash=476a2c9a4dc7d1fc97dd223b84338dbea3809a84caea2dcd887d9778725490e3
 $(package)_linux_dependencies=libusb eudev
 $(package)_patches=missing_win_include.patch

@@ -1,34 +0,0 @@
-package=ldns
-$(package)_version=1.7.1
-$(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/
-$(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=8ac84c16bdca60e710eea75782356f3ac3b55680d40e1530d7cea474ac208229
-$(package)_dependencies=openssl
-
-define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --enable-static --with-drill
-  $(package)_config_opts+=--with-ssl=$(host_prefix)
-  $(package)_config_opts_release=--disable-debug-mode
-  $(package)_config_opts_linux=--with-pic
-endef
-
-define $(package)_preprocess_cmds
-   cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub .
-endef
-
-define $(package)_config_cmds
-  $($(package)_autoconf)
-endef
-
-define $(package)_build_cmds
-  $(MAKE)
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install-h install-lib
-endef
-
-define $(package)_postprocess_cmds
-  rm lib/*.la
-endef
-
@@ -1,8 +1,8 @@
 package=libusb
-$(package)_version=1.0.22
-$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
+$(package)_version=1.0.26
+$(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version)
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
+$(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5

 define $(package)_preprocess_cmds
  autoreconf -i
@@ -1,21 +1,19 @@
 package=openssl
-$(package)_version=1.1.1l
+$(package)_version=3.0.13
 $(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
-$(package)_patches=fix_darwin.patch
+$(package)_sha256_hash=88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313

 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
-$(package)_config_env_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
-$(package)_build_env_android=ANDROID_NDK_HOME="$(host_prefix)/native"
-$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
+$(package)_config_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native"
+$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl --libdir=$(host_prefix)/lib
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-heartbeats
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
@@ -23,8 +21,8 @@ $(package)_config_opts+=no-rfc3779
 $(package)_config_opts+=no-sctp
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
-$(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
+$(package)_config_opts+=no-tests
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -50,8 +48,7 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef

 define $(package)_preprocess_cmds
-  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \
-  patch -p1 < $($(package)_patch_dir)/fix_darwin.patch
+  sed -i.old 's|crypto ssl apps util tools fuzz providers doc|crypto ssl util tools providers|' build.info
 endef

 define $(package)_config_cmds
@@ -1,4 +1,4 @@
-packages:=boost openssl zeromq libiconv expat ldns unbound
+packages:=boost openssl zeromq libiconv expat unbound

 # ccache is useless in gitian builds
 ifneq ($(GITIAN),1)
@@ -20,7 +20,6 @@ freebsd_packages = ncurses readline sodium

 linux_packages = eudev ncurses readline sodium $(hardware_packages)
 linux_native_packages = $(hardware_native_packages)
-qt_packages = qt

 ifeq ($(build_tests),ON)
 packages += gtest
@@ -1,175 +0,0 @@
-PACKAGE=qt
-$(package)_version=5.15.1
-$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules
-$(package)_suffix=everywhere-src-$($(package)_version).tar.xz
-$(package)_file_name=qtbase-$($(package)_suffix)
-$(package)_sha256_hash=33960404d579675b7210de103ed06a72613bfc4305443e278e2d32a3eb1f3d8c
-$(package)_build_subdir=qtbase
-$(package)_qt_libs=corelib
-$(package)_patches=fix_qt_pkgconfig.patch fix_no_printer.patch fix_rcc_determinism.patch no-xlib.patch
-
-$(package)_qttranslations_file_name=qttranslations-$($(package)_suffix)
-$(package)_qttranslations_sha256_hash=46e0c0e3a511fbcc803a4146204062e47f6ed43b34d98a3c27372a03b8746bd8
-
-$(package)_qttools_file_name=qttools-$($(package)_suffix)
-$(package)_qttools_sha256_hash=c98ee5f0f980bf68cbf0c94d62434816a92441733de50bd9adbe9b9055f03498
-
-$(package)_extra_sources  = $($(package)_qttranslations_file_name)
-$(package)_extra_sources += $($(package)_qttools_file_name)
-
-define $(package)_set_vars
-$(package)_config_opts_release = -release
-$(package)_config_opts_debug = -debug
-$(package)_config_opts += -bindir $(build_prefix)/bin
-$(package)_config_opts += -c++std c++11
-$(package)_config_opts += -confirm-license
-$(package)_config_opts += -dbus-runtime
-$(package)_config_opts += -hostprefix $(build_prefix)
-$(package)_config_opts += -no-compile-examples
-$(package)_config_opts += -no-cups
-$(package)_config_opts += -no-egl
-$(package)_config_opts += -no-eglfs
-$(package)_config_opts += -no-evdev
-$(package)_config_opts += -no-gui
-$(package)_config_opts += -no-freetype
-$(package)_config_opts += -no-gif
-$(package)_config_opts += -no-glib
-$(package)_config_opts += -no-icu
-$(package)_config_opts += -no-ico
-$(package)_config_opts += -no-iconv
-$(package)_config_opts += -no-kms
-$(package)_config_opts += -no-linuxfb
-$(package)_config_opts += -no-libjpeg
-$(package)_config_opts += -no-libudev
-$(package)_config_opts += -no-mtdev
-$(package)_config_opts += -no-openvg
-$(package)_config_opts += -no-reduce-relocations
-$(package)_config_opts += -no-sql-db2
-$(package)_config_opts += -no-sql-ibase
-$(package)_config_opts += -no-sql-oci
-$(package)_config_opts += -no-sql-tds
-$(package)_config_opts += -no-sql-mysql
-$(package)_config_opts += -no-sql-odbc
-$(package)_config_opts += -no-sql-psql
-$(package)_config_opts += -no-sql-sqlite
-$(package)_config_opts += -no-sql-sqlite2
-$(package)_config_opts += -no-use-gold-linker
-$(package)_config_opts += -nomake examples
-$(package)_config_opts += -nomake tests
-$(package)_config_opts += -opensource
-$(package)_config_opts += -no-openssl
-$(package)_config_opts += -optimized-qmake
-$(package)_config_opts += -pch
-$(package)_config_opts += -pkg-config
-$(package)_config_opts += -prefix $(host_prefix)
-$(package)_config_opts += -no-libpng
-$(package)_config_opts += -qt-pcre
-$(package)_config_opts += -qt-harfbuzz
-$(package)_config_opts += -no-zlib
-$(package)_config_opts += -static
-$(package)_config_opts += -silent
-$(package)_config_opts += -v
-$(package)_config_opts += -no-feature-bearermanagement
-$(package)_config_opts += -no-feature-colordialog
-$(package)_config_opts += -no-feature-dial
-$(package)_config_opts += -no-feature-filesystemwatcher
-$(package)_config_opts += -no-feature-fontcombobox
-$(package)_config_opts += -no-feature-ftp
-$(package)_config_opts += -no-feature-image_heuristic_mask
-$(package)_config_opts += -no-feature-keysequenceedit
-$(package)_config_opts += -no-feature-lcdnumber
-$(package)_config_opts += -no-feature-pdf
-$(package)_config_opts += -no-feature-printdialog
-$(package)_config_opts += -no-feature-printer
-$(package)_config_opts += -no-feature-printpreviewdialog
-$(package)_config_opts += -no-feature-printpreviewwidget
-$(package)_config_opts += -no-feature-sessionmanager
-$(package)_config_opts += -no-feature-sql
-$(package)_config_opts += -no-feature-statemachine
-$(package)_config_opts += -no-feature-syntaxhighlighter
-$(package)_config_opts += -no-feature-textbrowser
-$(package)_config_opts += -no-feature-textodfwriter
-$(package)_config_opts += -no-feature-topleveldomain
-$(package)_config_opts += -no-feature-udpsocket
-$(package)_config_opts += -no-feature-undocommand
-$(package)_config_opts += -no-feature-undogroup
-$(package)_config_opts += -no-feature-undostack
-$(package)_config_opts += -no-feature-undoview
-$(package)_config_opts += -no-feature-vnc
-$(package)_config_opts += -no-feature-wizard
-$(package)_config_opts_linux = -no-fontconfig
-$(package)_config_opts_linux += -no-opengl
-$(package)_config_opts_linux += -no-xcb
-$(package)_config_opts_linux += -no-feature-xlib
-endef
-
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttranslations_file_name),$($(package)_qttranslations_file_name),$($(package)_qttranslations_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_qttools_file_name),$($(package)_qttools_file_name),$($(package)_qttools_sha256_hash))
-endef
-
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_qttranslations_sha256_hash)  $($(package)_source_dir)/$($(package)_qttranslations_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_qttools_sha256_hash)  $($(package)_source_dir)/$($(package)_qttools_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir qtbase && \
-  tar --strip-components=1 -xf $($(package)_source) -C qtbase && \
-  mkdir qttranslations && \
-  tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttranslations_file_name) -C qttranslations && \
-  mkdir qttools && \
-  tar --strip-components=1 -xf $($(package)_source_dir)/$($(package)_qttools_file_name) -C qttools
-endef
-
-
-define $(package)_preprocess_cmds
-  sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \
-  sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \
-  sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \
-  sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \
-  cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \
-  sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \
-  patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \
-  patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \
-  echo "!host_build: QMAKE_CFLAGS     += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  echo "!host_build: QMAKE_CXXFLAGS   += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  echo "!host_build: QMAKE_LFLAGS     += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \
-  echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf && \
-  echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "s|QMAKE_CFLAGS           += |!host_build: QMAKE_CFLAGS            = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "s|QMAKE_CXXFLAGS         += |!host_build: QMAKE_CXXFLAGS            = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS            = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \
-  sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf
-endef
-
-define $(package)_config_cmds
-  export PKG_CONFIG_SYSROOT_DIR=/ && \
-  export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \
-  export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig  && \
-  ./configure $($(package)_config_opts) && \
-  echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \
-  $(MAKE) sub-src-clean && \
-  cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \
-  cd translations && ../../qtbase/bin/qmake translations.pro -o Makefile && cd ../.. &&\
-  cd qttools/src/linguist/lrelease/ && ../../../../qtbase/bin/qmake lrelease.pro -o Makefile
-endef
-
-define $(package)_build_cmds
-  $(MAKE) -C src $(addprefix sub-,$($(package)_qt_libs)) && \
-  $(MAKE) -C ../qttools/src/linguist/lrelease && \
-  $(MAKE) -C ../qttranslations
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) -C src INSTALL_ROOT=$($(package)_staging_dir) $(addsuffix -install_subtargets,$(addprefix sub-,$($(package)_qt_libs))) && cd .. &&\
-  $(MAKE) -C qttools/src/linguist/lrelease INSTALL_ROOT=$($(package)_staging_dir) install_target && \
-  $(MAKE) -C qttranslations INSTALL_ROOT=$($(package)_staging_dir) install_subtargets
-endef
-
-define $(package)_postprocess_cmds
-  rm -rf native/mkspecs/ native/lib/ lib/cmake/ && \
-  rm -f lib/lib*.la lib/*.prl plugins/*/*.prl
-endef
@@ -1,17 +1,21 @@
 package=unbound
-$(package)_version=1.15.0
+$(package)_version=1.19.1
 $(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=a480dc6c8937447b98d161fe911ffc76cfaffa2da18788781314e81339f1126f
-$(package)_dependencies=openssl expat ldns
+$(package)_sha256_hash=bc1d576f3dd846a0739adc41ffaa702404c6767d2b6082deb9f2f97cbb24a3a9
+$(package)_dependencies=openssl expat
 $(package)_patches=disable-glibc-reallocarray.patch


 define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
+  $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix)
+  $(package)_config_opts+=--with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no
+  $(package)_config_opts+=--without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
  $(package)_config_opts_linux=--with-pic
  $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix)
+  $(package)_config_opts_x86_64_darwin=ac_cv_func_SHA384_Init=yes
  $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread"
+  $(package)_cflags_mingw32+="-D_WIN32_WINNT=0x600"
 endef

 define $(package)_preprocess_cmds
@@ -30,6 +34,3 @@ endef
 define $(package)_stage_cmds
  $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
-
-define $(package)_postprocess_cmds
-endef
@@ -1,60 +0,0 @@
-From 96ac8f13f4d0ee96baf5724d9f96c44c34b8606c Mon Sep 17 00:00:00 2001
-From: David Carlier <devnexen@gmail.com>
-Date: Tue, 24 Aug 2021 22:40:14 +0100
-Subject: [PATCH] Darwin platform allows to build on releases before
- Yosemite/ios 8.
-
-issue #16407 #16408
-
-Reviewed-by: Paul Dale <pauli@openssl.org>
-Reviewed-by: Tomas Mraz <tomas@openssl.org>
-(Merged from https://github.com/openssl/openssl/pull/16409)
---
- crypto/rand/rand_unix.c |  5 +----
- include/crypto/rand.h   | 10 ++++++++++
- 2 files changed, 11 insertions(+), 4 deletions(-)
-
-diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c
-index 43f1069d151d..0f4525106af7 100644
--- a/crypto/rand/rand_unix.c
-+++ b/crypto/rand/rand_unix.c
-@@ -34,9 +34,6 @@
- #if defined(__OpenBSD__)
- # include <sys/param.h>
- #endif
-#if defined(__APPLE__)
-# include <CommonCrypto/CommonRandom.h>
-#endif
- 
- #if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__)
- # include <sys/types.h>
-@@ -381,7 +378,7 @@ static ssize_t syscall_random(void *buf, size_t buflen)
-         if (errno != ENOSYS)
-             return -1;
-     }
-#  elif defined(__APPLE__)
-+#  elif defined(OPENSSL_APPLE_CRYPTO_RANDOM)
-     if (CCRandomGenerateBytes(buf, buflen) == kCCSuccess)
- 	    return (ssize_t)buflen;
- 
-diff --git a/include/crypto/rand.h b/include/crypto/rand.h
-index 5350d3a93119..674f840fd13c 100644
--- a/include/crypto/rand.h
-+++ b/include/crypto/rand.h
-@@ -20,6 +20,16 @@
- 
- # include <openssl/rand.h>
- 
-+# if defined(__APPLE__) && !defined(OPENSSL_NO_APPLE_CRYPTO_RANDOM)
-+#  include <Availability.h>
-+#  if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101000) || \
-+     (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 80000)
-+#   define OPENSSL_APPLE_CRYPTO_RANDOM 1
-+#   include <CommonCrypto/CommonCryptoError.h>
-+#   include <CommonCrypto/CommonRandom.h>
-+#  endif
-+# endif
-+
- /* forward declaration */
- typedef struct rand_pool_st RAND_POOL;
- 
@@ -1,19 +0,0 @@
--- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
-+++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
-@@ -52,6 +52,7 @@
- //
-
- #include <QtCore/qglobal.h>
-+#include <qpa/qplatformprintdevice.h>
-
- #ifndef QT_NO_PRINTER
-
--- x/qtbase/src/plugins/plugins.pro
-+++ y/qtbase/src/plugins/plugins.pro
-@@ -9,6 +9,3 @@ qtHaveModule(gui) {
-     !android:qtConfig(library): SUBDIRS *= generic
- }
- qtHaveModule(widgets): SUBDIRS += styles
-
-!winrt:qtHaveModule(printsupport): \
-    SUBDIRS += printsupport
@@ -1,11 +0,0 @@
--- old/qtbase/mkspecs/features/qt_module.prf
-+++ new/qtbase/mkspecs/features/qt_module.prf
-@@ -269,7 +269,7 @@ load(qt_installs)
- load(qt_targets)
- 
- # this builds on top of qt_common
-!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
-+if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
-     CONFIG += create_pc
-     QMAKE_PKGCONFIG_DESTDIR = pkgconfig
-     host_build: \
@@ -1,15 +0,0 @@
--- old/qtbase/src/tools/rcc/rcc.cpp
-+++ new/qtbase/src/tools/rcc/rcc.cpp
-@@ -207,7 +207,11 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib)
-     if (lib.formatVersion() >= 2) {
-         // last modified time stamp
-         const QDateTime lastModified = m_fileInfo.lastModified();
-        lib.writeNumber8(quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0));
-+        quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0);
-+        static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong();
-+        if (sourceDate != 0)
-+            lastmod = sourceDate;
-+        lib.writeNumber8(lastmod);
-         if (text || pass1)
-             lib.writeChar('\n');
-     }
@@ -1,69 +0,0 @@
-From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001
-From: Carl Dong <contact@carldong.me>
-Date: Thu, 18 Jul 2019 17:22:05 -0400
-Subject: [PATCH] Wrap xlib related code blocks in #if's
-
-They are not necessary to compile QT.
---
- qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++
- 1 file changed, 8 insertions(+)
-
-diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
-index 7c62c2e2b3..c05c6c0a07 100644
--- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
-+++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
-@@ -49,7 +49,9 @@
- #include <QtGui/QWindow>
- #include <QtGui/QBitmap>
- #include <QtGui/private/qguiapplication_p.h>
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
- #include <X11/cursorfont.h>
-+#endif
- #include <xcb/xfixes.h>
- #include <xcb/xcb_image.h>
- 
-@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window)
-     xcb_flush(xcb_connection());
- }
- 
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
- static int cursorIdForShape(int cshape)
- {
-     int cursorId = 0;
-@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape)
-     }
-     return cursorId;
- }
-+#endif
- 
- xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape)
- {
-@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape)
- xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
- {
-     xcb_connection_t *conn = xcb_connection();
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
-     int cursorId = cursorIdForShape(cshape);
-+#endif
-     xcb_cursor_t cursor = XCB_NONE;
- 
-     // Try Xcursor first
-@@ -586,6 +592,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
-     // Non-standard X11 cursors are created from bitmaps
-     cursor = createNonStandardCursor(cshape);
- 
-+#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
-     // Create a glpyh cursor if everything else failed
-     if (!cursor && cursorId) {
-         cursor = xcb_generate_id(conn);
-@@ -593,6 +600,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
-                                 cursorId, cursorId + 1,
-                                 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0);
-     }
-+#endif
- 
-     if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) {
-         const char *name = cursorNames[cshape].front();
---
-2.22.0
-
@@ -27,8 +27,6 @@ SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a)
 SET(UNBOUND_INCLUDE_DIR @prefix@/include)
 SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a)

-SET(LRELEASE_PATH @prefix@/native/bin CACHE FILEPATH "path to lrelease" FORCE)
-
 if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android")
 SET(LIBUNWIND_INCLUDE_DIR @prefix@/include)
 SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a)
@@ -146,8 +144,11 @@ elseif(ARCHITECTURE STREQUAL "aarch64")
 endif()

 if(ARCHITECTURE STREQUAL "riscv64")
-    set(NO_AES ON)
-    set(ARCH "rv64imafdc")
+  if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    set(BUILD_TAG "linux-riscv64")
+  endif()
+  set(ARCH_ID "riscv64")
+  set(ARCH "rv64gc")
 endif()

 if(ARCHITECTURE STREQUAL "i686")
@@ -29,6 +29,7 @@

 #include <string>
 #include <ctime>
+#include <cstdint>

 namespace epee
 {
@@ -583,11 +583,8 @@ namespace net_utils
            break;
        }
      }
-      else if (ec.value())
-        terminate();
      else {
-        cancel_timer();
-        on_interrupted();
+        terminate();
      }
    };
    m_strand.post(
@@ -147,6 +147,16 @@ namespace epee
    return {reinterpret_cast<const std::uint8_t*>(src.data()), src.size_bytes()}; 
  }

+  //! \return `span<std::uint8_t>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<std::uint8_t> to_mut_byte_span(T& src)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_empty<value_type>(), "empty value types will not work -> sizeof == 1");
+    static_assert(!has_padding<value_type>(), "source value type may have padding");
+    return {reinterpret_cast<std::uint8_t*>(src.data()), src.size() * sizeof(value_type)};
+  }
+
  //! \return `span<const std::uint8_t>` which represents the bytes at `&src`.
  template<typename T>
  span<const std::uint8_t> as_byte_span(const T& src) noexcept
@@ -30,6 +30,7 @@

 #include <boost/utility/string_ref_fwd.hpp>
 #include <string>
+#include <cstdint>

 namespace epee 
 {
@@ -33,6 +33,9 @@
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"

+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #else 
@@ -31,6 +31,9 @@
 #include "parserse_base_utils.h"
 #include "file_io_utils.h"

+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100

 namespace epee
@@ -37,6 +37,7 @@
 #include "misc_log_ex.h"

 #include <boost/lexical_cast.hpp>
+#include <boost/numeric/conversion/bounds.hpp>
 #include <typeinfo>
 #include <iomanip>

@@ -31,6 +31,7 @@
 #include "mlocker.h"

 #include <boost/utility/string_ref.hpp>
+#include <boost/algorithm/string.hpp> 
 #include <sstream>
 #include <string>
 #include <cstdint>
@@ -69,23 +70,19 @@ namespace string_tools
 #ifdef _WIN32
   std::string get_current_module_path();
 #endif
-  bool set_module_name_and_folder(const std::string& path_to_process_);
-  bool trim_left(std::string& str);
-  bool trim_right(std::string& str);
+  void set_module_name_and_folder(const std::string& path_to_process_);
+  void trim_left(std::string& str);
+  void trim_right(std::string& str);
  //----------------------------------------------------------------------------
  inline std::string& trim(std::string& str)
  {
-    trim_left(str);
-    trim_right(str);
+    boost::trim(str);
    return str;
  }
  //----------------------------------------------------------------------------
-  inline std::string trim(const std::string& str_)
+  inline std::string trim(const std::string& str)
  {
-    std::string str = str_;
-    trim_left(str);
-    trim_right(str);
-    return str;
+    return boost::trim_copy(str);
  }
  std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false);
  
@@ -29,7 +29,7 @@
 #include <fstream>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/operations.hpp>
-#ifdef WIN32
+#ifdef _WIN32
 #include <windows.h>
 #include "string_tools.h"
 #endif
@@ -70,7 +70,7 @@ namespace file_io_utils

 	bool save_string_to_file(const std::string& path_to_file, const std::string& str)
 	{
-#ifdef WIN32
+#ifdef _WIN32
                std::wstring wide_path;
                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -104,7 +104,7 @@ namespace file_io_utils

 	bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size)
 	{
-#ifdef WIN32
+#ifdef _WIN32
                std::wstring wide_path;
                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -153,7 +153,7 @@ namespace file_io_utils

 	bool get_file_size(const std::string& path_to_file, uint64_t &size)
 	{
-#ifdef WIN32
+#ifdef _WIN32
                std::wstring wide_path;
                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -176,11 +176,12 @@ void mlog_configure(const std::string &filename_base, bool console, const std::s
      std::vector<boost::filesystem::path> found_files;
      const boost::filesystem::directory_iterator end_itr;
      const boost::filesystem::path filename_base_path(filename_base);
+      const std::string filename_base_name = filename_base_path.filename().string();
      const boost::filesystem::path parent_path = filename_base_path.has_parent_path() ? filename_base_path.parent_path() : ".";
      for (boost::filesystem::directory_iterator iter(parent_path); iter != end_itr; ++iter)
      {
-        const std::string filename = iter->path().string();
-        if (filename.size() >= filename_base.size() && std::memcmp(filename.data(), filename_base.data(), filename_base.size()) == 0)
+        const std::string filename = iter->path().filename().string();
+        if (filename.size() >= filename_base_name.size() && std::memcmp(filename.data(), filename_base_name.data(), filename_base_name.size()) == 0)
        {
          found_files.push_back(iter->path());
        }
@@ -338,11 +339,21 @@ bool is_stdout_a_tty()
  return is_a_tty.load(std::memory_order_relaxed);
 }

+static bool is_nocolor()
+{
+  static const char *no_color_var = getenv("NO_COLOR");
+  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
+  return no_color;
+}
+
 void set_console_color(int color, bool bright)
 {
  if (!is_stdout_a_tty())
    return;

+  if (is_nocolor())
+    return;
+
  switch(color)
  {
  case console_color_default:
@@ -461,6 +472,9 @@ void reset_console_color() {
  if (!is_stdout_a_tty())
    return;

+  if (is_nocolor())
+    return;
+
 #ifdef WIN32
  HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
  SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);
@@ -496,6 +496,13 @@ void ssl_options_t::configure(
  const std::string& host) const
 {
  socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+  {
+    // in case server is doing "virtual" domains, set hostname
+    SSL* const ssl_ctx = socket.native_handle();
+    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
+      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
+  }
+

  /* Using system-wide CA store for client verification is funky - there is
     no expected hostname for server to verify against. If server doesn't have
@@ -513,11 +520,7 @@ void ssl_options_t::configure(
  {
    socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);

-    // in case server is doing "virtual" domains, set hostname
-    SSL* const ssl_ctx = socket.native_handle();
-    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
-      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
-
+    
    socket.set_verify_callback([&](const bool preverified, boost::asio::ssl::verify_context &ctx)
    {
      // preverified means it passed system or user CA check. System CA is never loaded
@@ -49,7 +49,7 @@ namespace serialization
    byte_stream ss;
    ss.reserve(initial_buffer_size);
    store_to_binary(ss);
-    target = epee::byte_slice{std::move(ss)};
+    target = epee::byte_slice{std::move(ss), false};
    return true;
    CATCH_ENTRY("portable_storage::store_to_binary", false);
  }
@@ -1,4 +1,5 @@
 #include "readline_buffer.h"
+#include "string_tools.h"
 #include <readline/readline.h>
 #include <readline/history.h>
 #include <iostream>
@@ -173,7 +174,7 @@ static void handle_line(char* line)
    line_stat = rdln::full;
    the_line = line;
    std::string test_line = line;
-    boost::trim_right(test_line);
+    epee::string_tools::trim_right(test_line);
    if(!test_line.empty())
    {
      if (!same_as_last_line(test_line))
@@ -238,6 +239,10 @@ static char** attempted_completion(const char* text, int start, int end)

 static void install_line_handler()
 {
+#if RL_READLINE_VERSION >= 0x0801
+  rl_variable_bind("enable-bracketed-paste", "off");
+#endif
+
  rl_attempted_completion_function = attempted_completion;
  rl_callback_handler_install("", handle_line);
  stifle_history(500);
@@ -38,9 +38,12 @@
 #include <cstdlib>
 #include <string>
 #include <type_traits>
+#include <system_error>
 #include <boost/lexical_cast.hpp>
+#include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/utility/string_ref.hpp>
+#include <boost/filesystem.hpp>
 #include "misc_log_ex.h"
 #include "storages/parserse_base_utils.h"
 #include "hex.h"
@@ -157,46 +160,33 @@ namespace string_tools
    return pname;
  }
 #endif
-	
-    bool set_module_name_and_folder(const std::string& path_to_process_)
-	{
-    std::string path_to_process = path_to_process_;
+
+  void set_module_name_and_folder(const std::string& path_to_process_)
+  {
+    boost::filesystem::path path_to_process = path_to_process_;
+
 #ifdef _WIN32
    path_to_process = get_current_module_path();
 #endif 
-		std::string::size_type a = path_to_process.rfind( '\\' );
-		if(a == std::string::npos )
-		{
-			a = path_to_process.rfind( '/' );
-		}
-		if ( a != std::string::npos )
-		{	
-			get_current_module_name() = path_to_process.substr(a+1, path_to_process.size());
-			get_current_module_folder() = path_to_process.substr(0, a);
-			return true;
-		}else
-			return false;

-	}
+    get_current_module_name() = path_to_process.filename().string();
+    get_current_module_folder() = path_to_process.parent_path().string();
+  }

 	//----------------------------------------------------------------------------
-	bool trim_left(std::string& str)
-	{
-		for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));)
-			str.erase(str.begin());
-			
-		return true;
-	}
+  void trim_left(std::string& str)
+  {
+    boost::trim_left(str);
+    return;
+  }
+
 	//----------------------------------------------------------------------------
-	bool trim_right(std::string& str)
-	{
+  void trim_right(std::string& str)
+  {
+    boost::trim_right(str);
+    return;
+  }

-		for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));)
-			str.erase( --((it++).base()));
-
-		return true;
-	}
-  //----------------------------------------------------------------------------
  std::string pad_string(std::string s, size_t n, char c, bool prepend)
  {
    if (s.size() < n)
@@ -209,28 +199,22 @@ namespace string_tools
    return s;
  }
  
-    std::string get_extension(const std::string& str)
-	{
-		std::string res;
-		std::string::size_type pos = str.rfind('.');
-		if(std::string::npos == pos)
-			return res;
-		
-		res = str.substr(pos+1, str.size()-pos);
-		return res;
-	}
-	//----------------------------------------------------------------------------
-	std::string cut_off_extension(const std::string& str)
-	{
-		std::string res;
-		std::string::size_type pos = str.rfind('.');
-		if(std::string::npos == pos)
-			return str;
+  std::string get_extension(const std::string& str)
+  {
+    std::string ext_with_dot = boost::filesystem::path(str).extension().string();
+
+    if (ext_with_dot.empty())
+      return {};
+
+    return ext_with_dot.erase(0, 1);
+  }
+
+	//----------------------------------------------------------------------------
+  std::string cut_off_extension(const std::string& str)
+  {
+    return boost::filesystem::path(str).replace_extension("").string();
+  }

-		res = str.substr(0, pos);
-		return res;
-	}
-  //----------------------------------------------------------------------------
 #ifdef _WIN32
  std::wstring utf8_to_utf16(const std::string& str)
  {
@@ -57,7 +57,8 @@ The dockrun.sh script will do everything to build the binaries. Just specify the
 version to build as its only argument, e.g.

 ```bash
-./dockrun.sh v0.18.1.0
+VERSION=v0.18.3.4
+./dockrun.sh $VERSION
 ```

 The build should run to completion with no errors, and will display the SHA256 checksums
@@ -78,7 +79,7 @@ e.g.

 ```bash
 # Run build processes with 8 threads
-OPT="-j 8" ./dockrun.sh v0.18.1.0
+OPT="-j 8" ./dockrun.sh $VERSION
 ```

 Post-build
@@ -98,16 +99,16 @@ more builder/var/install-linux.log
 more builder/var/build-linux.log
 ```

-You can find the compiled archives inside of the container at the following directory (be sure to replace `v0.18.1.0` with the version being built):
+You can find the compiled archives inside of the container at the following directory:

 ```bash
 docker exec -it gitrun /bin/bash
-ls -la out/v0.18.1.0/
+ls -la out/$VERSION/
 ```

-To copy the compiled archives to the local host out of the Docker container, you can run the following (be sure to replace `v0.18.1.0` with the version being built):
+To copy the compiled archives to the local host out of the Docker container, you can run the following:

 ```bash
 mkdir out
-docker cp gitrun:/home/ubuntu/out/v0.18.1.0 out
+docker cp gitrun:/home/ubuntu/out/$VERSION out
 ```
@@ -133,7 +133,7 @@ Common setup part:
 su - gitianuser

 GH_USER=YOUR_GITHUB_USER_NAME
-VERSION=v0.18.1.0
+VERSION=v0.18.3.4
 ```

 Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. 
@@ -21,6 +21,7 @@ packages:
 - "g++-7-arm-linux-gnueabihf"
 - "gcc-arm-linux-gnueabihf"
 - "g++-arm-linux-gnueabihf"
+- "g++-riscv64-linux-gnu"
 - "g++-7-multilib"
 - "gcc-7-multilib"
 - "binutils-arm-linux-gnueabihf"
@@ -43,7 +44,7 @@ files: []
 script: |

  WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu"
+  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu riscv64-linux-gnu"
  FAKETIME_HOST_PROGS=""
  FAKETIME_PROGS="date"
  HOST_CFLAGS="-O2 -g"
@@ -159,7 +160,13 @@ script: |
    fi
    export C_INCLUDE_PATH="$EXTRA_INCLUDES"
    export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON
+    #  glibc only added riscv support in 2.27, disable backwards compatibility
+    if [ "$i" == "riscv64-linux-gnu" ]; then
+      BACKCOMPAT_OPTION=OFF
+    else
+      BACKCOMPAT_OPTION=ON
+    fi
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=${BACKCOMPAT_OPTION} -DCMAKE_SKIP_RPATH=ON
    make ${MAKEOPTS}
    chmod 755 bin/*
    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
@@ -71,13 +71,13 @@ type, and max connections:

 ```
 --anonymous-inbound rveahdfho7wo4b2m.onion:28083,127.0.0.1:28083,25
--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000,127.0.0.1:30000
+--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p,127.0.0.1:30000
 ```

 which tells `monerod` that a max of 25 inbound Tor connections are being
 received at address "rveahdfho7wo4b2m.onion:28083" and forwarded to `monerod`
 localhost port 28083, and a default max I2P connections are being received at
-address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000" and
+address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p" and
 forwarded to `monerod` localhost port 30000.
 These addresses will be shared with outgoing peers, over the same network type,
 otherwise the peer will not be notified of the peer address by the proxy.
@@ -149,6 +149,11 @@ static el::Color colorFromLevel(el::Level level)

 static void setConsoleColor(el::Color color, bool bright)
 {
+  static const char *no_color_var = getenv("NO_COLOR");
+  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
+  if (no_color)
+    return;
+
 #if ELPP_OS_WINDOWS
  HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
  switch (color)
@@ -46,6 +46,77 @@ using namespace cryptonote;

 static bool stop_requested = false;

+static bool do_inputs, do_outputs, do_ringsize, do_hours, do_emission, do_fees, do_diff;
+
+static struct tm prevtm, currtm;
+static uint64_t prevsz, currsz;
+static uint64_t prevtxs, currtxs;
+static uint64_t currblks;
+static uint64_t h;
+static uint64_t totins, totouts, totrings;
+static boost::multiprecision::uint128_t prevemission, prevfees;
+static boost::multiprecision::uint128_t emission, fees;
+static boost::multiprecision::uint128_t totdiff, mindiff, maxdiff;
+
+#define MAX_INOUT	0xffffffff
+#define MAX_RINGS	0xffffffff
+
+static uint32_t minins = MAX_INOUT, maxins;
+static uint32_t minouts = MAX_INOUT, maxouts;
+static uint32_t minrings = MAX_RINGS, maxrings;
+static uint32_t io, tottxs;
+static uint32_t txhr[24];
+
+static void doprint()
+{
+  char timebuf[64];
+
+  strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm);
+  prevtm = currtm;
+  std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
+  prevsz += currsz;
+  currsz = 0;
+  prevtxs += currtxs;
+  currtxs = 0;
+  if (!tottxs)
+    tottxs = 1;
+  if (do_emission) {
+    std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
+    prevemission += emission;
+    emission = 0;
+  }
+  if (do_fees) {
+    std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
+    prevfees += fees;
+    fees = 0;
+  }
+  if (do_diff) {
+    std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
+    mindiff = 0; maxdiff = 0; totdiff = 0;
+  }
+  if (do_inputs) {
+    std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
+    minins = MAX_INOUT; maxins = 0; totins = 0;
+  }
+  if (do_outputs) {
+    std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
+    minouts = MAX_INOUT; maxouts = 0; totouts = 0;
+  }
+  if (do_ringsize) {
+    std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
+    minrings = MAX_RINGS; maxrings = 0; totrings = 0;
+  }
+  if (do_hours) {
+    for (int i=0; i<24; i++) {
+      std::cout << "\t" << txhr[i];
+      txhr[i] = 0;
+    }
+  }
+  currblks = 0;
+  tottxs = 0;
+  std::cout << ENDL;
+}
+
 int main(int argc, char* argv[])
 {
  TRY_ENTRY();
@@ -123,13 +194,13 @@ int main(int argc, char* argv[])
  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
  block_start = command_line::get_arg(vm, arg_block_start);
  block_stop = command_line::get_arg(vm, arg_block_stop);
-  bool do_inputs = command_line::get_arg(vm, arg_inputs);
-  bool do_outputs = command_line::get_arg(vm, arg_outputs);
-  bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
-  bool do_hours = command_line::get_arg(vm, arg_hours);
-  bool do_emission = command_line::get_arg(vm, arg_emission);
-  bool do_fees = command_line::get_arg(vm, arg_fees);
-  bool do_diff = command_line::get_arg(vm, arg_diff);
+  do_inputs = command_line::get_arg(vm, arg_inputs);
+  do_outputs = command_line::get_arg(vm, arg_outputs);
+  do_ringsize = command_line::get_arg(vm, arg_ringsize);
+  do_hours = command_line::get_arg(vm, arg_hours);
+  do_emission = command_line::get_arg(vm, arg_emission);
+  do_fees = command_line::get_arg(vm, arg_fees);
+  do_diff = command_line::get_arg(vm, arg_diff);

  LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
  std::unique_ptr<Blockchain> core_storage;
@@ -211,25 +282,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
  }
  std::cout << ENDL;

-#define MAX_INOUT	0xffffffff
-#define MAX_RINGS	0xffffffff
-
-  struct tm prevtm = {0}, currtm;
-  uint64_t prevsz = 0, currsz = 0;
-  uint64_t prevtxs = 0, currtxs = 0;
-  uint64_t currblks = 0;
-  uint64_t totins = 0, totouts = 0, totrings = 0;
-  boost::multiprecision::uint128_t prevemission = 0, prevfees = 0;
-  boost::multiprecision::uint128_t emission = 0, fees = 0;
-  boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0;
-  uint32_t minins = MAX_INOUT, maxins = 0;
-  uint32_t minouts = MAX_INOUT, maxouts = 0;
-  uint32_t minrings = MAX_RINGS, maxrings = 0;
-  uint32_t io, tottxs = 0;
-  uint32_t txhr[24] = {0};
-  unsigned int i;
-
-  for (uint64_t h = block_start; h < block_stop; ++h)
+  for (h = block_start; h < block_stop; ++h)
  {
    cryptonote::blobdata bd = db->get_block_blob_from_height(h);
    cryptonote::block blk;
@@ -239,7 +292,6 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
      return 1;
    }
    time_t tt = blk.timestamp;
-    char timebuf[64];
    epee::misc_utils::get_gmt_time(tt, currtm);
    if (!prevtm.tm_year)
      prevtm = currtm;
@@ -247,54 +299,9 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
    if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27))
    {
      // check for timestamp fudging around month ends
-      if (prevtm.tm_mday == 1 && currtm.tm_mday > 27)
-        goto skip;
-      strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm);
-      prevtm = currtm;
-      std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
-      prevsz += currsz;
-      currsz = 0;
-      prevtxs += currtxs;
-      currtxs = 0;
-      if (!tottxs)
-        tottxs = 1;
-      if (do_emission) {
-        std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
-        prevemission += emission;
-        emission = 0;
-      }
-      if (do_fees) {
-        std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
-        prevfees += fees;
-        fees = 0;
-      }
-      if (do_diff) {
-        std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
-        mindiff = 0; maxdiff = 0; totdiff = 0;
-      }
-      if (do_inputs) {
-        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
-        minins = MAX_INOUT; maxins = 0; totins = 0;
-      }
-      if (do_outputs) {
-        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
-        minouts = MAX_INOUT; maxouts = 0; totouts = 0;
-      }
-      if (do_ringsize) {
-        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
-        minrings = MAX_RINGS; maxrings = 0; totrings = 0;
-      }
-      if (do_hours) {
-        for (i=0; i<24; i++) {
-          std::cout << "\t" << txhr[i];
-          txhr[i] = 0;
-        }
-      }
-      currblks = 0;
-      tottxs = 0;
-      std::cout << ENDL;
+      if (!(prevtm.tm_mday == 1 && currtm.tm_mday > 27))
+        doprint();
    }
-skip:
    currsz += bd.size();
    uint64_t coinbase_amount;
    uint64_t tx_fee_amount = 0;
@@ -371,6 +378,8 @@ skip:
    if (stop_requested)
      break;
  }
+  if (currblks)
+    doprint();

  core_storage->deinit();
  return 0;
@@ -241,6 +241,16 @@ namespace cryptonote
    ADD_CHECKPOINT2(2182500, "0d22b5f81982eff21d094af9e821dc2007e6342069e3b1a37b15d97646353124", "0xead4a874083492");
    ADD_CHECKPOINT2(2661600, "41c9060e8426012238e8a26da26fcb90797436896cc70886a894c2c560bcccf2", "0x2e0d87526ff161f");
    ADD_CHECKPOINT2(2677000, "1b9fee6246eeb176bd17d637bf252e9af54a4218675f01b4449cc0901867f9eb", "0x2f165bc1a5163ba");
+    ADD_CHECKPOINT2(2706000, "d8eb144c5e1fe6b329ecc900ec95e7792fccff84175fb23a25ed59d7299a511c", "0x310f7d89372f705");
+    ADD_CHECKPOINT2(2720000, "b19fb41dff15bd1016afbee9f8469f05aab715c9e5d1b974466a11fd58ecbb86", "0x3216b5851ddbb61");
+    ADD_CHECKPOINT2(2817000, "39726d19ccaac01d150bec827b877ffae710b516bd633503662036ef4422e577", "0x3900669561954c1");
+    ADD_CHECKPOINT2(2844000, "28fc7b446dfef5b469f5778eb72ddf32a307a5f5a9823d1c394e772349e05d40", "0x3af384ec0e97d12");
+    ADD_CHECKPOINT2(2851000, "5bf0e47fc782263191a33f63a67db6c711781dc2a3c442e17ed901ec401be5c9", "0x3b6cd8a8ed610e8");
+    ADD_CHECKPOINT2(2971000, "3d4cac5ac515eeabd18769ab943af85f36db51d28720def0d0e6effc2c8f5ce3", "0x436e532738b8b5b");
+    ADD_CHECKPOINT2(2985000, "08f5e6b7301c1b6ed88268a28f8677a06e8ff943b3f9e48d3080f71f9c134bfb", "0x444b7b42a633c96");
+    ADD_CHECKPOINT2(3088000, "bddf8ca09110d33d6d497f13a113630c2b6af1c84d4f3a6f35cb1446f2604ade", "0x4aed3615c2f8c3e");
+    ADD_CHECKPOINT2(3102800, "083f4a34f9490403b564286e7f13fd1ed45c52c86fa47195f151594e5bc87504", "0x4bbed52d4da5dfb");
+    ADD_CHECKPOINT2(3198000, "1d685b39be51e4e84e0af69fa78e023c7cb21de7d33acd012d0371d5f78712d5", "0x517d415fee3a816");
    return true;
  }

@@ -35,6 +35,7 @@
 #include <boost/archive/portable_binary_iarchive.hpp>
 #include <boost/filesystem/operations.hpp>

+#include "common/util.h"

 namespace tools
 {
@@ -110,7 +111,7 @@ namespace tools
    catch(...)
    {
      // if failed, try reading in unportable mode
-      boost::filesystem::copy_file(file_path, file_path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
+      tools::copy_file(file_path, file_path + ".unportable");
      data_file.close();
      data_file.open( file_path, std::ios_base::binary | std::ios_base::in);
      if(data_file.fail())
@@ -34,6 +34,7 @@
 #include <iostream>
 #include <vector>
 #include <stdexcept>
+#include <cstdint>

 namespace tools {

@@ -0,0 +1,65 @@
+// Copyright (c) 2014-2022, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once 
+
+#include <unordered_set>
+#include <mutex>
+
+namespace tools
+{
+  template<typename T, size_t MAX_SIZE>
+  class data_cache
+  {
+  public:
+    void add(const T& value)
+    {
+      std::lock_guard<std::mutex> lock(m);
+      if (data.insert(value).second)
+      {
+        T& old_value = buf[counter++ % MAX_SIZE];
+        data.erase(old_value);
+        old_value = value;
+      }
+    }
+
+    bool has(const T& value) const
+    {
+      std::lock_guard<std::mutex> lock(m);
+      return (data.find(value) != data.end());
+    }
+
+  private:
+    mutable std::mutex m;
+    std::unordered_set<T> data;
+    T buf[MAX_SIZE] = {};
+    size_t counter = 0;
+  };
+}
@@ -30,6 +30,8 @@
 // check local first (in the event of static or in-source compilation of libunbound)
 #include "unbound.h"

+#include <deque>
+#include <set>
 #include <stdlib.h>
 #include "include_base_utils.h"
 #include "common/threadpool.h"
@@ -327,11 +329,6 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec
  dnssec_available = false;
  dnssec_valid = false;

-  if (!check_address_syntax(url.c_str()))
-  {
-    return addresses;
-  }
-
  // destructor takes care of cleanup
  ub_result_ptr result;

@@ -414,16 +411,6 @@ DNSResolver DNSResolver::create()
  return DNSResolver();
 }

-bool DNSResolver::check_address_syntax(const char *addr) const
-{
-  // if string doesn't contain a dot, we won't consider it a url for now.
-  if (strchr(addr,'.') == NULL)
-  {
-    return false;
-  }
-  return true;
-}
-
 namespace dns_utils
 {

@@ -521,7 +508,7 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std

  // send all requests in parallel
  std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false);
-  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool& tpool = tools::threadpool::getInstanceForIO();
  tools::threadpool::waiter waiter(tpool);
  for (size_t n = 0; n < dns_urls.size(); ++n)
  {
@@ -159,15 +159,6 @@ private:
  // TODO: modify this to accommodate DNSSEC
  std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid);

-  /**
-   * @brief Checks a string to see if it looks like a URL
-   *
-   * @param addr the string to be checked
-   *
-   * @return true if it looks enough like a URL, false if not
-   */
-  bool check_address_syntax(const char *addr) const;
-
  DNSResolverData *m_data;
 }; // class DNSResolver

@@ -62,7 +62,7 @@ namespace tools
    while (1)
    {
      t1 = epee::misc_utils::get_ns_count();
-      if (t1 - t0 > 1*1000000000) break; // work one second
+      if (t1 - t0 > 1*100000000) break; // work 0.1 seconds
    }

    uint64_t r1 = get_tick_count();
@@ -31,6 +31,7 @@
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/thread.hpp>
 #include <cstddef>
+#include <deque>
 #include <functional>
 #include <utility>
 #include <vector>
@@ -42,10 +43,14 @@ namespace tools
 class threadpool
 {
 public:
-  static threadpool& getInstance() {
+  static threadpool& getInstanceForCompute() {
    static threadpool instance;
    return instance;
  }
+  static threadpool& getInstanceForIO() {
+    static threadpool instance(8);
+    return instance;
+  }
  static threadpool *getNewForUnitTests(unsigned max_threads = 0) {
    return new threadpool(max_threads);
  }
@@ -115,6 +115,24 @@ static int flock_exnb(int fd)

 namespace tools
 {
+
+  void copy_file(const std::string& from, const std::string& to)
+  {
+    using boost::filesystem::path;
+  #if BOOST_VERSION < 107400
+    // Remove this preprocessor if/else when we are bumping the boost version.
+    boost::filesystem::copy_file(
+        path(from),
+        path(to),
+        boost::filesystem::copy_option::overwrite_if_exists);
+  #else
+    boost::filesystem::copy_file(
+        path(from),
+        path(to),
+        boost::filesystem::copy_options::overwrite_existing);
+  #endif
+  }
+
  std::function<void(int)> signal_handler::m_handler;

  private_file::private_file() noexcept : m_handle(), m_filename() {}
@@ -122,7 +140,7 @@ namespace tools
  private_file::private_file(std::FILE* handle, std::string&& filename) noexcept
    : m_handle(handle), m_filename(std::move(filename)) {}

-  private_file private_file::create(std::string name)
+  private_file private_file::create(std::string name, uint32_t extra_flags)
  {
 #ifdef WIN32
    struct close_handle
@@ -175,7 +193,7 @@ namespace tools
        name.c_str(),
        GENERIC_WRITE, FILE_SHARE_READ,
        std::addressof(attributes),
-        CREATE_NEW, (FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE),
+        CREATE_NEW, (FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE | extra_flags),
        nullptr
      )
    };
@@ -194,7 +212,7 @@ namespace tools
      }
    }
 #else
-    const int fdr = open(name.c_str(), (O_RDONLY | O_CREAT), S_IRUSR);
+    const int fdr = open(name.c_str(), (O_RDONLY | O_CREAT | extra_flags), S_IRUSR);
    if (0 <= fdr)
    {
      struct stat rstats = {};
@@ -225,6 +243,23 @@ namespace tools
    return {};
  }

+  private_file private_file::drop_and_recreate(std::string filename)
+  {
+    if (epee::file_io_utils::is_file_exist(filename)) {
+      boost::system::error_code ec{};
+      boost::filesystem::remove(filename, ec);
+      if (ec) {
+        MERROR("Failed to remove " << filename << ": " << ec.message());
+        return {};
+      }
+    }
+#ifdef WIN32
+    return create(filename);
+#else
+    return create(filename, O_EXCL);
+#endif
+  }
+
  private_file::~private_file() noexcept
  {
    try
@@ -882,13 +917,6 @@ std::string get_nix_version_display_string()

  bool is_local_address(const std::string &address)
  {
-    // always assume Tor/I2P addresses to be untrusted by default
-    if (is_privacy_preserving_network(address))
-    {
-      MDEBUG("Address '" << address << "' is Tor/I2P, non local");
-      return false;
-    }
-
    // extract host
    epee::net_utils::http::url_content u_c;
    if (!epee::net_utils::parse_url(address, u_c))
@@ -902,20 +930,22 @@ std::string get_nix_version_display_string()
      return false;
    }

-    // resolve to IP
-    boost::asio::io_service io_service;
-    boost::asio::ip::tcp::resolver resolver(io_service);
-    boost::asio::ip::tcp::resolver::query query(u_c.host, "");
-    boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query);
-    while (i != boost::asio::ip::tcp::resolver::iterator())
+    if (u_c.host == "localhost" || boost::ends_with(u_c.host, ".localhost")) { // RFC 6761 (6.3)
+      MDEBUG("Address '" << address << "' is local");
+      return true;
+    }
+
+    boost::system::error_code ec;
+    const auto parsed_ip = boost::asio::ip::address::from_string(u_c.host, ec);
+    if (ec) {
+      MDEBUG("Failed to parse '" << address << "' as IP address: " << ec.message() << ". Considering it not local");
+      return false;
+    }
+
+    if (parsed_ip.is_loopback())
    {
-      const boost::asio::ip::tcp::endpoint &ep = *i;
-      if (ep.address().is_loopback())
-      {
-        MDEBUG("Address '" << address << "' is local");
-        return true;
-      }
-      ++i;
+      MDEBUG("Address '" << address << "' is local");
+      return true;
    }

    MDEBUG("Address '" << address << "' is not local");
@@ -67,6 +67,8 @@ namespace tools
    }
  };

+  void copy_file(const std::string& from, const std::string& to);
+
  //! A file restricted to process owner AND process. Deletes file on destruction.
  class private_file {
    std::unique_ptr<std::FILE, close_file> m_handle;
@@ -80,7 +82,11 @@ namespace tools

    /*! \return File only readable by owner and only used by this process
      OR `private_file{}` on error. */
-    static private_file create(std::string filename);
+    static private_file create(std::string filename, uint32_t extra_flags = 0);
+
+    /*! \return Drop and create file only readable by owner and only used
+      by this process OR `private_file{}` on error. */
+    static private_file drop_and_recreate(std::string filename);    

    private_file(private_file&&) = default;
    private_file& operator=(private_file&&) = default;
@@ -30,29 +30,41 @@
 #pragma once

 #ifdef _WIN32
+
 #include <windows.h>
-#define CTHR_MUTEX_TYPE	HANDLE
-#define CTHR_MUTEX_INIT	NULL
-#define CTHR_MUTEX_LOCK(x)	do { if (x == NULL) { \
-    HANDLE p = CreateMutex(NULL, FALSE, NULL); \
-    if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \
-      CloseHandle(p); \
-  } WaitForSingleObject(x, INFINITE); } while(0)
-#define CTHR_MUTEX_UNLOCK(x)	ReleaseMutex(x)
+
+#define CTHR_RWLOCK_TYPE	SRWLOCK
+#define CTHR_RWLOCK_INIT	SRWLOCK_INIT
+#define CTHR_RWLOCK_LOCK_WRITE(x)	AcquireSRWLockExclusive(&x)
+#define CTHR_RWLOCK_UNLOCK_WRITE(x)	ReleaseSRWLockExclusive(&x)
+#define CTHR_RWLOCK_LOCK_READ(x)	AcquireSRWLockShared(&x)
+#define CTHR_RWLOCK_UNLOCK_READ(x)	ReleaseSRWLockShared(&x)
+#define CTHR_RWLOCK_TRYLOCK_READ(x)	TryAcquireSRWLockShared(&x)
+
 #define CTHR_THREAD_TYPE	HANDLE
-#define CTHR_THREAD_RTYPE	void
-#define CTHR_THREAD_RETURN	return
-#define CTHR_THREAD_CREATE(thr, func, arg)	thr = (HANDLE)_beginthread(func, 0, arg)
-#define CTHR_THREAD_JOIN(thr)			WaitForSingleObject(thr, INFINITE)
+#define CTHR_THREAD_RTYPE	unsigned __stdcall
+#define CTHR_THREAD_RETURN	_endthreadex(0); return 0;
+#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L)
+#define CTHR_THREAD_JOIN(thr)			do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0)
+#define CTHR_THREAD_CLOSE(thr)			CloseHandle((HANDLE)thr);
+
 #else
+
 #include <pthread.h>
-#define CTHR_MUTEX_TYPE pthread_mutex_t
-#define CTHR_MUTEX_INIT	PTHREAD_MUTEX_INITIALIZER
-#define CTHR_MUTEX_LOCK(x)	pthread_mutex_lock(&x)
-#define CTHR_MUTEX_UNLOCK(x)	pthread_mutex_unlock(&x)
+
+#define CTHR_RWLOCK_TYPE	pthread_rwlock_t
+#define CTHR_RWLOCK_INIT	PTHREAD_RWLOCK_INITIALIZER
+#define CTHR_RWLOCK_LOCK_WRITE(x)	pthread_rwlock_wrlock(&x)
+#define CTHR_RWLOCK_UNLOCK_WRITE(x)	pthread_rwlock_unlock(&x)
+#define CTHR_RWLOCK_LOCK_READ(x)	pthread_rwlock_rdlock(&x)
+#define CTHR_RWLOCK_UNLOCK_READ(x)	pthread_rwlock_unlock(&x)
+#define CTHR_RWLOCK_TRYLOCK_READ(x)	(pthread_rwlock_tryrdlock(&x) == 0)
+
 #define CTHR_THREAD_TYPE pthread_t
 #define CTHR_THREAD_RTYPE	void *
 #define CTHR_THREAD_RETURN	return NULL
-#define CTHR_THREAD_CREATE(thr, func, arg)	pthread_create(&thr, NULL, func, arg)
+#define CTHR_THREAD_CREATE(thr, func, arg)	(pthread_create(&thr, NULL, func, arg) == 0)
 #define CTHR_THREAD_JOIN(thr)			pthread_join(thr, NULL)
+#define CTHR_THREAD_CLOSE(thr)
+
 #endif
@@ -97,5 +97,9 @@ void rx_slow_hash_allocate_state(void);
 void rx_slow_hash_free_state(void);
 uint64_t rx_seedheight(const uint64_t height);
 void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height);
-void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash, int miners, int is_alt);
-void rx_reorg(const uint64_t split_height);
+
+void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads);
+void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash);
+
+void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads);
+uint32_t rx_get_miner_thread(void);
@@ -34,7 +34,7 @@ typedef struct {
 	unsigned long long databitlen;    /*the message size in bits*/
 	unsigned long long datasize_in_buffer;      /*the size of the message remained in buffer; assumed to be multiple of 8bits except for the last partial block at the end of the message*/
 	DATA_ALIGN16(uint64 x[8][2]);     /*the 1024-bit state, ( x[i][0] || x[i][1] ) is the ith row of the state in the pseudocode*/
-	unsigned char buffer[64];         /*the 512-bit message block to be hashed;*/
+	DATA_ALIGN16(unsigned char buffer[64]);         /*the 512-bit message block to be hashed;*/
 } hashState;


@@ -213,16 +213,24 @@ static void E8(hashState *state)
 /*The compression function F8 */
 static void F8(hashState *state)
 {
-      uint64  i;
+      uint64_t* x = (uint64_t*)state->x;

      /*xor the 512-bit message with the fist half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[i >> 1][i & 1] ^= ((uint64*)state->buffer)[i];
+      for (int i = 0; i < 8; ++i) {
+            uint64 b;
+            memcpy(&b, &state->buffer[i << 3], sizeof(b));
+            x[i] ^= b;
+      }

      /*the bijective function E8 */
      E8(state);

      /*xor the 512-bit message with the second half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[(8+i) >> 1][(8+i) & 1] ^= ((uint64*)state->buffer)[i];
+      for (int i = 0; i < 8; ++i) {
+            uint64 b;
+            memcpy(&b, &state->buffer[i << 3], sizeof(b));
+            x[i + 8] ^= b;
+      }
 }

 /*before hashing a message, initialize the hash state as H0 */
@@ -240,6 +248,7 @@ static HashReturn Init(hashState *state, int hashbitlen)
            case 224: memcpy(state->x,JH224_H0,128); break;
            case 256: memcpy(state->x,JH256_H0,128); break;
            case 384: memcpy(state->x,JH384_H0,128); break;
+            default:
            case 512: memcpy(state->x,JH512_H0,128); break;
      }

@@ -43,32 +43,41 @@

 #define RX_LOGCAT	"randomx"

+// Report large page allocation failures as debug messages
+#define alloc_err_msg(x) mdebug(RX_LOGCAT, x);
+
+static CTHR_RWLOCK_TYPE main_dataset_lock = CTHR_RWLOCK_INIT;
+static CTHR_RWLOCK_TYPE main_cache_lock = CTHR_RWLOCK_INIT;
+
+static randomx_dataset *main_dataset = NULL;
+static randomx_cache *main_cache = NULL;
+static char main_seedhash[HASH_SIZE];
+static int main_seedhash_set = 0;
+
+static CTHR_RWLOCK_TYPE secondary_cache_lock = CTHR_RWLOCK_INIT;
+
+static randomx_cache *secondary_cache = NULL;
+static char secondary_seedhash[HASH_SIZE];
+static int secondary_seedhash_set = 0;
+
 #if defined(_MSC_VER)
 #define THREADV __declspec(thread)
 #else
 #define THREADV __thread
 #endif

-typedef struct rx_state {
-  CTHR_MUTEX_TYPE rs_mutex;
-  char rs_hash[HASH_SIZE];
-  uint64_t  rs_height;
-  randomx_cache *rs_cache;
-} rx_state;
+static THREADV randomx_vm *main_vm_full = NULL;
+static THREADV randomx_vm *main_vm_light = NULL;
+static THREADV randomx_vm *secondary_vm_light = NULL;

-static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT;
-static CTHR_MUTEX_TYPE rx_dataset_mutex = CTHR_MUTEX_INIT;
+static THREADV uint32_t miner_thread = 0;

-static rx_state rx_s[2] = {{CTHR_MUTEX_INIT,{0},0,0},{CTHR_MUTEX_INIT,{0},0,0}};
-
-static randomx_dataset *rx_dataset;
-static int rx_dataset_nomem;
-static int rx_dataset_nolp;
-static uint64_t rx_dataset_height;
-static THREADV randomx_vm *rx_vm = NULL;
+static bool is_main(const char* seedhash) { return main_seedhash_set && (memcmp(seedhash, main_seedhash, HASH_SIZE) == 0); }
+static bool is_secondary(const char* seedhash) { return secondary_seedhash_set && (memcmp(seedhash, secondary_seedhash, HASH_SIZE) == 0); }

 static void local_abort(const char *msg)
 {
+  merror(RX_LOGCAT, "%s", msg);
  fprintf(stderr, "%s\n", msg);
 #ifdef NDEBUG
  _exit(1);
@@ -77,6 +86,16 @@ static void local_abort(const char *msg)
 #endif
 }

+static void hash2hex(const char* hash, char* hex) {
+  const char* d = "0123456789abcdef";
+  for (int i = 0; i < HASH_SIZE; ++i) {
+    const uint8_t b = hash[i];
+    hex[i * 2 + 0] = d[b >> 4];
+    hex[i * 2 + 1] = d[b & 15];
+  }
+  hex[HASH_SIZE * 2] = '\0';
+}
+
 static inline int disabled_flags(void) {
  static int flags = -1;

@@ -157,19 +176,6 @@ static unsigned int get_seedhash_epoch_blocks(void)
  return blocks;
 }

-void rx_reorg(const uint64_t split_height) {
-  int i;
-  CTHR_MUTEX_LOCK(rx_mutex);
-  for (i=0; i<2; i++) {
-    if (split_height <= rx_s[i].rs_height) {
-      if (rx_s[i].rs_height == rx_dataset_height)
-        rx_dataset_height = 1;
-      rx_s[i].rs_height = 1;	/* set to an invalid seed height */
-    }
-  }
-  CTHR_MUTEX_UNLOCK(rx_mutex);
-}
-
 uint64_t rx_seedheight(const uint64_t height) {
  const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag();
  const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks();
@@ -183,6 +189,103 @@ void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nexth
  *nextheight = rx_seedheight(height + get_seedhash_epoch_lag());
 }

+static void rx_alloc_dataset(randomx_flags flags, randomx_dataset** dataset, int ignore_env)
+{
+  if (*dataset) {
+    return;
+  }
+
+  if (disabled_flags() & RANDOMX_FLAG_FULL_MEM) {
+    static int shown = 0;
+    if (!shown) {
+      shown = 1;
+      minfo(RX_LOGCAT, "RandomX dataset is disabled by MONERO_RANDOMX_UMASK environment variable.");
+    }
+    return;
+  }
+
+  if (!ignore_env && !getenv("MONERO_RANDOMX_FULL_MEM")) {
+    static int shown = 0;
+    if (!shown) {
+      shown = 1;
+      minfo(RX_LOGCAT, "RandomX dataset is not enabled by default. Use MONERO_RANDOMX_FULL_MEM environment variable to enable it.");
+    }
+    return;
+  }
+
+  *dataset = randomx_alloc_dataset((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags());
+  if (!*dataset) {
+    alloc_err_msg("Couldn't allocate RandomX dataset using large pages");
+    *dataset = randomx_alloc_dataset(flags & ~disabled_flags());
+    if (!*dataset) {
+      merror(RX_LOGCAT, "Couldn't allocate RandomX dataset");
+    }
+  }
+}
+
+static void rx_alloc_cache(randomx_flags flags, randomx_cache** cache)
+{
+  if (*cache) {
+    return;
+  }
+
+  *cache = randomx_alloc_cache((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags());
+  if (!*cache) {
+    alloc_err_msg("Couldn't allocate RandomX cache using large pages");
+    *cache = randomx_alloc_cache(flags & ~disabled_flags());
+    if (!*cache) local_abort("Couldn't allocate RandomX cache");
+  }
+}
+
+static void rx_init_full_vm(randomx_flags flags, randomx_vm** vm)
+{
+  if (*vm || !main_dataset || (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) {
+    return;
+  }
+
+  if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) {
+    flags |= RANDOMX_FLAG_SECURE;
+  }
+
+  *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset);
+  if (!*vm) {
+    static int shown = 0;
+    if (!shown) {
+        shown = 1;
+        alloc_err_msg("Couldn't allocate RandomX full VM using large pages (will print only once)");
+    }
+    *vm = randomx_create_vm((flags | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset);
+    if (!*vm) {
+      merror(RX_LOGCAT, "Couldn't allocate RandomX full VM");
+    }
+  }
+}
+
+static void rx_init_light_vm(randomx_flags flags, randomx_vm** vm, randomx_cache* cache)
+{
+  if (*vm) {
+    randomx_vm_set_cache(*vm, cache);
+    return;
+  }
+
+  if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) {
+    flags |= RANDOMX_FLAG_SECURE;
+  }
+
+  flags &= ~RANDOMX_FLAG_FULL_MEM;
+
+  *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags(), cache, NULL);
+  if (!*vm) {
+    static int shown = 0;
+    if (!shown) {
+        shown = 1;
+        alloc_err_msg("Couldn't allocate RandomX light VM using large pages (will print only once)");
+    }
+    *vm = randomx_create_vm(flags & ~disabled_flags(), cache, NULL);
+    if (!*vm) local_abort("Couldn't allocate RandomX light VM");
+  }
+}
+
 typedef struct seedinfo {
  randomx_cache *si_cache;
  unsigned long si_start;
@@ -191,187 +294,231 @@ typedef struct seedinfo {

 static CTHR_THREAD_RTYPE rx_seedthread(void *arg) {
  seedinfo *si = arg;
-  randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count);
+  randomx_init_dataset(main_dataset, si->si_cache, si->si_start, si->si_count);
  CTHR_THREAD_RETURN;
 }

-static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_t seedheight) {
-  if (miners > 1) {
-    unsigned long delta = randomx_dataset_item_count() / miners;
-    unsigned long start = 0;
-    int i;
-    seedinfo *si;
-    CTHR_THREAD_TYPE *st;
-    si = malloc(miners * sizeof(seedinfo));
-    if (si == NULL)
-      local_abort("Couldn't allocate RandomX mining threadinfo");
-    st = malloc(miners * sizeof(CTHR_THREAD_TYPE));
-    if (st == NULL) {
-      free(si);
-      local_abort("Couldn't allocate RandomX mining threadlist");
-    }
-    for (i=0; i<miners-1; i++) {
-      si[i].si_cache = rs_cache;
-      si[i].si_start = start;
-      si[i].si_count = delta;
-      start += delta;
-    }
-    si[i].si_cache = rs_cache;
+static void rx_init_dataset(size_t max_threads) {
+  if (!main_dataset) {
+    return;
+  }
+
+  // leave 2 CPU cores for other tasks
+  const size_t num_threads = (max_threads < 4) ? 1 : (max_threads - 2);
+  seedinfo* si = malloc(num_threads * sizeof(seedinfo));
+  if (!si) local_abort("Couldn't allocate RandomX mining threadinfo");
+
+  const uint32_t delta = randomx_dataset_item_count() / num_threads;
+  uint32_t start = 0;
+
+  const size_t n1 = num_threads - 1;
+  for (size_t i = 0; i < n1; ++i) {
+    si[i].si_cache = main_cache;
    si[i].si_start = start;
-    si[i].si_count = randomx_dataset_item_count() - start;
-    for (i=1; i<miners; i++) {
-      CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i]);
-    }
-    randomx_init_dataset(rx_dataset, rs_cache, 0, si[0].si_count);
-    for (i=1; i<miners; i++) {
-      CTHR_THREAD_JOIN(st[i]);
-    }
-    free(st);
-    free(si);
-  } else {
-    randomx_init_dataset(rx_dataset, rs_cache, 0, randomx_dataset_item_count());
+    si[i].si_count = delta;
+    start += delta;
  }
-  rx_dataset_height = seedheight;
+
+  si[n1].si_cache = main_cache;
+  si[n1].si_start = start;
+  si[n1].si_count = randomx_dataset_item_count() - start;
+
+  CTHR_THREAD_TYPE *st = malloc(num_threads * sizeof(CTHR_THREAD_TYPE));
+  if (!st) local_abort("Couldn't allocate RandomX mining threadlist");
+
+  CTHR_RWLOCK_LOCK_READ(main_cache_lock);
+  for (size_t i = 0; i < n1; ++i) {
+    if (!CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i])) {
+      local_abort("Couldn't start RandomX seed thread");
+    }
+  }
+  randomx_init_dataset(main_dataset, si[n1].si_cache, si[n1].si_start, si[n1].si_count);
+  for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]);
+  CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
+
+  free(st);
+  free(si);
+
+  minfo(RX_LOGCAT, "RandomX dataset initialized");
 }

-void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length,
-  char *hash, int miners, int is_alt) {
-  uint64_t s_height = rx_seedheight(mainheight);
-  int toggle = (s_height & get_seedhash_epoch_blocks()) != 0;
-  randomx_flags flags = enabled_flags() & ~disabled_flags();
-  rx_state *rx_sp;
-  randomx_cache *cache;
+typedef struct thread_info {
+  char seedhash[HASH_SIZE];
+  size_t max_threads;
+} thread_info;

-  CTHR_MUTEX_LOCK(rx_mutex);
+static CTHR_THREAD_RTYPE rx_set_main_seedhash_thread(void *arg) {
+  thread_info* info = arg;

-  /* if alt block but with same seed as mainchain, no need for alt cache */
-  if (is_alt) {
-    if (s_height == seedheight && !memcmp(rx_s[toggle].rs_hash, seedhash, HASH_SIZE))
-      is_alt = 0;
-  } else {
-  /* RPC could request an earlier block on mainchain */
-    if (s_height > seedheight)
-      is_alt = 1;
-    /* miner can be ahead of mainchain */
-    else if (s_height < seedheight)
-      toggle ^= 1;
+  CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock);
+  CTHR_RWLOCK_LOCK_WRITE(main_cache_lock);
+
+  // Double check that seedhash wasn't already updated
+  if (is_main(info->seedhash)) {
+    CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock);
+    CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+    free(info);
+    CTHR_THREAD_RETURN;
+  }
+  memcpy(main_seedhash, info->seedhash, HASH_SIZE);
+  main_seedhash_set = 1;
+
+  char buf[HASH_SIZE * 2 + 1];
+  hash2hex(main_seedhash, buf);
+  minfo(RX_LOGCAT, "RandomX new main seed hash is %s", buf);
+
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  rx_alloc_dataset(flags, &main_dataset, 0);
+  rx_alloc_cache(flags, &main_cache);
+
+  randomx_init_cache(main_cache, info->seedhash, HASH_SIZE);
+  minfo(RX_LOGCAT, "RandomX main cache initialized");
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock);
+
+  // From this point, rx_slow_hash can calculate hashes in light mode, but dataset is not initialized yet
+  rx_init_dataset(info->max_threads);
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+
+  free(info);
+  CTHR_THREAD_RETURN;
+}
+
+void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads) {
+  // Early out if seedhash didn't change
+  if (is_main(seedhash)) {
+    return;
  }

-  toggle ^= (is_alt != 0);
+  // Update main cache and dataset in the background
+  thread_info* info = malloc(sizeof(thread_info));
+  if (!info) local_abort("Couldn't allocate RandomX mining threadinfo");

-  rx_sp = &rx_s[toggle];
-  CTHR_MUTEX_LOCK(rx_sp->rs_mutex);
-  CTHR_MUTEX_UNLOCK(rx_mutex);
+  memcpy(info->seedhash, seedhash, HASH_SIZE);
+  info->max_threads = max_dataset_init_threads;

-  cache = rx_sp->rs_cache;
-  if (cache == NULL) {
-    if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) {
-      cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES);
-      if (cache == NULL) {
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache");
-      }
-    }
-    if (cache == NULL) {
-      cache = randomx_alloc_cache(flags);
-      if (cache == NULL)
-        local_abort("Couldn't allocate RandomX cache");
-    }
+  CTHR_THREAD_TYPE t;
+  if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) {
+    local_abort("Couldn't start RandomX seed thread");
  }
-  if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) {
-    randomx_init_cache(cache, seedhash, HASH_SIZE);
-    rx_sp->rs_cache = cache;
-    rx_sp->rs_height = seedheight;
-    memcpy(rx_sp->rs_hash, seedhash, HASH_SIZE);
-  }
-  if (rx_vm == NULL) {
-    if ((flags & RANDOMX_FLAG_JIT) && !miners) {
-        flags |= RANDOMX_FLAG_SECURE & ~disabled_flags();
-    }
-    if (miners && (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) {
-      miners = 0;
-    }
-    if (miners) {
-      CTHR_MUTEX_LOCK(rx_dataset_mutex);
-      if (!rx_dataset_nomem) {
-        if (rx_dataset == NULL) {
-          if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) {
-            rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES);
-            if (rx_dataset == NULL) {
-              mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset");
-            }
-          }
-          if (rx_dataset == NULL)
-            rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT);
-          if (rx_dataset != NULL)
-            rx_initdata(rx_sp->rs_cache, miners, seedheight);
+  CTHR_THREAD_CLOSE(t);
+}
+
+void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) {
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  int success = 0;
+
+  // Fast path (seedhash == main_seedhash)
+  // Multiple threads can run in parallel in fast or light mode, 1-2 ms or 10-15 ms per hash per thread
+  if (is_main(seedhash)) {
+    // If CTHR_RWLOCK_TRYLOCK_READ fails it means dataset is being initialized now, so use the light mode
+    if (main_dataset && CTHR_RWLOCK_TRYLOCK_READ(main_dataset_lock)) {
+      // Double check that main_seedhash didn't change
+      if (is_main(seedhash)) {
+        rx_init_full_vm(flags, &main_vm_full);
+        if (main_vm_full) {
+          randomx_calculate_hash(main_vm_full, data, length, result_hash);
+          success = 1;
        }
      }
-      if (rx_dataset != NULL)
-        flags |= RANDOMX_FLAG_FULL_MEM;
-      else {
-        miners = 0;
-        if (!rx_dataset_nomem) {
-          rx_dataset_nomem = 1;
-          mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner");
-        }
+      CTHR_RWLOCK_UNLOCK_READ(main_dataset_lock);
+    } else {
+      CTHR_RWLOCK_LOCK_READ(main_cache_lock);
+      // Double check that main_seedhash didn't change
+      if (is_main(seedhash)) {
+        rx_init_light_vm(flags, &main_vm_light, main_cache);
+        randomx_calculate_hash(main_vm_light, data, length, result_hash);
+        success = 1;
      }
-      CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
+      CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
    }
-    if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES) && !rx_dataset_nolp) {
-      rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset);
-      if(rx_vm == NULL) { //large pages failed
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM");
-        rx_dataset_nolp = 1;
-      }
-    }
-    if (rx_vm == NULL)
-      rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
-    if(rx_vm == NULL) {//fallback if everything fails
-      flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0);
-      rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
-    }
-    if (rx_vm == NULL)
-      local_abort("Couldn't allocate RandomX VM");
-  } else if (miners) {
-    CTHR_MUTEX_LOCK(rx_dataset_mutex);
-    if (rx_dataset != NULL && rx_dataset_height != seedheight)
-      rx_initdata(cache, miners, seedheight);
-    else if (rx_dataset == NULL) {
-      /* this is a no-op if the cache hasn't changed */
-      randomx_vm_set_cache(rx_vm, rx_sp->rs_cache);
-    }
-    CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
-  } else {
-    /* this is a no-op if the cache hasn't changed */
-    randomx_vm_set_cache(rx_vm, rx_sp->rs_cache);
  }
-  /* mainchain users can run in parallel */
-  if (!is_alt)
-    CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex);
-  randomx_calculate_hash(rx_vm, data, length, hash);
-  /* altchain slot users always get fully serialized */
-  if (is_alt)
-    CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex);
+
+  if (success) {
+    return;
+  }
+
+  char buf[HASH_SIZE * 2 + 1];
+
+  // Slow path (seedhash != main_seedhash, but seedhash == secondary_seedhash)
+  // Multiple threads can run in parallel in light mode, 10-15 ms per hash per thread
+  if (!secondary_cache) {
+    CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock);
+    if (!secondary_cache) {
+      hash2hex(seedhash, buf);
+      minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf);
+
+      rx_alloc_cache(flags, &secondary_cache);
+      randomx_init_cache(secondary_cache, seedhash, HASH_SIZE);
+      minfo(RX_LOGCAT, "RandomX secondary cache updated");
+      memcpy(secondary_seedhash, seedhash, HASH_SIZE);
+      secondary_seedhash_set = 1;
+    }
+    CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock);
+  }
+
+  CTHR_RWLOCK_LOCK_READ(secondary_cache_lock);
+  if (is_secondary(seedhash)) {
+    rx_init_light_vm(flags, &secondary_vm_light, secondary_cache);
+    randomx_calculate_hash(secondary_vm_light, data, length, result_hash);
+    success = 1;
+  }
+  CTHR_RWLOCK_UNLOCK_READ(secondary_cache_lock);
+
+  if (success) {
+    return;
+  }
+
+  // Slowest path (seedhash != main_seedhash, seedhash != secondary_seedhash)
+  // Only one thread runs at a time and updates secondary_seedhash if needed, up to 200-500 ms per hash
+  CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock);
+  if (!is_secondary(seedhash)) {
+    hash2hex(seedhash, buf);
+    minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf);
+
+    randomx_init_cache(secondary_cache, seedhash, HASH_SIZE);
+    minfo(RX_LOGCAT, "RandomX secondary cache updated");
+    memcpy(secondary_seedhash, seedhash, HASH_SIZE);
+    secondary_seedhash_set = 1;
+  }
+  rx_init_light_vm(flags, &secondary_vm_light, secondary_cache);
+  randomx_calculate_hash(secondary_vm_light, data, length, result_hash);
+  CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock);
 }

-void rx_slow_hash_allocate_state(void) {
+void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads) {
+  miner_thread = value;
+
+  // If dataset is not allocated yet, try to allocate and initialize it
+  CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock);
+  if (main_dataset) {
+    CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+    return;
+  }
+
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  rx_alloc_dataset(flags, &main_dataset, 1);
+  rx_init_dataset(max_dataset_init_threads);
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
 }

-void rx_slow_hash_free_state(void) {
-  if (rx_vm != NULL) {
-    randomx_destroy_vm(rx_vm);
-    rx_vm = NULL;
+uint32_t rx_get_miner_thread() {
+  return miner_thread;
+}
+
+void rx_slow_hash_allocate_state() {}
+
+static void rx_destroy_vm(randomx_vm** vm) {
+  if (*vm) {
+    randomx_destroy_vm(*vm);
+    *vm = NULL;
  }
 }

-void rx_stop_mining(void) {
-  CTHR_MUTEX_LOCK(rx_dataset_mutex);
-  if (rx_dataset != NULL) {
-    randomx_dataset *rd = rx_dataset;
-    rx_dataset = NULL;
-    randomx_release_dataset(rd);
-  }
-  rx_dataset_nomem = 0;
-  rx_dataset_nolp = 0;
-  CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
+void rx_slow_hash_free_state() {
+  rx_destroy_vm(&main_vm_full);
+  rx_destroy_vm(&main_vm_light);
+  rx_destroy_vm(&secondary_vm_light);
 }
@@ -1229,7 +1229,7 @@ namespace cryptonote
    char *end = NULL;
    errno = 0;
    const unsigned long long ull = strtoull(buf, &end, 10);
-    CHECK_AND_ASSERT_THROW_MES(ull != ULONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
+    CHECK_AND_ASSERT_THROW_MES(ull != ULLONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
    CHECK_AND_ASSERT_THROW_MES(ull != 0 || amount == 0, "Overflow in rounding");
    return ull;
  }
@@ -231,6 +231,11 @@ namespace cryptonote
     */
    uint64_t get_window_size() const { return window_size; }

+    /**
+     * @brief returns info for all known hard forks
+     */
+    const std::vector<hardfork_t>& get_hardforks() const { return heights; }
+
  private:

    uint8_t get_block_version(uint64_t height) const;
@@ -82,6 +82,7 @@
 using namespace epee;

 #include "miner.h"
+#include "crypto/hash.h"


 extern "C" void slow_hash_allocate_state();
@@ -436,7 +437,6 @@ namespace cryptonote
  {
    m_stop = true;
  }
-  extern "C" void rx_stop_mining(void);
  //-----------------------------------------------------------------------------------------------------
  bool miner::stop()
  {
@@ -469,7 +469,6 @@ namespace cryptonote
    MINFO("Mining has been stopped, " << m_threads.size() << " finished" );
    m_threads.clear();
    m_threads_autodetect.clear();
-    rx_stop_mining();
    return true;
  }
  //-----------------------------------------------------------------------------------------------------
@@ -524,6 +523,8 @@ namespace cryptonote
  bool miner::worker_thread()
  {
    const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment
+    bool rx_set = false;
+
    MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]");
    MGINFO("Miner thread was started ["<< th_local_index << "]");
    uint32_t nonce = m_starter_nonce + th_local_index;
@@ -574,6 +575,13 @@ namespace cryptonote

      b.nonce = nonce;
      crypto::hash h;
+
+      if ((b.major_version >= RX_BLOCK_VERSION) && !rx_set)
+      {
+        crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
+        rx_set = true;
+      }
+
      m_gbh(b, height, NULL, tools::get_max_concurrency(), h);

      if(check_hash(h, local_diff))
@@ -52,7 +52,7 @@ namespace cryptonote

    // load
    template <template <bool> class Archive>
-    bool do_serialize(Archive<false>& ar)
+    bool member_do_serialize(Archive<false>& ar)
    {
      // size - 1 - because of variant tag
      for (size = 1; size <= TX_EXTRA_PADDING_MAX_COUNT; ++size)
@@ -73,7 +73,7 @@ namespace cryptonote

    // store
    template <template <bool> class Archive>
-    bool do_serialize(Archive<true>& ar)
+    bool member_do_serialize(Archive<true>& ar)
    {
      if(TX_EXTRA_PADDING_MAX_COUNT < size)
        return false;
@@ -129,7 +129,7 @@ namespace cryptonote

    // load
    template <template <bool> class Archive>
-    bool do_serialize(Archive<false>& ar)
+    bool member_do_serialize(Archive<false>& ar)
    {
      std::string field;
      if(!::do_serialize(ar, field))
@@ -142,7 +142,7 @@ namespace cryptonote

    // store
    template <template <bool> class Archive>
-    bool do_serialize(Archive<true>& ar)
+    bool member_do_serialize(Archive<true>& ar)
    {
      std::ostringstream oss;
      binary_archive<true> oar(oss);
@@ -42,7 +42,12 @@ namespace cryptonote
    static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none");

    relay_method m_relay; // gives indication on how tx should be relayed (if at all)
-    bool m_verifivation_failed; //bad tx, should drop connection
+    bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense
+    // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop
+    // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be
+    // confused with breaking soft fork rules, because tx could be later added to the chain if mined
+    // because it does not violate consensus rules.
+    bool m_no_drop_offense;
    bool m_verifivation_impossible; //the transaction is related with an alternative blockchain
    bool m_added_to_pool; 
    bool m_low_mixin;
@@ -53,6 +58,8 @@ namespace cryptonote
    bool m_overspend;
    bool m_fee_too_low;
    bool m_too_few_outputs;
+    bool m_tx_extra_too_big;
+    bool m_nonzero_unlock_time;
  };

  struct block_verification_context
@@ -30,6 +30,7 @@

 #pragma once

+#include <cstdint>
 #include <stdexcept>
 #include <string>
 #include <boost/uuid/uuid.hpp>
@@ -206,6 +207,11 @@

 #define DNS_BLOCKLIST_LIFETIME (86400 * 8)

+//The limit is enough for the mandatory transaction content with 16 outputs (547 bytes),
+//a custom tag (1 byte) and up to 32 bytes of custom data for each recipient.
+// (1+32) + (1+1+16*32) + (1+16*32) = 1060
+#define MAX_TX_EXTRA_SIZE                       1060
+
 // New constants are intended to go here
 namespace config
 {
@@ -248,6 +254,7 @@ namespace config
  const unsigned char HASH_KEY_MM_SLOT = 'm';
  const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS_SEED[] = "multisig_tx_privkeys_seed";
  const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS[] = "multisig_tx_privkeys";
+  const constexpr char HASH_KEY_TXHASH_AND_MIXRING[] = "txhash_and_mixring";

  // Multisig
  const uint32_t MULTISIG_MAX_SIGNERS{16};
@@ -31,7 +31,9 @@ set(cryptonote_core_sources
  cryptonote_core.cpp
  tx_pool.cpp
  tx_sanity_check.cpp
-  cryptonote_tx_utils.cpp)
+  cryptonote_tx_utils.cpp
+  tx_verification_utils.cpp
+)

 set(cryptonote_core_headers)

@@ -57,6 +57,7 @@
 #include "common/notify.h"
 #include "common/varint.h"
 #include "common/pruning.h"
+#include "common/data_cache.h"
 #include "time_helper.h"

 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -98,7 +99,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) :
  m_difficulty_for_next_block(1),
  m_btc_valid(false),
  m_batch_success(true),
-  m_prepare_height(0)
+  m_prepare_height(0),
+  m_rct_ver_cache()
 {
  LOG_PRINT_L3("Blockchain::" << __func__);
 }
@@ -456,6 +458,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline
    if (!update_next_cumulative_weight_limit())
      return false;
  }
+
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  {
+    const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height()));
+    if (seedhash != crypto::null_hash)
+      rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+  }
+
  return true;
 }
 //------------------------------------------------------------------
@@ -570,6 +580,12 @@ void Blockchain::pop_blocks(uint64_t nblocks)

  if (stop_batch)
    m_db->batch_stop();
+
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  {
+    const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height()));
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+  }
 }
 //------------------------------------------------------------------
 // This function tells BlockchainDB to remove the top block from the
@@ -1239,18 +1255,20 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
  }

  m_hardfork->reorganize_from_chain_height(split_height);
-  get_block_longhash_reorg(split_height);

  std::shared_ptr<tools::Notify> reorg_notify = m_reorg_notify;
  if (reorg_notify)
    reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(),
        "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL);

+  const uint64_t new_height = m_db->height();
+  const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+
  crypto::hash prev_id;
  if (!get_block_hash(alt_chain.back().bl, prev_id))
    MERROR("Failed to get block hash of an alternative chain's tip");
  else
-    send_miner_notifications(prev_id, alt_chain.back().already_generated_coins);
+    send_miner_notifications(new_height, seedhash, prev_id, alt_chain.back().already_generated_coins);

  for (const auto& notifier : m_block_notifiers)
  {
@@ -1262,6 +1280,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
    }
  }

+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+
  MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height());
  return true;
 }
@@ -2001,7 +2022,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
      {
        seedhash = get_block_id_by_height(seedheight);
      }
-      get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash);
+      get_altblock_longhash(bei.bl, proof_of_work, seedhash);
    } else
    {
      get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0);
@@ -2044,7 +2065,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
      cryptonote::blobdata blob;
      if (m_tx_pool.have_tx(txid, relay_category::legacy))
      {
-        if (m_tx_pool.get_transaction_info(txid, td))
+        if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
        {
          bei.block_cumulative_weight += td.weight;
        }
@@ -3192,7 +3213,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
  }
  return false;
 }
-bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const
+bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys)
 {
  PERF_TIMER(expand_transaction_2);
  CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2");
@@ -3434,7 +3455,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
  std::vector < uint64_t > results;
  results.resize(tx.vin.size(), 0);

-  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
  tools::threadpool::waiter waiter(tpool);
  int threads = tpool.get_max_concurrency();

@@ -3515,6 +3536,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
        false, "Transaction spends at least one output which is too young");
  }

+  // Warn that new RCT types are present, and thus the cache is not being used effectively
+  static constexpr const std::uint8_t RCT_CACHE_TYPE = rct::RCTTypeBulletproofPlus;
+  if (tx.rct_signatures.type > RCT_CACHE_TYPE)
+  {
+    MWARNING("RCT cache is not caching new verification results. Please update RCT_CACHE_TYPE!");
+  }
+
  if (tx.version == 1)
  {
    if (threads > 1)
@@ -3536,12 +3564,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
  }
  else
  {
-    if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
-    {
-      MERROR_VER("Failed to expand rct signatures!");
-      return false;
-    }
-
    // from version 2, check ringct signatures
    // obviously, the original and simple rct APIs use a mixRing that's indexes
    // in opposite orders, because it'd be too simple otherwise...
@@ -3559,61 +3581,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
    case rct::RCTTypeCLSAG:
    case rct::RCTTypeBulletproofPlus:
    {
-      // check all this, either reconstructed (so should really pass), or not
-      {
-        if (pubkeys.size() != rv.mixRing.size())
-        {
-          MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-          return false;
-        }
-        for (size_t i = 0; i < pubkeys.size(); ++i)
-        {
-          if (pubkeys[i].size() != rv.mixRing[i].size())
-          {
-            MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-            return false;
-          }
-        }
-
-        for (size_t n = 0; n < pubkeys.size(); ++n)
-        {
-          for (size_t m = 0; m < pubkeys[n].size(); ++m)
-          {
-            if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[n][m].dest))
-            {
-              MERROR_VER("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
-              return false;
-            }
-            if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[n][m].mask))
-            {
-              MERROR_VER("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
-              return false;
-            }
-          }
-        }
-      }
-
-      const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size();
-      if (n_sigs != tx.vin.size())
-      {
-        MERROR_VER("Failed to check ringct signatures: mismatched MGs/vin sizes");
-        return false;
-      }
-      for (size_t n = 0; n < tx.vin.size(); ++n)
-      {
-        bool error;
-        if (rct::is_rct_clsag(rv.type))
-          error = memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.CLSAGs[n].I, 32);
-        else
-          error = rv.p.MGs[n].II.empty() || memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.MGs[n].II[0], 32);
-        if (error)
-        {
-          MERROR_VER("Failed to check ringct signatures: mismatched key image");
-          return false;
-        }
-      }
-
-      if (!rct::verRctNonSemanticsSimple(rv))
+      if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, RCT_CACHE_TYPE))
      {
        MERROR_VER("Failed to check ringct signatures!");
        return false;
@@ -3622,6 +3590,12 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
    }
    case rct::RCTTypeFull:
    {
+      if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
+      {
+        MERROR_VER("Failed to expand rct signatures!");
+        return false;
+      }
+
      // check all this, either reconstructed (so should really pass), or not
      {
        bool size_matches = true;
@@ -3736,7 +3710,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b
      div128_64(hi, lo, median_block_weight, &hi, &lo, NULL, NULL);
      assert(hi == 0);
      lo -= lo / 20;
-      return lo;
+      return lo == 0 ? 1 : lo;
    }
    else
    {
@@ -3858,6 +3832,8 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block
  const uint8_t version = get_current_hard_fork_version();
  const uint64_t db_height = m_db->height();

+  CHECK_AND_ASSERT_THROW_MES(grace_blocks <= CRYPTONOTE_REWARD_BLOCKS_WINDOW, "Grace blocks invalid In 2021 fee scaling estimate.");
+
  // we want Mlw = median of max((min(Mbw, 1.7 * Ml), Zm), Ml / 1.7)
  // Mbw: block weight for the last 99990 blocks, 0 for the next 10
  // Ml: penalty free zone (dynamic), aka long_term_median, aka median of max((min(Mb, 1.7 * Ml), Zm), Ml / 1.7)
@@ -3871,7 +3847,6 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block
  const uint64_t Mlw_penalty_free_zone_for_wallet = std::max<uint64_t>(rm.median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5);

  // Msw: median over [100 - grace blocks] past + [grace blocks] future blocks
-  CHECK_AND_ASSERT_THROW_MES(grace_blocks <= 100, "Grace blocks invalid In 2021 fee scaling estimate.");
  std::vector<uint64_t> weights;
  get_last_n_blocks_weights(weights, 100 - grace_blocks);
  weights.reserve(100);
@@ -4552,11 +4527,15 @@ leave:
    }
  }

-  send_miner_notifications(id, already_generated_coins);
+  const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+  send_miner_notifications(new_height, seedhash, id, already_generated_coins);

  for (const auto& notifier: m_block_notifiers)
    notifier(new_height - 1, {std::addressof(bl), 1});

+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+
  return true;
 }
 //------------------------------------------------------------------
@@ -4638,40 +4617,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
  }
  else
  {
-    const uint64_t block_weight = m_db->get_block_weight(db_height - 1);
+    const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
+    const uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks);

-    uint64_t long_term_median;
-    if (db_height == 1)
-    {
-      long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
-    }
-    else
-    {
-      uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
-      if (nblocks == db_height)
-        --nblocks;
-      long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks);
-    }
-
-    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
-
-    uint64_t short_term_constraint = m_long_term_effective_median_block_weight;
-    if (hf_version >= HF_VERSION_2021_SCALING)
-      short_term_constraint += m_long_term_effective_median_block_weight * 7 / 10;
-    else
-      short_term_constraint += m_long_term_effective_median_block_weight * 2 / 5;
-    uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint);
-
-    if (db_height == 1)
-    {
-      long_term_median = long_term_block_weight;
-    }
-    else
-    {
-      m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1);
-      m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight);
-      long_term_median = m_long_term_block_weights_cache_rolling_median.median();
-    }
    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);

    std::vector<uint64_t> weights;
@@ -5137,7 +5085,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
    return true;

  bool blocks_exist = false;
-  tools::threadpool& tpool = tools::threadpool::getInstance();
+  tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
  unsigned threads = tpool.get_max_concurrency();
  blocks.resize(blocks_entry.size());

@@ -5604,7 +5552,7 @@ void Blockchain::cancel()
 }

 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "a8b24ef4eeea7241b374d4526a3f7c351b53abe7006a3d7eee02ce0af2cc6d66";
+static const char expected_block_hashes_hash[] = "8ada865350270fd008397684d978dac75ea4029a8a1ffcaa9975c43be119ec19";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
  if (get_checkpoints == nullptr || !m_fast_sync)
@@ -5761,24 +5709,15 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_
  m_btc_valid = true;
 }

-void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins)
+void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins)
 {
  if (m_miner_notifiers.empty())
    return;

-  const uint64_t height = m_db->height();
  const uint8_t major_version = m_hardfork->get_ideal_version(height);
  const difficulty_type diff = get_difficulty_for_next_block();
  const uint64_t median_weight = m_current_block_cumul_weight_median;

-  crypto::hash seed_hash = crypto::null_hash;
-  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
-  {
-    uint64_t seed_height, next_height;
-    crypto::rx_seedheights(height, &seed_height, &next_height);
-    seed_hash = get_block_id_by_height(seed_height);
-  }
-
  std::vector<tx_block_template_backlog_entry> tx_backlog;
  m_tx_pool.get_block_template_backlog(tx_backlog);

@@ -57,6 +57,7 @@
 #include "rpc/core_rpc_server_commands_defs.h"
 #include "cryptonote_basic/difficulty.h"
 #include "cryptonote_tx_utils.h"
+#include "tx_verification_utils.h"
 #include "cryptonote_basic/verification_context.h"
 #include "crypto/hash.h"
 #include "checkpoints/checkpoints.h"
@@ -158,6 +159,13 @@ namespace cryptonote
     */
    bool deinit();

+    /**
+     * @brief get a set of blockchain checkpoint hashes
+     *
+     * @return set of blockchain checkpoint hashes
+     */
+    const checkpoints& get_checkpoints() const { return m_checkpoints; }
+
    /**
     * @brief assign a set of blockchain checkpoint hashes
     *
@@ -589,6 +597,15 @@ namespace cryptonote
     */
    bool store_blockchain();

+    /**
+     * @brief expands v2 transaction data from blockchain
+     *
+     * RingCT transactions do not transmit some of their data if it
+     * can be reconstituted by the receiver. This function expands
+     * that implicit data.
+     */
+    static bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys);
+
    /**
     * @brief validates a transaction's inputs
     *
@@ -892,6 +909,13 @@ namespace cryptonote
     */
    uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return m_hardfork->get_earliest_ideal_height_for_version(version); }

+    /**
+     * @brief returns info for all known hard forks
+     *
+     * @return the hardforks
+     */
+    const std::vector<hardfork_t>& get_hardforks() const { return m_hardfork->get_hardforks(); }
+
    /**
     * @brief get information about hardfork voting for a version
     *
@@ -1208,6 +1232,9 @@ namespace cryptonote
    uint64_t m_prepare_nblocks;
    std::vector<block> *m_prepare_blocks;

+    // cache for verifying transaction RCT non semantics
+    mutable rct_ver_cache_t m_rct_ver_cache;
+
    /**
     * @brief collects the keys for all outputs being "spent" as an input
     *
@@ -1560,15 +1587,6 @@ namespace cryptonote
     */
    void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints);

-    /**
-     * @brief expands v2 transaction data from blockchain
-     *
-     * RingCT transactions do not transmit some of their data if it
-     * can be reconstituted by the receiver. This function expands
-     * that implicit data.
-     */
-    bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const;
-
    /**
     * @brief invalidates any cached block template
     */
@@ -1584,9 +1602,11 @@ namespace cryptonote
    /**
     * @brief sends new block notifications to ZMQ `miner_data` subscribers
     *
+     * @param height current blockchain height
+     * @param seed_hash seed hash to use for mining
     * @param prev_id hash of new blockchain tip
     * @param already_generated_coins total coins mined by the network so far
     */
-    void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins);
+    void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins);
  };
 }  // namespace cryptonote
@@ -252,6 +252,10 @@ namespace cryptonote
      m_pprotocol = &m_protocol_stub;
  }
  //-----------------------------------------------------------------------------------
+  const checkpoints& core::get_checkpoints() const
+  {
+    return m_blockchain_storage.get_checkpoints();
+  }
  void core::set_checkpoints(checkpoints&& chk_pts)
  {
    m_blockchain_storage.set_checkpoints(std::move(chk_pts));
@@ -1009,7 +1013,7 @@ namespace cryptonote

    CRITICAL_REGION_LOCAL(m_incoming_tx_lock);

-    tools::threadpool& tpool = tools::threadpool::getInstance();
+    tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
    tools::threadpool::waiter waiter(tpool);
    epee::span<tx_blob_entry>::const_iterator it = tx_blobs.begin();
    for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
@@ -1095,7 +1099,7 @@ namespace cryptonote
      else if(tvc[i].m_verifivation_impossible)
      {MERROR_VER("Transaction verification impossible: " << results[i].hash);}

-      if(tvc[i].m_added_to_pool)
+      if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE)
      {
        MDEBUG("tx added: " << results[i].hash);
        valid_events = true;
@@ -1652,10 +1656,6 @@ namespace cryptonote
    if (((size_t)-1) <= 0xffffffff && block_blob.size() >= 0x3fffffff)
      MWARNING("This block's size is " << block_blob.size() << ", closing on the 32 bit limit");

-    // load json & DNS checkpoints every 10min/hour respectively,
-    // and verify them with respect to what blocks we already have
-    CHECK_AND_ASSERT_MES(update_checkpoints(), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
-
    block lb;
    if (!b)
    {
@@ -1727,6 +1727,11 @@ namespace cryptonote
    return true;
  }
  //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
+  {
+    return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
+  }
+  //-----------------------------------------------------------------------------------------------
  bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const
  {
    m_mempool.get_transactions(txs, include_sensitive_data);
@@ -1739,6 +1744,11 @@ namespace cryptonote
    return true;
  }
  //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental);
+  }
+  //-----------------------------------------------------------------------------------------------
  bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const
  {
    m_mempool.get_transaction_stats(stats, include_sensitive_data);
@@ -436,6 +436,13 @@ namespace cryptonote
      */
     void set_cryptonote_protocol(i_cryptonote_protocol* pprotocol);

+     /**
+      * @copydoc Blockchain::get_checkpoints
+      *
+      * @note see Blockchain::get_checkpoints()
+      */
+     const checkpoints& get_checkpoints() const;
+
     /**
      * @copydoc Blockchain::set_checkpoints
      *
@@ -503,6 +510,23 @@ namespace cryptonote
     bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const;

     /**
+      * @copydoc tx_memory_pool::get_pool_transactions_info
+      * @param include_sensitive_txes include private transactions
+      *
+      * @note see tx_memory_pool::get_pool_transactions_info
+      */
+     bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_pool_info
+      * @param include_sensitive_txes include private transactions
+      * @param max_tx_count max allowed added_txs in response
+      *
+      * @note see tx_memory_pool::get_pool_info
+      */
+     bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
+    /**
      * @copydoc tx_memory_pool::get_transactions
      * @param include_sensitive_txes include private transactions
      *
@@ -203,7 +203,7 @@ namespace cryptonote
    return addr.m_view_public_key;
  }
  //---------------------------------------------------------------
-  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool shuffle_outs, bool use_view_tags)
+  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool shuffle_outs, bool use_view_tags)
  {
    hw::device &hwdev = sender_account_keys.get_device();

@@ -218,7 +218,7 @@ namespace cryptonote
    amount_keys.clear();

    tx.version = rct ? 2 : 1;
-    tx.unlock_time = unlock_time;
+    tx.unlock_time = 0;

    tx.extra = extra;
    crypto::public_key txkey_pub;
@@ -437,6 +437,8 @@ namespace cryptonote
    if (!sort_tx_extra(tx.extra, tx.extra))
      return false;

+    CHECK_AND_ASSERT_MES(tx.extra.size() <= MAX_TX_EXTRA_SIZE, false, "TX extra size (" << tx.extra.size() << ") is greater than max allowed (" << MAX_TX_EXTRA_SIZE << ")");
+
    //check money
    if(summary_outs_money > summary_inputs_money )
    {
@@ -604,7 +606,7 @@ namespace cryptonote
    return true;
  }
  //---------------------------------------------------------------
-  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool use_view_tags)
+  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, bool use_view_tags)
  {
    hw::device &hwdev = sender_account_keys.get_device();
    hwdev.open_tx(tx_key);
@@ -625,7 +627,7 @@ namespace cryptonote
      }

      bool shuffle_outs = true;
-      bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, shuffle_outs, use_view_tags);
+      bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, tx_key, additional_tx_keys, rct, rct_config, shuffle_outs, use_view_tags);
      hwdev.close_tx();
      return r;
    } catch(...) {
@@ -634,14 +636,14 @@ namespace cryptonote
    }
  }
  //---------------------------------------------------------------
-  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time)
+  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx)
  {
     std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses;
     subaddresses[sender_account_keys.m_account_address.m_spend_public_key] = {0,0};
     crypto::secret_key tx_key;
     std::vector<crypto::secret_key> additional_tx_keys;
     std::vector<tx_destination_entry> destinations_copy = destinations;
-     return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0});
+     return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0});
  }
  //---------------------------------------------------------------
  bool generate_genesis_block(
@@ -669,10 +671,10 @@ namespace cryptonote
    return true;
  }
  //---------------------------------------------------------------
-  void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash)
+  void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash)
  {
    blobdata bd = get_block_hashing_blob(b);
-    rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1);
+    rx_slow_hash(seed_hash.data, bd.data(), bd.size(), res.data);
  }

  bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners)
@@ -686,20 +688,16 @@ namespace cryptonote
    }
    if (major_version >= RX_BLOCK_VERSION)
    {
-      uint64_t seed_height, main_height;
      crypto::hash hash;
      if (pbc != NULL)
      {
-        seed_height = rx_seedheight(height);
+        const uint64_t seed_height = rx_seedheight(height);
        hash = seed_hash ? *seed_hash : pbc->get_pending_block_id_by_height(seed_height);
-        main_height = pbc->get_current_blockchain_height();
      } else
      {
        memset(&hash, 0, sizeof(hash));  // only happens when generating genesis block
-        seed_height = 0;
-        main_height = 0;
      }
-      rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash);
+      rx_slow_hash(hash.data, bd.data(), bd.size(), res.data);
    } else {
      const int pow_variant = major_version >= 7 ? major_version - 6 : 0;
      crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height);
@@ -713,20 +711,10 @@ namespace cryptonote
 	return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners);
  }

-  bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners)
-  {
-    return get_block_longhash(pbc, b, res, height, NULL, miners);
-  }
-
-  crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners)
+  crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const crypto::hash *seed_hash, const int miners)
  {
    crypto::hash p = crypto::null_hash;
-    get_block_longhash(pbc, b, p, height, miners);
+    get_block_longhash(pbc, b, p, height, seed_hash, miners);
    return p;
  }
-
-  void get_block_longhash_reorg(const uint64_t split_height)
-  {
-    rx_reorg(split_height);
-  }
 }
@@ -118,9 +118,9 @@ namespace cryptonote

  //---------------------------------------------------------------
  crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr);
-  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time);
-  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool shuffle_outs = true, bool use_view_tags = false);
-  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool use_view_tags = false);
+  bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx);
+  bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool shuffle_outs = true, bool use_view_tags = false);
+  bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, bool use_view_tags = false);
  bool generate_output_ephemeral_keys(const size_t tx_version, const cryptonote::account_keys &sender_account_keys, const crypto::public_key &txkey_pub,  const crypto::secret_key &tx_key,
                                      const cryptonote::tx_destination_entry &dst_entr, const boost::optional<cryptonote::account_public_address> &change_addr, const size_t output_index,
                                      const bool &need_additional_txkeys, const std::vector<crypto::secret_key> &additional_tx_keys,
@@ -144,14 +144,10 @@ namespace cryptonote
    );

  class Blockchain;
-  bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height,
-    const int major_version, const crypto::hash *seed_hash, const int miners);
-  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners);
-  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners);
-  void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height,
-    const uint64_t seed_height, const crypto::hash& seed_hash);
-  crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners);
-  void get_block_longhash_reorg(const uint64_t split_height);
+  bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners = 0);
+  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0);
+  crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0);
+  void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash);

 }

@@ -133,6 +133,12 @@ namespace cryptonote
    // class code expects unsigned values throughout
    if (m_next_check < time_t(0))
      throw std::runtime_error{"Unexpected time_t (system clock) value"};
+
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_start_time = (time_t)0;
+    // We don't set these to "now" already here as we don't know how long it takes from construction
+    // of the pool until it "goes to work". It's safer to set when the first actual txs enter the
+    // corresponding lists.
  }
  //---------------------------------------------------------------------------------
  bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
@@ -207,6 +213,7 @@ namespace cryptonote
    {
      tvc.m_verifivation_failed = true;
      tvc.m_fee_too_low = true;
+      tvc.m_no_drop_offense = true;
      return false;
    }

@@ -219,6 +226,25 @@ namespace cryptonote
      return false;
    }

+    size_t tx_extra_size = tx.extra.size();
+    if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE)
+    {
+      LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
+      tvc.m_verifivation_failed = true;
+      tvc.m_tx_extra_too_big = true;
+      tvc.m_no_drop_offense = true;
+      return false;
+    }
+
+    if (!kept_by_block && tx.unlock_time)
+    {
+      LOG_PRINT_L1("transaction unlock time is not zero: " << tx.unlock_time);
+      tvc.m_verifivation_failed = true;
+      tvc.m_nonzero_unlock_time = true;
+      tvc.m_no_drop_offense = true;
+      return false;
+    }
+
    // if the transaction came from a block popped from the chain,
    // don't check if we have its key images as spent.
    // TODO: Investigate why not?
@@ -230,6 +256,7 @@ namespace cryptonote
        LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images");
        tvc.m_verifivation_failed = true;
        tvc.m_double_spend = true;
+        tvc.m_no_drop_offense = true;
        return false;
      }
    }
@@ -281,7 +308,7 @@ namespace cryptonote
            return false;

          m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time);
          lock.commit();
        }
        catch (const std::exception &e)
@@ -352,7 +379,7 @@ namespace cryptonote

          m_blockchain.remove_txpool_tx(id);
          m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time);
        }
        lock.commit();
      }
@@ -373,7 +400,7 @@ namespace cryptonote

    ++m_cookie;

-    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)));
+    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size());

    prune(m_txpool_max_weight);

@@ -418,8 +445,14 @@ namespace cryptonote
  void tx_memory_pool::prune(size_t bytes)
  {
    CRITICAL_REGION_LOCAL(m_transactions_lock);
+
+    // Nothing to do if already empty
+    if (m_txs_by_fee_and_receive_time.empty())
+      return;
+
    if (bytes == 0)
      bytes = m_txpool_max_weight;
+
    CRITICAL_REGION_LOCAL1(m_blockchain);
    LockedTXN lock(m_blockchain.get_db());
    bool changed = false;
@@ -464,7 +497,13 @@ namespace cryptonote
        reduce_txpool_weight(meta.weight);
        remove_transaction_keyimages(tx, txid);
        MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first);
-        m_txs_by_fee_and_receive_time.erase(it--);
+
+        auto it_prev = it;
+        --it_prev;
+
+        remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted));
+        it = it_prev;
+
        changed = true;
      }
      catch (const std::exception &e)
@@ -546,8 +585,7 @@ namespace cryptonote
    CRITICAL_REGION_LOCAL(m_transactions_lock);
    CRITICAL_REGION_LOCAL1(m_blockchain);

-    auto sorted_it = find_tx_in_sorted_container(id);
-
+    bool sensitive = false;
    try
    {
      LockedTXN lock(m_blockchain.get_db());
@@ -578,6 +616,7 @@ namespace cryptonote
      do_not_relay = meta.do_not_relay;
      double_spend_seen = meta.double_spend_seen;
      pruned = meta.pruned;
+      sensitive = !meta.matches(relay_category::broadcasted);

      // remove first, in case this throws, so key images aren't removed
      m_blockchain.remove_txpool_tx(id);
@@ -591,13 +630,12 @@ namespace cryptonote
      return false;
    }

-    if (sorted_it != m_txs_by_fee_and_receive_time.end())
-      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive);
    ++m_cookie;
    return true;
  }
  //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const
+  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
  {
    PERF_TIMER(get_transaction_info);
    CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -609,7 +647,12 @@ namespace cryptonote
      txpool_tx_meta_t meta;
      if (!m_blockchain.get_txpool_tx_meta(txid, meta))
      {
-        MERROR("Failed to find tx in txpool");
+        LOG_PRINT_L2("Failed to find tx in txpool: " << txid);
+        return false;
+      }
+      if (!include_sensitive_data && !meta.matches(relay_category::broadcasted))
+      {
+        // We don't want sensitive data && the tx is sensitive, so no need to return it
        return false;
      }
      cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
@@ -635,11 +678,13 @@ namespace cryptonote
      td.kept_by_block = meta.kept_by_block;
      td.last_failed_height = meta.last_failed_height;
      td.last_failed_id = meta.last_failed_id;
-      td.receive_time = meta.receive_time;
-      td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time;
+      td.receive_time = include_sensitive_data ? meta.receive_time : 0;
+      td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0;
      td.relayed = meta.relayed;
      td.do_not_relay = meta.do_not_relay;
      td.double_spend_seen = meta.double_spend_seen;
+      if (include_blob)
+        td.tx_blob = std::move(txblob);
    }
    catch (const std::exception &e)
    {
@@ -649,6 +694,25 @@ namespace cryptonote

    return true;
  }
+  //------------------------------------------------------------------
+  bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    txs.clear();
+
+    for (const auto &it: txids)
+    {
+      tx_details details;
+      bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/);
+      if (success)
+      {
+        txs.push_back(std::make_pair(it, std::move(details)));
+      }
+    }
+    return true;
+  }
  //---------------------------------------------------------------------------------
  bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const
  {
@@ -710,15 +774,7 @@ namespace cryptonote
         (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) )
      {
        LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age );
-        auto sorted_it = find_tx_in_sorted_container(txid);
-        if (sorted_it == m_txs_by_fee_and_receive_time.end())
-        {
-          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
-        }
-        else
-        {
-          m_txs_by_fee_and_receive_time.erase(sorted_it);
-        }
+        remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted));
        m_timed_out_transactions.insert(txid);
        remove.push_back(std::make_pair(txid, meta.weight));
      }
@@ -872,9 +928,12 @@ namespace cryptonote
            meta.last_relayed_time = std::chrono::system_clock::to_time_t(now);

          m_blockchain.update_txpool_tx(hash, meta);
-
          // wait until db update succeeds to ensure tx is visible in the pool
          was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted);
+
+          if (was_just_broadcasted)
+            // Make sure the tx gets re-added with an updated time
+            add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now));
        }
      }
      catch (const std::exception &e)
@@ -927,6 +986,81 @@ namespace cryptonote
    }, false, category);
  }
  //------------------------------------------------------------------
+  bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    incremental = true;
+    if (start_time == (time_t)0)
+    {
+      // Giving no start time means give back whole pool
+      incremental = false;
+    }
+    else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0))
+    {
+      if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time))
+      {
+        // If either of the two lists do not go back far enough it's not possible to
+        // deliver incremental pool info
+        incremental = false;
+      }
+      // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time
+    }
+    else
+    {
+      // Some incremental info still missing completely
+      incremental = false;
+    }
+
+    added_txs.clear();
+    remaining_added_txids.clear();
+    removed_txs.clear();
+
+    std::vector<crypto::hash> txids;
+    if (!incremental)
+    {
+      LOG_PRINT_L2("Giving back the whole pool");
+      // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the
+      // anonymous method somehow results in an LMDB error with transactions we have to build a list of
+      // ids first and get the full info afterwards
+      get_transaction_hashes(txids, include_sensitive);
+      if (txids.size() > max_tx_count)
+      {
+        remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end());
+        txids.erase(txids.begin() + max_tx_count, txids.end());
+      }
+      get_transactions_info(txids, added_txs, include_sensitive);
+      return true;
+    }
+
+    // Give back incrementally, based on time of entry into the map
+    for (const auto &pit : m_added_txs_by_id)
+    {
+      if (pit.second >= start_time)
+        txids.push_back(pit.first);
+    }
+    get_transactions_info(txids, added_txs, include_sensitive);
+    if (added_txs.size() > max_tx_count)
+    {
+      remaining_added_txids.reserve(added_txs.size() - max_tx_count);
+      for (size_t i = max_tx_count; i < added_txs.size(); ++i)
+        remaining_added_txids.push_back(added_txs[i].first);
+      added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end());
+    }
+
+    std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time);
+    while (rit != m_removed_txs_by_time.end())
+    {
+      if (include_sensitive || !rit->second.sensitive)
+      {
+        removed_txs.push_back(rit->second.txid);
+      }
+      ++rit;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------
  void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const
  {
    CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -961,7 +1095,7 @@ namespace cryptonote

    // If the total weight is too high, choose the best paying transactions
    if (total_weight > max_backlog_weight)
-      std::sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; });
+      std::stable_sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; });

    backlog.clear();
    uint64_t w = 0;
@@ -1631,6 +1765,12 @@ namespace cryptonote
    CRITICAL_REGION_LOCAL(m_transactions_lock);
    CRITICAL_REGION_LOCAL1(m_blockchain);

+    // Simply throw away incremental info, too difficult to update
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
+
    MINFO("Validating txpool contents for v" << (unsigned)version);

    LockedTXN lock(m_blockchain.get_db());
@@ -1688,6 +1828,106 @@ namespace cryptonote
    return n_removed;
  }
  //---------------------------------------------------------------------------------
+  void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time)
+  {
+
+    time_t now = time(NULL);
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it == m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.insert(std::make_pair(txid, now));
+    }
+    else
+    {
+      // This tx was already added to the map earlier, probably because then it was in the "stem"
+      // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set
+      // a new time for clients that are not allowed to see sensitive txs to make sure they will
+      // see it now if they query incrementally
+      it->second = now;
+
+      auto sorted_it = find_tx_in_sorted_container(txid);
+      if (sorted_it == m_txs_by_fee_and_receive_time.end())
+      {
+        MDEBUG("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container");
+      }
+      else
+      {
+        m_txs_by_fee_and_receive_time.erase(sorted_it);
+      }
+    }
+    m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid);
+
+    // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time'
+    // whether we have that txid there and if yes remove it; this results in possible duplicates
+    // where we return certain txids as deleted AND in the pool at the same time which requires
+    // clients to process deleted ones BEFORE processing pool txs
+    if (m_added_txs_start_time == (time_t)0)
+    {
+      m_added_txs_start_time = now;
+    }
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive)
+  {
+    if (sorted_it == m_txs_by_fee_and_receive_time.end())
+    {
+      LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
+    }
+    else
+    {
+      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    }
+
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it != m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.erase(it);
+    }
+    else
+    {
+      MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs");
+    }
+    track_removed_tx(txid, sensitive);
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive)
+  {
+    time_t now = time(NULL);
+    m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive}));
+    MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size());
+    if (m_removed_txs_start_time == (time_t)0)
+    {
+      m_removed_txs_start_time = now;
+    }
+
+    // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set
+    // an absolute size limit plus delete entries that are x minutes old (which is ok because clients
+    // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back)
+    const int MAX_REMOVED = 20000;
+    if (m_removed_txs_by_time.size() > MAX_REMOVED)
+    {
+      auto erase_it = m_removed_txs_by_time.begin();
+      std::advance(erase_it, MAX_REMOVED / 4 + 1);
+      m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it);
+      m_removed_txs_start_time = m_removed_txs_by_time.begin()->first;
+      MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size());
+    }
+    else
+    {
+      time_t earliest = now - (30 * 60);  // 30 minutes
+      std::map<time_t, removed_tx_info>::iterator from, to;
+      from = m_removed_txs_by_time.begin();
+      to = m_removed_txs_by_time.lower_bound(earliest);
+      int distance = std::distance(from, to);
+      if (distance > 0)
+      {
+        m_removed_txs_by_time.erase(from, to);
+        m_removed_txs_start_time = earliest;
+        MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size());
+      }
+    }
+  }
+  //---------------------------------------------------------------------------------
  bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes)
  {
    CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1695,6 +1935,10 @@ namespace cryptonote

    m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT;
    m_txs_by_fee_and_receive_time.clear();
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
    m_spent_key_images.clear();
    m_txpool_weight = 0;
    std::vector<crypto::hash> remove;
@@ -1719,7 +1963,7 @@ namespace cryptonote
          MFATAL("Failed to insert key images from txpool tx");
          return false;
        }
-        m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid);
+        add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time);
        m_txpool_weight += meta.weight;
        return true;
      }, true, relay_category::all);
@@ -69,11 +69,12 @@ namespace cryptonote
    {
      // sort by greatest first, not least
      if (a.first.first > b.first.first) return true;
-      else if (a.first.first < b.first.first) return false;
-      else if (a.first.second < b.first.second) return true;
-      else if (a.first.second > b.first.second) return false;
-      else if (a.second != b.second) return true;
-      else return false;
+      if (a.first.first < b.first.first) return false;
+
+      if (a.first.second < b.first.second) return true;
+      if (a.first.second > b.first.second) return false;
+
+      return memcmp(a.second.data, b.second.data, sizeof(crypto::hash)) < 0;
    }
  };

@@ -428,6 +429,7 @@ namespace cryptonote
    struct tx_details
    {
      transaction tx;  //!< the transaction
+      cryptonote::blobdata tx_blob; //!< the transaction's binary blob
      size_t blob_size;  //!< the transaction's size
      size_t weight;  //!< the transaction's weight
      uint64_t fee;  //!< the transaction's fee amount
@@ -466,13 +468,25 @@ namespace cryptonote
    /**
     * @brief get infornation about a single transaction
     */
-    bool get_transaction_info(const crypto::hash &txid, tx_details &td) const;
+    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;
+
+    /**
+     * @brief get information about multiple transactions
+     */
+    bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const;

    /**
     * @brief get transactions not in the passed set
     */
    bool get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const;

+    /**
+     * @brief get info necessary for update of pool-related info in a wallet, preferably incremental
+     *
+     * @return true on success, false on error
+     */
+    bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
  private:

    /**
@@ -577,6 +591,10 @@ namespace cryptonote
     */
    void prune(size_t bytes = 0);

+    void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time);
+    void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive);
+    void track_removed_tx(const crypto::hash& txid, bool sensitive);
+
    //TODO: confirm the below comments and investigate whether or not this
    //      is the desired behavior
    //! map key images to transactions which spent them
@@ -609,6 +627,26 @@ private:

    std::atomic<uint64_t> m_cookie; //!< incremented at each change

+    // Info when transactions entered the pool, accessible by txid
+    std::unordered_map<crypto::hash, time_t> m_added_txs_by_id;
+
+    // Info at what time the pool started to track the adding of transactions
+    time_t m_added_txs_start_time;
+
+    struct removed_tx_info
+    {
+      crypto::hash txid;
+      bool sensitive;
+    };
+
+    // Info about transactions that were removed from the pool, ordered by the time
+    // of deletion
+    std::multimap<time_t, removed_tx_info> m_removed_txs_by_time;
+
+    // Info how far back in time the list of removed tx ids currently reaches
+    // (it gets shorted periodically to prevent overflow)
+    time_t m_removed_txs_start_time;
+
    /**
     * @brief get an iterator to a transaction in the sorted container
     *
@@ -0,0 +1,167 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "cryptonote_core/blockchain.h"
+#include "cryptonote_core/tx_verification_utils.h"
+#include "ringct/rctSigs.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "blockchain"
+
+#define VER_ASSERT(cond, msgexpr) CHECK_AND_ASSERT_MES(cond, false, msgexpr)
+
+using namespace cryptonote;
+
+// Do RCT expansion, then do post-expansion sanity checks, then do full non-semantics verification.
+static bool expand_tx_and_ver_rct_non_sem(transaction& tx, const rct::ctkeyM& mix_ring)
+{
+    // Pruned transactions can not be expanded and verified because they are missing RCT data
+    VER_ASSERT(!tx.pruned, "Pruned transaction will not pass verRctNonSemanticsSimple");
+
+    // Calculate prefix hash
+    const crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
+
+    // Expand mixring, tx inputs, tx key images, prefix hash message, etc into the RCT sig
+    const bool exp_res = Blockchain::expand_transaction_2(tx, tx_prefix_hash, mix_ring);
+    VER_ASSERT(exp_res, "Failed to expand rct signatures!");
+
+    const rct::rctSig& rv = tx.rct_signatures;
+
+    // Check that expanded RCT mixring == input mixring
+    VER_ASSERT(rv.mixRing == mix_ring, "Failed to check ringct signatures: mismatched pubkeys/mixRing");
+
+    // Check CLSAG/MLSAG size against transaction input
+    const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size();
+    VER_ASSERT(n_sigs == tx.vin.size(), "Failed to check ringct signatures: mismatched input sigs/vin sizes");
+
+    // For each input, check that the key images were copied into the expanded RCT sig correctly
+    for (size_t n = 0; n < n_sigs; ++n)
+    {
+        const crypto::key_image& nth_vin_image = boost::get<txin_to_key>(tx.vin[n]).k_image;
+
+        if (rct::is_rct_clsag(rv.type))
+        {
+            const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.CLSAGs[n].I, 32);
+            VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched CLSAG key image");
+        }
+        else
+        {
+            const bool mg_nonempty = !rv.p.MGs[n].II.empty();
+            VER_ASSERT(mg_nonempty, "Failed to check ringct signatures: missing MLSAG key image");
+            const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.MGs[n].II[0], 32);
+            VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched MLSAG key image");
+        }
+    }
+
+    // Mix ring data is now known to be correctly incorporated into the RCT sig inside tx.
+    return rct::verRctNonSemanticsSimple(rv);
+}
+
+// Create a unique identifier for pair of tx blob + mix ring
+static crypto::hash calc_tx_mixring_hash(const transaction& tx, const rct::ctkeyM& mix_ring)
+{
+    std::stringstream ss;
+
+    // Start with domain seperation
+    ss << config::HASH_KEY_TXHASH_AND_MIXRING;
+
+    // Then add TX hash
+    const crypto::hash tx_hash = get_transaction_hash(tx);
+    ss.write(tx_hash.data, sizeof(crypto::hash));
+
+    // Then serialize mix ring
+    binary_archive<true> ar(ss);
+    ::do_serialize(ar, const_cast<rct::ctkeyM&>(mix_ring));
+
+    // Calculate hash of TX hash and mix ring blob
+    crypto::hash tx_and_mixring_hash;
+    get_blob_hash(ss.str(), tx_and_mixring_hash);
+
+    return tx_and_mixring_hash;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace cryptonote
+{
+
+bool ver_rct_non_semantics_simple_cached
+(
+    transaction& tx,
+    const rct::ctkeyM& mix_ring,
+    rct_ver_cache_t& cache,
+    const std::uint8_t rct_type_to_cache
+)
+{
+    // Hello future Monero dev! If you got this assert, read the following carefully:
+    //
+    // For this version of RCT, the way we guaranteed that verification caches do not generate false
+    // positives (and thus possibly enabling double spends) is we take a hash of two things. One,
+    // we use get_transaction_hash() which gives us a (cryptographically secure) unique
+    // representation of all "knobs" controlled by the possibly malicious constructor of the
+    // transaction. Two, we take a hash of all *previously validated* blockchain data referenced by
+    // this transaction which is required to validate the ring signature. In our case, this is the
+    // mixring. Future versions of the protocol may differ in this regard, but if this assumptions
+    // holds true in the future, enable the verification hash by modifying the `untested_tx`
+    // condition below.
+    const bool untested_tx = tx.version > 2 || tx.rct_signatures.type > rct::RCTTypeBulletproofPlus;
+    VER_ASSERT(!untested_tx, "Unknown TX type. Make sure RCT cache works correctly with this type and then enable it in the code here.");
+
+    // Don't cache older (or newer) rctSig types
+    // This cache only makes sense when it caches data from mempool first,
+    // so only "current fork version-enabled" RCT types need to be cached
+    if (tx.rct_signatures.type != rct_type_to_cache)
+    {
+        MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " skipped");
+        return expand_tx_and_ver_rct_non_sem(tx, mix_ring);
+    }
+
+    // Generate unique hash for tx+mix_ring pair
+    const crypto::hash tx_mixring_hash = calc_tx_mixring_hash(tx, mix_ring);
+
+    // Search cache for successful verification of same TX + mix ring combination
+    if (cache.has(tx_mixring_hash))
+    {
+        MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " hit");
+        return true;
+    }
+
+    // We had a cache miss, so now we must expand the mix ring and do full verification
+    MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " missed");
+    if (!expand_tx_and_ver_rct_non_sem(tx, mix_ring))
+    {
+        return false;
+    }
+
+    // At this point, the TX RCT verified successfully, so add it to the cache and return true
+    cache.add(tx_mixring_hash);
+
+    return true;
+}
+
+} // namespace cryptonote
@@ -0,0 +1,78 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "common/data_cache.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+
+namespace cryptonote
+{
+
+// Modifying this value should not affect consensus. You can adjust it for performance needs
+static constexpr const size_t RCT_VER_CACHE_SIZE = 8192;
+
+using rct_ver_cache_t = ::tools::data_cache<::crypto::hash, RCT_VER_CACHE_SIZE>;
+
+/**
+ * @brief Cached version of rct::verRctNonSemanticsSimple
+ *
+ * This function will not affect how the transaction is serialized and it will never modify the
+ * transaction prefix.
+ *
+ * The reference to tx is mutable since the transaction's ring signatures may be expanded by
+ * Blockchain::expand_transaction_2. However, on cache hits, the transaction will not be
+ * expanded. This means that the caller does not need to call expand_transaction_2 on this
+ * transaction before passing it; the transaction will not successfully verify with "old" RCT data
+ * if the transaction has been otherwise modified since the last verification.
+ *
+ * But, if cryptonote::get_transaction_hash(tx) returns a "stale" hash, this function is not
+ * guaranteed to work. So make sure that the cryptonote::transaction passed has not had
+ * modifications to it since the last time its hash was fetched without properly invalidating the
+ * hashes.
+ *
+ * rct_type_to_cache can be any RCT version value as long as rct::verRctNonSemanticsSimple works for
+ * this RCT version, but for most applications, it doesn't make sense to not make this version
+ * the "current" RCT version (i.e. the version that transactions in the mempool are).
+ *
+ * @param tx transaction which contains RCT signature to verify
+ * @param mix_ring mixring referenced by this tx. THIS DATA MUST BE PREVIOUSLY VALIDATED
+ * @param cache saves tx+mixring hashes used to cache calls
+ * @param rct_type_to_cache Only RCT sigs with version (e.g. RCTTypeBulletproofPlus) will be cached
+ * @return true when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return true
+ * @return false when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return false
+ */
+bool ver_rct_non_semantics_simple_cached
+(
+    transaction& tx,
+    const rct::ctkeyM& mix_ring,
+    rct_ver_cache_t& cache,
+    std::uint8_t rct_type_to_cache
+);
+
+} // namespace cryptonote
@@ -537,6 +537,10 @@ namespace cryptonote
      MLOG_PEER_STATE("requesting chain");
    }

+    // load json & DNS checkpoints every 10min/hour respectively,
+    // and verify them with respect to what blocks we already have
+    CHECK_AND_ASSERT_MES(m_core.update_checkpoints(), 1, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
+
    return 1;
  }
  //------------------------------------------------------------------------------------------------------------------------
@@ -819,6 +823,10 @@ namespace cryptonote
          post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
          MLOG_PEER_STATE("requesting chain");
        }            
+
+        // load json & DNS checkpoints every 10min/hour respectively,
+        // and verify them with respect to what blocks we already have
+        CHECK_AND_ASSERT_MES(m_core.update_checkpoints(), 1, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints.");
      }
    } 
    else
@@ -971,8 +979,18 @@ namespace cryptonote
  int t_cryptonote_protocol_handler<t_core>::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& context)
  {
    MLOG_P2P_MESSAGE("Received NOTIFY_NEW_TRANSACTIONS (" << arg.txs.size() << " txes)");
+    std::unordered_set<blobdata> seen;
    for (const auto &blob: arg.txs)
+    {
      MLOGIF_P2P_MESSAGE(cryptonote::transaction tx; crypto::hash hash; bool ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx, hash);, ret, "Including transaction " << hash);
+      if (seen.find(blob) != seen.end())
+      {
+        LOG_PRINT_CCONTEXT_L1("Duplicate transaction in notification, dropping connection");
+        drop_connection(context, false, false);
+        return 1;
+      }
+      seen.insert(blob);
+    }

    if(context.m_state != cryptonote_connection_context::state_normal)
      return 1;
@@ -1012,7 +1030,7 @@ namespace cryptonote
    for (auto& tx : arg.txs)
    {
      tx_verification_context tvc{};
-      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true))
+      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense)
      {
        LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection");
        drop_connection(context, false, false);
@@ -396,6 +396,8 @@ namespace levin
        for (auto& connection : connections)
        {
          std::sort(connection.first.begin(), connection.first.end()); // don't leak receive order
+          connection.first.erase(std::unique(connection.first.begin(), connection.first.end()),
+                                  connection.first.end());
          make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs, true);
        }

@@ -542,6 +544,7 @@ namespace levin
      i_core_events* core_;
      std::vector<blobdata> txs_;
      boost::uuids::uuid source_;
+      relay_method tx_relay;

      //! \pre Called in `zone_->strand`
      void operator()()
@@ -549,7 +552,7 @@ namespace levin
        if (!zone_ || !core_ || txs_.empty())
          return;

-        if (!zone_->fluffing)
+        if (!zone_->fluffing || tx_relay == relay_method::local)
        {
          core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem);
          for (int tries = 2; 0 < tries; tries--)
@@ -589,7 +592,7 @@ namespace levin

      change_channels(change_channels&&) = default;
      change_channels(const change_channels& source)
-        : zone_(source.zone_), map_(source.map_.clone())
+        : zone_(source.zone_), map_(source.map_.clone()), fluffing_(source.fluffing_)
      {}

      //! \pre Called within `zone_->strand`.
@@ -740,9 +743,14 @@ namespace levin
  notify::status notify::get_status() const noexcept
  {
    if (!zone_)
-      return {false, false};
+      return {false, false, false};

-    return {!zone_->noise.empty(), CRYPTONOTE_NOISE_CHANNELS <= zone_->connection_count};
+    // `connection_count` is only set when `!noise.empty()`.
+    const std::size_t connection_count = zone_->connection_count;
+    bool has_outgoing = connection_count;
+    if (zone_->noise.empty())
+      has_outgoing = zone_->p2p->get_out_connections_count();
+    return {!zone_->noise.empty(), CRYPTONOTE_NOISE_CHANNELS <= connection_count, has_outgoing};
  }

  void notify::new_out_connection()
@@ -871,7 +879,7 @@ namespace levin
          {
            // this will change a local/forward tx to stem or fluff ...
            zone_->strand.dispatch(
-              dandelionpp_notify{zone_, core_, std::move(txs), source}
+              dandelionpp_notify{zone_, core_, std::move(txs), source, tx_relay}
            );
            break;
          }
@@ -75,7 +75,8 @@ namespace levin
    struct status
    {
      bool has_noise;
-      bool connections_filled;
+      bool connections_filled; //!< True when has zone has `CRYPTONOTE_NOISE_CHANNELS` outgoing noise channels
+      bool has_outgoing; //!< True when zone has outgoing connections
    };

    //! Construct an instance that cannot notify.
@@ -219,6 +219,19 @@ int main(int argc, char const * argv[])
      {
        po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm);
      }
+      catch (const po::unknown_option &e)
+      {
+        std::string unrecognized_option = e.get_option_name();
+        if (all_options.find_nothrow(unrecognized_option, false))
+        {
+          std::cerr << "Option '" << unrecognized_option << "' is not allowed in the config file, please use it as a command line flag." << std::endl;
+        }
+        else
+        {
+          std::cerr << "Unrecognized option '" << unrecognized_option << "' in config file." << std::endl;
+        }
+        return 1;
+      }
      catch (const std::exception &e)
      {
        // log system isn't initialized yet
@@ -526,6 +526,8 @@ namespace hw {
        {0x2c97, 0x0001, 0, 0xffa0}, 
        {0x2c97, 0x0004, 0, 0xffa0},       
        {0x2c97, 0x0005, 0, 0xffa0},
+        {0x2c97, 0x0006, 0, 0xffa0},
+        {0x2c97, 0x0007, 0, 0xffa0},
    };

    bool device_ledger::connect(void) {
@@ -511,7 +511,7 @@ namespace trezor {
                                tools::wallet2::signed_tx_set & signed_tx,
                                hw::tx_aux_data & aux_data)
    {
-      CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset");
+      CHECK_AND_ASSERT_THROW_MES(std::get<0>(unsigned_tx.transfers) == 0, "Unsuported non zero offset");

      TREZOR_AUTO_LOCK_CMD();
      require_connected();
@@ -522,7 +522,7 @@ namespace trezor {
      const size_t num_tx = unsigned_tx.txes.size();
      m_num_transations_to_sign = num_tx;
      signed_tx.key_images.clear();
-      signed_tx.key_images.resize(unsigned_tx.transfers.second.size());
+      signed_tx.key_images.resize(std::get<2>(unsigned_tx.transfers).size());

      for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) {
        std::shared_ptr<protocol::tx::Signer> signer;
@@ -566,8 +566,8 @@ namespace trezor {
        cpend.key_images = key_images;

        // KI sync
-        for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){
-          signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image;
+        for(size_t cidx=0, trans_max=std::get<2>(unsigned_tx.transfers).size(); cidx < trans_max; ++cidx){
+          signed_tx.key_images[cidx] = std::get<2>(unsigned_tx.transfers)[cidx].m_key_image;
        }

        size_t num_sources = cdata.tx_data.sources.size();
@@ -579,9 +579,9 @@ namespace trezor {
          CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped");

          size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped];
-          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset");
+          CHECK_AND_ASSERT_THROW_MES(idx_map_src >= std::get<0>(unsigned_tx.transfers), "Invalid offset");

-          idx_map_src -= unsigned_tx.transfers.first;
+          idx_map_src -= std::get<0>(unsigned_tx.transfers);
          CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index");

          const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]);
@@ -230,8 +230,8 @@ namespace tx {
    }

    const tools::wallet2::transfer_details & get_transfer(size_t idx) const {
-      CHECK_AND_ASSERT_THROW_MES(idx < m_unsigned_tx->transfers.second.size() + m_unsigned_tx->transfers.first && idx >= m_unsigned_tx->transfers.first, "Invalid transfer index");
-      return m_unsigned_tx->transfers.second[idx - m_unsigned_tx->transfers.first];
+      CHECK_AND_ASSERT_THROW_MES(idx < std::get<2>(m_unsigned_tx->transfers).size() + std::get<0>(m_unsigned_tx->transfers) && idx >= std::get<0>(m_unsigned_tx->transfers), "Invalid transfer index");
+      return std::get<2>(m_unsigned_tx->transfers)[idx - std::get<0>(m_unsigned_tx->transfers)];
    }

    const tools::wallet2::transfer_details & get_source_transfer(size_t idx) const {
@@ -50,7 +50,6 @@
 using namespace std;
 using namespace epee;
 using namespace cryptonote;
-using boost::lexical_cast;
 namespace po = boost::program_options;

 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -84,6 +83,9 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str

  try
  {
+    if (total == 0)
+      throw std::runtime_error("Signer group of size 0 is not allowed.");
+
    // create M wallets first
    std::vector<boost::shared_ptr<tools::wallet2>> wallets(total);
    for (size_t n = 0; n < total; ++n)
@@ -118,13 +120,17 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str
      ss << "  " << name << std::endl;
    }

-    //exchange keys unless exchange_multisig_keys returns no extra info
-    while (!kex_msgs_intermediate[0].empty())
+    // exchange keys until the wallets are done
+    bool ready{false};
+    wallets[0]->multisig(&ready);
+    while (!ready)
    {
      for (size_t n = 0; n < total; ++n)
      {
          kex_msgs_intermediate[n] = wallets[n]->exchange_multisig_keys(pwd_container->password(), kex_msgs_intermediate);
      }
+
+      wallets[0]->multisig(&ready);
    }

    std::string address = wallets[0]->get_account().get_public_address_str(wallets[0]->nettype());
@@ -127,7 +127,7 @@ namespace multisig
  bool multisig_account::multisig_is_ready() const
  {
    if (main_kex_rounds_done())
-      return m_kex_rounds_complete >= multisig_kex_rounds_required(m_signers.size(), m_threshold) + 1;
+      return m_kex_rounds_complete >= multisig_setup_rounds_required(m_signers.size(), m_threshold);
    else
      return false;
  }
@@ -175,19 +175,20 @@ namespace multisig
    // only mutate account if update succeeds
    multisig_account temp_account{*this};
    temp_account.set_multisig_config(threshold, std::move(signers));
-    temp_account.kex_update_impl(expanded_msgs_rnd1);
+    temp_account.kex_update_impl(expanded_msgs_rnd1, false);
    *this = std::move(temp_account);
  }
  //----------------------------------------------------------------------------------------------------------------------
  // multisig_account: EXTERNAL
  //----------------------------------------------------------------------------------------------------------------------
-  void multisig_account::kex_update(const std::vector<multisig_kex_msg> &expanded_msgs)
+  void multisig_account::kex_update(const std::vector<multisig_kex_msg> &expanded_msgs,
+    const bool force_update_use_with_caution /*= false*/)
  {
    CHECK_AND_ASSERT_THROW_MES(account_is_active(), "multisig account: tried to update kex, but kex isn't initialized yet.");
    CHECK_AND_ASSERT_THROW_MES(!multisig_is_ready(), "multisig account: tried to update kex, but kex is already complete.");

    multisig_account temp_account{*this};
-    temp_account.kex_update_impl(expanded_msgs);
+    temp_account.kex_update_impl(expanded_msgs, force_update_use_with_caution);
    *this = std::move(temp_account);
  }
  //----------------------------------------------------------------------------------------------------------------------
@@ -200,4 +201,11 @@ namespace multisig
    return num_signers - threshold + 1;
  }
  //----------------------------------------------------------------------------------------------------------------------
+  // EXTERNAL
+  //----------------------------------------------------------------------------------------------------------------------
+  std::uint32_t multisig_setup_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold)
+  {
+    return multisig_kex_rounds_required(num_signers, threshold) + 1;
+  }
+  //----------------------------------------------------------------------------------------------------------------------
 } //namespace multisig
@@ -169,12 +169,20 @@ namespace multisig
    *    - The main interface for multisig key exchange, this handles all the work of processing input messages,
    *      creating new messages for new rounds, and finalizing the multisig shared public key when kex is complete.
    * param: expanded_msgs - kex messages corresponding to the account's 'in progress' round
+    * param: force_update_use_with_caution - try to force the account to update with messages from an incomplete signer set.
+    *    - If this is the post-kex verification round, only require one input message.
+    *      - Force updating here should only be done if we can safely assume an honest signer subgroup of size 'threshold'
+    *        will complete the account.
+    *    - If this is an intermediate round, only require messages from 'num signers - 1 - (round - 1)' other signers.
+    *      - If force updating with maliciously-crafted messages, the resulting account will be invalid (either unable
+    *        to complete signatures, or a 'hostage' to the malicious signer [i.e. can't sign without his participation]).
    */
-    void kex_update(const std::vector<multisig_kex_msg> &expanded_msgs);
+    void kex_update(const std::vector<multisig_kex_msg> &expanded_msgs,
+      const bool force_update_use_with_caution = false);

  private:
    // implementation of kex_update() (non-transactional)
-    void kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs);
+    void kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs, const bool incomplete_signer_set);
    /**
    * brief: initialize_kex_update - Helper for kex_update_impl()
    *    - Collect the local signer's shared keys to ignore in incoming messages, build the aggregate ancillary key
@@ -245,4 +253,13 @@ namespace multisig
  * return: number of kex rounds required
  */
  std::uint32_t multisig_kex_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold);
+
+  /**
+  * brief: multisig_setup_rounds_required - The number of setup rounds required to produce an M-of-N shared key.
+  *    - A participant must complete all kex rounds and 1 initialization round.
+  * param: num_signers - number of participants in multisig (N)
+  * param: threshold - threshold of multisig (M)
+  * return: number of setup rounds required
+  */
+  std::uint32_t multisig_setup_rounds_required(const std::uint32_t num_signers, const std::uint32_t threshold);
 } //namespace multisig
@@ -74,7 +74,7 @@ namespace multisig
      "Multisig threshold may not be larger than number of signers.");
    CHECK_AND_ASSERT_THROW_MES(threshold > 0, "Multisig threshold must be > 0.");
    CHECK_AND_ASSERT_THROW_MES(round > 0, "Multisig kex round must be > 0.");
-    CHECK_AND_ASSERT_THROW_MES(round <= multisig_kex_rounds_required(num_signers, threshold) + 1,
+    CHECK_AND_ASSERT_THROW_MES(round <= multisig_setup_rounds_required(num_signers, threshold),
      "Trying to process multisig kex for an invalid round.");
  }
  //----------------------------------------------------------------------------------------------------------------------
@@ -181,7 +181,8 @@ namespace multisig
  *    Key aggregation via aggregation coefficients prevents key cancellation attacks.
  *    See: https://www.getmonero.org/resources/research-lab/pubs/MRL-0009.pdf
  * param: final_keys - address components (public keys) obtained from other participants (not shared with local)
-  * param: privkeys_inout - private keys of address components known by local; each key will be multiplied by an aggregation coefficient (return by reference)
+  * param: privkeys_inout - private keys of address components known by local; each key will be multiplied by an aggregation
+  *                         coefficient (return by reference)
  * return: final multisig public spend key for the account
  */
  //----------------------------------------------------------------------------------------------------------------------
@@ -199,7 +200,8 @@ namespace multisig
    for (std::size_t multisig_keys_index{0}; multisig_keys_index < privkeys_inout.size(); ++multisig_keys_index)
    {
      crypto::public_key pubkey;
-      CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(privkeys_inout[multisig_keys_index], pubkey), "Failed to derive public key");
+      CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(privkeys_inout[multisig_keys_index], pubkey),
+        "Failed to derive public key");

      own_keys_mapping[pubkey] = multisig_keys_index;

@@ -307,8 +309,7 @@ namespace multisig
  * INTERNAL
  *
  * brief: multisig_kex_msgs_sanitize_pubkeys - Sanitize multisig kex messages.
-  *    - Removes duplicates from msg pubkeys, ignores pubkeys equal to the local account's signing key,
-  *      ignores messages signed by the local account, ignores keys found in input 'exclusion set',
+  *    - Removes duplicates from msg pubkeys, ignores keys found in input 'exclusion set',
  *      constructs map of pubkey:origins.
  *    - Requires that all input msgs have the same round number.
  *
@@ -316,15 +317,13 @@ namespace multisig
  *
  *    - If the messages' round numbers are all '1', then only the message signing pubkey is considered
  *      'recommended'. Furthermore, the 'exclusion set' is ignored.
-  * param: own_pubkey - local account's signing key (key used to sign multisig messages)
  * param: expanded_msgs - set of multisig kex messages to process
  * param: exclude_pubkeys - pubkeys to exclude from output set
  * outparam: sanitized_pubkeys_out - processed pubkeys obtained from msgs, mapped to their origins
  * return: round number shared by all input msgs
  */
  //----------------------------------------------------------------------------------------------------------------------
-  static std::uint32_t multisig_kex_msgs_sanitize_pubkeys(const crypto::public_key &own_pubkey,
-    const std::vector<multisig_kex_msg> &expanded_msgs,
+  static std::uint32_t multisig_kex_msgs_sanitize_pubkeys(const std::vector<multisig_kex_msg> &expanded_msgs,
    const std::vector<crypto::public_key> &exclude_pubkeys,
    multisig_keyset_map_memsafe_t &sanitized_pubkeys_out)
  {
@@ -339,10 +338,6 @@ namespace multisig
    // - origins = all the signing pubkeys that recommended a given msg pubkey
    for (const auto &expanded_msg : expanded_msgs)
    {
-      // ignore messages from self
-      if (expanded_msg.get_signing_pubkey() == own_pubkey)
-        continue;
-
      // in round 1, only the signing pubkey is treated as a msg pubkey
      if (round == 1)
      {
@@ -355,10 +350,6 @@ namespace multisig
        // copy all pubkeys from message into list
        for (const auto &pubkey : expanded_msg.get_msg_pubkeys())
        {
-          // ignore own pubkey
-          if (pubkey == own_pubkey)
-            continue;
-
          // ignore pubkeys in 'ignore' set
          if (std::find(exclude_pubkeys.begin(), exclude_pubkeys.end(), pubkey) != exclude_pubkeys.end())
            continue;
@@ -375,6 +366,31 @@ namespace multisig
  /**
  * INTERNAL
  *
+  * brief: remove_key_from_mapped_sets - Remove a specified key from the mapped sets in a multisig keyset map.
+  * param: key_to_remove - specified key to remove
+  * inoutparam: keyset_inout - keyset to update
+  */
+  //----------------------------------------------------------------------------------------------------------------------
+  static void remove_key_from_mapped_sets(const crypto::public_key &key_to_remove,
+    multisig_keyset_map_memsafe_t &keyset_inout)
+  {
+    // remove specified key from each mapped set
+    for (auto keyset_it = keyset_inout.begin(); keyset_it != keyset_inout.end();)
+    {
+      // remove specified key from this set
+      keyset_it->second.erase(key_to_remove);
+
+      // remove empty keyset positions or increment iterator
+      if (keyset_it->second.size() == 0)
+        keyset_it = keyset_inout.erase(keyset_it);
+      else
+        ++keyset_it;
+    }
+  }
+  //----------------------------------------------------------------------------------------------------------------------
+  /**
+  * INTERNAL
+  *
  * brief: evaluate_multisig_kex_round_msgs - Evaluate pubkeys from a kex round in order to prepare for the next round.
  *    - Sanitizes input msgs.
  *    - Require uniqueness in: 'exclude_pubkeys'.
@@ -392,6 +408,8 @@ namespace multisig
  * param: signers - expected participants in multisig kex
  * param: expanded_msgs - set of multisig kex messages to process
  * param: exclude_pubkeys - derivations held by the local account corresponding to round 'expected_round'
+  * param: incomplete_signer_set - only require the minimum number of signers to complete this round
+  *                                minimum = num_signers - (round num - 1)   (including local signer)
  * return: fully sanitized and validated pubkey:origins map for building the account's next kex round message
  */
  //----------------------------------------------------------------------------------------------------------------------
@@ -400,7 +418,8 @@ namespace multisig
    const std::uint32_t expected_round,
    const std::vector<crypto::public_key> &signers,
    const std::vector<multisig_kex_msg> &expanded_msgs,
-    const std::vector<crypto::public_key> &exclude_pubkeys)
+    const std::vector<crypto::public_key> &exclude_pubkeys,
+    const bool incomplete_signer_set)
  {
    // exclude_pubkeys should all be unique
    for (auto it = exclude_pubkeys.begin(); it != exclude_pubkeys.end(); ++it)
@@ -410,21 +429,31 @@ namespace multisig
    }

    // sanitize input messages
-    multisig_keyset_map_memsafe_t pubkey_origins_map;
-    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(base_pubkey, expanded_msgs, exclude_pubkeys, pubkey_origins_map);
+    multisig_keyset_map_memsafe_t pubkey_origins_map;  //map: [pubkey : [origins]]
+    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(expanded_msgs, exclude_pubkeys, pubkey_origins_map);
    CHECK_AND_ASSERT_THROW_MES(round == expected_round,
      "Kex messages were for round [" << round << "], but expected round is [" << expected_round << "]");

+    // remove the local signer from each origins set in the sanitized pubkey map
+    // note: intermediate kex rounds only need keys from other signers to make progress (keys from self are useless)
+    remove_key_from_mapped_sets(base_pubkey, pubkey_origins_map);
+
    // evaluate pubkeys collected
-    std::unordered_map<crypto::public_key, std::unordered_set<crypto::public_key>> origin_pubkeys_map;
+    std::unordered_map<crypto::public_key, std::unordered_set<crypto::public_key>> origin_pubkeys_map;  //map: [origin: [pubkeys]]

    // 1. each pubkey should be recommended by a precise number of signers
+    const std::size_t num_recommendations_per_pubkey_required{
+        incomplete_signer_set
+        ? 1
+        : round
+      };
+
    for (const auto &pubkey_and_origins : pubkey_origins_map)
    {
      // expected amount = round_num
      // With each successive round, pubkeys are shared by incrementally larger groups,
      //  starting at 1 in round 1 (i.e. the local multisig key to start kex with).
-      CHECK_AND_ASSERT_THROW_MES(pubkey_and_origins.second.size() == round,
+      CHECK_AND_ASSERT_THROW_MES(pubkey_and_origins.second.size() >= num_recommendations_per_pubkey_required,
        "A pubkey recommended by multisig kex messages had an unexpected number of recommendations.");

      // map (sanitized) pubkeys back to origins
@@ -433,8 +462,18 @@ namespace multisig
    }

    // 2. the number of unique signers recommending pubkeys should equal the number of signers passed in (minus the local signer)
-    CHECK_AND_ASSERT_THROW_MES(origin_pubkeys_map.size() == signers.size() - 1,
-      "Number of unique other signers does not equal number of other signers that recommended pubkeys.");
+    // - if an incomplete set is allowed, then we need at least one signer to represent each subgroup in this round that
+    //   doesn't include the local signer
+    const std::size_t num_signers_required{
+        incomplete_signer_set
+        ? signers.size() - 1 - (round - 1)
+        : signers.size() - 1
+      };
+
+    CHECK_AND_ASSERT_THROW_MES(origin_pubkeys_map.size() >= num_signers_required,
+      "Number of unique other signers recommending pubkeys does not equal number of required other signers "
+      "(kex round: " << round << ", num signers found: " << origin_pubkeys_map.size() << ", num signers required: " <<
+      num_signers_required << ").");

    // 3. each origin should recommend a precise number of pubkeys

@@ -461,19 +500,20 @@ namespace multisig

    // other signers: (N - 2) choose (msg_round_num - 1)
      // - Each signer recommends keys they share with other signers.
-      // - In each round, a signer shares a key with 'round num - 1' other signers.
-      // - Since 'origins pubkey map' excludes keys shared with the local account,
-      //   only keys shared with participants 'other than local and self' will be in the map (e.g. N - 2 signers).
-      // - So other signers will recommend (N - 2) choose (msg_round_num - 1) pubkeys (after removing keys shared with local).
-      // - Each origin should have a shared key with each group of size 'round - 1'.
-      // Note: Keys shared with local are ignored to facilitate kex round boosting, where one or more signers may
+      // - In each round, every group of size 'round num' will have a key. From a single signer's perspective,
+      //   they will share a key with every group of size 'round num - 1' of other signers.
+      // - Since 'origins pubkey map' excludes keys shared with the local account, only keys shared with participants
+      //   'other than local and self' will be in the map (e.g. N - 2 signers).
+      // - Other signers will recommend (N - 2) choose (msg_round_num - 1) pubkeys (after removing keys shared with local).
+      // Note: Keys shared with local are filtered out to facilitate kex round boosting, where one or more signers may
      //       have boosted the local signer (implying they didn't have access to the local signer's previous round msg).
    const std::uint32_t expected_recommendations_others = n_choose_k_f(signers.size() - 2, round - 1);

    // local: (N - 1) choose (msg_round_num - 1)
    const std::uint32_t expected_recommendations_self = n_choose_k_f(signers.size() - 1, round - 1);

-    // note: expected_recommendations_others would be 0 in the last round of 1-of-N, but we return early for that case
+    // note: expected_recommendations_others would be 0 in the last round of 1-of-N, but we don't call this function for
+    //       that case
    CHECK_AND_ASSERT_THROW_MES(expected_recommendations_self > 0 && expected_recommendations_others > 0,
      "Bad num signers or round num (possibly numerical limits exceeded).");

@@ -485,7 +525,7 @@ namespace multisig
    for (const auto &origin_and_pubkeys : origin_pubkeys_map)
    {
      CHECK_AND_ASSERT_THROW_MES(origin_and_pubkeys.second.size() == expected_recommendations_others,
-        "A pubkey recommended by multisig kex messages had an unexpected number of recommendations.");
+        "A multisig signer recommended an unexpected number of pubkeys.");

      // 2 (continued). only expected signers should be recommending keys
      CHECK_AND_ASSERT_THROW_MES(std::find(signers.begin(), signers.end(), origin_and_pubkeys.first) != signers.end(),
@@ -507,6 +547,7 @@ namespace multisig
  * param: expected_round - expected kex round of input messages
  * param: signers - expected participants in multisig kex
  * param: expanded_msgs - set of multisig kex messages to process
+  * param: incomplete_signer_set - only require the minimum amount of messages to complete this round (1 message)
  * return: sanitized and validated pubkey:origins map
  */
  //----------------------------------------------------------------------------------------------------------------------
@@ -514,15 +555,20 @@ namespace multisig
    const crypto::public_key &base_pubkey,
    const std::uint32_t expected_round,
    const std::vector<crypto::public_key> &signers,
-    const std::vector<multisig_kex_msg> &expanded_msgs)
+    const std::vector<multisig_kex_msg> &expanded_msgs,
+    const bool incomplete_signer_set)
  {
    // sanitize input messages
    const std::vector<crypto::public_key> dummy;
-    multisig_keyset_map_memsafe_t pubkey_origins_map;
-    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(base_pubkey, expanded_msgs, dummy, pubkey_origins_map);
+    multisig_keyset_map_memsafe_t pubkey_origins_map;  //map: [pubkey : [origins]]
+    const std::uint32_t round = multisig_kex_msgs_sanitize_pubkeys(expanded_msgs, dummy, pubkey_origins_map);
    CHECK_AND_ASSERT_THROW_MES(round == expected_round,
      "Kex messages were for round [" << round << "], but expected round is [" << expected_round << "]");

+    // note: do NOT remove the local signer from the pubkey origins map, since the post-kex round can be force-updated with
+    //       just the local signer's post-kex message (if the local signer were removed, then the post-kex message's pubkeys
+    //       would be completely lost)
+
    // evaluate pubkeys collected

    // 1) there should only be two pubkeys
@@ -533,17 +579,26 @@ namespace multisig
    CHECK_AND_ASSERT_THROW_MES(pubkey_origins_map.begin()->second == (++(pubkey_origins_map.begin()))->second,
      "Multisig post-kex round messages from other signers did not all recommend the same pubkey pair.");

-    // 3) all signers should be present in the recommendation list
+    // 3) all signers should be present in the recommendation list (unless an incomplete list is permitted)
    auto origins = pubkey_origins_map.begin()->second;
-    origins.insert(base_pubkey);  //add self
+    origins.insert(base_pubkey);  //add self if missing

-    CHECK_AND_ASSERT_THROW_MES(origins.size() == signers.size(),
-      "Multisig post-kex round message origins don't line up with multisig signer set.");
+    const std::size_t num_signers_required{
+        incomplete_signer_set
+        ? 1
+        : signers.size()
+      };

-    for (const crypto::public_key &signer : signers)
+    CHECK_AND_ASSERT_THROW_MES(origins.size() >= num_signers_required,
+      "Multisig post-kex round message origins don't line up with multisig signer set "
+      "(num signers found: " << origins.size() << ", num signers required: " << num_signers_required << ").");
+
+    for (const crypto::public_key &origin : origins)
    {
-      CHECK_AND_ASSERT_THROW_MES(origins.find(signer) != origins.end(),
-        "Could not find an expected signer in multisig post-kex round messages (all signers expected).");
+      // note: if num_signers_required == signers.size(), then this test will ensure all signers are present in 'origins',
+      //       which contains only unique pubkeys
+      CHECK_AND_ASSERT_THROW_MES(std::find(signers.begin(), signers.end(), origin) != signers.end(),
+        "An unknown origin recommended a multisig post-kex verification messsage.");
    }

    return pubkey_origins_map;
@@ -564,6 +619,7 @@ namespace multisig
  * param: expanded_msgs - set of multisig kex messages to process
  * param: exclude_pubkeys - keys held by the local account corresponding to round 'current_round'
  *    - If 'current_round' is the final round, these are the local account's shares of the final aggregate key.
+  * param: incomplete_signer_set - allow messages from an incomplete signer set
  * outparam: keys_to_origins_map_out - map between round keys and identity keys
  *    - If in the final round, these are key shares recommended by other signers for the final aggregate key.
  *    - Otherwise, these are the local account's DH derivations for the next round.
@@ -578,6 +634,7 @@ namespace multisig
    const std::vector<crypto::public_key> &signers,
    const std::vector<multisig_kex_msg> &expanded_msgs,
    const std::vector<crypto::public_key> &exclude_pubkeys,
+    const bool incomplete_signer_set,
    multisig_keyset_map_memsafe_t &keys_to_origins_map_out)
  {
    check_multisig_config(current_round, threshold, signers.size());
@@ -598,7 +655,8 @@ namespace multisig
        current_round,
        signers,
        expanded_msgs,
-        exclude_pubkeys);
+        exclude_pubkeys,
+        incomplete_signer_set);
    }
    else //(current_round == kex_rounds_required + 1)
    {
@@ -606,7 +664,8 @@ namespace multisig
      evaluated_pubkeys = evaluate_multisig_post_kex_round_msgs(base_pubkey,
        current_round,
        signers,
-        expanded_msgs);
+        expanded_msgs,
+        incomplete_signer_set);
    }

    // prepare keys-to-origins map for updating the multisig account
@@ -693,9 +752,9 @@ namespace multisig
    {
      // post-kex verification round: check that the multisig pubkey and common pubkey were recommended by other signers
      CHECK_AND_ASSERT_THROW_MES(result_keys_to_origins_map.count(m_multisig_pubkey) > 0,
-        "Multisig post-kex round: expected multisig pubkey wasn't found in other signers' messages.");
+        "Multisig post-kex round: expected multisig pubkey wasn't found in input messages.");
      CHECK_AND_ASSERT_THROW_MES(result_keys_to_origins_map.count(m_common_pubkey) > 0,
-        "Multisig post-kex round: expected common pubkey wasn't found in other signers' messages.");
+        "Multisig post-kex round: expected common pubkey wasn't found in input messages.");

      // save keys that should be recommended to other signers
      // - for convenience, re-recommend the post-kex verification message once an account is complete
@@ -790,7 +849,8 @@ namespace multisig
  //----------------------------------------------------------------------------------------------------------------------
  // multisig_account: INTERNAL
  //----------------------------------------------------------------------------------------------------------------------
-  void multisig_account::kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs)
+  void multisig_account::kex_update_impl(const std::vector<multisig_kex_msg> &expanded_msgs,
+    const bool incomplete_signer_set)
  {
    // check messages are for the expected kex round
    check_messages_round(expanded_msgs, m_kex_rounds_complete + 1);
@@ -816,6 +876,7 @@ namespace multisig
      m_signers,
      expanded_msgs,
      exclude_pubkeys,
+      incomplete_signer_set,
      result_keys_to_origins_map);

    // finish account update
@@ -206,8 +206,13 @@ namespace multisig
  //----------------------------------------------------------------------------------------------------------------------
  void multisig_kex_msg::parse_and_validate_msg()
  {
+    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(),
+      "Multisig kex msg magic inconsistency.");
+    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() >= MULTISIG_KEX_V1_MAGIC.size(),
+      "Multisig kex msg magic inconsistency.");
+
    // check message type
-    CHECK_AND_ASSERT_THROW_MES(m_msg.size() > 0, "Kex message unexpectedly empty.");
+    CHECK_AND_ASSERT_THROW_MES(m_msg.size() >= MULTISIG_KEX_MSG_V2_MAGIC_1.size(), "Kex message unexpectedly small.");
    CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_V1_MAGIC.size()) != MULTISIG_KEX_V1_MAGIC,
      "V1 multisig kex messages are deprecated (unsafe).");
    CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_MSG_V1_MAGIC.size()) != MULTISIG_KEX_MSG_V1_MAGIC,
@@ -215,8 +220,6 @@ namespace multisig

    // deserialize the message
    std::string msg_no_magic;
-    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(),
-      "Multisig kex msg magic inconsistency.");
    CHECK_AND_ASSERT_THROW_MES(tools::base58::decode(m_msg.substr(MULTISIG_KEX_MSG_V2_MAGIC_1.size()), msg_no_magic),
      "Multisig kex msg decoding error.");
    binary_archive<false> b_archive{epee::strspan<std::uint8_t>(msg_no_magic)};
@@ -820,7 +820,6 @@ tx_builder_ringct_t::~tx_builder_ringct_t()
 bool tx_builder_ringct_t::init(
  const cryptonote::account_keys& account_keys,
  const std::vector<std::uint8_t>& extra,
-  const std::uint64_t unlock_time,
  const std::uint32_t subaddr_account,
  const std::set<std::uint32_t>& subaddr_minor_indices,
  std::vector<cryptonote::tx_source_entry>& sources,
@@ -854,7 +853,7 @@ bool tx_builder_ringct_t::init(

  // misc. fields
  unsigned_tx.version = 2;  //rct = 2
-  unsigned_tx.unlock_time = unlock_time;
+  unsigned_tx.unlock_time = 0;

  // sort inputs
  sort_sources(sources);
@@ -71,7 +71,6 @@ public:
  bool init(
    const cryptonote::account_keys& account_keys,
    const std::vector<std::uint8_t>& extra,
-    const std::uint64_t unlock_time,
    const std::uint32_t subaddr_account,
    const std::set<std::uint32_t>& subaddr_minor_indices,
    std::vector<cryptonote::tx_source_entry>& sources,
@@ -71,7 +71,7 @@ namespace net
        struct i2p_serialized
        {
            std::string host;
-            std::uint16_t port;
+            std::uint16_t port; //! Leave for compatability with older clients

            BEGIN_KV_SERIALIZE_MAP()
                KV_SERIALIZE(host)
@@ -80,8 +80,7 @@ namespace net
        };
    }

-    i2p_address::i2p_address(const boost::string_ref host, const std::uint16_t port) noexcept
-      : port_(port)
+    i2p_address::i2p_address(const boost::string_ref host) noexcept
    {
        // this is a private constructor, throw if moved to public
        assert(host.size() < sizeof(host_));
@@ -97,27 +96,19 @@ namespace net
    }

    i2p_address::i2p_address() noexcept
-      : port_(0)
    {
        static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size");
        std::memcpy(host_, unknown_host, sizeof(unknown_host));
        std::memset(host_ + sizeof(unknown_host), 0, sizeof(host_) - sizeof(unknown_host));
    }

-    expect<i2p_address> i2p_address::make(const boost::string_ref address, const std::uint16_t default_port)
+    expect<i2p_address> i2p_address::make(const boost::string_ref address)
    {
        boost::string_ref host = address.substr(0, address.rfind(':'));
-        const boost::string_ref port =
-            address.substr(host.size() + (host.size() == address.size() ? 0 : 1));
-
        MONERO_CHECK(host_check(host));

-        std::uint16_t porti = default_port;
-        if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port}))
-            return {net::error::invalid_port};
-
        static_assert(b32_length + sizeof(tld) == sizeof(i2p_address::host_), "bad internal host size");
-        return i2p_address{host, porti};
+        return i2p_address{host};
    }

    bool i2p_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent)
@@ -127,23 +118,21 @@ namespace net
        {
            std::memcpy(host_, in.host.data(), in.host.size());
            std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size());
-            port_ = in.port;
            return true;
        }
        static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size");
        std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator
-        port_ = 0;
        return false;
    }

    bool i2p_address::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const
    {
-        const i2p_serialized out{std::string{host_}, port_};
+        // Set port to 1 for backwards compatability; zero is invalid port
+        const i2p_serialized out{std::string{host_}, 1};
        return out.store(dest, hparent);
    }

    i2p_address::i2p_address(const i2p_address& rhs) noexcept
-      : port_(rhs.port_)
    {
        std::memcpy(host_, rhs.host_, sizeof(host_));
    }
@@ -152,7 +141,6 @@ namespace net
    {
        if (this != std::addressof(rhs))
        {
-            port_ = rhs.port_;
            std::memcpy(host_, rhs.host_, sizeof(host_));
        }
        return *this;
@@ -166,13 +154,12 @@ namespace net

    bool i2p_address::equal(const i2p_address& rhs) const noexcept
    {
-        return port_ == rhs.port_ && is_same_host(rhs);
+        return is_same_host(rhs);
    }

    bool i2p_address::less(const i2p_address& rhs) const noexcept
    {
-        int res = std::strcmp(host_str(), rhs.host_str());
-        return res < 0 || (res == 0 && port() < rhs.port());
+        return std::strcmp(host_str(), rhs.host_str()) < 0;
    }

    bool i2p_address::is_same_host(const i2p_address& rhs) const noexcept
@@ -182,20 +169,6 @@ namespace net

    std::string i2p_address::str() const
    {
-        const std::size_t host_length = std::strlen(host_str());
-        const std::size_t port_length =
-            port_ == 0 ? 0 : std::numeric_limits<std::uint16_t>::digits10 + 2;
-
-        std::string out{};
-        out.reserve(host_length + port_length);
-        out.assign(host_str(), host_length);
-
-        if (port_ != 0)
-        {
-            out.push_back(':');
-            namespace karma = boost::spirit::karma;
-            karma::generate(std::back_inserter(out), karma::ushort_, port());
-        }
-        return out;
+        return host_str();
    }
 }
@@ -50,11 +50,10 @@ namespace net
    //! b32 i2p address; internal format not condensed/decoded.
    class i2p_address
    {
-        std::uint16_t port_;
        char host_[61]; // null-terminated

        //! Keep in private, `host.size()` has no runtime check
-        i2p_address(boost::string_ref host, std::uint16_t port) noexcept;
+        i2p_address(boost::string_ref host) noexcept;

    public:
        //! \return Size of internal buffer for host.
@@ -74,7 +73,7 @@ namespace net
            with `default_port` being used if port is not specified in
            `address`.
        */
-        static expect<i2p_address> make(boost::string_ref address, std::uint16_t default_port = 0);
+        static expect<i2p_address> make(boost::string_ref address);

        //! Load from epee p2p format, and \return false if not valid tor address
        bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent);
@@ -103,8 +102,8 @@ namespace net
        //! \return Null-terminated `x.b32.i2p` value or `unknown_str()`.
        const char* host_str() const noexcept { return host_; }

-        //! \return Port value or `0` if unspecified.
-        std::uint16_t port() const noexcept { return port_; }
+        //! \return `1` to work with I2P socks which considers `0` error.
+        std::uint16_t port() const noexcept { return 1; }

        static constexpr bool is_loopback() noexcept { return false; }
        static constexpr bool is_local() noexcept { return false; }
@@ -38,7 +38,7 @@ namespace net
 {
    void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port)
    {
-        // require ipv6 address format "[addr:addr:addr:...:addr]:port"
+        // If IPv6 address format with port "[addr:addr:addr:...:addr]:port"
        if (address.find(']') != std::string::npos)
        {
            host = address.substr(1, address.rfind(']') - 1);
@@ -47,6 +47,12 @@ namespace net
                port = address.substr(address.rfind(':') + 1);
            }
        }
+        // Else if IPv6 address format without port e.g. "addr:addr:addr:...:addr"
+        else if (std::count(address.begin(), address.end(), ':') >= 2)
+        {
+            host = address;
+        }
+        // Else IPv4, Tor, I2P address or hostname
        else
        {
            host = address.substr(0, address.rfind(':'));
@@ -75,7 +81,7 @@ namespace net
        if (host_str_ref.ends_with(".onion"))
            return tor_address::make(address, default_port);
        if (host_str_ref.ends_with(".i2p"))
-            return i2p_address::make(address, default_port);
+            return i2p_address::make(address);

        boost::system::error_code ec;
        boost::asio::ip::address_v6 v6 = boost::asio::ip::address_v6::from_string(host_str, ec);
@@ -38,6 +38,16 @@

 namespace net
 {
+    /*!
+     * \brief Takes a valid address string (IP, Tor, I2P, or DNS name) and splits it into host and port
+     *
+     * The host of an IPv6 addresses in the format "[x:x:..:x]:port" will have the braces stripped.
+     * For example, when the address is "[ffff::2023]", host will be set to "ffff::2023".
+     *
+     * \param address The address string one wants to split
+     * \param[out] host The host part of the address string. Is always set.
+     * \param[out] port The port part of the address string. Is only set when address string contains a port.
+    */
    void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port);

    /*!
@@ -247,7 +247,23 @@ namespace nodetool
    if (it == m_blocked_hosts.end())
    {
      m_blocked_hosts[host_str] = limit;
-      added = true;
+
+      // if the host was already blocked due to being in a blocked subnet, let it be silent
+      bool matches_blocked_subnet = false;
+      if (addr.get_type_id() == epee::net_utils::address_type::ipv4)
+      {
+        auto ipv4_address = addr.template as<epee::net_utils::ipv4_network_address>();
+        for (auto jt = m_blocked_subnets.begin(); jt != m_blocked_subnets.end(); ++jt)
+        {
+          if (jt->first.matches(ipv4_address))
+          {
+            matches_blocked_subnet = true;
+            break;
+          }
+        }
+      }
+      if (!matches_blocked_subnet)
+        added = true;
    }
    else if (it->second < limit || !add_only)
      it->second = limit;
@@ -317,6 +333,7 @@ namespace nodetool
      limit = std::numeric_limits<time_t>::max();
    else
      limit = now + seconds;
+    const bool added = m_blocked_subnets.find(subnet) == m_blocked_subnets.end();
    m_blocked_subnets[subnet] = limit;

    // drop any connection to that subnet. This should only have to look into
@@ -349,7 +366,10 @@ namespace nodetool
      conns.clear();
    }

-    MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked.");
+    if (added)
+      MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked.");
+    else
+      MINFO("Subnet " << subnet.host_str() << " blocked.");
    return true;
  }
  //-----------------------------------------------------------------------------------
@@ -645,20 +665,10 @@ namespace nodetool
  {
    using namespace boost::asio;

-    std::string host = addr;
+    // Split addr string into host string and port string
+    std::string host;
    std::string port = std::to_string(default_port);
-    size_t colon_pos = addr.find_last_of(':');
-    size_t dot_pos = addr.find_last_of('.');
-    size_t square_brace_pos = addr.find('[');
-
-    // IPv6 will have colons regardless.  IPv6 and IPv4 address:port will have a colon but also either a . or a [
-    // as IPv6 addresses specified as address:port are to be specified as "[addr:addr:...:addr]:port"
-    // One may also specify an IPv6 address as simply "[addr:addr:...:addr]" without the port; in that case
-    // the square braces will be stripped here.
-    if ((std::string::npos != colon_pos && std::string::npos != dot_pos) || std::string::npos != square_brace_pos)
-    {
-      net::get_network_address_host_and_port(addr, host, port);
-    }
+    net::get_network_address_host_and_port(addr, host, port);
    MINFO("Resolving node address: host=" << host << ", port=" << port);

    io_service io_srv;
@@ -695,34 +705,32 @@ namespace nodetool
    std::set<std::string> full_addrs;
    if (m_nettype == cryptonote::TESTNET)
    {
-      full_addrs.insert("212.83.175.67:28080");
-      full_addrs.insert("212.83.172.165:28080");
      full_addrs.insert("176.9.0.187:28080");
-      full_addrs.insert("88.99.173.38:28080");
      full_addrs.insert("51.79.173.165:28080");
+      full_addrs.insert("192.99.8.110:28080");
+      full_addrs.insert("37.187.74.171:28080");
+      full_addrs.insert("77.172.183.193:28080");
    }
    else if (m_nettype == cryptonote::STAGENET)
    {
-      full_addrs.insert("162.210.173.150:38080");
      full_addrs.insert("176.9.0.187:38080");
-      full_addrs.insert("88.99.173.38:38080");
      full_addrs.insert("51.79.173.165:38080");
+      full_addrs.insert("192.99.8.110:38080");
+      full_addrs.insert("37.187.74.171:38080");
+      full_addrs.insert("77.172.183.193:38080");
    }
    else if (m_nettype == cryptonote::FAKECHAIN)
    {
    }
    else
    {
-      full_addrs.insert("212.83.175.67:18080");
-      full_addrs.insert("212.83.172.165:18080");
      full_addrs.insert("176.9.0.187:18080");
      full_addrs.insert("88.198.163.90:18080");
-      full_addrs.insert("95.217.25.101:18080");
-      full_addrs.insert("136.244.105.131:18080");
-      full_addrs.insert("104.238.221.81:18080");
      full_addrs.insert("66.85.74.134:18080");
-      full_addrs.insert("88.99.173.38:18080");
      full_addrs.insert("51.79.173.165:18080");
+      full_addrs.insert("192.99.8.110:18080");
+      full_addrs.insert("37.187.74.171:18080");
+      full_addrs.insert("77.172.183.193:18080");
    }
    return full_addrs;
  }
@@ -853,10 +861,12 @@ namespace nodetool
      if (m_nettype == cryptonote::MAINNET)
      {
        return {
-          "xwvz3ekocr3dkyxfkmgm2hvbpzx2ysqmaxgter7znnqrhoicygkfswid.onion:18083",
-          "4pixvbejrvihnkxmduo2agsnmc3rrulrqc7s3cbwwrep6h6hrzsibeqd.onion:18083",
          "zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083",
          "qz43zul2x56jexzoqgkx2trzwcfnr6l3hbtfcfx54g4r3eahy3bssjyd.onion:18083",
+          "plowsof3t5hogddwabaeiyrno25efmzfxyro2vligremt7sxpsclfaid.onion:18083",
+          "plowsoffjexmxalw73tkjmf422gq6575fc7vicuu4javzn2ynnte6tyd.onion:18083",
+          "plowsofe6cleftfmk2raiw5h2x66atrik3nja4bfd3zrfa2hdlgworad.onion:18083",
+          "aclc4e2jhhtr44guufbnwk5bzwhaecinax4yip4wr4tjn27sjsfg6zqd.onion:18083",
        };
      }
      return {};
@@ -864,8 +874,9 @@ namespace nodetool
      if (m_nettype == cryptonote::MAINNET)
      {
        return {
-          "s3l6ke4ed3df466khuebb4poienoingwof7oxtbo6j4n56sghe3a.b32.i2p:18080",
-          "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080"
+          "uqj3aphckqtjsitz7kxx5flqpwjlq5ppr3chazfued7xucv3nheq.b32.i2p",
+          "vdmnehdjkpkg57nthgnjfuaqgku673r5bpbqg56ix6fyqoywgqrq.b32.i2p",
+          "ugnlcdciyhghh2zert7c3kl4biwkirc43ke33jiy5slnd3mv2trq.b32.i2p",
        };
      }
      return {};
@@ -2288,11 +2299,12 @@ namespace nodetool
      if (enet::zone::tor < network->first)
        break; // unknown network

-      if (network->second.m_connect)
+      const auto status = network->second.m_notifier.get_status();
+      if (network->second.m_connect && status.has_outgoing)
        return send(*network);
    }

-    // configuration should not allow this scenario
+    MWARNING("Unable to send " << txs.size() << " transaction(s): anonymity networks had no outgoing connections");
    return enet::zone::invalid;
  }
  //-----------------------------------------------------------------------------------
@@ -2413,7 +2425,7 @@ namespace nodetool
        return false;
      }
      return true;
-    });
+    }, "0.0.0.0", m_ssl_support);
    if(!r)
    {
      LOG_WARNING_CC(context, "Failed to call connect_async, network error.");
@@ -42,6 +42,7 @@
 #include <boost/serialization/version.hpp>

 #include "net_peerlist_boost_serialization.h"
+#include "common/util.h"


 namespace nodetool
@@ -200,7 +201,7 @@ namespace nodetool
    if (!out)
    {
      // if failed, try reading in unportable mode
-      boost::filesystem::copy_file(path, path + ".unportable", boost::filesystem::copy_option::overwrite_if_exists);
+      tools::copy_file(path, path + ".unportable");
      src_file.close();
      src_file.open( path , std::ios_base::binary | std::ios_base::in);
      if(src_file.fail())
@@ -196,7 +196,7 @@ namespace boost
      if (std::strcmp(host, net::i2p_address::unknown_str()) == 0)
        na = net::i2p_address::unknown();
      else
-        na = MONERO_UNWRAP(net::i2p_address::make(host, port));
+        na = MONERO_UNWRAP(net::i2p_address::make(host));
    }

    template <class Archive, class ver_type>
@@ -1333,7 +1333,7 @@ namespace rct {
        try
        {
          if (semantics) {
-            tools::threadpool& tpool = tools::threadpool::getInstance();
+            tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
            tools::threadpool::waiter waiter(tpool);
            std::deque<bool> results(rv.outPk.size(), false);
            DP("range proofs verified?");
@@ -1383,7 +1383,7 @@ namespace rct {
      {
        PERF_TIMER(verRctSemanticsSimple);

-        tools::threadpool& tpool = tools::threadpool::getInstance();
+        tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
        tools::threadpool::waiter waiter(tpool);
        std::deque<bool> results;
        std::vector<const Bulletproof*> bp_proofs;
@@ -1536,7 +1536,7 @@ namespace rct {
        const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size());

        std::deque<bool> results(threads);
-        tools::threadpool& tpool = tools::threadpool::getInstance();
+        tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
        tools::threadpool::waiter waiter(tpool);

        const keyV &pseudoOuts = bulletproof || bulletproof_plus ? rv.p.pseudoOuts : rv.pseudoOuts;
--- a/Show More
+++ b/Show More