Merge pull request #9226

b5b72ae Fixed mempool pruning (SChernykh)
Fixed mempool pruning
2024-03-08 20:23:52 -05:00 · 2024-03-08 21:03:41 +01:00 · 2024-03-08 13:46:09 -05:00 · 2024-03-08 10:49:17 -05:00 · 2024-03-08 10:44:12 -05:00 · 2024-03-08 08:14:06 -06:00
123 changed files with 5156 additions and 1664 deletions
@@ -27,10 +27,10 @@ jobs:
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: /Users/runner/Library/Caches/ccache
        key: ccache-${{ runner.os }}-build-${{ github.sha }}
@@ -51,15 +51,15 @@ jobs:
      run:
        shell: msys2 {0}
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: C:\Users\runneradmin\.ccache
        key: ccache-${{ runner.os }}-build-${{ github.sha }}
        restore-keys: ccache-${{ runner.os }}-build-
-    - uses: eine/setup-msys2@v2
+    - uses: msys2/setup-msys2@v2
      with:
        update: true
        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
@@ -77,12 +77,12 @@ jobs:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    strategy:
      matrix:
-        os: [ubuntu-latest, ubuntu-18.04]
+        os: [ubuntu-22.04, ubuntu-20.04]
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
@@ -101,14 +101,14 @@ jobs:
        ${{env.BUILD_DEFAULT_LINUX}}

  libwallet-ubuntu:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
-    - uses: actions/cache@v2
+    - uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
@@ -129,15 +129,15 @@ jobs:

  test-ubuntu:
    needs: build-ubuntu
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
        submodules: recursive
    - name: ccache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
@@ -151,10 +151,11 @@ jobs:
    - name: install monero dependencies
      run: ${{env.APT_INSTALL_LINUX}}
    - name: install Python dependencies
-      run: pip install requests psutil monotonic zmq
+      run: pip install requests psutil monotonic zmq deepdiff
    - name: tests
      env:
        CTEST_OUTPUT_ON_FAILURE: ON
+        DNS_PUBLIC: tcp://9.9.9.9
      run: |
        ${{env.CCACHE_SETTINGS}}
        ${{env.BUILD_DEFAULT_LINUX}}
@@ -166,8 +167,9 @@ jobs:
  source-archive:
    runs-on: ubuntu-20.04
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
+        fetch-depth: 0
        submodules: recursive
    - name: archive
      run: |
@@ -176,7 +178,7 @@ jobs:
        export OUTPUT="$VERSION.tar"
        echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV
        /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT"
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
      with:
        name: ${{ env.OUTPUT }}
        path: /home/runner/work/monero/monero/${{ env.OUTPUT }}
@@ -18,7 +18,7 @@ env:

 jobs:
  build-cross:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
    env:
      CCACHE_TEMPDIR: /tmp/.ccache-temp
    strategy:
@@ -57,19 +57,20 @@ jobs:
            packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
    name: ${{ matrix.toolchain.name }}
    steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
      with:
+        fetch-depth: 0
        submodules: recursive
 # Most volatile cache
    - name: ccache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: ~/.ccache
        key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }}
        restore-keys: ccache-${{ matrix.toolchain.host }}-
 # Less volatile cache
    - name: depends cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: contrib/depends/built
        key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
@@ -78,7 +79,7 @@ jobs:
          depends-${{ matrix.toolchain.host }}-
 # Static cache
    - name: OSX SDK cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
      with:
        path: contrib/depends/sdk-sources
        key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }}
@@ -96,7 +97,7 @@ jobs:
      run: |
        ${{env.CCACHE_SETTINGS}}
        make depends target=${{ matrix.toolchain.host }} -j2
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
      if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
      with:
        name: ${{ matrix.toolchain.name }}
@@ -0,0 +1,49 @@
+name: ci/gh-actions/gitian
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build-gitian:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        operating-system:
+          - name: "Linux"
+            option: "l"
+          - name: "Windows"
+            option: "w"
+          - name: "Android"
+            option: "a"
+          - name: "FreeBSD"
+            option: "f"
+          - name: "macOS"
+            option: "m"
+    name: ${{ matrix.operating-system.name }}
+    steps:
+    - name: prepare
+      run: |
+        sudo apt update
+        curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py
+        chmod +x gitian-build.py
+    - name: setup
+      run: |
+        ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }}
+    - name: build
+      run: |
+        ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }}
+    - name: post build
+      run: |
+        cd out/${{ github.ref_name }}
+        shasum -a256 *
+        echo \`\`\` >> $GITHUB_STEP_SUMMARY
+        shasum -a256 * >> $GITHUB_STEP_SUMMARY
+        echo \`\`\` >> $GITHUB_STEP_SUMMARY
+    - uses: actions/upload-artifact@v3
+      with:
+        name: ${{ matrix.operating-system.name }}
+        path: |
+          out/${{ github.ref_name }}/*
@@ -794,7 +794,7 @@ else()
    set(USE_LTO_DEFAULT false)
    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack,10485760")
    if(NOT BUILD_64)
-      add_definitions(-DWINVER=0x0501 -D_WIN32_WINNT=0x0501)
+      add_definitions(-DWINVER=0x0600 -D_WIN32_WINNT=0x0600)
    endif()
  endif()
  set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes")
@@ -1178,6 +1178,9 @@ find_library(NORM_LIBRARY norm)
 find_library(GSSAPI_LIBRARY gssapi_krb5)
 find_library(PROTOLIB_LIBRARY protolib)
 find_library(SODIUM_LIBRARY sodium)
+find_library(BSD_LIBRARY bsd)
+find_library(MD_LIBRARY md)
+find_library(PROTOKIT_LIBRARY protokit)

 if(NOT ZMQ_INCLUDE_PATH)
  message(FATAL_ERROR "Could not find required header zmq.h")
@@ -1200,6 +1203,15 @@ endif()
 if(SODIUM_LIBRARY)
  set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}")
 endif()
+if(BSD_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${BSD_LIBRARY}")
+endif()
+if(MD_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${MD_LIBRARY}")
+endif()
+if(PROTOKIT_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${PROTOKIT_LIBRARY}")
+endif()

 include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion
 add_subdirectory(contrib)
@@ -104,7 +104,7 @@ release-all:

 release-static:
 	mkdir -p $(builddir)/release
-	cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
+	cd $(builddir)/release && cmake -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)

 coverage:
 	mkdir -p $(builddir)/debug
@@ -138,8 +138,8 @@ Dates are provided in the format YYYY-MM-DD.
 | 1978433                        | 2019-11-30 | v12               | v0.15.0.0              | v0.16.0.0                  | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs
 | 2210000                        | 2020-10-17 | v13               | v0.17.0.0              | v0.17.3.2                  | New CLSAG transaction format
 | 2210720                        | 2020-10-18 | v14               | v0.17.1.1              | v0.17.3.2                  | forbid old MLSAG transaction format
-| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.1.2                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
-| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.1.2                  | forbid old v14 transaction format
+| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.3.2                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
+| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.3.2                  | forbid old v14 transaction format
 | XXXXXXX                        | XXX-XX-XX | XXX                | vX.XX.X.X              | vX.XX.X.X                  | XXX |

 X's indicate that these details have not been determined as of commit date.
@@ -344,7 +344,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (
    ```bash
    git clone https://github.com/monero-project/monero.git
    cd monero
-    git checkout v0.18.1.2
+    git checkout v0.18.3.2
    ```

 * Build:
@@ -463,10 +463,10 @@ application.
    cd monero
    ```

-* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.1.2'. If you don't care about the version and just want binaries from master, skip this step:
+* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.3.2'. If you don't care about the version and just want binaries from master, skip this step:

    ```bash
-    git checkout v0.18.1.2
+    git checkout v0.18.3.2
    ```

 * If you are on a 64-bit system, run:
@@ -113,7 +113,7 @@ if ( LibUSB_FOUND )
    if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC)
        if (APPLE)
            if(DEPENDS)
-                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security")
            else()
                find_library(COREFOUNDATION CoreFoundation)
                find_library(IOKIT IOKit)
@@ -143,8 +143,11 @@ $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig
 $(1)_config_env+=PATH="$(build_prefix)/bin:$(PATH)"
 $(1)_build_env+=PATH="$(build_prefix)/bin:$(PATH)"
 $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)"
-$(1)_autoconf=./configure --host=$($($(1)_type)_host) --disable-dependency-tracking --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"
+$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)"

+ifeq ($(filter $(1),libusb unbound),)
+$(1)_autoconf += --disable-dependency-tracking
+endif
 ifneq ($($(1)_nm),)
 $(1)_autoconf += NM="$$($(1)_nm)"
 endif
@@ -1,12 +1,12 @@
 package=expat
-$(package)_version=2.4.1
-$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_4_1
+$(package)_version=2.6.0
+$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$($(package)_version))/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=2f9b6a580b94577b150a7d5617ad4643a4301a6616ff459307df3e225bcfbf40
+$(package)_sha256_hash=ff60e6a6b6ce570ae012dc7b73169c7fdf4b6bf08c12ed0ec6f55736b78d85ba

 define $(package)_set_vars
-$(package)_config_opts=--enable-static
-$(package)_config_opts=--disable-shared
+$(package)_config_opts=--disable-shared --without-docbook --without-tests --without-examples
+$(package)_config_opts+=--enable-option-checking --without-xmlwf --with-pic
 $(package)_config_opts+=--prefix=$(host_prefix)
 endef

@@ -23,6 +23,6 @@ define $(package)_stage_cmds
 endef

 define $(package)_postprocess_cmds
-  rm lib/*.la
+  rm -rf share lib/cmake lib/*.la
 endef

@@ -1,8 +1,8 @@
 package=hidapi
-$(package)_version=0.11.0
-$(package)_download_path=https://github.com/libusb/hidapi/archive
+$(package)_version=0.13.1
+$(package)_download_path=https://github.com/libusb/hidapi/archive/refs/tags
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=391d8e52f2d6a5cf76e2b0c079cfefe25497ba1d4659131297081fc0cd744632
+$(package)_sha256_hash=476a2c9a4dc7d1fc97dd223b84338dbea3809a84caea2dcd887d9778725490e3
 $(package)_linux_dependencies=libusb eudev
 $(package)_patches=missing_win_include.patch

@@ -1,8 +1,8 @@
 package=libusb
-$(package)_version=1.0.22
-$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
+$(package)_version=1.0.26
+$(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version)
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
+$(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5

 define $(package)_preprocess_cmds
  autoreconf -i
@@ -1,21 +1,19 @@
 package=openssl
-$(package)_version=1.1.1l
+$(package)_version=3.0.13
 $(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1
-$(package)_patches=fix_darwin.patch
+$(package)_sha256_hash=88525753f79d3bec27d2fa7c66aa0b92b3aa9498dafd93d7cfa4b3780cdae313

 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
-$(package)_config_env_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
-$(package)_build_env_android=ANDROID_NDK_HOME="$(host_prefix)/native"
-$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
+$(package)_config_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native"
+$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl --libdir=$(host_prefix)/lib
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-heartbeats
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
@@ -23,8 +21,8 @@ $(package)_config_opts+=no-rfc3779
 $(package)_config_opts+=no-sctp
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
-$(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
+$(package)_config_opts+=no-tests
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -50,8 +48,7 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef

 define $(package)_preprocess_cmds
-  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \
-  patch -p1 < $($(package)_patch_dir)/fix_darwin.patch
+  sed -i.old 's|crypto ssl apps util tools fuzz providers doc|crypto ssl util tools providers|' build.info
 endef

 define $(package)_config_cmds
@@ -1,17 +1,21 @@
 package=unbound
-$(package)_version=1.15.0
+$(package)_version=1.19.1
 $(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=a480dc6c8937447b98d161fe911ffc76cfaffa2da18788781314e81339f1126f
+$(package)_sha256_hash=bc1d576f3dd846a0739adc41ffaa702404c6767d2b6082deb9f2f97cbb24a3a9
 $(package)_dependencies=openssl expat
 $(package)_patches=disable-glibc-reallocarray.patch


 define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
+  $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix)
+  $(package)_config_opts+=--with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no
+  $(package)_config_opts+=--without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
  $(package)_config_opts_linux=--with-pic
  $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix)
+  $(package)_config_opts_x86_64_darwin=ac_cv_func_SHA384_Init=yes
  $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread"
+  $(package)_cflags_mingw32+="-D_WIN32_WINNT=0x600"
 endef

 define $(package)_preprocess_cmds
@@ -30,6 +34,3 @@ endef
 define $(package)_stage_cmds
  $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
-
-define $(package)_postprocess_cmds
-endef
@@ -1,60 +0,0 @@
-From 96ac8f13f4d0ee96baf5724d9f96c44c34b8606c Mon Sep 17 00:00:00 2001
-From: David Carlier <devnexen@gmail.com>
-Date: Tue, 24 Aug 2021 22:40:14 +0100
-Subject: [PATCH] Darwin platform allows to build on releases before
- Yosemite/ios 8.
-
-issue #16407 #16408
-
-Reviewed-by: Paul Dale <pauli@openssl.org>
-Reviewed-by: Tomas Mraz <tomas@openssl.org>
-(Merged from https://github.com/openssl/openssl/pull/16409)
---
- crypto/rand/rand_unix.c |  5 +----
- include/crypto/rand.h   | 10 ++++++++++
- 2 files changed, 11 insertions(+), 4 deletions(-)
-
-diff --git a/crypto/rand/rand_unix.c b/crypto/rand/rand_unix.c
-index 43f1069d151d..0f4525106af7 100644
--- a/crypto/rand/rand_unix.c
-+++ b/crypto/rand/rand_unix.c
-@@ -34,9 +34,6 @@
- #if defined(__OpenBSD__)
- # include <sys/param.h>
- #endif
-#if defined(__APPLE__)
-# include <CommonCrypto/CommonRandom.h>
-#endif
- 
- #if defined(OPENSSL_SYS_UNIX) || defined(__DJGPP__)
- # include <sys/types.h>
-@@ -381,7 +378,7 @@ static ssize_t syscall_random(void *buf, size_t buflen)
-         if (errno != ENOSYS)
-             return -1;
-     }
-#  elif defined(__APPLE__)
-+#  elif defined(OPENSSL_APPLE_CRYPTO_RANDOM)
-     if (CCRandomGenerateBytes(buf, buflen) == kCCSuccess)
- 	    return (ssize_t)buflen;
- 
-diff --git a/include/crypto/rand.h b/include/crypto/rand.h
-index 5350d3a93119..674f840fd13c 100644
--- a/include/crypto/rand.h
-+++ b/include/crypto/rand.h
-@@ -20,6 +20,16 @@
- 
- # include <openssl/rand.h>
- 
-+# if defined(__APPLE__) && !defined(OPENSSL_NO_APPLE_CRYPTO_RANDOM)
-+#  include <Availability.h>
-+#  if (defined(__MAC_OS_X_VERSION_MIN_REQUIRED) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101000) || \
-+     (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 80000)
-+#   define OPENSSL_APPLE_CRYPTO_RANDOM 1
-+#   include <CommonCrypto/CommonCryptoError.h>
-+#   include <CommonCrypto/CommonRandom.h>
-+#  endif
-+# endif
-+
- /* forward declaration */
- typedef struct rand_pool_st RAND_POOL;
- 
@@ -144,8 +144,8 @@ elseif(ARCHITECTURE STREQUAL "aarch64")
 endif()

 if(ARCHITECTURE STREQUAL "riscv64")
-    set(NO_AES ON)
-    set(ARCH "rv64imafdc")
+  set(ARCH_ID "riscv64")
+  set(ARCH "rv64gc")
 endif()

 if(ARCHITECTURE STREQUAL "i686")
@@ -29,6 +29,7 @@

 #include <string>
 #include <ctime>
+#include <cstdint>

 namespace epee
 {
@@ -583,11 +583,8 @@ namespace net_utils
            break;
        }
      }
-      else if (ec.value())
-        terminate();
      else {
-        cancel_timer();
-        on_interrupted();
+        terminate();
      }
    };
    m_strand.post(
@@ -245,8 +245,18 @@ namespace net_utils
 					}
 				}

+				// This magic var determines the maximum length for when copying the body message in
+				// memory is faster/more preferable than the round-trip time for one packet
+				constexpr size_t BODY_NO_COPY_CUTOFF = 128 * 1024; // ~262 KB or ~175 packets
+
+				// Maximum expected total headers bytes
+				constexpr size_t HEADER_RESERVE_SIZE = 2048;
+
+				const bool do_copy_body = body.size() <= BODY_NO_COPY_CUTOFF;
+				const size_t req_buff_cap = HEADER_RESERVE_SIZE + (do_copy_body ? body.size() : 0);
+
 				std::string req_buff{};
-				req_buff.reserve(2048);
+				req_buff.reserve(req_buff_cap);
 				req_buff.append(method.data(), method.size()).append(" ").append(uri.data(), uri.size()).append(" HTTP/1.1\r\n");
 				add_field(req_buff, "Host", m_host_buff);
 				add_field(req_buff, "Content-Length", std::to_string(body.size()));
@@ -255,9 +265,7 @@ namespace net_utils
 				for(const auto& field : additional_params)
 					add_field(req_buff, field);

-				for (unsigned sends = 0; sends < 2; ++sends)
 				{
-					const std::size_t initial_size = req_buff.size();
 					const auto auth = m_auth.get_auth_field(method, uri);
 					if (auth)
 						add_field(req_buff, *auth);
@@ -265,11 +273,21 @@ namespace net_utils
 					req_buff += "\r\n";
 					//--

-					bool res = m_net_client.send(req_buff, timeout);
-					CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
-					if(body.size())
+					if (do_copy_body) // small body
+					{
+						// Copy headers + body together and potentially send one fewer packet
+						req_buff.append(body.data(), body.size());
+						const bool res = m_net_client.send(req_buff, timeout);
+						CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+					}
+					else // large body
+					{
+						// Send headers and body seperately to avoid copying heavy body message
+						bool res = m_net_client.send(req_buff, timeout);
+						CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
 						res = m_net_client.send(body, timeout);
-					CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+						CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+					}

 					m_response_info.clear();
 					m_state = reciev_machine_state_header;
@@ -282,19 +300,11 @@ namespace net_utils
 						return true;
 					}

-					switch (m_auth.handle_401(m_response_info))
+					if (m_auth.handle_401(m_response_info) == http_client_auth::kParseFailure)
 					{
-					case http_client_auth::kSuccess:
-						break;
-					case http_client_auth::kBadPassword:
-                                                sends = 2;
-						break;
-					default:
-					case http_client_auth::kParseFailure:
 						LOG_ERROR("Bad server response for authentication");
 						return false;
 					}
-					req_buff.resize(initial_size); // rollback for new auth generation
 				}
 				LOG_ERROR("Client has incorrect username/password for server requiring authentication");
 				return false;
@@ -147,6 +147,16 @@ namespace epee
    return {reinterpret_cast<const std::uint8_t*>(src.data()), src.size_bytes()}; 
  }

+  //! \return `span<std::uint8_t>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<std::uint8_t> to_mut_byte_span(T& src)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_empty<value_type>(), "empty value types will not work -> sizeof == 1");
+    static_assert(!has_padding<value_type>(), "source value type may have padding");
+    return {reinterpret_cast<std::uint8_t*>(src.data()), src.size() * sizeof(value_type)};
+  }
+
  //! \return `span<const std::uint8_t>` which represents the bytes at `&src`.
  template<typename T>
  span<const std::uint8_t> as_byte_span(const T& src) noexcept
@@ -30,6 +30,7 @@

 #include <boost/utility/string_ref_fwd.hpp>
 #include <string>
+#include <cstdint>

 namespace epee 
 {
@@ -33,6 +33,9 @@
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"

+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #else 
@@ -31,6 +31,9 @@
 #include "parserse_base_utils.h"
 #include "file_io_utils.h"

+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100

 namespace epee
@@ -338,11 +338,21 @@ bool is_stdout_a_tty()
  return is_a_tty.load(std::memory_order_relaxed);
 }

+static bool is_nocolor()
+{
+  static const char *no_color_var = getenv("NO_COLOR");
+  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
+  return no_color;
+}
+
 void set_console_color(int color, bool bright)
 {
  if (!is_stdout_a_tty())
    return;

+  if (is_nocolor())
+    return;
+
  switch(color)
  {
  case console_color_default:
@@ -461,6 +471,9 @@ void reset_console_color() {
  if (!is_stdout_a_tty())
    return;

+  if (is_nocolor())
+    return;
+
 #ifdef WIN32
  HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
  SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);
@@ -496,6 +496,13 @@ void ssl_options_t::configure(
  const std::string& host) const
 {
  socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+  {
+    // in case server is doing "virtual" domains, set hostname
+    SSL* const ssl_ctx = socket.native_handle();
+    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
+      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
+  }
+

  /* Using system-wide CA store for client verification is funky - there is
     no expected hostname for server to verify against. If server doesn't have
@@ -513,11 +520,7 @@ void ssl_options_t::configure(
  {
    socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);

-    // in case server is doing "virtual" domains, set hostname
-    SSL* const ssl_ctx = socket.native_handle();
-    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
-      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
-
+    
    socket.set_verify_callback([&](const bool preverified, boost::asio::ssl::verify_context &ctx)
    {
      // preverified means it passed system or user CA check. System CA is never loaded
@@ -49,7 +49,7 @@ namespace serialization
    byte_stream ss;
    ss.reserve(initial_buffer_size);
    store_to_binary(ss);
-    target = epee::byte_slice{std::move(ss)};
+    target = epee::byte_slice{std::move(ss), false};
    return true;
    CATCH_ENTRY("portable_storage::store_to_binary", false);
  }
@@ -238,6 +238,10 @@ static char** attempted_completion(const char* text, int start, int end)

 static void install_line_handler()
 {
+#if RL_READLINE_VERSION >= 0x0801
+  rl_variable_bind("enable-bracketed-paste", "off");
+#endif
+
  rl_attempted_completion_function = attempted_completion;
  rl_callback_handler_install("", handle_line);
  stifle_history(500);
@@ -57,7 +57,7 @@ The dockrun.sh script will do everything to build the binaries. Just specify the
 version to build as its only argument, e.g.

 ```bash
-VERSION=v0.18.1.2
+VERSION=v0.18.3.2
 ./dockrun.sh $VERSION
 ```

@@ -133,7 +133,7 @@ Common setup part:
 su - gitianuser

 GH_USER=YOUR_GITHUB_USER_NAME
-VERSION=v0.18.1.2
+VERSION=v0.18.3.2
 ```

 Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. 
@@ -21,6 +21,7 @@ packages:
 - "g++-7-arm-linux-gnueabihf"
 - "gcc-arm-linux-gnueabihf"
 - "g++-arm-linux-gnueabihf"
+- "g++-riscv64-linux-gnu"
 - "g++-7-multilib"
 - "gcc-7-multilib"
 - "binutils-arm-linux-gnueabihf"
@@ -43,7 +44,7 @@ files: []
 script: |

  WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu"
+  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu riscv64-linux-gnu"
  FAKETIME_HOST_PROGS=""
  FAKETIME_PROGS="date"
  HOST_CFLAGS="-O2 -g"
@@ -159,7 +160,13 @@ script: |
    fi
    export C_INCLUDE_PATH="$EXTRA_INCLUDES"
    export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON
+    #  glibc only added riscv support in 2.27, disable backwards compatibility
+    if [ "$i" == "riscv64-linux-gnu" ]; then
+      BACKCOMPAT_OPTION=OFF
+    else
+      BACKCOMPAT_OPTION=ON
+    fi
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=${BACKCOMPAT_OPTION} -DCMAKE_SKIP_RPATH=ON
    make ${MAKEOPTS}
    chmod 755 bin/*
    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
@@ -71,13 +71,13 @@ type, and max connections:

 ```
 --anonymous-inbound rveahdfho7wo4b2m.onion:28083,127.0.0.1:28083,25
--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000,127.0.0.1:30000
+--anonymous-inbound cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p,127.0.0.1:30000
 ```

 which tells `monerod` that a max of 25 inbound Tor connections are being
 received at address "rveahdfho7wo4b2m.onion:28083" and forwarded to `monerod`
 localhost port 28083, and a default max I2P connections are being received at
-address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p:5000" and
+address "cmeua5767mz2q5jsaelk2rxhf67agrwuetaso5dzbenyzwlbkg2q.b32.i2p" and
 forwarded to `monerod` localhost port 30000.
 These addresses will be shared with outgoing peers, over the same network type,
 otherwise the peer will not be notified of the peer address by the proxy.
@@ -149,6 +149,11 @@ static el::Color colorFromLevel(el::Level level)

 static void setConsoleColor(el::Color color, bool bright)
 {
+  static const char *no_color_var = getenv("NO_COLOR");
+  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
+  if (no_color)
+    return;
+
 #if ELPP_OS_WINDOWS
  HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
  switch (color)
@@ -46,6 +46,77 @@ using namespace cryptonote;

 static bool stop_requested = false;

+static bool do_inputs, do_outputs, do_ringsize, do_hours, do_emission, do_fees, do_diff;
+
+static struct tm prevtm, currtm;
+static uint64_t prevsz, currsz;
+static uint64_t prevtxs, currtxs;
+static uint64_t currblks;
+static uint64_t h;
+static uint64_t totins, totouts, totrings;
+static boost::multiprecision::uint128_t prevemission, prevfees;
+static boost::multiprecision::uint128_t emission, fees;
+static boost::multiprecision::uint128_t totdiff, mindiff, maxdiff;
+
+#define MAX_INOUT	0xffffffff
+#define MAX_RINGS	0xffffffff
+
+static uint32_t minins = MAX_INOUT, maxins;
+static uint32_t minouts = MAX_INOUT, maxouts;
+static uint32_t minrings = MAX_RINGS, maxrings;
+static uint32_t io, tottxs;
+static uint32_t txhr[24];
+
+static void doprint()
+{
+  char timebuf[64];
+
+  strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm);
+  prevtm = currtm;
+  std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
+  prevsz += currsz;
+  currsz = 0;
+  prevtxs += currtxs;
+  currtxs = 0;
+  if (!tottxs)
+    tottxs = 1;
+  if (do_emission) {
+    std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
+    prevemission += emission;
+    emission = 0;
+  }
+  if (do_fees) {
+    std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
+    prevfees += fees;
+    fees = 0;
+  }
+  if (do_diff) {
+    std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
+    mindiff = 0; maxdiff = 0; totdiff = 0;
+  }
+  if (do_inputs) {
+    std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
+    minins = MAX_INOUT; maxins = 0; totins = 0;
+  }
+  if (do_outputs) {
+    std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
+    minouts = MAX_INOUT; maxouts = 0; totouts = 0;
+  }
+  if (do_ringsize) {
+    std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
+    minrings = MAX_RINGS; maxrings = 0; totrings = 0;
+  }
+  if (do_hours) {
+    for (int i=0; i<24; i++) {
+      std::cout << "\t" << txhr[i];
+      txhr[i] = 0;
+    }
+  }
+  currblks = 0;
+  tottxs = 0;
+  std::cout << ENDL;
+}
+
 int main(int argc, char* argv[])
 {
  TRY_ENTRY();
@@ -123,13 +194,13 @@ int main(int argc, char* argv[])
  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
  block_start = command_line::get_arg(vm, arg_block_start);
  block_stop = command_line::get_arg(vm, arg_block_stop);
-  bool do_inputs = command_line::get_arg(vm, arg_inputs);
-  bool do_outputs = command_line::get_arg(vm, arg_outputs);
-  bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
-  bool do_hours = command_line::get_arg(vm, arg_hours);
-  bool do_emission = command_line::get_arg(vm, arg_emission);
-  bool do_fees = command_line::get_arg(vm, arg_fees);
-  bool do_diff = command_line::get_arg(vm, arg_diff);
+  do_inputs = command_line::get_arg(vm, arg_inputs);
+  do_outputs = command_line::get_arg(vm, arg_outputs);
+  do_ringsize = command_line::get_arg(vm, arg_ringsize);
+  do_hours = command_line::get_arg(vm, arg_hours);
+  do_emission = command_line::get_arg(vm, arg_emission);
+  do_fees = command_line::get_arg(vm, arg_fees);
+  do_diff = command_line::get_arg(vm, arg_diff);

  LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
  std::unique_ptr<Blockchain> core_storage;
@@ -211,25 +282,7 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
  }
  std::cout << ENDL;

-#define MAX_INOUT	0xffffffff
-#define MAX_RINGS	0xffffffff
-
-  struct tm prevtm = {0}, currtm;
-  uint64_t prevsz = 0, currsz = 0;
-  uint64_t prevtxs = 0, currtxs = 0;
-  uint64_t currblks = 0;
-  uint64_t totins = 0, totouts = 0, totrings = 0;
-  boost::multiprecision::uint128_t prevemission = 0, prevfees = 0;
-  boost::multiprecision::uint128_t emission = 0, fees = 0;
-  boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0;
-  uint32_t minins = MAX_INOUT, maxins = 0;
-  uint32_t minouts = MAX_INOUT, maxouts = 0;
-  uint32_t minrings = MAX_RINGS, maxrings = 0;
-  uint32_t io, tottxs = 0;
-  uint32_t txhr[24] = {0};
-  unsigned int i;
-
-  for (uint64_t h = block_start; h < block_stop; ++h)
+  for (h = block_start; h < block_stop; ++h)
  {
    cryptonote::blobdata bd = db->get_block_blob_from_height(h);
    cryptonote::block blk;
@@ -239,7 +292,6 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
      return 1;
    }
    time_t tt = blk.timestamp;
-    char timebuf[64];
    epee::misc_utils::get_gmt_time(tt, currtm);
    if (!prevtm.tm_year)
      prevtm = currtm;
@@ -247,54 +299,9 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
    if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27))
    {
      // check for timestamp fudging around month ends
-      if (prevtm.tm_mday == 1 && currtm.tm_mday > 27)
-        goto skip;
-      strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm);
-      prevtm = currtm;
-      std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
-      prevsz += currsz;
-      currsz = 0;
-      prevtxs += currtxs;
-      currtxs = 0;
-      if (!tottxs)
-        tottxs = 1;
-      if (do_emission) {
-        std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
-        prevemission += emission;
-        emission = 0;
-      }
-      if (do_fees) {
-        std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
-        prevfees += fees;
-        fees = 0;
-      }
-      if (do_diff) {
-        std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
-        mindiff = 0; maxdiff = 0; totdiff = 0;
-      }
-      if (do_inputs) {
-        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
-        minins = MAX_INOUT; maxins = 0; totins = 0;
-      }
-      if (do_outputs) {
-        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
-        minouts = MAX_INOUT; maxouts = 0; totouts = 0;
-      }
-      if (do_ringsize) {
-        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
-        minrings = MAX_RINGS; maxrings = 0; totrings = 0;
-      }
-      if (do_hours) {
-        for (i=0; i<24; i++) {
-          std::cout << "\t" << txhr[i];
-          txhr[i] = 0;
-        }
-      }
-      currblks = 0;
-      tottxs = 0;
-      std::cout << ENDL;
+      if (!(prevtm.tm_mday == 1 && currtm.tm_mday > 27))
+        doprint();
    }
-skip:
    currsz += bd.size();
    uint64_t coinbase_amount;
    uint64_t tx_fee_amount = 0;
@@ -371,6 +378,8 @@ skip:
    if (stop_requested)
      break;
  }
+  if (currblks)
+    doprint();

  core_storage->deinit();
  return 0;
@@ -243,6 +243,12 @@ namespace cryptonote
    ADD_CHECKPOINT2(2677000, "1b9fee6246eeb176bd17d637bf252e9af54a4218675f01b4449cc0901867f9eb", "0x2f165bc1a5163ba");
    ADD_CHECKPOINT2(2706000, "d8eb144c5e1fe6b329ecc900ec95e7792fccff84175fb23a25ed59d7299a511c", "0x310f7d89372f705");
    ADD_CHECKPOINT2(2720000, "b19fb41dff15bd1016afbee9f8469f05aab715c9e5d1b974466a11fd58ecbb86", "0x3216b5851ddbb61");
+    ADD_CHECKPOINT2(2817000, "39726d19ccaac01d150bec827b877ffae710b516bd633503662036ef4422e577", "0x3900669561954c1");
+    ADD_CHECKPOINT2(2844000, "28fc7b446dfef5b469f5778eb72ddf32a307a5f5a9823d1c394e772349e05d40", "0x3af384ec0e97d12");
+    ADD_CHECKPOINT2(2851000, "5bf0e47fc782263191a33f63a67db6c711781dc2a3c442e17ed901ec401be5c9", "0x3b6cd8a8ed610e8");
+    ADD_CHECKPOINT2(2971000, "3d4cac5ac515eeabd18769ab943af85f36db51d28720def0d0e6effc2c8f5ce3", "0x436e532738b8b5b");
+    ADD_CHECKPOINT2(2985000, "08f5e6b7301c1b6ed88268a28f8677a06e8ff943b3f9e48d3080f71f9c134bfb", "0x444b7b42a633c96");
+    ADD_CHECKPOINT2(3088000, "bddf8ca09110d33d6d497f13a113630c2b6af1c84d4f3a6f35cb1446f2604ade", "0x4aed3615c2f8c3e");
    return true;
  }

@@ -34,6 +34,7 @@
 #include <iostream>
 #include <vector>
 #include <stdexcept>
+#include <cstdint>

 namespace tools {

@@ -0,0 +1,65 @@
+// Copyright (c) 2014-2022, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once 
+
+#include <unordered_set>
+#include <mutex>
+
+namespace tools
+{
+  template<typename T, size_t MAX_SIZE>
+  class data_cache
+  {
+  public:
+    void add(const T& value)
+    {
+      std::lock_guard<std::mutex> lock(m);
+      if (data.insert(value).second)
+      {
+        T& old_value = buf[counter++ % MAX_SIZE];
+        data.erase(old_value);
+        old_value = value;
+      }
+    }
+
+    bool has(const T& value) const
+    {
+      std::lock_guard<std::mutex> lock(m);
+      return (data.find(value) != data.end());
+    }
+
+  private:
+    mutable std::mutex m;
+    std::unordered_set<T> data;
+    T buf[MAX_SIZE] = {};
+    size_t counter = 0;
+  };
+}
@@ -30,6 +30,8 @@
 // check local first (in the event of static or in-source compilation of libunbound)
 #include "unbound.h"

+#include <deque>
+#include <set>
 #include <stdlib.h>
 #include "include_base_utils.h"
 #include "common/threadpool.h"
@@ -327,11 +329,6 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec
  dnssec_available = false;
  dnssec_valid = false;

-  if (!check_address_syntax(url.c_str()))
-  {
-    return addresses;
-  }
-
  // destructor takes care of cleanup
  ub_result_ptr result;

@@ -414,16 +411,6 @@ DNSResolver DNSResolver::create()
  return DNSResolver();
 }

-bool DNSResolver::check_address_syntax(const char *addr) const
-{
-  // if string doesn't contain a dot, we won't consider it a url for now.
-  if (strchr(addr,'.') == NULL)
-  {
-    return false;
-  }
-  return true;
-}
-
 namespace dns_utils
 {

@@ -159,15 +159,6 @@ private:
  // TODO: modify this to accommodate DNSSEC
  std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid);

-  /**
-   * @brief Checks a string to see if it looks like a URL
-   *
-   * @param addr the string to be checked
-   *
-   * @return true if it looks enough like a URL, false if not
-   */
-  bool check_address_syntax(const char *addr) const;
-
  DNSResolverData *m_data;
 }; // class DNSResolver

@@ -62,7 +62,7 @@ namespace tools
    while (1)
    {
      t1 = epee::misc_utils::get_ns_count();
-      if (t1 - t0 > 1*1000000000) break; // work one second
+      if (t1 - t0 > 1*100000000) break; // work 0.1 seconds
    }

    uint64_t r1 = get_tick_count();
@@ -31,6 +31,7 @@
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/thread.hpp>
 #include <cstddef>
+#include <deque>
 #include <functional>
 #include <utility>
 #include <vector>
@@ -882,13 +882,6 @@ std::string get_nix_version_display_string()

  bool is_local_address(const std::string &address)
  {
-    // always assume Tor/I2P addresses to be untrusted by default
-    if (is_privacy_preserving_network(address))
-    {
-      MDEBUG("Address '" << address << "' is Tor/I2P, non local");
-      return false;
-    }
-
    // extract host
    epee::net_utils::http::url_content u_c;
    if (!epee::net_utils::parse_url(address, u_c))
@@ -902,20 +895,22 @@ std::string get_nix_version_display_string()
      return false;
    }

-    // resolve to IP
-    boost::asio::io_service io_service;
-    boost::asio::ip::tcp::resolver resolver(io_service);
-    boost::asio::ip::tcp::resolver::query query(u_c.host, "");
-    boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query);
-    while (i != boost::asio::ip::tcp::resolver::iterator())
+    if (u_c.host == "localhost" || boost::ends_with(u_c.host, ".localhost")) { // RFC 6761 (6.3)
+      MDEBUG("Address '" << address << "' is local");
+      return true;
+    }
+
+    boost::system::error_code ec;
+    const auto parsed_ip = boost::asio::ip::address::from_string(u_c.host, ec);
+    if (ec) {
+      MDEBUG("Failed to parse '" << address << "' as IP address: " << ec.message() << ". Considering it not local");
+      return false;
+    }
+
+    if (parsed_ip.is_loopback())
    {
-      const boost::asio::ip::tcp::endpoint &ep = *i;
-      if (ep.address().is_loopback())
-      {
-        MDEBUG("Address '" << address << "' is local");
-        return true;
-      }
-      ++i;
+      MDEBUG("Address '" << address << "' is local");
+      return true;
    }

    MDEBUG("Address '" << address << "' is not local");
@@ -30,29 +30,41 @@
 #pragma once

 #ifdef _WIN32
+
 #include <windows.h>
-#define CTHR_MUTEX_TYPE	HANDLE
-#define CTHR_MUTEX_INIT	NULL
-#define CTHR_MUTEX_LOCK(x)	do { if (x == NULL) { \
-    HANDLE p = CreateMutex(NULL, FALSE, NULL); \
-    if (InterlockedCompareExchangePointer((PVOID*)&x, (PVOID)p, NULL) != NULL) \
-      CloseHandle(p); \
-  } WaitForSingleObject(x, INFINITE); } while(0)
-#define CTHR_MUTEX_UNLOCK(x)	ReleaseMutex(x)
+
+#define CTHR_RWLOCK_TYPE	SRWLOCK
+#define CTHR_RWLOCK_INIT	SRWLOCK_INIT
+#define CTHR_RWLOCK_LOCK_WRITE(x)	AcquireSRWLockExclusive(&x)
+#define CTHR_RWLOCK_UNLOCK_WRITE(x)	ReleaseSRWLockExclusive(&x)
+#define CTHR_RWLOCK_LOCK_READ(x)	AcquireSRWLockShared(&x)
+#define CTHR_RWLOCK_UNLOCK_READ(x)	ReleaseSRWLockShared(&x)
+#define CTHR_RWLOCK_TRYLOCK_READ(x)	TryAcquireSRWLockShared(&x)
+
 #define CTHR_THREAD_TYPE	HANDLE
-#define CTHR_THREAD_RTYPE	void
-#define CTHR_THREAD_RETURN	return
-#define CTHR_THREAD_CREATE(thr, func, arg)	thr = (HANDLE)_beginthread(func, 0, arg)
-#define CTHR_THREAD_JOIN(thr)			WaitForSingleObject(thr, INFINITE)
+#define CTHR_THREAD_RTYPE	unsigned __stdcall
+#define CTHR_THREAD_RETURN	_endthreadex(0); return 0;
+#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L)
+#define CTHR_THREAD_JOIN(thr)			do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0)
+#define CTHR_THREAD_CLOSE(thr)			CloseHandle((HANDLE)thr);
+
 #else
+
 #include <pthread.h>
-#define CTHR_MUTEX_TYPE pthread_mutex_t
-#define CTHR_MUTEX_INIT	PTHREAD_MUTEX_INITIALIZER
-#define CTHR_MUTEX_LOCK(x)	pthread_mutex_lock(&x)
-#define CTHR_MUTEX_UNLOCK(x)	pthread_mutex_unlock(&x)
+
+#define CTHR_RWLOCK_TYPE	pthread_rwlock_t
+#define CTHR_RWLOCK_INIT	PTHREAD_RWLOCK_INITIALIZER
+#define CTHR_RWLOCK_LOCK_WRITE(x)	pthread_rwlock_wrlock(&x)
+#define CTHR_RWLOCK_UNLOCK_WRITE(x)	pthread_rwlock_unlock(&x)
+#define CTHR_RWLOCK_LOCK_READ(x)	pthread_rwlock_rdlock(&x)
+#define CTHR_RWLOCK_UNLOCK_READ(x)	pthread_rwlock_unlock(&x)
+#define CTHR_RWLOCK_TRYLOCK_READ(x)	(pthread_rwlock_tryrdlock(&x) == 0)
+
 #define CTHR_THREAD_TYPE pthread_t
 #define CTHR_THREAD_RTYPE	void *
 #define CTHR_THREAD_RETURN	return NULL
-#define CTHR_THREAD_CREATE(thr, func, arg)	pthread_create(&thr, NULL, func, arg)
+#define CTHR_THREAD_CREATE(thr, func, arg)	(pthread_create(&thr, NULL, func, arg) == 0)
 #define CTHR_THREAD_JOIN(thr)			pthread_join(thr, NULL)
+#define CTHR_THREAD_CLOSE(thr)
+
 #endif
@@ -97,5 +97,9 @@ void rx_slow_hash_allocate_state(void);
 void rx_slow_hash_free_state(void);
 uint64_t rx_seedheight(const uint64_t height);
 void rx_seedheights(const uint64_t height, uint64_t *seed_height, uint64_t *next_height);
-void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length, char *hash, int miners, int is_alt);
-void rx_reorg(const uint64_t split_height);
+
+void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads);
+void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash);
+
+void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads);
+uint32_t rx_get_miner_thread(void);
@@ -34,7 +34,7 @@ typedef struct {
 	unsigned long long databitlen;    /*the message size in bits*/
 	unsigned long long datasize_in_buffer;      /*the size of the message remained in buffer; assumed to be multiple of 8bits except for the last partial block at the end of the message*/
 	DATA_ALIGN16(uint64 x[8][2]);     /*the 1024-bit state, ( x[i][0] || x[i][1] ) is the ith row of the state in the pseudocode*/
-	unsigned char buffer[64];         /*the 512-bit message block to be hashed;*/
+	DATA_ALIGN16(unsigned char buffer[64]);         /*the 512-bit message block to be hashed;*/
 } hashState;


@@ -213,16 +213,24 @@ static void E8(hashState *state)
 /*The compression function F8 */
 static void F8(hashState *state)
 {
-      uint64  i;
+      uint64_t* x = (uint64_t*)state->x;

      /*xor the 512-bit message with the fist half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[i >> 1][i & 1] ^= ((uint64*)state->buffer)[i];
+      for (int i = 0; i < 8; ++i) {
+            uint64 b;
+            memcpy(&b, &state->buffer[i << 3], sizeof(b));
+            x[i] ^= b;
+      }

      /*the bijective function E8 */
      E8(state);

      /*xor the 512-bit message with the second half of the 1024-bit hash state*/
-      for (i = 0; i < 8; i++)  state->x[(8+i) >> 1][(8+i) & 1] ^= ((uint64*)state->buffer)[i];
+      for (int i = 0; i < 8; ++i) {
+            uint64 b;
+            memcpy(&b, &state->buffer[i << 3], sizeof(b));
+            x[i + 8] ^= b;
+      }
 }

 /*before hashing a message, initialize the hash state as H0 */
@@ -240,6 +248,7 @@ static HashReturn Init(hashState *state, int hashbitlen)
            case 224: memcpy(state->x,JH224_H0,128); break;
            case 256: memcpy(state->x,JH256_H0,128); break;
            case 384: memcpy(state->x,JH384_H0,128); break;
+            default:
            case 512: memcpy(state->x,JH512_H0,128); break;
      }

@@ -43,32 +43,41 @@

 #define RX_LOGCAT	"randomx"

+// Report large page allocation failures as debug messages
+#define alloc_err_msg(x) mdebug(RX_LOGCAT, x);
+
+static CTHR_RWLOCK_TYPE main_dataset_lock = CTHR_RWLOCK_INIT;
+static CTHR_RWLOCK_TYPE main_cache_lock = CTHR_RWLOCK_INIT;
+
+static randomx_dataset *main_dataset = NULL;
+static randomx_cache *main_cache = NULL;
+static char main_seedhash[HASH_SIZE];
+static int main_seedhash_set = 0;
+
+static CTHR_RWLOCK_TYPE secondary_cache_lock = CTHR_RWLOCK_INIT;
+
+static randomx_cache *secondary_cache = NULL;
+static char secondary_seedhash[HASH_SIZE];
+static int secondary_seedhash_set = 0;
+
 #if defined(_MSC_VER)
 #define THREADV __declspec(thread)
 #else
 #define THREADV __thread
 #endif

-typedef struct rx_state {
-  CTHR_MUTEX_TYPE rs_mutex;
-  char rs_hash[HASH_SIZE];
-  uint64_t  rs_height;
-  randomx_cache *rs_cache;
-} rx_state;
+static THREADV randomx_vm *main_vm_full = NULL;
+static THREADV randomx_vm *main_vm_light = NULL;
+static THREADV randomx_vm *secondary_vm_light = NULL;

-static CTHR_MUTEX_TYPE rx_mutex = CTHR_MUTEX_INIT;
-static CTHR_MUTEX_TYPE rx_dataset_mutex = CTHR_MUTEX_INIT;
+static THREADV uint32_t miner_thread = 0;

-static rx_state rx_s[2] = {{CTHR_MUTEX_INIT,{0},0,0},{CTHR_MUTEX_INIT,{0},0,0}};
-
-static randomx_dataset *rx_dataset;
-static int rx_dataset_nomem;
-static int rx_dataset_nolp;
-static uint64_t rx_dataset_height;
-static THREADV randomx_vm *rx_vm = NULL;
+static bool is_main(const char* seedhash) { return main_seedhash_set && (memcmp(seedhash, main_seedhash, HASH_SIZE) == 0); }
+static bool is_secondary(const char* seedhash) { return secondary_seedhash_set && (memcmp(seedhash, secondary_seedhash, HASH_SIZE) == 0); }

 static void local_abort(const char *msg)
 {
+  merror(RX_LOGCAT, "%s", msg);
  fprintf(stderr, "%s\n", msg);
 #ifdef NDEBUG
  _exit(1);
@@ -77,6 +86,16 @@ static void local_abort(const char *msg)
 #endif
 }

+static void hash2hex(const char* hash, char* hex) {
+  const char* d = "0123456789abcdef";
+  for (int i = 0; i < HASH_SIZE; ++i) {
+    const uint8_t b = hash[i];
+    hex[i * 2 + 0] = d[b >> 4];
+    hex[i * 2 + 1] = d[b & 15];
+  }
+  hex[HASH_SIZE * 2] = '\0';
+}
+
 static inline int disabled_flags(void) {
  static int flags = -1;

@@ -157,19 +176,6 @@ static unsigned int get_seedhash_epoch_blocks(void)
  return blocks;
 }

-void rx_reorg(const uint64_t split_height) {
-  int i;
-  CTHR_MUTEX_LOCK(rx_mutex);
-  for (i=0; i<2; i++) {
-    if (split_height <= rx_s[i].rs_height) {
-      if (rx_s[i].rs_height == rx_dataset_height)
-        rx_dataset_height = 1;
-      rx_s[i].rs_height = 1;	/* set to an invalid seed height */
-    }
-  }
-  CTHR_MUTEX_UNLOCK(rx_mutex);
-}
-
 uint64_t rx_seedheight(const uint64_t height) {
  const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag();
  const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks();
@@ -183,6 +189,103 @@ void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nexth
  *nextheight = rx_seedheight(height + get_seedhash_epoch_lag());
 }

+static void rx_alloc_dataset(randomx_flags flags, randomx_dataset** dataset, int ignore_env)
+{
+  if (*dataset) {
+    return;
+  }
+
+  if (disabled_flags() & RANDOMX_FLAG_FULL_MEM) {
+    static int shown = 0;
+    if (!shown) {
+      shown = 1;
+      minfo(RX_LOGCAT, "RandomX dataset is disabled by MONERO_RANDOMX_UMASK environment variable.");
+    }
+    return;
+  }
+
+  if (!ignore_env && !getenv("MONERO_RANDOMX_FULL_MEM")) {
+    static int shown = 0;
+    if (!shown) {
+      shown = 1;
+      minfo(RX_LOGCAT, "RandomX dataset is not enabled by default. Use MONERO_RANDOMX_FULL_MEM environment variable to enable it.");
+    }
+    return;
+  }
+
+  *dataset = randomx_alloc_dataset((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags());
+  if (!*dataset) {
+    alloc_err_msg("Couldn't allocate RandomX dataset using large pages");
+    *dataset = randomx_alloc_dataset(flags & ~disabled_flags());
+    if (!*dataset) {
+      merror(RX_LOGCAT, "Couldn't allocate RandomX dataset");
+    }
+  }
+}
+
+static void rx_alloc_cache(randomx_flags flags, randomx_cache** cache)
+{
+  if (*cache) {
+    return;
+  }
+
+  *cache = randomx_alloc_cache((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags());
+  if (!*cache) {
+    alloc_err_msg("Couldn't allocate RandomX cache using large pages");
+    *cache = randomx_alloc_cache(flags & ~disabled_flags());
+    if (!*cache) local_abort("Couldn't allocate RandomX cache");
+  }
+}
+
+static void rx_init_full_vm(randomx_flags flags, randomx_vm** vm)
+{
+  if (*vm || !main_dataset || (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) {
+    return;
+  }
+
+  if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) {
+    flags |= RANDOMX_FLAG_SECURE;
+  }
+
+  *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset);
+  if (!*vm) {
+    static int shown = 0;
+    if (!shown) {
+        shown = 1;
+        alloc_err_msg("Couldn't allocate RandomX full VM using large pages (will print only once)");
+    }
+    *vm = randomx_create_vm((flags | RANDOMX_FLAG_FULL_MEM) & ~disabled_flags(), NULL, main_dataset);
+    if (!*vm) {
+      merror(RX_LOGCAT, "Couldn't allocate RandomX full VM");
+    }
+  }
+}
+
+static void rx_init_light_vm(randomx_flags flags, randomx_vm** vm, randomx_cache* cache)
+{
+  if (*vm) {
+    randomx_vm_set_cache(*vm, cache);
+    return;
+  }
+
+  if ((flags & RANDOMX_FLAG_JIT) && !miner_thread) {
+    flags |= RANDOMX_FLAG_SECURE;
+  }
+
+  flags &= ~RANDOMX_FLAG_FULL_MEM;
+
+  *vm = randomx_create_vm((flags | RANDOMX_FLAG_LARGE_PAGES) & ~disabled_flags(), cache, NULL);
+  if (!*vm) {
+    static int shown = 0;
+    if (!shown) {
+        shown = 1;
+        alloc_err_msg("Couldn't allocate RandomX light VM using large pages (will print only once)");
+    }
+    *vm = randomx_create_vm(flags & ~disabled_flags(), cache, NULL);
+    if (!*vm) local_abort("Couldn't allocate RandomX light VM");
+  }
+}
+
 typedef struct seedinfo {
  randomx_cache *si_cache;
  unsigned long si_start;
@@ -191,187 +294,231 @@ typedef struct seedinfo {

 static CTHR_THREAD_RTYPE rx_seedthread(void *arg) {
  seedinfo *si = arg;
-  randomx_init_dataset(rx_dataset, si->si_cache, si->si_start, si->si_count);
+  randomx_init_dataset(main_dataset, si->si_cache, si->si_start, si->si_count);
  CTHR_THREAD_RETURN;
 }

-static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_t seedheight) {
-  if (miners > 1) {
-    unsigned long delta = randomx_dataset_item_count() / miners;
-    unsigned long start = 0;
-    int i;
-    seedinfo *si;
-    CTHR_THREAD_TYPE *st;
-    si = malloc(miners * sizeof(seedinfo));
-    if (si == NULL)
-      local_abort("Couldn't allocate RandomX mining threadinfo");
-    st = malloc(miners * sizeof(CTHR_THREAD_TYPE));
-    if (st == NULL) {
-      free(si);
-      local_abort("Couldn't allocate RandomX mining threadlist");
-    }
-    for (i=0; i<miners-1; i++) {
-      si[i].si_cache = rs_cache;
-      si[i].si_start = start;
-      si[i].si_count = delta;
-      start += delta;
-    }
-    si[i].si_cache = rs_cache;
+static void rx_init_dataset(size_t max_threads) {
+  if (!main_dataset) {
+    return;
+  }
+
+  // leave 2 CPU cores for other tasks
+  const size_t num_threads = (max_threads < 4) ? 1 : (max_threads - 2);
+  seedinfo* si = malloc(num_threads * sizeof(seedinfo));
+  if (!si) local_abort("Couldn't allocate RandomX mining threadinfo");
+
+  const uint32_t delta = randomx_dataset_item_count() / num_threads;
+  uint32_t start = 0;
+
+  const size_t n1 = num_threads - 1;
+  for (size_t i = 0; i < n1; ++i) {
+    si[i].si_cache = main_cache;
    si[i].si_start = start;
-    si[i].si_count = randomx_dataset_item_count() - start;
-    for (i=1; i<miners; i++) {
-      CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i]);
-    }
-    randomx_init_dataset(rx_dataset, rs_cache, 0, si[0].si_count);
-    for (i=1; i<miners; i++) {
-      CTHR_THREAD_JOIN(st[i]);
-    }
-    free(st);
-    free(si);
-  } else {
-    randomx_init_dataset(rx_dataset, rs_cache, 0, randomx_dataset_item_count());
+    si[i].si_count = delta;
+    start += delta;
  }
-  rx_dataset_height = seedheight;
+
+  si[n1].si_cache = main_cache;
+  si[n1].si_start = start;
+  si[n1].si_count = randomx_dataset_item_count() - start;
+
+  CTHR_THREAD_TYPE *st = malloc(num_threads * sizeof(CTHR_THREAD_TYPE));
+  if (!st) local_abort("Couldn't allocate RandomX mining threadlist");
+
+  CTHR_RWLOCK_LOCK_READ(main_cache_lock);
+  for (size_t i = 0; i < n1; ++i) {
+    if (!CTHR_THREAD_CREATE(st[i], rx_seedthread, &si[i])) {
+      local_abort("Couldn't start RandomX seed thread");
+    }
+  }
+  randomx_init_dataset(main_dataset, si[n1].si_cache, si[n1].si_start, si[n1].si_count);
+  for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]);
+  CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
+
+  free(st);
+  free(si);
+
+  minfo(RX_LOGCAT, "RandomX dataset initialized");
 }

-void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length,
-  char *hash, int miners, int is_alt) {
-  uint64_t s_height = rx_seedheight(mainheight);
-  int toggle = (s_height & get_seedhash_epoch_blocks()) != 0;
-  randomx_flags flags = enabled_flags() & ~disabled_flags();
-  rx_state *rx_sp;
-  randomx_cache *cache;
+typedef struct thread_info {
+  char seedhash[HASH_SIZE];
+  size_t max_threads;
+} thread_info;

-  CTHR_MUTEX_LOCK(rx_mutex);
+static CTHR_THREAD_RTYPE rx_set_main_seedhash_thread(void *arg) {
+  thread_info* info = arg;

-  /* if alt block but with same seed as mainchain, no need for alt cache */
-  if (is_alt) {
-    if (s_height == seedheight && !memcmp(rx_s[toggle].rs_hash, seedhash, HASH_SIZE))
-      is_alt = 0;
-  } else {
-  /* RPC could request an earlier block on mainchain */
-    if (s_height > seedheight)
-      is_alt = 1;
-    /* miner can be ahead of mainchain */
-    else if (s_height < seedheight)
-      toggle ^= 1;
+  CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock);
+  CTHR_RWLOCK_LOCK_WRITE(main_cache_lock);
+
+  // Double check that seedhash wasn't already updated
+  if (is_main(info->seedhash)) {
+    CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock);
+    CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+    free(info);
+    CTHR_THREAD_RETURN;
+  }
+  memcpy(main_seedhash, info->seedhash, HASH_SIZE);
+  main_seedhash_set = 1;
+
+  char buf[HASH_SIZE * 2 + 1];
+  hash2hex(main_seedhash, buf);
+  minfo(RX_LOGCAT, "RandomX new main seed hash is %s", buf);
+
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  rx_alloc_dataset(flags, &main_dataset, 0);
+  rx_alloc_cache(flags, &main_cache);
+
+  randomx_init_cache(main_cache, info->seedhash, HASH_SIZE);
+  minfo(RX_LOGCAT, "RandomX main cache initialized");
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_cache_lock);
+
+  // From this point, rx_slow_hash can calculate hashes in light mode, but dataset is not initialized yet
+  rx_init_dataset(info->max_threads);
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+
+  free(info);
+  CTHR_THREAD_RETURN;
+}
+
+void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads) {
+  // Early out if seedhash didn't change
+  if (is_main(seedhash)) {
+    return;
  }

-  toggle ^= (is_alt != 0);
+  // Update main cache and dataset in the background
+  thread_info* info = malloc(sizeof(thread_info));
+  if (!info) local_abort("Couldn't allocate RandomX mining threadinfo");

-  rx_sp = &rx_s[toggle];
-  CTHR_MUTEX_LOCK(rx_sp->rs_mutex);
-  CTHR_MUTEX_UNLOCK(rx_mutex);
+  memcpy(info->seedhash, seedhash, HASH_SIZE);
+  info->max_threads = max_dataset_init_threads;

-  cache = rx_sp->rs_cache;
-  if (cache == NULL) {
-    if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) {
-      cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES);
-      if (cache == NULL) {
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache");
-      }
-    }
-    if (cache == NULL) {
-      cache = randomx_alloc_cache(flags);
-      if (cache == NULL)
-        local_abort("Couldn't allocate RandomX cache");
-    }
+  CTHR_THREAD_TYPE t;
+  if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) {
+    local_abort("Couldn't start RandomX seed thread");
  }
-  if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) {
-    randomx_init_cache(cache, seedhash, HASH_SIZE);
-    rx_sp->rs_cache = cache;
-    rx_sp->rs_height = seedheight;
-    memcpy(rx_sp->rs_hash, seedhash, HASH_SIZE);
-  }
-  if (rx_vm == NULL) {
-    if ((flags & RANDOMX_FLAG_JIT) && !miners) {
-        flags |= RANDOMX_FLAG_SECURE & ~disabled_flags();
-    }
-    if (miners && (disabled_flags() & RANDOMX_FLAG_FULL_MEM)) {
-      miners = 0;
-    }
-    if (miners) {
-      CTHR_MUTEX_LOCK(rx_dataset_mutex);
-      if (!rx_dataset_nomem) {
-        if (rx_dataset == NULL) {
-          if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) {
-            rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES);
-            if (rx_dataset == NULL) {
-              mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset");
-            }
-          }
-          if (rx_dataset == NULL)
-            rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT);
-          if (rx_dataset != NULL)
-            rx_initdata(rx_sp->rs_cache, miners, seedheight);
+  CTHR_THREAD_CLOSE(t);
+}
+
+void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) {
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  int success = 0;
+
+  // Fast path (seedhash == main_seedhash)
+  // Multiple threads can run in parallel in fast or light mode, 1-2 ms or 10-15 ms per hash per thread
+  if (is_main(seedhash)) {
+    // If CTHR_RWLOCK_TRYLOCK_READ fails it means dataset is being initialized now, so use the light mode
+    if (main_dataset && CTHR_RWLOCK_TRYLOCK_READ(main_dataset_lock)) {
+      // Double check that main_seedhash didn't change
+      if (is_main(seedhash)) {
+        rx_init_full_vm(flags, &main_vm_full);
+        if (main_vm_full) {
+          randomx_calculate_hash(main_vm_full, data, length, result_hash);
+          success = 1;
        }
      }
-      if (rx_dataset != NULL)
-        flags |= RANDOMX_FLAG_FULL_MEM;
-      else {
-        miners = 0;
-        if (!rx_dataset_nomem) {
-          rx_dataset_nomem = 1;
-          mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner");
-        }
+      CTHR_RWLOCK_UNLOCK_READ(main_dataset_lock);
+    } else {
+      CTHR_RWLOCK_LOCK_READ(main_cache_lock);
+      // Double check that main_seedhash didn't change
+      if (is_main(seedhash)) {
+        rx_init_light_vm(flags, &main_vm_light, main_cache);
+        randomx_calculate_hash(main_vm_light, data, length, result_hash);
+        success = 1;
      }
-      CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
+      CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
    }
-    if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES) && !rx_dataset_nolp) {
-      rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset);
-      if(rx_vm == NULL) { //large pages failed
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM");
-        rx_dataset_nolp = 1;
-      }
-    }
-    if (rx_vm == NULL)
-      rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
-    if(rx_vm == NULL) {//fallback if everything fails
-      flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0);
-      rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
-    }
-    if (rx_vm == NULL)
-      local_abort("Couldn't allocate RandomX VM");
-  } else if (miners) {
-    CTHR_MUTEX_LOCK(rx_dataset_mutex);
-    if (rx_dataset != NULL && rx_dataset_height != seedheight)
-      rx_initdata(cache, miners, seedheight);
-    else if (rx_dataset == NULL) {
-      /* this is a no-op if the cache hasn't changed */
-      randomx_vm_set_cache(rx_vm, rx_sp->rs_cache);
-    }
-    CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
-  } else {
-    /* this is a no-op if the cache hasn't changed */
-    randomx_vm_set_cache(rx_vm, rx_sp->rs_cache);
  }
-  /* mainchain users can run in parallel */
-  if (!is_alt)
-    CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex);
-  randomx_calculate_hash(rx_vm, data, length, hash);
-  /* altchain slot users always get fully serialized */
-  if (is_alt)
-    CTHR_MUTEX_UNLOCK(rx_sp->rs_mutex);
+
+  if (success) {
+    return;
+  }
+
+  char buf[HASH_SIZE * 2 + 1];
+
+  // Slow path (seedhash != main_seedhash, but seedhash == secondary_seedhash)
+  // Multiple threads can run in parallel in light mode, 10-15 ms per hash per thread
+  if (!secondary_cache) {
+    CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock);
+    if (!secondary_cache) {
+      hash2hex(seedhash, buf);
+      minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf);
+
+      rx_alloc_cache(flags, &secondary_cache);
+      randomx_init_cache(secondary_cache, seedhash, HASH_SIZE);
+      minfo(RX_LOGCAT, "RandomX secondary cache updated");
+      memcpy(secondary_seedhash, seedhash, HASH_SIZE);
+      secondary_seedhash_set = 1;
+    }
+    CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock);
+  }
+
+  CTHR_RWLOCK_LOCK_READ(secondary_cache_lock);
+  if (is_secondary(seedhash)) {
+    rx_init_light_vm(flags, &secondary_vm_light, secondary_cache);
+    randomx_calculate_hash(secondary_vm_light, data, length, result_hash);
+    success = 1;
+  }
+  CTHR_RWLOCK_UNLOCK_READ(secondary_cache_lock);
+
+  if (success) {
+    return;
+  }
+
+  // Slowest path (seedhash != main_seedhash, seedhash != secondary_seedhash)
+  // Only one thread runs at a time and updates secondary_seedhash if needed, up to 200-500 ms per hash
+  CTHR_RWLOCK_LOCK_WRITE(secondary_cache_lock);
+  if (!is_secondary(seedhash)) {
+    hash2hex(seedhash, buf);
+    minfo(RX_LOGCAT, "RandomX new secondary seed hash is %s", buf);
+
+    randomx_init_cache(secondary_cache, seedhash, HASH_SIZE);
+    minfo(RX_LOGCAT, "RandomX secondary cache updated");
+    memcpy(secondary_seedhash, seedhash, HASH_SIZE);
+    secondary_seedhash_set = 1;
+  }
+  rx_init_light_vm(flags, &secondary_vm_light, secondary_cache);
+  randomx_calculate_hash(secondary_vm_light, data, length, result_hash);
+  CTHR_RWLOCK_UNLOCK_WRITE(secondary_cache_lock);
 }

-void rx_slow_hash_allocate_state(void) {
+void rx_set_miner_thread(uint32_t value, size_t max_dataset_init_threads) {
+  miner_thread = value;
+
+  // If dataset is not allocated yet, try to allocate and initialize it
+  CTHR_RWLOCK_LOCK_WRITE(main_dataset_lock);
+  if (main_dataset) {
+    CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
+    return;
+  }
+
+  const randomx_flags flags = enabled_flags() & ~disabled_flags();
+  rx_alloc_dataset(flags, &main_dataset, 1);
+  rx_init_dataset(max_dataset_init_threads);
+
+  CTHR_RWLOCK_UNLOCK_WRITE(main_dataset_lock);
 }

-void rx_slow_hash_free_state(void) {
-  if (rx_vm != NULL) {
-    randomx_destroy_vm(rx_vm);
-    rx_vm = NULL;
+uint32_t rx_get_miner_thread() {
+  return miner_thread;
+}
+
+void rx_slow_hash_allocate_state() {}
+
+static void rx_destroy_vm(randomx_vm** vm) {
+  if (*vm) {
+    randomx_destroy_vm(*vm);
+    *vm = NULL;
  }
 }

-void rx_stop_mining(void) {
-  CTHR_MUTEX_LOCK(rx_dataset_mutex);
-  if (rx_dataset != NULL) {
-    randomx_dataset *rd = rx_dataset;
-    rx_dataset = NULL;
-    randomx_release_dataset(rd);
-  }
-  rx_dataset_nomem = 0;
-  rx_dataset_nolp = 0;
-  CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
+void rx_slow_hash_free_state() {
+  rx_destroy_vm(&main_vm_full);
+  rx_destroy_vm(&main_vm_light);
+  rx_destroy_vm(&secondary_vm_light);
 }
@@ -1229,7 +1229,7 @@ namespace cryptonote
    char *end = NULL;
    errno = 0;
    const unsigned long long ull = strtoull(buf, &end, 10);
-    CHECK_AND_ASSERT_THROW_MES(ull != ULONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
+    CHECK_AND_ASSERT_THROW_MES(ull != ULLONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
    CHECK_AND_ASSERT_THROW_MES(ull != 0 || amount == 0, "Overflow in rounding");
    return ull;
  }
@@ -82,6 +82,7 @@
 using namespace epee;

 #include "miner.h"
+#include "crypto/hash.h"


 extern "C" void slow_hash_allocate_state();
@@ -436,7 +437,6 @@ namespace cryptonote
  {
    m_stop = true;
  }
-  extern "C" void rx_stop_mining(void);
  //-----------------------------------------------------------------------------------------------------
  bool miner::stop()
  {
@@ -469,7 +469,6 @@ namespace cryptonote
    MINFO("Mining has been stopped, " << m_threads.size() << " finished" );
    m_threads.clear();
    m_threads_autodetect.clear();
-    rx_stop_mining();
    return true;
  }
  //-----------------------------------------------------------------------------------------------------
@@ -524,6 +523,8 @@ namespace cryptonote
  bool miner::worker_thread()
  {
    const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment
+    bool rx_set = false;
+
    MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]");
    MGINFO("Miner thread was started ["<< th_local_index << "]");
    uint32_t nonce = m_starter_nonce + th_local_index;
@@ -574,6 +575,13 @@ namespace cryptonote

      b.nonce = nonce;
      crypto::hash h;
+
+      if ((b.major_version >= RX_BLOCK_VERSION) && !rx_set)
+      {
+        crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
+        rx_set = true;
+      }
+
      m_gbh(b, height, NULL, tools::get_max_concurrency(), h);

      if(check_hash(h, local_diff))
@@ -52,7 +52,7 @@ namespace cryptonote

    // load
    template <template <bool> class Archive>
-    bool do_serialize(Archive<false>& ar)
+    bool member_do_serialize(Archive<false>& ar)
    {
      // size - 1 - because of variant tag
      for (size = 1; size <= TX_EXTRA_PADDING_MAX_COUNT; ++size)
@@ -73,7 +73,7 @@ namespace cryptonote

    // store
    template <template <bool> class Archive>
-    bool do_serialize(Archive<true>& ar)
+    bool member_do_serialize(Archive<true>& ar)
    {
      if(TX_EXTRA_PADDING_MAX_COUNT < size)
        return false;
@@ -129,7 +129,7 @@ namespace cryptonote

    // load
    template <template <bool> class Archive>
-    bool do_serialize(Archive<false>& ar)
+    bool member_do_serialize(Archive<false>& ar)
    {
      std::string field;
      if(!::do_serialize(ar, field))
@@ -142,7 +142,7 @@ namespace cryptonote

    // store
    template <template <bool> class Archive>
-    bool do_serialize(Archive<true>& ar)
+    bool member_do_serialize(Archive<true>& ar)
    {
      std::ostringstream oss;
      binary_archive<true> oar(oss);
@@ -42,7 +42,12 @@ namespace cryptonote
    static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none");

    relay_method m_relay; // gives indication on how tx should be relayed (if at all)
-    bool m_verifivation_failed; //bad tx, should drop connection
+    bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense
+    // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop
+    // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be
+    // confused with breaking soft fork rules, because tx could be later added to the chain if mined
+    // because it does not violate consensus rules.
+    bool m_no_drop_offense;
    bool m_verifivation_impossible; //the transaction is related with an alternative blockchain
    bool m_added_to_pool; 
    bool m_low_mixin;
@@ -53,6 +58,7 @@ namespace cryptonote
    bool m_overspend;
    bool m_fee_too_low;
    bool m_too_few_outputs;
+    bool m_tx_extra_too_big;
  };

  struct block_verification_context
@@ -30,6 +30,7 @@

 #pragma once

+#include <cstdint>
 #include <stdexcept>
 #include <string>
 #include <boost/uuid/uuid.hpp>
@@ -206,6 +207,11 @@

 #define DNS_BLOCKLIST_LIFETIME (86400 * 8)

+//The limit is enough for the mandatory transaction content with 16 outputs (547 bytes),
+//a custom tag (1 byte) and up to 32 bytes of custom data for each recipient.
+// (1+32) + (1+1+16*32) + (1+16*32) = 1060
+#define MAX_TX_EXTRA_SIZE                       1060
+
 // New constants are intended to go here
 namespace config
 {
@@ -248,6 +254,7 @@ namespace config
  const unsigned char HASH_KEY_MM_SLOT = 'm';
  const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS_SEED[] = "multisig_tx_privkeys_seed";
  const constexpr char HASH_KEY_MULTISIG_TX_PRIVKEYS[] = "multisig_tx_privkeys";
+  const constexpr char HASH_KEY_TXHASH_AND_MIXRING[] = "txhash_and_mixring";

  // Multisig
  const uint32_t MULTISIG_MAX_SIGNERS{16};
@@ -31,7 +31,9 @@ set(cryptonote_core_sources
  cryptonote_core.cpp
  tx_pool.cpp
  tx_sanity_check.cpp
-  cryptonote_tx_utils.cpp)
+  cryptonote_tx_utils.cpp
+  tx_verification_utils.cpp
+)

 set(cryptonote_core_headers)

@@ -57,6 +57,7 @@
 #include "common/notify.h"
 #include "common/varint.h"
 #include "common/pruning.h"
+#include "common/data_cache.h"
 #include "time_helper.h"

 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -98,7 +99,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) :
  m_difficulty_for_next_block(1),
  m_btc_valid(false),
  m_batch_success(true),
-  m_prepare_height(0)
+  m_prepare_height(0),
+  m_rct_ver_cache()
 {
  LOG_PRINT_L3("Blockchain::" << __func__);
 }
@@ -456,6 +458,14 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline
    if (!update_next_cumulative_weight_limit())
      return false;
  }
+
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  {
+    const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height()));
+    if (seedhash != crypto::null_hash)
+      rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+  }
+
  return true;
 }
 //------------------------------------------------------------------
@@ -570,6 +580,12 @@ void Blockchain::pop_blocks(uint64_t nblocks)

  if (stop_batch)
    m_db->batch_stop();
+
+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+  {
+    const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(m_db->height()));
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+  }
 }
 //------------------------------------------------------------------
 // This function tells BlockchainDB to remove the top block from the
@@ -1239,18 +1255,20 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
  }

  m_hardfork->reorganize_from_chain_height(split_height);
-  get_block_longhash_reorg(split_height);

  std::shared_ptr<tools::Notify> reorg_notify = m_reorg_notify;
  if (reorg_notify)
    reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(),
        "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL);

+  const uint64_t new_height = m_db->height();
+  const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+
  crypto::hash prev_id;
  if (!get_block_hash(alt_chain.back().bl, prev_id))
    MERROR("Failed to get block hash of an alternative chain's tip");
  else
-    send_miner_notifications(prev_id, alt_chain.back().already_generated_coins);
+    send_miner_notifications(new_height, seedhash, prev_id, alt_chain.back().already_generated_coins);

  for (const auto& notifier : m_block_notifiers)
  {
@@ -1262,6 +1280,9 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
    }
  }

+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+
  MGINFO_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height());
  return true;
 }
@@ -2001,7 +2022,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
      {
        seedhash = get_block_id_by_height(seedheight);
      }
-      get_altblock_longhash(bei.bl, proof_of_work, get_current_blockchain_height(), bei.height, seedheight, seedhash);
+      get_altblock_longhash(bei.bl, proof_of_work, seedhash);
    } else
    {
      get_block_longhash(this, bei.bl, proof_of_work, bei.height, 0);
@@ -2044,7 +2065,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
      cryptonote::blobdata blob;
      if (m_tx_pool.have_tx(txid, relay_category::legacy))
      {
-        if (m_tx_pool.get_transaction_info(txid, td))
+        if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
        {
          bei.block_cumulative_weight += td.weight;
        }
@@ -3192,7 +3213,7 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
  }
  return false;
 }
-bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const
+bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys)
 {
  PERF_TIMER(expand_transaction_2);
  CHECK_AND_ASSERT_MES(tx.version == 2, false, "Transaction version is not 2");
@@ -3515,6 +3536,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
        false, "Transaction spends at least one output which is too young");
  }

+  // Warn that new RCT types are present, and thus the cache is not being used effectively
+  static constexpr const std::uint8_t RCT_CACHE_TYPE = rct::RCTTypeBulletproofPlus;
+  if (tx.rct_signatures.type > RCT_CACHE_TYPE)
+  {
+    MWARNING("RCT cache is not caching new verification results. Please update RCT_CACHE_TYPE!");
+  }
+
  if (tx.version == 1)
  {
    if (threads > 1)
@@ -3536,12 +3564,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
  }
  else
  {
-    if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
-    {
-      MERROR_VER("Failed to expand rct signatures!");
-      return false;
-    }
-
    // from version 2, check ringct signatures
    // obviously, the original and simple rct APIs use a mixRing that's indexes
    // in opposite orders, because it'd be too simple otherwise...
@@ -3559,61 +3581,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
    case rct::RCTTypeCLSAG:
    case rct::RCTTypeBulletproofPlus:
    {
-      // check all this, either reconstructed (so should really pass), or not
-      {
-        if (pubkeys.size() != rv.mixRing.size())
-        {
-          MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-          return false;
-        }
-        for (size_t i = 0; i < pubkeys.size(); ++i)
-        {
-          if (pubkeys[i].size() != rv.mixRing[i].size())
-          {
-            MERROR_VER("Failed to check ringct signatures: mismatched pubkeys/mixRing size");
-            return false;
-          }
-        }
-
-        for (size_t n = 0; n < pubkeys.size(); ++n)
-        {
-          for (size_t m = 0; m < pubkeys[n].size(); ++m)
-          {
-            if (pubkeys[n][m].dest != rct::rct2pk(rv.mixRing[n][m].dest))
-            {
-              MERROR_VER("Failed to check ringct signatures: mismatched pubkey at vin " << n << ", index " << m);
-              return false;
-            }
-            if (pubkeys[n][m].mask != rct::rct2pk(rv.mixRing[n][m].mask))
-            {
-              MERROR_VER("Failed to check ringct signatures: mismatched commitment at vin " << n << ", index " << m);
-              return false;
-            }
-          }
-        }
-      }
-
-      const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size();
-      if (n_sigs != tx.vin.size())
-      {
-        MERROR_VER("Failed to check ringct signatures: mismatched MGs/vin sizes");
-        return false;
-      }
-      for (size_t n = 0; n < tx.vin.size(); ++n)
-      {
-        bool error;
-        if (rct::is_rct_clsag(rv.type))
-          error = memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.CLSAGs[n].I, 32);
-        else
-          error = rv.p.MGs[n].II.empty() || memcmp(&boost::get<txin_to_key>(tx.vin[n]).k_image, &rv.p.MGs[n].II[0], 32);
-        if (error)
-        {
-          MERROR_VER("Failed to check ringct signatures: mismatched key image");
-          return false;
-        }
-      }
-
-      if (!rct::verRctNonSemanticsSimple(rv))
+      if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, RCT_CACHE_TYPE))
      {
        MERROR_VER("Failed to check ringct signatures!");
        return false;
@@ -3622,6 +3590,12 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
    }
    case rct::RCTTypeFull:
    {
+      if (!expand_transaction_2(tx, tx_prefix_hash, pubkeys))
+      {
+        MERROR_VER("Failed to expand rct signatures!");
+        return false;
+      }
+
      // check all this, either reconstructed (so should really pass), or not
      {
        bool size_matches = true;
@@ -3736,7 +3710,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b
      div128_64(hi, lo, median_block_weight, &hi, &lo, NULL, NULL);
      assert(hi == 0);
      lo -= lo / 20;
-      return lo;
+      return lo == 0 ? 1 : lo;
    }
    else
    {
@@ -3858,6 +3832,8 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block
  const uint8_t version = get_current_hard_fork_version();
  const uint64_t db_height = m_db->height();

+  CHECK_AND_ASSERT_THROW_MES(grace_blocks <= CRYPTONOTE_REWARD_BLOCKS_WINDOW, "Grace blocks invalid In 2021 fee scaling estimate.");
+
  // we want Mlw = median of max((min(Mbw, 1.7 * Ml), Zm), Ml / 1.7)
  // Mbw: block weight for the last 99990 blocks, 0 for the next 10
  // Ml: penalty free zone (dynamic), aka long_term_median, aka median of max((min(Mb, 1.7 * Ml), Zm), Ml / 1.7)
@@ -3871,7 +3847,6 @@ void Blockchain::get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_block
  const uint64_t Mlw_penalty_free_zone_for_wallet = std::max<uint64_t>(rm.median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5);

  // Msw: median over [100 - grace blocks] past + [grace blocks] future blocks
-  CHECK_AND_ASSERT_THROW_MES(grace_blocks <= 100, "Grace blocks invalid In 2021 fee scaling estimate.");
  std::vector<uint64_t> weights;
  get_last_n_blocks_weights(weights, 100 - grace_blocks);
  weights.reserve(100);
@@ -4552,11 +4527,15 @@ leave:
    }
  }

-  send_miner_notifications(id, already_generated_coins);
+  const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+  send_miner_notifications(new_height, seedhash, id, already_generated_coins);

  for (const auto& notifier: m_block_notifiers)
    notifier(new_height - 1, {std::addressof(bl), 1});

+  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
+    rx_set_main_seedhash(seedhash.data, tools::get_max_concurrency());
+
  return true;
 }
 //------------------------------------------------------------------
@@ -4638,40 +4617,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
  }
  else
  {
-    const uint64_t block_weight = m_db->get_block_weight(db_height - 1);
+    const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
+    const uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks);

-    uint64_t long_term_median;
-    if (db_height == 1)
-    {
-      long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
-    }
-    else
-    {
-      uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
-      if (nblocks == db_height)
-        --nblocks;
-      long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks);
-    }
-
-    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
-
-    uint64_t short_term_constraint = m_long_term_effective_median_block_weight;
-    if (hf_version >= HF_VERSION_2021_SCALING)
-      short_term_constraint += m_long_term_effective_median_block_weight * 7 / 10;
-    else
-      short_term_constraint += m_long_term_effective_median_block_weight * 2 / 5;
-    uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint);
-
-    if (db_height == 1)
-    {
-      long_term_median = long_term_block_weight;
-    }
-    else
-    {
-      m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1);
-      m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight);
-      long_term_median = m_long_term_block_weights_cache_rolling_median.median();
-    }
    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);

    std::vector<uint64_t> weights;
@@ -5604,7 +5552,7 @@ void Blockchain::cancel()
 }

 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "52105e0e45013e4c6705d5b7902268f5989b6eea71f5a04160176297a79f4b32";
+static const char expected_block_hashes_hash[] = "374d836b26c46b9d8fdbc977da7a89a43c2eefb65659901ee4d0053d302e1468";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
  if (get_checkpoints == nullptr || !m_fast_sync)
@@ -5761,24 +5709,15 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_
  m_btc_valid = true;
 }

-void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins)
+void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins)
 {
  if (m_miner_notifiers.empty())
    return;

-  const uint64_t height = m_db->height();
  const uint8_t major_version = m_hardfork->get_ideal_version(height);
  const difficulty_type diff = get_difficulty_for_next_block();
  const uint64_t median_weight = m_current_block_cumul_weight_median;

-  crypto::hash seed_hash = crypto::null_hash;
-  if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
-  {
-    uint64_t seed_height, next_height;
-    crypto::rx_seedheights(height, &seed_height, &next_height);
-    seed_hash = get_block_id_by_height(seed_height);
-  }
-
  std::vector<tx_block_template_backlog_entry> tx_backlog;
  m_tx_pool.get_block_template_backlog(tx_backlog);

@@ -57,6 +57,7 @@
 #include "rpc/core_rpc_server_commands_defs.h"
 #include "cryptonote_basic/difficulty.h"
 #include "cryptonote_tx_utils.h"
+#include "tx_verification_utils.h"
 #include "cryptonote_basic/verification_context.h"
 #include "crypto/hash.h"
 #include "checkpoints/checkpoints.h"
@@ -596,6 +597,15 @@ namespace cryptonote
     */
    bool store_blockchain();

+    /**
+     * @brief expands v2 transaction data from blockchain
+     *
+     * RingCT transactions do not transmit some of their data if it
+     * can be reconstituted by the receiver. This function expands
+     * that implicit data.
+     */
+    static bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys);
+
    /**
     * @brief validates a transaction's inputs
     *
@@ -1222,6 +1232,9 @@ namespace cryptonote
    uint64_t m_prepare_nblocks;
    std::vector<block> *m_prepare_blocks;

+    // cache for verifying transaction RCT non semantics
+    mutable rct_ver_cache_t m_rct_ver_cache;
+
    /**
     * @brief collects the keys for all outputs being "spent" as an input
     *
@@ -1574,15 +1587,6 @@ namespace cryptonote
     */
    void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints);

-    /**
-     * @brief expands v2 transaction data from blockchain
-     *
-     * RingCT transactions do not transmit some of their data if it
-     * can be reconstituted by the receiver. This function expands
-     * that implicit data.
-     */
-    bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys) const;
-
    /**
     * @brief invalidates any cached block template
     */
@@ -1598,9 +1602,11 @@ namespace cryptonote
    /**
     * @brief sends new block notifications to ZMQ `miner_data` subscribers
     *
+     * @param height current blockchain height
+     * @param seed_hash seed hash to use for mining
     * @param prev_id hash of new blockchain tip
     * @param already_generated_coins total coins mined by the network so far
     */
-    void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins);
+    void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins);
  };
 }  // namespace cryptonote
@@ -1099,7 +1099,7 @@ namespace cryptonote
      else if(tvc[i].m_verifivation_impossible)
      {MERROR_VER("Transaction verification impossible: " << results[i].hash);}

-      if(tvc[i].m_added_to_pool)
+      if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE)
      {
        MDEBUG("tx added: " << results[i].hash);
        valid_events = true;
@@ -1727,6 +1727,11 @@ namespace cryptonote
    return true;
  }
  //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
+  {
+    return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
+  }
+  //-----------------------------------------------------------------------------------------------
  bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const
  {
    m_mempool.get_transactions(txs, include_sensitive_data);
@@ -1739,6 +1744,11 @@ namespace cryptonote
    return true;
  }
  //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental);
+  }
+  //-----------------------------------------------------------------------------------------------
  bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const
  {
    m_mempool.get_transaction_stats(stats, include_sensitive_data);
@@ -510,6 +510,23 @@ namespace cryptonote
     bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const;

     /**
+      * @copydoc tx_memory_pool::get_pool_transactions_info
+      * @param include_sensitive_txes include private transactions
+      *
+      * @note see tx_memory_pool::get_pool_transactions_info
+      */
+     bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_pool_info
+      * @param include_sensitive_txes include private transactions
+      * @param max_tx_count max allowed added_txs in response
+      *
+      * @note see tx_memory_pool::get_pool_info
+      */
+     bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
+    /**
      * @copydoc tx_memory_pool::get_transactions
      * @param include_sensitive_txes include private transactions
      *
@@ -437,6 +437,8 @@ namespace cryptonote
    if (!sort_tx_extra(tx.extra, tx.extra))
      return false;

+    CHECK_AND_ASSERT_MES(tx.extra.size() <= MAX_TX_EXTRA_SIZE, false, "TX extra size (" << tx.extra.size() << ") is greater than max allowed (" << MAX_TX_EXTRA_SIZE << ")");
+
    //check money
    if(summary_outs_money > summary_inputs_money )
    {
@@ -669,10 +671,10 @@ namespace cryptonote
    return true;
  }
  //---------------------------------------------------------------
-  void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height, const uint64_t seed_height, const crypto::hash& seed_hash)
+  void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash)
  {
    blobdata bd = get_block_hashing_blob(b);
-    rx_slow_hash(main_height, seed_height, seed_hash.data, bd.data(), bd.size(), res.data, 0, 1);
+    rx_slow_hash(seed_hash.data, bd.data(), bd.size(), res.data);
  }

  bool get_block_longhash(const Blockchain *pbc, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners)
@@ -686,20 +688,16 @@ namespace cryptonote
    }
    if (major_version >= RX_BLOCK_VERSION)
    {
-      uint64_t seed_height, main_height;
      crypto::hash hash;
      if (pbc != NULL)
      {
-        seed_height = rx_seedheight(height);
+        const uint64_t seed_height = rx_seedheight(height);
        hash = seed_hash ? *seed_hash : pbc->get_pending_block_id_by_height(seed_height);
-        main_height = pbc->get_current_blockchain_height();
      } else
      {
        memset(&hash, 0, sizeof(hash));  // only happens when generating genesis block
-        seed_height = 0;
-        main_height = 0;
      }
-      rx_slow_hash(main_height, seed_height, hash.data, bd.data(), bd.size(), res.data, seed_hash ? 0 : miners, !!seed_hash);
+      rx_slow_hash(hash.data, bd.data(), bd.size(), res.data);
    } else {
      const int pow_variant = major_version >= 7 ? major_version - 6 : 0;
      crypto::cn_slow_hash(bd.data(), bd.size(), res, pow_variant, height);
@@ -713,20 +711,10 @@ namespace cryptonote
 	return get_block_longhash(pbc, bd, res, height, b.major_version, seed_hash, miners);
  }

-  bool get_block_longhash(const Blockchain *pbc, const block& b, crypto::hash& res, const uint64_t height, const int miners)
-  {
-    return get_block_longhash(pbc, b, res, height, NULL, miners);
-  }
-
-  crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const int miners)
+  crypto::hash get_block_longhash(const Blockchain *pbc, const block& b, const uint64_t height, const crypto::hash *seed_hash, const int miners)
  {
    crypto::hash p = crypto::null_hash;
-    get_block_longhash(pbc, b, p, height, miners);
+    get_block_longhash(pbc, b, p, height, seed_hash, miners);
    return p;
  }
-
-  void get_block_longhash_reorg(const uint64_t split_height)
-  {
-    rx_reorg(split_height);
-  }
 }
@@ -144,14 +144,10 @@ namespace cryptonote
    );

  class Blockchain;
-  bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height,
-    const int major_version, const crypto::hash *seed_hash, const int miners);
-  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const int miners);
-  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash, const int miners);
-  void get_altblock_longhash(const block& b, crypto::hash& res, const uint64_t main_height, const uint64_t height,
-    const uint64_t seed_height, const crypto::hash& seed_hash);
-  crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const int miners);
-  void get_block_longhash_reorg(const uint64_t split_height);
+  bool get_block_longhash(const Blockchain *pb, const blobdata& bd, crypto::hash& res, const uint64_t height, const int major_version, const crypto::hash *seed_hash, const int miners = 0);
+  bool get_block_longhash(const Blockchain *pb, const block& b, crypto::hash& res, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0);
+  crypto::hash get_block_longhash(const Blockchain *pb, const block& b, const uint64_t height, const crypto::hash *seed_hash = nullptr, const int miners = 0);
+  void get_altblock_longhash(const block& b, crypto::hash& res, const crypto::hash& seed_hash);

 }

@@ -133,6 +133,12 @@ namespace cryptonote
    // class code expects unsigned values throughout
    if (m_next_check < time_t(0))
      throw std::runtime_error{"Unexpected time_t (system clock) value"};
+
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_start_time = (time_t)0;
+    // We don't set these to "now" already here as we don't know how long it takes from construction
+    // of the pool until it "goes to work". It's safer to set when the first actual txs enter the
+    // corresponding lists.
  }
  //---------------------------------------------------------------------------------
  bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
@@ -207,6 +213,7 @@ namespace cryptonote
    {
      tvc.m_verifivation_failed = true;
      tvc.m_fee_too_low = true;
+      tvc.m_no_drop_offense = true;
      return false;
    }

@@ -219,6 +226,16 @@ namespace cryptonote
      return false;
    }

+    size_t tx_extra_size = tx.extra.size();
+    if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE)
+    {
+      LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
+      tvc.m_verifivation_failed = true;
+      tvc.m_tx_extra_too_big = true;
+      tvc.m_no_drop_offense = true;
+      return false;
+    }
+
    // if the transaction came from a block popped from the chain,
    // don't check if we have its key images as spent.
    // TODO: Investigate why not?
@@ -230,6 +247,7 @@ namespace cryptonote
        LOG_PRINT_L1("Transaction with id= "<< id << " used already spent key images");
        tvc.m_verifivation_failed = true;
        tvc.m_double_spend = true;
+        tvc.m_no_drop_offense = true;
        return false;
      }
    }
@@ -281,7 +299,7 @@ namespace cryptonote
            return false;

          m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time);
          lock.commit();
        }
        catch (const std::exception &e)
@@ -352,7 +370,7 @@ namespace cryptonote

          m_blockchain.remove_txpool_tx(id);
          m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time);
        }
        lock.commit();
      }
@@ -373,7 +391,7 @@ namespace cryptonote

    ++m_cookie;

-    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)));
+    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size());

    prune(m_txpool_max_weight);

@@ -418,8 +436,14 @@ namespace cryptonote
  void tx_memory_pool::prune(size_t bytes)
  {
    CRITICAL_REGION_LOCAL(m_transactions_lock);
+
+    // Nothing to do if already empty
+    if (m_txs_by_fee_and_receive_time.empty())
+      return;
+
    if (bytes == 0)
      bytes = m_txpool_max_weight;
+
    CRITICAL_REGION_LOCAL1(m_blockchain);
    LockedTXN lock(m_blockchain.get_db());
    bool changed = false;
@@ -464,7 +488,13 @@ namespace cryptonote
        reduce_txpool_weight(meta.weight);
        remove_transaction_keyimages(tx, txid);
        MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first);
-        m_txs_by_fee_and_receive_time.erase(it--);
+
+        auto it_prev = it;
+        --it_prev;
+
+        remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted));
+        it = it_prev;
+
        changed = true;
      }
      catch (const std::exception &e)
@@ -546,8 +576,7 @@ namespace cryptonote
    CRITICAL_REGION_LOCAL(m_transactions_lock);
    CRITICAL_REGION_LOCAL1(m_blockchain);

-    auto sorted_it = find_tx_in_sorted_container(id);
-
+    bool sensitive = false;
    try
    {
      LockedTXN lock(m_blockchain.get_db());
@@ -578,6 +607,7 @@ namespace cryptonote
      do_not_relay = meta.do_not_relay;
      double_spend_seen = meta.double_spend_seen;
      pruned = meta.pruned;
+      sensitive = !meta.matches(relay_category::broadcasted);

      // remove first, in case this throws, so key images aren't removed
      m_blockchain.remove_txpool_tx(id);
@@ -591,13 +621,12 @@ namespace cryptonote
      return false;
    }

-    if (sorted_it != m_txs_by_fee_and_receive_time.end())
-      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive);
    ++m_cookie;
    return true;
  }
  //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const
+  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
  {
    PERF_TIMER(get_transaction_info);
    CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -609,7 +638,12 @@ namespace cryptonote
      txpool_tx_meta_t meta;
      if (!m_blockchain.get_txpool_tx_meta(txid, meta))
      {
-        MERROR("Failed to find tx in txpool");
+        LOG_PRINT_L2("Failed to find tx in txpool: " << txid);
+        return false;
+      }
+      if (!include_sensitive_data && !meta.matches(relay_category::broadcasted))
+      {
+        // We don't want sensitive data && the tx is sensitive, so no need to return it
        return false;
      }
      cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
@@ -635,11 +669,13 @@ namespace cryptonote
      td.kept_by_block = meta.kept_by_block;
      td.last_failed_height = meta.last_failed_height;
      td.last_failed_id = meta.last_failed_id;
-      td.receive_time = meta.receive_time;
-      td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time;
+      td.receive_time = include_sensitive_data ? meta.receive_time : 0;
+      td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0;
      td.relayed = meta.relayed;
      td.do_not_relay = meta.do_not_relay;
      td.double_spend_seen = meta.double_spend_seen;
+      if (include_blob)
+        td.tx_blob = std::move(txblob);
    }
    catch (const std::exception &e)
    {
@@ -649,6 +685,25 @@ namespace cryptonote

    return true;
  }
+  //------------------------------------------------------------------
+  bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    txs.clear();
+
+    for (const auto &it: txids)
+    {
+      tx_details details;
+      bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/);
+      if (success)
+      {
+        txs.push_back(std::make_pair(it, std::move(details)));
+      }
+    }
+    return true;
+  }
  //---------------------------------------------------------------------------------
  bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const
  {
@@ -710,15 +765,7 @@ namespace cryptonote
         (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) )
      {
        LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age );
-        auto sorted_it = find_tx_in_sorted_container(txid);
-        if (sorted_it == m_txs_by_fee_and_receive_time.end())
-        {
-          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
-        }
-        else
-        {
-          m_txs_by_fee_and_receive_time.erase(sorted_it);
-        }
+        remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted));
        m_timed_out_transactions.insert(txid);
        remove.push_back(std::make_pair(txid, meta.weight));
      }
@@ -872,9 +919,12 @@ namespace cryptonote
            meta.last_relayed_time = std::chrono::system_clock::to_time_t(now);

          m_blockchain.update_txpool_tx(hash, meta);
-
          // wait until db update succeeds to ensure tx is visible in the pool
          was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted);
+
+          if (was_just_broadcasted)
+            // Make sure the tx gets re-added with an updated time
+            add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now));
        }
      }
      catch (const std::exception &e)
@@ -927,6 +977,81 @@ namespace cryptonote
    }, false, category);
  }
  //------------------------------------------------------------------
+  bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    incremental = true;
+    if (start_time == (time_t)0)
+    {
+      // Giving no start time means give back whole pool
+      incremental = false;
+    }
+    else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0))
+    {
+      if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time))
+      {
+        // If either of the two lists do not go back far enough it's not possible to
+        // deliver incremental pool info
+        incremental = false;
+      }
+      // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time
+    }
+    else
+    {
+      // Some incremental info still missing completely
+      incremental = false;
+    }
+
+    added_txs.clear();
+    remaining_added_txids.clear();
+    removed_txs.clear();
+
+    std::vector<crypto::hash> txids;
+    if (!incremental)
+    {
+      LOG_PRINT_L2("Giving back the whole pool");
+      // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the
+      // anonymous method somehow results in an LMDB error with transactions we have to build a list of
+      // ids first and get the full info afterwards
+      get_transaction_hashes(txids, include_sensitive);
+      if (txids.size() > max_tx_count)
+      {
+        remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end());
+        txids.erase(txids.begin() + max_tx_count, txids.end());
+      }
+      get_transactions_info(txids, added_txs, include_sensitive);
+      return true;
+    }
+
+    // Give back incrementally, based on time of entry into the map
+    for (const auto &pit : m_added_txs_by_id)
+    {
+      if (pit.second >= start_time)
+        txids.push_back(pit.first);
+    }
+    get_transactions_info(txids, added_txs, include_sensitive);
+    if (added_txs.size() > max_tx_count)
+    {
+      remaining_added_txids.reserve(added_txs.size() - max_tx_count);
+      for (size_t i = max_tx_count; i < added_txs.size(); ++i)
+        remaining_added_txids.push_back(added_txs[i].first);
+      added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end());
+    }
+
+    std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time);
+    while (rit != m_removed_txs_by_time.end())
+    {
+      if (include_sensitive || !rit->second.sensitive)
+      {
+        removed_txs.push_back(rit->second.txid);
+      }
+      ++rit;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------
  void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const
  {
    CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -961,7 +1086,7 @@ namespace cryptonote

    // If the total weight is too high, choose the best paying transactions
    if (total_weight > max_backlog_weight)
-      std::sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; });
+      std::stable_sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; });

    backlog.clear();
    uint64_t w = 0;
@@ -1631,6 +1756,12 @@ namespace cryptonote
    CRITICAL_REGION_LOCAL(m_transactions_lock);
    CRITICAL_REGION_LOCAL1(m_blockchain);

+    // Simply throw away incremental info, too difficult to update
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
+
    MINFO("Validating txpool contents for v" << (unsigned)version);

    LockedTXN lock(m_blockchain.get_db());
@@ -1688,6 +1819,106 @@ namespace cryptonote
    return n_removed;
  }
  //---------------------------------------------------------------------------------
+  void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time)
+  {
+
+    time_t now = time(NULL);
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it == m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.insert(std::make_pair(txid, now));
+    }
+    else
+    {
+      // This tx was already added to the map earlier, probably because then it was in the "stem"
+      // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set
+      // a new time for clients that are not allowed to see sensitive txs to make sure they will
+      // see it now if they query incrementally
+      it->second = now;
+
+      auto sorted_it = find_tx_in_sorted_container(txid);
+      if (sorted_it == m_txs_by_fee_and_receive_time.end())
+      {
+        MDEBUG("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container");
+      }
+      else
+      {
+        m_txs_by_fee_and_receive_time.erase(sorted_it);
+      }
+    }
+    m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid);
+
+    // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time'
+    // whether we have that txid there and if yes remove it; this results in possible duplicates
+    // where we return certain txids as deleted AND in the pool at the same time which requires
+    // clients to process deleted ones BEFORE processing pool txs
+    if (m_added_txs_start_time == (time_t)0)
+    {
+      m_added_txs_start_time = now;
+    }
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive)
+  {
+    if (sorted_it == m_txs_by_fee_and_receive_time.end())
+    {
+      LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
+    }
+    else
+    {
+      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    }
+
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it != m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.erase(it);
+    }
+    else
+    {
+      MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs");
+    }
+    track_removed_tx(txid, sensitive);
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive)
+  {
+    time_t now = time(NULL);
+    m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive}));
+    MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size());
+    if (m_removed_txs_start_time == (time_t)0)
+    {
+      m_removed_txs_start_time = now;
+    }
+
+    // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set
+    // an absolute size limit plus delete entries that are x minutes old (which is ok because clients
+    // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back)
+    const int MAX_REMOVED = 20000;
+    if (m_removed_txs_by_time.size() > MAX_REMOVED)
+    {
+      auto erase_it = m_removed_txs_by_time.begin();
+      std::advance(erase_it, MAX_REMOVED / 4 + 1);
+      m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it);
+      m_removed_txs_start_time = m_removed_txs_by_time.begin()->first;
+      MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size());
+    }
+    else
+    {
+      time_t earliest = now - (30 * 60);  // 30 minutes
+      std::map<time_t, removed_tx_info>::iterator from, to;
+      from = m_removed_txs_by_time.begin();
+      to = m_removed_txs_by_time.lower_bound(earliest);
+      int distance = std::distance(from, to);
+      if (distance > 0)
+      {
+        m_removed_txs_by_time.erase(from, to);
+        m_removed_txs_start_time = earliest;
+        MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size());
+      }
+    }
+  }
+  //---------------------------------------------------------------------------------
  bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes)
  {
    CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1695,6 +1926,10 @@ namespace cryptonote

    m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT;
    m_txs_by_fee_and_receive_time.clear();
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
    m_spent_key_images.clear();
    m_txpool_weight = 0;
    std::vector<crypto::hash> remove;
@@ -1719,7 +1954,7 @@ namespace cryptonote
          MFATAL("Failed to insert key images from txpool tx");
          return false;
        }
-        m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid);
+        add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time);
        m_txpool_weight += meta.weight;
        return true;
      }, true, relay_category::all);
@@ -69,11 +69,12 @@ namespace cryptonote
    {
      // sort by greatest first, not least
      if (a.first.first > b.first.first) return true;
-      else if (a.first.first < b.first.first) return false;
-      else if (a.first.second < b.first.second) return true;
-      else if (a.first.second > b.first.second) return false;
-      else if (a.second != b.second) return true;
-      else return false;
+      if (a.first.first < b.first.first) return false;
+
+      if (a.first.second < b.first.second) return true;
+      if (a.first.second > b.first.second) return false;
+
+      return memcmp(a.second.data, b.second.data, sizeof(crypto::hash)) < 0;
    }
  };

@@ -428,6 +429,7 @@ namespace cryptonote
    struct tx_details
    {
      transaction tx;  //!< the transaction
+      cryptonote::blobdata tx_blob; //!< the transaction's binary blob
      size_t blob_size;  //!< the transaction's size
      size_t weight;  //!< the transaction's weight
      uint64_t fee;  //!< the transaction's fee amount
@@ -466,13 +468,25 @@ namespace cryptonote
    /**
     * @brief get infornation about a single transaction
     */
-    bool get_transaction_info(const crypto::hash &txid, tx_details &td) const;
+    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;
+
+    /**
+     * @brief get information about multiple transactions
+     */
+    bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const;

    /**
     * @brief get transactions not in the passed set
     */
    bool get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const;

+    /**
+     * @brief get info necessary for update of pool-related info in a wallet, preferably incremental
+     *
+     * @return true on success, false on error
+     */
+    bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
  private:

    /**
@@ -577,6 +591,10 @@ namespace cryptonote
     */
    void prune(size_t bytes = 0);

+    void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time);
+    void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive);
+    void track_removed_tx(const crypto::hash& txid, bool sensitive);
+
    //TODO: confirm the below comments and investigate whether or not this
    //      is the desired behavior
    //! map key images to transactions which spent them
@@ -609,6 +627,26 @@ private:

    std::atomic<uint64_t> m_cookie; //!< incremented at each change

+    // Info when transactions entered the pool, accessible by txid
+    std::unordered_map<crypto::hash, time_t> m_added_txs_by_id;
+
+    // Info at what time the pool started to track the adding of transactions
+    time_t m_added_txs_start_time;
+
+    struct removed_tx_info
+    {
+      crypto::hash txid;
+      bool sensitive;
+    };
+
+    // Info about transactions that were removed from the pool, ordered by the time
+    // of deletion
+    std::multimap<time_t, removed_tx_info> m_removed_txs_by_time;
+
+    // Info how far back in time the list of removed tx ids currently reaches
+    // (it gets shorted periodically to prevent overflow)
+    time_t m_removed_txs_start_time;
+
    /**
     * @brief get an iterator to a transaction in the sorted container
     *
@@ -0,0 +1,167 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "cryptonote_core/blockchain.h"
+#include "cryptonote_core/tx_verification_utils.h"
+#include "ringct/rctSigs.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "blockchain"
+
+#define VER_ASSERT(cond, msgexpr) CHECK_AND_ASSERT_MES(cond, false, msgexpr)
+
+using namespace cryptonote;
+
+// Do RCT expansion, then do post-expansion sanity checks, then do full non-semantics verification.
+static bool expand_tx_and_ver_rct_non_sem(transaction& tx, const rct::ctkeyM& mix_ring)
+{
+    // Pruned transactions can not be expanded and verified because they are missing RCT data
+    VER_ASSERT(!tx.pruned, "Pruned transaction will not pass verRctNonSemanticsSimple");
+
+    // Calculate prefix hash
+    const crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
+
+    // Expand mixring, tx inputs, tx key images, prefix hash message, etc into the RCT sig
+    const bool exp_res = Blockchain::expand_transaction_2(tx, tx_prefix_hash, mix_ring);
+    VER_ASSERT(exp_res, "Failed to expand rct signatures!");
+
+    const rct::rctSig& rv = tx.rct_signatures;
+
+    // Check that expanded RCT mixring == input mixring
+    VER_ASSERT(rv.mixRing == mix_ring, "Failed to check ringct signatures: mismatched pubkeys/mixRing");
+
+    // Check CLSAG/MLSAG size against transaction input
+    const size_t n_sigs = rct::is_rct_clsag(rv.type) ? rv.p.CLSAGs.size() : rv.p.MGs.size();
+    VER_ASSERT(n_sigs == tx.vin.size(), "Failed to check ringct signatures: mismatched input sigs/vin sizes");
+
+    // For each input, check that the key images were copied into the expanded RCT sig correctly
+    for (size_t n = 0; n < n_sigs; ++n)
+    {
+        const crypto::key_image& nth_vin_image = boost::get<txin_to_key>(tx.vin[n]).k_image;
+
+        if (rct::is_rct_clsag(rv.type))
+        {
+            const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.CLSAGs[n].I, 32);
+            VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched CLSAG key image");
+        }
+        else
+        {
+            const bool mg_nonempty = !rv.p.MGs[n].II.empty();
+            VER_ASSERT(mg_nonempty, "Failed to check ringct signatures: missing MLSAG key image");
+            const bool ki_match = 0 == memcmp(&nth_vin_image, &rv.p.MGs[n].II[0], 32);
+            VER_ASSERT(ki_match, "Failed to check ringct signatures: mismatched MLSAG key image");
+        }
+    }
+
+    // Mix ring data is now known to be correctly incorporated into the RCT sig inside tx.
+    return rct::verRctNonSemanticsSimple(rv);
+}
+
+// Create a unique identifier for pair of tx blob + mix ring
+static crypto::hash calc_tx_mixring_hash(const transaction& tx, const rct::ctkeyM& mix_ring)
+{
+    std::stringstream ss;
+
+    // Start with domain seperation
+    ss << config::HASH_KEY_TXHASH_AND_MIXRING;
+
+    // Then add TX hash
+    const crypto::hash tx_hash = get_transaction_hash(tx);
+    ss.write(tx_hash.data, sizeof(crypto::hash));
+
+    // Then serialize mix ring
+    binary_archive<true> ar(ss);
+    ::do_serialize(ar, const_cast<rct::ctkeyM&>(mix_ring));
+
+    // Calculate hash of TX hash and mix ring blob
+    crypto::hash tx_and_mixring_hash;
+    get_blob_hash(ss.str(), tx_and_mixring_hash);
+
+    return tx_and_mixring_hash;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+
+namespace cryptonote
+{
+
+bool ver_rct_non_semantics_simple_cached
+(
+    transaction& tx,
+    const rct::ctkeyM& mix_ring,
+    rct_ver_cache_t& cache,
+    const std::uint8_t rct_type_to_cache
+)
+{
+    // Hello future Monero dev! If you got this assert, read the following carefully:
+    //
+    // For this version of RCT, the way we guaranteed that verification caches do not generate false
+    // positives (and thus possibly enabling double spends) is we take a hash of two things. One,
+    // we use get_transaction_hash() which gives us a (cryptographically secure) unique
+    // representation of all "knobs" controlled by the possibly malicious constructor of the
+    // transaction. Two, we take a hash of all *previously validated* blockchain data referenced by
+    // this transaction which is required to validate the ring signature. In our case, this is the
+    // mixring. Future versions of the protocol may differ in this regard, but if this assumptions
+    // holds true in the future, enable the verification hash by modifying the `untested_tx`
+    // condition below.
+    const bool untested_tx = tx.version > 2 || tx.rct_signatures.type > rct::RCTTypeBulletproofPlus;
+    VER_ASSERT(!untested_tx, "Unknown TX type. Make sure RCT cache works correctly with this type and then enable it in the code here.");
+
+    // Don't cache older (or newer) rctSig types
+    // This cache only makes sense when it caches data from mempool first,
+    // so only "current fork version-enabled" RCT types need to be cached
+    if (tx.rct_signatures.type != rct_type_to_cache)
+    {
+        MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " skipped");
+        return expand_tx_and_ver_rct_non_sem(tx, mix_ring);
+    }
+
+    // Generate unique hash for tx+mix_ring pair
+    const crypto::hash tx_mixring_hash = calc_tx_mixring_hash(tx, mix_ring);
+
+    // Search cache for successful verification of same TX + mix ring combination
+    if (cache.has(tx_mixring_hash))
+    {
+        MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " hit");
+        return true;
+    }
+
+    // We had a cache miss, so now we must expand the mix ring and do full verification
+    MDEBUG("RCT cache: tx " << get_transaction_hash(tx) << " missed");
+    if (!expand_tx_and_ver_rct_non_sem(tx, mix_ring))
+    {
+        return false;
+    }
+
+    // At this point, the TX RCT verified successfully, so add it to the cache and return true
+    cache.add(tx_mixring_hash);
+
+    return true;
+}
+
+} // namespace cryptonote
@@ -0,0 +1,78 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "common/data_cache.h"
+#include "cryptonote_basic/cryptonote_basic.h"
+
+namespace cryptonote
+{
+
+// Modifying this value should not affect consensus. You can adjust it for performance needs
+static constexpr const size_t RCT_VER_CACHE_SIZE = 8192;
+
+using rct_ver_cache_t = ::tools::data_cache<::crypto::hash, RCT_VER_CACHE_SIZE>;
+
+/**
+ * @brief Cached version of rct::verRctNonSemanticsSimple
+ *
+ * This function will not affect how the transaction is serialized and it will never modify the
+ * transaction prefix.
+ *
+ * The reference to tx is mutable since the transaction's ring signatures may be expanded by
+ * Blockchain::expand_transaction_2. However, on cache hits, the transaction will not be
+ * expanded. This means that the caller does not need to call expand_transaction_2 on this
+ * transaction before passing it; the transaction will not successfully verify with "old" RCT data
+ * if the transaction has been otherwise modified since the last verification.
+ *
+ * But, if cryptonote::get_transaction_hash(tx) returns a "stale" hash, this function is not
+ * guaranteed to work. So make sure that the cryptonote::transaction passed has not had
+ * modifications to it since the last time its hash was fetched without properly invalidating the
+ * hashes.
+ *
+ * rct_type_to_cache can be any RCT version value as long as rct::verRctNonSemanticsSimple works for
+ * this RCT version, but for most applications, it doesn't make sense to not make this version
+ * the "current" RCT version (i.e. the version that transactions in the mempool are).
+ *
+ * @param tx transaction which contains RCT signature to verify
+ * @param mix_ring mixring referenced by this tx. THIS DATA MUST BE PREVIOUSLY VALIDATED
+ * @param cache saves tx+mixring hashes used to cache calls
+ * @param rct_type_to_cache Only RCT sigs with version (e.g. RCTTypeBulletproofPlus) will be cached
+ * @return true when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return true
+ * @return false when verRctNonSemanticsSimple() w/ expanded tx.rct_signatures would return false
+ */
+bool ver_rct_non_semantics_simple_cached
+(
+    transaction& tx,
+    const rct::ctkeyM& mix_ring,
+    rct_ver_cache_t& cache,
+    std::uint8_t rct_type_to_cache
+);
+
+} // namespace cryptonote
@@ -979,8 +979,18 @@ namespace cryptonote
  int t_cryptonote_protocol_handler<t_core>::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& context)
  {
    MLOG_P2P_MESSAGE("Received NOTIFY_NEW_TRANSACTIONS (" << arg.txs.size() << " txes)");
+    std::unordered_set<blobdata> seen;
    for (const auto &blob: arg.txs)
+    {
      MLOGIF_P2P_MESSAGE(cryptonote::transaction tx; crypto::hash hash; bool ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx, hash);, ret, "Including transaction " << hash);
+      if (seen.find(blob) != seen.end())
+      {
+        LOG_PRINT_CCONTEXT_L1("Duplicate transaction in notification, dropping connection");
+        drop_connection(context, false, false);
+        return 1;
+      }
+      seen.insert(blob);
+    }

    if(context.m_state != cryptonote_connection_context::state_normal)
      return 1;
@@ -1020,7 +1030,7 @@ namespace cryptonote
    for (auto& tx : arg.txs)
    {
      tx_verification_context tvc{};
-      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true))
+      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense)
      {
        LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection");
        drop_connection(context, false, false);
@@ -542,6 +542,7 @@ namespace levin
      i_core_events* core_;
      std::vector<blobdata> txs_;
      boost::uuids::uuid source_;
+      relay_method tx_relay;

      //! \pre Called in `zone_->strand`
      void operator()()
@@ -549,7 +550,7 @@ namespace levin
        if (!zone_ || !core_ || txs_.empty())
          return;

-        if (!zone_->fluffing)
+        if (!zone_->fluffing || tx_relay == relay_method::local)
        {
          core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem);
          for (int tries = 2; 0 < tries; tries--)
@@ -589,7 +590,7 @@ namespace levin

      change_channels(change_channels&&) = default;
      change_channels(const change_channels& source)
-        : zone_(source.zone_), map_(source.map_.clone())
+        : zone_(source.zone_), map_(source.map_.clone()), fluffing_(source.fluffing_)
      {}

      //! \pre Called within `zone_->strand`.
@@ -871,7 +872,7 @@ namespace levin
          {
            // this will change a local/forward tx to stem or fluff ...
            zone_->strand.dispatch(
-              dandelionpp_notify{zone_, core_, std::move(txs), source}
+              dandelionpp_notify{zone_, core_, std::move(txs), source, tx_relay}
            );
            break;
          }
@@ -219,6 +219,19 @@ int main(int argc, char const * argv[])
      {
        po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm);
      }
+      catch (const po::unknown_option &e)
+      {
+        std::string unrecognized_option = e.get_option_name();
+        if (all_options.find_nothrow(unrecognized_option, false))
+        {
+          std::cerr << "Option '" << unrecognized_option << "' is not allowed in the config file, please use it as a command line flag." << std::endl;
+        }
+        else
+        {
+          std::cerr << "Unrecognized option '" << unrecognized_option << "' in config file." << std::endl;
+        }
+        return 1;
+      }
      catch (const std::exception &e)
      {
        // log system isn't initialized yet
@@ -526,6 +526,7 @@ namespace hw {
        {0x2c97, 0x0001, 0, 0xffa0}, 
        {0x2c97, 0x0004, 0, 0xffa0},       
        {0x2c97, 0x0005, 0, 0xffa0},
+        {0x2c97, 0x0006, 0, 0xffa0},
    };

    bool device_ledger::connect(void) {
@@ -206,8 +206,13 @@ namespace multisig
  //----------------------------------------------------------------------------------------------------------------------
  void multisig_kex_msg::parse_and_validate_msg()
  {
+    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(),
+      "Multisig kex msg magic inconsistency.");
+    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() >= MULTISIG_KEX_V1_MAGIC.size(),
+      "Multisig kex msg magic inconsistency.");
+
    // check message type
-    CHECK_AND_ASSERT_THROW_MES(m_msg.size() > 0, "Kex message unexpectedly empty.");
+    CHECK_AND_ASSERT_THROW_MES(m_msg.size() >= MULTISIG_KEX_MSG_V2_MAGIC_1.size(), "Kex message unexpectedly small.");
    CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_V1_MAGIC.size()) != MULTISIG_KEX_V1_MAGIC,
      "V1 multisig kex messages are deprecated (unsafe).");
    CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_MSG_V1_MAGIC.size()) != MULTISIG_KEX_MSG_V1_MAGIC,
@@ -215,8 +220,6 @@ namespace multisig

    // deserialize the message
    std::string msg_no_magic;
-    CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(),
-      "Multisig kex msg magic inconsistency.");
    CHECK_AND_ASSERT_THROW_MES(tools::base58::decode(m_msg.substr(MULTISIG_KEX_MSG_V2_MAGIC_1.size()), msg_no_magic),
      "Multisig kex msg decoding error.");
    binary_archive<false> b_archive{epee::strspan<std::uint8_t>(msg_no_magic)};
@@ -71,7 +71,7 @@ namespace net
        struct i2p_serialized
        {
            std::string host;
-            std::uint16_t port;
+            std::uint16_t port; //! Leave for compatability with older clients

            BEGIN_KV_SERIALIZE_MAP()
                KV_SERIALIZE(host)
@@ -80,8 +80,7 @@ namespace net
        };
    }

-    i2p_address::i2p_address(const boost::string_ref host, const std::uint16_t port) noexcept
-      : port_(port)
+    i2p_address::i2p_address(const boost::string_ref host) noexcept
    {
        // this is a private constructor, throw if moved to public
        assert(host.size() < sizeof(host_));
@@ -97,27 +96,19 @@ namespace net
    }

    i2p_address::i2p_address() noexcept
-      : port_(0)
    {
        static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size");
        std::memcpy(host_, unknown_host, sizeof(unknown_host));
        std::memset(host_ + sizeof(unknown_host), 0, sizeof(host_) - sizeof(unknown_host));
    }

-    expect<i2p_address> i2p_address::make(const boost::string_ref address, const std::uint16_t default_port)
+    expect<i2p_address> i2p_address::make(const boost::string_ref address)
    {
        boost::string_ref host = address.substr(0, address.rfind(':'));
-        const boost::string_ref port =
-            address.substr(host.size() + (host.size() == address.size() ? 0 : 1));
-
        MONERO_CHECK(host_check(host));

-        std::uint16_t porti = default_port;
-        if (!port.empty() && !epee::string_tools::get_xtype_from_string(porti, std::string{port}))
-            return {net::error::invalid_port};
-
        static_assert(b32_length + sizeof(tld) == sizeof(i2p_address::host_), "bad internal host size");
-        return i2p_address{host, porti};
+        return i2p_address{host};
    }

    bool i2p_address::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent)
@@ -127,23 +118,21 @@ namespace net
        {
            std::memcpy(host_, in.host.data(), in.host.size());
            std::memset(host_ + in.host.size(), 0, sizeof(host_) - in.host.size());
-            port_ = in.port;
            return true;
        }
        static_assert(sizeof(unknown_host) <= sizeof(host_), "bad buffer size");
        std::memcpy(host_, unknown_host, sizeof(unknown_host)); // include null terminator
-        port_ = 0;
        return false;
    }

    bool i2p_address::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const
    {
-        const i2p_serialized out{std::string{host_}, port_};
+        // Set port to 1 for backwards compatability; zero is invalid port
+        const i2p_serialized out{std::string{host_}, 1};
        return out.store(dest, hparent);
    }

    i2p_address::i2p_address(const i2p_address& rhs) noexcept
-      : port_(rhs.port_)
    {
        std::memcpy(host_, rhs.host_, sizeof(host_));
    }
@@ -152,7 +141,6 @@ namespace net
    {
        if (this != std::addressof(rhs))
        {
-            port_ = rhs.port_;
            std::memcpy(host_, rhs.host_, sizeof(host_));
        }
        return *this;
@@ -166,13 +154,12 @@ namespace net

    bool i2p_address::equal(const i2p_address& rhs) const noexcept
    {
-        return port_ == rhs.port_ && is_same_host(rhs);
+        return is_same_host(rhs);
    }

    bool i2p_address::less(const i2p_address& rhs) const noexcept
    {
-        int res = std::strcmp(host_str(), rhs.host_str());
-        return res < 0 || (res == 0 && port() < rhs.port());
+        return std::strcmp(host_str(), rhs.host_str()) < 0;
    }

    bool i2p_address::is_same_host(const i2p_address& rhs) const noexcept
@@ -182,20 +169,6 @@ namespace net

    std::string i2p_address::str() const
    {
-        const std::size_t host_length = std::strlen(host_str());
-        const std::size_t port_length =
-            port_ == 0 ? 0 : std::numeric_limits<std::uint16_t>::digits10 + 2;
-
-        std::string out{};
-        out.reserve(host_length + port_length);
-        out.assign(host_str(), host_length);
-
-        if (port_ != 0)
-        {
-            out.push_back(':');
-            namespace karma = boost::spirit::karma;
-            karma::generate(std::back_inserter(out), karma::ushort_, port());
-        }
-        return out;
+        return host_str();
    }
 }
@@ -50,11 +50,10 @@ namespace net
    //! b32 i2p address; internal format not condensed/decoded.
    class i2p_address
    {
-        std::uint16_t port_;
        char host_[61]; // null-terminated

        //! Keep in private, `host.size()` has no runtime check
-        i2p_address(boost::string_ref host, std::uint16_t port) noexcept;
+        i2p_address(boost::string_ref host) noexcept;

    public:
        //! \return Size of internal buffer for host.
@@ -74,7 +73,7 @@ namespace net
            with `default_port` being used if port is not specified in
            `address`.
        */
-        static expect<i2p_address> make(boost::string_ref address, std::uint16_t default_port = 0);
+        static expect<i2p_address> make(boost::string_ref address);

        //! Load from epee p2p format, and \return false if not valid tor address
        bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent);
@@ -103,8 +102,8 @@ namespace net
        //! \return Null-terminated `x.b32.i2p` value or `unknown_str()`.
        const char* host_str() const noexcept { return host_; }

-        //! \return Port value or `0` if unspecified.
-        std::uint16_t port() const noexcept { return port_; }
+        //! \return `1` to work with I2P socks which considers `0` error.
+        std::uint16_t port() const noexcept { return 1; }

        static constexpr bool is_loopback() noexcept { return false; }
        static constexpr bool is_local() noexcept { return false; }
@@ -38,7 +38,7 @@ namespace net
 {
    void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port)
    {
-        // require ipv6 address format "[addr:addr:addr:...:addr]:port"
+        // If IPv6 address format with port "[addr:addr:addr:...:addr]:port"
        if (address.find(']') != std::string::npos)
        {
            host = address.substr(1, address.rfind(']') - 1);
@@ -47,6 +47,12 @@ namespace net
                port = address.substr(address.rfind(':') + 1);
            }
        }
+        // Else if IPv6 address format without port e.g. "addr:addr:addr:...:addr"
+        else if (std::count(address.begin(), address.end(), ':') >= 2)
+        {
+            host = address;
+        }
+        // Else IPv4, Tor, I2P address or hostname
        else
        {
            host = address.substr(0, address.rfind(':'));
@@ -75,7 +81,7 @@ namespace net
        if (host_str_ref.ends_with(".onion"))
            return tor_address::make(address, default_port);
        if (host_str_ref.ends_with(".i2p"))
-            return i2p_address::make(address, default_port);
+            return i2p_address::make(address);

        boost::system::error_code ec;
        boost::asio::ip::address_v6 v6 = boost::asio::ip::address_v6::from_string(host_str, ec);
@@ -38,6 +38,16 @@

 namespace net
 {
+    /*!
+     * \brief Takes a valid address string (IP, Tor, I2P, or DNS name) and splits it into host and port
+     *
+     * The host of an IPv6 addresses in the format "[x:x:..:x]:port" will have the braces stripped.
+     * For example, when the address is "[ffff::2023]", host will be set to "ffff::2023".
+     *
+     * \param address The address string one wants to split
+     * \param[out] host The host part of the address string. Is always set.
+     * \param[out] port The port part of the address string. Is only set when address string contains a port.
+    */
    void get_network_address_host_and_port(const std::string& address, std::string& host, std::string& port);

    /*!
@@ -247,7 +247,23 @@ namespace nodetool
    if (it == m_blocked_hosts.end())
    {
      m_blocked_hosts[host_str] = limit;
-      added = true;
+
+      // if the host was already blocked due to being in a blocked subnet, let it be silent
+      bool matches_blocked_subnet = false;
+      if (addr.get_type_id() == epee::net_utils::address_type::ipv4)
+      {
+        auto ipv4_address = addr.template as<epee::net_utils::ipv4_network_address>();
+        for (auto jt = m_blocked_subnets.begin(); jt != m_blocked_subnets.end(); ++jt)
+        {
+          if (jt->first.matches(ipv4_address))
+          {
+            matches_blocked_subnet = true;
+            break;
+          }
+        }
+      }
+      if (!matches_blocked_subnet)
+        added = true;
    }
    else if (it->second < limit || !add_only)
      it->second = limit;
@@ -317,6 +333,7 @@ namespace nodetool
      limit = std::numeric_limits<time_t>::max();
    else
      limit = now + seconds;
+    const bool added = m_blocked_subnets.find(subnet) == m_blocked_subnets.end();
    m_blocked_subnets[subnet] = limit;

    // drop any connection to that subnet. This should only have to look into
@@ -349,7 +366,10 @@ namespace nodetool
      conns.clear();
    }

-    MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked.");
+    if (added)
+      MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked.");
+    else
+      MINFO("Subnet " << subnet.host_str() << " blocked.");
    return true;
  }
  //-----------------------------------------------------------------------------------
@@ -645,20 +665,10 @@ namespace nodetool
  {
    using namespace boost::asio;

-    std::string host = addr;
+    // Split addr string into host string and port string
+    std::string host;
    std::string port = std::to_string(default_port);
-    size_t colon_pos = addr.find_last_of(':');
-    size_t dot_pos = addr.find_last_of('.');
-    size_t square_brace_pos = addr.find('[');
-
-    // IPv6 will have colons regardless.  IPv6 and IPv4 address:port will have a colon but also either a . or a [
-    // as IPv6 addresses specified as address:port are to be specified as "[addr:addr:...:addr]:port"
-    // One may also specify an IPv6 address as simply "[addr:addr:...:addr]" without the port; in that case
-    // the square braces will be stripped here.
-    if ((std::string::npos != colon_pos && std::string::npos != dot_pos) || std::string::npos != square_brace_pos)
-    {
-      net::get_network_address_host_and_port(addr, host, port);
-    }
+    net::get_network_address_host_and_port(addr, host, port);
    MINFO("Resolving node address: host=" << host << ", port=" << port);

    io_service io_srv;
@@ -695,34 +705,32 @@ namespace nodetool
    std::set<std::string> full_addrs;
    if (m_nettype == cryptonote::TESTNET)
    {
-      full_addrs.insert("212.83.175.67:28080");
-      full_addrs.insert("212.83.172.165:28080");
      full_addrs.insert("176.9.0.187:28080");
      full_addrs.insert("88.99.173.38:28080");
      full_addrs.insert("51.79.173.165:28080");
+      full_addrs.insert("192.99.8.110:28080");
+      full_addrs.insert("37.187.74.171:28080");
    }
    else if (m_nettype == cryptonote::STAGENET)
    {
-      full_addrs.insert("162.210.173.150:38080");
      full_addrs.insert("176.9.0.187:38080");
      full_addrs.insert("88.99.173.38:38080");
      full_addrs.insert("51.79.173.165:38080");
+      full_addrs.insert("192.99.8.110:38080");
+      full_addrs.insert("37.187.74.171:38080");
    }
    else if (m_nettype == cryptonote::FAKECHAIN)
    {
    }
    else
    {
-      full_addrs.insert("212.83.175.67:18080");
-      full_addrs.insert("212.83.172.165:18080");
      full_addrs.insert("176.9.0.187:18080");
      full_addrs.insert("88.198.163.90:18080");
-      full_addrs.insert("95.217.25.101:18080");
-      full_addrs.insert("136.244.105.131:18080");
-      full_addrs.insert("104.238.221.81:18080");
      full_addrs.insert("66.85.74.134:18080");
      full_addrs.insert("88.99.173.38:18080");
      full_addrs.insert("51.79.173.165:18080");
+      full_addrs.insert("192.99.8.110:18080");
+      full_addrs.insert("37.187.74.171:18080");
    }
    return full_addrs;
  }
@@ -857,6 +865,8 @@ namespace nodetool
          "4pixvbejrvihnkxmduo2agsnmc3rrulrqc7s3cbwwrep6h6hrzsibeqd.onion:18083",
          "zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083",
          "qz43zul2x56jexzoqgkx2trzwcfnr6l3hbtfcfx54g4r3eahy3bssjyd.onion:18083",
+          "plowsof3t5hogddwabaeiyrno25efmzfxyro2vligremt7sxpsclfaid.onion:18083",
+          "plowsoffjexmxalw73tkjmf422gq6575fc7vicuu4javzn2ynnte6tyd.onion:18083",
        };
      }
      return {};
@@ -865,7 +875,9 @@ namespace nodetool
      {
        return {
          "s3l6ke4ed3df466khuebb4poienoingwof7oxtbo6j4n56sghe3a.b32.i2p:18080",
-          "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080"
+          "sel36x6fibfzujwvt4hf5gxolz6kd3jpvbjqg6o3ud2xtionyl2q.b32.i2p:18080",
+          "uqj3aphckqtjsitz7kxx5flqpwjlq5ppr3chazfued7xucv3nheq.b32.i2p:18080",
+          "vdmnehdjkpkg57nthgnjfuaqgku673r5bpbqg56ix6fyqoywgqrq.b32.i2p:18080",
        };
      }
      return {};
@@ -2413,7 +2425,7 @@ namespace nodetool
        return false;
      }
      return true;
-    });
+    }, "0.0.0.0", m_ssl_support);
    if(!r)
    {
      LOG_WARNING_CC(context, "Failed to call connect_async, network error.");
@@ -196,7 +196,7 @@ namespace boost
      if (std::strcmp(host, net::i2p_address::unknown_str()) == 0)
        na = net::i2p_address::unknown();
      else
-        na = MONERO_UNWRAP(net::i2p_address::make(host, port));
+        na = MONERO_UNWRAP(net::i2p_address::make(host));
    }

    template <class Archive, class ver_type>
@@ -97,6 +97,14 @@ namespace rct {
    struct ctkey {
        key dest;
        key mask; //C here if public
+
+        bool operator==(const ctkey &other) const {
+          return (dest == other.dest) && (mask == other.mask);
+        }
+
+        bool operator!=(const ctkey &other) const {
+          return !(*this == other);
+        }
    };
    typedef std::vector<ctkey> ctkeyV;
    typedef std::vector<ctkeyV> ctkeyM;
@@ -317,6 +325,10 @@ namespace rct {
        ctkeyV outPk;
        xmr_amount txnFee; // contains b

+        rctSigBase() :
+          type(RCTTypeNull), message{}, mixRing{}, pseudoOuts{}, ecdhInfo{}, outPk{}, txnFee(0)
+        {}
+
        template<bool W, template <bool> class Archive>
        bool serialize_rctsig_base(Archive<W> &ar, size_t inputs, size_t outputs)
        {
@@ -354,11 +366,17 @@ namespace rct {
          {
            if (type == RCTTypeBulletproof2 || type == RCTTypeCLSAG || type == RCTTypeBulletproofPlus)
            {
+              // Since RCTTypeBulletproof2 enote types, we don't serialize the blinding factor, and only serialize the
+              // first 8 bytes of ecdhInfo[i].amount
              ar.begin_object();
-              if (!typename Archive<W>::is_saving())
+              crypto::hash8 trunc_amount; // placeholder variable needed to maintain "strict aliasing"
+              if (!typename Archive<W>::is_saving()) // loading
                memset(ecdhInfo[i].amount.bytes, 0, sizeof(ecdhInfo[i].amount.bytes));
-              crypto::hash8 &amount = (crypto::hash8&)ecdhInfo[i].amount;
-              FIELD(amount);
+              else // saving
+                memcpy(trunc_amount.data, ecdhInfo[i].amount.bytes, sizeof(trunc_amount));
+              FIELD(trunc_amount);
+              if (!typename Archive<W>::is_saving()) // loading
+                memcpy(ecdhInfo[i].amount.bytes, trunc_amount.data, sizeof(trunc_amount));
              ar.end_object();
            }
            else
@@ -598,88 +598,165 @@ namespace cryptonote

    CHECK_PAYMENT(req, res, 1);

-    // quick check for noop
-    if (!req.block_ids.empty())
+    res.daemon_time = (uint64_t)time(NULL);
+    // Always set daemon time, and set it early rather than late, as delivering some incremental pool
+    // info twice because of slightly overlapping time intervals is no problem, whereas producing gaps
+    // and never delivering something is
+
+    bool get_blocks = false;
+    bool get_pool = false;
+    switch (req.requested_info)
    {
-      uint64_t last_block_height;
-      crypto::hash last_block_hash;
-      m_core.get_blockchain_top(last_block_height, last_block_hash);
-      if (last_block_hash == req.block_ids.front())
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY:
+        // Compatibility value 0: Clients that do not set 'requested_info' want blocks, and only blocks
+        get_blocks = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL:
+        get_blocks = true;
+        get_pool = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY:
+        get_pool = true;
+        break;
+      default:
+        res.status = "Failed, wrong requested info";
+        return true;
+    }
+
+    res.pool_info_extent = COMMAND_RPC_GET_BLOCKS_FAST::NONE;
+
+    if (get_pool)
+    {
+      const bool restricted = m_restricted && ctx;
+      const bool request_has_rpc_origin = ctx != NULL;
+      const bool allow_sensitive = !request_has_rpc_origin || !restricted;
+      const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits<size_t>::max();
+
+      bool incremental;
+      std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> added_pool_txs;
+      bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental);
+      if (success)
      {
-        res.start_height = 0;
-        res.current_height = m_core.get_current_blockchain_height();
-        res.status = CORE_RPC_STATUS_OK;
+        res.added_pool_txs.clear();
+        if (m_rpc_payment)
+        {
+          CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH);
+        }
+        for (const auto &added_pool_tx: added_pool_txs)
+        {
+          COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info;
+          info.tx_hash = added_pool_tx.first;
+          std::stringstream oss;
+          binary_archive<true> ar(oss);
+          bool r = req.prune
+            ? const_cast<cryptonote::transaction&>(added_pool_tx.second.tx).serialize_base(ar)
+            : ::serialization::serialize(ar, const_cast<cryptonote::transaction&>(added_pool_tx.second.tx));
+          if (!r)
+          {
+            res.status = "Failed to serialize transaction";
+            return true;
+          }
+          info.tx_blob = oss.str();
+          info.double_spend_seen = added_pool_tx.second.double_spend_seen;
+          res.added_pool_txs.push_back(std::move(info));
+        }
+      }
+      if (success)
+      {
+        res.pool_info_extent = incremental ? COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL : COMMAND_RPC_GET_BLOCKS_FAST::FULL;
+      }
+      else
+      {
+        res.status = "Failed to get pool info";
        return true;
      }
    }

-    size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-    if (m_rpc_payment)
+    if (get_blocks)
    {
-      max_blocks = res.credits / COST_PER_BLOCK;
-      if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
-        max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-      if (max_blocks == 0)
+      // quick check for noop
+      if (!req.block_ids.empty())
      {
-        res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+        uint64_t last_block_height;
+        crypto::hash last_block_hash;
+        m_core.get_blockchain_top(last_block_height, last_block_hash);
+        if (last_block_hash == req.block_ids.front())
+        {
+          res.start_height = 0;
+          res.current_height = last_block_height + 1;
+          res.status = CORE_RPC_STATUS_OK;
+          return true;
+        }
+      }
+
+      size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+      if (m_rpc_payment)
+      {
+        max_blocks = res.credits / COST_PER_BLOCK;
+        if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
+          max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+        if (max_blocks == 0)
+        {
+          res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+          return true;
+        }
+      }
+
+      std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
+      if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
+      {
+        res.status = "Failed";
+        add_host_fail(ctx);
        return true;
      }
-    }

-    std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
-    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
-    {
-      res.status = "Failed";
-      add_host_fail(ctx);
-      return true;
-    }
+      CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);

-    CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
-
-    size_t size = 0, ntxes = 0;
-    res.blocks.reserve(bs.size());
-    res.output_indices.reserve(bs.size());
-    for(auto& bd: bs)
-    {
-      res.blocks.resize(res.blocks.size()+1);
-      res.blocks.back().pruned = req.prune;
-      res.blocks.back().block = bd.first.first;
-      size += bd.first.first.size();
-      res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
-      ntxes += bd.second.size();
-      res.output_indices.back().indices.reserve(1 + bd.second.size());
-      if (req.no_miner_tx)
-        res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
-      res.blocks.back().txs.reserve(bd.second.size());
-      for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+      size_t size = 0, ntxes = 0;
+      res.blocks.reserve(bs.size());
+      res.output_indices.reserve(bs.size());
+      for(auto& bd: bs)
      {
-        res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
-        i->second.clear();
-        i->second.shrink_to_fit();
-        size += res.blocks.back().txs.back().blob.size();
-      }
+        res.blocks.resize(res.blocks.size()+1);
+        res.blocks.back().pruned = req.prune;
+        res.blocks.back().block = bd.first.first;
+        size += bd.first.first.size();
+        res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
+        ntxes += bd.second.size();
+        res.output_indices.back().indices.reserve(1 + bd.second.size());
+        if (req.no_miner_tx)
+          res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
+        res.blocks.back().txs.reserve(bd.second.size());
+        for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+        {
+          res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
+          i->second.clear();
+          i->second.shrink_to_fit();
+          size += res.blocks.back().txs.back().blob.size();
+        }

-      const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
-      if (n_txes_to_lookup > 0)
-      {
-        std::vector<std::vector<uint64_t>> indices;
-        bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
-        if (!r)
+        const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
+        if (n_txes_to_lookup > 0)
        {
-          res.status = "Failed";
-          return true;
+          std::vector<std::vector<uint64_t>> indices;
+          bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
+          if (!r)
+          {
+            res.status = "Failed";
+            return true;
+          }
+          if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
+          {
+            res.status = "Failed";
+            return true;
+          }
+          for (size_t i = 0; i < indices.size(); ++i)
+            res.output_indices.back().indices.push_back({std::move(indices[i])});
        }
-        if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
-        {
-          res.status = "Failed";
-          return true;
-        }
-        for (size_t i = 0; i < indices.size(); ++i)
-          res.output_indices.back().indices.push_back({std::move(indices[i])});
      }
+      MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
    }

-    MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
    res.status = CORE_RPC_STATUS_OK;
    return true;
  }
@@ -919,17 +996,16 @@ namespace cryptonote
    // try the pool for any missing txes
    size_t found_in_pool = 0;
    std::unordered_set<crypto::hash> pool_tx_hashes;
-    std::unordered_map<crypto::hash, tx_info> per_tx_pool_tx_info;
+    std::unordered_map<crypto::hash, tx_memory_pool::tx_details> per_tx_pool_tx_details;
    if (!missed_txs.empty())
    {
-      std::vector<tx_info> pool_tx_info;
-      std::vector<spent_key_image_info> pool_key_image_info;
-      bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted);
+      std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> pool_txs;
+      bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted);
      if(r)
      {
        // sort to match original request
        std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs;
-        std::vector<tx_info>::const_iterator i;
+        std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>::const_iterator i;
        unsigned txs_processed = 0;
        for (const crypto::hash &h: vh)
        {
@@ -949,36 +1025,23 @@ namespace cryptonote
            sorted_txs.push_back(std::move(txs[txs_processed]));
            ++txs_processed;
          }
-          else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end())
+          else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair<crypto::hash, tx_memory_pool::tx_details> &pt) { return h == pt.first; })) != pool_txs.end())
          {
-            cryptonote::transaction tx;
-            if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx))
-            {
-              res.status = "Failed to parse and validate tx from blob";
-              return true;
-            }
+            const tx_memory_pool::tx_details &td = i->second;
            std::stringstream ss;
            binary_archive<true> ba(ss);
-            bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
+            bool r = const_cast<cryptonote::transaction&>(td.tx).serialize_base(ba);
            if (!r)
            {
              res.status = "Failed to serialize transaction base";
              return true;
            }
            const cryptonote::blobdata pruned = ss.str();
-            const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx);
-            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size())));
+            const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx);
+            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size())));
            missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h));
            pool_tx_hashes.insert(h);
-            const std::string hash_string = epee::string_tools::pod_to_hex(h);
-            for (const auto &ti: pool_tx_info)
-            {
-              if (ti.id_hash == hash_string)
-              {
-                per_tx_pool_tx_info.insert(std::make_pair(h, ti));
-                break;
-              }
-            }
+            per_tx_pool_tx_details.insert(std::make_pair(h, td));
            ++found_in_pool;
          }
        }
@@ -1074,8 +1137,8 @@ namespace cryptonote
      {
        e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max();
        e.confirmations = 0;
-        auto it = per_tx_pool_tx_info.find(tx_hash);
-        if (it != per_tx_pool_tx_info.end())
+        auto it = per_tx_pool_tx_details.find(tx_hash);
+        if (it != per_tx_pool_tx_details.end())
        {
          e.double_spend_seen = it->second.double_spend_seen;
          e.relayed = it->second.relayed;
@@ -1275,6 +1338,8 @@ namespace cryptonote
          add_reason(reason, "fee too low");
        if ((res.too_few_outputs = tvc.m_too_few_outputs))
          add_reason(reason, "too few outputs");
+        if ((res.tx_extra_too_big = tvc.m_tx_extra_too_big))
+          add_reason(reason, "tx-extra too big");
        const std::string punctuation = reason.empty() ? "" : ": ";
        if (tvc.m_verifivation_failed)
        {
@@ -2113,7 +2178,8 @@ namespace cryptonote
    // Fixing of high orphan issue for most pools
    // Thanks Boolberry!
    block b;
-    if(!parse_and_validate_block_from_blob(blockblob, b))
+    crypto::hash blk_id;
+    if(!parse_and_validate_block_from_blob(blockblob, b, blk_id))
    {
      error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB;
      error_resp.message = "Wrong block blob";
@@ -2136,6 +2202,7 @@ namespace cryptonote
      error_resp.message = "Block not accepted";
      return false;
    }
+    res.block_id = epee::string_tools::pod_to_hex(blk_id);
    res.status = CORE_RPC_STATUS_OK;
    return true;
  }
@@ -88,7 +88,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 3
-#define CORE_RPC_VERSION_MINOR 11
+#define CORE_RPC_VERSION_MINOR 13
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)

@@ -162,18 +162,29 @@ namespace cryptonote
  struct COMMAND_RPC_GET_BLOCKS_FAST
  {

+    enum REQUESTED_INFO
+    {
+      BLOCKS_ONLY = 0,
+      BLOCKS_AND_POOL = 1,
+      POOL_ONLY = 2
+    };
+
    struct request_t: public rpc_access_request_base
    {
+      uint8_t     requested_info;
      std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
      uint64_t    start_height;
      bool        prune;
      bool        no_miner_tx;
+      uint64_t    pool_info_since;
      BEGIN_KV_SERIALIZE_MAP()
        KV_SERIALIZE_PARENT(rpc_access_request_base)
+        KV_SERIALIZE_OPT(requested_info, (uint8_t)0)
        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
        KV_SERIALIZE(start_height)
        KV_SERIALIZE(prune)
        KV_SERIALIZE_OPT(no_miner_tx, false)
+        KV_SERIALIZE_OPT(pool_info_since, (uint64_t)0)
      END_KV_SERIALIZE_MAP()
    };
    typedef epee::misc_utils::struct_init<request_t> request;
@@ -196,12 +207,37 @@ namespace cryptonote
      END_KV_SERIALIZE_MAP()
    };

+    struct pool_tx_info
+    {
+      crypto::hash tx_hash;
+      blobdata tx_blob;
+      bool double_spend_seen;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_VAL_POD_AS_BLOB(tx_hash)
+        KV_SERIALIZE(tx_blob)
+        KV_SERIALIZE(double_spend_seen)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    enum POOL_INFO_EXTENT
+    {
+      NONE = 0,
+      INCREMENTAL = 1,
+      FULL = 2
+    };
+
    struct response_t: public rpc_access_response_base
    {
      std::vector<block_complete_entry> blocks;
      uint64_t    start_height;
      uint64_t    current_height;
      std::vector<block_output_indices> output_indices;
+      uint64_t    daemon_time;
+      uint8_t     pool_info_extent;
+      std::vector<pool_tx_info> added_pool_txs;
+      std::vector<crypto::hash> remaining_added_pool_txids;
+      std::vector<crypto::hash> removed_pool_txids;

      BEGIN_KV_SERIALIZE_MAP()
        KV_SERIALIZE_PARENT(rpc_access_response_base)
@@ -209,6 +245,17 @@ namespace cryptonote
        KV_SERIALIZE(start_height)
        KV_SERIALIZE(current_height)
        KV_SERIALIZE(output_indices)
+        KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0)
+        KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0)
+        if (pool_info_extent != POOL_INFO_EXTENT::NONE)
+        {
+          KV_SERIALIZE(added_pool_txs)
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids)
+        }
+        if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL)
+        {
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
+        }
      END_KV_SERIALIZE_MAP()
    };
    typedef epee::misc_utils::struct_init<response_t> response;
@@ -592,6 +639,7 @@ namespace cryptonote
      bool fee_too_low;
      bool too_few_outputs;
      bool sanity_check_failed;
+      bool tx_extra_too_big;

      BEGIN_KV_SERIALIZE_MAP()
        KV_SERIALIZE_PARENT(rpc_access_response_base)
@@ -606,6 +654,7 @@ namespace cryptonote
        KV_SERIALIZE(fee_too_low)
        KV_SERIALIZE(too_few_outputs)
        KV_SERIALIZE(sanity_check_failed)
+        KV_SERIALIZE(tx_extra_too_big)
      END_KV_SERIALIZE_MAP()
    };
    typedef epee::misc_utils::struct_init<response_t> response;
@@ -1066,8 +1115,11 @@ namespace cryptonote
    
    struct response_t: public rpc_response_base
    {
+      std::string block_id;
+
      BEGIN_KV_SERIALIZE_MAP()
        KV_SERIALIZE_PARENT(rpc_response_base)
+        KV_SERIALIZE(block_id)
      END_KV_SERIALIZE_MAP()
    };
    typedef epee::misc_utils::struct_init<response_t> response;
@@ -48,6 +48,11 @@ void GetHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) c

 void GetHeight::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, height, height);
 }

@@ -61,6 +66,11 @@ void GetBlocksFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest

 void GetBlocksFast::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, block_ids, block_ids);
  GET_FROM_JSON_OBJECT(val, start_height, start_height);
  GET_FROM_JSON_OBJECT(val, prune, prune);
@@ -76,6 +86,11 @@ void GetBlocksFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des

 void GetBlocksFast::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, blocks, blocks);
  GET_FROM_JSON_OBJECT(val, start_height, start_height);
  GET_FROM_JSON_OBJECT(val, current_height, current_height);
@@ -91,6 +106,11 @@ void GetHashesFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest

 void GetHashesFast::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, known_hashes, known_hashes);
  GET_FROM_JSON_OBJECT(val, start_height, start_height);
 }
@@ -100,10 +120,16 @@ void GetHashesFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des
  INSERT_INTO_JSON_OBJECT(dest, hashes, hashes);
  INSERT_INTO_JSON_OBJECT(dest, start_height, start_height);
  INSERT_INTO_JSON_OBJECT(dest, current_height, current_height);
+
 }

 void GetHashesFast::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, hashes, hashes);
  GET_FROM_JSON_OBJECT(val, start_height, start_height);
  GET_FROM_JSON_OBJECT(val, current_height, current_height);
@@ -117,6 +143,11 @@ void GetTransactions::Request::doToJson(rapidjson::Writer<epee::byte_stream>& de

 void GetTransactions::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, tx_hashes, tx_hashes);
 }

@@ -128,6 +159,11 @@ void GetTransactions::Response::doToJson(rapidjson::Writer<epee::byte_stream>& d

 void GetTransactions::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, txs, txs);
  GET_FROM_JSON_OBJECT(val, missed_hashes, missed_hashes);
 }
@@ -140,6 +176,11 @@ void KeyImagesSpent::Request::doToJson(rapidjson::Writer<epee::byte_stream>& des

 void KeyImagesSpent::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, key_images, key_images);
 }

@@ -150,6 +191,11 @@ void KeyImagesSpent::Response::doToJson(rapidjson::Writer<epee::byte_stream>& de

 void KeyImagesSpent::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, spent_status, spent_status);
 }

@@ -161,6 +207,11 @@ void GetTxGlobalOutputIndices::Request::doToJson(rapidjson::Writer<epee::byte_st

 void GetTxGlobalOutputIndices::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, tx_hash, tx_hash);
 }

@@ -171,6 +222,11 @@ void GetTxGlobalOutputIndices::Response::doToJson(rapidjson::Writer<epee::byte_s

 void GetTxGlobalOutputIndices::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, output_indices, output_indices);
 }

@@ -182,6 +238,11 @@ void SendRawTx::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) co

 void SendRawTx::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, tx, tx);
  GET_FROM_JSON_OBJECT(val, relay, relay);
 }
@@ -194,6 +255,11 @@ void SendRawTx::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) c

 void SendRawTx::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, relayed, relayed);
 }

@@ -205,6 +271,11 @@ void SendRawTxHex::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest)

 void SendRawTxHex::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, tx_as_hex, tx_as_hex);
  GET_FROM_JSON_OBJECT(val, relay, relay);
 }
@@ -219,6 +290,11 @@ void StartMining::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest)

 void StartMining::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, miner_address, miner_address);
  GET_FROM_JSON_OBJECT(val, threads_count, threads_count);
  GET_FROM_JSON_OBJECT(val, do_background_mining, do_background_mining);
@@ -266,6 +342,11 @@ void MiningStatus::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest

 void MiningStatus::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, active, active);
  GET_FROM_JSON_OBJECT(val, speed, speed);
  GET_FROM_JSON_OBJECT(val, threads_count, threads_count);
@@ -288,6 +369,11 @@ void GetInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) con

 void GetInfo::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, info, info);
 }

@@ -314,6 +400,11 @@ void GetBlockHash::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest)

 void GetBlockHash::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, height, height);
 }

@@ -324,6 +415,11 @@ void GetBlockHash::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest

 void GetBlockHash::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, hash, hash);
 }

@@ -342,6 +438,11 @@ void GetLastBlockHeader::Response::doToJson(rapidjson::Writer<epee::byte_stream>

 void GetLastBlockHeader::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, header, header);
 }

@@ -353,6 +454,11 @@ void GetBlockHeaderByHash::Request::doToJson(rapidjson::Writer<epee::byte_stream

 void GetBlockHeaderByHash::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, hash, hash);
 }

@@ -363,6 +469,11 @@ void GetBlockHeaderByHash::Response::doToJson(rapidjson::Writer<epee::byte_strea

 void GetBlockHeaderByHash::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, header, header);
 }

@@ -374,6 +485,11 @@ void GetBlockHeaderByHeight::Request::doToJson(rapidjson::Writer<epee::byte_stre

 void GetBlockHeaderByHeight::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, height, height);
 }

@@ -384,6 +500,11 @@ void GetBlockHeaderByHeight::Response::doToJson(rapidjson::Writer<epee::byte_str

 void GetBlockHeaderByHeight::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, header, header);
 }

@@ -395,6 +516,11 @@ void GetBlockHeadersByHeight::Request::doToJson(rapidjson::Writer<epee::byte_str

 void GetBlockHeadersByHeight::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, heights, heights);
 }

@@ -405,6 +531,11 @@ void GetBlockHeadersByHeight::Response::doToJson(rapidjson::Writer<epee::byte_st

 void GetBlockHeadersByHeight::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, headers, headers);
 }

@@ -424,6 +555,11 @@ void GetPeerList::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest)

 void GetPeerList::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, white_list, white_list);
  GET_FROM_JSON_OBJECT(val, gray_list, gray_list);
 }
@@ -436,6 +572,11 @@ void SetLogLevel::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest)

 void SetLogLevel::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, level, level);
 }

@@ -462,6 +603,11 @@ void GetTransactionPool::Response::doToJson(rapidjson::Writer<epee::byte_stream>

 void GetTransactionPool::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, transactions, transactions);
  GET_FROM_JSON_OBJECT(val, key_images, key_images);
 }
@@ -474,6 +620,11 @@ void HardForkInfo::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest)

 void HardForkInfo::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, version, version);
 }

@@ -484,6 +635,11 @@ void HardForkInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest

 void HardForkInfo::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, info, info);
 }

@@ -499,6 +655,11 @@ void GetOutputHistogram::Request::doToJson(rapidjson::Writer<epee::byte_stream>&

 void GetOutputHistogram::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, amounts, amounts);
  GET_FROM_JSON_OBJECT(val, min_count, min_count);
  GET_FROM_JSON_OBJECT(val, max_count, max_count);
@@ -513,6 +674,11 @@ void GetOutputHistogram::Response::doToJson(rapidjson::Writer<epee::byte_stream>

 void GetOutputHistogram::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, histogram, histogram);
 }

@@ -524,6 +690,11 @@ void GetOutputKeys::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest

 void GetOutputKeys::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, outputs, outputs);
 }

@@ -534,6 +705,11 @@ void GetOutputKeys::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des

 void GetOutputKeys::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, keys, keys);
 }

@@ -552,6 +728,11 @@ void GetRPCVersion::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des

 void GetRPCVersion::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, version, version);
 }

@@ -562,6 +743,11 @@ void GetFeeEstimate::Request::doToJson(rapidjson::Writer<epee::byte_stream>& des

 void GetFeeEstimate::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks);
 }

@@ -575,6 +761,11 @@ void GetFeeEstimate::Response::doToJson(rapidjson::Writer<epee::byte_stream>& de

 void GetFeeEstimate::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee);
  GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask);
  GET_FROM_JSON_OBJECT(val, size_scale, size_scale);
@@ -591,6 +782,11 @@ void GetOutputDistribution::Request::doToJson(rapidjson::Writer<epee::byte_strea

 void GetOutputDistribution::Request::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, amounts, amounts);
  GET_FROM_JSON_OBJECT(val, from_height, from_height);
  GET_FROM_JSON_OBJECT(val, to_height, to_height);
@@ -605,6 +801,11 @@ void GetOutputDistribution::Response::doToJson(rapidjson::Writer<epee::byte_stre

 void GetOutputDistribution::Response::fromJson(const rapidjson::Value& val)
 {
+  if (!val.IsObject())
+  {
+    throw json::WRONG_TYPE("json object");
+  }
+
  GET_FROM_JSON_OBJECT(val, status, status);
  GET_FROM_JSON_OBJECT(val, distributions, distributions);
 }
@@ -236,10 +236,8 @@ namespace cryptonote
    *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(nonce);
    if (block.major_version >= RX_BLOCK_VERSION)
    {
-      const uint64_t seed_height = is_current ? info.seed_height : info.previous_seed_height;
      const crypto::hash &seed_hash = is_current ? info.seed_hash : info.previous_seed_hash;
-      const uint64_t height = cryptonote::get_block_height(block);
-      crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, 0, 0);
+      crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data);
    }
    else
    {
@@ -158,13 +158,22 @@ void ZmqServer::serve()

      if (!pub || sockets[2].revents)
      {
-        std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags));
-        MDEBUG("Received RPC request: \"" << message << "\"");
-        epee::byte_slice response = handler.handle(std::move(message));
+        expect<std::string> message = net::zmq::receive(rep.get(), read_flags);
+        if (!message)
+        {
+          // EAGAIN can occur when using `zmq_poll`, which doesn't inspect for message validity
+          if (message != net::zmq::make_error_code(EAGAIN))
+            MONERO_THROW(message.error(), "Read failure on ZMQ-RPC");
+        }
+        else // no errors
+        {
+          MDEBUG("Received RPC request: \"" << *message << "\"");
+          epee::byte_slice response = handler.handle(std::move(*message));

-        const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()};
-        MDEBUG("Sending RPC reply: \"" << response_view << "\"");
-        MONERO_UNWRAP(net::zmq::send(std::move(response), rep.get()));
+          const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()};
+          MDEBUG("Sending RPC reply: \"" << response_view << "\"");
+          MONERO_UNWRAP(net::zmq::send(std::move(response), rep.get()));
+        }
      }
    }
  }
@@ -42,7 +42,7 @@ namespace serialization
    typename std::enable_if<!use_container_varint<T>(), bool>::type
    serialize_container_element(Archive& ar, T& e)
    {
-      return ::do_serialize(ar, e);
+      return do_serialize(ar, e);
    }

    template<typename Archive, typename T>
@@ -52,7 +52,7 @@ namespace serialization
      static constexpr const bool previously_varint = std::is_same<uint64_t, T>() || std::is_same<uint32_t, T>();

      if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving())
-        return ::do_serialize(ar, e);
+        return do_serialize(ar, e);
      ar.serialize_varint(e);
      return true;
    }
@@ -42,14 +42,12 @@ struct debug_archive : public json_archive<W> {
 };

 template <class T>
-struct serializer<debug_archive<true>, T>
+static inline bool do_serialize(debug_archive<true> &ar, T &v)
 {
-  static void serialize(debug_archive<true> &ar, T &v)
-  {
    ar.begin_object();
    ar.tag(variant_serialization_traits<debug_archive<true>, T>::get_tag());
-    serializer<json_archive<true>, T>::serialize(ar, v);
+    do_serialize(static_cast<json_archive<true>&>(ar), v);
    ar.end_object();
    ar.stream() << std::endl;
-  }
-};
+    return true;
+}
@@ -31,8 +31,6 @@
 #include "cryptonote_basic/difficulty.h"
 #include "serialization.h"

-template<> struct is_basic_type<cryptonote::difficulty_type> { typedef boost::true_type type; };
-
 template <template <bool> class Archive>
 inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff)
 {
@@ -1140,6 +1140,7 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig&
    INSERT_INTO_JSON_OBJECT(dest, bulletproofs, sig.p.bulletproofs);
    INSERT_INTO_JSON_OBJECT(dest, bulletproofs_plus, sig.p.bulletproofs_plus);
    INSERT_INTO_JSON_OBJECT(dest, mlsags, sig.p.MGs);
+    INSERT_INTO_JSON_OBJECT(dest, clsags, sig.p.CLSAGs);
    INSERT_INTO_JSON_OBJECT(dest, pseudo_outs, sig.get_pseudo_outs());

    dest.EndObject();
@@ -1175,6 +1176,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
    GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs, bulletproofs);
    GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs_plus, bulletproofs_plus);
    GET_FROM_JSON_OBJECT(prunable->value, sig.p.MGs, mlsags);
+    GET_FROM_JSON_OBJECT(prunable->value, sig.p.CLSAGs, clsags);
    GET_FROM_JSON_OBJECT(prunable->value, pseudo_outs, pseudo_outs);

    sig.get_pseudo_outs() = std::move(pseudo_outs);
@@ -1185,6 +1187,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
    sig.p.bulletproofs.clear();
    sig.p.bulletproofs_plus.clear();
    sig.p.MGs.clear();
+    sig.p.CLSAGs.clear();
    sig.get_pseudo_outs().clear();
  }
 }
@@ -1393,6 +1396,29 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig)
  GET_FROM_JSON_OBJECT(val, sig.cc, cc);
 }

+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::clsag& sig)
+{
+  dest.StartObject();
+
+  INSERT_INTO_JSON_OBJECT(dest, s, sig.s);
+  INSERT_INTO_JSON_OBJECT(dest, c1, sig.c1);
+  INSERT_INTO_JSON_OBJECT(dest, D, sig.D);
+
+  dest.EndObject();
+}
+
+void fromJsonValue(const rapidjson::Value& val, rct::clsag& sig)
+{
+  if (!val.IsObject())
+  {
+    throw WRONG_TYPE("key64 (rct::key[64])");
+  }
+
+  GET_FROM_JSON_OBJECT(val, sig.s, s);
+  GET_FROM_JSON_OBJECT(val, sig.c1, c1);
+  GET_FROM_JSON_OBJECT(val, sig.D, D);
+}
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info)
 {
  dest.StartObject();
@@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig);
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig);
 void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig);

+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::clsag& sig);
+void fromJsonValue(const rapidjson::Value& val, rct::clsag& sig);
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info);

@@ -47,7 +47,7 @@ namespace serialization
    typename std::enable_if<!use_pair_varint<T>(), bool>::type
    serialize_pair_element(Archive& ar, T& e)
    {
-      return ::do_serialize(ar, e);
+      return do_serialize(ar, e);
    }

    template<typename Archive, typename T>
@@ -57,7 +57,7 @@ namespace serialization
      static constexpr const bool previously_varint = std::is_same<uint64_t, T>();

      if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving())
-        return ::do_serialize(ar, e);
+        return do_serialize(ar, e);
      ar.serialize_varint(e);
      return true;
    }
@@ -57,73 +57,30 @@
 template <class T>
 struct is_blob_type { typedef boost::false_type type; };

-/*! \struct has_free_serializer
- *
- * \brief a descriptor for dispatching serialize
- */
-template <class T>
-struct has_free_serializer { typedef boost::true_type type; };
-
-/*! \struct is_basic_type
- *
- * \brief a descriptor for dispatching serialize
- */
-template <class T>
-struct is_basic_type { typedef boost::false_type type; };
-
-template<typename F, typename S>
-struct is_basic_type<std::pair<F,S>> { typedef boost::true_type type; };
-template<>
-struct is_basic_type<std::string> { typedef boost::true_type type; };
-
-/*! \struct serializer
- *
- * \brief ... wouldn't a class be better?
- * 
- * \detailed The logic behind serializing data. Places the archive
- * data into the supplied parameter. This dispatches based on the
- * supplied \a T template parameter's traits of is_blob_type or it is
- * an integral (as defined by the is_integral trait). Depends on the
- * \a Archive parameter to have overloaded the serialize_blob(T v,
- * size_t size) and serialize_int(T v) base on which trait it
- * applied. When the class has neither types, it falls to the
- * overloaded method do_serialize(Archive ar) in T to do the work.
- */
-template <class Archive, class T>
-struct serializer{
-  static bool serialize(Archive &ar, T &v) {
-    return serialize(ar, v, typename boost::is_integral<T>::type(), typename is_blob_type<T>::type(), typename is_basic_type<T>::type());
-  }
-  template<typename A>
-  static bool serialize(Archive &ar, T &v, boost::false_type, boost::true_type, A a) {
-    ar.serialize_blob(&v, sizeof(v));
-    return true;
-  }
-  template<typename A>
-  static bool serialize(Archive &ar, T &v, boost::true_type, boost::false_type, A a) {
-    ar.serialize_int(v);
-    return true;
-  }
-  static bool serialize(Archive &ar, T &v, boost::false_type, boost::false_type, boost::false_type) {
-    //serialize_custom(ar, v, typename has_free_serializer<T>::type());
-    return v.do_serialize(ar);
-  }
-  static bool serialize(Archive &ar, T &v, boost::false_type, boost::false_type, boost::true_type) {
-    //serialize_custom(ar, v, typename has_free_serializer<T>::type());
-    return do_serialize(ar, v);
-  }
-  static void serialize_custom(Archive &ar, T &v, boost::true_type) {
-  }
-};
-
 /*! \fn do_serialize(Archive &ar, T &v)
 *
- * \brief just calls the serialize function defined for ar and v...
+ * \brief main function for dispatching serialization for a given pair of archive and value types
+ * 
+ * Types marked true with is_blob_type<T> will be serialized as a blob, integral types will be
+ * serialized as integers, and types who have a `member_do_serialize` method will be serialized
+ * using that method. Booleans are serialized like blobs.
 */
 template <class Archive, class T>
-inline bool do_serialize(Archive &ar, T &v)
+inline std::enable_if_t<is_blob_type<T>::type::value, bool> do_serialize(Archive &ar, T &v)
 {
-  return ::serializer<Archive, T>::serialize(ar, v);
+  ar.serialize_blob(&v, sizeof(v));
+  return true;
+}
+template <class Archive, class T>
+inline std::enable_if_t<boost::is_integral<T>::value, bool> do_serialize(Archive &ar, T &v)
+{
+  ar.serialize_int(v);
+  return true;
+}
+template <class Archive, class T>
+inline auto do_serialize(Archive &ar, T &v) -> decltype(v.member_do_serialize(ar), true)
+{
+  return v.member_do_serialize(ar);
 }
 template <class Archive>
 inline bool do_serialize(Archive &ar, bool &v)
@@ -144,16 +101,6 @@ inline bool do_serialize(Archive &ar, bool &v)
    typedef boost::true_type type;					\
  }

-/*! \macro FREE_SERIALIZER
- *
- * \brief adds the has_free_serializer to the type
- */
-#define FREE_SERIALIZER(T)						\
-  template<>								\
-  struct has_free_serializer<T> {					\
-    typedef boost::true_type type;					\
-  }
-
 /*! \macro VARIANT_TAG
 *
 * \brief Adds the tag \tag to the \a Archive of \a Type
@@ -174,7 +121,7 @@ inline bool do_serialize(Archive &ar, bool &v)
 */
 #define BEGIN_SERIALIZE()						\
  template <bool W, template <bool> class Archive>			\
-  bool do_serialize(Archive<W> &ar) {
+  bool member_do_serialize(Archive<W> &ar) {

 /*! \macro BEGIN_SERIALIZE_OBJECT
 *
@@ -183,7 +130,7 @@ inline bool do_serialize(Archive &ar, bool &v)
 */
 #define BEGIN_SERIALIZE_OBJECT()					\
  template <bool W, template <bool> class Archive>			\
-  bool do_serialize(Archive<W> &ar) {					\
+  bool member_do_serialize(Archive<W> &ar) {					\
    ar.begin_object();							\
    bool r = do_serialize_object(ar);					\
    ar.end_object();							\
@@ -197,11 +144,6 @@ inline bool do_serialize(Archive &ar, bool &v)
 #define PREPARE_CUSTOM_VECTOR_SERIALIZATION(size, vec)			\
  ::serialization::detail::prepare_custom_vector_serialization(size, vec, typename Archive<W>::is_saving())

-/*! \macro PREPARE_CUSTOM_DEQUE_SERIALIZATION
- */
-#define PREPARE_CUSTOM_DEQUE_SERIALIZATION(size, vec)			\
-  ::serialization::detail::prepare_custom_deque_serialization(size, vec, typename Archive<W>::is_saving())
-
 /*! \macro END_SERIALIZE
 * \brief self-explanatory
 */
@@ -209,16 +151,6 @@ inline bool do_serialize(Archive &ar, bool &v)
  return ar.good();				\
  }

-/*! \macro VALUE(f)
- * \brief the same as FIELD(f)
- */
-#define VALUE(f)					\
-  do {							\
-    ar.tag(#f);						\
-    bool r = ::do_serialize(ar, f);			\
-    if (!r || !ar.good()) return false;			\
-  } while(0);
-
 /*! \macro FIELD_N(t,f)
 *
 * \brief serializes a field \a f tagged \a t  
@@ -226,7 +158,7 @@ inline bool do_serialize(Archive &ar, bool &v)
 #define FIELD_N(t, f)					\
  do {							\
    ar.tag(t);						\
-    bool r = ::do_serialize(ar, f);			\
+    bool r = do_serialize(ar, f);			\
    if (!r || !ar.good()) return false;			\
  } while(0);

@@ -237,7 +169,7 @@ inline bool do_serialize(Archive &ar, bool &v)
 #define FIELD(f)					\
  do {							\
    ar.tag(#f);						\
-    bool r = ::do_serialize(ar, f);			\
+    bool r = do_serialize(ar, f);			\
    if (!r || !ar.good()) return false;			\
  } while(0);

@@ -247,7 +179,7 @@ inline bool do_serialize(Archive &ar, bool &v)
 */
 #define FIELDS(f)							\
  do {									\
-    bool r = ::do_serialize(ar, f);					\
+    bool r = do_serialize(ar, f);					\
    if (!r || !ar.good()) return false;					\
  } while(0);

@@ -317,17 +249,6 @@ namespace serialization {
      vec.resize(size);
    }

-    template <typename T>
-    void prepare_custom_deque_serialization(size_t size, std::deque<T>& vec, const boost::mpl::bool_<true>& /*is_saving*/)
-    {
-    }
-
-    template <typename T>
-    void prepare_custom_deque_serialization(size_t size, std::deque<T>& vec, const boost::mpl::bool_<false>& /*is_saving*/)
-    {
-      vec.resize(size);
-    }
-
    /*! \fn do_check_stream_state
     *
     * \brief self explanatory
@@ -39,7 +39,7 @@ namespace serialization
    template <typename Archive, class T>
    bool serialize_tuple_element(Archive& ar, T& e)
    {
-      return ::do_serialize(ar, e);
+      return do_serialize(ar, e);
    }

    template <typename Archive>
@@ -72,7 +72,7 @@ struct variant_reader
  {
    if(variant_serialization_traits<Archive, current_type>::get_tag() == t) {
      current_type x;
-      if(!::do_serialize(ar, x))
+      if(!do_serialize(ar, x))
      {
        ar.set_fail();
        return false;
@@ -100,19 +100,13 @@ struct variant_reader<Archive, Variant, TBegin, TBegin>
  }
 };

-
-template <template <bool> class Archive, BOOST_VARIANT_ENUM_PARAMS(typename T)>
-struct serializer<Archive<false>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>>
-{
-  typedef boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> variant_type;
-  typedef typename Archive<false>::variant_tag_type variant_tag_type;
-  typedef typename variant_type::types types;
-
-  static bool serialize(Archive<false> &ar, variant_type &v) {
-    variant_tag_type t;
+template <template <bool> class Archive, typename... T>
+static bool do_serialize(Archive<false> &ar, boost::variant<T...> &v) {
+    using types = typename boost::variant<T...>::types;
+    typename Archive<false>::variant_tag_type t;
    ar.begin_variant();
    ar.read_variant_tag(t);
-    if(!variant_reader<Archive<false>, variant_type,
+    if(!variant_reader<Archive<false>, boost::variant<T...>,
       typename boost::mpl::begin<types>::type,
       typename boost::mpl::end<types>::type>::read(ar, v, t))
    {
@@ -121,27 +115,21 @@ struct serializer<Archive<false>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>>
    }
    ar.end_variant();
    return true;
-  }
-};
+}

-template <template <bool> class Archive, BOOST_VARIANT_ENUM_PARAMS(typename T)>
-struct serializer<Archive<true>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>>
+template <template <bool> class Archive>
+struct variant_write_visitor : public boost::static_visitor<bool>
 {
-  typedef boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> variant_type;
-  //typedef typename Archive<true>::variant_tag_type variant_tag_type;
-
-  struct visitor : public boost::static_visitor<bool>
-  {
    Archive<true> &ar;

-    visitor(Archive<true> &a) : ar(a) { }
+    variant_write_visitor(Archive<true> &a) : ar(a) { }

    template <class T>
    bool operator ()(T &rv) const
    {
      ar.begin_variant();
      ar.write_variant_tag(variant_serialization_traits<Archive<true>, T>::get_tag());
-      if(!::do_serialize(ar, rv))
+      if(!do_serialize(ar, rv))
      {
        ar.set_fail();
        return false;
@@ -149,9 +137,10 @@ struct serializer<Archive<true>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>>
      ar.end_variant();
      return true;
    }
-  };
-
-  static bool serialize(Archive<true> &ar, variant_type &v) {
-    return boost::apply_visitor(visitor(ar), v);
-  }
 };
+
+template <template <bool> class Archive, typename... T>
+static bool do_serialize(Archive<true> &ar, boost::variant<T...> &v)
+{
+  return boost::apply_visitor(variant_write_visitor<Archive>(ar), v);
+}
@@ -196,7 +196,7 @@ namespace
  const char* USAGE_INCOMING_TRANSFERS("incoming_transfers [available|unavailable] [verbose] [uses] [index=<N1>[,<N2>[,...]]]");
  const char* USAGE_PAYMENTS("payments <PID_1> [<PID_2> ... <PID_N>]");
  const char* USAGE_PAYMENT_ID("payment_id");
-  const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]");
+  const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [subtractfeefrom=<D0>[,<D1>,all,...]] [<payment_id>]");
  const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]");
  const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]");
  const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]");
@@ -531,7 +531,52 @@ namespace
      fail_msg_writer() << sw::tr("invalid format for subaddress lookahead; must be <major>:<minor>");
    return r;
  }
-}
+
+  static constexpr const char* SFFD_ARG_NAME{"subtractfeefrom="};
+
+  bool parse_subtract_fee_from_outputs
+  (
+    const std::string& arg,
+    tools::wallet2::unique_index_container& subtract_fee_from_outputs,
+    bool& subtract_fee_from_all,
+    bool& matches
+  )
+  {
+    matches = false;
+    if (!boost::string_ref{arg}.starts_with(SFFD_ARG_NAME)) // if arg doesn't match
+      return true;
+    matches = true;
+
+    const char* arg_end = arg.c_str() + arg.size();
+    for (const char* p = arg.c_str() + strlen(SFFD_ARG_NAME); p < arg_end;)
+    {
+      const char* new_p = nullptr;
+      const unsigned long dest_index = strtoul(p, const_cast<char**>(&new_p), 10);
+      if (dest_index == 0 && new_p == p) // numerical conversion failed
+      {
+        if (0 != strncmp(p, "all", 3))
+        {
+          fail_msg_writer() << tr("Failed to parse subtractfeefrom list");
+          return false;
+        }
+        subtract_fee_from_all = true;
+        break;
+      }
+      else if (dest_index > std::numeric_limits<uint32_t>::max())
+      {
+        fail_msg_writer() << tr("Destination index is too large") << ": " << dest_index;
+        return false;
+      }
+      else
+      {
+        subtract_fee_from_outputs.insert(dest_index);
+        p = new_p + 1; // skip the comma
+      }
+    }
+
+    return true;
+  }
+} // anonymous namespace

 void simple_wallet::handle_transfer_exception(const std::exception_ptr &e, bool trusted_daemon)
 {
@@ -3215,7 +3260,6 @@ bool simple_wallet::scan_tx(const std::vector<std::string> &args)
    }
    txids.insert(txid);
  }
-  std::vector<crypto::hash> txids_v(txids.begin(), txids.end());

  if (!m_wallet->is_trusted_daemon()) {
    message_writer(console_color_red, true) << tr("WARNING: this operation may reveal the txids to the remote node and affect your privacy");
@@ -3228,7 +3272,9 @@ bool simple_wallet::scan_tx(const std::vector<std::string> &args)
  LOCK_IDLE_SCOPE();
  m_in_manual_refresh.store(true);
  try {
-    m_wallet->scan_tx(txids_v);
+    m_wallet->scan_tx(txids);
+  } catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e) {
+    fail_msg_writer() << e.what() << ". Either connect to a trusted daemon by passing --trusted-daemon when starting the wallet, or use rescan_bc to rescan the chain.";
  } catch (const std::exception &e) {
    fail_msg_writer() << e.what();
  }
@@ -3289,7 +3335,7 @@ simple_wallet::simple_wallet()
                           tr("Show the blockchain height."));
  m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::transfer, _1),
                           tr(USAGE_TRANSFER),
-                           tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)"));
+                           tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included). The \"subtractfeefrom=\" list allows you to choose which destinations to fund the tx fee from instead of the change output. The fee will be split across the chosen destinations proportionally equally. For example, to make 3 transfers where the fee is taken from the first and third destinations, one could do: \"transfer <addr1> 3 <addr2> 0.5 <addr3> 1 subtractfeefrom=0,2\". Let's say the tx fee is 0.1. The balance would drop by exactly 4.5 XMR including fees, and addr1 & addr3 would receive 2.925 & 0.975 XMR, respectively. Use \"subtractfeefrom=all\" to spread the fee across all destinations."));
  m_cmd_binder.set_handler("locked_transfer",
                           boost::bind(&simple_wallet::on_command, this, &simple_wallet::locked_transfer,_1),
                           tr(USAGE_LOCKED_TRANSFER),
@@ -5894,7 +5940,10 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo
  {
    m_in_manual_refresh.store(true, std::memory_order_relaxed);
    epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);});
-    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money);
+    // For manual refresh don't allow incremental checking of the pool: Because we did not process the txs
+    // for us in the pool during automatic refresh we could miss some of them if we checked the pool
+    // incrementally here
+    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money, true, false);

    if (reset == ResetSoftKeepKI)
    {
@@ -6555,7 +6604,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
    local_args.erase(local_args.begin());
  }

-  uint32_t priority = 0;
+  uint32_t priority = m_wallet->get_default_priority();
  if (local_args.size() > 0 && parse_priority(local_args[0], priority))
    local_args.erase(local_args.begin());

@@ -6636,6 +6685,27 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
    local_args.pop_back();
  }

+  // Parse subtractfeefrom destination list
+  tools::wallet2::unique_index_container subtract_fee_from_outputs;
+  bool subtract_fee_from_all = false;
+  for (auto it = local_args.begin(); it < local_args.end();)
+  {
+    bool matches = false;
+    if (!parse_subtract_fee_from_outputs(*it, subtract_fee_from_outputs, subtract_fee_from_all, matches))
+    {
+      return false;
+    }
+    else if (matches)
+    {
+      it = local_args.erase(it);
+      break;
+    }
+    else
+    {
+      ++it;
+    }
+  }
+
  vector<cryptonote::address_parse_info> dsts_info;
  vector<cryptonote::tx_destination_entry> dsts;
  for (size_t i = 0; i < local_args.size(); )
@@ -6732,6 +6802,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
    dsts.push_back(de);
  }

+  if (subtract_fee_from_all)
+  {
+    subtract_fee_from_outputs.clear();
+    for (decltype(subtract_fee_from_outputs)::value_type i = 0; i < dsts.size(); ++i)
+      subtract_fee_from_outputs.insert(i);
+  }
+
  SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;);

  try
@@ -6750,13 +6827,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri
          return false;
        }
        unlock_block = bc_height + locked_blocks;
-        ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices);
+        ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, subtract_fee_from_outputs);
      break;
      default:
        LOG_ERROR("Unknown transfer method, using default");
        /* FALLTHRU */
      case Transfer:
-        ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices);
+        ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, subtract_fee_from_outputs);
      break;
    }

@@ -1,5 +1,5 @@
 #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@"
-#define DEF_MONERO_VERSION "0.18.1.2"
+#define DEF_MONERO_VERSION "0.18.3.2"
 #define DEF_MONERO_RELEASE_NAME "Fluorine Fermi"
 #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG
 #define DEF_MONERO_VERSION_IS_RELEASE @VERSION_IS_RELEASE@
@@ -535,7 +535,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas
        view_wallet->generate(path, password, address, viewkey);

        // Export/Import outputs
-        auto outputs = m_wallet->export_outputs();
+        auto outputs = m_wallet->export_outputs(true/*all*/);
        view_wallet->import_outputs(outputs);

        // Copy scanned blockchain
@@ -553,7 +553,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas

        // Export/Import key images
        // We already know the spent status from the outputs we exported, thus no need to check them again
-        auto key_images = m_wallet->export_key_images();
+        auto key_images = m_wallet->export_key_images(true/*all*/);
        uint64_t spent = 0;
        uint64_t unspent = 0;
        view_wallet->import_key_images(key_images.second, key_images.first, spent, unspent, false);
@@ -1302,11 +1302,15 @@ bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
        }
        txids_u.insert(txid);
    }
-    std::vector<crypto::hash> txids_v(txids_u.begin(), txids_u.end());

    try
    {
-        m_wallet->scan_tx(txids_v);
+        m_wallet->scan_tx(txids_u);
+    }
+    catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e)
+    {
+        setStatusError(e.what());
+        return false;
    }
    catch (const std::exception &e)
    {
@@ -1782,7 +1786,7 @@ uint64_t WalletImpl::estimateTransactionFee(const std::vector<std::pair<std::str
        m_wallet->use_fork_rules(HF_VERSION_CLSAG, 0),
        m_wallet->use_fork_rules(HF_VERSION_BULLETPROOF_PLUS, 0),
        m_wallet->use_fork_rules(HF_VERSION_VIEW_TAGS, 0),
-        m_wallet->get_base_fee(),
+        m_wallet->get_base_fee(priority),
        m_wallet->get_fee_quantization_mask());
 }

@@ -38,6 +38,7 @@
 #include <ctime>
 #include <iostream>
 #include <stdexcept>
+#include <cstdint>

 //  Public interface for libwallet library
 namespace Monero {
@@ -392,4 +392,56 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_payment_info(bool mining, boo
  return boost::none;
 }

+boost::optional<std::string> NodeRPCProxy::get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f)
+{
+  const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp
+  for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE)
+  {
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req_t = AUTO_VAL_INIT(req_t);
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response resp_t = AUTO_VAL_INIT(resp_t);
+
+    const size_t n_txids = std::min<size_t>(SLICE_SIZE, txids.size() - offset);
+    for (size_t n = offset; n < (offset + n_txids); ++n)
+      req_t.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[n]));
+    MDEBUG("asking for " << req_t.txs_hashes.size() << " transactions");
+    req_t.decode_as_json = false;
+    req_t.prune = true;
+
+    bool r = false;
+    {
+      const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
+      uint64_t pre_call_credits = m_rpc_payment_state.credits;
+      req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key);
+      r = net_utils::invoke_http_json("/gettransactions", req_t, resp_t, m_http_client, rpc_timeout);
+      if (r && resp_t.status == CORE_RPC_STATUS_OK)
+        check_rpc_cost(m_rpc_payment_state, "/gettransactions", resp_t.credits, pre_call_credits, resp_t.txs.size() * COST_PER_TX);
+    }
+
+    f(req_t, resp_t, r);
+  }
+  return boost::optional<std::string>();
+}
+
+boost::optional<std::string> NodeRPCProxy::get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header)
+{
+  if (m_offline)
+    return boost::optional<std::string>("offline");
+
+  cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request req_t = AUTO_VAL_INIT(req_t);
+  cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response resp_t = AUTO_VAL_INIT(resp_t);
+  req_t.height = height;
+
+  {
+    const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
+    uint64_t pre_call_credits = m_rpc_payment_state.credits;
+    req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key);
+    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req_t, resp_t, m_http_client, rpc_timeout);
+    RETURN_ON_RPC_RESPONSE_ERROR(r, epee::json_rpc::error{}, resp_t, "getblockheaderbyheight");
+    check_rpc_cost(m_rpc_payment_state, "getblockheaderbyheight", resp_t.credits, pre_call_credits, COST_PER_BLOCK_HEADER);
+  }
+
+  block_header = std::move(resp_t.block_header);
+  return boost::optional<std::string>();
+}
+
 }
@@ -59,6 +59,8 @@ public:
  boost::optional<std::string> get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_blocks, std::vector<uint64_t> &fees);
  boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask);
  boost::optional<std::string> get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie);
+  boost::optional<std::string> get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f);
+  boost::optional<std::string> get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header);

 private:
  template<typename T> void handle_payment_changes(const T &res, std::true_type) {
@@ -101,6 +101,7 @@ namespace tools
    uint64_t pick();
    gamma_picker(const std::vector<uint64_t> &rct_offsets);
    gamma_picker(const std::vector<uint64_t> &rct_offsets, double shape, double scale);
+    uint64_t get_num_rct_outs() const { return num_rct_outputs; }

  private:
    struct gamma_engine
@@ -138,6 +139,7 @@ private:
  public:
    // Full wallet callbacks
    virtual void on_new_block(uint64_t height, const cryptonote::block& block) {}
+    virtual void on_reorg(uint64_t height, uint64_t blocks_detached, size_t transfers_detached) {}
    virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, uint64_t burnt, const cryptonote::subaddress_index& subaddr_index, bool is_change, uint64_t unlock_time) {}
    virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {}
    virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {}
@@ -475,7 +477,7 @@ private:
      time_t m_sent_time;
      std::vector<cryptonote::tx_destination_entry> m_dests;
      crypto::hash m_payment_id;
-      enum { pending, pending_not_in_pool, failed } m_state;
+      enum { pending, pending_in_pool, failed } m_state;
      uint64_t m_timestamp;
      uint32_t m_subaddr_account;   // subaddress account of your wallet to be used in this transfer
      std::set<uint32_t> m_subaddr_indices;  // set of address indices used as inputs in this transfer
@@ -597,6 +599,7 @@ private:

    typedef std::vector<transfer_details> transfer_container;
    typedef serializable_unordered_multimap<crypto::hash, payment_details> payment_container;
+    typedef std::set<uint32_t> unique_index_container;

    struct multisig_sig
    {
@@ -816,6 +819,30 @@ private:
      bool empty() const { return tx_extra_fields.empty() && primary.empty() && additional.empty(); }
    };

+    struct detached_blockchain_data
+    {
+      hashchain detached_blockchain;
+      size_t original_chain_size;
+      std::unordered_set<crypto::hash> detached_tx_hashes;
+      std::unordered_map<crypto::hash, std::vector<cryptonote::tx_destination_entry>> detached_confirmed_txs_dests;
+    };
+
+    struct process_tx_entry_t
+    {
+      cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry tx_entry;
+      cryptonote::transaction tx;
+      crypto::hash tx_hash;
+    };
+
+    struct tx_entry_data
+    {
+      std::vector<process_tx_entry_t> tx_entries;
+      uint64_t lowest_height;
+      uint64_t highest_height;
+
+      tx_entry_data(): lowest_height((uint64_t)-1), highest_height(0) {}
+    };
+
    /*!
     * \brief  Generates a wallet or restores one. Assumes the multisig setup
      *        has already completed for the provided multisig info.
@@ -915,22 +942,32 @@ private:
    /*!
     * \brief store_to  Stores wallet to another file(s), deleting old ones
     * \param path      Path to the wallet file (keys and address filenames will be generated based on this filename)
-     * \param password  Password to protect new wallet (TODO: probably better save the password in the wallet object?)
+     * \param password  Password that currently locks the wallet
+     * \param force_rewrite_keys if true, always rewrite keys file
+     *
+     * Leave both "path" and "password" blank to restore the cache file to the current position in the disk
+     * (which is the same as calling `store()`). If you want to store the wallet with a new password,
+     * use the method `change_password()`.
+     *
+     * Normally the keys file is not overwritten when storing, except when force_rewrite_keys is true
+     * or when `path` is a new wallet file.
+     *
+     * \throw error::invalid_password If storing keys file and old password is incorrect
     */
-    void store_to(const std::string &path, const epee::wipeable_string &password);
+    void store_to(const std::string &path, const epee::wipeable_string &password, bool force_rewrite_keys = false);
    /*!
     * \brief get_keys_file_data  Get wallet keys data which can be stored to a wallet file.
-     * \param password            Password of the encrypted wallet buffer (TODO: probably better save the password in the wallet object?)
+     * \param password            Password that currently locks the wallet
     * \param watch_only          true to include only view key, false to include both spend and view keys
     * \return                    Encrypted wallet keys data which can be stored to a wallet file
+     * \throw                     error::invalid_password if password does not match current wallet
     */
    boost::optional<wallet2::keys_file_data> get_keys_file_data(const epee::wipeable_string& password, bool watch_only);
    /*!
     * \brief get_cache_file_data   Get wallet cache data which can be stored to a wallet file.
-     * \param password              Password to protect the wallet cache data (TODO: probably better save the password in the wallet object?)
-     * \return                      Encrypted wallet cache data which can be stored to a wallet file
+     * \return                      Encrypted wallet cache data which can be stored to a wallet file (using current password)
     */
-    boost::optional<wallet2::cache_file_data> get_cache_file_data(const epee::wipeable_string& password);
+    boost::optional<wallet2::cache_file_data> get_cache_file_data();

    std::string path() const;

@@ -1023,7 +1060,7 @@ private:
    bool is_deprecated() const;
    void refresh(bool trusted_daemon);
    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched);
-    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true);
+    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true, bool try_incremental = true, uint64_t max_blocks = std::numeric_limits<uint64_t>::max());
    bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok);

    void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; }
@@ -1034,7 +1071,7 @@ private:
    bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const;
    bool has_multisig_partial_key_images() const;
    bool has_unknown_key_images() const;
-    bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const;
+    bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string()) const;
    bool key_on_device() const { return get_device_type() != hw::device::device_type::SOFTWARE; }
    hw::device::device_type get_device_type() const { return m_key_device_type; }
    bool reconnect_device();
@@ -1076,11 +1113,11 @@ private:
    bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const;
    bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL);
    bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func);
-    std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices);     // pass subaddr_indices by value on purpose
+    std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const unique_index_container& subtract_fee_from_outputs = {});     // pass subaddr_indices by value on purpose
    std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices);
    std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra);
    std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra);
-    bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const;
+    bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, const std::vector<cryptonote::tx_destination_entry>& dsts, const unique_index_container& subtract_fee_from_outputs = {}) const;
    void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux);
    void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux);
    uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent);
@@ -1380,7 +1417,7 @@ private:
    std::string get_spend_proof(const crypto::hash &txid, const std::string &message);
    bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str);

-    void scan_tx(const std::vector<crypto::hash> &txids);
+    void scan_tx(const std::unordered_set<crypto::hash> &txids);

    /*!
     * \brief  Generates a proof that proves the reserve of unspent funds
@@ -1506,9 +1543,9 @@ private:
    bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false);
    crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const;

-    void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false);
+    void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false, bool try_incremental = false);
    void process_pool_state(const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &txs);
-    void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes);
+    void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes, bool remove_if_found);

    std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const;
    std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const;
@@ -1644,6 +1681,7 @@ private:
    void thaw(const crypto::key_image &ki);
    bool frozen(const crypto::key_image &ki) const;
    bool frozen(const transfer_details &td) const;
+    bool frozen(const multisig_tx_set& txs) const; // does partially signed txset contain frozen enotes?

    bool save_to_file(const std::string& path_to_file, const std::string& binary, bool is_printable = false) const;
    static bool load_from_file(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000);
@@ -1699,18 +1737,24 @@ private:
     */
    bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password);
    bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt);
-    void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL, bool ignore_callbacks = false);
    bool should_skip_block(const cryptonote::block &b, uint64_t height) const;
    void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
-    void detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    detached_blockchain_data detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    void handle_reorg(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
    void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const;
    bool clear();
    void clear_soft(bool keep_key_images=false);
-    void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height);
+    void pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height);
    void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes);
    void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false);
-    void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception);
+    void pull_and_parse_next_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception);
    void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    bool accept_pool_tx_for_processing(const crypto::hash &txid);
+    void process_unconfirmed_transfer(bool incremental, const crypto::hash &txid, wallet2::unconfirmed_transfer_details &tx_details, bool seen_in_pool, std::chrono::system_clock::time_point now, bool refreshed);
+    void process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed);
+    void update_pool_state_by_pool_query(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false);
+    void update_pool_state_from_pool_data(bool incremental, const std::vector<crypto::hash> &removed_pool_txids, const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &added_pool_txs, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed);
    uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const;
    bool prepare_file_names(const std::string& file_path);
    void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height);
@@ -1753,6 +1797,9 @@ private:
    crypto::chacha_key get_ringdb_key();
    void setup_keys(const epee::wipeable_string &password);
    size_t get_transfer_details(const crypto::key_image &ki) const;
+    tx_entry_data get_tx_entries(const std::unordered_set<crypto::hash> &txids);
+    void sort_scan_tx_entries(std::vector<process_tx_entry_t> &unsorted_tx_entries);
+    void process_scan_txs(const tx_entry_data &txs_to_scan, const tx_entry_data &txs_to_reprocess, const std::unordered_set<crypto::hash> &tx_hashes_to_reprocess, detached_blockchain_data &dbd);

    void register_devices();
    hw::device& lookup_device(const std::string & device_descriptor);
@@ -1845,6 +1892,11 @@ private:
    // If m_refresh_from_block_height is explicitly set to zero we need this to differentiate it from the case that
    // m_refresh_from_block_height was defaulted to zero.*/
    bool m_explicit_refresh_from_block_height;
+    uint64_t m_pool_info_query_time;
+    std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> m_process_pool_txs;
+    uint64_t m_skip_to_height;
+    // m_skip_to_height is useful when we don't want to modify the wallet's restore height.
+    // m_refresh_from_block_height is also a wallet's restore height which should remain constant unless explicitly modified by the user.
    bool m_confirm_non_default_ring_size;
    AskPasswordType m_ask_password;
    uint64_t m_max_reorg_depth;
@@ -85,6 +85,7 @@ namespace tools
    //         tx_too_big
    //         zero_amount
    //         zero_destination
+    //         subtract_fee_from_bad_index
    //       wallet_rpc_error *
    //         daemon_busy
    //         no_connection_to_daemon
@@ -93,6 +94,8 @@ namespace tools
    //         get_output_distribution
    //         payment_required
    //       wallet_files_doesnt_correspond
+    //       scan_tx_error *
+    //         wont_reprocess_recent_txs_via_untrusted_daemon
    //
    // * - class with protected ctor

@@ -777,6 +780,15 @@ namespace tools
      }
    };
    //----------------------------------------------------------------------------------------------------
+    struct subtract_fee_from_bad_index : public transfer_error
+    {
+      explicit subtract_fee_from_bad_index(std::string&& loc, long bad_index)
+        : transfer_error(std::move(loc),
+          "subtractfeefrom: bad index: " + std::to_string(bad_index) + " (indexes are 0-based)")
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
    struct wallet_rpc_error : public wallet_logic_error
    {
      const std::string& request() const { return m_request; }
@@ -915,6 +927,23 @@ namespace tools
      }
    };
    //----------------------------------------------------------------------------------------------------
+    struct scan_tx_error : public wallet_logic_error
+    {
+    protected:
+      explicit scan_tx_error(std::string&& loc, const std::string& message)
+        : wallet_logic_error(std::move(loc), message)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
+    struct wont_reprocess_recent_txs_via_untrusted_daemon : public scan_tx_error
+    {
+      explicit wont_reprocess_recent_txs_via_untrusted_daemon(std::string&& loc)
+        : scan_tx_error(std::move(loc), "The wallet has already seen 1 or more recent transactions than the scanned tx")
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------

 #if !defined(_MSC_VER)

@@ -155,8 +155,7 @@ bool wallet2::search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads
        const uint8_t major_version = hashing_blob[0];
        if (major_version >= RX_BLOCK_VERSION)
        {
-          const int miners = 1;
-          crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data, miners, 0);
+          crypto::rx_slow_hash(seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data);
        }
        else
        {
@@ -60,6 +60,7 @@ using namespace epee;
 #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc"

 #define DEFAULT_AUTO_REFRESH_PERIOD 20 // seconds
+#define REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE 256    // just to split refresh in separate calls to play nicer with other threads

 #define CHECK_MULTISIG_ENABLED() \
  do \
@@ -79,6 +80,7 @@ namespace
  const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false};
  const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"};
  const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false};
+  const command_line::arg_descriptor<bool> arg_no_initial_sync = {"no-initial-sync", "Skips the initial sync before listening for connections", false};

  constexpr const char default_rpc_username[] = "monero";

@@ -149,12 +151,17 @@ namespace tools
        return true;
      if (boost::posix_time::microsec_clock::universal_time() < m_last_auto_refresh_time + boost::posix_time::seconds(m_auto_refresh_period))
        return true;
+      uint64_t blocks_fetched = 0;
      try {
-        if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon());
+        bool received_money = false;
+        if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, blocks_fetched, received_money, true, true, REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE);
      } catch (const std::exception& ex) {
        LOG_ERROR("Exception at while refreshing, what=" << ex.what());
      }
-      m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time();
+      // if we got the max amount of blocks, do not set the last refresh time, we did only part of the refresh and will
+      // continue asap, and only set the last refresh time once the refresh is actually finished
+      if (blocks_fetched < REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE)
+        m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time();
      return true;
    }, 1000);
    m_net_server.add_idle_handler([this](){
@@ -981,9 +988,9 @@ namespace tools
    return amount;
  }
  //------------------------------------------------------------------------------------------------------------------------------
-  template<typename Ts, typename Tu, typename Tk>
+  template<typename Ts, typename Tu, typename Tk, typename Ta>
  bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector,
-      bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay,
+      bool get_tx_key, Ts& tx_key, Tu &amount, Ta &amounts_by_dest, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay,
      Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er)
  {
    for (const auto & ptx : ptx_vector)
@@ -1000,6 +1007,12 @@ namespace tools
      fill(fee, ptx.fee);
      fill(weight, cryptonote::get_transaction_weight(ptx.tx));

+      // add amounts by destination
+      tools::wallet_rpc::amounts_list abd;
+      for (const auto& dst : ptx.dests)
+        abd.amounts.push_back(dst.amount);
+      fill(amounts_by_dest, abd);
+
      // add spent key images
      tools::wallet_rpc::key_image_list key_image_list;
      bool all_are_txin_to_key = std::all_of(ptx.tx.vin.begin(), ptx.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool
@@ -1080,7 +1093,7 @@ namespace tools
    {
      uint64_t mixin = m_wallet->adjust_mixin(req.ring_size ? req.ring_size - 1 : 0);
      uint32_t priority = m_wallet->adjust_priority(req.priority);
-      std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices);
+      std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, req.subtract_fee_from_outputs);

      if (ptx_vector.empty())
      {
@@ -1097,7 +1110,7 @@ namespace tools
        return false;
      }

-      return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
+      return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.amounts_by_dest, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
          res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er);
    }
    catch (const std::exception& e)
@@ -1145,7 +1158,7 @@ namespace tools
        return false;
      }

-      return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
+      return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.amounts_by_dest_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
          res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er);
    }
    catch (const std::exception& e)
@@ -1534,7 +1547,7 @@ namespace tools
    {
      std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions();

-      return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
+      return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.amounts_by_dest_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
          res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er);
    }
    catch (const std::exception& e)
@@ -1594,7 +1607,7 @@ namespace tools
      uint32_t priority = m_wallet->adjust_priority(req.priority);
      std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra, req.account_index, subaddr_indices);

-      return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
+      return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.amounts_by_dest_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
          res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er);
    }
    catch (const std::exception& e)
@@ -1671,7 +1684,7 @@ namespace tools
        return false;
      }

-      return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
+      return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.amounts_by_dest, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay,
          res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er);
    }
    catch (const std::exception& e)
@@ -3167,7 +3180,7 @@ namespace tools
          return false;
      }

-      std::vector<crypto::hash> txids;
+      std::unordered_set<crypto::hash> txids;
      std::list<std::string>::const_iterator i = req.txids.begin();
      while (i != req.txids.end())
      {
@@ -3180,11 +3193,15 @@ namespace tools
          }

          crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data());
-          txids.push_back(txid);
+          txids.insert(txid);
      }

      try {
          m_wallet->scan_tx(txids);
+      }  catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e) {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = e.what() + std::string(". Either connect to a trusted daemon or rescan the chain.");
+          return false;
      } catch (const std::exception &e) {
          handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
          return false;
@@ -3802,7 +3819,7 @@ namespace tools
    std::string old_language;

    // check the given seed
-    {
+    if (!req.enable_multisig_experimental) {
      if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, old_language))
      {
        er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
@@ -3825,6 +3842,13 @@ namespace tools

    // process seed_offset if given
    {
+      if (req.enable_multisig_experimental && !req.seed_offset.empty())
+      {
+        er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+        er.message = "Multisig seeds are not compatible with seed offsets";
+        return false;
+      }
+
      if (!req.seed_offset.empty())
      {
        recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset);
@@ -3888,7 +3912,27 @@ namespace tools
    crypto::secret_key recovery_val;
    try
    {
-      recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false);
+      if (req.enable_multisig_experimental)
+      {
+        // Parse multisig seed into raw multisig data
+        epee::wipeable_string multisig_data;
+        multisig_data.resize(req.seed.size() / 2);
+        if (!epee::from_hex::to_buffer(epee::to_mut_byte_span(multisig_data), req.seed))
+        {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = "Multisig seed not represented as hexadecimal string";
+          return false;
+        }
+
+        // Generate multisig wallet
+        wal->generate(wallet_file, std::move(rc.second).password(), multisig_data, false);
+        wal->enable_multisig(true);
+      }
+      else
+      {
+        // Generate normal wallet
+        recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false);
+      }
      MINFO("Wallet has been restored.\n");
    }
    catch (const std::exception &e)
@@ -3899,7 +3943,7 @@ namespace tools

    // // Convert the secret key back to seed
    epee::wipeable_string electrum_words;
-    if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language))
+    if (!req.enable_multisig_experimental && !crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language))
    {
      er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
      er.message = "Failed to encode seed";
@@ -4521,6 +4565,7 @@ public:
      const auto password_file = command_line::get_arg(vm, arg_password_file);
      const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password);
      const auto password_prompt = prompt_for_password ? password_prompter : nullptr;
+      const auto no_initial_sync = command_line::get_arg(vm, arg_no_initial_sync);

      if(!wallet_file.empty() && !from_json.empty())
      {
@@ -4589,7 +4634,8 @@ public:

      try
      {
-        wal->refresh(wal->is_trusted_daemon());
+        if (!no_initial_sync)
+          wal->refresh(wal->is_trusted_daemon());
      }
      catch (const std::exception& e)
      {
@@ -4700,6 +4746,7 @@ int main(int argc, char** argv) {
  command_line::add_arg(desc_params, arg_wallet_dir);
  command_line::add_arg(desc_params, arg_prompt_for_password);
  command_line::add_arg(desc_params, arg_rpc_client_secret_key);
+  command_line::add_arg(desc_params, arg_no_initial_sync);

  daemonizer::init_options(hidden_options, desc_params);
  desc_params.add(hidden_options);
@@ -263,9 +263,9 @@ namespace tools
      bool not_open(epee::json_rpc::error& er);
      void handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code);

-      template<typename Ts, typename Tu, typename Tk>
+      template<typename Ts, typename Tu, typename Tk, typename Ta>
      bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector,
-          bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay,
+          bool get_tx_key, Ts& tx_key, Tu &amount, Ta &amounts_by_dest, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay,
          Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er);

      bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er);
--- a/Show More
+++ b/Show More