Merge pull request #9014

69de381 add a test for the long term weight cache (Boog900)
810f6a6 Fix: long term block weight cache The long term block weight cache was doing a wrong calculation when adding a new block to the cache. (Boog900)

.github/workflows/build.yml

@@ -151,7 +151,7 @@ jobs:
     - name: install monero dependencies
       run: ${{env.APT_INSTALL_LINUX}}
     - name: install Python dependencies
-      run: pip install requests psutil monotonic zmq
+      run: pip install requests psutil monotonic zmq deepdiff
     - name: tests
       env:
         CTEST_OUTPUT_ON_FAILURE: ON

README.md

@@ -138,8 +138,8 @@ Dates are provided in the format YYYY-MM-DD.
 | 1978433                        | 2019-11-30 | v12               | v0.15.0.0              | v0.16.0.0                  | New PoW based on RandomX, only allow >= 2 outputs, change to the block median used to calculate penalty, v1 coinbases are forbidden, rct sigs in coinbase forbidden, 10 block lock time for incoming outputs
 | 2210000                        | 2020-10-17 | v13               | v0.17.0.0              | v0.17.3.2                  | New CLSAG transaction format
 | 2210720                        | 2020-10-18 | v14               | v0.17.1.1              | v0.17.3.2                  | forbid old MLSAG transaction format
-| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.2.1                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
-| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.2.1                  | forbid old v14 transaction format
+| 2688888                        | 2022-08-13 | v15               | v0.18.0.0              | v0.18.3.1                  | ringsize = 16, bulletproofs+, view tags, adjusted dynamic block weight algorithm
+| 2689608                        | 2022-08-14 | v16               | v0.18.0.0              | v0.18.3.1                  | forbid old v14 transaction format
 | XXXXXXX                        | XXX-XX-XX | XXX                | vX.XX.X.X              | vX.XX.X.X                  | XXX |
 
 X's indicate that these details have not been determined as of commit date.
@@ -344,7 +344,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (
     ```bash
     git clone https://github.com/monero-project/monero.git
     cd monero
-    git checkout v0.18.2.1
+    git checkout v0.18.3.1
     ```
 
 * Build:
@@ -463,10 +463,10 @@ application.
     cd monero
     ```
 
-* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.2.1'. If you don't care about the version and just want binaries from master, skip this step:
+* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.18.3.1'. If you don't care about the version and just want binaries from master, skip this step:
 
     ```bash
-    git checkout v0.18.2.1
+    git checkout v0.18.3.1
     ```
 
 * If you are on a 64-bit system, run:

contrib/depends/packages/openssl.mk

@@ -1,8 +1,8 @@
 package=openssl
-$(package)_version=1.1.1t
+$(package)_version=1.1.1u
 $(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b
+$(package)_sha256_hash=e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"

contrib/epee/include/file_io_utils.h

@@ -29,6 +29,7 @@
 
 #include <string>
 #include <ctime>
+#include <cstdint>
 
 namespace epee
 {

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -583,11 +583,8 @@ namespace net_utils
             break;
         }
       }
-      else if (ec.value())
-        terminate();
       else {
-        cancel_timer();
-        on_interrupted();
+        terminate();
       }
     };
     m_strand.post(

contrib/epee/include/span.h

@@ -147,6 +147,16 @@ namespace epee
     return {reinterpret_cast<const std::uint8_t*>(src.data()), src.size_bytes()}; 
   }
 
+  //! \return `span<std::uint8_t>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<std::uint8_t> to_mut_byte_span(T& src)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_empty<value_type>(), "empty value types will not work -> sizeof == 1");
+    static_assert(!has_padding<value_type>(), "source value type may have padding");
+    return {reinterpret_cast<std::uint8_t*>(src.data()), src.size() * sizeof(value_type)};
+  }
+
   //! \return `span<const std::uint8_t>` which represents the bytes at `&src`.
   template<typename T>
   span<const std::uint8_t> as_byte_span(const T& src) noexcept

contrib/epee/include/storages/parserse_base_utils.h

@@ -30,6 +30,7 @@
 
 #include <boost/utility/string_ref_fwd.hpp>
 #include <string>
+#include <cstdint>
 
 namespace epee 
 {

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -33,6 +33,9 @@
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #else 

contrib/epee/include/storages/portable_storage_from_json.h

@@ -31,6 +31,9 @@
 #include "parserse_base_utils.h"
 #include "file_io_utils.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100
 
 namespace epee

contrib/epee/src/net_ssl.cpp

@@ -496,6 +496,13 @@ void ssl_options_t::configure(
   const std::string& host) const
 {
   socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+  {
+    // in case server is doing "virtual" domains, set hostname
+    SSL* const ssl_ctx = socket.native_handle();
+    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
+      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
+  }
+
 
   /* Using system-wide CA store for client verification is funky - there is
      no expected hostname for server to verify against. If server doesn't have
@@ -513,11 +520,7 @@ void ssl_options_t::configure(
   {
     socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);
 
-    // in case server is doing "virtual" domains, set hostname
-    SSL* const ssl_ctx = socket.native_handle();
-    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
-      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
-
+    
     socket.set_verify_callback([&](const bool preverified, boost::asio::ssl::verify_context &ctx)
     {
       // preverified means it passed system or user CA check. System CA is never loaded

contrib/gitian/DOCKRUN.md

@@ -57,7 +57,7 @@ The dockrun.sh script will do everything to build the binaries. Just specify the
 version to build as its only argument, e.g.
 
 ```bash
-VERSION=v0.18.2.1
+VERSION=v0.18.3.1
 ./dockrun.sh $VERSION
 ```
 

contrib/gitian/README.md

@@ -133,7 +133,7 @@ Common setup part:
 su - gitianuser
 
 GH_USER=YOUR_GITHUB_USER_NAME
-VERSION=v0.18.2.1
+VERSION=v0.18.3.1
 ```
 
 Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. 

src/checkpoints/checkpoints.cpp

@@ -245,6 +245,9 @@ namespace cryptonote
     ADD_CHECKPOINT2(2720000, "b19fb41dff15bd1016afbee9f8469f05aab715c9e5d1b974466a11fd58ecbb86", "0x3216b5851ddbb61");
     ADD_CHECKPOINT2(2817000, "39726d19ccaac01d150bec827b877ffae710b516bd633503662036ef4422e577", "0x3900669561954c1");
     ADD_CHECKPOINT2(2844000, "28fc7b446dfef5b469f5778eb72ddf32a307a5f5a9823d1c394e772349e05d40", "0x3af384ec0e97d12");
+    ADD_CHECKPOINT2(2851000, "5bf0e47fc782263191a33f63a67db6c711781dc2a3c442e17ed901ec401be5c9", "0x3b6cd8a8ed610e8");
+    ADD_CHECKPOINT2(2971000, "3d4cac5ac515eeabd18769ab943af85f36db51d28720def0d0e6effc2c8f5ce3", "0x436e532738b8b5b");
+    ADD_CHECKPOINT2(2985000, "08f5e6b7301c1b6ed88268a28f8677a06e8ff943b3f9e48d3080f71f9c134bfb", "0x444b7b42a633c96");
     return true;
   }
 

src/common/combinator.h

@@ -34,6 +34,7 @@
 #include <iostream>
 #include <vector>
 #include <stdexcept>
+#include <cstdint>
 
 namespace tools {
 

src/common/perf_timer.cpp

@@ -62,7 +62,7 @@ namespace tools
     while (1)
     {
       t1 = epee::misc_utils::get_ns_count();
-      if (t1 - t0 > 1*1000000000) break; // work one second
+      if (t1 - t0 > 1*100000000) break; // work 0.1 seconds
     }
 
     uint64_t r1 = get_tick_count();

src/common/util.cpp

@@ -882,13 +882,6 @@ std::string get_nix_version_display_string()
 
   bool is_local_address(const std::string &address)
   {
-    // always assume Tor/I2P addresses to be untrusted by default
-    if (is_privacy_preserving_network(address))
-    {
-      MDEBUG("Address '" << address << "' is Tor/I2P, non local");
-      return false;
-    }
-
     // extract host
     epee::net_utils::http::url_content u_c;
     if (!epee::net_utils::parse_url(address, u_c))
@@ -902,20 +895,22 @@ std::string get_nix_version_display_string()
       return false;
     }
 
-    // resolve to IP
-    boost::asio::io_service io_service;
-    boost::asio::ip::tcp::resolver resolver(io_service);
-    boost::asio::ip::tcp::resolver::query query(u_c.host, "");
-    boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query);
-    while (i != boost::asio::ip::tcp::resolver::iterator())
+    if (u_c.host == "localhost" || boost::ends_with(u_c.host, ".localhost")) { // RFC 6761 (6.3)
+      MDEBUG("Address '" << address << "' is local");
+      return true;
+    }
+
+    boost::system::error_code ec;
+    const auto parsed_ip = boost::asio::ip::address::from_string(u_c.host, ec);
+    if (ec) {
+      MDEBUG("Failed to parse '" << address << "' as IP address: " << ec.message() << ". Considering it not local");
+      return false;
+    }
+
+    if (parsed_ip.is_loopback())
     {
-      const boost::asio::ip::tcp::endpoint &ep = *i;
-      if (ep.address().is_loopback())
-      {
-        MDEBUG("Address '" << address << "' is local");
-        return true;
-      }
-      ++i;
+      MDEBUG("Address '" << address << "' is local");
+      return true;
     }
 
     MDEBUG("Address '" << address << "' is not local");

src/crypto/c_threads.h

@@ -42,10 +42,11 @@
 #define CTHR_RWLOCK_TRYLOCK_READ(x)	TryAcquireSRWLockShared(&x)
 
 #define CTHR_THREAD_TYPE	HANDLE
-#define CTHR_THREAD_RTYPE	void
-#define CTHR_THREAD_RETURN	return
-#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthread(func, 0, arg)) != -1L)
-#define CTHR_THREAD_JOIN(thr)			WaitForSingleObject((HANDLE)thr, INFINITE)
+#define CTHR_THREAD_RTYPE	unsigned __stdcall
+#define CTHR_THREAD_RETURN	_endthreadex(0); return 0;
+#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L)
+#define CTHR_THREAD_JOIN(thr)			do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0)
+#define CTHR_THREAD_CLOSE(thr)			CloseHandle((HANDLE)thr);
 
 #else
 
@@ -64,5 +65,6 @@
 #define CTHR_THREAD_RETURN	return NULL
 #define CTHR_THREAD_CREATE(thr, func, arg)	(pthread_create(&thr, NULL, func, arg) == 0)
 #define CTHR_THREAD_JOIN(thr)			pthread_join(thr, NULL)
+#define CTHR_THREAD_CLOSE(thr)
 
 #endif

src/crypto/rx-slow-hash.c

@@ -332,7 +332,7 @@ static void rx_init_dataset(size_t max_threads) {
       local_abort("Couldn't start RandomX seed thread");
     }
   }
-  rx_seedthread(&si[n1]);
+  randomx_init_dataset(main_dataset, si[n1].si_cache, si[n1].si_start, si[n1].si_count);
   for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]);
   CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
 
@@ -402,6 +402,7 @@ void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads)
   if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) {
     local_abort("Couldn't start RandomX seed thread");
   }
+  CTHR_THREAD_CLOSE(t);
 }
 
 void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) {

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -1229,7 +1229,7 @@ namespace cryptonote
     char *end = NULL;
     errno = 0;
     const unsigned long long ull = strtoull(buf, &end, 10);
-    CHECK_AND_ASSERT_THROW_MES(ull != ULONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
+    CHECK_AND_ASSERT_THROW_MES(ull != ULLONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
     CHECK_AND_ASSERT_THROW_MES(ull != 0 || amount == 0, "Overflow in rounding");
     return ull;
   }

src/cryptonote_basic/miner.cpp

@@ -523,7 +523,7 @@ namespace cryptonote
   bool miner::worker_thread()
   {
     const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment
-    crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
+    bool rx_set = false;
 
     MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]");
     MGINFO("Miner thread was started ["<< th_local_index << "]");
@@ -575,6 +575,13 @@ namespace cryptonote
 
       b.nonce = nonce;
       crypto::hash h;
+
+      if ((b.major_version >= RX_BLOCK_VERSION) && !rx_set)
+      {
+        crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
+        rx_set = true;
+      }
+
       m_gbh(b, height, NULL, tools::get_max_concurrency(), h);
 
       if(check_hash(h, local_diff))

src/cryptonote_basic/verification_context.h

@@ -42,7 +42,12 @@ namespace cryptonote
     static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none");
 
     relay_method m_relay; // gives indication on how tx should be relayed (if at all)
-    bool m_verifivation_failed; //bad tx, should drop connection
+    bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense
+    // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop
+    // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be
+    // confused with breaking soft fork rules, because tx could be later added to the chain if mined
+    // because it does not violate consensus rules.
+    bool m_no_drop_offense;
     bool m_verifivation_impossible; //the transaction is related with an alternative blockchain
     bool m_added_to_pool; 
     bool m_low_mixin;

src/cryptonote_core/blockchain.cpp

@@ -2065,7 +2065,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
       cryptonote::blobdata blob;
       if (m_tx_pool.have_tx(txid, relay_category::legacy))
       {
-        if (m_tx_pool.get_transaction_info(txid, td))
+        if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
         {
           bei.block_cumulative_weight += td.weight;
         }
@@ -3710,7 +3710,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b
       div128_64(hi, lo, median_block_weight, &hi, &lo, NULL, NULL);
       assert(hi == 0);
       lo -= lo / 20;
-      return lo;
+      return lo == 0 ? 1 : lo;
     }
     else
     {
@@ -4616,40 +4616,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
   }
   else
   {
-    const uint64_t block_weight = m_db->get_block_weight(db_height - 1);
+    const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
+    const uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks);
 
-    uint64_t long_term_median;
-    if (db_height == 1)
-    {
-      long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
-    }
-    else
-    {
-      uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
-      if (nblocks == db_height)
-        --nblocks;
-      long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks);
-    }
-
-    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
-
-    uint64_t short_term_constraint = m_long_term_effective_median_block_weight;
-    if (hf_version >= HF_VERSION_2021_SCALING)
-      short_term_constraint += m_long_term_effective_median_block_weight * 7 / 10;
-    else
-      short_term_constraint += m_long_term_effective_median_block_weight * 2 / 5;
-    uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint);
-
-    if (db_height == 1)
-    {
-      long_term_median = long_term_block_weight;
-    }
-    else
-    {
-      m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1);
-      m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight);
-      long_term_median = m_long_term_block_weights_cache_rolling_median.median();
-    }
     m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
 
     std::vector<uint64_t> weights;
@@ -5582,7 +5551,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "af0467d5d8ac1ad7232ebfd8610ec775df3438cc3b7e8033c10ad71874b72e15";
+static const char expected_block_hashes_hash[] = "bc9c91329af96137390d9c709fa3cecc924f1b25dadb7589f0d751cd93f3cc39";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
   if (get_checkpoints == nullptr || !m_fast_sync)

src/cryptonote_core/cryptonote_core.cpp

@@ -1099,7 +1099,7 @@ namespace cryptonote
       else if(tvc[i].m_verifivation_impossible)
       {MERROR_VER("Transaction verification impossible: " << results[i].hash);}
 
-      if(tvc[i].m_added_to_pool)
+      if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE)
       {
         MDEBUG("tx added: " << results[i].hash);
         valid_events = true;
@@ -1727,6 +1727,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
+  {
+    return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const
   {
     m_mempool.get_transactions(txs, include_sensitive_data);
@@ -1739,6 +1744,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const
   {
     m_mempool.get_transaction_stats(stats, include_sensitive_data);

src/cryptonote_core/cryptonote_core.h

@@ -510,6 +510,23 @@ namespace cryptonote
      bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const;
 
      /**
+      * @copydoc tx_memory_pool::get_pool_transactions_info
+      * @param include_sensitive_txes include private transactions
+      *
+      * @note see tx_memory_pool::get_pool_transactions_info
+      */
+     bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_pool_info
+      * @param include_sensitive_txes include private transactions
+      * @param max_tx_count max allowed added_txs in response
+      *
+      * @note see tx_memory_pool::get_pool_info
+      */
+     bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
+    /**
       * @copydoc tx_memory_pool::get_transactions
       * @param include_sensitive_txes include private transactions
       *

src/cryptonote_core/tx_pool.cpp

@@ -133,6 +133,12 @@ namespace cryptonote
     // class code expects unsigned values throughout
     if (m_next_check < time_t(0))
       throw std::runtime_error{"Unexpected time_t (system clock) value"};
+
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_start_time = (time_t)0;
+    // We don't set these to "now" already here as we don't know how long it takes from construction
+    // of the pool until it "goes to work". It's safer to set when the first actual txs enter the
+    // corresponding lists.
   }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
@@ -207,6 +213,7 @@ namespace cryptonote
     {
       tvc.m_verifivation_failed = true;
       tvc.m_fee_too_low = true;
+      tvc.m_no_drop_offense = true;
       return false;
     }
 
@@ -225,6 +232,7 @@ namespace cryptonote
       LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
       tvc.m_verifivation_failed = true;
       tvc.m_tx_extra_too_big = true;
+      tvc.m_no_drop_offense = true;
       return false;
     }
 
@@ -290,7 +298,7 @@ namespace cryptonote
             return false;
 
           m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time);
           lock.commit();
         }
         catch (const std::exception &e)
@@ -361,7 +369,7 @@ namespace cryptonote
 
           m_blockchain.remove_txpool_tx(id);
           m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time);
         }
         lock.commit();
       }
@@ -382,7 +390,7 @@ namespace cryptonote
 
     ++m_cookie;
 
-    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)));
+    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size());
 
     prune(m_txpool_max_weight);
 
@@ -473,7 +481,8 @@ namespace cryptonote
         reduce_txpool_weight(meta.weight);
         remove_transaction_keyimages(tx, txid);
         MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first);
-        m_txs_by_fee_and_receive_time.erase(it--);
+        remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted));
+        it--;
         changed = true;
       }
       catch (const std::exception &e)
@@ -555,8 +564,7 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
-    auto sorted_it = find_tx_in_sorted_container(id);
-
+    bool sensitive = false;
     try
     {
       LockedTXN lock(m_blockchain.get_db());
@@ -587,6 +595,7 @@ namespace cryptonote
       do_not_relay = meta.do_not_relay;
       double_spend_seen = meta.double_spend_seen;
       pruned = meta.pruned;
+      sensitive = !meta.matches(relay_category::broadcasted);
 
       // remove first, in case this throws, so key images aren't removed
       m_blockchain.remove_txpool_tx(id);
@@ -600,13 +609,12 @@ namespace cryptonote
       return false;
     }
 
-    if (sorted_it != m_txs_by_fee_and_receive_time.end())
-      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive);
     ++m_cookie;
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const
+  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
   {
     PERF_TIMER(get_transaction_info);
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -618,7 +626,12 @@ namespace cryptonote
       txpool_tx_meta_t meta;
       if (!m_blockchain.get_txpool_tx_meta(txid, meta))
       {
-        MERROR("Failed to find tx in txpool");
+        LOG_PRINT_L2("Failed to find tx in txpool: " << txid);
+        return false;
+      }
+      if (!include_sensitive_data && !meta.matches(relay_category::broadcasted))
+      {
+        // We don't want sensitive data && the tx is sensitive, so no need to return it
         return false;
       }
       cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
@@ -644,11 +657,13 @@ namespace cryptonote
       td.kept_by_block = meta.kept_by_block;
       td.last_failed_height = meta.last_failed_height;
       td.last_failed_id = meta.last_failed_id;
-      td.receive_time = meta.receive_time;
-      td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time;
+      td.receive_time = include_sensitive_data ? meta.receive_time : 0;
+      td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0;
       td.relayed = meta.relayed;
       td.do_not_relay = meta.do_not_relay;
       td.double_spend_seen = meta.double_spend_seen;
+      if (include_blob)
+        td.tx_blob = std::move(txblob);
     }
     catch (const std::exception &e)
     {
@@ -658,6 +673,25 @@ namespace cryptonote
 
     return true;
   }
+  //------------------------------------------------------------------
+  bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    txs.clear();
+
+    for (const auto &it: txids)
+    {
+      tx_details details;
+      bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/);
+      if (success)
+      {
+        txs.push_back(std::make_pair(it, std::move(details)));
+      }
+    }
+    return true;
+  }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const
   {
@@ -719,15 +753,7 @@ namespace cryptonote
          (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) )
       {
         LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age );
-        auto sorted_it = find_tx_in_sorted_container(txid);
-        if (sorted_it == m_txs_by_fee_and_receive_time.end())
-        {
-          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
-        }
-        else
-        {
-          m_txs_by_fee_and_receive_time.erase(sorted_it);
-        }
+        remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted));
         m_timed_out_transactions.insert(txid);
         remove.push_back(std::make_pair(txid, meta.weight));
       }
@@ -881,9 +907,12 @@ namespace cryptonote
             meta.last_relayed_time = std::chrono::system_clock::to_time_t(now);
 
           m_blockchain.update_txpool_tx(hash, meta);
-
           // wait until db update succeeds to ensure tx is visible in the pool
           was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted);
+
+          if (was_just_broadcasted)
+            // Make sure the tx gets re-added with an updated time
+            add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now));
         }
       }
       catch (const std::exception &e)
@@ -936,6 +965,81 @@ namespace cryptonote
     }, false, category);
   }
   //------------------------------------------------------------------
+  bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    incremental = true;
+    if (start_time == (time_t)0)
+    {
+      // Giving no start time means give back whole pool
+      incremental = false;
+    }
+    else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0))
+    {
+      if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time))
+      {
+        // If either of the two lists do not go back far enough it's not possible to
+        // deliver incremental pool info
+        incremental = false;
+      }
+      // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time
+    }
+    else
+    {
+      // Some incremental info still missing completely
+      incremental = false;
+    }
+
+    added_txs.clear();
+    remaining_added_txids.clear();
+    removed_txs.clear();
+
+    std::vector<crypto::hash> txids;
+    if (!incremental)
+    {
+      LOG_PRINT_L2("Giving back the whole pool");
+      // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the
+      // anonymous method somehow results in an LMDB error with transactions we have to build a list of
+      // ids first and get the full info afterwards
+      get_transaction_hashes(txids, include_sensitive);
+      if (txids.size() > max_tx_count)
+      {
+        remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end());
+        txids.erase(txids.begin() + max_tx_count, txids.end());
+      }
+      get_transactions_info(txids, added_txs, include_sensitive);
+      return true;
+    }
+
+    // Give back incrementally, based on time of entry into the map
+    for (const auto &pit : m_added_txs_by_id)
+    {
+      if (pit.second >= start_time)
+        txids.push_back(pit.first);
+    }
+    get_transactions_info(txids, added_txs, include_sensitive);
+    if (added_txs.size() > max_tx_count)
+    {
+      remaining_added_txids.reserve(added_txs.size() - max_tx_count);
+      for (size_t i = max_tx_count; i < added_txs.size(); ++i)
+        remaining_added_txids.push_back(added_txs[i].first);
+      added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end());
+    }
+
+    std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time);
+    while (rit != m_removed_txs_by_time.end())
+    {
+      if (include_sensitive || !rit->second.sensitive)
+      {
+        removed_txs.push_back(rit->second.txid);
+      }
+      ++rit;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------
   void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1640,6 +1744,12 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
+    // Simply throw away incremental info, too difficult to update
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
+
     MINFO("Validating txpool contents for v" << (unsigned)version);
 
     LockedTXN lock(m_blockchain.get_db());
@@ -1697,6 +1807,106 @@ namespace cryptonote
     return n_removed;
   }
   //---------------------------------------------------------------------------------
+  void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time)
+  {
+
+    time_t now = time(NULL);
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it == m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.insert(std::make_pair(txid, now));
+    }
+    else
+    {
+      // This tx was already added to the map earlier, probably because then it was in the "stem"
+      // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set
+      // a new time for clients that are not allowed to see sensitive txs to make sure they will
+      // see it now if they query incrementally
+      it->second = now;
+
+      auto sorted_it = find_tx_in_sorted_container(txid);
+      if (sorted_it == m_txs_by_fee_and_receive_time.end())
+      {
+        MERROR("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container");
+      }
+      else
+      {
+        m_txs_by_fee_and_receive_time.erase(sorted_it);
+      }
+    }
+    m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid);
+
+    // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time'
+    // whether we have that txid there and if yes remove it; this results in possible duplicates
+    // where we return certain txids as deleted AND in the pool at the same time which requires
+    // clients to process deleted ones BEFORE processing pool txs
+    if (m_added_txs_start_time == (time_t)0)
+    {
+      m_added_txs_start_time = now;
+    }
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive)
+  {
+    if (sorted_it == m_txs_by_fee_and_receive_time.end())
+    {
+      LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
+    }
+    else
+    {
+      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    }
+
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it != m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.erase(it);
+    }
+    else
+    {
+      MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs");
+    }
+    track_removed_tx(txid, sensitive);
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive)
+  {
+    time_t now = time(NULL);
+    m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive}));
+    MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size());
+    if (m_removed_txs_start_time == (time_t)0)
+    {
+      m_removed_txs_start_time = now;
+    }
+
+    // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set
+    // an absolute size limit plus delete entries that are x minutes old (which is ok because clients
+    // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back)
+    const int MAX_REMOVED = 20000;
+    if (m_removed_txs_by_time.size() > MAX_REMOVED)
+    {
+      auto erase_it = m_removed_txs_by_time.begin();
+      std::advance(erase_it, MAX_REMOVED / 4 + 1);
+      m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it);
+      m_removed_txs_start_time = m_removed_txs_by_time.begin()->first;
+      MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size());
+    }
+    else
+    {
+      time_t earliest = now - (30 * 60);  // 30 minutes
+      std::map<time_t, removed_tx_info>::iterator from, to;
+      from = m_removed_txs_by_time.begin();
+      to = m_removed_txs_by_time.lower_bound(earliest);
+      int distance = std::distance(from, to);
+      if (distance > 0)
+      {
+        m_removed_txs_by_time.erase(from, to);
+        m_removed_txs_start_time = earliest;
+        MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size());
+      }
+    }
+  }
+  //---------------------------------------------------------------------------------
   bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1704,6 +1914,10 @@ namespace cryptonote
 
     m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT;
     m_txs_by_fee_and_receive_time.clear();
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
     m_spent_key_images.clear();
     m_txpool_weight = 0;
     std::vector<crypto::hash> remove;
@@ -1728,7 +1942,7 @@ namespace cryptonote
           MFATAL("Failed to insert key images from txpool tx");
           return false;
         }
-        m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid);
+        add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time);
         m_txpool_weight += meta.weight;
         return true;
       }, true, relay_category::all);

src/cryptonote_core/tx_pool.h

@@ -428,6 +428,7 @@ namespace cryptonote
     struct tx_details
     {
       transaction tx;  //!< the transaction
+      cryptonote::blobdata tx_blob; //!< the transaction's binary blob
       size_t blob_size;  //!< the transaction's size
       size_t weight;  //!< the transaction's weight
       uint64_t fee;  //!< the transaction's fee amount
@@ -466,13 +467,25 @@ namespace cryptonote
     /**
      * @brief get infornation about a single transaction
      */
-    bool get_transaction_info(const crypto::hash &txid, tx_details &td) const;
+    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;
+
+    /**
+     * @brief get information about multiple transactions
+     */
+    bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const;
 
     /**
      * @brief get transactions not in the passed set
      */
     bool get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const;
 
+    /**
+     * @brief get info necessary for update of pool-related info in a wallet, preferably incremental
+     *
+     * @return true on success, false on error
+     */
+    bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
   private:
 
     /**
@@ -577,6 +590,10 @@ namespace cryptonote
      */
     void prune(size_t bytes = 0);
 
+    void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time);
+    void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive);
+    void track_removed_tx(const crypto::hash& txid, bool sensitive);
+
     //TODO: confirm the below comments and investigate whether or not this
     //      is the desired behavior
     //! map key images to transactions which spent them
@@ -609,6 +626,26 @@ private:
 
     std::atomic<uint64_t> m_cookie; //!< incremented at each change
 
+    // Info when transactions entered the pool, accessible by txid
+    std::unordered_map<crypto::hash, time_t> m_added_txs_by_id;
+
+    // Info at what time the pool started to track the adding of transactions
+    time_t m_added_txs_start_time;
+
+    struct removed_tx_info
+    {
+      crypto::hash txid;
+      bool sensitive;
+    };
+
+    // Info about transactions that were removed from the pool, ordered by the time
+    // of deletion
+    std::multimap<time_t, removed_tx_info> m_removed_txs_by_time;
+
+    // Info how far back in time the list of removed tx ids currently reaches
+    // (it gets shorted periodically to prevent overflow)
+    time_t m_removed_txs_start_time;
+
     /**
      * @brief get an iterator to a transaction in the sorted container
      *

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -979,8 +979,18 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_NEW_TRANSACTIONS (" << arg.txs.size() << " txes)");
+    std::unordered_set<blobdata> seen;
     for (const auto &blob: arg.txs)
+    {
       MLOGIF_P2P_MESSAGE(cryptonote::transaction tx; crypto::hash hash; bool ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx, hash);, ret, "Including transaction " << hash);
+      if (seen.find(blob) != seen.end())
+      {
+        LOG_PRINT_CCONTEXT_L1("Duplicate transaction in notification, dropping connection");
+        drop_connection(context, false, false);
+        return 1;
+      }
+      seen.insert(blob);
+    }
 
     if(context.m_state != cryptonote_connection_context::state_normal)
       return 1;
@@ -1020,7 +1030,7 @@ namespace cryptonote
     for (auto& tx : arg.txs)
     {
       tx_verification_context tvc{};
-      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true))
+      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense)
       {
         LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection");
         drop_connection(context, false, false);

src/daemon/main.cpp

@@ -219,6 +219,19 @@ int main(int argc, char const * argv[])
       {
         po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm);
       }
+      catch (const po::unknown_option &e)
+      {
+        std::string unrecognized_option = e.get_option_name();
+        if (all_options.find_nothrow(unrecognized_option, false))
+        {
+          std::cerr << "Option '" << unrecognized_option << "' is not allowed in the config file, please use it as a command line flag." << std::endl;
+        }
+        else
+        {
+          std::cerr << "Unrecognized option '" << unrecognized_option << "' in config file." << std::endl;
+        }
+        return 1;
+      }
       catch (const std::exception &e)
       {
         // log system isn't initialized yet

src/rpc/core_rpc_server.cpp

@@ -598,88 +598,165 @@ namespace cryptonote
 
     CHECK_PAYMENT(req, res, 1);
 
-    // quick check for noop
-    if (!req.block_ids.empty())
+    res.daemon_time = (uint64_t)time(NULL);
+    // Always set daemon time, and set it early rather than late, as delivering some incremental pool
+    // info twice because of slightly overlapping time intervals is no problem, whereas producing gaps
+    // and never delivering something is
+
+    bool get_blocks = false;
+    bool get_pool = false;
+    switch (req.requested_info)
     {
-      uint64_t last_block_height;
-      crypto::hash last_block_hash;
-      m_core.get_blockchain_top(last_block_height, last_block_hash);
-      if (last_block_hash == req.block_ids.front())
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY:
+        // Compatibility value 0: Clients that do not set 'requested_info' want blocks, and only blocks
+        get_blocks = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL:
+        get_blocks = true;
+        get_pool = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY:
+        get_pool = true;
+        break;
+      default:
+        res.status = "Failed, wrong requested info";
+        return true;
+    }
+
+    res.pool_info_extent = COMMAND_RPC_GET_BLOCKS_FAST::NONE;
+
+    if (get_pool)
+    {
+      const bool restricted = m_restricted && ctx;
+      const bool request_has_rpc_origin = ctx != NULL;
+      const bool allow_sensitive = !request_has_rpc_origin || !restricted;
+      const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits<size_t>::max();
+
+      bool incremental;
+      std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> added_pool_txs;
+      bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental);
+      if (success)
       {
-        res.start_height = 0;
-        res.current_height = m_core.get_current_blockchain_height();
-        res.status = CORE_RPC_STATUS_OK;
+        res.added_pool_txs.clear();
+        if (m_rpc_payment)
+        {
+          CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH);
+        }
+        for (const auto &added_pool_tx: added_pool_txs)
+        {
+          COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info;
+          info.tx_hash = added_pool_tx.first;
+          std::stringstream oss;
+          binary_archive<true> ar(oss);
+          bool r = req.prune
+            ? const_cast<cryptonote::transaction&>(added_pool_tx.second.tx).serialize_base(ar)
+            : ::serialization::serialize(ar, const_cast<cryptonote::transaction&>(added_pool_tx.second.tx));
+          if (!r)
+          {
+            res.status = "Failed to serialize transaction";
+            return true;
+          }
+          info.tx_blob = oss.str();
+          info.double_spend_seen = added_pool_tx.second.double_spend_seen;
+          res.added_pool_txs.push_back(std::move(info));
+        }
+      }
+      if (success)
+      {
+        res.pool_info_extent = incremental ? COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL : COMMAND_RPC_GET_BLOCKS_FAST::FULL;
+      }
+      else
+      {
+        res.status = "Failed to get pool info";
         return true;
       }
     }
 
-    size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-    if (m_rpc_payment)
+    if (get_blocks)
     {
-      max_blocks = res.credits / COST_PER_BLOCK;
-      if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
-        max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-      if (max_blocks == 0)
+      // quick check for noop
+      if (!req.block_ids.empty())
       {
-        res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+        uint64_t last_block_height;
+        crypto::hash last_block_hash;
+        m_core.get_blockchain_top(last_block_height, last_block_hash);
+        if (last_block_hash == req.block_ids.front())
+        {
+          res.start_height = 0;
+          res.current_height = last_block_height + 1;
+          res.status = CORE_RPC_STATUS_OK;
+          return true;
+        }
+      }
+
+      size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+      if (m_rpc_payment)
+      {
+        max_blocks = res.credits / COST_PER_BLOCK;
+        if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
+          max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+        if (max_blocks == 0)
+        {
+          res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+          return true;
+        }
+      }
+
+      std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
+      if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
+      {
+        res.status = "Failed";
+        add_host_fail(ctx);
         return true;
       }
-    }
 
-    std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
-    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
-    {
-      res.status = "Failed";
-      add_host_fail(ctx);
-      return true;
-    }
+      CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
 
-    CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
-
-    size_t size = 0, ntxes = 0;
-    res.blocks.reserve(bs.size());
-    res.output_indices.reserve(bs.size());
-    for(auto& bd: bs)
-    {
-      res.blocks.resize(res.blocks.size()+1);
-      res.blocks.back().pruned = req.prune;
-      res.blocks.back().block = bd.first.first;
-      size += bd.first.first.size();
-      res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
-      ntxes += bd.second.size();
-      res.output_indices.back().indices.reserve(1 + bd.second.size());
-      if (req.no_miner_tx)
-        res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
-      res.blocks.back().txs.reserve(bd.second.size());
-      for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+      size_t size = 0, ntxes = 0;
+      res.blocks.reserve(bs.size());
+      res.output_indices.reserve(bs.size());
+      for(auto& bd: bs)
       {
-        res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
-        i->second.clear();
-        i->second.shrink_to_fit();
-        size += res.blocks.back().txs.back().blob.size();
-      }
+        res.blocks.resize(res.blocks.size()+1);
+        res.blocks.back().pruned = req.prune;
+        res.blocks.back().block = bd.first.first;
+        size += bd.first.first.size();
+        res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
+        ntxes += bd.second.size();
+        res.output_indices.back().indices.reserve(1 + bd.second.size());
+        if (req.no_miner_tx)
+          res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
+        res.blocks.back().txs.reserve(bd.second.size());
+        for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+        {
+          res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
+          i->second.clear();
+          i->second.shrink_to_fit();
+          size += res.blocks.back().txs.back().blob.size();
+        }
 
-      const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
-      if (n_txes_to_lookup > 0)
-      {
-        std::vector<std::vector<uint64_t>> indices;
-        bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
-        if (!r)
+        const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
+        if (n_txes_to_lookup > 0)
         {
-          res.status = "Failed";
-          return true;
+          std::vector<std::vector<uint64_t>> indices;
+          bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
+          if (!r)
+          {
+            res.status = "Failed";
+            return true;
+          }
+          if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
+          {
+            res.status = "Failed";
+            return true;
+          }
+          for (size_t i = 0; i < indices.size(); ++i)
+            res.output_indices.back().indices.push_back({std::move(indices[i])});
         }
-        if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
-        {
-          res.status = "Failed";
-          return true;
-        }
-        for (size_t i = 0; i < indices.size(); ++i)
-          res.output_indices.back().indices.push_back({std::move(indices[i])});
       }
+      MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
     }
 
-    MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }
@@ -919,17 +996,16 @@ namespace cryptonote
     // try the pool for any missing txes
     size_t found_in_pool = 0;
     std::unordered_set<crypto::hash> pool_tx_hashes;
-    std::unordered_map<crypto::hash, tx_info> per_tx_pool_tx_info;
+    std::unordered_map<crypto::hash, tx_memory_pool::tx_details> per_tx_pool_tx_details;
     if (!missed_txs.empty())
     {
-      std::vector<tx_info> pool_tx_info;
-      std::vector<spent_key_image_info> pool_key_image_info;
-      bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted);
+      std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> pool_txs;
+      bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted);
       if(r)
       {
         // sort to match original request
         std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs;
-        std::vector<tx_info>::const_iterator i;
+        std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>::const_iterator i;
         unsigned txs_processed = 0;
         for (const crypto::hash &h: vh)
         {
@@ -949,36 +1025,23 @@ namespace cryptonote
             sorted_txs.push_back(std::move(txs[txs_processed]));
             ++txs_processed;
           }
-          else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end())
+          else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair<crypto::hash, tx_memory_pool::tx_details> &pt) { return h == pt.first; })) != pool_txs.end())
           {
-            cryptonote::transaction tx;
-            if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx))
-            {
-              res.status = "Failed to parse and validate tx from blob";
-              return true;
-            }
+            const tx_memory_pool::tx_details &td = i->second;
             std::stringstream ss;
             binary_archive<true> ba(ss);
-            bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
+            bool r = const_cast<cryptonote::transaction&>(td.tx).serialize_base(ba);
             if (!r)
             {
               res.status = "Failed to serialize transaction base";
               return true;
             }
             const cryptonote::blobdata pruned = ss.str();
-            const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx);
-            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size())));
+            const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx);
+            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size())));
             missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h));
             pool_tx_hashes.insert(h);
-            const std::string hash_string = epee::string_tools::pod_to_hex(h);
-            for (const auto &ti: pool_tx_info)
-            {
-              if (ti.id_hash == hash_string)
-              {
-                per_tx_pool_tx_info.insert(std::make_pair(h, ti));
-                break;
-              }
-            }
+            per_tx_pool_tx_details.insert(std::make_pair(h, td));
             ++found_in_pool;
           }
         }
@@ -1074,8 +1137,8 @@ namespace cryptonote
       {
         e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max();
         e.confirmations = 0;
-        auto it = per_tx_pool_tx_info.find(tx_hash);
-        if (it != per_tx_pool_tx_info.end())
+        auto it = per_tx_pool_tx_details.find(tx_hash);
+        if (it != per_tx_pool_tx_details.end())
         {
           e.double_spend_seen = it->second.double_spend_seen;
           e.relayed = it->second.relayed;
@@ -2115,7 +2178,8 @@ namespace cryptonote
     // Fixing of high orphan issue for most pools
     // Thanks Boolberry!
     block b;
-    if(!parse_and_validate_block_from_blob(blockblob, b))
+    crypto::hash blk_id;
+    if(!parse_and_validate_block_from_blob(blockblob, b, blk_id))
     {
       error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB;
       error_resp.message = "Wrong block blob";
@@ -2138,6 +2202,7 @@ namespace cryptonote
       error_resp.message = "Block not accepted";
       return false;
     }
+    res.block_id = epee::string_tools::pod_to_hex(blk_id);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }

src/rpc/core_rpc_server_commands_defs.h

@@ -88,7 +88,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 3
-#define CORE_RPC_VERSION_MINOR 12
+#define CORE_RPC_VERSION_MINOR 13
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -162,18 +162,29 @@ namespace cryptonote
   struct COMMAND_RPC_GET_BLOCKS_FAST
   {
 
+    enum REQUESTED_INFO
+    {
+      BLOCKS_ONLY = 0,
+      BLOCKS_AND_POOL = 1,
+      POOL_ONLY = 2
+    };
+
     struct request_t: public rpc_access_request_base
     {
+      uint8_t     requested_info;
       std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
       uint64_t    start_height;
       bool        prune;
       bool        no_miner_tx;
+      uint64_t    pool_info_since;
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_request_base)
+        KV_SERIALIZE_OPT(requested_info, (uint8_t)0)
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(prune)
         KV_SERIALIZE_OPT(no_miner_tx, false)
+        KV_SERIALIZE_OPT(pool_info_since, (uint64_t)0)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -196,12 +207,37 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
 
+    struct pool_tx_info
+    {
+      crypto::hash tx_hash;
+      blobdata tx_blob;
+      bool double_spend_seen;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_VAL_POD_AS_BLOB(tx_hash)
+        KV_SERIALIZE(tx_blob)
+        KV_SERIALIZE(double_spend_seen)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    enum POOL_INFO_EXTENT
+    {
+      NONE = 0,
+      INCREMENTAL = 1,
+      FULL = 2
+    };
+
     struct response_t: public rpc_access_response_base
     {
       std::vector<block_complete_entry> blocks;
       uint64_t    start_height;
       uint64_t    current_height;
       std::vector<block_output_indices> output_indices;
+      uint64_t    daemon_time;
+      uint8_t     pool_info_extent;
+      std::vector<pool_tx_info> added_pool_txs;
+      std::vector<crypto::hash> remaining_added_pool_txids;
+      std::vector<crypto::hash> removed_pool_txids;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_response_base)
@@ -209,6 +245,17 @@ namespace cryptonote
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(current_height)
         KV_SERIALIZE(output_indices)
+        KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0)
+        KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0)
+        if (pool_info_extent != POOL_INFO_EXTENT::NONE)
+        {
+          KV_SERIALIZE(added_pool_txs)
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids)
+        }
+        if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL)
+        {
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
+        }
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;
@@ -1068,8 +1115,11 @@ namespace cryptonote
     
     struct response_t: public rpc_response_base
     {
+      std::string block_id;
+
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_response_base)
+        KV_SERIALIZE(block_id)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;

src/serialization/json_object.cpp

@@ -1140,6 +1140,7 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig&
     INSERT_INTO_JSON_OBJECT(dest, bulletproofs, sig.p.bulletproofs);
     INSERT_INTO_JSON_OBJECT(dest, bulletproofs_plus, sig.p.bulletproofs_plus);
     INSERT_INTO_JSON_OBJECT(dest, mlsags, sig.p.MGs);
+    INSERT_INTO_JSON_OBJECT(dest, clsags, sig.p.CLSAGs);
     INSERT_INTO_JSON_OBJECT(dest, pseudo_outs, sig.get_pseudo_outs());
 
     dest.EndObject();
@@ -1175,6 +1176,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
     GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs, bulletproofs);
     GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs_plus, bulletproofs_plus);
     GET_FROM_JSON_OBJECT(prunable->value, sig.p.MGs, mlsags);
+    GET_FROM_JSON_OBJECT(prunable->value, sig.p.CLSAGs, clsags);
     GET_FROM_JSON_OBJECT(prunable->value, pseudo_outs, pseudo_outs);
 
     sig.get_pseudo_outs() = std::move(pseudo_outs);
@@ -1185,6 +1187,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
     sig.p.bulletproofs.clear();
     sig.p.bulletproofs_plus.clear();
     sig.p.MGs.clear();
+    sig.p.CLSAGs.clear();
     sig.get_pseudo_outs().clear();
   }
 }
@@ -1393,6 +1396,29 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig)
   GET_FROM_JSON_OBJECT(val, sig.cc, cc);
 }
 
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::clsag& sig)
+{
+  dest.StartObject();
+
+  INSERT_INTO_JSON_OBJECT(dest, s, sig.s);
+  INSERT_INTO_JSON_OBJECT(dest, c1, sig.c1);
+  INSERT_INTO_JSON_OBJECT(dest, D, sig.D);
+
+  dest.EndObject();
+}
+
+void fromJsonValue(const rapidjson::Value& val, rct::clsag& sig)
+{
+  if (!val.IsObject())
+  {
+    throw WRONG_TYPE("key64 (rct::key[64])");
+  }
+
+  GET_FROM_JSON_OBJECT(val, sig.s, s);
+  GET_FROM_JSON_OBJECT(val, sig.c1, c1);
+  GET_FROM_JSON_OBJECT(val, sig.D, D);
+}
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info)
 {
   dest.StartObject();

src/serialization/json_object.h

@@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig);
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig);
 void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig);
 
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::clsag& sig);
+void fromJsonValue(const rapidjson::Value& val, rct::clsag& sig);
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info);
 

src/simplewallet/simplewallet.cpp

@@ -3215,7 +3215,6 @@ bool simple_wallet::scan_tx(const std::vector<std::string> &args)
     }
     txids.insert(txid);
   }
-  std::vector<crypto::hash> txids_v(txids.begin(), txids.end());
 
   if (!m_wallet->is_trusted_daemon()) {
     message_writer(console_color_red, true) << tr("WARNING: this operation may reveal the txids to the remote node and affect your privacy");
@@ -3228,7 +3227,9 @@ bool simple_wallet::scan_tx(const std::vector<std::string> &args)
   LOCK_IDLE_SCOPE();
   m_in_manual_refresh.store(true);
   try {
-    m_wallet->scan_tx(txids_v);
+    m_wallet->scan_tx(txids);
+  } catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e) {
+    fail_msg_writer() << e.what() << ". Either connect to a trusted daemon by passing --trusted-daemon when starting the wallet, or use rescan_bc to rescan the chain.";
   } catch (const std::exception &e) {
     fail_msg_writer() << e.what();
   }
@@ -5894,7 +5895,10 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo
   {
     m_in_manual_refresh.store(true, std::memory_order_relaxed);
     epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);});
-    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money);
+    // For manual refresh don't allow incremental checking of the pool: Because we did not process the txs
+    // for us in the pool during automatic refresh we could miss some of them if we checked the pool
+    // incrementally here
+    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money, true, false);
 
     if (reset == ResetSoftKeepKI)
     {

src/version.cpp.in

@@ -1,5 +1,5 @@
 #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@"
-#define DEF_MONERO_VERSION "0.18.2.1"
+#define DEF_MONERO_VERSION "0.18.3.1"
 #define DEF_MONERO_RELEASE_NAME "Fluorine Fermi"
 #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG
 #define DEF_MONERO_VERSION_IS_RELEASE @VERSION_IS_RELEASE@

src/wallet/api/wallet.cpp

@@ -1302,11 +1302,15 @@ bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
         }
         txids_u.insert(txid);
     }
-    std::vector<crypto::hash> txids_v(txids_u.begin(), txids_u.end());
 
     try
     {
-        m_wallet->scan_tx(txids_v);
+        m_wallet->scan_tx(txids_u);
+    }
+    catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e)
+    {
+        setStatusError(e.what());
+        return false;
     }
     catch (const std::exception &e)
     {

src/wallet/api/wallet2_api.h

@@ -38,6 +38,7 @@
 #include <ctime>
 #include <iostream>
 #include <stdexcept>
+#include <cstdint>
 
 //  Public interface for libwallet library
 namespace Monero {

src/wallet/node_rpc_proxy.cpp

@@ -392,4 +392,56 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_payment_info(bool mining, boo
   return boost::none;
 }
 
+boost::optional<std::string> NodeRPCProxy::get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f)
+{
+  const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp
+  for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE)
+  {
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req_t = AUTO_VAL_INIT(req_t);
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response resp_t = AUTO_VAL_INIT(resp_t);
+
+    const size_t n_txids = std::min<size_t>(SLICE_SIZE, txids.size() - offset);
+    for (size_t n = offset; n < (offset + n_txids); ++n)
+      req_t.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[n]));
+    MDEBUG("asking for " << req_t.txs_hashes.size() << " transactions");
+    req_t.decode_as_json = false;
+    req_t.prune = true;
+
+    bool r = false;
+    {
+      const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
+      uint64_t pre_call_credits = m_rpc_payment_state.credits;
+      req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key);
+      r = net_utils::invoke_http_json("/gettransactions", req_t, resp_t, m_http_client, rpc_timeout);
+      if (r && resp_t.status == CORE_RPC_STATUS_OK)
+        check_rpc_cost(m_rpc_payment_state, "/gettransactions", resp_t.credits, pre_call_credits, resp_t.txs.size() * COST_PER_TX);
+    }
+
+    f(req_t, resp_t, r);
+  }
+  return boost::optional<std::string>();
+}
+
+boost::optional<std::string> NodeRPCProxy::get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header)
+{
+  if (m_offline)
+    return boost::optional<std::string>("offline");
+
+  cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request req_t = AUTO_VAL_INIT(req_t);
+  cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response resp_t = AUTO_VAL_INIT(resp_t);
+  req_t.height = height;
+
+  {
+    const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
+    uint64_t pre_call_credits = m_rpc_payment_state.credits;
+    req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key);
+    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req_t, resp_t, m_http_client, rpc_timeout);
+    RETURN_ON_RPC_RESPONSE_ERROR(r, epee::json_rpc::error{}, resp_t, "getblockheaderbyheight");
+    check_rpc_cost(m_rpc_payment_state, "getblockheaderbyheight", resp_t.credits, pre_call_credits, COST_PER_BLOCK_HEADER);
+  }
+
+  block_header = std::move(resp_t.block_header);
+  return boost::optional<std::string>();
+}
+
 }

src/wallet/node_rpc_proxy.h

@@ -59,6 +59,8 @@ public:
   boost::optional<std::string> get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_blocks, std::vector<uint64_t> &fees);
   boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask);
   boost::optional<std::string> get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie);
+  boost::optional<std::string> get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f);
+  boost::optional<std::string> get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header);
 
 private:
   template<typename T> void handle_payment_changes(const T &res, std::true_type) {

src/wallet/wallet2.h

@@ -476,7 +476,7 @@ private:
       time_t m_sent_time;
       std::vector<cryptonote::tx_destination_entry> m_dests;
       crypto::hash m_payment_id;
-      enum { pending, pending_not_in_pool, failed } m_state;
+      enum { pending, pending_in_pool, failed } m_state;
       uint64_t m_timestamp;
       uint32_t m_subaddr_account;   // subaddress account of your wallet to be used in this transfer
       std::set<uint32_t> m_subaddr_indices;  // set of address indices used as inputs in this transfer
@@ -817,6 +817,30 @@ private:
       bool empty() const { return tx_extra_fields.empty() && primary.empty() && additional.empty(); }
     };
 
+    struct detached_blockchain_data
+    {
+      hashchain detached_blockchain;
+      size_t original_chain_size;
+      std::unordered_set<crypto::hash> detached_tx_hashes;
+      std::unordered_map<crypto::hash, std::vector<cryptonote::tx_destination_entry>> detached_confirmed_txs_dests;
+    };
+
+    struct process_tx_entry_t
+    {
+      cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry tx_entry;
+      cryptonote::transaction tx;
+      crypto::hash tx_hash;
+    };
+
+    struct tx_entry_data
+    {
+      std::vector<process_tx_entry_t> tx_entries;
+      uint64_t lowest_height;
+      uint64_t highest_height;
+
+      tx_entry_data(): lowest_height((uint64_t)-1), highest_height(0) {}
+    };
+
     /*!
      * \brief  Generates a wallet or restores one. Assumes the multisig setup
       *        has already completed for the provided multisig info.
@@ -916,22 +940,32 @@ private:
     /*!
      * \brief store_to  Stores wallet to another file(s), deleting old ones
      * \param path      Path to the wallet file (keys and address filenames will be generated based on this filename)
-     * \param password  Password to protect new wallet (TODO: probably better save the password in the wallet object?)
+     * \param password  Password that currently locks the wallet
+     * \param force_rewrite_keys if true, always rewrite keys file
+     *
+     * Leave both "path" and "password" blank to restore the cache file to the current position in the disk
+     * (which is the same as calling `store()`). If you want to store the wallet with a new password,
+     * use the method `change_password()`.
+     *
+     * Normally the keys file is not overwritten when storing, except when force_rewrite_keys is true
+     * or when `path` is a new wallet file.
+     *
+     * \throw error::invalid_password If storing keys file and old password is incorrect
      */
-    void store_to(const std::string &path, const epee::wipeable_string &password);
+    void store_to(const std::string &path, const epee::wipeable_string &password, bool force_rewrite_keys = false);
     /*!
      * \brief get_keys_file_data  Get wallet keys data which can be stored to a wallet file.
-     * \param password            Password of the encrypted wallet buffer (TODO: probably better save the password in the wallet object?)
+     * \param password            Password that currently locks the wallet
      * \param watch_only          true to include only view key, false to include both spend and view keys
      * \return                    Encrypted wallet keys data which can be stored to a wallet file
+     * \throw                     error::invalid_password if password does not match current wallet
      */
     boost::optional<wallet2::keys_file_data> get_keys_file_data(const epee::wipeable_string& password, bool watch_only);
     /*!
      * \brief get_cache_file_data   Get wallet cache data which can be stored to a wallet file.
-     * \param password              Password to protect the wallet cache data (TODO: probably better save the password in the wallet object?)
-     * \return                      Encrypted wallet cache data which can be stored to a wallet file
+     * \return                      Encrypted wallet cache data which can be stored to a wallet file (using current password)
      */
-    boost::optional<wallet2::cache_file_data> get_cache_file_data(const epee::wipeable_string& password);
+    boost::optional<wallet2::cache_file_data> get_cache_file_data();
 
     std::string path() const;
 
@@ -1024,7 +1058,7 @@ private:
     bool is_deprecated() const;
     void refresh(bool trusted_daemon);
     void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched);
-    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true);
+    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true, bool try_incremental = true, uint64_t max_blocks = std::numeric_limits<uint64_t>::max());
     bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok);
 
     void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; }
@@ -1035,7 +1069,7 @@ private:
     bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const;
     bool has_multisig_partial_key_images() const;
     bool has_unknown_key_images() const;
-    bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const;
+    bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string()) const;
     bool key_on_device() const { return get_device_type() != hw::device::device_type::SOFTWARE; }
     hw::device::device_type get_device_type() const { return m_key_device_type; }
     bool reconnect_device();
@@ -1381,7 +1415,7 @@ private:
     std::string get_spend_proof(const crypto::hash &txid, const std::string &message);
     bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str);
 
-    void scan_tx(const std::vector<crypto::hash> &txids);
+    void scan_tx(const std::unordered_set<crypto::hash> &txids);
 
     /*!
      * \brief  Generates a proof that proves the reserve of unspent funds
@@ -1507,9 +1541,9 @@ private:
     bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false);
     crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const;
 
-    void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false);
+    void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false, bool try_incremental = false);
     void process_pool_state(const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &txs);
-    void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes);
+    void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes, bool remove_if_found);
 
     std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const;
     std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const;
@@ -1645,6 +1679,7 @@ private:
     void thaw(const crypto::key_image &ki);
     bool frozen(const crypto::key_image &ki) const;
     bool frozen(const transfer_details &td) const;
+    bool frozen(const multisig_tx_set& txs) const; // does partially signed txset contain frozen enotes?
 
     bool save_to_file(const std::string& path_to_file, const std::string& binary, bool is_printable = false) const;
     static bool load_from_file(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000);
@@ -1700,18 +1735,24 @@ private:
      */
     bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password);
     bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt);
-    void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL, bool ignore_callbacks = false);
     bool should_skip_block(const cryptonote::block &b, uint64_t height) const;
     void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
-    void detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    detached_blockchain_data detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    void handle_reorg(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
     void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const;
     bool clear();
     void clear_soft(bool keep_key_images=false);
-    void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height);
+    void pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height);
     void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes);
     void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false);
-    void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception);
+    void pull_and_parse_next_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception);
     void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    bool accept_pool_tx_for_processing(const crypto::hash &txid);
+    void process_unconfirmed_transfer(bool incremental, const crypto::hash &txid, wallet2::unconfirmed_transfer_details &tx_details, bool seen_in_pool, std::chrono::system_clock::time_point now, bool refreshed);
+    void process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed);
+    void update_pool_state_by_pool_query(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false);
+    void update_pool_state_from_pool_data(bool incremental, const std::vector<crypto::hash> &removed_pool_txids, const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &added_pool_txs, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed);
     uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const;
     bool prepare_file_names(const std::string& file_path);
     void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height);
@@ -1754,6 +1795,9 @@ private:
     crypto::chacha_key get_ringdb_key();
     void setup_keys(const epee::wipeable_string &password);
     size_t get_transfer_details(const crypto::key_image &ki) const;
+    tx_entry_data get_tx_entries(const std::unordered_set<crypto::hash> &txids);
+    void sort_scan_tx_entries(std::vector<process_tx_entry_t> &unsorted_tx_entries);
+    void process_scan_txs(const tx_entry_data &txs_to_scan, const tx_entry_data &txs_to_reprocess, const std::unordered_set<crypto::hash> &tx_hashes_to_reprocess, detached_blockchain_data &dbd);
 
     void register_devices();
     hw::device& lookup_device(const std::string & device_descriptor);
@@ -1846,6 +1890,11 @@ private:
     // If m_refresh_from_block_height is explicitly set to zero we need this to differentiate it from the case that
     // m_refresh_from_block_height was defaulted to zero.*/
     bool m_explicit_refresh_from_block_height;
+    uint64_t m_pool_info_query_time;
+    std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> m_process_pool_txs;
+    uint64_t m_skip_to_height;
+    // m_skip_to_height is useful when we don't want to modify the wallet's restore height.
+    // m_refresh_from_block_height is also a wallet's restore height which should remain constant unless explicitly modified by the user.
     bool m_confirm_non_default_ring_size;
     AskPasswordType m_ask_password;
     uint64_t m_max_reorg_depth;

src/wallet/wallet_errors.h

@@ -93,6 +93,8 @@ namespace tools
     //         get_output_distribution
     //         payment_required
     //       wallet_files_doesnt_correspond
+    //       scan_tx_error *
+    //         wont_reprocess_recent_txs_via_untrusted_daemon
     //
     // * - class with protected ctor
 
@@ -915,6 +917,23 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct scan_tx_error : public wallet_logic_error
+    {
+    protected:
+      explicit scan_tx_error(std::string&& loc, const std::string& message)
+        : wallet_logic_error(std::move(loc), message)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
+    struct wont_reprocess_recent_txs_via_untrusted_daemon : public scan_tx_error
+    {
+      explicit wont_reprocess_recent_txs_via_untrusted_daemon(std::string&& loc)
+        : scan_tx_error(std::move(loc), "The wallet has already seen 1 or more recent transactions than the scanned tx")
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
 
 #if !defined(_MSC_VER)
 

src/wallet/wallet_rpc_server.cpp

@@ -60,6 +60,7 @@ using namespace epee;
 #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc"
 
 #define DEFAULT_AUTO_REFRESH_PERIOD 20 // seconds
+#define REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE 256    // just to split refresh in separate calls to play nicer with other threads
 
 #define CHECK_MULTISIG_ENABLED() \
   do \
@@ -79,6 +80,7 @@ namespace
   const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false};
   const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"};
   const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false};
+  const command_line::arg_descriptor<bool> arg_no_initial_sync = {"no-initial-sync", "Skips the initial sync before listening for connections", false};
 
   constexpr const char default_rpc_username[] = "monero";
 
@@ -149,12 +151,17 @@ namespace tools
         return true;
       if (boost::posix_time::microsec_clock::universal_time() < m_last_auto_refresh_time + boost::posix_time::seconds(m_auto_refresh_period))
         return true;
+      uint64_t blocks_fetched = 0;
       try {
-        if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon());
+        bool received_money = false;
+        if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, blocks_fetched, received_money, true, true, REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE);
       } catch (const std::exception& ex) {
         LOG_ERROR("Exception at while refreshing, what=" << ex.what());
       }
-      m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time();
+      // if we got the max amount of blocks, do not set the last refresh time, we did only part of the refresh and will
+      // continue asap, and only set the last refresh time once the refresh is actually finished
+      if (blocks_fetched < REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE)
+        m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time();
       return true;
     }, 1000);
     m_net_server.add_idle_handler([this](){
@@ -3167,7 +3174,7 @@ namespace tools
           return false;
       }
 
-      std::vector<crypto::hash> txids;
+      std::unordered_set<crypto::hash> txids;
       std::list<std::string>::const_iterator i = req.txids.begin();
       while (i != req.txids.end())
       {
@@ -3180,11 +3187,15 @@ namespace tools
           }
 
           crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data());
-          txids.push_back(txid);
+          txids.insert(txid);
       }
 
       try {
           m_wallet->scan_tx(txids);
+      }  catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e) {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = e.what() + std::string(". Either connect to a trusted daemon or rescan the chain.");
+          return false;
       } catch (const std::exception &e) {
           handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
           return false;
@@ -3802,7 +3813,7 @@ namespace tools
     std::string old_language;
 
     // check the given seed
-    {
+    if (!req.enable_multisig_experimental) {
       if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, old_language))
       {
         er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
@@ -3825,6 +3836,13 @@ namespace tools
 
     // process seed_offset if given
     {
+      if (req.enable_multisig_experimental && !req.seed_offset.empty())
+      {
+        er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+        er.message = "Multisig seeds are not compatible with seed offsets";
+        return false;
+      }
+
       if (!req.seed_offset.empty())
       {
         recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset);
@@ -3888,7 +3906,27 @@ namespace tools
     crypto::secret_key recovery_val;
     try
     {
-      recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false);
+      if (req.enable_multisig_experimental)
+      {
+        // Parse multisig seed into raw multisig data
+        epee::wipeable_string multisig_data;
+        multisig_data.resize(req.seed.size() / 2);
+        if (!epee::from_hex::to_buffer(epee::to_mut_byte_span(multisig_data), req.seed))
+        {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = "Multisig seed not represented as hexadecimal string";
+          return false;
+        }
+
+        // Generate multisig wallet
+        wal->generate(wallet_file, std::move(rc.second).password(), multisig_data, false);
+        wal->enable_multisig(true);
+      }
+      else
+      {
+        // Generate normal wallet
+        recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false);
+      }
       MINFO("Wallet has been restored.\n");
     }
     catch (const std::exception &e)
@@ -3899,7 +3937,7 @@ namespace tools
 
     // // Convert the secret key back to seed
     epee::wipeable_string electrum_words;
-    if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language))
+    if (!req.enable_multisig_experimental && !crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language))
     {
       er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
       er.message = "Failed to encode seed";
@@ -4521,6 +4559,7 @@ public:
       const auto password_file = command_line::get_arg(vm, arg_password_file);
       const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password);
       const auto password_prompt = prompt_for_password ? password_prompter : nullptr;
+      const auto no_initial_sync = command_line::get_arg(vm, arg_no_initial_sync);
 
       if(!wallet_file.empty() && !from_json.empty())
       {
@@ -4589,7 +4628,8 @@ public:
 
       try
       {
-        wal->refresh(wal->is_trusted_daemon());
+        if (!no_initial_sync)
+          wal->refresh(wal->is_trusted_daemon());
       }
       catch (const std::exception& e)
       {
@@ -4700,6 +4740,7 @@ int main(int argc, char** argv) {
   command_line::add_arg(desc_params, arg_wallet_dir);
   command_line::add_arg(desc_params, arg_prompt_for_password);
   command_line::add_arg(desc_params, arg_rpc_client_secret_key);
+  command_line::add_arg(desc_params, arg_no_initial_sync);
 
   daemonizer::init_options(hidden_options, desc_params);
   desc_params.add(hidden_options);

src/wallet/wallet_rpc_server_commands_defs.h

@@ -530,6 +530,33 @@ namespace wallet_rpc
     END_KV_SERIALIZE_MAP()
   };
 
+  struct single_transfer_response
+  {
+    std::string tx_hash;
+    std::string tx_key;
+    uint64_t amount;
+    uint64_t fee;
+    uint64_t weight;
+    std::string tx_blob;
+    std::string tx_metadata;
+    std::string multisig_txset;
+    std::string unsigned_txset;
+    key_image_list spent_key_images;
+
+    BEGIN_KV_SERIALIZE_MAP()
+      KV_SERIALIZE(tx_hash)
+      KV_SERIALIZE(tx_key)
+      KV_SERIALIZE(amount)
+      KV_SERIALIZE(fee)
+      KV_SERIALIZE(weight)
+      KV_SERIALIZE(tx_blob)
+      KV_SERIALIZE(tx_metadata)
+      KV_SERIALIZE(multisig_txset)
+      KV_SERIALIZE(unsigned_txset)
+      KV_SERIALIZE(spent_key_images)
+    END_KV_SERIALIZE_MAP()
+  };
+
   struct COMMAND_RPC_TRANSFER
   {
     struct request_t
@@ -562,35 +589,37 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct response_t
-    {
-      std::string tx_hash;
-      std::string tx_key;
-      uint64_t amount;
-      uint64_t fee;
-      uint64_t weight;
-      std::string tx_blob;
-      std::string tx_metadata;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      key_image_list spent_key_images;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash)
-        KV_SERIALIZE(tx_key)
-        KV_SERIALIZE(amount)
-        KV_SERIALIZE(fee)
-        KV_SERIALIZE(weight)
-        KV_SERIALIZE(tx_blob)
-        KV_SERIALIZE(tx_metadata)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef single_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
+  struct split_transfer_response
+  {
+    std::list<std::string> tx_hash_list;
+    std::list<std::string> tx_key_list;
+    std::list<uint64_t> amount_list;
+    std::list<uint64_t> fee_list;
+    std::list<uint64_t> weight_list;
+    std::list<std::string> tx_blob_list;
+    std::list<std::string> tx_metadata_list;
+    std::string multisig_txset;
+    std::string unsigned_txset;
+    std::list<key_image_list> spent_key_images_list;
+
+    BEGIN_KV_SERIALIZE_MAP()
+      KV_SERIALIZE(tx_hash_list)
+      KV_SERIALIZE(tx_key_list)
+      KV_SERIALIZE(amount_list)
+      KV_SERIALIZE(fee_list)
+      KV_SERIALIZE(weight_list)
+      KV_SERIALIZE(tx_blob_list)
+      KV_SERIALIZE(tx_metadata_list)
+      KV_SERIALIZE(multisig_txset)
+      KV_SERIALIZE(unsigned_txset)
+      KV_SERIALIZE(spent_key_images_list)
+    END_KV_SERIALIZE_MAP()
+  };
+
   struct COMMAND_RPC_TRANSFER_SPLIT
   {
     struct request_t
@@ -623,41 +652,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct key_list
-    {
-      std::list<std::string> keys;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(keys)
-      END_KV_SERIALIZE_MAP()
-    };
-
-    struct response_t
-    {
-      std::list<std::string> tx_hash_list;
-      std::list<std::string> tx_key_list;
-      std::list<uint64_t> amount_list;
-      std::list<uint64_t> fee_list;
-      std::list<uint64_t> weight_list;
-      std::list<std::string> tx_blob_list;
-      std::list<std::string> tx_metadata_list;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      std::list<key_image_list> spent_key_images_list;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash_list)
-        KV_SERIALIZE(tx_key_list)
-        KV_SERIALIZE(amount_list)
-        KV_SERIALIZE(fee_list)
-        KV_SERIALIZE(weight_list)
-        KV_SERIALIZE(tx_blob_list)
-        KV_SERIALIZE(tx_metadata_list)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images_list)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef split_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -821,41 +816,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct key_list
-    {
-      std::list<std::string> keys;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(keys)
-      END_KV_SERIALIZE_MAP()
-    };
-
-    struct response_t
-    {
-      std::list<std::string> tx_hash_list;
-      std::list<std::string> tx_key_list;
-      std::list<uint64_t> amount_list;
-      std::list<uint64_t> fee_list;
-      std::list<uint64_t> weight_list;
-      std::list<std::string> tx_blob_list;
-      std::list<std::string> tx_metadata_list;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      std::list<key_image_list> spent_key_images_list;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash_list)
-        KV_SERIALIZE(tx_key_list)
-        KV_SERIALIZE(amount_list)
-        KV_SERIALIZE(fee_list)
-        KV_SERIALIZE(weight_list)
-        KV_SERIALIZE(tx_blob_list)
-        KV_SERIALIZE(tx_metadata_list)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images_list)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef split_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -897,41 +858,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct key_list
-    {
-      std::list<std::string> keys;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(keys)
-      END_KV_SERIALIZE_MAP()
-    };
-
-    struct response_t
-    {
-      std::list<std::string> tx_hash_list;
-      std::list<std::string> tx_key_list;
-      std::list<uint64_t> amount_list;
-      std::list<uint64_t> fee_list;
-      std::list<uint64_t> weight_list;
-      std::list<std::string> tx_blob_list;
-      std::list<std::string> tx_metadata_list;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      std::list<key_image_list> spent_key_images_list;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash_list)
-        KV_SERIALIZE(tx_key_list)
-        KV_SERIALIZE(amount_list)
-        KV_SERIALIZE(fee_list)
-        KV_SERIALIZE(weight_list)
-        KV_SERIALIZE(tx_blob_list)
-        KV_SERIALIZE(tx_metadata_list)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images_list)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef split_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -967,32 +894,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct response_t
-    {
-      std::string tx_hash;
-      std::string tx_key;
-      uint64_t amount;
-      uint64_t fee;
-      uint64_t weight;
-      std::string tx_blob;
-      std::string tx_metadata;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      key_image_list spent_key_images;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash)
-        KV_SERIALIZE(tx_key)
-        KV_SERIALIZE(amount)
-        KV_SERIALIZE(fee)
-        KV_SERIALIZE(weight)
-        KV_SERIALIZE(tx_blob)
-        KV_SERIALIZE(tx_metadata)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef single_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -2360,6 +2262,7 @@ namespace wallet_rpc
       std::string password;
       std::string language;
       bool autosave_current;
+      bool enable_multisig_experimental;
 
       BEGIN_KV_SERIALIZE_MAP()
       KV_SERIALIZE_OPT(restore_height, (uint64_t)0)
@@ -2369,6 +2272,7 @@ namespace wallet_rpc
       KV_SERIALIZE(password)
       KV_SERIALIZE(language)
       KV_SERIALIZE_OPT(autosave_current, true)
+      KV_SERIALIZE_OPT(enable_multisig_experimental, false)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;

tests/CMakeLists.txt

@@ -72,14 +72,8 @@ else ()
   include_directories(SYSTEM "${CMAKE_CURRENT_SOURCE_DIR}/gtest/include")
 endif (GTest_FOUND)
 
-file(COPY
-  data/wallet_9svHk1.keys
-  data/wallet_9svHk1
-  data/outputs
-  data/unsigned_monero_tx
-  data/signed_monero_tx
-  data/sha256sum
-  DESTINATION data)
+message(STATUS "Copying test data directory...")
+file(COPY data DESTINATION .) # Copy data directory from source root to build root
 
 if (CMAKE_BUILD_TYPE STREQUAL "fuzz" OR OSSFUZZ)
   add_subdirectory(fuzz)

tests/README.md

@@ -54,7 +54,7 @@ Functional tests are located under the `tests/functional_tests` directory.
 
 Building all the tests requires installing the following dependencies:
 ```bash
-pip install requests psutil monotonic zmq
+pip install requests psutil monotonic zmq deepdiff
 ```
 
 First, run a regtest daemon in the offline mode and with a fixed difficulty:

tests/functional_tests/CMakeLists.txt

@@ -67,7 +67,7 @@ target_link_libraries(make_test_signature
 monero_add_minimal_executable(cpu_power_test cpu_power_test.cpp)
 find_program(PYTHON3_FOUND python3 REQUIRED)
 
-execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; import zmq; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; import zmq; import deepdiff; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
 if (REQUESTS_OUTPUT STREQUAL "OK")
   add_test(
     NAME    functional_tests_rpc
@@ -76,6 +76,6 @@ if (REQUESTS_OUTPUT STREQUAL "OK")
     NAME    check_missing_rpc_methods
     COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}")
 else()
-  message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil', 'monotonic', and 'zmq' python modules")
+  message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil', 'monotonic', 'zmq', and 'deepdiff' python modules")
   set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc check_missing_rpc_methods)
 endif()

tests/functional_tests/mining.py

@@ -36,6 +36,7 @@ import math
 import monotonic
 import util_resources
 import multiprocessing
+import string
 
 """Test daemon mining RPC calls
 
@@ -52,6 +53,11 @@ Control the behavior with these environment variables:
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+def assert_non_null_hash(s):
+    assert len(s) == 64 # correct length
+    assert all((c in string.hexdigits for c in s)) # is parseable as hex
+    assert s != ('0' * 64) # isn't null hash
+
 class MiningTest():
     def run_test(self):
         self.reset()
@@ -250,6 +256,8 @@ class MiningTest():
             block_hash = hashes[i]
             assert len(block_hash) == 64
             res = daemon.submitblock(blocks[i])
+            submitted_block_id = res.block_id
+            assert_non_null_hash(submitted_block_id)
             res = daemon.get_height()
             assert res.height == height + i + 1
             assert res.hash == block_hash
@@ -346,6 +354,8 @@ class MiningTest():
         t0 = time.time()
         for h in range(len(block_hashes)):
             res = daemon.submitblock(blocks[h])
+            submitted_block_id = res.block_id
+            assert_non_null_hash(submitted_block_id)
         t0 = time.time() - t0
         res = daemon.get_info()
         assert height == res.height

tests/functional_tests/multisig.py

@@ -29,6 +29,7 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from __future__ import print_function
+import random
 
 """Test multisig transfers
 """
@@ -36,47 +37,61 @@ from __future__ import print_function
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+TEST_CASES = \
+[
+#  M  N  Primary Address
+  [2, 2, '45J58b7PmKJFSiNPFFrTdtfMcFGnruP7V4CMuRpX7NsH4j3jGHKAjo3YJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ3gr7sG'],
+  [2, 3, '44G2TQNfsiURKkvxp7gbgaJY8WynZvANnhmyMAwv6WeEbAvyAWMfKXRhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5duN94i'],
+  [3, 3, '41mro238grj56GnrWkakAKTkBy2yDcXYsUZ2iXCM9pe5Ueajd2RRc6Fhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5ief4ZP'],
+  [3, 4, '44vZSprQKJQRFe6t1VHgU4ESvq2dv7TjBLVGE7QscKxMdFSiyyPCEV64NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6dakeff'],
+  [2, 4, '47puypSwsV1gvUDratmX4y58fSwikXVehEiBhVLxJA1gRCxHyrRgTDr4NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6aRPj5U'],
+  [1, 2, '4A8RnBQixry4VXkqeWhmg8L7vWJVDJj4FN9PV4E7Mgad5ZZ6LKQdn8dYJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ4S8RSB']
+]
+
+PUB_ADDRS = [case[2] for case in TEST_CASES]
+
 class MultisigTest():
     def run_test(self):
         self.reset()
-        self.mine('45J58b7PmKJFSiNPFFrTdtfMcFGnruP7V4CMuRpX7NsH4j3jGHKAjo3YJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ3gr7sG', 5)
-        self.mine('44G2TQNfsiURKkvxp7gbgaJY8WynZvANnhmyMAwv6WeEbAvyAWMfKXRhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5duN94i', 5)
-        self.mine('41mro238grj56GnrWkakAKTkBy2yDcXYsUZ2iXCM9pe5Ueajd2RRc6Fhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5ief4ZP', 5)
-        self.mine('44vZSprQKJQRFe6t1VHgU4ESvq2dv7TjBLVGE7QscKxMdFSiyyPCEV64NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6dakeff', 5)
-        self.mine('47puypSwsV1gvUDratmX4y58fSwikXVehEiBhVLxJA1gRCxHyrRgTDr4NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6aRPj5U', 5)
+        for pub_addr in PUB_ADDRS:
+            self.mine(pub_addr, 4)
         self.mine('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80)
 
         self.test_states()
 
-        self.create_multisig_wallets(2, 2, '45J58b7PmKJFSiNPFFrTdtfMcFGnruP7V4CMuRpX7NsH4j3jGHKAjo3YJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ3gr7sG')
-        self.import_multisig_info([1, 0], 5)
-        txid = self.transfer([1, 0])
-        self.import_multisig_info([0, 1], 6)
-        self.check_transaction(txid)
+        self.fund_addrs_with_normal_wallet(PUB_ADDRS)
 
-        self.create_multisig_wallets(2, 3, '44G2TQNfsiURKkvxp7gbgaJY8WynZvANnhmyMAwv6WeEbAvyAWMfKXRhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5duN94i')
-        self.import_multisig_info([0, 2], 5)
-        txid = self.transfer([0, 2])
-        self.import_multisig_info([0, 1, 2], 6)
-        self.check_transaction(txid)
+        for M, N, pub_addr in TEST_CASES:
+            assert M <= N
+            shuffled_participants = list(range(N))
+            random.shuffle(shuffled_participants)
+            shuffled_signers = shuffled_participants[:M]
 
-        self.create_multisig_wallets(3, 3, '41mro238grj56GnrWkakAKTkBy2yDcXYsUZ2iXCM9pe5Ueajd2RRc6Fhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5ief4ZP')
-        self.import_multisig_info([2, 0, 1], 5)
-        txid = self.transfer([2, 1, 0])
-        self.import_multisig_info([0, 2, 1], 6)
-        self.check_transaction(txid)
+            expected_outputs = 5 # each wallet owns four mined outputs & one transferred output
 
-        self.create_multisig_wallets(3, 4, '44vZSprQKJQRFe6t1VHgU4ESvq2dv7TjBLVGE7QscKxMdFSiyyPCEV64NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6dakeff')
-        self.import_multisig_info([0, 2, 3], 5)
-        txid = self.transfer([0, 2, 3])
-        self.import_multisig_info([0, 1, 2, 3], 6)
-        self.check_transaction(txid)
+            # Create multisig wallet and test transferring
+            self.create_multisig_wallets(M, N, pub_addr)
+            self.import_multisig_info(shuffled_signers if M != 1 else shuffled_participants, expected_outputs)
+            txid = self.transfer(shuffled_signers)
+            expected_outputs += 1
+            self.import_multisig_info(shuffled_participants, expected_outputs)
+            self.check_transaction(txid)
 
-        self.create_multisig_wallets(2, 4, '47puypSwsV1gvUDratmX4y58fSwikXVehEiBhVLxJA1gRCxHyrRgTDr4NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6aRPj5U')
-        self.import_multisig_info([1, 2], 5)
-        txid = self.transfer([1, 2])
-        self.import_multisig_info([0, 1, 2, 3], 6)
-        self.check_transaction(txid)
+            # If more than 1 signer, try to freeze key image of one signer, make tx using that key
+            # image on another signer, then have first signer sign multisg_txset. Should fail
+            if M != 1:
+                txid = self.try_transfer_frozen(shuffled_signers)
+                expected_outputs += 1
+                self.import_multisig_info(shuffled_participants, expected_outputs)
+                self.check_transaction(txid)
+
+            # Recreate wallet from multisig seed and test transferring
+            self.remake_some_multisig_wallets_by_multsig_seed(M)
+            self.import_multisig_info(shuffled_signers if M != 1 else shuffled_participants, expected_outputs)
+            txid = self.transfer(shuffled_signers)
+            expected_outputs += 1
+            self.import_multisig_info(shuffled_participants, expected_outputs)
+            self.check_transaction(txid)
 
     def reset(self):
         print('Resetting blockchain')
@@ -90,6 +105,11 @@ class MultisigTest():
         daemon = Daemon()
         daemon.generateblocks(address, blocks)
 
+    # This method sets up N_total wallets with a threshold of M_threshold doing the following steps:
+    #   * restore_deterministic_wallet(w/ hardcoded seeds)
+    #   * prepare_multisig(enable_multisig_experimental = True)
+    #   * make_multisig()
+    #   * exchange_multisig_keys()
     def create_multisig_wallets(self, M_threshold, N_total, expected_address):
       print('Creating ' + str(M_threshold) + '/' + str(N_total) + ' multisig wallet')
       seeds = [
@@ -100,6 +120,8 @@ class MultisigTest():
       ]
       assert M_threshold <= N_total
       assert N_total <= len(seeds)
+
+      # restore_deterministic_wallet() & prepare_multisig()
       self.wallet = [None] * N_total
       info = []
       for i in range(N_total):
@@ -111,10 +133,12 @@ class MultisigTest():
         assert len(res.multisig_info) > 0
         info.append(res.multisig_info)
 
+      # Assert that all wallets are multisig
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
         assert res.multisig == False
 
+      # make_multisig() with each other's info
       addresses = []
       next_stage = []
       for i in range(N_total):
@@ -122,6 +146,7 @@ class MultisigTest():
         addresses.append(res.address)
         next_stage.append(res.multisig_info)
 
+      # Assert multisig paramaters M/N for each wallet
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
         assert res.multisig == True
@@ -129,13 +154,15 @@ class MultisigTest():
         assert res.threshold == M_threshold
         assert res.total == N_total
 
-      while True:
+      # exchange_multisig_keys()
+      num_exchange_multisig_keys_stages = 0
+      while True: # while not all wallets are ready
         n_ready = 0
         for i in range(N_total):
           res = self.wallet[i].is_multisig()
           if res.ready == True:
             n_ready += 1
-        assert n_ready == 0 or n_ready == N_total
+        assert n_ready == 0 or n_ready == N_total # No partial readiness
         if n_ready == N_total:
           break
         info = next_stage
@@ -145,10 +172,17 @@ class MultisigTest():
           res = self.wallet[i].exchange_multisig_keys(info)
           next_stage.append(res.multisig_info)
           addresses.append(res.address)
+        num_exchange_multisig_keys_stages += 1
+
+      # We should only need N - M + 1 key exchange rounds
+      assert num_exchange_multisig_keys_stages == N_total - M_threshold + 1
+
+      # Assert that the all wallets have expected public address
       for i in range(N_total):
-        assert addresses[i] == expected_address
+        assert addresses[i] == expected_address, addresses[i]
       self.wallet_address = expected_address
 
+      # Assert multisig paramaters M/N and "ready" for each wallet
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
         assert res.multisig == True
@@ -156,6 +190,73 @@ class MultisigTest():
         assert res.threshold == M_threshold
         assert res.total == N_total
 
+    # We want to test if multisig wallets can receive normal transfers as well and mining transfers
+    def fund_addrs_with_normal_wallet(self, addrs):
+      print("Funding multisig wallets with normal wallet-to-wallet transfers")
+
+      # Generate normal deterministic wallet
+      normal_seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
+      assert not hasattr(self, 'wallet') or not self.wallet
+      self.wallet = [Wallet(idx = 0)]
+      res = self.wallet[0].restore_deterministic_wallet(seed = normal_seed)
+      assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+
+      self.wallet[0].refresh()
+
+      # Check that we own enough spendable enotes
+      res = self.wallet[0].incoming_transfers(transfer_type = 'available')
+      assert 'transfers' in res
+      num_outs_spendable = 0
+      min_out_amount = None
+      for t in res.transfers:
+          if not t.spent:
+            num_outs_spendable += 1
+            min_out_amount = min(min_out_amount, t.amount) if min_out_amount is not None else t.amount
+      assert num_outs_spendable >= 2 * len(addrs)
+
+      # Transfer to addrs and mine to confirm tx
+      dsts = [{'address': addr, 'amount': int(min_out_amount * 0.95)} for addr in addrs]
+      res = self.wallet[0].transfer(dsts, get_tx_metadata = True)
+      tx_hex = res.tx_metadata
+      res = self.wallet[0].relay_tx(tx_hex)
+      self.mine('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 10)
+
+    def remake_some_multisig_wallets_by_multsig_seed(self, threshold):
+      N = len(self.wallet)
+      num_signers_to_remake = random.randint(1, N) # Do at least one
+      signers_to_remake = list(range(N))
+      random.shuffle(signers_to_remake)
+      signers_to_remake = signers_to_remake[:num_signers_to_remake]
+
+      for i in signers_to_remake:
+        print("Remaking {}/{} multsig wallet from multisig seed: #{}".format(threshold, N, i+1))
+
+        otherwise_unused_seed = \
+          'factual wiggle awakened maul sash biscuit pause reinvest fonts sleepless knowledge tossed jewels request gusts dagger gumball onward dotted amended powder cynical strained topic request'
+
+        # Get information about wallet, will compare against later
+        old_viewkey = self.wallet[i].query_key('view_key').key
+        old_spendkey = self.wallet[i].query_key('spend_key').key
+        old_multisig_seed = self.wallet[i].query_key('mnemonic').key
+
+        # Close old wallet and restore w/ random seed so we know that restoring actually did something
+        self.wallet[i].close_wallet()
+        self.wallet[i].restore_deterministic_wallet(seed=otherwise_unused_seed)
+        mid_viewkey = self.wallet[i].query_key('view_key').key
+        assert mid_viewkey != old_viewkey
+
+        # Now restore w/ old multisig seed and check against original
+        self.wallet[i].close_wallet()
+        self.wallet[i].restore_deterministic_wallet(seed=old_multisig_seed, enable_multisig_experimental=True)
+        new_viewkey = self.wallet[i].query_key('view_key').key
+        new_spendkey = self.wallet[i].query_key('spend_key').key
+        new_multisig_seed = self.wallet[i].query_key('mnemonic').key
+        assert new_viewkey == old_viewkey
+        assert new_spendkey == old_spendkey
+        assert new_multisig_seed == old_multisig_seed
+
+        self.wallet[i].refresh()
+
     def test_states(self):
         print('Testing multisig states')
         seeds = [
@@ -248,7 +349,7 @@ class MultisigTest():
           assert res.n_outputs == expected_outputs
 
     def transfer(self, signers):
-        assert len(signers) >= 2
+        assert len(signers) >= 1
 
         daemon = Daemon()
 
@@ -316,6 +417,104 @@ class MultisigTest():
         daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
         return txid
 
+    def try_transfer_frozen(self, signers):
+        assert len(signers) >= 2
+
+        daemon = Daemon()
+
+        print("Creating multisig transaction from wallet " + str(signers[0]))
+
+        dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000}
+        res = self.wallet[signers[0]].transfer([dst])
+        assert len(res.tx_hash) == 0 # not known yet
+        txid = res.tx_hash
+        assert len(res.tx_key) == 32*2
+        assert res.amount > 0
+        amount = res.amount
+        assert res.fee > 0
+        fee = res.fee
+        assert len(res.tx_blob) == 0
+        assert len(res.tx_metadata) == 0
+        assert len(res.multisig_txset) > 0
+        assert len(res.unsigned_txset) == 0
+        spent_key_images = res.spent_key_images.key_images
+        multisig_txset = res.multisig_txset
+
+        daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
+        for i in range(len(self.wallet)):
+          self.wallet[i].refresh()
+
+        for i in range(len(signers[1:])):
+          # Check that each signer wallet has key image and it is not frozen
+          for ki in spent_key_images:
+            frozen = self.wallet[signers[i+1]].frozen(ki).frozen
+            assert not frozen
+
+        # Freeze key image involved with initiated transfer
+        assert len(spent_key_images)
+        ki0 = spent_key_images[0]
+        print("Freezing involved key image:", ki0)
+        self.wallet[signers[1]].freeze(ki0)
+        frozen = self.wallet[signers[1]].frozen(ki).frozen
+        assert frozen
+
+        # Try signing multisig (this operation should fail b/c of the frozen key image)
+        print("Attemping to sign with frozen key image. This should fail")
+        try:
+          res = self.wallet[signers[1]].sign_multisig(multisig_txset)
+          raise ValueError('sign_multisig should not have succeeded w/ frozen enotes')
+        except AssertionError:
+          pass
+
+        # Thaw key image and continue transfer as normal
+        print("Thawing key image and continuing transfer as normal")
+        self.wallet[signers[1]].thaw(ki0)
+        frozen = self.wallet[signers[1]].frozen(ki).frozen
+        assert not frozen
+
+        for i in range(len(signers[1:])):
+          print('Signing multisig transaction with wallet ' + str(signers[i+1]))
+          res = self.wallet[signers[i+1]].describe_transfer(multisig_txset = multisig_txset)
+          assert len(res.desc) == 1
+          desc = res.desc[0]
+          assert desc.amount_in >= amount + fee
+          assert desc.amount_out == desc.amount_in - fee
+          assert desc.ring_size == 16
+          assert desc.unlock_time == 0
+          assert not 'payment_id' in desc or desc.payment_id in ['', '0000000000000000']
+          assert desc.change_amount == desc.amount_in - 1000000000000 - fee
+          assert desc.change_address == self.wallet_address
+          assert desc.fee == fee
+          assert len(desc.recipients) == 1
+          rec = desc.recipients[0]
+          assert rec.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+          assert rec.amount == 1000000000000
+
+          res = self.wallet[signers[i+1]].sign_multisig(multisig_txset)
+          multisig_txset = res.tx_data_hex
+          assert len(res.tx_hash_list if 'tx_hash_list' in res else []) == (i == len(signers[1:]) - 1)
+
+          if i < len(signers[1:]) - 1:
+            print('Submitting multisig transaction prematurely with wallet ' + str(signers[-1]))
+            ok = False
+            try: self.wallet[signers[-1]].submit_multisig(multisig_txset)
+            except: ok = True
+            assert ok
+
+        print('Submitting multisig transaction with wallet ' + str(signers[-1]))
+        res = self.wallet[signers[-1]].submit_multisig(multisig_txset)
+        assert len(res.tx_hash_list) == 1
+        txid = res.tx_hash_list[0]
+
+        for i in range(len(self.wallet)):
+          self.wallet[i].refresh()
+          res = self.wallet[i].get_transfers()
+          assert len([x for x in (res['pending'] if 'pending' in res else []) if x.txid == txid]) == (1 if i == signers[-1] else 0)
+          assert len([x for x in (res['out'] if 'out' in res else []) if x.txid == txid]) == 0
+
+        daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
+        return txid
+
     def check_transaction(self, txid):
         for i in range(len(self.wallet)):
           self.wallet[i].refresh()

tests/functional_tests/transfer.py

@@ -30,6 +30,9 @@
 
 from __future__ import print_function
 import json
+import pprint
+from deepdiff import DeepDiff
+pp = pprint.PrettyPrinter(indent=2)
 
 """Test simple transfers
 """
@@ -37,6 +40,12 @@ import json
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+seeds = [
+    'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted',
+    'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout',
+    'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
+]
+
 class TransferTest():
     def run_test(self):
         self.reset()
@@ -52,6 +61,7 @@ class TransferTest():
         self.check_tx_notes()
         self.check_rescan()
         self.check_is_key_image_spent()
+        self.check_scan_tx()
 
     def reset(self):
         print('Resetting blockchain')
@@ -62,11 +72,6 @@ class TransferTest():
 
     def create(self):
         print('Creating wallets')
-        seeds = [
-          'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted',
-          'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout',
-          'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
-        ]
         self.wallet = [None] * len(seeds)
         for i in range(len(seeds)):
             self.wallet[i] = Wallet(idx = i)
@@ -829,6 +834,217 @@ class TransferTest():
         res = daemon.is_key_image_spent(ki)
         assert res.spent_status == expected
 
+    def check_scan_tx(self):
+        daemon = Daemon()
+
+        print('Testing scan_tx')
+
+        def diff_transfers(actual_transfers, expected_transfers):
+            diff = DeepDiff(actual_transfers, expected_transfers)
+            if diff != {}:
+                pp.pprint(diff)
+            assert diff == {}
+
+        # set up sender_wallet
+        sender_wallet = self.wallet[0]
+        try: sender_wallet.close_wallet()
+        except: pass
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0])
+        sender_wallet.auto_refresh(enable = False)
+        sender_wallet.refresh()
+        res = sender_wallet.get_transfers()
+        out_len = 0 if 'out' not in res else len(res.out)
+        sender_starting_balance = sender_wallet.get_balance().balance
+        amount = 1000000000000
+        assert sender_starting_balance > amount
+
+        # set up receiver_wallet
+        receiver_wallet = self.wallet[1]
+        try: receiver_wallet.close_wallet()
+        except: pass
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1])
+        receiver_wallet.auto_refresh(enable = False)
+        receiver_wallet.refresh()
+        res = receiver_wallet.get_transfers()
+        in_len = 0 if 'in' not in res else len(res['in'])
+        receiver_starting_balance = receiver_wallet.get_balance().balance
+
+        # transfer from sender_wallet to receiver_wallet
+        dst = {'address': '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 'amount': amount}
+        res = sender_wallet.transfer([dst])
+        assert len(res.tx_hash) == 32*2
+        txid = res.tx_hash
+        assert res.amount == amount
+        assert res.fee > 0
+        fee = res.fee
+
+        expected_sender_balance = sender_starting_balance - (amount + fee)
+        expected_receiver_balance = receiver_starting_balance + amount
+
+        test = 'Checking scan_tx on outgoing pool tx'
+        for attempt in range(2): # test re-scanning
+            print(test + ' (' + ('first attempt' if attempt == 0 else 're-scanning tx') + ')')
+            sender_wallet.scan_tx([txid])
+            res = sender_wallet.get_transfers()
+            assert 'pool' not in res or len(res.pool) == 0
+            if out_len == 0:
+                assert 'out' not in res
+            else:
+                assert len(res.out) == out_len
+            assert len(res.pending) == 1
+            tx = [x for x in res.pending if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert tx.amount == amount
+            assert tx.fee == fee
+            assert len(tx.destinations) == 1
+            assert tx.destinations[0].amount == amount
+            assert tx.destinations[0].address == dst['address']
+            assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        test = 'Checking scan_tx on incoming pool tx'
+        for attempt in range(2): # test re-scanning
+            print(test + ' (' + ('first attempt' if attempt == 0 else 're-scanning tx') + ')')
+            receiver_wallet.scan_tx([txid])
+            res = receiver_wallet.get_transfers()
+            assert 'pending' not in res or len(res.pending) == 0
+            if in_len == 0:
+                assert 'in' not in res
+            else:
+                assert len(res['in']) == in_len
+            assert 'pool' in res and len(res.pool) == 1
+            tx = [x for x in res.pool if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert tx.amount == amount
+            assert tx.fee == fee
+            assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        # mine the tx
+        height = daemon.generateblocks(dst['address'], 1).height
+        block_header = daemon.getblockheaderbyheight(height = height).block_header
+        miner_txid = block_header.miner_tx_hash
+        expected_receiver_balance += block_header.reward
+
+        print('Checking scan_tx on outgoing tx before refresh')
+        sender_wallet.scan_tx([txid])
+        res = sender_wallet.get_transfers()
+        assert 'pending' not in res or len(res.pending) == 0
+        assert 'pool' not in res or len (res.pool) == 0
+        assert len(res.out) == out_len + 1
+        tx = [x for x in res.out if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount
+        assert tx.fee == fee
+        assert len(tx.destinations) == 1
+        assert tx.destinations[0].amount == amount
+        assert tx.destinations[0].address == dst['address']
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Checking scan_tx on outgoing tx after refresh')
+        sender_wallet.refresh()
+        sender_wallet.scan_tx([txid])
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print("Checking scan_tx on outgoing wallet's earliest tx")
+        earliest_height = height
+        earliest_txid = txid
+        for x in res['in']:
+            if x.height < earliest_height:
+                earliest_height = x.height
+                earliest_txid = x.txid
+        sender_wallet.scan_tx([earliest_txid])
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        test = 'Checking scan_tx on outgoing wallet restored at current height'
+        for i, out_tx in enumerate(res.out):
+            if 'destinations' in out_tx:
+                del res.out[i]['destinations'] # destinations are not expected after wallet restore
+        out_txids = [x.txid for x in res.out]
+        in_txids = [x.txid for x in res['in']]
+        all_txs = out_txids + in_txids
+        for test_type in ["all txs", "incoming first", "duplicates within", "duplicates across"]:
+            print(test + ' (' + test_type + ')')
+            sender_wallet.close_wallet()
+            sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = height)
+            assert sender_wallet.get_transfers() == {}
+            if test_type == "all txs":
+                sender_wallet.scan_tx(all_txs)
+            elif test_type == "incoming first":
+                sender_wallet.scan_tx(in_txids)
+                sender_wallet.scan_tx(out_txids)
+            # TODO: test_type == "outgoing first"
+            elif test_type == "duplicates within":
+                sender_wallet.scan_tx(all_txs + all_txs)
+            elif test_type == "duplicates across":
+                sender_wallet.scan_tx(all_txs)
+                sender_wallet.scan_tx(all_txs)
+            else:
+                assert True == False
+            diff_transfers(sender_wallet.get_transfers(), res)
+            assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Sanity check against outgoing wallet restored at height 0')
+        sender_wallet.close_wallet()
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = 0)
+        sender_wallet.refresh()
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Checking scan_tx on incoming txs before refresh')
+        receiver_wallet.scan_tx([txid, miner_txid])
+        res = receiver_wallet.get_transfers()
+        assert 'pending' not in res or len(res.pending) == 0
+        assert 'pool' not in res or len (res.pool) == 0
+        assert len(res['in']) == in_len + 2
+        tx = [x for x in res['in'] if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount
+        assert tx.fee == fee
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Checking scan_tx on incoming txs after refresh')
+        receiver_wallet.refresh()
+        receiver_wallet.scan_tx([txid, miner_txid])
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print("Checking scan_tx on incoming wallet's earliest tx")
+        earliest_height = height
+        earliest_txid = txid
+        for x in res['in']:
+            if x.height < earliest_height:
+                earliest_height = x.height
+                earliest_txid = x.txid
+        receiver_wallet.scan_tx([earliest_txid])
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Checking scan_tx on incoming wallet restored at current height')
+        txids = [x.txid for x in res['in']]
+        if 'out' in res:
+            txids = txids + [x.txid for x in res.out]
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = height)
+        assert receiver_wallet.get_transfers() == {}
+        receiver_wallet.scan_tx(txids)
+        if 'out' in res:
+            for i, out_tx in enumerate(res.out):
+                if 'destinations' in out_tx:
+                    del res.out[i]['destinations'] # destinations are not expected after wallet restore
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Sanity check against incoming wallet restored at height 0')
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = 0)
+        receiver_wallet.refresh()
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
 
 if __name__ == '__main__':
     TransferTest().run_test()

tests/unit_tests/CMakeLists.txt

@@ -90,12 +90,14 @@ set(unit_tests_sources
   hardfork.cpp
   unbound.cpp
   uri.cpp
+  util.cpp
   varint.cpp
   ver_rct_non_semantics_simple_cached.cpp
   ringct.cpp
   output_selection.cpp
   vercmp.cpp
   ringdb.cpp
+  wallet_storage.cpp
   wipeable_string.cpp
   is_hdd.cpp
   aligned.cpp

tests/unit_tests/long_term_block_weight.cpp

@@ -407,3 +407,38 @@ TEST(long_term_block_weight, long_growth_spike_and_drop)
   ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07);
   ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09);
 }
+
+TEST(long_term_block_weight, cache_matches_true_value)
+{
+  PREFIX(16);
+
+  // Add big blocks to increase the block weight limit 
+  for (uint64_t h = 0; h <= 2000; ++h)
+  {
+    size_t w = bc->get_current_cumulative_block_weight_limit();
+    uint64_t ltw = bc->get_next_long_term_block_weight(w);
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
+    bc->update_next_cumulative_weight_limit();
+  }
+
+  ASSERT_GT(bc->get_current_cumulative_block_weight_limit() * 10/17 , 300000);
+
+  // Add small blocks to the top of the chain
+  for (uint64_t h = 2000; h <= 5001; ++h)
+  {
+    size_t w = (bc->get_current_cumulative_block_weight_median() * 10/17) - 1000;
+    uint64_t ltw = bc->get_next_long_term_block_weight(w);
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
+    bc->update_next_cumulative_weight_limit();
+  }
+ 
+  // get the weight limit
+  uint64_t weight_limit = bc->get_current_cumulative_block_weight_limit();
+  // refresh the cache
+  bc->m_long_term_block_weights_cache_rolling_median.clear();
+  bc->get_long_term_block_weight_median(bc->get_db().height() - TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW, TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW);
+  bc->update_next_cumulative_weight_limit();
+
+  // make sure the weight limit is the same
+  ASSERT_EQ(weight_limit, bc->get_current_cumulative_block_weight_limit());
+}

tests/unit_tests/util.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2023-2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "gtest/gtest.h"
+
+#include "common/util.h"
+
+TEST(LocalAddress, localhost) { ASSERT_TRUE(tools::is_local_address("localhost")); }
+TEST(LocalAddress, localhost_port) { ASSERT_TRUE(tools::is_local_address("localhost:18081")); }
+TEST(LocalAddress, localhost_suffix) { ASSERT_TRUE(tools::is_local_address("test.localhost")); }
+TEST(LocalAddress, loopback) { ASSERT_TRUE(tools::is_local_address("127.0.0.1")); }
+TEST(LocalAddress, loopback_port) { ASSERT_TRUE(tools::is_local_address("127.0.0.1:18081")); }
+TEST(LocalAddress, loopback_protocol) { ASSERT_TRUE(tools::is_local_address("http://127.0.0.1")); }
+TEST(LocalAddress, loopback_hi) { ASSERT_TRUE(tools::is_local_address("127.255.255.255")); }
+TEST(LocalAddress, loopback_lo) { ASSERT_TRUE(tools::is_local_address("127.0.0.0")); }
+TEST(LocalAddress, loopback_ipv6) { ASSERT_TRUE(tools::is_local_address("[0:0:0:0:0:0:0:1]")); }
+
+TEST(LocalAddress, onion) { ASSERT_FALSE(tools::is_local_address("vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion")); }
+TEST(LocalAddress, i2p) { ASSERT_FALSE(tools::is_local_address("xmrto2bturnore26xmrto2bturnore26xmrto2bturnore26xmr2.b32.i2p")); }
+TEST(LocalAddress, valid_ip) { ASSERT_FALSE(tools::is_local_address("1.2.3.4")); }
+TEST(LocalAddress, valid_ipv6) { ASSERT_FALSE(tools::is_local_address("[0:0:0:0:0:0:0:2]")); }
+TEST(LocalAddress, valid_domain) { ASSERT_FALSE(tools::is_local_address("getmonero.org")); }
+TEST(LocalAddress, local_prefix) { ASSERT_FALSE(tools::is_local_address("localhost.com")); }
+TEST(LocalAddress, invalid) { ASSERT_FALSE(tools::is_local_address("test")); }
+TEST(LocalAddress, empty) { ASSERT_FALSE(tools::is_local_address("")); }

tests/unit_tests/wallet_storage.cpp

@@ -0,0 +1,266 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "unit_tests_utils.h"
+#include "gtest/gtest.h"
+
+#include "file_io_utils.h"
+#include "wallet/wallet2.h"
+
+using namespace boost::filesystem;
+using namespace epee::file_io_utils;
+
+static constexpr const char WALLET_00fd416a_PRIMARY_ADDRESS[] =
+    "45p2SngJAPSJbqSiUvYfS3BfhEdxZmv8pDt25oW1LzxrZv9Uq6ARagiFViMGUE3gJk5VPWingCXVf1p2tyAy6SUeSHPhbve";
+
+TEST(wallet_storage, store_to_file2file)
+{
+    const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a";
+    const path interm_wallet_file = unit_test::data_dir / "wallet_00fd416a_copy_file2file";
+    const path target_wallet_file = unit_test::data_dir / "wallet_00fd416a_new_file2file";
+
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string() + ".keys"));
+
+    copy_file(source_wallet_file, interm_wallet_file, copy_option::overwrite_if_exists);
+    copy_file(source_wallet_file.string() + ".keys", interm_wallet_file.string() + ".keys", copy_option::overwrite_if_exists);
+
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string() + ".keys"));
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string password("beepbeep");
+
+    const auto files_are_expected = [&]()
+    {
+        EXPECT_FALSE(is_file_exist(interm_wallet_file.string()));
+        EXPECT_FALSE(is_file_exist(interm_wallet_file.string() + ".keys"));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+    };
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.store_to(target_wallet_file.string(), password);
+        files_are_expected();
+    }
+
+    files_are_expected();
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.store_to("", "");
+        files_are_expected();
+    }
+
+    files_are_expected();
+}
+
+TEST(wallet_storage, store_to_mem2file)
+{
+    const path target_wallet_file = unit_test::data_dir / "wallet_mem2file";
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string password("beepbeep2");
+
+    {
+        tools::wallet2 w;
+        w.generate("", password);
+        w.store_to(target_wallet_file.string(), password);
+
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+    }
+
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), password);
+
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+    }
+
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+}
+
+TEST(wallet_storage, change_password_same_file)
+{
+    const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a";
+    const path interm_wallet_file = unit_test::data_dir / "wallet_00fd416a_copy_change_password_same";
+
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string() + ".keys"));
+
+    copy_file(source_wallet_file, interm_wallet_file, copy_option::overwrite_if_exists);
+    copy_file(source_wallet_file.string() + ".keys", interm_wallet_file.string() + ".keys", copy_option::overwrite_if_exists);
+
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string old_password("beepbeep");
+    epee::wipeable_string new_password("meepmeep");
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), old_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.change_password(w.get_wallet_file(), old_password, new_password);
+    }
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), new_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+    }
+
+    {
+        tools::wallet2 w;
+        EXPECT_THROW(w.load(interm_wallet_file.string(), old_password), tools::error::invalid_password);
+    }
+}
+
+TEST(wallet_storage, change_password_different_file)
+{
+    const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a";
+    const path interm_wallet_file = unit_test::data_dir / "wallet_00fd416a_copy_change_password_diff";
+    const path target_wallet_file = unit_test::data_dir / "wallet_00fd416a_new_change_password_diff";
+
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string() + ".keys"));
+
+    copy_file(source_wallet_file, interm_wallet_file, copy_option::overwrite_if_exists);
+    copy_file(source_wallet_file.string() + ".keys", interm_wallet_file.string() + ".keys", copy_option::overwrite_if_exists);
+
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string() + ".keys"));
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string old_password("beepbeep");
+    epee::wipeable_string new_password("meepmeep");
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), old_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.change_password(target_wallet_file.string(), old_password, new_password);
+    }
+
+    EXPECT_FALSE(is_file_exist(interm_wallet_file.string()));
+    EXPECT_FALSE(is_file_exist(interm_wallet_file.string() + ".keys"));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), new_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+    }
+}
+
+TEST(wallet_storage, change_password_in_memory)
+{
+    const epee::wipeable_string password1("monero");
+    const epee::wipeable_string password2("means money");
+    const epee::wipeable_string password_wrong("is traceable");
+
+    tools::wallet2 w;
+    w.generate("", password1);
+    const std::string primary_address_1 = w.get_address_as_str();
+    w.change_password("", password1, password2);
+    const std::string primary_address_2 = w.get_address_as_str();
+    EXPECT_EQ(primary_address_1, primary_address_2);
+
+    EXPECT_THROW(w.change_password("", password_wrong, password1), tools::error::invalid_password);
+}
+
+TEST(wallet_storage, change_password_mem2file)
+{
+    const path target_wallet_file = unit_test::data_dir / "wallet_change_password_mem2file";
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    const epee::wipeable_string password1("https://safecurves.cr.yp.to/rigid.html");
+    const epee::wipeable_string password2(
+        "https://csrc.nist.gov/csrc/media/projects/crypto-standards-development-process/documents/dualec_in_x982_and_sp800-90.pdf");
+    
+    std::string primary_address_1, primary_address_2;
+    {
+        tools::wallet2 w;
+        w.generate("", password1);
+        primary_address_1 = w.get_address_as_str();
+        w.change_password(target_wallet_file.string(), password1, password2);
+    }
+
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), password2);
+        primary_address_2 = w.get_address_as_str();
+    }
+
+    EXPECT_EQ(primary_address_1, primary_address_2);
+}

utils/python-rpc/framework/wallet.py

@@ -297,7 +297,7 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(query_key)
 
-    def restore_deterministic_wallet(self, seed = '', seed_offset = '', filename = '', restore_height = 0, password = '', language = '', autosave_current = True):
+    def restore_deterministic_wallet(self, seed = '', seed_offset = '', filename = '', restore_height = 0, password = '', language = '', autosave_current = True, enable_multisig_experimental = False):
         restore_deterministic_wallet = {
             'method': 'restore_deterministic_wallet',
             'params' : {
@@ -308,6 +308,7 @@ class Wallet(object):
                 'password': password,
                 'language': language,
                 'autosave_current': autosave_current,
+                'enable_multisig_experimental': enable_multisig_experimental
             },
             'jsonrpc': '2.0', 
             'id': '0'