Adds kcn dev reward to bt

Boost compatibility fix

README.md

@@ -1,6 +1,12 @@
 Node-CryptoForkNote-Util with Merged Mining support
 ===================================================
 
+Installing locally and testing
+-----
+```
+npm install https://github.com/MoneroOcean/node-cryptoforknote-util
+```
+
 Dependencies
 ------------
 

binding.gyp

@@ -5,6 +5,8 @@
             "sources": [
                 "src/main.cc",
                 "src/cryptonote_core/cryptonote_format_utils.cpp",
+                "src/offshore/pricing_record.cpp",
+                "src/zephyr_oracle/pricing_record.cpp",
                 "src/crypto/tree-hash.c",
                 "src/crypto/crypto.cpp",
                 "src/crypto/crypto-ops.c",
@@ -29,7 +31,7 @@
                 "-fno-exceptions -std=gnu11 -march=native -fPIC -DNDEBUG -Ofast -funroll-loops -fvariable-expansion-in-unroller -ftree-loop-if-convert-stores -fmerge-all-constants -fbranch-target-load-optimize2"
             ],
             "cflags_cc": [
-                "-fexceptions -frtti -std=gnu++11 -march=native -fPIC -DNDEBUG -Ofast -s -funroll-loops -fvariable-expansion-in-unroller -ftree-loop-if-convert-stores -fmerge-all-constants -fbranch-target-load-optimize2"
+                "-fexceptions -frtti -std=c++17 -march=native -fPIC -DNDEBUG -Ofast -s -funroll-loops -fvariable-expansion-in-unroller -ftree-loop-if-convert-stores -fmerge-all-constants -fbranch-target-load-optimize2"
             ],
             "xcode_settings": {
                 "OTHER_CFLAGS": [ "-fexceptions -frtti" ]

index.js

@@ -1 +1,228 @@
-module.exports = require('bindings')('cryptoforknote.node')
+module.exports = require('bindings')('cryptoforknote.node');
+
+const SHA3    = require('sha3');
+const bignum  = require('bignum');
+const bitcoin = require('bitcoinjs-lib');
+const varuint = require('varuint-bitcoin');
+const crypto  = require('crypto');
+const fastMerkleRoot = require('merkle-lib/fastRoot');
+
+const rtm = require('cryptoforknote-util/rtm');
+
+function scriptCompile(addrHash) {
+  return bitcoin.script.compile([
+    bitcoin.opcodes.OP_DUP,
+    bitcoin.opcodes.OP_HASH160,
+    addrHash,
+    bitcoin.opcodes.OP_EQUALVERIFY,
+    bitcoin.opcodes.OP_CHECKSIG
+  ]);
+}
+
+function reverseBuffer(buff) {
+  let reversed = Buffer.alloc(buff.length);
+  for (var i = buff.length - 1; i >= 0; i--) reversed[buff.length - i - 1] = buff[i];
+  return reversed;
+}
+
+function txesHaveWitnessCommit(transactions) {
+  return (
+    transactions instanceof Array &&
+    transactions[0] &&
+    transactions[0].ins &&
+    transactions[0].ins instanceof Array &&
+    transactions[0].ins[0] &&
+    transactions[0].ins[0].witness &&
+    transactions[0].ins[0].witness instanceof Array &&
+    transactions[0].ins[0].witness.length > 0
+  );
+}
+
+function sha256(buffer) {
+  return crypto.createHash('sha256').update(buffer).digest();
+};
+
+function hash256(buffer) {
+  return sha256(sha256(buffer));
+};
+
+function getMerkleRoot(transactions) {
+  if (transactions.length === 0) return Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex')
+  const forWitness = txesHaveWitnessCommit(transactions);
+  const hashes = transactions.map(transaction => transaction.getHash(forWitness));
+  const rootHash = fastMerkleRoot(hashes, hash256);
+  return forWitness ? hash256(Buffer.concat([rootHash, transactions[0].ins[0].witness[0]])) : rootHash;
+}
+
+let last_epoch_number;
+let last_seed_hash;
+
+module.exports.baseDiff = function() {
+  return bignum('FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF', 16);
+};
+
+module.exports.baseRavenDiff = function() {
+  return parseInt('0x00000000ff000000000000000000000000000000000000000000000000000000');
+};
+
+module.exports.RavenBlockTemplate = function(rpcData, poolAddress) {
+  const poolAddrHash = bitcoin.address.fromBase58Check(poolAddress).hash;
+
+  let txCoinbase = new bitcoin.Transaction();
+  let bytesHeight;
+  { // input for coinbase tx
+    let blockHeightSerial = rpcData.height.toString(16).length % 2 === 0 ?
+                                  rpcData.height.toString(16) :
+                            '0' + rpcData.height.toString(16);
+    bytesHeight = Math.ceil((rpcData.height << 1).toString(2).length / 8);
+    const lengthDiff  = blockHeightSerial.length/2 - bytesHeight;
+    for (let i = 0; i < lengthDiff; i++) blockHeightSerial = blockHeightSerial + '00';
+    const serializedBlockHeight = Buffer.concat([
+      Buffer.from('0' + bytesHeight, 'hex'),
+      reverseBuffer(Buffer.from(blockHeightSerial, 'hex')),
+      Buffer.from('00', 'hex') // OP_0
+    ]);
+
+    txCoinbase.addInput(
+      // will be used for our reserved_offset extra_nonce
+      Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'),
+      0xFFFFFFFF, 0xFFFFFFFF,
+      Buffer.concat([serializedBlockHeight, Buffer.alloc(17, 0xCC)]) // 17 bytes
+    );
+
+    txCoinbase.addOutput(scriptCompile(poolAddrHash), Math.floor(rpcData.coinbasevalue));
+
+    // For CLORE
+    if (rpcData.CommunityAutonomousAddress && rpcData.CommunityAutonomousValue) {
+      txCoinbase.addOutput(scriptCompile(bitcoin.address.fromBase58Check(rpcData.CommunityAutonomousAddress).hash), Math.floor(rpcData.CommunityAutonomousValue));
+    }
+
+    if (rpcData.default_witness_commitment) {
+      txCoinbase.addOutput(Buffer.from(rpcData.default_witness_commitment, 'hex'), 0);
+    }
+  }
+
+  let header = Buffer.alloc(80);
+  { let position = 0;
+    header.writeUInt32BE(rpcData.height, position, 4);                  // height         42-46
+    header.write(rpcData.bits, position += 4, 4, 'hex');                // bits           47-50
+    header.writeUInt32BE(rpcData.curtime, position += 4, 4, 'hex');     // nTime          51-54
+    header.write('DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD', position += 4, 32, 'hex');                 // merkelRoot     55-87
+    header.write(rpcData.previousblockhash, position += 32, 32, 'hex'); // prevblockhash  88-120
+    header.writeUInt32BE(rpcData.version, position += 32, 4);           // version        121-153
+    header = reverseBuffer(header);
+  }
+  
+  let blob = Buffer.concat([
+    header, // 80 bytes
+    Buffer.from('AAAAAAAAAAAAAAAA', 'hex'), // 8 bytes
+    Buffer.from('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB', 'hex'), // 32 bytes
+    varuint.encode(rpcData.transactions.length + 1, Buffer.alloc(varuint.encodingLength(rpcData.transactions.length + 1)), 0)
+  ]);
+  const offset1 = blob.length; 
+  blob = Buffer.concat([ blob, Buffer.from(txCoinbase.toHex(), 'hex') ]);
+
+  rpcData.transactions.forEach(function (value) {
+    blob = Buffer.concat([ blob, Buffer.from(value.data, 'hex') ]);
+  });
+
+  const EPOCH_LENGTH = 7500;
+  const epoch_number = Math.floor(rpcData.height / EPOCH_LENGTH);
+  if (last_epoch_number !== epoch_number) {
+    let sha3 = new SHA3.SHA3Hash(256);
+    if (last_epoch_number && last_epoch_number + 1 === epoch_number) {
+      last_seed_hash = sha3.update(last_seed_hash).digest();
+    } else {
+      last_seed_hash = Buffer.alloc(32, 0);
+      for (let i = 0; i < epoch_number; i++) {
+        last_seed_hash = sha3.update(last_seed_hash).digest();
+        sha3.reset();
+      }
+    }
+    last_epoch_number = epoch_number;
+  }
+
+  const difficulty = parseFloat((module.exports.baseRavenDiff() / bignum(rpcData.target, 16).toNumber()).toFixed(9));
+
+  return {
+    blocktemplate_blob: blob.toString('hex'),
+    // reserved_offset to CCCCCC....
+    reserved_offset:    offset1 + 4 /* txCoinbase.version */ + 1 /* vinLen */  + 32 /* hash */ + 4 /* index  */ +
+                        1 /* vScript len */ + 1 /* coinbase height len */ + bytesHeight + 1 /* trailing zero byte */,
+    seed_hash:          last_seed_hash.toString('hex'),
+    difficulty:         difficulty,
+    height:             rpcData.height,
+    bits:               rpcData.bits,
+    prev_hash:          rpcData.previousblockhash,
+  };
+};
+
+function update_merkle_root_hash(offset, payload, blob_in, blob_out) {
+  const nTransactions = varuint.decode(blob_in, offset);
+  offset += varuint.decode.bytes;
+  let transactions = [];
+  for (let i = 0; i < nTransactions; ++i) {
+    const tx = bitcoin.Transaction.fromBuffer(blob_in.slice(offset), true, payload && i == 0);
+    transactions.push(tx);
+    offset += tx.byteLength();
+  }
+  getMerkleRoot(transactions).copy(blob_out, 4 + 32);
+};
+
+module.exports.blockHashBuff = function(blobBuffer) {
+  return reverseBuffer(hash256(blobBuffer));
+};
+
+module.exports.convertRavenBlob = function(blobBuffer) {
+  let header = blobBuffer.slice(0, 80);
+  update_merkle_root_hash(80 + 8 + 32, false, blobBuffer, header);
+  return module.exports.blockHashBuff(header);
+};
+
+module.exports.constructNewRavenBlob = function(blockTemplate, nonceBuff, mixhashBuff) {
+  update_merkle_root_hash(80 + 8 + 32, false, blockTemplate, blockTemplate);
+  nonceBuff.copy  (blockTemplate, 80, 0, 8);
+  mixhashBuff.copy(blockTemplate, 88, 0, 32);
+  return blockTemplate;
+};
+
+module.exports.constructNewDeroBlob = function(blockTemplate, nonceBuff) {
+  nonceBuff.copy(blockTemplate, 39, 0, 4);
+  return blockTemplate;
+};
+
+module.exports.EthBlockTemplate = function(rpcData) {
+  const difficulty = module.exports.baseDiff().div(bignum(rpcData[2].substr(2), 16)).toNumber();
+  return {
+    hash:               rpcData[0].substr(2),
+    seed_hash:          rpcData[1].substr(2),
+    difficulty:         difficulty,
+    height:             parseInt(rpcData[3])
+  };
+};
+
+module.exports.ErgBlockTemplate = function(rpcData) {
+  const difficulty = module.exports.baseDiff().div(bignum(rpcData.b)).toNumber();
+  return {
+    hash:               rpcData.msg,
+    hash2:              rpcData.pk,
+    difficulty:         difficulty,
+    height:             parseInt(rpcData.h)
+  };
+};
+
+module.exports.RtmBlockTemplate = function(rpcData, poolAddress) {
+  return rtm.RtmBlockTemplate(rpcData, poolAddress);
+};
+
+module.exports.convertRtmBlob = function(blobBuffer) {
+  let header = blobBuffer.slice(0, 80);
+  update_merkle_root_hash(80, true, blobBuffer, header);
+  return header;
+};
+
+module.exports.constructNewRtmBlob = function(blockTemplate, nonceBuff) {
+  update_merkle_root_hash(80, true, blockTemplate, blockTemplate);
+  nonceBuff.copy(blockTemplate, 76, 0, 4);
+  return blockTemplate;
+};

package.json

@@ -1,18 +1,25 @@
 {
   "name": "cryptoforknote-util",
-  "version": "7.0.2",
-  "main": "cryptoforknote-util",
+  "version": "15.3.3",
   "author": {
     "name": "LucasJones",
     "email": "lucasjonesdev@hotmail.co.uk"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/MoneroOcean/node-cryptoforknote-util.git"
+    "url": "https://github.com/haven-protocol-org/node-cryptoforknote-util.git"
   },
   "dependencies": {
+    "promise": "*",
     "bindings": "*",
-    "nan": "^2.0.0"
+    "nan": "^2.14.2",
+    "bignum": "^0.13.1",
+    "sha3": "*",
+    "base58-native": "*",
+    "bech32": "*",
+    "varuint-bitcoin": "^1.0.4",
+    "merkle-lib": "^2.0.10",
+    "bitcoinjs-lib": "git+https://github.com/MoneroOcean/bitcoinjs-lib.git"
   },
   "keywords": [
     "cryptonight",

rtm.js

@@ -0,0 +1,302 @@
+const bignum  = require('bignum');
+const base58  = require('base58-native');
+const bech32  = require('bech32');
+const bitcoin = require('bitcoinjs-lib');
+
+const diff1 = 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF;
+
+function reverseBuffer(buff) {
+  let reversed = Buffer.alloc(buff.length);
+  for (let i = buff.length - 1; i >= 0; i--) reversed[buff.length - i - 1] = buff[i];
+  return reversed;
+}
+
+function reverseByteOrder(buff) {
+  for (let i = 0; i < 8; i++) buff.writeUInt32LE(buff.readUInt32BE(i * 4), i * 4);
+  return reverseBuffer(buff);
+}
+
+function packInt32LE(num) {
+  let buff = Buffer.alloc(4);
+  buff.writeInt32LE(num, 0);
+  return buff;
+}
+
+function packInt32BE(num) {
+  let buff = Buffer.alloc(4);
+  buff.writeInt32BE(num, 0);
+  return buff;
+}
+
+function packUInt16LE(num) {
+  let buff = Buffer.alloc(2);
+  buff.writeUInt16LE(num, 0);
+  return buff;
+}
+
+function packUInt32LE(num) {
+  let buff = Buffer.alloc(4);
+  buff.writeUInt32LE(num, 0);
+  return buff;
+}
+
+function packUInt32BE(num) {
+  let buff = Buffer.alloc(4);
+  buff.writeUInt32BE(num, 0);
+  return buff;
+}
+
+function packInt64LE(num){
+  let buff = Buffer.alloc(8);
+  buff.writeUInt32LE(num % Math.pow(2, 32), 0);
+  buff.writeUInt32LE(Math.floor(num / Math.pow(2, 32)), 4);
+  return buff;
+}
+
+// Defined in bitcoin protocol here:
+// https://en.bitcoin.it/wiki/Protocol_specification#Variable_length_integer
+function varIntBuffer(n) {
+  if (n < 0xfd) {
+    return Buffer.from([n]);
+  } else if (n <= 0xffff) {
+    let buff = Buffer.alloc(3);
+    buff[0] = 0xfd;
+    buff.writeUInt16LE(n, 1);
+    return buff;
+  } else if (n <= 0xffffffff) {
+    let buff = Buffer.alloc(5);
+    buff[0] = 0xfe;
+    buff.writeUInt32LE(n, 1);
+    return buff;
+  } else{
+    let buff = Buffer.alloc(9);
+    buff[0] = 0xff;
+    packUInt16LE(n).copy(buff, 1);
+    return buff;
+  }
+}
+
+// "serialized CScript" formatting as defined here:
+// https://github.com/bitcoin/bips/blob/master/bip-0034.mediawiki#specification
+// Used to format height and date when putting into script signature:
+// https://en.bitcoin.it/wiki/Script
+function serializeNumber(n) {
+  // New version from TheSeven
+  if (n >= 1 && n <= 16) return Buffer.from([0x50 + n]);
+  var l = 1;
+  var buff = Buffer.alloc(9);
+  while (n > 0x7f) {
+      buff.writeUInt8(n & 0xff, l++);
+      n >>= 8;
+  }
+  buff.writeUInt8(l, 0);
+  buff.writeUInt8(n, l++);
+  return buff.slice(0, l);
+}
+
+// Used for serializing strings used in script signature
+function serializeString(s) {
+  if (s.length < 253) {
+    return Buffer.concat([ Buffer.from([s.length]), Buffer.from(s) ]);
+  } else if (s.length < 0x10000) {
+    return Buffer.concat([ Buffer.from([253]), packUInt16LE(s.length), Buffer.from(s) ]);
+  } else if (s.length < 0x100000000) {
+    return Buffer.concat([ Buffer.from([254]), packUInt32LE(s.length), Buffer.from(s) ]);
+  } else {
+    return Buffer.concat([ Buffer.from([255]), packUInt16LE(s.length), Buffer.from(s) ]);
+  }
+}
+
+// An exact copy of python's range feature. Written by Tadeck:
+// http://stackoverflow.com/a/8273091
+function range(start, stop, step) {
+  if (typeof stop === 'undefined') {
+    stop = start;
+    start = 0;
+  }
+  if (typeof step === 'undefined') {
+    step = 1;
+  }
+  if ((step > 0 && start >= stop) || (step < 0 && start <= stop)) {
+    return [];
+  }
+  let result = [];
+  for (let i = start; step > 0 ? i < stop : i > stop; i += step) {
+    result.push(i);
+  }
+  return result;
+}
+
+function uint256BufferFromHash(hex) {
+  let fromHex = Buffer.from(hex, 'hex');
+  if (fromHex.length != 32) {
+    let empty = Buffer.alloc(32);
+    empty.fill(0);
+    fromHex.copy(empty);
+    fromHex = empty;
+  }
+  return reverseBuffer(fromHex);
+}
+
+function getTransactionBuffers(txs) {
+  let txHashes = txs.map(function(tx) {
+    if (tx.txid !== undefined) {
+      return uint256BufferFromHash(tx.txid);
+    }
+    return uint256BufferFromHash(tx.hash);
+  });
+  return [null].concat(txHashes);
+}
+
+function addressToScript(addr) {
+  const decoded = base58.decode(addr);
+  if (!decoded || decoded.length != 25) {
+    const decoded2 = Buffer.from(bech32.bech32.fromWords(bech32.bech32.decode(addr).words.slice(1)));
+    if (decoded2.length != 20) throw new Error('Invalid address ' + addr);
+    return Buffer.concat([Buffer.from([0x0, 0x14]), decoded2]);
+  }
+  const pubkey = decoded.slice(1, -4);
+  return Buffer.concat([Buffer.from([0x76, 0xa9, 0x14]), pubkey, Buffer.from([0x88, 0xac])]);
+}
+
+function createOutputTransaction(amount, payee, rewardToPool, reward, txOutputBuffers, payeeScript) {
+  const payeeReward = amount;
+  if (!payeeScript) payeeScript = addressToScript(payee);
+  txOutputBuffers.push(Buffer.concat([
+    packInt64LE(payeeReward),
+    varIntBuffer(payeeScript.length),
+    payeeScript
+  ]));
+  return { reward: reward - amount, rewardToPool: rewardToPool - amount };
+}
+
+function generateOutputTransactions(rpcData, poolAddress) {
+  let reward       = rpcData.coinbasevalue;
+  let rewardToPool = reward;
+  let txOutputBuffers = [];
+
+  if (rpcData.coinbasedevreward) {
+    const rewards = createOutputTransaction(rpcData.coinbasedevreward.value, rpcData.coinbasedevreward.address, rewardToPool, reward, txOutputBuffers);
+    reward        = rewards.reward;
+    rewardToPool  = rewards.rewardToPool;
+  }
+
+  if (rpcData.smartnode) {
+    if (rpcData.smartnode.payee) {
+      const rewards = createOutputTransaction(rpcData.smartnode.amount, rpcData.smartnode.payee, rewardToPool, reward, txOutputBuffers);
+      reward        = rewards.reward;
+      rewardToPool  = rewards.rewardToPool;
+    } else if (Array.isArray(rpcData.smartnode)) {
+      for (let i in rpcData.smartnode) {
+        const rewards = createOutputTransaction(rpcData.smartnode[i].amount, rpcData.smartnode[i].payee, rewardToPool, reward, txOutputBuffers);
+	reward        = rewards.reward;
+        rewardToPool  = rewards.rewardToPool;
+      }
+    } 
+  }
+
+  if (rpcData.superblock) {
+    for (let i in rpcData.superblock) {
+      const rewards = createOutputTransaction(rpcData.superblock[i].amount, rpcData.superblock[i].payee, rewardToPool, reward, txOutputBuffers);
+      reward        = rewards.reward;
+      rewardToPool  = rewards.rewardToPool;
+    }
+  }
+
+  if (rpcData.founder_payments_started && rpcData.founder) {
+    const founderReward = rpcData.founder.amount || 0;
+    const rewards = createOutputTransaction(founderReward, rpcData.founder.payee, rewardToPool, reward, txOutputBuffers);
+    reward        = rewards.reward;
+    rewardToPool  = rewards.rewardToPool;
+  }
+
+  createOutputTransaction(rewardToPool, null, rewardToPool, reward, txOutputBuffers, Buffer.from(addressToScript(poolAddress), "hex"));
+
+  if (rpcData.default_witness_commitment !== undefined) {
+    const witness_commitment = Buffer.from(rpcData.default_witness_commitment, 'hex');
+    txOutputBuffers.unshift(Buffer.concat([
+      packInt64LE(0),
+      varIntBuffer(witness_commitment.length),
+      witness_commitment
+    ]));
+  }
+
+  return Buffer.concat([ varIntBuffer(txOutputBuffers.length), Buffer.concat(txOutputBuffers)]);
+}
+
+module.exports.RtmBlockTemplate = function(rpcData, poolAddress) {
+  const extraNoncePlaceholderLength = 17;
+  const coinbaseVersion = Buffer.concat([packUInt16LE(3), packUInt16LE(5)]);
+
+  const scriptSigPart1 = Buffer.concat([
+    serializeNumber(rpcData.height),
+    Buffer.from(rpcData.coinbaseaux.flags ? rpcData.coinbaseaux.flags : "", 'hex'),
+    serializeNumber(Date.now() / 1000 | 0),
+    Buffer.from([extraNoncePlaceholderLength])
+  ]);
+
+  const scriptSigPart2 = serializeString('/nodeStratum/');
+
+  const blob1 = Buffer.concat([
+    coinbaseVersion,
+    // transaction input
+    varIntBuffer(1), // txInputsCount
+    uint256BufferFromHash(""), // txInPrevOutHash
+    packUInt32LE(Math.pow(2, 32) - 1), // txInPrevOutIndex
+    varIntBuffer(scriptSigPart1.length + extraNoncePlaceholderLength + scriptSigPart2.length),
+    scriptSigPart1
+  ]);
+
+  let blob2 = Buffer.concat([
+    scriptSigPart2,
+    packUInt32LE(0), // txInSequence
+    // end transaction input
+    // transaction output
+    generateOutputTransactions(rpcData, poolAddress),
+    // end transaction ouput
+    packUInt32LE(0) // txLockTime
+  ]);
+
+  if (rpcData.coinbase_payload) {
+     blob2 = Buffer.concat([
+       blob2,
+       varIntBuffer(rpcData.coinbase_payload.length / 2),
+       Buffer.from(rpcData.coinbase_payload, 'hex')
+     ]);
+  }
+
+  const prev_hash = reverseBuffer(Buffer.from(rpcData.previousblockhash, 'hex')).toString('hex');
+  const version = packInt32LE(rpcData.version).toString('hex');
+  const curtime = packUInt32LE(rpcData.curtime).toString('hex');
+  let bits = Buffer.from(rpcData.bits, 'hex');
+  bits.writeUInt32LE(bits.readUInt32BE());
+  let txs = [];
+  // skip version 1 transaction because they contain some OP_RETURN(0x6A) opcode in the beginning of
+  // tx input scripts instead of size of script part so not sure how to parse them
+  // just drop them for now
+  // example: https://explorer.raptoreum.com/tx/1461d70fa8362b0896e2e9be6312521f2684f22c9b0f9152695f33f67d9f9d3f
+  rpcData.transactions.forEach(function(tx) {
+    if (tx.version != 1) {
+      try {
+        bitcoin.Transaction.fromBuffer(Buffer.from(tx.data, 'hex'), false, false);
+      } catch(err) {
+        console.error("Skip RTM tx due to parse error: " + tx.data);
+        return; // skip transaction if it is not parsed OK (varint coding seems to be different for RTM)
+      }
+      txs.push(tx);
+    } else {
+      console.error("Skip RTM v1 tx: " + tx.data);
+    }
+  });
+  const txn = varIntBuffer(txs.length + 1);
+
+  return {
+    difficulty:         parseFloat((diff1 / bignum(rpcData.target, 16).toNumber()).toFixed(9)),
+    height:             rpcData.height,
+    prev_hash:          prev_hash,
+    blocktemplate_blob: version + prev_hash + Buffer.alloc(32, 0).toString('hex') + curtime + bits.toString('hex') + Buffer.alloc(4, 0).toString('hex') +
+                        txn.toString('hex') + blob1.toString('hex') + Buffer.alloc(extraNoncePlaceholderLength, 0xCC).toString('hex') + blob2.toString('hex')  +
+                        Buffer.concat(txs.map(function(tx) { return Buffer.from(tx.data, 'hex'); })).toString('hex'),
+    reserved_offset:    80 + txn.length + blob1.length
+  }
+}

src/common/base58.cpp

@@ -1,7 +1,32 @@
-// Copyright (c) 2012-2013 The Cryptonote developers
-// Distributed under the MIT/X11 software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
+// Copyright (c) 2014-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
 #include "base58.h"
 
@@ -11,7 +36,6 @@
 
 #include "crypto/hash.h"
 #include "int-util.h"
-#include "util.h"
 #include "varint.h"
 
 namespace tools
@@ -84,20 +108,8 @@ namespace tools
         assert(1 <= size && size <= sizeof(uint64_t));
 
         uint64_t res = 0;
-        switch (9 - size)
-        {
-        case 1:            res |= *data++;
-        case 2: res <<= 8; res |= *data++;
-        case 3: res <<= 8; res |= *data++;
-        case 4: res <<= 8; res |= *data++;
-        case 5: res <<= 8; res |= *data++;
-        case 6: res <<= 8; res |= *data++;
-        case 7: res <<= 8; res |= *data++;
-        case 8: res <<= 8; res |= *data; break;
-        default: assert(false);
-        }
-
-        return res;
+        memcpy(reinterpret_cast<uint8_t*>(&res) + sizeof(uint64_t) - size, data, size);
+        return SWAP64BE(res);
       }
 
       void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data)
@@ -110,7 +122,7 @@ namespace tools
 
       void encode_block(const char* block, size_t size, char* res)
       {
-        assert(1 <= size && size <= sizeof(full_block_size));
+        assert(1 <= size && size <= full_block_size);
 
         uint64_t num = uint_8be_to_64(reinterpret_cast<const uint8_t*>(block), size);
         int i = static_cast<int>(encoded_block_sizes[size]) - 1;
@@ -222,7 +234,7 @@ namespace tools
       return encode(buf);
     }
 
-    bool decode_addr(std::string addr, uint64_t& tag, std::string& data)
+    bool decode_addr(const std::string &addr, uint64_t& tag, std::string& data)
     {
       std::string addr_data;
       bool r = decode(addr, addr_data);

src/common/base58.h

@@ -1,6 +1,32 @@
-// Copyright (c) 2012-2013 The Cryptonote developers
-// Distributed under the MIT/X11 software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+// Copyright (c) 2014-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
 #pragma once
 
@@ -15,6 +41,6 @@ namespace tools
     bool decode(const std::string& enc, std::string& data);
 
     std::string encode_addr(uint64_t tag, const std::string& data);
-    bool decode_addr(std::string addr, uint64_t& tag, std::string& data);
+    bool decode_addr(const std::string &addr, uint64_t& tag, std::string& data);
   }
 }

src/contrib/epee/include/file_io_utils.h

@@ -0,0 +1,235 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+#ifndef _FILE_IO_UTILS_H_
+#define _FILE_IO_UTILS_H_
+
+#include <fstream>
+#include <boost/filesystem/path.hpp>
+#include <boost/filesystem/operations.hpp>
+#ifdef WIN32
+#include <windows.h>
+#include "string_tools.h"
+#endif
+
+// On Windows there is a problem with non-ASCII characters in path and file names
+// as far as support by the standard components used is concerned:
+
+// The various file stream classes, e.g. std::ifstream and std::ofstream, are
+// part of the GNU C++ Library / libstdc++. On the most basic level they use the
+// fopen() call as defined / made accessible to programs compiled within MSYS2
+// by the stdio.h header file maintained by the MinGW project.
+
+// The critical point: The implementation of fopen() is part of MSVCRT, the
+// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any
+// Unicode support.
+
+// Monero code that would want to continue to use the normal file stream classes
+// but WITH Unicode support could therefore not solve this problem on its own,
+// but 2 different projects from 2 different maintaining groups would need changes
+// in this particular direction - something probably difficult to achieve and
+// with a long time to wait until all new versions / releases arrive.
+
+// Implemented solution approach: Circumvent the problem by stopping to use std
+// file stream classes on Windows and directly use Unicode-capable WIN32 API
+// calls. Most of the code doing so is concentrated in this header file here.
+
+namespace epee
+{
+namespace file_io_utils
+{
+	inline 
+		bool is_file_exist(const std::string& path)
+	{
+		boost::filesystem::path p(path);
+		return boost::filesystem::exists(p);
+	}
+
+	inline
+		bool save_string_to_file(const std::string& path_to_file, const std::string& str)
+	{
+#ifdef WIN32
+                std::wstring wide_path;
+                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
+                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+                if (file_handle == INVALID_HANDLE_VALUE)
+                    return false;
+                DWORD bytes_written;
+                DWORD bytes_to_write = (DWORD)str.size();
+                BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL);
+                CloseHandle(file_handle);
+                if (bytes_written != bytes_to_write)
+                    result = FALSE;
+                return result;
+#else
+		try
+		{
+			std::ofstream fstream;
+			fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+			fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc);
+			fstream << str;
+			fstream.close();
+			return true;
+		}
+
+		catch(...)
+		{
+			return false;
+		}
+#endif
+	}
+
+	inline
+	bool get_file_time(const std::string& path_to_file, time_t& ft)
+	{
+		boost::system::error_code ec;
+		ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec);
+		if(!ec)
+			return true;
+		else
+			return false;
+	}
+
+	inline
+		bool set_file_time(const std::string& path_to_file, const time_t& ft)
+	{
+		boost::system::error_code ec;
+		boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec);
+		if(!ec)
+			return true;
+		else
+			return false;
+	}
+
+
+	inline
+		bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000)
+	{
+#ifdef WIN32
+                std::wstring wide_path;
+                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
+                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+                if (file_handle == INVALID_HANDLE_VALUE)
+                    return false;
+                DWORD file_size = GetFileSize(file_handle, NULL);
+                if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) {
+                    CloseHandle(file_handle);
+                    return false;
+                }
+                target_str.resize(file_size);
+                DWORD bytes_read;
+                BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL);
+                CloseHandle(file_handle);
+                if (bytes_read != file_size)
+                    result = FALSE;
+                return result;
+#else
+		try
+		{
+			std::ifstream fstream;
+			fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+			fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate);
+
+			std::ifstream::pos_type file_size = fstream.tellg();
+			
+			if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large
+				return false;//don't go crazy
+			size_t file_size_t = static_cast<size_t>(file_size);
+
+			target_str.resize(file_size_t);
+
+			fstream.seekg (0, std::ios::beg);
+			fstream.read((char*)target_str.data(), target_str.size());
+			fstream.close();
+			return true;
+		}
+
+		catch(...)
+		{
+			return false;
+		}
+#endif
+	}
+
+	inline
+		bool append_string_to_file(const std::string& path_to_file, const std::string& str)
+	{
+                // No special Windows implementation because so far not used in Monero code
+		try
+		{
+			std::ofstream fstream;
+			fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+			fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app);
+			fstream << str;
+			fstream.close();
+			return true;
+		}
+
+		catch(...)
+		{
+			return false;
+		}
+	}
+
+	inline
+		bool get_file_size(const std::string& path_to_file, uint64_t &size)
+	{
+#ifdef WIN32
+                std::wstring wide_path;
+                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
+                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+                if (file_handle == INVALID_HANDLE_VALUE)
+                    return false;
+                LARGE_INTEGER file_size;
+                BOOL result = GetFileSizeEx(file_handle, &file_size);
+                CloseHandle(file_handle);
+                if (result) {
+                    size = file_size.QuadPart;
+                }
+                return size;
+#else
+		try
+		{
+			std::ifstream fstream;
+			fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
+			fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate);
+			size = fstream.tellg();
+			fstream.close();
+			return true;
+		}
+
+		catch(...)
+		{
+			return false;
+		}
+#endif
+	}
+
+}
+}
+
+#endif //_FILE_IO_UTILS_H_

src/contrib/epee/include/fnv1.h

@@ -0,0 +1,45 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+namespace epee
+{
+
+namespace fnv
+{
+  inline uint64_t FNV1a(const char *ptr, size_t sz)
+  {
+    uint64_t h = 0xcbf29ce484222325;
+    for (size_t i = 0; i < sz; ++i)
+      h = (h ^ *(const uint8_t*)ptr++) * 0x100000001b3;
+    return h;
+  }
+}
+
+}

src/contrib/epee/include/hex.h

@@ -0,0 +1,78 @@
+// Copyright (c) 2017-2019, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include <iosfwd>
+#include <string>
+#include <boost/utility/string_ref.hpp>
+
+#include "wipeable_string.h"
+#include "span.h"
+
+namespace epee
+{
+  struct to_hex
+  {
+    //! \return A std::string containing hex of `src`.
+    static std::string string(const span<const std::uint8_t> src);
+    //! \return A epee::wipeable_string containing hex of `src`.
+    static epee::wipeable_string wipeable_string(const span<const std::uint8_t> src);
+    template<typename T> static epee::wipeable_string wipeable_string(const T &pod) { return wipeable_string(span<const uint8_t>((const uint8_t*)&pod, sizeof(pod))); }
+
+    //! \return An array containing hex of `src`.
+    template<std::size_t N>
+    static std::array<char, N * 2> array(const std::array<std::uint8_t, N>& src) noexcept
+    {
+      std::array<char, N * 2> out{{}};
+      static_assert(N <= 128, "keep the stack size down");
+      buffer_unchecked(out.data(), {src.data(), src.size()});
+      return out;
+    }
+
+    //! Append `src` as hex to `out`.
+    static void buffer(std::ostream& out, const span<const std::uint8_t> src);
+
+    //! Append `< + src + >` as hex to `out`.
+    static void formatted(std::ostream& out, const span<const std::uint8_t> src);
+
+  private:
+    template<typename T> T static convert(const span<const std::uint8_t> src);
+
+    //! Write `src` bytes as hex to `out`. `out` must be twice the length
+    static void buffer_unchecked(char* out, const span<const std::uint8_t> src) noexcept;
+  };
+
+  struct from_hex
+  {
+      //! \return An std::vector of unsigned integers from the `src`
+      static std::vector<uint8_t> vector(boost::string_ref src);
+  };
+}

src/contrib/epee/include/int-util.h

@@ -0,0 +1,258 @@
+// Copyright (c) 2014-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+
+#ifndef _MSC_VER
+#include <sys/param.h>
+#endif
+
+#if defined(__ANDROID__)
+#include <byteswap.h>
+#endif
+
+#if defined(__sun) && defined(__SVR4)
+#include <endian.h>
+#endif
+
+#if defined(_MSC_VER)
+#include <stdlib.h>
+
+static inline uint32_t rol32(uint32_t x, int r) {
+  static_assert(sizeof(uint32_t) == sizeof(unsigned int), "this code assumes 32-bit integers");
+  return _rotl(x, r);
+}
+
+static inline uint64_t rol64(uint64_t x, int r) {
+  return _rotl64(x, r);
+}
+
+#else
+
+static inline uint32_t rol32(uint32_t x, int r) {
+  return (x << (r & 31)) | (x >> (-r & 31));
+}
+
+static inline uint64_t rol64(uint64_t x, int r) {
+  return (x << (r & 63)) | (x >> (-r & 63));
+}
+
+#endif
+
+static inline uint64_t hi_dword(uint64_t val) {
+  return val >> 32;
+}
+
+static inline uint64_t lo_dword(uint64_t val) {
+  return val & 0xFFFFFFFF;
+}
+
+static inline uint64_t mul128(uint64_t multiplier, uint64_t multiplicand, uint64_t* product_hi) {
+  // multiplier   = ab = a * 2^32 + b
+  // multiplicand = cd = c * 2^32 + d
+  // ab * cd = a * c * 2^64 + (a * d + b * c) * 2^32 + b * d
+  uint64_t a = hi_dword(multiplier);
+  uint64_t b = lo_dword(multiplier);
+  uint64_t c = hi_dword(multiplicand);
+  uint64_t d = lo_dword(multiplicand);
+
+  uint64_t ac = a * c;
+  uint64_t ad = a * d;
+  uint64_t bc = b * c;
+  uint64_t bd = b * d;
+
+  uint64_t adbc = ad + bc;
+  uint64_t adbc_carry = adbc < ad ? 1 : 0;
+
+  // multiplier * multiplicand = product_hi * 2^64 + product_lo
+  uint64_t product_lo = bd + (adbc << 32);
+  uint64_t product_lo_carry = product_lo < bd ? 1 : 0;
+  *product_hi = ac + (adbc >> 32) + (adbc_carry << 32) + product_lo_carry;
+  assert(ac <= *product_hi);
+
+  return product_lo;
+}
+
+static inline uint64_t div_with_reminder(uint64_t dividend, uint32_t divisor, uint32_t* remainder) {
+  dividend |= ((uint64_t)*remainder) << 32;
+  *remainder = dividend % divisor;
+  return dividend / divisor;
+}
+
+// Long division with 2^32 base
+static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uint32_t divisor, uint64_t* quotient_hi, uint64_t* quotient_lo) {
+  uint64_t dividend_dwords[4];
+  uint32_t remainder = 0;
+
+  dividend_dwords[3] = hi_dword(dividend_hi);
+  dividend_dwords[2] = lo_dword(dividend_hi);
+  dividend_dwords[1] = hi_dword(dividend_lo);
+  dividend_dwords[0] = lo_dword(dividend_lo);
+
+  *quotient_hi  = div_with_reminder(dividend_dwords[3], divisor, &remainder) << 32;
+  *quotient_hi |= div_with_reminder(dividend_dwords[2], divisor, &remainder);
+  *quotient_lo  = div_with_reminder(dividend_dwords[1], divisor, &remainder) << 32;
+  *quotient_lo |= div_with_reminder(dividend_dwords[0], divisor, &remainder);
+
+  return remainder;
+}
+
+#define IDENT32(x) ((uint32_t) (x))
+#define IDENT64(x) ((uint64_t) (x))
+
+#define SWAP32(x) ((((uint32_t) (x) & 0x000000ff) << 24) | \
+  (((uint32_t) (x) & 0x0000ff00) <<  8) | \
+  (((uint32_t) (x) & 0x00ff0000) >>  8) | \
+  (((uint32_t) (x) & 0xff000000) >> 24))
+#define SWAP64(x) ((((uint64_t) (x) & 0x00000000000000ff) << 56) | \
+  (((uint64_t) (x) & 0x000000000000ff00) << 40) | \
+  (((uint64_t) (x) & 0x0000000000ff0000) << 24) | \
+  (((uint64_t) (x) & 0x00000000ff000000) <<  8) | \
+  (((uint64_t) (x) & 0x000000ff00000000) >>  8) | \
+  (((uint64_t) (x) & 0x0000ff0000000000) >> 24) | \
+  (((uint64_t) (x) & 0x00ff000000000000) >> 40) | \
+  (((uint64_t) (x) & 0xff00000000000000) >> 56))
+
+static inline uint32_t ident32(uint32_t x) { return x; }
+static inline uint64_t ident64(uint64_t x) { return x; }
+
+#ifndef __OpenBSD__
+#  if defined(__ANDROID__) && defined(__swap32) && !defined(swap32)
+#      define swap32 __swap32
+#  elif !defined(swap32)
+static inline uint32_t swap32(uint32_t x) {
+  x = ((x & 0x00ff00ff) << 8) | ((x & 0xff00ff00) >> 8);
+  return (x << 16) | (x >> 16);
+}
+#  endif
+#  if defined(__ANDROID__) && defined(__swap64) && !defined(swap64)
+#      define swap64 __swap64
+#  elif !defined(swap64)
+static inline uint64_t swap64(uint64_t x) {
+  x = ((x & 0x00ff00ff00ff00ff) <<  8) | ((x & 0xff00ff00ff00ff00) >>  8);
+  x = ((x & 0x0000ffff0000ffff) << 16) | ((x & 0xffff0000ffff0000) >> 16);
+  return (x << 32) | (x >> 32);
+}
+#  endif
+#endif /* __OpenBSD__ */
+
+#if defined(__GNUC__)
+#define UNUSED __attribute__((unused))
+#else
+#define UNUSED
+#endif
+static inline void mem_inplace_ident(void *mem UNUSED, size_t n UNUSED) { }
+#undef UNUSED
+
+static inline void mem_inplace_swap32(void *mem, size_t n) {
+  size_t i;
+  for (i = 0; i < n; i++) {
+    ((uint32_t *) mem)[i] = swap32(((const uint32_t *) mem)[i]);
+  }
+}
+static inline void mem_inplace_swap64(void *mem, size_t n) {
+  size_t i;
+  for (i = 0; i < n; i++) {
+    ((uint64_t *) mem)[i] = swap64(((const uint64_t *) mem)[i]);
+  }
+}
+
+static inline void memcpy_ident32(void *dst, const void *src, size_t n) {
+  memcpy(dst, src, 4 * n);
+}
+static inline void memcpy_ident64(void *dst, const void *src, size_t n) {
+  memcpy(dst, src, 8 * n);
+}
+
+static inline void memcpy_swap32(void *dst, const void *src, size_t n) {
+  size_t i;
+  for (i = 0; i < n; i++) {
+    ((uint32_t *) dst)[i] = swap32(((const uint32_t *) src)[i]);
+  }
+}
+static inline void memcpy_swap64(void *dst, const void *src, size_t n) {
+  size_t i;
+  for (i = 0; i < n; i++) {
+    ((uint64_t *) dst)[i] = swap64(((const uint64_t *) src)[i]);
+  }
+}
+
+#ifdef _MSC_VER
+# define LITTLE_ENDIAN	1234
+# define BIG_ENDIAN	4321
+# define BYTE_ORDER	LITTLE_ENDIAN
+#endif
+
+#if !defined(BYTE_ORDER) || !defined(LITTLE_ENDIAN) || !defined(BIG_ENDIAN)
+static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not enabled");
+#endif
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+#define SWAP32LE IDENT32
+#define SWAP32BE SWAP32
+#define swap32le ident32
+#define swap32be swap32
+#define mem_inplace_swap32le mem_inplace_ident
+#define mem_inplace_swap32be mem_inplace_swap32
+#define memcpy_swap32le memcpy_ident32
+#define memcpy_swap32be memcpy_swap32
+#define SWAP64LE IDENT64
+#define SWAP64BE SWAP64
+#define swap64le ident64
+#define swap64be swap64
+#define mem_inplace_swap64le mem_inplace_ident
+#define mem_inplace_swap64be mem_inplace_swap64
+#define memcpy_swap64le memcpy_ident64
+#define memcpy_swap64be memcpy_swap64
+#endif
+
+#if BYTE_ORDER == BIG_ENDIAN
+#define SWAP32BE IDENT32
+#define SWAP32LE SWAP32
+#define swap32be ident32
+#define swap32le swap32
+#define mem_inplace_swap32be mem_inplace_ident
+#define mem_inplace_swap32le mem_inplace_swap32
+#define memcpy_swap32be memcpy_ident32
+#define memcpy_swap32le memcpy_swap32
+#define SWAP64BE IDENT64
+#define SWAP64LE SWAP64
+#define swap64be ident64
+#define swap64le swap64
+#define mem_inplace_swap64be mem_inplace_ident
+#define mem_inplace_swap64le mem_inplace_swap64
+#define memcpy_swap64be memcpy_ident64
+#define memcpy_swap64le memcpy_swap64
+#endif

src/contrib/epee/include/math_helper.h

@@ -32,11 +32,12 @@
 
 #include <list>
 #include <numeric>
-#include <boost/timer.hpp>
+#include <boost/timer/timer.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <boost/uuid/random_generator.hpp>
 
 #include "misc_os_dependent.h"
+#include "syncobj.h"
 
 namespace epee
 {
@@ -229,44 +230,57 @@ namespace math_helper
 		}
 
 	}
-PRAGMA_WARNING_PUSH
-PRAGMA_GCC("GCC diagnostic ignored \"-Wstrict-aliasing\"")
-  inline
-  uint64_t generated_random_uint64()
-  {
-    boost::uuids::uuid id___ = boost::uuids::random_generator()();
-    return  *reinterpret_cast<uint64_t*>(&id___.data[0]); //(*reinterpret_cast<uint64_t*>(&id___.data[0]) ^ *reinterpret_cast<uint64_t*>(&id___.data[8]));
-  }
-PRAGMA_WARNING_POP
-	template<int default_interval, bool start_immediate = true>
-	class once_a_time_seconds
+	template<uint64_t scale, int default_interval, bool start_immediate = true>
+	class once_a_time
 	{
+    uint64_t get_time() const
+    {
+#ifdef _WIN32
+      FILETIME fileTime;
+      GetSystemTimeAsFileTime(&fileTime);
+      unsigned __int64 present = 0;
+      present |= fileTime.dwHighDateTime;
+      present = present << 32;
+      present |= fileTime.dwLowDateTime;
+      present /= 10;  // mic-sec
+      return present;
+#else
+      struct timeval tv;
+      gettimeofday(&tv, NULL);
+      return tv.tv_sec * 1000000 + tv.tv_usec;
+#endif
+    }
+
 	public:
-		once_a_time_seconds():m_interval(default_interval)
+		once_a_time():m_interval(default_interval * scale)
 		{
 			m_last_worked_time = 0;
       if(!start_immediate)
-        time(&m_last_worked_time);
+        m_last_worked_time = get_time();
 		}
 
 		template<class functor_t>
 		bool do_call(functor_t functr)
 		{
-			time_t current_time = 0;
-			time(&current_time);
+			uint64_t current_time = get_time();
 
       if(current_time - m_last_worked_time > m_interval)
 			{
 				bool res = functr();
-				time(&m_last_worked_time);
+				m_last_worked_time = get_time();
 				return res;
 			}
 			return true;
 		}
 
 	private:
-		time_t m_last_worked_time;
-		time_t m_interval;
+		uint64_t m_last_worked_time;
+		uint64_t m_interval;
 	};
+
+  template<int default_interval, bool start_immediate = true>
+  class once_a_time_seconds: public once_a_time<1000000, default_interval, start_immediate> {};
+  template<int default_interval, bool start_immediate = true>
+  class once_a_time_milliseconds: public once_a_time<1000, default_interval, start_immediate> {};
+}
 }
-}

src/contrib/epee/include/memwipe.h

@@ -0,0 +1,80 @@
+// Copyright (c) 2017-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#ifdef __cplusplus
+#include <array>
+
+extern "C" {
+#endif
+
+void *memwipe(void *src, size_t n);
+
+#ifdef __cplusplus
+}
+#endif
+
+#ifdef __cplusplus
+namespace tools {
+
+  /// Scrubs data in the contained type upon destruction.
+  ///
+  /// Primarily useful for making sure that private keys don't stick around in
+  /// memory after the objects that held them have gone out of scope.
+  template <class T>
+  struct scrubbed : public T {
+    using type = T;
+
+    ~scrubbed() {
+      scrub();
+    }
+
+    /// Destroy the contents of the contained type.
+    void scrub() {
+      static_assert(std::is_pod<T>::value,
+                    "T cannot be auto-scrubbed. T must be POD.");
+      static_assert(std::is_trivially_destructible<T>::value,
+                    "T cannot be auto-scrubbed. T must be trivially destructable.");
+      memwipe(this, sizeof(T));
+    }
+  };
+
+  template<typename T>
+  T& unwrap(scrubbed<T>& src) { return src; }
+
+  template<typename T>
+  const T& unwrap(scrubbed<T> const& src) { return src; }
+
+  template <class T, size_t N>
+  using scrubbed_arr = scrubbed<std::array<T, N>>;
+} // namespace tools
+
+#endif // __cplusplus

src/contrib/epee/include/misc_language.h

@@ -147,7 +147,8 @@ namespace misc_utils
     {}
     ~call_befor_die()
     {
-      m_func();
+      try { m_func(); }
+      catch (...) { /* ignore */ }
     }
   };
 
@@ -158,5 +159,10 @@ namespace misc_utils
     return slc;
   }
 
+  template<typename T> struct struct_init: T
+  {
+    struct_init(): T{} {}
+  };
+
 }
 }

src/contrib/epee/include/misc_os_dependent.h

@@ -23,6 +23,10 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 // 
+#ifdef _WIN32
+#include <winsock2.h>
+#endif
+
 #ifdef WIN32
   #ifndef WIN32_LEAN_AND_MEAN 
   #define WIN32_LEAN_AND_MEAN
@@ -42,16 +46,27 @@
 #include <mach/mach.h>
 #endif
 
+#include <iostream>
+#include <boost/lexical_cast.hpp>
+
 #pragma once 
 namespace epee
 {
 namespace misc_utils
 {
 
-        inline uint64_t get_tick_count()
+        inline uint64_t get_ns_count()
         {
 #if defined(_MSC_VER)
-                return ::GetTickCount64();
+                return ::GetTickCount64() * 1000000;
+#elif defined(WIN32)
+                static LARGE_INTEGER pcfreq = {0};
+                LARGE_INTEGER ticks;
+                if (!pcfreq.QuadPart)
+                    QueryPerformanceFrequency(&pcfreq);
+                QueryPerformanceCounter(&ticks);
+                ticks.QuadPart *= 1000000000; /* we want nsec */
+                return ticks.QuadPart / pcfreq.QuadPart;
 #elif defined(__MACH__)
                 clock_serv_t cclock;
                 mach_timespec_t mts;
@@ -60,16 +75,21 @@ namespace misc_utils
                 clock_get_time(cclock, &mts);
                 mach_port_deallocate(mach_task_self(), cclock);
 
-                return (mts.tv_sec * 1000) + (mts.tv_nsec/1000000);
+                return ((uint64_t)mts.tv_sec * 1000000000) + (mts.tv_nsec);
 #else
                 struct timespec ts;
                 if(clock_gettime(CLOCK_MONOTONIC, &ts) != 0) {
                         return 0;
                 }
-                return (ts.tv_sec * 1000) + (ts.tv_nsec/1000000);
+                return ((uint64_t)ts.tv_sec * 1000000000) + (ts.tv_nsec);
 #endif
         }
 
+        inline uint64_t get_tick_count()
+        {
+                return get_ns_count() / 1000000;
+        }
+
 
         inline int call_sys_cmd(const std::string& cmd)
 	{      
@@ -98,10 +118,19 @@ namespace misc_utils
 
 	inline std::string get_thread_string_id()
 	{
-#if defined(_MSC_VER)
+#if defined(_WIN32)
 		return boost::lexical_cast<std::string>(GetCurrentThreadId());
 #elif defined(__GNUC__)  
 		return boost::lexical_cast<std::string>(pthread_self());
+#endif
+	}
+
+	inline bool get_gmt_time(time_t t, struct tm &tm)
+	{
+#ifdef _WIN32
+		return gmtime_s(&tm, &t);
+#else
+		return gmtime_r(&t, &tm);
 #endif
 	}
 }

src/contrib/epee/include/mlocker.h

@@ -0,0 +1,87 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once 
+
+#include <map>
+#include <boost/thread/mutex.hpp>
+
+namespace epee
+{
+  class mlocker
+  {
+  public:
+    mlocker(void *ptr, size_t len);
+    ~mlocker();
+
+    static size_t get_page_size();
+    static size_t get_num_locked_pages();
+    static size_t get_num_locked_objects();
+
+    static void lock(void *ptr, size_t len);
+    static void unlock(void *ptr, size_t len);
+
+  private:
+    static size_t page_size;
+    static size_t num_locked_objects;
+
+    static boost::mutex &mutex();
+    static std::map<size_t, unsigned int> &map();
+    static void lock_page(size_t page);
+    static void unlock_page(size_t page);
+
+    void *ptr;
+    size_t len;
+  };
+
+  /// Locks memory while in scope
+  ///
+  /// Primarily useful for making sure that private keys don't get swapped out
+  //  to disk
+  template <class T>
+  struct mlocked : public T {
+    using type = T;
+
+    mlocked(): T() { mlocker::lock(this, sizeof(T)); }
+    mlocked(const T &t): T(t) { mlocker::lock(this, sizeof(T)); }
+    mlocked(const mlocked<T> &mt): T(mt) { mlocker::lock(this, sizeof(T)); }
+    mlocked(const T &&t): T(t) { mlocker::lock(this, sizeof(T)); }
+    mlocked(const mlocked<T> &&mt): T(mt) { mlocker::lock(this, sizeof(T)); }
+    mlocked<T> &operator=(const mlocked<T> &mt) { T::operator=(mt); return *this; }
+    ~mlocked() { try { mlocker::unlock(this, sizeof(T)); } catch (...) { /* do not propagate */ } }
+  };
+
+  template<typename T>
+  T& unwrap(mlocked<T>& src) { return src; }
+
+  template<typename T>
+  const T& unwrap(mlocked<T> const& src) { return src; }
+
+  template <class T, size_t N>
+  using mlocked_arr = mlocked<std::array<T, N>>;
+}

src/contrib/epee/include/serialization/enableable.h

@@ -50,4 +50,4 @@ namespace epee
     {	// construct from specified values
     }
   };
-}
+}

src/contrib/epee/include/serialization/keyvalue_serialization.h

@@ -31,6 +31,10 @@
 #include "misc_log_ex.h"
 #include "enableable.h"
 #include "keyvalue_serialization_overloads.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 namespace epee
 {
   /************************************************************************/
@@ -41,12 +45,12 @@ public: \
   template<class t_storage> \
   bool store( t_storage& st, typename t_storage::hsection hparent_section = nullptr) const\
   {\
-  return serialize_map<true>(*this, st, hparent_section);\
+    return serialize_map<true>(*this, st, hparent_section);\
   }\
   template<class t_storage> \
   bool _load( t_storage& stg, typename t_storage::hsection hparent_section = nullptr)\
   {\
-  return serialize_map<false>(*this, stg, hparent_section);\
+    return serialize_map<false>(*this, stg, hparent_section);\
   }\
   template<class t_storage> \
   bool load( t_storage& stg, typename t_storage::hsection hparent_section = nullptr)\
@@ -68,6 +72,15 @@ public: \
 #define KV_SERIALIZE_N(varialble, val_name) \
   epee::serialization::selector<is_store>::serialize(this_ref.varialble, stg, hparent_section, val_name);
 
+  template<typename T> inline void serialize_default(const T &t, T v) { }
+  template<typename T> inline void serialize_default(T &t, T v) { t = v; }
+
+#define KV_SERIALIZE_OPT_N(variable, val_name, default_value) \
+  do { \
+    if (!epee::serialization::selector<is_store>::serialize(this_ref.variable, stg, hparent_section, val_name)) \
+      epee::serialize_default(this_ref.variable, default_value); \
+  } while (0);
+
 #define KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name) \
   epee::serialization::selector<is_store>::serialize_t_val_as_blob(this_ref.varialble, stg, hparent_section, val_name); 
 
@@ -75,6 +88,14 @@ public: \
   static_assert(std::is_pod<decltype(this_ref.varialble)>::value, "t_type must be a POD type."); \
   KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name)
 
+#define KV_SERIALIZE_VAL_POD_AS_BLOB_OPT_N(varialble, val_name, default_value) \
+  do { \
+    static_assert(std::is_pod<decltype(this_ref.varialble)>::value, "t_type must be a POD type."); \
+    bool ret = KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name); \
+    if (!ret) \
+      epee::serialize_default(this_ref.varialble, default_value); \
+  } while(0);
+
 #define KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(varialble, val_name) \
   epee::serialization::selector<is_store>::serialize_stl_container_pod_val_as_blob(this_ref.varialble, stg, hparent_section, val_name);
 
@@ -82,8 +103,10 @@ public: \
 
 #define KV_SERIALIZE(varialble)                           KV_SERIALIZE_N(varialble, #varialble)
 #define KV_SERIALIZE_VAL_POD_AS_BLOB(varialble)           KV_SERIALIZE_VAL_POD_AS_BLOB_N(varialble, #varialble)
+#define KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(varialble, def)  KV_SERIALIZE_VAL_POD_AS_BLOB_OPT_N(varialble, #varialble, def)
 #define KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(varialble)     KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, #varialble) //skip is_pod compile time check
 #define KV_SERIALIZE_CONTAINER_POD_AS_BLOB(varialble)     KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(varialble, #varialble)
+#define KV_SERIALIZE_OPT(variable,default_value)          KV_SERIALIZE_OPT_N(variable, #variable, default_value)
 
 }
 

src/contrib/epee/include/serialization/keyvalue_serialization_overloads.h

@@ -26,8 +26,24 @@
 
 #pragma once
 
+#include <cstdint>
+#include <set>
+#include <list>
+#include <vector>
+#include <deque>
+#include <boost/mpl/vector.hpp>
+#include <boost/mpl/contains_fwd.hpp>
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 namespace epee
 {
+  namespace
+  {
+    template<class C> void hint_resize(C &container, size_t size) {}
+    template<class C> void hint_resize(std::vector<C> &container, size_t size) { container.reserve(size); }
+  }
   namespace serialization
   {
 
@@ -73,7 +89,7 @@ namespace epee
     template<class serializible_type, class t_storage>
     static bool unserialize_t_obj(serializible_type& obj, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
     {
-      typename t_storage::hsection	hchild_section = stg.open_section(pname, hparent_section, true);
+      typename t_storage::hsection	hchild_section = stg.open_section(pname, hparent_section, false);
       if(!hchild_section) return false;
       return obj._load(stg, hchild_section);
     }
@@ -90,7 +106,7 @@ namespace epee
     static bool unserialize_t_obj(enableable<serializible_type>& obj, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
     {
       obj.enabled = false;
-      typename t_storage::hsection	hchild_section = stg.open_section(pname, hparent_section, true);
+      typename t_storage::hsection	hchild_section = stg.open_section(pname, hparent_section, false);
       if(!hchild_section) return false;
       obj.enabled = true;
       return obj.v._load(stg, hchild_section);
@@ -117,16 +133,15 @@ namespace epee
       typename stl_container::value_type exchange_val;
       typename t_storage::harray hval_array = stg.get_first_value(pname, exchange_val, hparent_section);
       if(!hval_array) return false;
-      container.push_back(std::move(exchange_val));
+      container.insert(container.end(), std::move(exchange_val));
       while(stg.get_next_value(hval_array, exchange_val))
-        container.push_back(std::move(exchange_val));
+        container.insert(container.end(), std::move(exchange_val));
       return true;
     }//--------------------------------------------------------------------------------------------------------------------
     template<class stl_container, class t_storage>
     static bool serialize_stl_container_pod_val_as_blob(const stl_container& container, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
     {
       if(!container.size()) return true;
-      typename stl_container::const_iterator it = container.begin();
       std::string mb;
       mb.resize(sizeof(typename stl_container::value_type)*container.size());
       typename stl_container::value_type* p_elem = (typename stl_container::value_type*)mb.data();
@@ -150,10 +165,11 @@ namespace epee
         typename stl_container::value_type* pelem =  (typename stl_container::value_type*)buff.data();
         CHECK_AND_ASSERT_MES(!(loaded_size%sizeof(typename stl_container::value_type)), 
           false, 
-          "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type));
+          "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type) << ", type " << typeid(typename stl_container::value_type).name());
         size_t count = (loaded_size/sizeof(typename stl_container::value_type));
+        hint_resize(container, count);
         for(size_t i = 0; i < count; i++)
-          container.push_back(*(pelem++));
+          container.insert(container.end(), *(pelem++));
       }
       return res;
     }
@@ -187,12 +203,12 @@ namespace epee
       typename t_storage::harray hsec_array = stg.get_first_section(pname, hchild_section, hparent_section);
       if(!hsec_array || !hchild_section) return false;
       res = val._load(stg, hchild_section);
-      container.push_back(val);
+      container.insert(container.end(), val);
       while(stg.get_next_section(hsec_array, hchild_section))
       {
         typename stl_container::value_type val_l = typename stl_container::value_type();
         res |= val_l._load(stg, hchild_section);
-        container.push_back(std::move(val_l));
+        container.insert(container.end(), std::move(val_l));
       }
       return res;
     }
@@ -228,6 +244,18 @@ namespace epee
       } 
       //-------------------------------------------------------------------------------------------------------------------
       template<class t_type, class t_storage>
+      static bool kv_serialize(const std::deque<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return serialize_stl_container_t_val(d, stg, hparent_section, pname);
+      }
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
+      static bool kv_unserialize(std::deque<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return unserialize_stl_container_t_val(d, stg, hparent_section, pname);
+      } 
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
       static bool kv_serialize(const std::list<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
       {
         return serialize_stl_container_t_val(d, stg, hparent_section, pname);
@@ -239,6 +267,18 @@ namespace epee
         return unserialize_stl_container_t_val(d, stg, hparent_section, pname);
       } 
       //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
+      static bool kv_serialize(const std::set<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return serialize_stl_container_t_val(d, stg, hparent_section, pname);
+      }
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
+      static bool kv_unserialize(std::set<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return unserialize_stl_container_t_val(d, stg, hparent_section, pname);
+      } 
+      //-------------------------------------------------------------------------------------------------------------------
     };
     template<>
     struct kv_serialization_overloads_impl_is_base_serializable_types<false>
@@ -269,6 +309,18 @@ namespace epee
       } 
       //-------------------------------------------------------------------------------------------------------------------
       template<class t_type, class t_storage>
+      static bool kv_serialize(const std::deque<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return serialize_stl_container_t_obj(d, stg, hparent_section, pname);
+      }
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
+      static bool kv_unserialize(std::deque<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return unserialize_stl_container_t_obj(d, stg, hparent_section, pname);
+      } 
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
       static bool kv_serialize(const std::list<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
       {
         return serialize_stl_container_t_obj(d, stg, hparent_section, pname);
@@ -279,6 +331,18 @@ namespace epee
       {
         return unserialize_stl_container_t_obj(d, stg, hparent_section, pname);
       } 
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
+      static bool kv_serialize(const std::set<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return serialize_stl_container_t_obj(d, stg, hparent_section, pname);
+      }
+      //-------------------------------------------------------------------------------------------------------------------
+      template<class t_type, class t_storage>
+      static bool kv_unserialize(std::set<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+      {
+        return unserialize_stl_container_t_obj(d, stg, hparent_section, pname);
+      } 
     };
     template<class t_storage>
     struct base_serializable_types: public boost::mpl::vector<uint64_t, uint32_t, uint16_t, uint8_t, int64_t, int32_t, int16_t, int8_t, double, bool, std::string, typename t_storage::meta_entry>::type
@@ -354,6 +418,18 @@ namespace epee
     } 
     //-------------------------------------------------------------------------------------------------------------------
     template<class t_type, class t_storage>
+    bool kv_serialize(const std::deque<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+    {
+      return kv_serialization_overloads_impl_is_base_serializable_types<boost::mpl::contains<base_serializable_types<t_storage>, typename std::remove_const<t_type>::type>::value>::kv_serialize(d, stg, hparent_section, pname);
+    }
+    //-------------------------------------------------------------------------------------------------------------------
+    template<class t_type, class t_storage>
+    bool kv_unserialize(std::deque<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+    {
+      return kv_serialization_overloads_impl_is_base_serializable_types<boost::mpl::contains<base_serializable_types<t_storage>, typename std::remove_const<t_type>::type>::value>::kv_unserialize(d, stg, hparent_section, pname);
+    } 
+    //-------------------------------------------------------------------------------------------------------------------
+    template<class t_type, class t_storage>
     bool kv_serialize(const std::list<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
     {
       return kv_serialization_overloads_impl_is_base_serializable_types<boost::mpl::contains<base_serializable_types<t_storage>, typename std::remove_const<t_type>::type>::value>::kv_serialize(d, stg, hparent_section, pname);
@@ -364,5 +440,17 @@ namespace epee
     {
       return kv_serialization_overloads_impl_is_base_serializable_types<boost::mpl::contains<base_serializable_types<t_storage>, typename std::remove_const<t_type>::type>::value>::kv_unserialize(d, stg, hparent_section, pname);
     } 
+    //-------------------------------------------------------------------------------------------------------------------
+    template<class t_type, class t_storage>
+    bool kv_serialize(const std::set<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+    {
+      return kv_serialization_overloads_impl_is_base_serializable_types<boost::mpl::contains<base_serializable_types<t_storage>, typename std::remove_const<t_type>::type>::value>::kv_serialize(d, stg, hparent_section, pname);
+    }
+    //-------------------------------------------------------------------------------------------------------------------
+    template<class t_type, class t_storage>
+    bool kv_unserialize(std::set<t_type>& d, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname)
+    {
+      return kv_serialization_overloads_impl_is_base_serializable_types<boost::mpl::contains<base_serializable_types<t_storage>, typename std::remove_const<t_type>::type>::value>::kv_unserialize(d, stg, hparent_section, pname);
+    } 
   }
 }

src/contrib/epee/include/span.h

@@ -0,0 +1,175 @@
+// Copyright (c) 2017-2019, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <type_traits>
+
+namespace epee
+{
+  /*!
+    \brief Non-owning sequence of data. Does not deep copy
+
+    Inspired by `gsl::span` and/or `boost::iterator_range`. This class is
+    intended to be used as a parameter type for functions that need to take a
+    writable or read-only sequence of data. Most common cases are `span<char>`
+    and `span<std::uint8_t>`. Using as a class member is only recommended if
+    clearly documented as not doing a deep-copy. C-arrays are easily convertible
+    to this type.
+
+    \note Conversion from C string literal to `span<const char>` will include
+      the NULL-terminator.
+    \note Never allows derived-to-base pointer conversion; an array of derived
+      types is not an array of base types.
+   */
+  template<typename T>
+  class span
+  {
+    template<typename U>
+    static constexpr bool safe_conversion() noexcept
+    {
+      // Allow exact matches or `T*` -> `const T*`.
+      using with_const = typename std::add_const<U>::type;
+      return std::is_same<T, U>() ||
+        (std::is_const<T>() && std::is_same<T, with_const>());
+    }
+
+  public:
+    using value_type = T;
+    using size_type = std::size_t;
+    using difference_type = std::ptrdiff_t;
+    using pointer = T*;
+    using const_pointer = const T*;
+    using reference = T&;
+    using const_reference = const T&;
+    using iterator = pointer;
+    using const_iterator = const_pointer;
+
+    constexpr span() noexcept : ptr(nullptr), len(0) {}
+    constexpr span(std::nullptr_t) noexcept : span() {}
+
+    //! Prevent derived-to-base conversions; invalid in this context.
+    template<typename U, typename = typename std::enable_if<safe_conversion<U>()>::type>
+    constexpr span(U* const src_ptr, const std::size_t count) noexcept
+      : ptr(src_ptr), len(count) {}
+
+    //! Conversion from C-array. Prevents common bugs with sizeof + arrays.
+    template<std::size_t N>
+    constexpr span(T (&src)[N]) noexcept : span(src, N) {}
+
+    constexpr span(const span&) noexcept = default;
+    span& operator=(const span&) noexcept = default;
+
+    /*! Try to remove `amount` elements from beginning of span.
+    \return Number of elements removed. */
+    std::size_t remove_prefix(std::size_t amount) noexcept
+    {
+        amount = std::min(len, amount);
+        ptr += amount;
+        len -= amount;
+        return amount;
+    }
+
+    constexpr iterator begin() const noexcept { return ptr; }
+    constexpr const_iterator cbegin() const noexcept { return ptr; }
+
+    constexpr iterator end() const noexcept { return begin() + size(); }
+    constexpr const_iterator cend() const noexcept { return cbegin() + size(); }
+
+    constexpr bool empty() const noexcept { return size() == 0; }
+    constexpr pointer data() const noexcept { return ptr; }
+    constexpr std::size_t size() const noexcept { return len; }
+    constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); }
+
+    const T &operator[](size_t idx) const { return ptr[idx]; }
+
+  private:
+    T* ptr;
+    std::size_t len;
+  };
+
+  //! \return `span<const T::value_type>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<const typename T::value_type> to_span(const T& src)
+  {
+    // compiler provides diagnostic if size() is not size_t.
+    return {src.data(), src.size()};
+  }
+
+  //! \return `span<T::value_type>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<typename T::value_type> to_mut_span(T& src)
+  {
+    // compiler provides diagnostic if size() is not size_t.
+    return {src.data(), src.size()};
+  }
+
+  template<typename T>
+  constexpr bool has_padding() noexcept
+  {
+    return !std::is_standard_layout<T>() || alignof(T) != 1;
+  }
+
+  //! \return Cast data from `src` as `span<const std::uint8_t>`.
+  template<typename T>
+  span<const std::uint8_t> to_byte_span(const span<const T> src) noexcept
+  {
+    static_assert(!has_padding<T>(), "source type may have padding");
+    return {reinterpret_cast<const std::uint8_t*>(src.data()), src.size_bytes()}; 
+  }
+
+  //! \return `span<const std::uint8_t>` which represents the bytes at `&src`.
+  template<typename T>
+  span<const std::uint8_t> as_byte_span(const T& src) noexcept
+  {
+    static_assert(!std::is_empty<T>(), "empty types will not work -> sizeof == 1");
+    static_assert(!has_padding<T>(), "source type may have padding");
+    return {reinterpret_cast<const std::uint8_t*>(std::addressof(src)), sizeof(T)};
+  }
+
+  //! \return `span<std::uint8_t>` which represents the bytes at `&src`.
+  template<typename T>
+  span<std::uint8_t> as_mut_byte_span(T& src) noexcept
+  {
+    static_assert(!std::is_empty<T>(), "empty types will not work -> sizeof == 1");
+    static_assert(!has_padding<T>(), "source type may have padding");
+    return {reinterpret_cast<std::uint8_t*>(std::addressof(src)), sizeof(T)};
+  }
+
+  //! make a span from a std::string
+  template<typename T>
+  span<const T> strspan(const std::string &s) noexcept
+  {
+    static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected type");
+    return {reinterpret_cast<const T*>(s.data()), s.size()};
+  }
+}

src/contrib/epee/include/storages/crypted_storage.h

@@ -0,0 +1,62 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+#ifndef _CRYPTED_STORAGE_H_
+#define _CRYPTED_STORAGE_H_
+
+#include "cryptopp_helper.h"
+
+namespace epee
+{
+template<class t_base_storage, class crypt_provider, class t_key_provider>
+class crypted_storage: public t_base_storage
+{
+public: 
+	size_t	PackToSolidBuffer(std::string& targetObj)
+	{
+		size_t res = t_base_storage::PackToSolidBuffer(targetObj);
+		if(res <= 0)
+			return res;
+		
+		if(!crypt_provider::encrypt(targetObj, t_key_provider::get_storage_default_key()))
+			return 0;
+
+		return targetObj.size();
+	}
+
+	size_t		LoadFromSolidBuffer(const std::string& pTargetObj)
+	{
+		std::string buff_to_decrypt = pTargetObj;
+		if(crypt_provider::decrypt(buff_to_decrypt, t_key_provider::get_storage_default_key()))
+			return t_base_storage::LoadFromSolidBuffer(buff_to_decrypt);
+
+		return 0;
+	}
+};
+}
+
+#endif //_CRYPTED_STORAGE_H_

src/contrib/epee/include/storages/gzipped_inmemstorage.h

@@ -0,0 +1,68 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+#ifndef _GZIPPED_INMEMSTORAGE_H_
+#define _GZIPPED_INMEMSTORAGE_H_
+
+#include "zlib_helper.h"
+namespace epee
+{
+namespace StorageNamed
+{
+
+	template<class t_base_storage>
+	class gziped_storage: public t_base_storage
+	{
+	public: 
+		size_t	PackToSolidBuffer(std::string& targetObj)
+		{
+			size_t res = t_base_storage::PackToSolidBuffer(targetObj);
+			if(res <= 0)
+				return res;
+
+			if(!zlib_helper::pack(targetObj))
+				return 0;
+		
+			return targetObj.size();
+		}
+
+		size_t		LoadFromSolidBuffer(const std::string& pTargetObj)
+		{
+			std::string buff_to_ungzip = pTargetObj;
+			if(zlib_helper::unpack(buff_to_ungzip))
+				return t_base_storage::LoadFromSolidBuffer(buff_to_ungzip);
+
+			return 0;
+		}
+
+	private:
+	};
+
+}
+}
+
+#endif 

src/contrib/epee/include/storages/http_abstract_invoke.h

@@ -0,0 +1,132 @@
+
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+#pragma once
+#include <boost/utility/string_ref.hpp>
+#include <chrono>
+#include <string>
+#include "portable_storage_template_helper.h"
+#include "net/http_base.h"
+#include "net/http_server_handlers_map2.h"
+
+namespace epee
+{
+  namespace net_utils
+  {
+    template<class t_request, class t_response, class t_transport>
+    bool invoke_http_json(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET")
+    {
+      std::string req_param;
+      if(!serialization::store_t_to_json(out_struct, req_param))
+        return false;
+
+      http::fields_list additional_params;
+      additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8"));
+
+      const http::http_response_info* pri = NULL;
+      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri), std::move(additional_params)))
+      {
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri);
+        return false;
+      }
+
+      if(!pri)
+      {
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", internal error (null response ptr)");
+        return false;
+      }
+
+      if(pri->m_response_code != 200)
+      {
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", wrong response code: " << pri->m_response_code);
+        return false;
+      }
+
+      return serialization::load_t_from_json(result_struct, pri->m_body);
+    }
+
+
+
+    template<class t_request, class t_response, class t_transport>
+    bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "GET")
+    {
+      std::string req_param;
+      if(!serialization::store_t_to_binary(out_struct, req_param))
+        return false;
+
+      const http::http_response_info* pri = NULL;
+      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri)))
+      {
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri);
+        return false;
+      }
+
+      if(!pri)
+      {
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", internal error (null response ptr)");
+        return false;
+      }
+
+      if(pri->m_response_code != 200)
+      {
+        LOG_PRINT_L1("Failed to invoke http request to  " << uri << ", wrong response code: " << pri->m_response_code);
+        return false;
+      }
+
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
+    }
+
+    template<class t_request, class t_response, class t_transport>
+    bool invoke_http_json_rpc(const boost::string_ref uri, std::string method_name, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    {
+      epee::json_rpc::request<t_request> req_t = AUTO_VAL_INIT(req_t);
+      req_t.jsonrpc = "2.0";
+      req_t.id = req_id;
+      req_t.method = std::move(method_name);
+      req_t.params = out_struct;
+      epee::json_rpc::response<t_response, epee::json_rpc::error> resp_t = AUTO_VAL_INIT(resp_t);
+      if(!epee::net_utils::invoke_http_json(uri, req_t, resp_t, transport, timeout, http_method))
+      {
+        return false;
+      }
+      if(resp_t.error.code || resp_t.error.message.size())
+      {
+        LOG_ERROR("RPC call of \"" << req_t.method << "\" returned error: " << resp_t.error.code << ", message: " << resp_t.error.message);
+        return false;
+      }
+      result_struct = resp_t.result;
+      return true;
+    }
+
+    template<class t_command, class t_transport>
+    bool invoke_http_json_rpc(const boost::string_ref uri, typename t_command::request& out_struct, typename t_command::response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0")
+    {
+      return invoke_http_json_rpc(uri, t_command::methodname(), out_struct, result_struct, transport, timeout, http_method, req_id);
+    }
+
+  }
+}

src/contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -0,0 +1,303 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+#pragma once
+
+#include "portable_storage_template_helper.h"
+#include <boost/utility/value_init.hpp>
+#include "span.h"
+#include "net/levin_base.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net"
+
+namespace epee
+{
+  namespace net_utils
+  {
+    template<class t_arg, class t_result, class t_transport>
+    bool invoke_remote_command2(int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport)
+    {
+      if(!transport.is_connected())
+        return false;
+
+      serialization::portable_storage stg;
+      out_struct.store(stg);
+      std::string buff_to_send, buff_to_recv;
+      stg.store_to_binary(buff_to_send);
+
+      int res = transport.invoke(command, buff_to_send, buff_to_recv);
+      if( res <=0 )
+      {
+        MERROR("Failed to invoke command " << command << " return code " << res);
+        return false;
+      }
+      serialization::portable_storage stg_ret;
+      if(!stg_ret.load_from_binary(buff_to_recv))
+      {
+        LOG_ERROR("Failed to load_from_binary on command " << command);
+        return false;
+      }
+      return result_struct.load(stg_ret);
+    }
+
+    template<class t_arg, class t_transport>
+    bool notify_remote_command2(int command, const t_arg& out_struct, t_transport& transport)
+    {
+      if(!transport.is_connected())
+        return false;
+
+      serialization::portable_storage stg;
+      out_struct.store(&stg);
+      std::string buff_to_send;
+      stg.store_to_binary(buff_to_send);
+
+      int res = transport.notify(command, buff_to_send);
+      if(res <=0 )
+      {
+        LOG_ERROR("Failed to notify command " << command << " return code " << res);
+        return false;
+      }
+      return true;
+    }
+
+    template<class t_arg, class t_result, class t_transport>
+    bool invoke_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_result& result_struct, t_transport& transport)
+    {
+
+      typename serialization::portable_storage stg;
+      out_struct.store(stg);
+      std::string buff_to_send, buff_to_recv;
+      stg.store_to_binary(buff_to_send);
+
+      int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id);
+      if( res <=0 )
+      {
+        LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
+        return false;
+      }
+      typename serialization::portable_storage stg_ret;
+      if(!stg_ret.load_from_binary(buff_to_recv))
+      {
+        LOG_ERROR("Failed to load_from_binary on command " << command);
+        return false;
+      }
+      return result_struct.load(stg_ret);
+    }
+
+    template<class t_result, class t_arg, class callback_t, class t_transport>
+    bool async_invoke_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_transport& transport, const callback_t &cb, size_t inv_timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
+    {
+      typename serialization::portable_storage stg;
+      const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation
+      std::string buff_to_send;
+      stg.store_to_binary(buff_to_send);
+      int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool 
+      {
+        t_result result_struct = AUTO_VAL_INIT(result_struct);
+        if( code <=0 )
+        {
+          LOG_PRINT_L1("Failed to invoke command " << command << " return code " << code);
+          cb(code, result_struct, context);
+          return false;
+        }
+        serialization::portable_storage stg_ret;
+        if(!stg_ret.load_from_binary(buff))
+        {
+          LOG_ERROR("Failed to load_from_binary on command " << command);
+          cb(LEVIN_ERROR_FORMAT, result_struct, context);
+          return false;
+        }
+        if (!result_struct.load(stg_ret))
+        {
+          LOG_ERROR("Failed to load result struct on command " << command);
+          cb(LEVIN_ERROR_FORMAT, result_struct, context);
+          return false;
+        }
+        cb(code, result_struct, context);
+        return true;
+      }, inv_timeout);
+      if( res <=0 )
+      {
+        LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
+        return false;
+      }
+      return true;
+    }
+
+    template<class t_arg, class t_transport>
+    bool notify_remote_command2(boost::uuids::uuid conn_id, int command, const t_arg& out_struct, t_transport& transport)
+    {
+
+      serialization::portable_storage stg;
+      out_struct.store(stg);
+      std::string buff_to_send;
+      stg.store_to_binary(buff_to_send);
+
+      int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
+      if(res <=0 )
+      {
+        MERROR("Failed to notify command " << command << " return code " << res);
+        return false;
+      }
+      return true;
+    }
+    //----------------------------------------------------------------------------------------------------
+    //----------------------------------------------------------------------------------------------------
+    template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t>
+    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context )
+    {
+      serialization::portable_storage strg;
+      if(!strg.load_from_binary(in_buff))
+      {
+        LOG_ERROR("Failed to load_from_binary in command " << command);
+        return -1;
+      }
+      boost::value_initialized<t_in_type> in_struct;
+      boost::value_initialized<t_out_type> out_struct;
+
+      if (!static_cast<t_in_type&>(in_struct).load(strg))
+      {
+        LOG_ERROR("Failed to load in_struct in command " << command);
+        return -1;
+      }
+      int res = cb(command, static_cast<t_in_type&>(in_struct), static_cast<t_out_type&>(out_struct), context);
+      serialization::portable_storage strg_out;
+      static_cast<t_out_type&>(out_struct).store(strg_out);
+
+      if(!strg_out.store_to_binary(buff_out))
+      {
+        LOG_ERROR("Failed to store_to_binary in command" << command);
+        return -1;
+      }
+
+      return res;
+    }
+
+    template<class t_owner, class t_in_type, class t_context, class callback_t>
+    int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
+    {
+      serialization::portable_storage strg;
+      if(!strg.load_from_binary(in_buff))
+      {
+        LOG_ERROR("Failed to load_from_binary in notify " << command);
+        return -1;
+      }
+      boost::value_initialized<t_in_type> in_struct;
+      if (!static_cast<t_in_type&>(in_struct).load(strg))
+      {
+        LOG_ERROR("Failed to load in_struct in notify " << command);
+        return -1;
+      }
+      return cb(command, in_struct, context);
+    }
+
+#define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \
+  { \
+  bool handled = false; \
+  return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
+  } 
+
+#define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \
+  { \
+  bool handled = false; std::string fake_str;\
+  return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
+  } 
+
+
+#define CHAIN_LEVIN_INVOKE_MAP() \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
+  { \
+  bool handled = false; \
+  return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
+  } 
+
+#define CHAIN_LEVIN_NOTIFY_MAP() \
+  int notify(int command, const epee::span<const uint8_t> in_buff, epee::net_utils::connection_context_base& context) \
+  { \
+  bool handled = false; std::string fake_str;\
+  return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
+  } 
+
+#define CHAIN_LEVIN_NOTIFY_STUB() \
+  int notify(int command, const epee::span<const uint8_t> in_buff, epee::net_utils::connection_context_base& context) \
+  { \
+  return -1; \
+  } 
+
+#define BEGIN_INVOKE_MAP2(owner_type) \
+  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
+  { \
+  typedef owner_type internal_owner_type_name;
+
+#define HANDLE_INVOKE2(command_id, func, type_name_in, typename_out) \
+  if(!is_notify && command_id == command) \
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, type_name_in, typename_out>(this, command, in_buff, buff_out, boost::bind(func, this, _1, _2, _3, _4), context);}
+
+#define HANDLE_INVOKE_T2(COMMAND, func) \
+  if(!is_notify && COMMAND::ID == command) \
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, typename COMMAND::request, typename COMMAND::response>(command, in_buff, buff_out, boost::bind(func, this, _1, _2, _3, _4), context);}
+
+
+#define HANDLE_NOTIFY2(command_id, func, type_name_in) \
+  if(is_notify && command_id == command) \
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, type_name_in>(this, command, in_buff, boost::bind(func, this, _1, _2, _3), context);}
+
+#define HANDLE_NOTIFY_T2(NOTIFY, func) \
+  if(is_notify && NOTIFY::ID == command) \
+  {handled=true;return epee::net_utils::buff_to_t_adapter<internal_owner_type_name, typename NOTIFY::request>(this, command, in_buff, boost::bind(func, this, _1, _2, _3), context);}
+
+
+#define CHAIN_INVOKE_MAP2(func) \
+  { \
+  int res = func(is_notify, command, in_buff, buff_out, context, handled); \
+  if(handled) \
+  return res; \
+  }
+
+#define CHAIN_INVOKE_MAP_TO_OBJ2(obj) \
+  { \
+  int res = obj.handle_invoke_map(is_notify, command, in_buff, buff_out, context, handled); \
+  if(handled) \
+  return res; \
+  }
+
+#define CHAIN_INVOKE_MAP_TO_OBJ_FORCE_CONTEXT(obj, context_type) \
+  { \
+  int res = obj.handle_invoke_map(is_notify, command, in_buff, buff_out, static_cast<context_type>(context), handled); \
+  if(handled) return res; \
+  }
+
+
+#define END_INVOKE_MAP2() \
+  LOG_ERROR("Unknown command:" << command); \
+  return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \
+  }
+  }
+}
+

src/contrib/epee/include/storages/parserse_base_utils.h

@@ -0,0 +1,318 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+
+#pragma once 
+
+#include <algorithm>
+#include <boost/utility/string_ref.hpp>
+
+namespace epee 
+{
+namespace misc_utils
+{
+  namespace parse
+  {
+    // 1: digit
+    // 2: .eE (floating point)
+    // 4: alpha
+    // 8: whitespace
+    // 16: allowed in float but doesn't necessarily mean it's a float
+    // 32: \ and " (end of verbatim string)
+    static const constexpr uint8_t lut[256]={
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 0, 0, // 16
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 32
+      8, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 16, 18, 0, // 48
+      17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, // 64
+      0, 4, 4, 4, 4, 22, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 80
+      4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 32, 0, 0, 0, // 96
+      0, 4, 4, 4, 4, 22, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 112
+      4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 128
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    inline bool isspace(char c)
+    {
+      return lut[(uint8_t)c] & 8;
+    }
+
+    inline bool isdigit(char c)
+    {
+      return lut[(uint8_t)c] & 1;
+    }
+
+    inline std::string transform_to_escape_sequence(const std::string& src)
+    {
+      static const char escaped[] = "\b\f\n\r\t\v\"\\/";
+      std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped));
+      if (it == src.end())
+        return src;
+
+      std::string res;
+      res.reserve(2 * src.size());
+      res.assign(src.begin(), it);
+      for(; it!=src.end(); ++it)
+      {
+        switch(*it)
+        {
+        case '\b':  //Backspace (ascii code 08)
+          res+="\\b"; break;
+        case '\f':  //Form feed (ascii code 0C)
+          res+="\\f"; break;
+        case '\n':  //New line
+          res+="\\n"; break;
+        case '\r':  //Carriage return
+          res+="\\r"; break;
+        case '\t':  //Tab
+          res+="\\t"; break;
+        case '\v':  //Vertical tab
+          res+="\\v"; break;
+        //case '\'':  //Apostrophe or single quote
+        //  res+="\\'"; break;
+        case '"':  //Double quote
+          res+="\\\""; break;
+        case '\\':  //Backslash caracter
+          res+="\\\\"; break;
+        case '/':  //Backslash caracter
+          res+="\\/"; break;
+        default:
+          res.push_back(*it);
+        }
+      }
+      return res;
+    }
+    /*
+      
+      \b  Backspace (ascii code 08)
+      \f  Form feed (ascii code 0C)
+      \n  New line
+      \r  Carriage return
+      \t  Tab
+      \v  Vertical tab
+      \'  Apostrophe or single quote
+      \"  Double quote
+      \\  Backslash character
+
+      */
+      inline void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
+      {
+        bool escape_mode = false;
+        std::string::const_iterator it = star_end_string;
+        ++it;
+        std::string::const_iterator fi = it;
+        while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0)
+          ++fi;
+        val.assign(it, fi);
+        val.reserve(std::distance(star_end_string, buf_end));
+        it = fi;
+        for(;it != buf_end;it++)
+        {
+          if(escape_mode/*prev_ch == '\\'*/)
+          {
+            switch(*it)
+            {
+            case 'b':  //Backspace (ascii code 08)
+              val.push_back(0x08);break;
+            case 'f':  //Form feed (ascii code 0C)
+              val.push_back(0x0C);break;
+            case 'n':  //New line
+              val.push_back('\n');break;
+            case 'r':  //Carriage return
+              val.push_back('\r');break;
+            case 't':  //Tab
+              val.push_back('\t');break;
+            case 'v':  //Vertical tab
+              val.push_back('\v');break;
+            case '\'':  //Apostrophe or single quote
+              val.push_back('\'');break;
+            case '"':  //Double quote
+              val.push_back('"');break;
+            case '\\':  //Backslash character
+              val.push_back('\\');break;
+            case '/':  //Slash character
+              val.push_back('/');break;
+            default:
+              val.push_back(*it);
+              LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\"");
+            }
+            escape_mode = false;
+          }else if(*it == '"')
+          {
+            star_end_string = it;
+            return;
+          }else if(*it == '\\')
+          {
+            escape_mode = true;
+          }          
+          else
+          {
+            val.push_back(*it); 
+          }
+        }
+        ASSERT_MES_AND_THROW("Failed to match string in json entry: " << std::string(star_end_string, buf_end));
+      }
+      inline bool match_string(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
+      {
+        try
+        {
+
+          match_string2(star_end_string, buf_end, val);
+          return true;
+        }
+        catch(...)
+        {
+          return false;
+        }
+      }
+      inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val)
+      {
+        val.clear();
+        uint8_t float_flag = 0;
+        is_signed_val = false;
+        size_t chars = 0;
+        std::string::const_iterator it = star_end_string;
+        if (it != buf_end && *it == '-')
+        {
+          is_signed_val = true;
+          ++chars;
+          ++it;
+        }
+        for(;it != buf_end;it++)
+        {
+          const uint8_t flags = lut[(uint8_t)*it];
+          if (flags & 16)
+          {
+            float_flag |= flags;
+            ++chars;
+          }
+          else
+          {
+            val = boost::string_ref(&*star_end_string, chars);
+            if(val.size())
+            {
+              star_end_string = --it;
+              is_float_val = !!(float_flag & 2);
+              return;
+            }
+            else 
+              ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end));
+          }
+        }
+        ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end));
+      }
+      inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val)
+      {
+        try
+        {
+          bool is_v_float = false;bool is_signed_val = false;
+          match_number2(star_end_string, buf_end, val, is_v_float, is_signed_val);
+          return !is_v_float;
+        }
+        catch(...)
+        {
+          return false;
+        }
+      }
+      inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val)
+      {
+        val.clear();
+
+        for(std::string::const_iterator it = star_end_string;it != buf_end;it++)
+        {
+          if (!(lut[(uint8_t)*it] & 4))
+          {
+            val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it));
+            if(val.size())
+            {
+              star_end_string = --it;
+              return;
+            }else 
+              ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end));
+          }
+        }
+        ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end));
+      }
+      inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val)
+      {
+        try
+        {
+          match_word2(star_end_string, buf_end, val);
+          return true;
+        }
+        catch(...)
+        {
+          return false;
+        }
+      }
+      inline bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
+      {
+        val.clear();
+
+        for(std::string::const_iterator it = star_end_string;it != buf_end;it++)
+        {
+          if(!isalnum(*it) && *it != '-' && *it != '_')
+          {
+            val.assign(star_end_string, it);
+            if(val.size())
+            {
+              star_end_string = --it;
+              return true;
+            }else 
+              return false;
+          }
+        }
+        return false;
+      }
+      inline bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end)
+      {
+        word_end = star_end_string;
+
+        for(std::string::const_iterator it = star_end_string;it != buf_end;it++)
+        {
+          if(isspace(*it))
+          {
+
+            continue;
+          }else if( *it == '=' )
+          {            
+            star_end_string = it;
+            word_end = it;
+            return true;
+          }
+        }
+        return false;
+      }
+  }
+}
+}

src/contrib/epee/include/storages/portable_storage.h

@@ -0,0 +1,483 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+
+#pragma once 
+
+#include "misc_language.h"
+#include "portable_storage_base.h"
+#include "portable_storage_to_bin.h"
+#include "portable_storage_from_bin.h"
+#include "portable_storage_to_json.h"
+#include "portable_storage_from_json.h"
+#include "portable_storage_val_converters.h"
+#include "span.h"
+#include "int-util.h"
+#include <boost/mpl/contains.hpp>
+
+namespace epee
+{
+  namespace serialization
+  {
+    /************************************************************************/
+    /*                                                                      */
+    /************************************************************************/
+    class portable_storage
+    {
+    public:
+      typedef epee::serialization::hsection hsection;
+      typedef epee::serialization::harray  harray;
+      typedef storage_entry meta_entry;
+
+      portable_storage(){}
+      virtual ~portable_storage(){}
+      hsection   open_section(const std::string& section_name,  hsection hparent_section, bool create_if_notexist = false);
+      template<class t_value>
+      bool       get_value(const std::string& value_name, t_value& val, hsection hparent_section);
+      bool       get_value(const std::string& value_name, storage_entry& val, hsection hparent_section);
+      template<class t_value>
+      bool       set_value(const std::string& value_name, const t_value& target, hsection hparent_section);
+
+      //serial access for arrays of values --------------------------------------
+      //values
+      template<class t_value>
+      harray get_first_value(const std::string& value_name, t_value& target, hsection hparent_section);
+      template<class t_value>
+      bool          get_next_value(harray hval_array, t_value& target);
+      template<class t_value>
+      harray insert_first_value(const std::string& value_name, const t_value& target, hsection hparent_section);
+      template<class t_value>
+      bool          insert_next_value(harray hval_array, const t_value& target);
+      //sections
+      harray get_first_section(const std::string& pSectionName, hsection& h_child_section, hsection hparent_section);
+      bool            get_next_section(harray hSecArray, hsection& h_child_section);
+      harray insert_first_section(const std::string& pSectionName, hsection& hinserted_childsection, hsection hparent_section);
+      bool            insert_next_section(harray hSecArray, hsection& hinserted_childsection);
+      //------------------------------------------------------------------------
+      //delete entry (section, value or array)
+      bool        delete_entry(const std::string& pentry_name, hsection hparent_section = nullptr);
+
+      //-------------------------------------------------------------------------------
+      bool		store_to_binary(binarybuffer& target);
+      bool		load_from_binary(const epee::span<const uint8_t> target);
+      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
+      template<class trace_policy>
+      bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
+      bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
+      bool		  load_from_json(const std::string& source);
+
+    private:
+      section m_root;
+      hsection	get_root_section() {return &m_root;}
+      storage_entry* find_storage_entry(const std::string& pentry_name, hsection psection);
+      template<class entry_type>
+      storage_entry* insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, const entry_type& entry);
+
+      hsection    insert_new_section(const std::string& pentry_name, hsection psection);
+
+#pragma pack(push)
+#pragma pack(1)
+      struct storage_block_header
+      {
+        uint32_t m_signature_a;
+        uint32_t m_signature_b;
+        uint8_t  m_ver;
+      };
+#pragma pack(pop)
+    };
+    inline
+    bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines)
+    {
+      TRY_ENTRY();
+      std::stringstream ss;
+      epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines);
+      buff = ss.str();
+      return true;
+      CATCH_ENTRY("portable_storage::dump_as_json", false)
+    }
+    inline
+    bool portable_storage::load_from_json(const std::string& source)
+    {
+      TRY_ENTRY();
+      return json::load_from_json(source, *this);
+      CATCH_ENTRY("portable_storage::load_from_json", false)
+    }
+
+    template<class trace_policy>
+    bool portable_storage::dump_as_xml(std::string& targetObj, const std::string& root_name)
+    {
+      return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
+    }
+
+    inline
+    bool portable_storage::store_to_binary(binarybuffer& target)
+    {
+      TRY_ENTRY();
+      std::stringstream ss;
+      storage_block_header sbh = AUTO_VAL_INIT(sbh);
+      sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
+      sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
+      sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
+      ss.write((const char*)&sbh, sizeof(storage_block_header));
+      pack_entry_to_buff(ss, m_root);
+      target = ss.str();
+      return true;
+      CATCH_ENTRY("portable_storage::store_to_binary", false)
+    }
+    inline
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
+    {
+      m_root.m_entries.clear();
+      if(source.size() < sizeof(storage_block_header))
+      {
+        LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header));
+        return false;
+      }
+      storage_block_header* pbuff = (storage_block_header*)source.data();
+      if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) ||
+        pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB)
+        )
+      {
+        LOG_ERROR("portable_storage: wrong binary format - signature mismatch");
+        return false;
+      }
+      if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER)
+      {
+        LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver);
+        return false;
+      }
+      TRY_ENTRY();
+      throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header));
+      buf_reader.read(m_root);
+      return true;//TODO:
+      CATCH_ENTRY("portable_storage::load_from_binary", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    hsection portable_storage::open_section(const std::string& section_name,  hsection hparent_section, bool create_if_notexist)
+    {
+      TRY_ENTRY();
+      hparent_section = hparent_section ? hparent_section:&m_root;
+      storage_entry* pentry = find_storage_entry(section_name, hparent_section);
+      if(!pentry)
+      {
+        if(!create_if_notexist)
+          return nullptr;
+        return insert_new_section(section_name, hparent_section);
+      }
+      CHECK_AND_ASSERT(pentry , nullptr);
+      //check that section_entry we find is real "CSSection"
+      if(pentry->type() != typeid(section))
+      {
+        if(create_if_notexist)
+          *pentry = storage_entry(section());//replace
+        else
+          return nullptr;
+      }
+      return &boost::get<section>(*pentry);
+      CATCH_ENTRY("portable_storage::open_section", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    template<class to_type>
+    struct get_value_visitor: boost::static_visitor<void>
+    {
+      to_type& m_target;
+      get_value_visitor(to_type& target):m_target(target){}
+      template<class from_type>
+      void operator()(const from_type& v){convert_t(v, m_target);}
+    };
+
+    template<class t_value>
+    bool portable_storage::get_value(const std::string& value_name, t_value& val, hsection hparent_section)
+    {
+      BOOST_MPL_ASSERT(( boost::mpl::contains<storage_entry::types, t_value> )); 
+      //TRY_ENTRY();
+      if(!hparent_section) hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(value_name, hparent_section);
+      if(!pentry)
+        return false;
+
+      get_value_visitor<t_value> gvv(val);
+      boost::apply_visitor(gvv, *pentry);
+      return true;
+      //CATCH_ENTRY("portable_storage::template<>get_value", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section)
+    {
+      //TRY_ENTRY();
+      if(!hparent_section) hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(value_name, hparent_section);
+      if(!pentry)
+        return false;
+
+      val = *pentry;
+      return true;
+      //CATCH_ENTRY("portable_storage::template<>get_value", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    template<class t_value>
+    bool portable_storage::set_value(const std::string& value_name, const t_value& v, hsection hparent_section)        
+    {
+      BOOST_MPL_ASSERT(( boost::mpl::contains<boost::mpl::push_front<storage_entry::types, storage_entry>::type, t_value> )); 
+      TRY_ENTRY();
+      if(!hparent_section)
+        hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(value_name, hparent_section);
+      if(!pentry)
+      {
+        pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, v);
+        if(!pentry)
+          return false;
+        return true;
+      }
+      *pentry = storage_entry(v);
+      return true;
+      CATCH_ENTRY("portable_storage::template<>set_value", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection)
+    {
+      TRY_ENTRY();
+      CHECK_AND_ASSERT(psection, nullptr);
+      auto it = psection->m_entries.find(pentry_name);
+      if(it == psection->m_entries.end())
+        return nullptr;
+
+      return &it->second;
+      CATCH_ENTRY("portable_storage::find_storage_entry", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    template<class entry_type>
+    storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, const entry_type& entry)
+    {
+      TRY_ENTRY();
+      CHECK_AND_ASSERT(psection, nullptr);
+      auto ins_res = psection->m_entries.insert(std::pair<std::string, storage_entry>(pentry_name, entry));
+      return &ins_res.first->second;
+      CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection)
+    {
+      TRY_ENTRY();
+      storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section());
+      if(!pse) return nullptr;
+      return &boost::get<section>(*pse);
+      CATCH_ENTRY("portable_storage::insert_new_section", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    template<class to_type>
+    struct get_first_value_visitor: boost::static_visitor<bool>
+    {
+      to_type& m_target;
+      get_first_value_visitor(to_type& target):m_target(target){}
+      template<class from_type>
+      bool operator()(const array_entry_t<from_type>& a)
+      {
+        const from_type* pv = a.get_first_val();
+        if(!pv)
+          return false;
+        convert_t(*pv, m_target);
+        return true;
+      }
+    };
+    //---------------------------------------------------------------------------------------------------------------
+    template<class t_value>
+    harray portable_storage::get_first_value(const std::string& value_name, t_value& target, hsection hparent_section)
+    {
+      BOOST_MPL_ASSERT(( boost::mpl::contains<storage_entry::types, t_value> )); 
+      //TRY_ENTRY();
+      if(!hparent_section) hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(value_name, hparent_section);
+      if(!pentry)
+        return nullptr;
+      if(pentry->type() != typeid(array_entry))
+        return nullptr;
+      array_entry& ar_entry = boost::get<array_entry>(*pentry);
+      
+      get_first_value_visitor<t_value> gfv(target);
+      if(!boost::apply_visitor(gfv, ar_entry))
+        return nullptr;
+      return &ar_entry;
+      //CATCH_ENTRY("portable_storage::get_first_value", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    template<class to_type>
+    struct get_next_value_visitor: boost::static_visitor<bool>
+    {
+      to_type& m_target;
+      get_next_value_visitor(to_type& target):m_target(target){}
+      template<class from_type>
+      bool operator()(const array_entry_t<from_type>& a)
+      {
+        //TODO: optimize code here: work without get_next_val function
+        const from_type* pv = a.get_next_val();
+        if(!pv)
+          return false;
+        convert_t(*pv, m_target);
+        return true;
+      }
+    };
+
+
+    template<class t_value>
+    bool portable_storage::get_next_value(harray hval_array, t_value& target)
+    {
+      BOOST_MPL_ASSERT(( boost::mpl::contains<storage_entry::types, t_value> )); 
+      //TRY_ENTRY();
+      CHECK_AND_ASSERT(hval_array, false);
+      array_entry& ar_entry = *hval_array;
+      get_next_value_visitor<t_value> gnv(target);
+      if(!boost::apply_visitor(gnv, ar_entry))
+        return false;
+      return true;
+      //CATCH_ENTRY("portable_storage::get_next_value", false);
+    } 
+    //---------------------------------------------------------------------------------------------------------------
+    template<class t_value>
+    harray portable_storage::insert_first_value(const std::string& value_name, const t_value& target, hsection hparent_section)
+    {
+      TRY_ENTRY();
+      if(!hparent_section) hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(value_name, hparent_section);
+      if(!pentry)
+      {
+        pentry = insert_new_entry_get_storage_entry(value_name, hparent_section, array_entry(array_entry_t<t_value>()));
+        if(!pentry)
+          return nullptr;
+      }
+      if(pentry->type() != typeid(array_entry))
+        *pentry = storage_entry(array_entry(array_entry_t<t_value>()));
+
+      array_entry& arr = boost::get<array_entry>(*pentry);
+      if(arr.type() != typeid(array_entry_t<t_value>))
+        arr = array_entry(array_entry_t<t_value>());
+
+      array_entry_t<t_value>& arr_typed = boost::get<array_entry_t<t_value> >(arr);
+      arr_typed.insert_first_val(target);
+      return &arr;
+      CATCH_ENTRY("portable_storage::insert_first_value", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    template<class t_value>
+    bool portable_storage::insert_next_value(harray hval_array, const t_value& target)
+    {
+      TRY_ENTRY();
+      CHECK_AND_ASSERT(hval_array, false);
+
+      CHECK_AND_ASSERT_MES(hval_array->type() == typeid(array_entry_t<t_value>), 
+        false, "unexpected type in insert_next_value: " << typeid(array_entry_t<t_value>).name());
+
+      array_entry_t<t_value>& arr_typed = boost::get<array_entry_t<t_value> >(*hval_array);
+      arr_typed.insert_next_value(target);
+      return true;
+      CATCH_ENTRY("portable_storage::insert_next_value", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    //sections
+    inline
+    harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section)
+    {
+      TRY_ENTRY();
+      if(!hparent_section) hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(sec_name, hparent_section);
+      if(!pentry)
+        return nullptr;
+      if(pentry->type() != typeid(array_entry))
+        return nullptr;
+      array_entry& ar_entry = boost::get<array_entry>(*pentry);
+      if(ar_entry.type() != typeid(array_entry_t<section>))
+        return nullptr;
+      array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry);
+      section* psec = sec_array.get_first_val();
+      if(!psec)
+        return nullptr;
+      h_child_section = psec;
+      return &ar_entry;
+      CATCH_ENTRY("portable_storage::get_first_section", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section)
+    {
+      TRY_ENTRY();
+      CHECK_AND_ASSERT(hsec_array, false);
+      if(hsec_array->type() != typeid(array_entry_t<section>))
+        return false;
+      array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array);
+      h_child_section = sec_array.get_next_val();
+      if(!h_child_section)
+        return false;
+      return true;
+      CATCH_ENTRY("portable_storage::get_next_section", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section)
+    {
+      TRY_ENTRY();
+      if(!hparent_section) hparent_section = &m_root;
+      storage_entry* pentry = find_storage_entry(sec_name, hparent_section);
+      if(!pentry)
+      {
+        pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>()));
+        if(!pentry)
+          return nullptr;
+      }
+      if(pentry->type() != typeid(array_entry))
+        *pentry = storage_entry(array_entry(array_entry_t<section>()));
+
+      array_entry& ar_entry = boost::get<array_entry>(*pentry);
+      if(ar_entry.type() != typeid(array_entry_t<section>))
+        ar_entry = array_entry(array_entry_t<section>());
+
+      array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry);
+      hinserted_childsection = &sec_array.insert_first_val(section());
+      return &ar_entry;
+      CATCH_ENTRY("portable_storage::insert_first_section", nullptr);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+    inline
+    bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection)
+    {
+      TRY_ENTRY();
+      CHECK_AND_ASSERT(hsec_array, false);
+      CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), 
+        false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name());
+
+      array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array);
+      hinserted_childsection = &sec_array.insert_next_value(section());
+      return true;
+      CATCH_ENTRY("portable_storage::insert_next_section", false);
+    }
+    //---------------------------------------------------------------------------------------------------------------
+  }
+}

src/contrib/epee/include/storages/portable_storage_base.h

@@ -0,0 +1,170 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+
+#pragma once 
+
+#include <boost/variant.hpp>
+#include <boost/any.hpp>
+#include <string>
+#include <vector>
+#include <deque>
+
+#define PORTABLE_STORAGE_SIGNATUREA 0x01011101
+#define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare 
+#define PORTABLE_STORAGE_FORMAT_VER 1
+
+#define PORTABLE_RAW_SIZE_MARK_MASK   0x03 
+#define PORTABLE_RAW_SIZE_MARK_BYTE   0
+#define PORTABLE_RAW_SIZE_MARK_WORD   1
+#define PORTABLE_RAW_SIZE_MARK_DWORD  2
+#define PORTABLE_RAW_SIZE_MARK_INT64  3
+
+#ifndef MAX_STRING_LEN_POSSIBLE       
+#define MAX_STRING_LEN_POSSIBLE       2000000000 //do not let string be so big
+#endif
+
+//data types 
+#define SERIALIZE_TYPE_INT64                1
+#define SERIALIZE_TYPE_INT32                2
+#define SERIALIZE_TYPE_INT16                3
+#define SERIALIZE_TYPE_INT8                 4
+#define SERIALIZE_TYPE_UINT64               5
+#define SERIALIZE_TYPE_UINT32               6
+#define SERIALIZE_TYPE_UINT16               7
+#define SERIALIZE_TYPE_UINT8                8
+#define SERIALIZE_TYPE_DUOBLE               9
+#define SERIALIZE_TYPE_STRING               10
+#define SERIALIZE_TYPE_BOOL                 11
+#define SERIALIZE_TYPE_OBJECT               12
+#define SERIALIZE_TYPE_ARRAY                13
+
+#define SERIALIZE_FLAG_ARRAY              0x80
+
+
+namespace epee
+{
+  namespace serialization
+  {
+    struct section;
+
+    template<typename T> struct entry_container { typedef std::vector<T> type; static void reserve(type &t, size_t n) { t.reserve(n); } };
+    template<> struct entry_container<bool> { typedef std::deque<bool> type; static void reserve(type &t, size_t n) {} };
+
+    /************************************************************************/
+    /*                                                                      */
+    /************************************************************************/
+    template<class t_entry_type>
+    struct array_entry_t
+    {
+      array_entry_t():m_it(m_array.end()){}        
+      array_entry_t(const array_entry_t& other):m_array(other.m_array), m_it(m_array.end()){}
+
+      const t_entry_type* get_first_val() const 
+      {
+        m_it = m_array.begin();
+        return get_next_val();
+      }
+
+      t_entry_type* get_first_val() 
+      {
+        m_it = m_array.begin();
+        return get_next_val();
+      }
+
+
+      const t_entry_type* get_next_val() const 
+      {
+        if(m_it == m_array.end())
+          return nullptr;
+        return &(*(m_it++));
+      }
+
+      t_entry_type* get_next_val() 
+      {
+        if(m_it == m_array.end())
+          return nullptr;
+        return (t_entry_type*)&(*(m_it++));//fuckoff
+      }
+
+      t_entry_type& insert_first_val(const t_entry_type& v)
+      {
+        m_array.clear();
+        m_it = m_array.end();
+        return insert_next_value(v);
+      }
+
+      t_entry_type& insert_next_value(const t_entry_type& v)
+      {
+        m_array.push_back(v);
+        return m_array.back();
+      }
+
+      void reserve(size_t n)
+      {
+        entry_container<t_entry_type>::reserve(m_array, n);
+      }
+
+      typename entry_container<t_entry_type>::type m_array;
+      mutable typename entry_container<t_entry_type>::type::const_iterator m_it;
+    };
+
+
+    typedef  boost::make_recursive_variant<
+      array_entry_t<section>, 
+      array_entry_t<uint64_t>, 
+      array_entry_t<uint32_t>, 
+      array_entry_t<uint16_t>, 
+      array_entry_t<uint8_t>, 
+      array_entry_t<int64_t>, 
+      array_entry_t<int32_t>, 
+      array_entry_t<int16_t>, 
+      array_entry_t<int8_t>, 
+      array_entry_t<double>, 
+      array_entry_t<bool>, 
+      array_entry_t<std::string>,
+      array_entry_t<section>, 
+      array_entry_t<boost::recursive_variant_> 
+    >::type array_entry;
+
+    typedef boost::variant<uint64_t, uint32_t, uint16_t, uint8_t, int64_t, int32_t, int16_t, int8_t, double, bool, std::string, section, array_entry> storage_entry;
+
+    typedef std::string binarybuffer;//it's ok      
+
+    /************************************************************************/
+    /*                                                                      */
+    /************************************************************************/
+    struct section
+    {
+      std::map<std::string, storage_entry> m_entries;
+    };
+
+    //handle-like aliases
+    typedef section*      hsection;  
+    typedef array_entry*  harray;
+  }
+}

src/contrib/epee/include/storages/portable_storage_bin_utils.h

@@ -0,0 +1,46 @@
+// Copyright (c) 2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once 
+
+#include "int-util.h"
+
+template<typename T> T convert_swapper(T t) { return t; }
+template<> inline uint16_t convert_swapper(uint16_t t) { return SWAP16LE(t); }
+template<> inline int16_t convert_swapper(int16_t t) { return SWAP16LE((uint16_t&)t); }
+template<> inline uint32_t convert_swapper(uint32_t t) { return SWAP32LE(t); }
+template<> inline int32_t convert_swapper(int32_t t) { return SWAP32LE((uint32_t&)t); }
+template<> inline uint64_t convert_swapper(uint64_t t) { return SWAP64LE(t); }
+template<> inline int64_t convert_swapper(int64_t t) { return SWAP64LE((uint64_t&)t); }
+template<> inline double convert_swapper(double t) { union { uint64_t u; double d; } u; u.d = t; u.u = SWAP64LE(u.u); return u.d; }
+
+#if BYTE_ORDER == BIG_ENDIAN
+#define CONVERT_POD(x) convert_swapper(x)
+#else
+#define CONVERT_POD(x) (x)
+#endif

src/contrib/epee/include/storages/portable_storage_from_bin.h

@@ -0,0 +1,291 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+
+#pragma once 
+
+#include "misc_language.h"
+#include "portable_storage_base.h"
+
+#ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
+#define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
+#else 
+#define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL 100
+#endif
+
+namespace epee
+{
+  namespace serialization
+  {
+    struct throwable_buffer_reader
+    {
+      throwable_buffer_reader(const void* ptr, size_t sz);
+      void read(void* target, size_t count);
+      void read_sec_name(std::string& sce_name);
+      template<class t_pod_type>
+      void read(t_pod_type& pod_val);
+      template<class t_type>
+      t_type read();
+      template<class type_name>
+      storage_entry read_ae();
+      storage_entry load_storage_array_entry(uint8_t type);
+      size_t read_varint();
+      template<class t_type>
+      storage_entry read_se();
+      storage_entry load_storage_entry();
+      void read(section& sec);
+      void read(std::string& str);
+      void read(array_entry &ae);
+    private:
+      struct recursuion_limitation_guard
+      {
+        size_t& m_counter_ref;
+        recursuion_limitation_guard(size_t& counter):m_counter_ref(counter)
+        {
+          ++m_counter_ref;
+          CHECK_AND_ASSERT_THROW_MES(m_counter_ref < EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL, "Wrong blob data in portable storage: recursion limitation (" << EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL << ") exceeded");
+        }
+        ~recursuion_limitation_guard() noexcept(false)
+        {
+          CHECK_AND_ASSERT_THROW_MES(m_counter_ref != 0, "Internal error: m_counter_ref == 0 while ~recursuion_limitation_guard()");
+          --m_counter_ref;
+        }
+      };
+#define RECURSION_LIMITATION()  recursuion_limitation_guard rl(m_recursion_count)
+
+      const uint8_t* m_ptr;
+      size_t m_count;
+      size_t m_recursion_count;
+    };
+
+    inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz)
+    {
+      if(!ptr) 
+        throw std::runtime_error("throwable_buffer_reader: ptr==nullptr");
+      if(!sz)
+        throw std::runtime_error("throwable_buffer_reader: sz==0");
+      m_ptr = (uint8_t*)ptr;
+      m_count = sz;
+      m_recursion_count = 0;
+    }
+    inline 
+    void throwable_buffer_reader::read(void* target, size_t count)
+    {
+      RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_count >= count, " attempt to read " << count << " bytes from buffer with " << m_count << " bytes remained");
+      memcpy(target, m_ptr, count);
+      m_ptr += count;
+      m_count -= count;
+    }
+    inline 
+    void throwable_buffer_reader::read_sec_name(std::string& sce_name)
+    {
+      RECURSION_LIMITATION();
+      uint8_t name_len = 0;
+      read(name_len);
+      sce_name.resize(name_len);
+      read((void*)sce_name.data(), name_len);
+    }
+
+    template<class t_pod_type>
+    void throwable_buffer_reader::read(t_pod_type& pod_val)
+    {
+      RECURSION_LIMITATION();
+      static_assert(std::is_pod<t_pod_type>::value, "POD type expected");
+      read(&pod_val, sizeof(pod_val));
+    }
+    
+    template<class t_type>
+    t_type throwable_buffer_reader::read()
+    {
+      RECURSION_LIMITATION();
+      t_type v;
+      read(v);
+      return v;
+    }
+
+
+    template<class type_name>
+    storage_entry throwable_buffer_reader::read_ae()
+    {
+      RECURSION_LIMITATION();
+      //for pod types
+      array_entry_t<type_name> sa;
+      size_t size = read_varint();
+      CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed");
+      sa.reserve(size);
+      //TODO: add some optimization here later
+      while(size--)
+        sa.m_array.push_back(read<type_name>());        
+      return storage_entry(array_entry(sa));
+    }
+
+    inline 
+    storage_entry throwable_buffer_reader::load_storage_array_entry(uint8_t type)
+    {
+      RECURSION_LIMITATION();
+      type &= ~SERIALIZE_FLAG_ARRAY;
+      switch(type)
+      {
+      case SERIALIZE_TYPE_INT64:  return read_ae<int64_t>();
+      case SERIALIZE_TYPE_INT32:  return read_ae<int32_t>();
+      case SERIALIZE_TYPE_INT16:  return read_ae<int16_t>();
+      case SERIALIZE_TYPE_INT8:   return read_ae<int8_t>();
+      case SERIALIZE_TYPE_UINT64: return read_ae<uint64_t>();
+      case SERIALIZE_TYPE_UINT32: return read_ae<uint32_t>();
+      case SERIALIZE_TYPE_UINT16: return read_ae<uint16_t>();
+      case SERIALIZE_TYPE_UINT8:  return read_ae<uint8_t>();
+      case SERIALIZE_TYPE_DUOBLE: return read_ae<double>();
+      case SERIALIZE_TYPE_BOOL:   return read_ae<bool>();
+      case SERIALIZE_TYPE_STRING: return read_ae<std::string>();
+      case SERIALIZE_TYPE_OBJECT: return read_ae<section>();
+      case SERIALIZE_TYPE_ARRAY:  return read_ae<array_entry>();
+      default: 
+        CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << type);
+      }
+    }
+
+    inline 
+    size_t throwable_buffer_reader::read_varint()
+    {
+      RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_count >= 1, "empty buff, expected place for varint");
+      size_t v = 0;
+      uint8_t size_mask = (*(uint8_t*)m_ptr) &PORTABLE_RAW_SIZE_MARK_MASK;
+      switch (size_mask)
+      {
+      case PORTABLE_RAW_SIZE_MARK_BYTE: v = read<uint8_t>();break;
+      case PORTABLE_RAW_SIZE_MARK_WORD: v = read<uint16_t>();break;
+      case PORTABLE_RAW_SIZE_MARK_DWORD: v = read<uint32_t>();break;
+      case PORTABLE_RAW_SIZE_MARK_INT64: v = read<uint64_t>();break;
+      default:
+        CHECK_AND_ASSERT_THROW_MES(false, "unknown varint size_mask = " << size_mask);
+      }
+      v >>= 2;
+      return v;
+    }
+
+    template<class t_type>
+    storage_entry throwable_buffer_reader::read_se()
+    {
+      RECURSION_LIMITATION();
+      t_type v;
+      read(v);
+      return storage_entry(v);
+    }
+
+    template<>
+    inline storage_entry throwable_buffer_reader::read_se<std::string>()
+    {
+      RECURSION_LIMITATION();
+      return storage_entry(read<std::string>());
+    }
+
+
+    template<>
+    inline storage_entry throwable_buffer_reader::read_se<section>()
+    {
+      RECURSION_LIMITATION();
+      section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
+      storage_entry se(s);
+      section& section_entry = boost::get<section>(se);
+      read(section_entry);
+      return se;
+    }
+
+    template<>
+    inline storage_entry throwable_buffer_reader::read_se<array_entry>()
+    {
+      RECURSION_LIMITATION();
+      uint8_t ent_type = 0;
+      read(ent_type);
+      CHECK_AND_ASSERT_THROW_MES(ent_type&SERIALIZE_FLAG_ARRAY, "wrong type sequenses");
+      return load_storage_array_entry(ent_type);
+    }
+
+    inline 
+    storage_entry throwable_buffer_reader::load_storage_entry()
+    {
+      RECURSION_LIMITATION();
+      uint8_t ent_type = 0;
+      read(ent_type);
+      if(ent_type&SERIALIZE_FLAG_ARRAY)
+        return load_storage_array_entry(ent_type);
+
+      switch(ent_type)
+      {
+      case SERIALIZE_TYPE_INT64:  return read_se<int64_t>();
+      case SERIALIZE_TYPE_INT32:  return read_se<int32_t>();
+      case SERIALIZE_TYPE_INT16:  return read_se<int16_t>();
+      case SERIALIZE_TYPE_INT8:   return read_se<int8_t>();
+      case SERIALIZE_TYPE_UINT64: return read_se<uint64_t>();
+      case SERIALIZE_TYPE_UINT32: return read_se<uint32_t>();
+      case SERIALIZE_TYPE_UINT16: return read_se<uint16_t>();
+      case SERIALIZE_TYPE_UINT8:  return read_se<uint8_t>();
+      case SERIALIZE_TYPE_DUOBLE: return read_se<double>();
+      case SERIALIZE_TYPE_BOOL:   return read_se<bool>();
+      case SERIALIZE_TYPE_STRING: return read_se<std::string>();
+      case SERIALIZE_TYPE_OBJECT: return read_se<section>();
+      case SERIALIZE_TYPE_ARRAY:  return read_se<array_entry>();
+      default: 
+        CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << ent_type);
+      }
+    }
+    inline 
+    void throwable_buffer_reader::read(section& sec)
+    {
+      RECURSION_LIMITATION();
+      sec.m_entries.clear();
+      size_t count = read_varint();
+      while(count--)
+      {
+        //read section name string
+        std::string sec_name;
+        read_sec_name(sec_name);
+        sec.m_entries.insert(std::make_pair(sec_name, load_storage_entry()));
+      }
+    }
+    inline 
+    void throwable_buffer_reader::read(std::string& str)
+    {
+      RECURSION_LIMITATION();
+      size_t len = read_varint();
+      CHECK_AND_ASSERT_THROW_MES(len < MAX_STRING_LEN_POSSIBLE, "to big string len value in storage: " << len);
+      CHECK_AND_ASSERT_THROW_MES(m_count >= len, "string len count value " << len << " goes out of remain storage len " << m_count);
+      //do this manually to avoid double memory write in huge strings (first time at resize, second at read)
+      str.assign((const char*)m_ptr, len);
+      m_ptr+=len;
+      m_count -= len;
+    }
+    inline
+    void throwable_buffer_reader::read(array_entry &ae)
+    {
+      RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported");
+    }
+  }
+}

src/contrib/epee/include/storages/portable_storage_from_json.h

@@ -0,0 +1,414 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+#pragma once
+#include <boost/lexical_cast.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include "parserse_base_utils.h"
+#include "file_io_utils.h"
+
+#define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100
+
+namespace epee
+{
+  using namespace misc_utils::parse;
+  namespace serialization
+  {
+    namespace json
+    {
+#define CHECK_ISSPACE()  if(!epee::misc_utils::parse::isspace(*it)){ ASSERT_MES_AND_THROW("Wrong JSON character at: " << std::string(it, buf_end));}
+
+      /*inline void parse_error()
+      {
+        ASSERT_MES_AND_THROW("json parse error");
+      }*/
+      template<class t_storage>
+      inline void run_handler(typename t_storage::hsection current_section, std::string::const_iterator& sec_buf_begin, std::string::const_iterator buf_end, t_storage& stg, unsigned int recursion)
+      {
+        CHECK_AND_ASSERT_THROW_MES(recursion < EPEE_JSON_RECURSION_LIMIT_INTERNAL, "Wrong JSON data: recursion limitation (" << EPEE_JSON_RECURSION_LIMIT_INTERNAL << ") exceeded");
+
+        std::string::const_iterator sub_element_start;
+        std::string name;        
+        typename t_storage::harray h_array = nullptr;
+        enum match_state
+        {
+          match_state_lookup_for_section_start, 
+          match_state_lookup_for_name, 
+          match_state_waiting_separator, 
+          match_state_wonder_after_separator, 
+          match_state_wonder_after_value, 
+          match_state_wonder_array, 
+          match_state_array_after_value,
+          match_state_array_waiting_value, 
+          match_state_error
+        };
+
+        enum array_mode
+        {
+          array_mode_undifined = 0,
+          array_mode_sections, 
+          array_mode_string, 
+          array_mode_numbers,
+          array_mode_booleans
+        };
+
+        match_state state = match_state_lookup_for_section_start;
+        array_mode array_md = array_mode_undifined;
+        std::string::const_iterator it = sec_buf_begin;
+        for(;it != buf_end;it++)
+        {
+          switch (state)
+          {
+          case match_state_lookup_for_section_start:
+            if(*it == '{')
+              state = match_state_lookup_for_name;
+            else CHECK_ISSPACE();
+            break;
+          case match_state_lookup_for_name:
+            switch(*it)
+            {
+            case '"':
+              match_string2(it, buf_end, name);
+              state = match_state_waiting_separator;
+              break;
+            case '}':
+              //this is it! section ends here.
+              //seems that it is empty section
+              sec_buf_begin = it;
+              return;
+            default:
+              CHECK_ISSPACE();
+            }
+            break;
+          case match_state_waiting_separator:
+            if(*it == ':')
+              state = match_state_wonder_after_separator;
+            else CHECK_ISSPACE();
+            break;
+          case match_state_wonder_after_separator:
+            if(*it == '"')
+            {//just a named string value started
+              std::string val;
+              match_string2(it, buf_end, val);
+              //insert text value 
+              stg.set_value(name, std::move(val), current_section);
+              state = match_state_wonder_after_value;
+            }else if (epee::misc_utils::parse::isdigit(*it) || *it == '-')
+            {//just a named number value started
+              boost::string_ref val;
+              bool is_v_float = false;bool is_signed = false;
+              match_number2(it, buf_end, val, is_v_float, is_signed);
+              if(!is_v_float)
+              {
+                if(is_signed)
+                {
+                  errno = 0;
+                  int64_t nval = strtoll(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  stg.set_value(name, nval, current_section);              
+                }else
+                {
+                  errno = 0;
+                  uint64_t nval = strtoull(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  stg.set_value(name, nval, current_section);              
+                }
+              }else
+              {
+                errno = 0;
+                double nval = strtod(val.data(), NULL);
+                if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                stg.set_value(name, nval, current_section);              
+              }
+              state = match_state_wonder_after_value;
+            }else if(isalpha(*it) )
+            {// could be null, true or false
+              boost::string_ref word;
+              match_word2(it, buf_end, word);
+              if(boost::iequals(word, "null"))
+              {
+                state = match_state_wonder_after_value;
+                //just skip this, 
+              }else if(boost::iequals(word, "true"))
+              {
+                stg.set_value(name, true, current_section);              
+                state = match_state_wonder_after_value;
+              }else if(boost::iequals(word, "false"))
+              {
+                stg.set_value(name, false, current_section);              
+                state = match_state_wonder_after_value;
+              }else ASSERT_MES_AND_THROW("Unknown value keyword " << word);
+            }else if(*it == '{')
+            {
+              //sub section here
+              typename t_storage::hsection new_sec = stg.open_section(name, current_section, true);
+              CHECK_AND_ASSERT_THROW_MES(new_sec, "Failed to insert new section in json: " << std::string(it, buf_end));
+              run_handler(new_sec, it, buf_end, stg, recursion + 1);
+              state = match_state_wonder_after_value;
+            }else if(*it == '[')
+            {//array of something
+              state = match_state_wonder_array;
+            }else CHECK_ISSPACE();
+            break;
+          case match_state_wonder_after_value:
+            if(*it == ',')
+              state = match_state_lookup_for_name;
+            else if(*it == '}')
+            {
+              //this is it! section ends here.
+              sec_buf_begin = it;
+              return;
+            }else CHECK_ISSPACE();
+            break;
+          case match_state_wonder_array:
+            if(*it == '[')
+            {
+              ASSERT_MES_AND_THROW("array of array not suppoerted yet :( sorry"); 
+              //mean array of array
+            }
+            if(*it == '{')
+            {
+              //mean array of sections
+              typename t_storage::hsection new_sec = nullptr;
+              h_array = stg.insert_first_section(name, new_sec, current_section);
+              CHECK_AND_ASSERT_THROW_MES(h_array&&new_sec, "failed to create new section");
+              run_handler(new_sec, it, buf_end, stg, recursion + 1);
+              state = match_state_array_after_value;
+              array_md = array_mode_sections;
+            }else if(*it == '"')
+            {
+              //mean array of strings
+              std::string val;
+              match_string2(it, buf_end, val);
+              h_array = stg.insert_first_value(name, std::move(val), current_section);
+              CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values entry");
+              state = match_state_array_after_value;
+              array_md = array_mode_string;
+            }else if (epee::misc_utils::parse::isdigit(*it) || *it == '-')
+            {//array of numbers value started
+              boost::string_ref val;
+              bool is_v_float = false;bool is_signed_val = false;
+              match_number2(it, buf_end, val, is_v_float, is_signed_val);
+              if(!is_v_float)
+              {
+                if (is_signed_val)
+                {
+                  errno = 0;
+                  int64_t nval = strtoll(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  h_array = stg.insert_first_value(name, nval, current_section);
+                }else
+                {
+                  errno = 0;
+                  uint64_t nval = strtoull(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  h_array = stg.insert_first_value(name, nval, current_section);
+                }
+                CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
+              }else
+              {
+                errno = 0;
+                double nval = strtod(val.data(), NULL);
+                if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                h_array = stg.insert_first_value(name, nval, current_section);
+                CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
+              }
+
+              state = match_state_array_after_value;
+              array_md = array_mode_numbers;
+            }else if(*it == ']')//empty array
+            {
+              array_md = array_mode_undifined;
+              state = match_state_wonder_after_value;
+            }else if(isalpha(*it) )
+            {// array of booleans
+              boost::string_ref word;
+              match_word2(it, buf_end, word);
+              if(boost::iequals(word, "true"))
+              {
+                h_array = stg.insert_first_value(name, true, current_section);              
+                CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
+                state = match_state_array_after_value;
+                array_md = array_mode_booleans;
+              }else if(boost::iequals(word, "false"))
+              {
+                h_array = stg.insert_first_value(name, false, current_section);              
+                CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
+                state = match_state_array_after_value;
+                array_md = array_mode_booleans;
+
+              }else ASSERT_MES_AND_THROW("Unknown value keyword " << word)
+            }else CHECK_ISSPACE();
+            break;
+          case match_state_array_after_value:
+            if(*it == ',')
+              state = match_state_array_waiting_value;
+            else if(*it == ']')
+            {
+              h_array = nullptr;
+              array_md = array_mode_undifined;
+              state = match_state_wonder_after_value;
+            }else CHECK_ISSPACE();
+            break;
+          case match_state_array_waiting_value:
+            switch(array_md)
+            {
+            case array_mode_sections:
+              if(*it == '{')
+              {
+                typename t_storage::hsection new_sec = NULL;
+                bool res = stg.insert_next_section(h_array, new_sec);
+                CHECK_AND_ASSERT_THROW_MES(res&&new_sec, "failed to insert next section");
+                run_handler(new_sec, it, buf_end, stg, recursion + 1);
+                state = match_state_array_after_value;
+              }else CHECK_ISSPACE();
+              break;
+            case array_mode_string:
+              if(*it == '"')
+              {
+                std::string val;
+                match_string2(it, buf_end, val);
+                bool res = stg.insert_next_value(h_array, std::move(val));
+                CHECK_AND_ASSERT_THROW_MES(res, "failed to insert values");
+                state = match_state_array_after_value;
+              }else CHECK_ISSPACE();
+              break;
+            case array_mode_numbers:
+              if (epee::misc_utils::parse::isdigit(*it) || *it == '-')
+              {//array of numbers value started
+                boost::string_ref val;
+                bool is_v_float = false;bool is_signed_val = false;
+                match_number2(it, buf_end, val, is_v_float, is_signed_val);
+                bool insert_res = false;
+                if(!is_v_float)
+                {
+                  if (is_signed_val)
+                  {
+                    errno = 0;
+                    int64_t nval = strtoll(val.data(), NULL, 10);
+                    if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                    insert_res = stg.insert_next_value(h_array, nval);
+                  }else
+                  {
+                    errno = 0;
+                    uint64_t nval = strtoull(val.data(), NULL, 10);
+                    if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                    insert_res = stg.insert_next_value(h_array, nval);
+                  }
+                }else
+                {
+                  errno = 0;
+                  double nval = strtod(val.data(), NULL);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  insert_res = stg.insert_next_value(h_array, nval);              
+                }
+                CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value");
+                state = match_state_array_after_value;
+                array_md = array_mode_numbers;
+              }else CHECK_ISSPACE();
+              break;
+            case array_mode_booleans:
+              if(isalpha(*it) )
+              {// array of booleans
+                boost::string_ref word;
+                match_word2(it, buf_end, word);
+                if(boost::iequals(word, "true"))
+                {
+                  bool r = stg.insert_next_value(h_array, true);              
+                  CHECK_AND_ASSERT_THROW_MES(r, " failed to insert values section entry");
+                  state = match_state_array_after_value;
+                }else if(boost::iequals(word, "false"))
+                {
+                  bool r = stg.insert_next_value(h_array, false);
+                  CHECK_AND_ASSERT_THROW_MES(r, " failed to insert values section entry");
+                  state = match_state_array_after_value;
+                }
+                else ASSERT_MES_AND_THROW("Unknown value keyword " << word);
+              }else CHECK_ISSPACE();
+              break;
+            case array_mode_undifined:
+            default:
+              ASSERT_MES_AND_THROW("Bad array state");
+            }
+            break;
+          case match_state_error:
+          default:
+            ASSERT_MES_AND_THROW("WRONG JSON STATE");
+          }
+        }
+      }
+/*
+{
+    "firstName": "John",
+    "lastName": "Smith",
+    "age": 25,
+    "address": {
+        "streetAddress": "21 2nd Street",
+        "city": "New York",
+        "state": "NY",
+        "postalCode": -10021, 
+        "have_boobs": true, 
+        "have_balls": false 
+    },
+    "phoneNumber": [
+        {
+            "type": "home",
+            "number": "212 555-1234"
+        },
+        {
+            "type": "fax",
+            "number": "646 555-4567"
+        }
+    ], 
+    "phoneNumbers": [
+    "812 123-1234",
+    "916 123-4567"
+    ]
+}
+*/
+      template<class t_storage>
+      inline bool load_from_json(const std::string& buff_json, t_storage& stg)
+      {
+        std::string::const_iterator sec_buf_begin  = buff_json.begin();
+        try
+        {
+          run_handler(nullptr, sec_buf_begin, buff_json.end(), stg, 0);
+          return true;
+        }
+        catch(const std::exception& ex)
+        {
+          MERROR("Failed to parse json, what: " << ex.what());
+          return false;
+        }
+        catch(...)
+        {
+          MERROR("Failed to parse json");
+          return false;
+        }
+      }
+    }
+  }
+}

src/contrib/epee/include/storages/portable_storage_template_helper.h

@@ -0,0 +1,129 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+#pragma once
+
+#include <string>
+
+#include "parserse_base_utils.h"
+#include "portable_storage.h"
+#include "file_io_utils.h"
+
+namespace epee
+{
+  namespace serialization
+  {
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool load_t_from_json(t_struct& out, const std::string& json_buff)
+    {
+      portable_storage ps;
+      bool rs = ps.load_from_json(json_buff);
+      if(!rs)
+        return false;
+
+      return out.load(ps);
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool load_t_from_json_file(t_struct& out, const std::string& json_file)
+    {
+      std::string f_buff;
+      if(!file_io_utils::load_file_to_string(json_file, f_buff))
+        return false;
+
+      return load_t_from_json(out, f_buff);
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool store_t_to_json(t_struct& str_in, std::string& json_buff, size_t indent = 0, bool insert_newlines = true)
+    {
+      portable_storage ps;
+      str_in.store(ps);
+      ps.dump_as_json(json_buff, indent, insert_newlines);
+      return true;
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    std::string store_t_to_json(t_struct& str_in, size_t indent = 0, bool insert_newlines = true)
+    {
+      std::string json_buff;
+      store_t_to_json(str_in, json_buff, indent, insert_newlines);
+      return json_buff;
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool store_t_to_json_file(t_struct& str_in, const std::string& fpath)
+    {
+      std::string json_buff;
+      store_t_to_json(str_in, json_buff);
+      return file_io_utils::save_string_to_file(fpath, json_buff);
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
+    {
+      portable_storage ps;
+      bool rs = ps.load_from_binary(binary_buff);
+      if(!rs)
+        return false;
+
+      return out.load(ps);
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool load_t_from_binary(t_struct& out, const std::string& binary_buff)
+    {
+      return load_t_from_binary(out, epee::strspan<uint8_t>(binary_buff));
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool load_t_from_binary_file(t_struct& out, const std::string& binary_file)
+    {
+      std::string f_buff;
+      if(!file_io_utils::load_file_to_string(binary_file, f_buff))
+        return false;
+
+      return load_t_from_binary(out, f_buff);
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    bool store_t_to_binary(t_struct& str_in, std::string& binary_buff, size_t indent = 0)
+    {
+      portable_storage ps;
+      str_in.store(ps);
+      return ps.store_to_binary(binary_buff);
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
+    std::string store_t_to_binary(t_struct& str_in, size_t indent = 0)
+    {
+      std::string binary_buff;
+      store_t_to_binary(str_in, binary_buff, indent);
+      return binary_buff;
+    }
+  }
+}

src/contrib/epee/include/storages/portable_storage_to_bin.h

@@ -0,0 +1,216 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+
+#pragma once 
+
+#include "pragma_comp_defs.h"
+#include "misc_language.h"
+#include "portable_storage_base.h"
+
+namespace epee
+{
+  namespace serialization
+  {
+
+    template<class pack_value, class t_stream>
+    size_t pack_varint_t(t_stream& strm, uint8_t type_or, size_t& pv)
+    {
+      pack_value v = (*((pack_value*)&pv)) << 2;
+      v |= type_or;
+      strm.write((const char*)&v, sizeof(pack_value));
+      return sizeof(pack_value);
+    }
+
+    PRAGMA_WARNING_PUSH
+      PRAGMA_GCC("GCC diagnostic ignored \"-Wstrict-aliasing\"")
+#ifdef __clang__
+      PRAGMA_GCC("GCC diagnostic ignored \"-Wtautological-constant-out-of-range-compare\"")
+#endif
+      template<class t_stream>
+    size_t pack_varint(t_stream& strm, size_t val)
+    {   //the first two bits always reserved for size information
+
+      if(val <= 63)
+      {//mean enough one byte
+        return pack_varint_t<uint8_t>(strm, PORTABLE_RAW_SIZE_MARK_BYTE, val);
+      }
+      else if(val <= 16383)
+      {//mean need word
+        return pack_varint_t<uint16_t>(strm, PORTABLE_RAW_SIZE_MARK_WORD, val);
+      }else if(val <= 1073741823)
+      {//mean need dword
+        return pack_varint_t<uint32_t>(strm, PORTABLE_RAW_SIZE_MARK_DWORD, val);
+      }else
+      {
+        CHECK_AND_ASSERT_THROW_MES(val <= 4611686018427387903, "failed to pack varint - too big amount = " << val);
+        return pack_varint_t<uint64_t>(strm, PORTABLE_RAW_SIZE_MARK_INT64, val);
+      }
+    }
+    PRAGMA_WARNING_POP
+
+      template<class t_stream>
+    bool put_string(t_stream& strm, const std::string& v)
+    {
+      pack_varint(strm, v.size());
+      if(v.size())
+        strm.write((const char*)v.data(), v.size());        
+      return true;
+    }
+
+    template<class t_stream>
+    struct array_entry_store_visitor: public boost::static_visitor<bool>
+    {
+      t_stream& m_strm;
+
+      template<class t_pod_type>
+      bool pack_pod_array_type(uint8_t contained_type, const array_entry_t<t_pod_type>& arr_pod)
+      {
+        uint8_t type = contained_type|SERIALIZE_FLAG_ARRAY;
+        m_strm.write((const char*)&type, 1);
+        pack_varint(m_strm, arr_pod.m_array.size());
+        for(const t_pod_type& x: arr_pod.m_array)
+          m_strm.write((const char*)&x, sizeof(t_pod_type));
+        return true;
+      }
+
+      array_entry_store_visitor(t_stream& strm):m_strm(strm){}
+      bool operator()(const array_entry_t<uint64_t>& v){ return pack_pod_array_type(SERIALIZE_TYPE_UINT64, v);}
+      bool operator()(const array_entry_t<uint32_t>& v){ return pack_pod_array_type(SERIALIZE_TYPE_UINT32, v);}
+      bool operator()(const array_entry_t<uint16_t>& v){ return pack_pod_array_type(SERIALIZE_TYPE_UINT16, v);}
+      bool operator()(const array_entry_t<uint8_t>& v) { return pack_pod_array_type(SERIALIZE_TYPE_UINT8, v);}
+      bool operator()(const array_entry_t<int64_t>& v) { return pack_pod_array_type(SERIALIZE_TYPE_INT64, v);}
+      bool operator()(const array_entry_t<int32_t>& v) { return pack_pod_array_type(SERIALIZE_TYPE_INT32, v);}
+      bool operator()(const array_entry_t<int16_t>& v) { return pack_pod_array_type(SERIALIZE_TYPE_INT16, v);}
+      bool operator()(const array_entry_t<int8_t>& v)  { return pack_pod_array_type(SERIALIZE_TYPE_INT8, v);}
+      bool operator()(const array_entry_t<double>& v)  { return pack_pod_array_type(SERIALIZE_TYPE_DUOBLE, v);}
+      bool operator()(const array_entry_t<bool>& v)    { return pack_pod_array_type(SERIALIZE_TYPE_BOOL, v);}
+      bool operator()(const array_entry_t<std::string>& arr_str)
+      {
+        uint8_t type = SERIALIZE_TYPE_STRING|SERIALIZE_FLAG_ARRAY;
+        m_strm.write((const char*)&type, 1);
+        pack_varint(m_strm, arr_str.m_array.size());
+        for(const std::string& s: arr_str.m_array)
+          put_string(m_strm, s);
+        return true;
+      }
+      bool operator()(const array_entry_t<section>& arr_sec)    
+      {
+        uint8_t type = SERIALIZE_TYPE_OBJECT|SERIALIZE_FLAG_ARRAY;
+        m_strm.write((const char*)&type, 1);
+        pack_varint(m_strm, arr_sec.m_array.size());
+        for(const section& s: arr_sec.m_array)
+          pack_entry_to_buff(m_strm, s);
+        return true;
+      }
+      bool operator()(const array_entry_t<array_entry>& arra_ar)    
+      {
+        uint8_t type = SERIALIZE_TYPE_ARRAY|SERIALIZE_FLAG_ARRAY;
+        m_strm.write((const char*)&type, 1);
+        pack_varint(m_strm, arra_ar.m_array.size());
+        for(const array_entry& s: arra_ar.m_array)
+          pack_entry_to_buff(m_strm, s);
+        return true;
+      }
+    };
+
+    template<class t_stream>
+    struct storage_entry_store_visitor: public boost::static_visitor<bool>
+    {
+      t_stream& m_strm;
+      storage_entry_store_visitor(t_stream& strm):m_strm(strm){}
+      template<class pod_type>
+      bool pack_pod_type(uint8_t type, const pod_type& v)
+      {
+        m_strm.write((const char*)&type, 1);
+        m_strm.write((const char*)&v, sizeof(pod_type));
+        return true;
+      }
+      //section, array_entry
+      bool operator()(const uint64_t& v){ return pack_pod_type(SERIALIZE_TYPE_UINT64, v);}
+      bool operator()(const uint32_t& v){ return pack_pod_type(SERIALIZE_TYPE_UINT32, v);}
+      bool operator()(const uint16_t& v){ return pack_pod_type(SERIALIZE_TYPE_UINT16, v);}
+      bool operator()(const uint8_t& v) { return pack_pod_type(SERIALIZE_TYPE_UINT8, v);}
+      bool operator()(const int64_t& v) { return pack_pod_type(SERIALIZE_TYPE_INT64, v);}
+      bool operator()(const int32_t& v) { return pack_pod_type(SERIALIZE_TYPE_INT32, v);}
+      bool operator()(const int16_t& v) { return pack_pod_type(SERIALIZE_TYPE_INT16, v);}
+      bool operator()(const int8_t& v)  { return pack_pod_type(SERIALIZE_TYPE_INT8, v);}
+      bool operator()(const double& v)  { return pack_pod_type(SERIALIZE_TYPE_DUOBLE, v);}
+      bool operator()(const bool& v)  { return pack_pod_type(SERIALIZE_TYPE_BOOL, v);}
+      bool operator()(const std::string& v)
+      {
+        uint8_t type = SERIALIZE_TYPE_STRING;
+        m_strm.write((const char*)&type, 1);
+        put_string(m_strm, v);
+        return true;
+      }
+      bool operator()(const section& v)  
+      {
+        uint8_t type = SERIALIZE_TYPE_OBJECT;
+        m_strm.write((const char*)&type, 1);
+        return pack_entry_to_buff(m_strm, v);
+      }
+
+      bool operator()(const array_entry& v)  
+      {
+        //uint8_t type = SERIALIZE_TYPE_ARRAY;
+        //m_strm.write((const char*)&type, 1);
+        return pack_entry_to_buff(m_strm, v);
+      }
+    };
+
+    template<class t_stream>
+    bool pack_entry_to_buff(t_stream& strm, const array_entry& ae)
+    {
+      array_entry_store_visitor<t_stream> aesv(strm);
+      return boost::apply_visitor(aesv, ae);
+    }
+
+    template<class t_stream>
+    bool pack_entry_to_buff(t_stream& strm, const storage_entry& se)
+    {
+      storage_entry_store_visitor<t_stream> sv(strm);
+      return boost::apply_visitor(sv, se);
+    }
+
+    template<class t_stream>
+    bool pack_entry_to_buff(t_stream& strm, const section& sec)
+    {
+      typedef std::map<std::string, storage_entry>::value_type section_pair;
+      pack_varint(strm, sec.m_entries.size());
+      for(const section_pair& se: sec.m_entries)
+      {
+        CHECK_AND_ASSERT_THROW_MES(se.first.size() < std::numeric_limits<uint8_t>::max(), "storage_entry_name is too long: " << se.first.size() << ", val: " << se.first);
+        uint8_t len = static_cast<uint8_t>(se.first.size());
+        strm.write((const char*)&len, sizeof(len));
+        strm.write(se.first.data(), size_t(len));
+        pack_entry_to_buff(strm, se.second);
+      }
+      return true;
+    }
+  }
+}

src/contrib/epee/include/storages/portable_storage_to_json.h

@@ -0,0 +1,181 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+
+
+
+#pragma once 
+
+#include "misc_language.h"
+#include "portable_storage_base.h"
+#include "parserse_base_utils.h"
+
+namespace epee
+{
+  namespace serialization
+  {
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const array_entry& ae, size_t indent, bool insert_newlines);
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const storage_entry& se, size_t indent, bool insert_newlines);
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const std::string& v, size_t indent, bool insert_newlines);
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const int8_t& v, size_t indent, bool insert_newlines);
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const uint8_t& v, size_t indent, bool insert_newlines);
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const bool& v, size_t indent, bool insert_newlines);
+    template<class t_stream, class t_type>
+    void dump_as_json(t_stream& strm, const t_type& v, size_t indent, bool insert_newlines);
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const section& sec, size_t indent, bool insert_newlines);
+
+
+    inline std::string make_indent(size_t indent)
+    {
+      return std::string(indent*2, ' ');
+    }
+
+    template<class t_stream>
+    struct array_entry_store_to_json_visitor: public boost::static_visitor<void>
+    {
+      t_stream& m_strm;
+      size_t m_indent;
+      bool m_insert_newlines;
+      array_entry_store_to_json_visitor(t_stream& strm, size_t indent,
+                                        bool insert_newlines = true)
+        : m_strm(strm), m_indent(indent), m_insert_newlines(insert_newlines)
+      {}
+
+      template<class t_type>
+      void operator()(const array_entry_t<t_type>& a)
+      {
+        m_strm << "[";
+        if(a.m_array.size())
+        {
+          auto last_it = --a.m_array.end();
+          for(auto it = a.m_array.begin(); it != a.m_array.end(); it++)
+          {
+            dump_as_json(m_strm, *it, m_indent, m_insert_newlines);
+            if(it != last_it)
+              m_strm << ",";
+          }
+        }
+        m_strm << "]";
+      }
+    };
+
+    template<class t_stream>
+    struct storage_entry_store_to_json_visitor: public boost::static_visitor<void>
+    {
+      t_stream& m_strm;
+      size_t m_indent;
+      bool m_insert_newlines;
+      storage_entry_store_to_json_visitor(t_stream& strm, size_t indent,
+                                          bool insert_newlines = true)
+          : m_strm(strm), m_indent(indent), m_insert_newlines(insert_newlines)
+      {}
+      //section, array_entry
+      template<class visited_type>
+      void operator()(const visited_type& v)
+      { 
+        dump_as_json(m_strm, v, m_indent, m_insert_newlines);
+      }
+    };
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const array_entry& ae, size_t indent, bool insert_newlines)
+    {
+      array_entry_store_to_json_visitor<t_stream> aesv(strm, indent, insert_newlines);
+      boost::apply_visitor(aesv, ae);
+    }
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const storage_entry& se, size_t indent, bool insert_newlines)
+    {
+      storage_entry_store_to_json_visitor<t_stream> sv(strm, indent, insert_newlines);
+      boost::apply_visitor(sv, se);
+    }
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const std::string& v, size_t indent, bool insert_newlines)
+    {
+      strm << "\"" << misc_utils::parse::transform_to_escape_sequence(v) << "\"";
+    }
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const int8_t& v, size_t indent, bool insert_newlines)
+    {
+      strm << static_cast<int32_t>(v);
+    }
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const uint8_t& v, size_t indent, bool insert_newlines)
+    {
+      strm << static_cast<int32_t>(v);
+    }
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const bool& v, size_t indent, bool insert_newlines)
+    {
+      if(v)
+        strm << "true";
+      else
+        strm << "false";
+    }
+
+
+
+    template<class t_stream, class t_type>
+    void dump_as_json(t_stream& strm, const t_type& v, size_t indent, bool insert_newlines)
+    {
+      strm << v;
+    }
+
+    template<class t_stream>
+    void dump_as_json(t_stream& strm, const section& sec, size_t indent, bool insert_newlines)
+    {
+      size_t local_indent = indent + 1;
+      std::string newline = insert_newlines ? "\r\n" : "";
+      strm << "{" << newline;
+      std::string indent_str = make_indent(local_indent);
+      if(sec.m_entries.size())
+      {
+        auto it_last = --sec.m_entries.end();
+        for(auto it = sec.m_entries.begin(); it!= sec.m_entries.end();it++)
+        {
+          strm << indent_str << "\"" << misc_utils::parse::transform_to_escape_sequence(it->first) << "\"" << ": ";
+          dump_as_json(strm, it->second, local_indent, insert_newlines);
+          if(it_last != it)
+            strm << ",";
+          strm << newline;
+        }
+      }
+      strm << make_indent(indent) <<  "}";
+    }
+  }
+}

src/contrib/epee/include/storages/portable_storage_val_converters.h

@@ -0,0 +1,202 @@
+// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// * Neither the name of the Andrey N. Sabelnikov nor the
+// names of its contributors may be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
+// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+
+
+
+#pragma once
+
+#include <time.h>
+#include <boost/regex.hpp>
+
+#include "misc_language.h"
+#include "portable_storage_base.h"
+#include "warnings.h"
+
+namespace epee
+{
+  namespace serialization
+  {
+#define ASSERT_AND_THROW_WRONG_CONVERSION() ASSERT_MES_AND_THROW("WRONG DATA CONVERSION: from type=" << typeid(from).name() << " to type " << typeid(to).name())
+
+    template<typename from_type, typename to_type>
+    void convert_int_to_uint(const from_type& from, to_type& to)
+    {
+PUSH_WARNINGS
+DISABLE_VS_WARNINGS(4018)
+      CHECK_AND_ASSERT_THROW_MES(from >=0, "unexpected int value with signed storage value less than 0, and unsigned receiver value");
+DISABLE_GCC_AND_CLANG_WARNING(sign-compare)
+      CHECK_AND_ASSERT_THROW_MES(from <= std::numeric_limits<to_type>::max(), "int value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with max possible value = " << std::numeric_limits<to_type>::max());
+      to = static_cast<to_type>(from);
+POP_WARNINGS
+    }
+    template<typename from_type, typename to_type>
+    void convert_int_to_int(const from_type& from, to_type& to)
+    {
+      CHECK_AND_ASSERT_THROW_MES(from >= boost::numeric::bounds<to_type>::lowest(), "int value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with lowest possible value = " << boost::numeric::bounds<to_type>::lowest());
+PUSH_WARNINGS
+DISABLE_CLANG_WARNING(tautological-constant-out-of-range-compare)
+      CHECK_AND_ASSERT_THROW_MES(from <= std::numeric_limits<to_type>::max(), "int value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with max possible value = " << std::numeric_limits<to_type>::max());
+POP_WARNINGS
+      to = static_cast<to_type>(from);
+    }
+    template<typename from_type, typename to_type>
+    void convert_uint_to_any_int(const from_type& from, to_type& to)
+    {
+PUSH_WARNINGS
+DISABLE_VS_WARNINGS(4018)
+DISABLE_CLANG_WARNING(tautological-constant-out-of-range-compare)
+        CHECK_AND_ASSERT_THROW_MES(from <= std::numeric_limits<to_type>::max(), "uint value overhead: try to set value " << from << " to type " << typeid(to_type).name() << " with max possible value = " << std::numeric_limits<to_type>::max());
+      to = static_cast<to_type>(from);
+POP_WARNINGS
+    }
+
+    template<typename from_type, typename to_type, bool, bool> //is from signed, is from to signed
+    struct convert_to_signed_unsigned;
+
+    template<typename from_type, typename to_type>
+    struct convert_to_signed_unsigned<from_type, to_type, true, true>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {//from signed to signed
+        convert_int_to_int(from, to);
+      }
+    };
+
+    template<typename from_type, typename to_type>
+    struct convert_to_signed_unsigned<from_type, to_type, true, false>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {//from signed to unsigned
+        convert_int_to_uint(from, to);
+      }
+    };
+
+    template<typename from_type, typename to_type>
+    struct convert_to_signed_unsigned<from_type, to_type, false, true>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {//from unsigned to signed
+        convert_uint_to_any_int(from, to);
+      }
+    };
+
+    template<typename from_type, typename to_type>
+    struct convert_to_signed_unsigned<from_type, to_type, false, false>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {
+        //from unsigned to unsigned
+        convert_uint_to_any_int(from, to);
+      }
+    };
+
+    template<typename from_type, typename to_type, bool>
+    struct convert_to_integral;
+
+    template<typename from_type, typename to_type>
+    struct convert_to_integral<from_type, to_type, true>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {
+        convert_to_signed_unsigned<from_type, to_type, std::is_signed<from_type>::value, std::is_signed<to_type>::value>::convert(from, to);
+      }
+    };
+
+    template<typename from_type, typename to_type>
+    struct convert_to_integral<from_type, to_type, false>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {
+        ASSERT_AND_THROW_WRONG_CONVERSION();
+      }
+    };
+
+    // For MyMonero/OpenMonero backend compatibility
+    // MyMonero backend sends amount, fees and timestamp values as strings.
+    // Until MM backend is updated, this is needed for compatibility between OpenMonero and MyMonero. 
+    template<>
+    struct convert_to_integral<std::string, uint64_t, false>
+    {
+      static void convert(const std::string& from, uint64_t& to)
+      {
+        MTRACE("Converting std::string to uint64_t. Source: " << from);
+        // String only contains digits
+        if(std::all_of(from.begin(), from.end(), epee::misc_utils::parse::isdigit))
+          to = boost::lexical_cast<uint64_t>(from);
+        // MyMonero ISO 8061 timestamp (2017-05-06T16:27:06Z)
+        else if (boost::regex_match (from, boost::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ")))
+        {
+          // Convert to unix timestamp
+#ifdef HAVE_STRPTIME
+          struct tm tm;
+          if (strptime(from.c_str(), "%Y-%m-%dT%H:%M:%S", &tm))
+#else
+          std::tm tm = {};
+          std::istringstream ss(from);
+          if (ss >> std::get_time(&tm, "%Y-%m-%dT%H:%M:%S"))
+#endif
+            to = std::mktime(&tm);
+        } else
+          ASSERT_AND_THROW_WRONG_CONVERSION();
+      }
+    };
+
+    template<class from_type, class to_type>
+    struct is_convertable: std::integral_constant<bool,
+                            std::is_integral<to_type>::value &&
+                            std::is_integral<from_type>::value &&
+                            !std::is_same<from_type, bool>::value &&
+                            !std::is_same<to_type, bool>::value > {};
+
+    template<typename from_type, typename to_type, bool>
+    struct convert_to_same;
+
+    template<typename from_type, typename to_type>
+    struct convert_to_same<from_type, to_type, true>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {
+        to = from;
+      }
+    };
+
+    template<typename from_type, typename to_type>
+    struct convert_to_same<from_type, to_type, false>
+    {
+      static void convert(const from_type& from, to_type& to)
+      {
+        convert_to_integral<from_type, to_type, is_convertable<from_type, to_type>::value>::convert(from, to);// ASSERT_AND_THROW_WRONG_CONVERSION();
+      }
+    };
+
+
+    template<class from_type, class to_type>
+    void convert_t(const from_type& from, to_type& to)
+    {
+      convert_to_same<from_type, to_type, std::is_same<to_type, from_type>::value>::convert(from, to);
+    }
+  }
+}

src/contrib/epee/include/string_tools.h

@@ -29,16 +29,25 @@
 #ifndef _STRING_TOOLS_H_
 #define _STRING_TOOLS_H_
 
-//#include <objbase.h>
+// Previously pulled in by ASIO, further cleanup still required ...
+#ifdef _WIN32
+# include <winsock2.h>
+# include <windows.h>
+#endif
+
+#include <string.h>
 #include <locale>
 #include <cstdlib>
-//#include <strsafe.h>
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_io.hpp>
+#include <string>
+#include <type_traits>
 #include <boost/lexical_cast.hpp>
-#include <boost/asio.hpp>
-#include <boost/algorithm/string/compare.hpp>
-#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+#include "misc_log_ex.h"
+#include "storages/parserse_base_utils.h"
+#include "hex.h"
+#include "memwipe.h"
+#include "mlocker.h"
+#include "span.h"
 #include "warnings.h"
 
 
@@ -50,135 +59,64 @@
 #pragma comment (lib, "Rpcrt4.lib")
 #endif
 
+static const constexpr unsigned char isx[256] =
+{
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    0,    1,    2,    3,    4,    5,    6,    7,    8,    9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff,   10,   11,   12,   13,   14,   15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff,   10,   11,   12,   13,   14,   15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+};
+
 namespace epee
 {
 namespace string_tools
 {
-	inline std::wstring get_str_from_guid(const boost::uuids::uuid& rid)
-	{
-		return boost::lexical_cast<std::wstring>(rid);
-	}
-	//----------------------------------------------------------------------------
-	inline std::string get_str_from_guid_a(const boost::uuids::uuid& rid)
-	{
-		return boost::lexical_cast<std::string>(rid);
-	}
-	//----------------------------------------------------------------------------
-	inline bool get_guid_from_string( boost::uuids::uuid& inetifer, std::wstring str_id)
-	{
-		if(str_id.size() < 36)
-			return false;
-
-		if('{' == *str_id.begin())
-			str_id.erase(0, 1);
-
-		if('}' == *(--str_id.end()))
-			str_id.erase(--str_id.end());
-
-		try
-		{
-			inetifer = boost::lexical_cast<boost::uuids::uuid>(str_id);
-			return true;
-		}
-		catch(...)
-		{
-			return false;
-		}
-	}
-	//----------------------------------------------------------------------------
-	inline bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id)
-	{
-		std::string local_str_id = str_id;
-		if(local_str_id.size() < 36)
-			return false;
-
-		if('{' == *local_str_id.begin())
-			local_str_id.erase(0, 1);
-
-		if('}' == *(--local_str_id.end()))
-			local_str_id.erase(--local_str_id.end());
-
-		try
-		{
-			inetifer = boost::lexical_cast<boost::uuids::uuid>(local_str_id);
-			return true;
-		}
-		catch(...)
-		{
-			return false;
-		}
-	}
-	//----------------------------------------------------------------------------
-  template<class CharT>
-  std::basic_string<CharT> buff_to_hex(const std::basic_string<CharT>& s)
+  //----------------------------------------------------------------------------
+  inline std::string buff_to_hex_nodelimer(const std::string& src)
   {
-    using namespace std;
-    basic_stringstream<CharT> hexStream;
-    hexStream << hex << noshowbase << setw(2);
-
-    for(typename std::basic_string<CharT>::const_iterator it = s.begin(); it != s.end(); it++)
-    {
-      hexStream << "0x"<< static_cast<unsigned int>(static_cast<unsigned char>(*it)) << " ";
-    }
-    return hexStream.str();
+    return to_hex::string(to_byte_span(to_span(src)));
   }
   //----------------------------------------------------------------------------
-  template<class CharT>
-  std::basic_string<CharT> buff_to_hex_nodelimer(const std::basic_string<CharT>& s)
+  inline bool parse_hexstr_to_binbuff(const epee::span<const char> s, epee::span<char>& res)
   {
-    using namespace std;
-    basic_stringstream<CharT> hexStream;
-    hexStream << hex << noshowbase;
+      if (s.size() != res.size() * 2)
+        return false;
 
-    for(typename std::basic_string<CharT>::const_iterator it = s.begin(); it != s.end(); it++)
-    {
-      hexStream << setw(2) << setfill('0') << static_cast<unsigned int>(static_cast<unsigned char>(*it));
-    }
-    return hexStream.str();
-  }
-  //----------------------------------------------------------------------------
-  template<class CharT>
-  bool parse_hexstr_to_binbuff(const std::basic_string<CharT>& s, std::basic_string<CharT>& res)
-  {
-    res.clear();
-    try
-    {
-      long v = 0;
-      for(size_t i = 0; i < (s.size() + 1) / 2; i++)
+      unsigned char *dst = (unsigned char *)&res[0];
+      const unsigned char *src = (const unsigned char *)s.data();
+      for(size_t i = 0; i < s.size(); i += 2)
       {
-        CharT byte_str[3];
-        size_t copied = s.copy(byte_str, 2, 2 * i);
-        byte_str[copied] = CharT(0);
-        CharT* endptr;
-        v = strtoul(byte_str, &endptr, 16);
-        if (v < 0 || 0xFF < v || endptr != byte_str + copied)
-        {
-          return false;
-        }
-        res.push_back(static_cast<unsigned char>(v));
+        int tmp = *src++;
+        tmp = isx[tmp];
+        if (tmp == 0xff) return false;
+        int t2 = *src++;
+        t2 = isx[t2];
+        if (t2 == 0xff) return false;
+        *dst++ = (tmp << 4) | t2;
       }
 
       return true;
-    }catch(...)
-    {
-      return false;
-    }
   }
   //----------------------------------------------------------------------------
-  template<class t_pod_type>
-  bool parse_tpod_from_hex_string(const std::string& str_hash, t_pod_type& t_pod)
+  inline bool parse_hexstr_to_binbuff(const std::string& s, std::string& res)
   {
-    std::string buf;
-    bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf);
-    if (!res || buf.size() != sizeof(t_pod_type))
-    {
+    if (s.size() & 1)
       return false;
-    }
-    else
-    {
-      buf.copy(reinterpret_cast<char *>(&t_pod), sizeof(t_pod_type));
-      return true;
-    }
+    res.resize(s.size() / 2);
+    epee::span<char> rspan((char*)&res[0], res.size());
+    return parse_hexstr_to_binbuff(epee::to_span(s), rspan);
   }
   //----------------------------------------------------------------------------
 PUSH_WARNINGS
@@ -190,7 +128,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized)
     {
       for (char c : str_id)
       {
-        if (!std::isdigit(c))
+        if (!epee::misc_utils::parse::isdigit(c))
           return false;
       }
     }
@@ -200,7 +138,7 @@ DISABLE_GCC_WARNING(maybe-uninitialized)
       val = boost::lexical_cast<XType>(str_id);
       return true;
     }
-    catch(std::exception& /*e*/)
+    catch(const std::exception& /*e*/)
     {
       //const char* pmsg = e.what();
       return false;
@@ -213,22 +151,6 @@ DISABLE_GCC_WARNING(maybe-uninitialized)
     return true;
   }
 POP_WARNINGS
-	//---------------------------------------------------
-	template<typename int_t>
-	bool get_xnum_from_hex_string(const std::string str, int_t& res )
-	{
-		try
-		{
-			std::stringstream ss;
-			ss << std::hex << str;
-			ss >> res;
-			return true;	
-		}
-		catch(...)
-		{
-			return false;
-		}
-	}
 	//----------------------------------------------------------------------------
 	template<class XType>
 	inline bool xtype_to_string(const  XType& val, std::string& str)
@@ -244,139 +166,39 @@ POP_WARNINGS
 
 		return true;
 	}
-    
-	typedef std::map<std::string, std::string> command_line_params_a;
-	typedef std::map<std::wstring, std::wstring> command_line_params_w;
-
-	template<class t_string>
-	bool parse_commandline(std::map<t_string, t_string>& res, int argc, char** argv)
-	{
-    t_string key;
-    for(int i = 1; i < argc; i++)
-    {
-      if(!argv[i])
-        break;
-      t_string s = argv[i];
-      std::string::size_type p = s.find('=');
-      if(std::string::npos == p)
-      {
-        res[s] = "";
-      }else
-      {
-        std::string ss;
-        t_string nm = s.substr(0, p);
-        t_string vl = s.substr(p+1, s.size());
-        res[nm] = vl;
-      }
-    }
-    return true;
-	}
-
-/*  template<typename t_type>
-  bool get_xparam_from_command_line(const std::map<std::string, std::string>& res, const std::basic_string<typename t_string::value_type> & key, t_type& val)
-  {
-
-  }
-  */
-
-	template<class t_string, typename t_type>
-	bool get_xparam_from_command_line(const std::map<t_string, t_string>& res, const t_string & key, t_type& val)
-	{
-		typename std::map<t_string, t_string>::const_iterator it = res.find(key);
-		if(it == res.end())
-			return false;
-
-		if(it->second.size())
-		{
-			return get_xtype_from_string(val, it->second);
-		}
-
-		return true;
-	}
-
-    template<class t_string, typename t_type>
-    t_type get_xparam_from_command_line(const std::map<t_string, t_string>& res, const t_string & key, const t_type& default_value)
-    {
-        typename std::map<t_string, t_string>::const_iterator it = res.find(key);
-        if(it == res.end())
-            return default_value;
-
-        if(it->second.size())
-        {
-            t_type s;
-            get_xtype_from_string(s, it->second);
-            return s;
-        }
-
-        return default_value;
-    }
-
-  template<class t_string>
-  bool have_in_command_line(const std::map<t_string, t_string>& res, const std::basic_string<typename t_string::value_type>& key)
-  {
-    typename std::map<t_string, t_string>::const_iterator it = res.find(key);
-    if(it == res.end())
-      return false;
-
-    return true;
-  }
-
 	//----------------------------------------------------------------------------
-//#ifdef _WINSOCK2API_
-	inline std::string get_ip_string_from_int32(uint32_t ip)
-	{
-		in_addr adr;
-		adr.s_addr = ip;
-		const char* pbuf = inet_ntoa(adr);
-		if(pbuf)
-			return pbuf;
-		else
-			return "[failed]";
-	}
+	std::string get_ip_string_from_int32(uint32_t ip);
 	//----------------------------------------------------------------------------
-	inline bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str)
-	{
-		ip = inet_addr(ip_str.c_str());
-		if(INADDR_NONE == ip)
-			return false;
-
-		return true;
-	}
+	bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str);
   //----------------------------------------------------------------------------
-  inline bool parse_peer_from_string(uint32_t& ip, uint32_t& port, const std::string& addres)
+  inline bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres)
   {
     //parse ip and address
     std::string::size_type p = addres.find(':');
+    std::string ip_str, port_str;
     if(p == std::string::npos)
     {
-      return false;
+      port = 0;
+      ip_str = addres;
+    }
+    else
+    {
+      ip_str = addres.substr(0, p);
+      port_str = addres.substr(p+1, addres.size());
     }
-    std::string ip_str = addres.substr(0, p);
-    std::string port_str = addres.substr(p+1, addres.size());
 
     if(!get_ip_int32_from_string(ip, ip_str))
     {
       return false;
     }
 
-    if(!get_xtype_from_string(port, port_str))
+    if(p != std::string::npos && !get_xtype_from_string(port, port_str))
     {
       return false;
     }
     return true;
   }
 
-//#endif
-	//----------------------------------------------------------------------------
-  template<typename t>
-	inline std::string get_t_as_hex_nwidth(const t& v, std::streamsize w = 8)
-  {
-    std::stringstream ss;
-    ss << std::setfill ('0') << std::setw (w) << std::hex << std::noshowbase;
-    ss << v;
-    return ss.str();
-  }
-
 	inline std::string num_to_string_fast(int64_t val)
 	{
 		/*
@@ -386,68 +208,15 @@ POP_WARNINGS
 		return boost::lexical_cast<std::string>(val);
 	}
 	//----------------------------------------------------------------------------
-	inline bool string_to_num_fast(const std::string& buff, int64_t& val)
+	inline std::string to_string_hex(uint32_t val)
 	{
-		//return get_xtype_from_string(val, buff);
-#if (defined _MSC_VER)
-    val = _atoi64(buff.c_str());
-#else
-    val = atoll(buff.c_str());
-#endif
-		/*
-		 * val = atoi64(buff.c_str());
-     */
-		if(buff != "0" && val == 0)
-			return false;
-		return true;
+		std::stringstream ss;
+		ss << std::hex << val;
+		std::string s;
+		ss >> s;
+		return s;
 	}
 	//----------------------------------------------------------------------------
-	inline bool string_to_num_fast(const std::string& buff, int& val)
-	{
-		val = atoi(buff.c_str());
-		if(buff != "0" && val == 0)
-			return false;
-
-		return true;
-	}
-	//----------------------------------------------------------------------------
-#ifdef WINDOWS_PLATFORM
-	inline std::string system_time_to_string(const SYSTEMTIME& st)
-	{
-	
-		/*
-		TIME_ZONE_INFORMATION tzi;
-		GetTimeZoneInformation(&tzi);
-		SystemTimeToTzSpecificLocalTime(&tzi, &stUTC, &stLocal);
-		*/
-
-		char szTime[25], szDate[25];
-		::GetTimeFormatA(
-			LOCALE_USER_DEFAULT,    // locale
-			TIME_FORCE24HOURFORMAT, // options
-			&st,					// time
-			NULL,                   // time format string
-			szTime,                 // formatted string buffer
-			25                      // size of string buffer
-			);
-
-		::GetDateFormatA(
-			LOCALE_USER_DEFAULT,    // locale
-			NULL,                   // options
-			&st,					// date
-			NULL,                   // date format
-			szDate,                 // formatted string buffer
-			25                      // size of buffer
-			);
-		szTime[24] = szDate[24] = 0; //be happy :)
-		
-		std::string res = szDate;
-		(res += " " )+= szTime;
-		return res; 
-
-	}
-#endif
-	//----------------------------------------------------------------------------
 	
 	inline bool compare_no_case(const std::string& str1, const std::string& str2)
 	{
@@ -455,33 +224,6 @@ POP_WARNINGS
 		return !boost::iequals(str1, str2);
 	}
 	//----------------------------------------------------------------------------
-	inline bool compare_no_case(const std::wstring& str1, const std::wstring& str2)
-	{
-		return !boost::iequals(str1, str2);
-	}
-	//----------------------------------------------------------------------------
-	inline bool is_match_prefix(const std::wstring& str1, const std::wstring& prefix)
-	{	
-		if(prefix.size()>str1.size())
-			return false;
-
-		if(!compare_no_case(str1.substr(0, prefix.size()), prefix))
-			return true;
-		else
-			return false;
-	}
-	//----------------------------------------------------------------------------
-	inline bool is_match_prefix(const std::string& str1, const std::string& prefix)
-	{	
-		if(prefix.size()>str1.size())
-			return false;
-
-		if(!compare_no_case(str1.substr(0, prefix.size()), prefix))
-			return true;
-		else
-			return false;
-	}
-	//----------------------------------------------------------------------------
 	inline std::string& get_current_module_name()
 	{
 		static std::string module_name;
@@ -559,29 +301,48 @@ POP_WARNINGS
     return str;
   }
   //----------------------------------------------------------------------------
+  inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false)
+  {
+    if (s.size() < n)
+    {
+      if (prepend)
+        s = std::string(n - s.size(), c) + s;
+      else
+        s.append(n - s.size(), c);
+    }
+    return s;
+  }
+  //----------------------------------------------------------------------------
   template<class t_pod_type>
   std::string pod_to_hex(const t_pod_type& s)
   {
-    std::string buff;
-    buff.assign(reinterpret_cast<const char*>(&s), sizeof(s));
-    return buff_to_hex_nodelimer(buff);
+    static_assert(std::is_standard_layout<t_pod_type>(), "expected standard layout type");
+    return to_hex::string(as_byte_span(s));
   }
   //----------------------------------------------------------------------------
   template<class t_pod_type>
   bool hex_to_pod(const std::string& hex_str, t_pod_type& s)
   {
-    std::string hex_str_tr = trim(hex_str);
+    static_assert(std::is_pod<t_pod_type>::value, "expected pod type");
     if(sizeof(s)*2 != hex_str.size())
       return false;
-    std::string bin_buff;
-    if(!parse_hexstr_to_binbuff(hex_str_tr, bin_buff))
-      return false;
-    if(bin_buff.size()!=sizeof(s))
-      return false;
-
-    s = *(t_pod_type*)bin_buff.data();
-    return true;
+    epee::span<char> rspan((char*)&s, sizeof(s));
+    return parse_hexstr_to_binbuff(epee::to_span(hex_str), rspan);
   }
+  //----------------------------------------------------------------------------
+  template<class t_pod_type>
+  bool hex_to_pod(const std::string& hex_str, tools::scrubbed<t_pod_type>& s)
+  {
+    return hex_to_pod(hex_str, unwrap(s));
+  }
+  //----------------------------------------------------------------------------
+  template<class t_pod_type>
+  bool hex_to_pod(const std::string& hex_str, epee::mlocked<t_pod_type>& s)
+  {
+    return hex_to_pod(hex_str, unwrap(s));
+  }
+  //----------------------------------------------------------------------------
+  bool validate_hex(uint64_t length, const std::string& str);
   //----------------------------------------------------------------------------
 	inline std::string get_extension(const std::string& str)
 	{
@@ -594,20 +355,6 @@ POP_WARNINGS
 		return res;
 	}
 	//----------------------------------------------------------------------------
-	inline std::string get_filename_from_path(const std::string& str)
-	{
-		std::string res;
-		std::string::size_type pos = str.rfind('\\');
-		if(std::string::npos == pos)
-			return str;
-
-		res = str.substr(pos+1, str.size()-pos);
-		return res;
-	}
-	//----------------------------------------------------------------------------
-
-
-
 	inline std::string cut_off_extension(const std::string& str)
 	{
 		std::string res;
@@ -618,126 +365,40 @@ POP_WARNINGS
 		res = str.substr(0, pos);
 		return res;
 	}
-
-	//----------------------------------------------------------------------------
-#ifdef _WININET_
-	inline std::string get_string_from_systemtime(const SYSTEMTIME& sys_time)
-	{
-		std::string result_string;
-
-		char buff[100] = {0};
-		BOOL res = ::InternetTimeFromSystemTimeA(&sys_time, INTERNET_RFC1123_FORMAT, buff, 99);
-		if(!res)
-		{
-			LOG_ERROR("Failed to load SytemTime to string");
-		}
-
-		result_string = buff;
-		return result_string;
-
-	}
-	//-------------------------------------------------------------------------------------
-	inline SYSTEMTIME get_systemtime_from_string(const std::string& buff)
-	{
-		SYSTEMTIME result_time = {0};
-
-		BOOL res = ::InternetTimeToSystemTimeA(buff.c_str(), &result_time, NULL);
-		if(!res)
-		{
-			LOG_ERROR("Failed to load SytemTime from string " << buff << "interval set to 15 minutes");
-		}
-
-		return result_time;
-	}
-#endif
-
-#ifdef WINDOWS_PLATFORM
-	static const DWORD INFO_BUFFER_SIZE = 10000;
-
-	static const wchar_t* get_pc_name()
-	{
-		static wchar_t	info[INFO_BUFFER_SIZE];
-		static DWORD	bufCharCount = INFO_BUFFER_SIZE;
-		static bool		init = false;
-
-		if (!init) {
-			if (!GetComputerNameW( info, &bufCharCount ))
-				info[0] = 0;
-			else
-				init = true;
-		}
-
-		return info;
-	}
-
-	static const wchar_t* get_user_name()
-	{
-		static wchar_t	info[INFO_BUFFER_SIZE];
-		static DWORD	bufCharCount = INFO_BUFFER_SIZE;
-		static bool		init = false;
-
-		if (!init) {
-			if (!GetUserNameW( info, &bufCharCount ))
-				info[0] = 0;
-			else
-				init = true;
-		}
-
-		return info;
-	}
-#endif
-
-#ifdef _LM_
-	static const wchar_t* get_domain_name()
-	{
-		static wchar_t	info[INFO_BUFFER_SIZE];
-		static DWORD	bufCharCount = 0;
-		static bool		init = false;
-
-		if (!init) {
-			LPWSTR domain( NULL );
-			NETSETUP_JOIN_STATUS status;
-			info[0] = 0;
-
-			if (NET_API_STATUS result = NetGetJoinInformation( NULL, &domain, &status )) 
-			{
-				LOG_ERROR("get_domain_name error: " << log_space::get_win32_err_descr(result));
-			} else
-			{
-				StringCchCopyW( info, sizeof(info)/sizeof( info[0] ), domain );
-				NetApiBufferFree((void*)domain);
-				init = true;
-			}
-		}
-
-		return info;
-	}
-#endif
-#ifdef WINDOWS_PLATFORM
-	inline
-	std::string load_resource_string_a(int id, const char* pmodule_name = NULL)
-	{
-		//slow realization 
-		HMODULE h = ::GetModuleHandleA( pmodule_name );
-		
-		char buff[2000] = {0};
-		
-		::LoadStringA( h, id, buff, sizeof(buff));
-		buff[sizeof(buff)-1] = 0; //be happy :)
-		return buff;
-	}
-	inline
-	std::wstring load_resource_string_w(int id, const char* pmodule_name = NULL)
-	{
-		//slow realization 
-		HMODULE h = ::GetModuleHandleA( pmodule_name );
-		
-		wchar_t buff[2000] = {0};
-		
-		::LoadStringW( h, id, buff, sizeof(buff) / sizeof( buff[0] ) );
-		buff[(sizeof(buff)/sizeof(buff[0]))-1] = 0; //be happy :)
-		return buff;	
-	}
+  //----------------------------------------------------------------------------
+#ifdef _WIN32
+  inline std::wstring utf8_to_utf16(const std::string& str)
+  {
+    if (str.empty())
+      return {};
+    int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0);
+    if (wstr_size == 0)
+    {
+      throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message());
+    }
+    std::wstring wstr(wstr_size, wchar_t{});
+    if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size))
+    {
+      throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message());
+    }
+    return wstr;
+  }
+  inline std::string utf16_to_utf8(const std::wstring& wstr)
+  {
+    if (wstr.empty())
+      return {};
+    int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL);
+    if (str_size == 0)
+    {
+      throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message());
+    }
+    std::string str(str_size, char{});
+    if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL))
+    {
+      throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message());
+    }
+    return str;
+  }
 #endif
 }
 }

src/contrib/epee/include/syncobj.h

@@ -30,15 +30,25 @@
 #ifndef __WINH_OBJ_H__
 #define __WINH_OBJ_H__
 
-#include <condition_variable>
-#include <mutex>
+#include <boost/chrono/duration.hpp>
+#include <boost/thread/condition_variable.hpp>
 #include <boost/thread/locks.hpp>
 #include <boost/thread/mutex.hpp>
 #include <boost/thread/recursive_mutex.hpp>
+#include <boost/thread/thread.hpp>
 
 namespace epee
 {
 
+  namespace debug
+  {
+    inline unsigned int &g_test_dbg_lock_sleep()
+    {
+      static unsigned int value = 0;
+      return value;
+    }
+  }
+  
   struct simple_event
   {
     simple_event() : m_rised(false)
@@ -47,22 +57,22 @@ namespace epee
 
     void raise()
     {
-      std::unique_lock<std::mutex> lock(m_mx);
+      boost::unique_lock<boost::mutex> lock(m_mx);
       m_rised = true;
       m_cond_var.notify_one();
     }
 
     void wait()
     {
-      std::unique_lock<std::mutex> lock(m_mx);
+      boost::unique_lock<boost::mutex> lock(m_mx);
       while (!m_rised) 
         m_cond_var.wait(lock);
       m_rised = false;
     }
 
   private:
-    std::mutex m_mx;
-    std::condition_variable m_cond_var;
+    boost::mutex m_mx;
+    boost::condition_variable m_cond_var;
     bool m_rised;
   };
 
@@ -215,10 +225,10 @@ namespace epee
 #define  SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard   critical_region_var(x)
 #define  EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard   critical_region_var(x)
 
-#define  CRITICAL_REGION_LOCAL(x) epee::critical_region_t<decltype(x)>   critical_region_var(x)
-#define  CRITICAL_REGION_BEGIN(x) { epee::critical_region_t<decltype(x)>   critical_region_var(x)
-#define  CRITICAL_REGION_LOCAL1(x) epee::critical_region_t<decltype(x)>   critical_region_var1(x)
-#define  CRITICAL_REGION_BEGIN1(x) { epee::critical_region_t<decltype(x)>   critical_region_var1(x)
+#define  CRITICAL_REGION_LOCAL(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));}   epee::critical_region_t<decltype(x)>   critical_region_var(x)
+#define  CRITICAL_REGION_BEGIN(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)>   critical_region_var(x)
+#define  CRITICAL_REGION_LOCAL1(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)>   critical_region_var1(x)
+#define  CRITICAL_REGION_BEGIN1(x) {  boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)>   critical_region_var1(x)
 
 #define  CRITICAL_REGION_END() }
 

src/contrib/epee/include/time_helper.h

@@ -156,4 +156,4 @@ PRAGMA_WARNING_POP
 
 #endif
 }
-}
+}

src/contrib/epee/include/warnings.h

@@ -27,4 +27,4 @@
 
 #define DISABLE_GCC_AND_CLANG_WARNING(w) _Pragma(BOOST_PP_STRINGIZE(GCC diagnostic ignored BOOST_PP_STRINGIZE(-W##w)))
 
-#endif
+#endif

src/contrib/epee/include/wipeable_string.h

@@ -0,0 +1,110 @@
+// Copyright (c) 2017-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/optional/optional.hpp>
+#include <stddef.h>
+#include <vector>
+#include <string>
+#include "memwipe.h"
+#include "fnv1.h"
+
+namespace epee
+{
+  class wipeable_string
+  {
+  public:
+    typedef char value_type;
+
+    wipeable_string() {}
+    wipeable_string(const wipeable_string &other);
+    wipeable_string(wipeable_string &&other);
+    wipeable_string(const std::string &other);
+    wipeable_string(std::string &&other);
+    wipeable_string(const char *s);
+    wipeable_string(const char *s, size_t len);
+    ~wipeable_string();
+    void wipe();
+    void push_back(char c);
+    void operator+=(char c);
+    void operator+=(const std::string &s);
+    void operator+=(const epee::wipeable_string &s);
+    void operator+=(const char *s);
+    void append(const char *ptr, size_t len);
+    char pop_back();
+    const char *data() const noexcept { return buffer.data(); }
+    char *data() noexcept { return buffer.data(); }
+    size_t size() const noexcept { return buffer.size(); }
+    size_t length() const noexcept { return buffer.size(); }
+    bool empty() const noexcept { return buffer.empty(); }
+    void trim();
+    void split(std::vector<wipeable_string> &fields) const;
+    boost::optional<wipeable_string> parse_hexstr() const;
+    template<typename T> inline bool hex_to_pod(T &pod) const;
+    template<typename T> inline bool hex_to_pod(tools::scrubbed<T> &pod) const { return hex_to_pod(unwrap(pod)); }
+    void resize(size_t sz);
+    void reserve(size_t sz);
+    void clear();
+    bool operator==(const wipeable_string &other) const noexcept { return buffer == other.buffer; }
+    bool operator!=(const wipeable_string &other) const noexcept { return buffer != other.buffer; }
+    wipeable_string &operator=(wipeable_string &&other);
+    wipeable_string &operator=(const wipeable_string &other);
+
+  private:
+    void grow(size_t sz, size_t reserved = 0);
+
+  private:
+    std::vector<char> buffer;
+  };
+
+  template<typename T> inline bool wipeable_string::hex_to_pod(T &pod) const
+  {
+    static_assert(std::is_pod<T>::value, "expected pod type");
+    if (size() != sizeof(T) * 2)
+      return false;
+    boost::optional<epee::wipeable_string> blob = parse_hexstr();
+    if (!blob)
+      return false;
+    if (blob->size() != sizeof(T))
+      return false;
+    pod = *(const T*)blob->data();
+    return true;
+  }
+}
+
+namespace std
+{
+  template<> struct hash<epee::wipeable_string>
+  {
+    size_t operator()(const epee::wipeable_string &s) const
+    {
+      return epee::fnv::FNV1a(s.data(), s.size());
+    }
+  };
+}

src/crypto/crypto.cpp

@@ -50,279 +50,8 @@ namespace crypto {
     return &reinterpret_cast<const unsigned char &>(scalar);
   }
 
-  static inline void random_scalar(ec_scalar &res) {
-    unsigned char tmp[64];
-    generate_random_bytes(64, tmp);
-    sc_reduce(tmp);
-    memcpy(&res, tmp, 32);
-  }
-
-  static inline void hash_to_scalar(const void *data, size_t length, ec_scalar &res) {
-    cn_fast_hash(data, length, reinterpret_cast<hash &>(res));
-    sc_reduce32(&res);
-  }
-
-  void crypto_ops::generate_keys(public_key &pub, secret_key &sec) {
-    lock_guard<mutex> lock(random_lock);
-    ge_p3 point;
-    random_scalar(sec);
-    ge_scalarmult_base(&point, &sec);
-    ge_p3_tobytes(&pub, &point);
-  }
-
   bool crypto_ops::check_key(const public_key &key) {
     ge_p3 point;
     return ge_frombytes_vartime(&point, &key) == 0;
   }
-
-  bool crypto_ops::secret_key_to_public_key(const secret_key &sec, public_key &pub) {
-    ge_p3 point;
-    if (sc_check(&sec) != 0) {
-      return false;
-    }
-    ge_scalarmult_base(&point, &sec);
-    ge_p3_tobytes(&pub, &point);
-    return true;
-  }
-
-  bool crypto_ops::generate_key_derivation(const public_key &key1, const secret_key &key2, key_derivation &derivation) {
-    ge_p3 point;
-    ge_p2 point2;
-    ge_p1p1 point3;
-    assert(sc_check(&key2) == 0);
-    if (ge_frombytes_vartime(&point, &key1) != 0) {
-      return false;
-    }
-    ge_scalarmult(&point2, &key2, &point);
-    ge_mul8(&point3, &point2);
-    ge_p1p1_to_p2(&point2, &point3);
-    ge_tobytes(&derivation, &point2);
-    return true;
-  }
-
-  static void derivation_to_scalar(const key_derivation &derivation, size_t output_index, ec_scalar &res) {
-    struct {
-      key_derivation derivation;
-      char output_index[(sizeof(size_t) * 8 + 6) / 7];
-    } buf;
-    char *end = buf.output_index;
-    buf.derivation = derivation;
-    tools::write_varint(end, output_index);
-    assert(end <= buf.output_index + sizeof buf.output_index);
-    hash_to_scalar(&buf, end - reinterpret_cast<char *>(&buf), res);
-  }
-
-  bool crypto_ops::derive_public_key(const key_derivation &derivation, size_t output_index,
-    const public_key &base, public_key &derived_key) {
-    ec_scalar scalar;
-    ge_p3 point1;
-    ge_p3 point2;
-    ge_cached point3;
-    ge_p1p1 point4;
-    ge_p2 point5;
-    if (ge_frombytes_vartime(&point1, &base) != 0) {
-      return false;
-    }
-    derivation_to_scalar(derivation, output_index, scalar);
-    ge_scalarmult_base(&point2, &scalar);
-    ge_p3_to_cached(&point3, &point2);
-    ge_add(&point4, &point1, &point3);
-    ge_p1p1_to_p2(&point5, &point4);
-    ge_tobytes(&derived_key, &point5);
-    return true;
-  }
-
-  void crypto_ops::derive_secret_key(const key_derivation &derivation, size_t output_index,
-    const secret_key &base, secret_key &derived_key) {
-    ec_scalar scalar;
-    assert(sc_check(&base) == 0);
-    derivation_to_scalar(derivation, output_index, scalar);
-    sc_add(&derived_key, &base, &scalar);
-  }
-
-  struct s_comm {
-    hash h;
-    ec_point key;
-    ec_point comm;
-  };
-
-  void crypto_ops::generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) {
-    lock_guard<mutex> lock(random_lock);
-    ge_p3 tmp3;
-    ec_scalar k;
-    s_comm buf;
-#if !defined(NDEBUG)
-    {
-      ge_p3 t;
-      public_key t2;
-      assert(sc_check(&sec) == 0);
-      ge_scalarmult_base(&t, &sec);
-      ge_p3_tobytes(&t2, &t);
-      assert(pub == t2);
-    }
-#endif
-    buf.h = prefix_hash;
-    buf.key = pub;
-    random_scalar(k);
-    ge_scalarmult_base(&tmp3, &k);
-    ge_p3_tobytes(&buf.comm, &tmp3);
-    hash_to_scalar(&buf, sizeof(s_comm), sig.c);
-    sc_mulsub(&sig.r, &sig.c, &sec, &k);
-  }
-
-  bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) {
-    ge_p2 tmp2;
-    ge_p3 tmp3;
-    ec_scalar c;
-    s_comm buf;
-    assert(check_key(pub));
-    buf.h = prefix_hash;
-    buf.key = pub;
-    if (ge_frombytes_vartime(&tmp3, &pub) != 0) {
-      abort();
-    }
-    if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) {
-      return false;
-    }
-    ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r);
-    ge_tobytes(&buf.comm, &tmp2);
-    hash_to_scalar(&buf, sizeof(s_comm), c);
-    sc_sub(&c, &c, &sig.c);
-    return sc_isnonzero(&c) == 0;
-  }
-
-  static void hash_to_ec(const public_key &key, ge_p3 &res) {
-    hash h;
-    ge_p2 point;
-    ge_p1p1 point2;
-    cn_fast_hash(std::addressof(key), sizeof(public_key), h);
-    ge_fromfe_frombytes_vartime(&point, reinterpret_cast<const unsigned char *>(&h));
-    ge_mul8(&point2, &point);
-    ge_p1p1_to_p3(&res, &point2);
-  }
-
-  void crypto_ops::generate_key_image(const public_key &pub, const secret_key &sec, key_image &image) {
-    ge_p3 point;
-    ge_p2 point2;
-    assert(sc_check(&sec) == 0);
-    hash_to_ec(pub, point);
-    ge_scalarmult(&point2, &sec, &point);
-    ge_tobytes(&image, &point2);
-  }
-
-PUSH_WARNINGS
-DISABLE_VS_WARNINGS(4200)
-  struct rs_comm {
-    hash h;
-    struct {
-      ec_point a, b;
-    } ab[];
-  } rcs;
-POP_WARNINGS
-
-  static inline size_t rs_comm_size(size_t pubs_count) {
-    return sizeof(rs_comm) + pubs_count * sizeof(rcs.ab[0]);
-  }
-
-  void crypto_ops::generate_ring_signature(const hash &prefix_hash, const key_image &image,
-    const public_key *const *pubs, size_t pubs_count,
-    const secret_key &sec, size_t sec_index,
-    signature *sig) {
-    lock_guard<mutex> lock(random_lock);
-    size_t i;
-    ge_p3 image_unp;
-    ge_dsmp image_pre;
-    ec_scalar sum, k, h;
-    rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count)));
-    assert(sec_index < pubs_count);
-#if !defined(NDEBUG)
-    {
-      ge_p3 t;
-      public_key t2;
-      key_image t3;
-      assert(sc_check(&sec) == 0);
-      ge_scalarmult_base(&t, &sec);
-      ge_p3_tobytes(&t2, &t);
-      assert(*pubs[sec_index] == t2);
-      generate_key_image(*pubs[sec_index], sec, t3);
-      assert(image == t3);
-      for (i = 0; i < pubs_count; i++) {
-        assert(check_key(*pubs[i]));
-      }
-    }
-#endif
-    if (ge_frombytes_vartime(&image_unp, &image) != 0) {
-      abort();
-    }
-    ge_dsm_precomp(image_pre, &image_unp);
-    sc_0(&sum);
-    buf->h = prefix_hash;
-    for (i = 0; i < pubs_count; i++) {
-      ge_p2 tmp2;
-      ge_p3 tmp3;
-      if (i == sec_index) {
-        random_scalar(k);
-        ge_scalarmult_base(&tmp3, &k);
-        ge_p3_tobytes(&buf->ab[i].a, &tmp3);
-        hash_to_ec(*pubs[i], tmp3);
-        ge_scalarmult(&tmp2, &k, &tmp3);
-        ge_tobytes(&buf->ab[i].b, &tmp2);
-      } else {
-        random_scalar(sig[i].c);
-        random_scalar(sig[i].r);
-        if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) {
-          abort();
-        }
-        ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r);
-        ge_tobytes(&buf->ab[i].a, &tmp2);
-        hash_to_ec(*pubs[i], tmp3);
-        ge_double_scalarmult_precomp_vartime(&tmp2, &sig[i].r, &tmp3, &sig[i].c, image_pre);
-        ge_tobytes(&buf->ab[i].b, &tmp2);
-        sc_add(&sum, &sum, &sig[i].c);
-      }
-    }
-    hash_to_scalar(buf, rs_comm_size(pubs_count), h);
-    sc_sub(&sig[sec_index].c, &h, &sum);
-    sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &sec, &k);
-  }
-
-  bool crypto_ops::check_ring_signature(const hash &prefix_hash, const key_image &image,
-    const public_key *const *pubs, size_t pubs_count,
-    const signature *sig) {
-    size_t i;
-    ge_p3 image_unp;
-    ge_dsmp image_pre;
-    ec_scalar sum, h;
-    rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count)));
-#if !defined(NDEBUG)
-    for (i = 0; i < pubs_count; i++) {
-      assert(check_key(*pubs[i]));
-    }
-#endif
-    if (ge_frombytes_vartime(&image_unp, &image) != 0) {
-      return false;
-    }
-    ge_dsm_precomp(image_pre, &image_unp);
-    sc_0(&sum);
-    buf->h = prefix_hash;
-    for (i = 0; i < pubs_count; i++) {
-      ge_p2 tmp2;
-      ge_p3 tmp3;
-      if (sc_check(&sig[i].c) != 0 || sc_check(&sig[i].r) != 0) {
-        return false;
-      }
-      if (ge_frombytes_vartime(&tmp3, &*pubs[i]) != 0) {
-        abort();
-      }
-      ge_double_scalarmult_base_vartime(&tmp2, &sig[i].c, &tmp3, &sig[i].r);
-      ge_tobytes(&buf->ab[i].a, &tmp2);
-      hash_to_ec(*pubs[i], tmp3);
-      ge_double_scalarmult_precomp_vartime(&tmp2, &sig[i].r, &tmp3, &sig[i].c, image_pre);
-      ge_tobytes(&buf->ab[i].b, &tmp2);
-      sc_add(&sum, &sum, &sig[i].c);
-    }
-    hash_to_scalar(buf, rs_comm_size(pubs_count), h);
-    sc_sub(&h, &h, &sum);
-    return sc_isnonzero(&h) == 0;
-  }
 }

src/crypto/crypto.h

@@ -49,12 +49,16 @@ namespace crypto {
     ec_scalar c, r;
     friend class crypto_ops;
   };
+
+  POD_CLASS view_tag {
+    char data;
+  };
 #pragma pack(pop)
 
   static_assert(sizeof(ec_point) == 32 && sizeof(ec_scalar) == 32 &&
     sizeof(public_key) == 32 && sizeof(secret_key) == 32 &&
     sizeof(key_derivation) == 32 && sizeof(key_image) == 32 &&
-    sizeof(signature) == 64, "Invalid structure size");
+    sizeof(signature) == 64 && sizeof(view_tag) == 1, "Invalid structure size");
 
   class crypto_ops {
     crypto_ops();
@@ -62,32 +66,8 @@ namespace crypto {
     void operator=(const crypto_ops &);
     ~crypto_ops();
 
-    static void generate_keys(public_key &, secret_key &);
-    friend void generate_keys(public_key &, secret_key &);
     static bool check_key(const public_key &);
     friend bool check_key(const public_key &);
-    static bool secret_key_to_public_key(const secret_key &, public_key &);
-    friend bool secret_key_to_public_key(const secret_key &, public_key &);
-    static bool generate_key_derivation(const public_key &, const secret_key &, key_derivation &);
-    friend bool generate_key_derivation(const public_key &, const secret_key &, key_derivation &);
-    static bool derive_public_key(const key_derivation &, std::size_t, const public_key &, public_key &);
-    friend bool derive_public_key(const key_derivation &, std::size_t, const public_key &, public_key &);
-    static void derive_secret_key(const key_derivation &, std::size_t, const secret_key &, secret_key &);
-    friend void derive_secret_key(const key_derivation &, std::size_t, const secret_key &, secret_key &);
-    static void generate_signature(const hash &, const public_key &, const secret_key &, signature &);
-    friend void generate_signature(const hash &, const public_key &, const secret_key &, signature &);
-    static bool check_signature(const hash &, const public_key &, const signature &);
-    friend bool check_signature(const hash &, const public_key &, const signature &);
-    static void generate_key_image(const public_key &, const secret_key &, key_image &);
-    friend void generate_key_image(const public_key &, const secret_key &, key_image &);
-    static void generate_ring_signature(const hash &, const key_image &,
-      const public_key *const *, std::size_t, const secret_key &, std::size_t, signature *);
-    friend void generate_ring_signature(const hash &, const key_image &,
-      const public_key *const *, std::size_t, const secret_key &, std::size_t, signature *);
-    static bool check_ring_signature(const hash &, const key_image &,
-      const public_key *const *, std::size_t, const signature *);
-    friend bool check_ring_signature(const hash &, const key_image &,
-      const public_key *const *, std::size_t, const signature *);
   };
 
   /* Generate a value filled with random bytes.
@@ -100,87 +80,14 @@ namespace crypto {
     return res;
   }
 
-  /* Generate a new key pair
-   */
-  inline void generate_keys(public_key &pub, secret_key &sec) {
-    crypto_ops::generate_keys(pub, sec);
-  }
-
   /* Check a public key. Returns true if it is valid, false otherwise.
    */
   inline bool check_key(const public_key &key) {
     return crypto_ops::check_key(key);
   }
-
-  /* Checks a private key and computes the corresponding public key.
-   */
-  inline bool secret_key_to_public_key(const secret_key &sec, public_key &pub) {
-    return crypto_ops::secret_key_to_public_key(sec, pub);
-  }
-
-  /* To generate an ephemeral key used to send money to:
-   * * The sender generates a new key pair, which becomes the transaction key. The public transaction key is included in "extra" field.
-   * * Both the sender and the receiver generate key derivation from the transaction key, the receivers' "view" key and the output index.
-   * * The sender uses key derivation and the receivers' "spend" key to derive an ephemeral public key.
-   * * The receiver can either derive the public key (to check that the transaction is addressed to him) or the private key (to spend the money).
-   */
-  inline bool generate_key_derivation(const public_key &key1, const secret_key &key2, key_derivation &derivation) {
-    return crypto_ops::generate_key_derivation(key1, key2, derivation);
-  }
-  inline bool derive_public_key(const key_derivation &derivation, std::size_t output_index,
-    const public_key &base, public_key &derived_key) {
-    return crypto_ops::derive_public_key(derivation, output_index, base, derived_key);
-  }
-  inline void derive_secret_key(const key_derivation &derivation, std::size_t output_index,
-    const secret_key &base, secret_key &derived_key) {
-    crypto_ops::derive_secret_key(derivation, output_index, base, derived_key);
-  }
-
-  /* Generation and checking of a standard signature.
-   */
-  inline void generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) {
-    crypto_ops::generate_signature(prefix_hash, pub, sec, sig);
-  }
-  inline bool check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) {
-    return crypto_ops::check_signature(prefix_hash, pub, sig);
-  }
-
-  /* To send money to a key:
-   * * The sender generates an ephemeral key and includes it in transaction output.
-   * * To spend the money, the receiver generates a key image from it.
-   * * Then he selects a bunch of outputs, including the one he spends, and uses them to generate a ring signature.
-   * To check the signature, it is necessary to collect all the keys that were used to generate it. To detect double spends, it is necessary to check that each key image is used at most once.
-   */
-  inline void generate_key_image(const public_key &pub, const secret_key &sec, key_image &image) {
-    crypto_ops::generate_key_image(pub, sec, image);
-  }
-  inline void generate_ring_signature(const hash &prefix_hash, const key_image &image,
-    const public_key *const *pubs, std::size_t pubs_count,
-    const secret_key &sec, std::size_t sec_index,
-    signature *sig) {
-    crypto_ops::generate_ring_signature(prefix_hash, image, pubs, pubs_count, sec, sec_index, sig);
-  }
-  inline bool check_ring_signature(const hash &prefix_hash, const key_image &image,
-    const public_key *const *pubs, std::size_t pubs_count,
-    const signature *sig) {
-    return crypto_ops::check_ring_signature(prefix_hash, image, pubs, pubs_count, sig);
-  }
-
-  /* Variants with vector<const public_key *> parameters.
-   */
-  inline void generate_ring_signature(const hash &prefix_hash, const key_image &image,
-    const std::vector<const public_key *> &pubs,
-    const secret_key &sec, std::size_t sec_index,
-    signature *sig) {
-    generate_ring_signature(prefix_hash, image, pubs.data(), pubs.size(), sec, sec_index, sig);
-  }
-  inline bool check_ring_signature(const hash &prefix_hash, const key_image &image,
-    const std::vector<const public_key *> &pubs,
-    const signature *sig) {
-    return check_ring_signature(prefix_hash, image, pubs.data(), pubs.size(), sig);
-  }
 }
 
 CRYPTO_MAKE_COMPARABLE(public_key)
 CRYPTO_MAKE_HASHABLE(key_image)
 CRYPTO_MAKE_COMPARABLE(signature)
+CRYPTO_MAKE_COMPARABLE(view_tag)

src/crypto/hash.h

@@ -24,6 +24,10 @@ namespace crypto {
     public:
     uint32_t data[40];
   };
+  POD_CLASS cycle48 {
+    public:
+    uint32_t data[48];
+  };
   POD_CLASS hash {
     char data[HASH_SIZE];
   };

src/cryptonote_config.h

@@ -1,6 +1,24 @@
 #pragma once
 
-#define CURRENT_TRANSACTION_VERSION 1
+#define CURRENT_TRANSACTION_VERSION      1
+#define POU_TRANSACTION_VERSION          6
+#define COLLATERAL_TRANSACTION_VERSION   7
+#define HAVEN_TYPES_TRANSACTION_VERSION  8
+#define OFFSHORE_TRANSACTION_VERSION     3
+#define HF_VERSION_XASSET_FEES_V2       17
+#define HF_VERSION_HAVEN2               18
+#define HF_VERSION_USE_COLLATERAL       20
+
+// UNLOCK TIMES
+#define TX_V6_OFFSHORE_UNLOCK_BLOCKS                    21*720  // 21 day unlock time
+#define TX_V6_ONSHORE_UNLOCK_BLOCKS                     360     // 12 hour unlock time
+#define TX_V7_ONSHORE_UNLOCK_BLOCKS                     21*720 // 21 day unlock time
+#define TX_V6_XASSET_UNLOCK_BLOCKS                      1440    // 2 day unlock time
+#define TX_V6_OFFSHORE_UNLOCK_BLOCKS_TESTNET            60     // 2 hour unlock time - FOR TESTING ONLY
+#define TX_V6_ONSHORE_UNLOCK_BLOCKS_TESTNET             30     // 1 hour unlock time - FOR TESTING ONLY
+#define TX_V6_XASSET_UNLOCK_BLOCKS_TESTNET              60     // 2 hour unlock time - FOR TESTING ONLY
+
+#define PRICING_RECORD_VALID_TIME_DIFF_FROM_BLOCK       120  // seconds
 
 enum BLOB_TYPE {
   BLOB_TYPE_CRYPTONOTE        = 0,
@@ -13,5 +31,9 @@ enum BLOB_TYPE {
   BLOB_TYPE_AEON              = 7, // Aeon
   BLOB_TYPE_CRYPTONOTE_CUCKOO = 8, // MoneroV / Swap
   BLOB_TYPE_CRYPTONOTE_XTNC   = 9, // XTNC
-  BLOB_TYPE_CRYPTONOTE_TUBE   = 10, // bittube
+  BLOB_TYPE_CRYPTONOTE_TUBE   = 10, // TUBE
+  BLOB_TYPE_CRYPTONOTE_XHV    = 11, // Haven
+  BLOB_TYPE_CRYPTONOTE_XTA    = 12, // ITALO
+  BLOB_TYPE_CRYPTONOTE_ZEPHYR = 13, // ZEPHYR
+  BLOB_TYPE_CRYPTONOTE_XLA    = 14, // XLA
 };

src/cryptonote_core/cryptonote_basic.h

@@ -14,9 +14,9 @@
 #include "serialization/variant.h"
 #include "serialization/vector.h"
 #include "serialization/binary_archive.h"
-#include "serialization/json_archive.h"
-#include "serialization/debug_archive.h"
 #include "serialization/crypto.h"
+#include "serialization/pricing_record.h"
+#include "serialization/zephyr_pricing_record.h"
 #include "serialization/keyvalue_serialization.h" // eepe named serialization
 #include "string_tools.h"
 #include "cryptonote_config.h"
@@ -26,6 +26,8 @@
 #include "tx_extra.h"
 #include "ringct/rctTypes.h"
 #include "cryptonote_protocol/blobdatatype.h"
+#include "offshore/pricing_record.h"
+#include "zephyr_oracle/pricing_record.h"
 
 
 namespace cryptonote
@@ -65,6 +67,7 @@ namespace cryptonote
     crypto::hash hash;
   };
 
+  // outputs <= HF_VERSION_VIEW_TAGS
   struct txout_to_key
   {
     txout_to_key() { }
@@ -72,6 +75,98 @@ namespace cryptonote
     crypto::public_key key;
   };
 
+  // outputs >= HF_VERSION_VIEW_TAGS
+  struct txout_to_tagged_key
+  {
+    txout_to_tagged_key() { }
+    txout_to_tagged_key(const crypto::public_key &_key, const crypto::view_tag &_view_tag) : key(_key), view_tag(_view_tag) { }
+    crypto::public_key key;
+    crypto::view_tag view_tag; // optimization to reduce scanning time
+
+    BEGIN_SERIALIZE_OBJECT()
+      FIELD(key)
+      FIELD(view_tag)
+    END_SERIALIZE()
+  };
+
+  // outputs <= HF_VERSION_VIEW_TAGS
+  struct txout_haven_key
+  {
+    txout_haven_key() { }
+    txout_haven_key(const crypto::public_key &_key, const std::string &_asset_type, const uint64_t &_unlock_time, const bool &_is_collateral, const bool &_is_collateral_change) : key(_key), asset_type(_asset_type), unlock_time(_unlock_time), is_collateral(_is_collateral), is_collateral_change(_is_collateral_change) { }
+    crypto::public_key key;
+    std::string asset_type;
+    uint64_t unlock_time;
+    bool is_collateral;
+    bool is_collateral_change;
+
+    BEGIN_SERIALIZE_OBJECT()
+      FIELD(key)
+      FIELD(asset_type)
+      VARINT_FIELD(unlock_time)
+      FIELD(is_collateral)
+      FIELD(is_collateral_change)
+    END_SERIALIZE()
+  };
+
+  // outputs >= HF_VERSION_VIEW_TAGS
+  struct txout_haven_tagged_key
+  {
+    txout_haven_tagged_key() { }
+    txout_haven_tagged_key(const crypto::public_key &_key, const std::string &_asset_type, const uint64_t &_unlock_time, const bool &_is_collateral, const bool &_is_collateral_change, const crypto::view_tag &_view_tag) : key(_key), asset_type(_asset_type), unlock_time(_unlock_time), is_collateral(_is_collateral), is_collateral_change(_is_collateral_change), view_tag(_view_tag) { }
+    crypto::public_key key;
+    std::string asset_type;
+    uint64_t unlock_time;
+    bool is_collateral;
+    bool is_collateral_change;
+    crypto::view_tag view_tag; // optimization to reduce scanning time
+
+    BEGIN_SERIALIZE_OBJECT()
+      FIELD(key)
+      FIELD(asset_type)
+      VARINT_FIELD(unlock_time)
+      FIELD(is_collateral)
+      FIELD(is_collateral_change)
+      FIELD(view_tag)
+    END_SERIALIZE()
+  };
+
+  struct txout_offshore
+  {
+    txout_offshore() { }
+    txout_offshore(const crypto::public_key &_key) : key(_key) { }
+    crypto::public_key key;
+  };
+
+  struct txout_xasset
+  {
+    txout_xasset() { }
+    txout_xasset(const crypto::public_key &_key, const std::string &_asset_type) : key(_key), asset_type(_asset_type) { }
+    crypto::public_key key;
+    std::string asset_type;
+
+    BEGIN_SERIALIZE_OBJECT()
+      FIELD(key)
+      FIELD(asset_type)
+    END_SERIALIZE()
+  };
+
+  // ZEPHYR
+  struct txout_zephyr_tagged_key
+  {
+    txout_zephyr_tagged_key() { }
+    txout_zephyr_tagged_key(const crypto::public_key &_key, const std::string &_asset_type, const crypto::view_tag &_view_tag) : key(_key), asset_type(_asset_type), view_tag(_view_tag) { }
+    crypto::public_key key;
+    std::string asset_type;
+    crypto::view_tag view_tag; // optimization to reduce scanning time
+
+    BEGIN_SERIALIZE_OBJECT()
+      FIELD(key)
+      FIELD(asset_type)
+      FIELD(view_tag)
+    END_SERIALIZE()
+  };
+
   /* inputs */
 
   struct txin_gen
@@ -124,12 +219,85 @@ namespace cryptonote
     END_SERIALIZE()
   };
 
+  struct txin_offshore
+  {
+    uint64_t amount;
+    std::vector<uint64_t> key_offsets;
+    crypto::key_image k_image;
 
-  typedef boost::variant<txin_gen, txin_to_script, txin_to_scripthash, txin_to_key> txin_v;
+    BEGIN_SERIALIZE_OBJECT()
+    VARINT_FIELD(amount)
+    FIELD(key_offsets)
+    FIELD(k_image)
+    END_SERIALIZE()
+  };
 
-  typedef boost::variant<txout_to_script, txout_to_scripthash, txout_to_key> txout_target_v;
+  struct txin_onshore
+  {
+    uint64_t amount;
+    std::vector<uint64_t> key_offsets;
+    crypto::key_image k_image;
+    
+    BEGIN_SERIALIZE_OBJECT()
+    VARINT_FIELD(amount)
+    FIELD(key_offsets)
+    FIELD(k_image)
+    END_SERIALIZE()
+  };
+
+  struct txin_haven_key
+  {
+    uint64_t amount;
+    std::string asset_type;
+    std::vector<uint64_t> key_offsets;
+    crypto::key_image k_image;      // double spending protection
+
+    BEGIN_SERIALIZE_OBJECT()
+    VARINT_FIELD(amount)
+    FIELD(asset_type)
+    FIELD(key_offsets)
+    FIELD(k_image)
+    END_SERIALIZE()
+  };
+
+  struct txin_xasset
+  {
+    uint64_t amount;
+    std::string asset_type;
+    std::vector<uint64_t> key_offsets;
+    crypto::key_image k_image;      // double spending protection
+
+    BEGIN_SERIALIZE_OBJECT()
+    VARINT_FIELD(amount)
+    FIELD(asset_type)
+    FIELD(key_offsets)
+    FIELD(k_image)
+    END_SERIALIZE()
+  };
+
+  struct txin_zephyr_key
+  {
+    uint64_t amount;
+    std::string asset_type;
+    std::vector<uint64_t> key_offsets;
+    crypto::key_image k_image;      // double spending protection
+
+    BEGIN_SERIALIZE_OBJECT()
+      VARINT_FIELD(amount)
+      FIELD(asset_type)
+      FIELD(key_offsets)
+      FIELD(k_image)
+    END_SERIALIZE()
+  };
+  
+  typedef boost::variant<txin_gen, txin_to_script, txin_to_scripthash, txin_to_key, txin_offshore, txin_onshore, txin_xasset, txin_haven_key> txin_v;
+  typedef boost::variant<txin_gen, txin_to_script, txin_to_scripthash, txin_zephyr_key> txin_zephyr_v;
+
+  typedef boost::variant<txout_to_script, txout_to_scripthash, txout_to_key, txout_to_tagged_key> txout_target_v;
+  typedef boost::variant<txout_to_script, txout_to_scripthash, txout_to_key, txout_offshore, txout_xasset, txout_haven_key, txout_haven_tagged_key> txout_xhv_target_v;
+
+  typedef boost::variant<txout_to_script, txout_to_scripthash, txout_zephyr_tagged_key> txout_stablero_target_v;
 
-  //typedef std::pair<uint64_t, txout> out_t;
   struct tx_out
   {
     uint64_t amount;
@@ -141,6 +309,29 @@ namespace cryptonote
     END_SERIALIZE()
   };
 
+  struct tx_out_xhv
+  {
+    uint64_t amount;
+    txout_xhv_target_v target;
+
+    BEGIN_SERIALIZE_OBJECT()
+      VARINT_FIELD(amount)
+      FIELD(target)
+    END_SERIALIZE()
+  };
+
+  struct tx_out_zephyr
+  {
+    uint64_t amount;
+    txout_stablero_target_v target;
+
+    BEGIN_SERIALIZE_OBJECT()
+      VARINT_FIELD(amount)
+      FIELD(target)
+    END_SERIALIZE()
+  };
+
+
   enum loki_version
   {
     loki_version_0 = 0,
@@ -160,14 +351,24 @@ namespace cryptonote
     uint64_t unlock_time;  //number of block (or time), used as a limitation like: spend this tx not early then block/time
 
     std::vector<txin_v> vin;
+    std::vector<txin_zephyr_v> vin_zephyr;
     std::vector<tx_out> vout;
+    std::vector<tx_out_xhv> vout_xhv;
+    std::vector<tx_out_zephyr> vout_zephyr;
     //extra
     std::vector<uint8_t> extra;
+    // Block height to use PR from
+    uint64_t pricing_record_height;
+    // Circulating supply information
+    std::vector<uint8_t> offshore_data;
+    uint64_t amount_burnt;
+    uint64_t amount_minted;
+    std::vector<uint64_t> output_unlock_times;
+    std::vector<uint32_t> collateral_indices;
 
     //
     // NOTE: Loki specific
     //
-    std::vector<uint64_t> output_unlock_times;
     enum loki_type_t
     {
       loki_type_standard,
@@ -183,25 +384,289 @@ namespace cryptonote
     };
 
     BEGIN_SERIALIZE()
-      VARINT_FIELD(version)
-      if (version > loki_version_2 && (blob_type == BLOB_TYPE_CRYPTONOTE_LOKI || blob_type == BLOB_TYPE_CRYPTONOTE_XTNC))
-      {
-        FIELD(output_unlock_times)
-        if (version == loki_version_3_per_output_unlock_times)
-          FIELD(is_deregister)
-      }
-      VARINT_FIELD(unlock_time)
-      FIELD(vin)
-      FIELD(vout)
-      if (blob_type == BLOB_TYPE_CRYPTONOTE_LOKI || blob_type == BLOB_TYPE_CRYPTONOTE_XTNC)
-      {
-        if (version >= loki_version_3_per_output_unlock_times && vout.size() != output_unlock_times.size()) return false;
-      }
-      FIELD(extra)
-      if ((blob_type == BLOB_TYPE_CRYPTONOTE_LOKI || blob_type == BLOB_TYPE_CRYPTONOTE_XTNC) && version >= loki_version_4_tx_types)
-      {
-        VARINT_FIELD(type)
-        if (static_cast<uint16_t>(type) >= loki_type_count) return false;
+      if (blob_type == BLOB_TYPE_CRYPTONOTE_XHV) {
+        VARINT_FIELD(version)
+          //if(version == 0 || CURRENT_TRANSACTION_VERSION < version) return false;
+      
+        // Only transactions prior to HAVEN_TYPES_TRANSACTION_VERSION are permitted to be anything other than HAVEN_TYPES and need translation
+        if (version < HAVEN_TYPES_TRANSACTION_VERSION) {
+  
+          if (version < POU_TRANSACTION_VERSION) {
+            VARINT_FIELD(unlock_time)
+          }
+          if (!typename Archive<W>::is_saving()) {
+            FIELD(vin)
+            FIELD(vout_xhv)
+            FIELD(extra)
+            if(version >= OFFSHORE_TRANSACTION_VERSION) {
+              VARINT_FIELD(pricing_record_height)
+              if (version < 5)
+                FIELD(offshore_data)
+              if (version >= POU_TRANSACTION_VERSION) {
+                FIELD(output_unlock_times)
+                if (vout_xhv.size() != output_unlock_times.size()) {
+                  return false;
+                }
+              }
+              VARINT_FIELD(amount_burnt)
+              VARINT_FIELD(amount_minted)
+              if (version >= COLLATERAL_TRANSACTION_VERSION && amount_burnt) {
+                FIELD(collateral_indices)
+                if (collateral_indices.size() != 2) {
+                  return false;
+                }
+                for (const auto vout_idx: collateral_indices) {
+                  if (vout_idx >= vout_xhv.size())
+                    return false;
+                }
+              }
+            }
+            std::vector<txin_v> vin_tmp(vin);
+            bool is_conversion_tx = (amount_burnt != 0);
+            bool is_offshore_tx = is_conversion_tx;
+            bool is_onshore_tx = false;
+            vin.clear();
+            for (auto &vin_entry: vin_tmp) {
+              if (vin_entry.type() == typeid(txin_gen)) {
+                vin.push_back(vin_entry);
+                continue;
+              }
+              txin_haven_key in;
+              if (vin_entry.type() == typeid(txin_to_key)) {
+                in.asset_type = "XHV";
+                in.amount = boost::get<txin_to_key>(vin_entry).amount;
+                in.key_offsets = boost::get<txin_to_key>(vin_entry).key_offsets;
+                in.k_image = boost::get<txin_to_key>(vin_entry).k_image;
+              } else if (vin_entry.type() == typeid(txin_offshore)) {
+                is_offshore_tx = false;
+                is_onshore_tx = false;
+                in.asset_type = "XUSD";
+                in.amount = boost::get<txin_offshore>(vin_entry).amount;
+                in.key_offsets = boost::get<txin_offshore>(vin_entry).key_offsets;
+                in.k_image = boost::get<txin_offshore>(vin_entry).k_image;
+              } else if (vin_entry.type() == typeid(txin_onshore)) {
+                is_offshore_tx = false;
+                is_onshore_tx = true;
+                in.asset_type = "XUSD";
+                in.amount = boost::get<txin_onshore>(vin_entry).amount;
+                in.key_offsets = boost::get<txin_onshore>(vin_entry).key_offsets;
+                in.k_image = boost::get<txin_onshore>(vin_entry).k_image;
+              } else if (vin_entry.type() == typeid(txin_xasset)) {
+                is_offshore_tx = false;
+                is_onshore_tx = false;
+                in.amount = boost::get<txin_xasset>(vin_entry).amount;
+                in.key_offsets = boost::get<txin_xasset>(vin_entry).key_offsets;
+                in.k_image = boost::get<txin_xasset>(vin_entry).k_image;
+                in.asset_type = boost::get<txin_xasset>(vin_entry).asset_type;
+              } else {
+                return false;
+              }
+              vin.push_back(in);
+            }
+            std::vector<tx_out_xhv> vout_tmp(vout_xhv);
+            vout_xhv.clear();
+            for (size_t i=0; i<vout_tmp.size(); i++) {
+              txout_haven_key out;
+              if (vout_tmp[i].target.type() == typeid(txout_to_key)) {
+                out.asset_type = "XHV";
+                out.key = boost::get<txout_to_key>(vout_tmp[i].target).key;
+              } else if (vout_tmp[i].target.type() == typeid(txout_offshore)) {
+                out.asset_type = "XUSD";
+                out.key = boost::get<txout_offshore>(vout_tmp[i].target).key;
+              } else if (vout_tmp[i].target.type() == typeid(txout_xasset)) {
+                out.asset_type = boost::get<txout_xasset>(vout_tmp[i].target).asset_type;
+                out.key = boost::get<txout_xasset>(vout_tmp[i].target).key;
+              } else {
+                return false;
+              }
+              out.unlock_time = (version >= POU_TRANSACTION_VERSION) ? output_unlock_times[i] : unlock_time;
+              out.is_collateral = false;
+              out.is_collateral_change = false;
+              if (version >= COLLATERAL_TRANSACTION_VERSION && amount_burnt) {
+                if (((is_onshore_tx) &&
+                     (collateral_indices[0] == i)) ||
+                    ((!is_onshore_tx) &&
+                     (is_offshore_tx) &&
+                     (collateral_indices[0] == i && collateral_indices[1] == 0))) {
+                    out.is_collateral = true;
+                }
+                if (is_onshore_tx && collateral_indices[1] == i) {
+                  out.is_collateral_change = true;
+                }
+              }
+              tx_out_xhv foo;
+              foo.amount = vout_tmp[i].amount;
+              foo.target = out;
+              vout_xhv.push_back(foo);
+            }
+            return true;
+          }
+          bool is_offshore_tx = (amount_burnt != 0);
+          bool is_onshore_tx = false;
+          std::vector<txin_v> vin_tmp;
+          vin_tmp.reserve(vin.size());
+          for (auto &vin_entry_v: vin) {
+            if (vin_entry_v.type() == typeid(txin_gen)) {
+              vin_tmp.push_back(vin_entry_v);
+              continue;
+            }
+            txin_haven_key vin_entry = boost::get<txin_haven_key>(vin_entry_v);
+            if (vin_entry.asset_type == "XHV") {
+              txin_to_key in;
+              in.amount = vin_entry.amount;
+              in.key_offsets = vin_entry.key_offsets;
+              in.k_image = vin_entry.k_image;
+              vin_tmp.push_back(in);
+            } else if (vin_entry.asset_type == "XUSD") {
+              is_offshore_tx = false;
+              int xhv_outputs = std::count_if(vout_xhv.begin(), vout_xhv.end(), [](tx_out_xhv &foo_v) {
+                if (foo_v.target.type() == typeid(txout_haven_key)) {
+                  txout_haven_key out = boost::get<txout_haven_key>(foo_v.target);
+                  return out.asset_type == "XHV";
+                } else if (foo_v.target.type() == typeid(txout_haven_tagged_key)) {
+                  txout_haven_tagged_key out = boost::get<txout_haven_tagged_key>(foo_v.target);
+                  return out.asset_type == "XHV";
+                } else {
+                  return false;
+                }
+              });
+              if (xhv_outputs) {
+                is_onshore_tx = true;
+                txin_onshore in;
+                in.amount = vin_entry.amount;
+                in.key_offsets = vin_entry.key_offsets;
+                in.k_image = vin_entry.k_image;
+                vin_tmp.push_back(in);
+              } else {
+                txin_offshore in;
+                in.amount = vin_entry.amount;
+                in.key_offsets = vin_entry.key_offsets;
+                in.k_image = vin_entry.k_image;
+                vin_tmp.push_back(in);
+              }
+            } else {
+              is_offshore_tx = false;
+              txin_xasset in;
+              in.amount = vin_entry.amount;
+              in.asset_type = vin_entry.asset_type;
+              in.key_offsets = vin_entry.key_offsets;
+              in.k_image = vin_entry.k_image;
+              vin_tmp.push_back(in);
+            }
+          }
+          std::vector<tx_out_xhv> vout_tmp;
+          vout_tmp.reserve(vout_xhv.size());
+          output_unlock_times.resize(vout_xhv.size());
+          std::vector<uint32_t> collateral_indices_temp;
+          collateral_indices_temp.resize(2);
+          for (size_t i=0; i<vout_xhv.size(); i++) {
+            txout_haven_key outhk = boost::get<txout_haven_key>(vout_xhv[i].target);
+            tx_out_xhv foo;
+            foo.amount = vout_xhv[i].amount;
+            if (outhk.asset_type == "XHV") {
+              txout_to_key out;
+              out.key = outhk.key;
+              foo.target = out;
+            } else if (outhk.asset_type == "XUSD") {
+              txout_offshore out;
+              out.key = outhk.key;
+              foo.target = out;
+            } else {
+              txout_xasset out;
+              out.asset_type = outhk.asset_type;
+              out.key = outhk.key;
+              foo.target = out;
+            }
+            output_unlock_times[i] = outhk.unlock_time;
+            if (outhk.is_collateral) {
+              collateral_indices_temp[0] = i;
+            } else if (outhk.is_collateral_change) {
+              collateral_indices_temp[1] = i;
+            }
+            vout_tmp.push_back(foo);
+          }
+          FIELD_N("vin", vin_tmp)
+          FIELD_N("vout", vout_tmp)
+          FIELD(extra)
+          if(version >= OFFSHORE_TRANSACTION_VERSION) {
+            VARINT_FIELD(pricing_record_height)
+            if (version < 5)
+              FIELD(offshore_data)
+            if (version >= POU_TRANSACTION_VERSION) {
+              FIELD(output_unlock_times)
+              if (vout_xhv.size() != output_unlock_times.size()) {
+                return false;
+              }
+            }
+            VARINT_FIELD(amount_burnt)
+            VARINT_FIELD(amount_minted)
+            if (version >= COLLATERAL_TRANSACTION_VERSION && amount_burnt) {
+              if (collateral_indices.size() != 2) {
+                if ((is_offshore_tx || is_onshore_tx) && collateral_indices_temp.size() != 2) {
+                  return false;
+                }
+                if (is_offshore_tx || is_onshore_tx) {
+                  collateral_indices = collateral_indices_temp;
+                } else {
+                  collateral_indices.clear();
+                  collateral_indices.push_back(0);
+                  collateral_indices.push_back(0);
+                }
+              }
+              FIELD(collateral_indices)
+              for (const auto vout_idx: collateral_indices) {
+                if (vout_idx >= vout_xhv.size())
+                  return false;
+              }
+            }
+          }
+          return true;
+        }
+  
+        FIELD(vin)
+        FIELD(vout_xhv)
+        FIELD(extra)
+        VARINT_FIELD(pricing_record_height)
+        VARINT_FIELD(amount_burnt)
+        VARINT_FIELD(amount_minted)
+  
+      } else {
+  
+        VARINT_FIELD(version)
+        if (version > loki_version_2 && (blob_type == BLOB_TYPE_CRYPTONOTE_LOKI || blob_type == BLOB_TYPE_CRYPTONOTE_XTNC))
+        {
+          FIELD(output_unlock_times)
+          if (version == loki_version_3_per_output_unlock_times)
+            FIELD(is_deregister)
+        }
+  
+        VARINT_FIELD(unlock_time)
+  
+        if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR)
+          FIELD(vin_zephyr)
+        else 
+          FIELD(vin)
+  
+        if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR)
+          FIELD(vout_zephyr)
+        else
+          FIELD(vout)
+  
+        if (blob_type == BLOB_TYPE_CRYPTONOTE_LOKI || blob_type == BLOB_TYPE_CRYPTONOTE_XTNC)
+        {
+          if (version >= loki_version_3_per_output_unlock_times && vout.size() != output_unlock_times.size()) return false;
+        }
+        FIELD(extra)
+        if ((blob_type == BLOB_TYPE_CRYPTONOTE_LOKI || blob_type == BLOB_TYPE_CRYPTONOTE_XTNC) && version >= loki_version_4_tx_types)
+        {
+          VARINT_FIELD(type)
+          if (static_cast<uint16_t>(type) >= loki_type_count) return false;
+        }
+        if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR) {
+          VARINT_FIELD(pricing_record_height)
+          VARINT_FIELD(amount_burnt)
+          VARINT_FIELD(amount_minted)
+        }
       }
     END_SERIALIZE()
 
@@ -257,18 +722,38 @@ namespace cryptonote
       else
       {
         ar.tag("rct_signatures");
-        if (!vin.empty())
+        if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR ? !vin_zephyr.empty() : !vin.empty())
         {
           ar.begin_object();
-          bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size());
+          bool r;
+          if (blob_type == BLOB_TYPE_CRYPTONOTE_XHV)
+            r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout_xhv.size());
+          else if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR)
+            r = rct_signatures.serialize_rctsig_base(ar, vin_zephyr.size(), vout_zephyr.size());
+          else
+            r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size());
           if (!r || !ar.stream().good()) return false;
           ar.end_object();
           if (rct_signatures.type != rct::RCTTypeNull)
           {
             ar.tag("rctsig_prunable");
             ar.begin_object();
-            r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin.size(), vout.size(),
-                vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(vin[0]).key_offsets.size() - 1 : 0);
+            if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR) {
+              r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin_zephyr.size(), vout_zephyr.size(),
+                  vin_zephyr[0].type() == typeid(txin_zephyr_key) ? boost::get<txin_zephyr_key>(vin_zephyr[0]).key_offsets.size() - 1 : 0);
+            } else if (blob_type == BLOB_TYPE_CRYPTONOTE_XHV) {
+              r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin.size(), vout_xhv.size(),
+                  vin.size() > 0 && vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(vin[0]).key_offsets.size() - 1 :
+                  vin.size() > 0 && vin[0].type() == typeid(txin_offshore) ? boost::get<txin_offshore>(vin[0]).key_offsets.size() - 1 :
+                  vin.size() > 0 && vin[0].type() == typeid(txin_onshore) ? boost::get<txin_onshore>(vin[0]).key_offsets.size() - 1 :
+                  vin.size() > 0 && vin[0].type() == typeid(txin_xasset) ? boost::get<txin_xasset>(vin[0]).key_offsets.size() - 1 :
+                  vin.size() > 0 && vin[0].type() == typeid(txin_haven_key) ? boost::get<txin_haven_key>(vin[0]).key_offsets.size() - 1 :
+                  0
+              );
+            } else {
+              r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin.size(), vout.size(),
+                  vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(vin[0]).key_offsets.size() - 1 : 0);
+            }
             if (!r || !ar.stream().good()) return false;
             ar.end_object();
           }
@@ -298,9 +783,18 @@ namespace cryptonote
     version = 0;
     unlock_time = 0;
     vin.clear();
+    vin_zephyr.clear();
     vout.clear();
+    vout_xhv.clear();
+    vout_zephyr.clear();
     extra.clear();
     signatures.clear();
+    pricing_record_height = 0;
+    offshore_data.clear();
+    amount_burnt = 0;
+    amount_minted = 0;
+    output_unlock_times.clear();
+    collateral_indices.clear();
   }
 
   inline
@@ -312,6 +806,11 @@ namespace cryptonote
       size_t operator()(const txin_to_script& txin) const{return 0;}
       size_t operator()(const txin_to_scripthash& txin) const{return 0;}
       size_t operator()(const txin_to_key& txin) const {return txin.key_offsets.size();}
+      size_t operator()(const txin_offshore& txin) const {return txin.key_offsets.size();}
+      size_t operator()(const txin_onshore& txin) const {return txin.key_offsets.size();}
+      size_t operator()(const txin_xasset& txin) const {return txin.key_offsets.size();}
+      size_t operator()(const txin_haven_key& txin) const {return txin.key_offsets.size();}
+      size_t operator()(const txin_zephyr_key& txin) const {return txin.key_offsets.size();}
     };
 
     return boost::apply_visitor(txin_signature_size_visitor(), tx_in);
@@ -366,7 +865,7 @@ namespace cryptonote
           const uint8_t data[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x36, 0x78, 0x9e, 0x7a, 0x1e, 0x28, 0x14, 0x36, 0x46, 0x42, 0x29, 0x82, 0x8f, 0x81, 0x7d, 0x66, 0x12, 0xf7, 0xb4, 0x77, 0xd6, 0x65, 0x91, 0xff, 0x96, 0xa9, 0xe0, 0x64, 0xbc, 0xc9, 0x8a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
           blobdata blobdata((char*)data, sizeof(data));
           const unsigned char* p = (unsigned char*)&miner_tx_hash;
-          for (int i = 0; i != HASH_SIZE; ++ i, ++ p) blobdata[i] = *p;
+          for (int i = 0; i != crypto::HASH_SIZE; ++ i, ++ p) blobdata[i] = *p;
           get_blob_hash(blobdata, miner_tx_hash);
         }
 
@@ -431,15 +930,19 @@ namespace cryptonote
     crypto::hash prev_id;
     uint64_t nonce;
     uint64_t nonce8;
+    offshore::pricing_record pricing_record;
+    zephyr_oracle::pricing_record zephyr_pricing_record;
     crypto::cycle cycle;
     crypto::cycle40 cycle40;
+    crypto::cycle48 cycle48;
+    crypto::signature signature;
 
     BEGIN_SERIALIZE()
       VARINT_FIELD(major_version)
       VARINT_FIELD(minor_version)
       if (blob_type != BLOB_TYPE_FORKNOTE2) VARINT_FIELD(timestamp)
       FIELD(prev_id)
-      if (blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO || blob_type == BLOB_TYPE_CRYPTONOTE_TUBE) FIELD(nonce8)
+      if (blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO || blob_type == BLOB_TYPE_CRYPTONOTE_TUBE || blob_type == BLOB_TYPE_CRYPTONOTE_XTA) FIELD(nonce8)
       if (blob_type != BLOB_TYPE_FORKNOTE2) {
         if (blob_type == BLOB_TYPE_AEON) {
           FIELD(nonce)
@@ -452,6 +955,30 @@ namespace cryptonote
       }
       if (blob_type == BLOB_TYPE_CRYPTONOTE_XTNC || blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO) FIELD(cycle)
       if (blob_type == BLOB_TYPE_CRYPTONOTE_TUBE) FIELD(cycle40)
+      if (blob_type == BLOB_TYPE_CRYPTONOTE_XTA) FIELD(cycle48)
+      if (blob_type == BLOB_TYPE_CRYPTONOTE_XHV) FIELD(pricing_record)
+      if (blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR) {
+        if (major_version >= 3)
+        {
+          FIELD_N("pricing_record", zephyr_pricing_record)
+        }
+        else
+        {
+          zephyr_oracle::pricing_record_v1 pr_v1;
+          if (!typename Archive<W>::is_saving())
+          {
+            FIELD(pr_v1)
+            pr_v1.write_to_pr(zephyr_pricing_record);
+          }
+          else
+          {
+            pr_v1.read_from_pr(zephyr_pricing_record);
+            FIELD(pr_v1)
+          }
+        }
+      }
+      if (blob_type == BLOB_TYPE_CRYPTONOTE_XLA && major_version >= 13) FIELD(signature)
+
     END_SERIALIZE()
   };
 
@@ -520,52 +1047,31 @@ namespace cryptonote
     KV_SERIALIZE(payment_id)
     END_KV_SERIALIZE_MAP()
   };
-
-  struct keypair
-  {
-    crypto::public_key pub;
-    crypto::secret_key sec;
-
-    static inline keypair generate()
-    {
-      keypair k;
-      generate_keys(k.pub, k.sec);
-      return k;
-    }
-  };
   //---------------------------------------------------------------
 
 }
 
 BLOB_SERIALIZER(cryptonote::txout_to_key);
+BLOB_SERIALIZER(cryptonote::txout_offshore);
 BLOB_SERIALIZER(cryptonote::txout_to_scripthash);
 
 VARIANT_TAG(binary_archive, cryptonote::txin_gen, 0xff);
 VARIANT_TAG(binary_archive, cryptonote::txin_to_script, 0x0);
 VARIANT_TAG(binary_archive, cryptonote::txin_to_scripthash, 0x1);
 VARIANT_TAG(binary_archive, cryptonote::txin_to_key, 0x2);
+VARIANT_TAG(binary_archive, cryptonote::txin_zephyr_key, 0x2);
+VARIANT_TAG(binary_archive, cryptonote::txin_offshore, 0x3);
+VARIANT_TAG(binary_archive, cryptonote::txin_onshore, 0x4);
+VARIANT_TAG(binary_archive, cryptonote::txin_xasset, 0x5);
+VARIANT_TAG(binary_archive, cryptonote::txin_haven_key, 0x6);
 VARIANT_TAG(binary_archive, cryptonote::txout_to_script, 0x0);
 VARIANT_TAG(binary_archive, cryptonote::txout_to_scripthash, 0x1);
 VARIANT_TAG(binary_archive, cryptonote::txout_to_key, 0x2);
+VARIANT_TAG(binary_archive, cryptonote::txout_zephyr_tagged_key, 0x2);
+VARIANT_TAG(binary_archive, cryptonote::txout_to_tagged_key, 0x3);
+VARIANT_TAG(binary_archive, cryptonote::txout_offshore, 0x3);
+VARIANT_TAG(binary_archive, cryptonote::txout_xasset, 0x5);
+VARIANT_TAG(binary_archive, cryptonote::txout_haven_key, 0x6);
+VARIANT_TAG(binary_archive, cryptonote::txout_haven_tagged_key, 0x7);
 VARIANT_TAG(binary_archive, cryptonote::transaction, 0xcc);
 VARIANT_TAG(binary_archive, cryptonote::block, 0xbb);
-
-VARIANT_TAG(json_archive, cryptonote::txin_gen, "gen");
-VARIANT_TAG(json_archive, cryptonote::txin_to_script, "script");
-VARIANT_TAG(json_archive, cryptonote::txin_to_scripthash, "scripthash");
-VARIANT_TAG(json_archive, cryptonote::txin_to_key, "key");
-VARIANT_TAG(json_archive, cryptonote::txout_to_script, "script");
-VARIANT_TAG(json_archive, cryptonote::txout_to_scripthash, "scripthash");
-VARIANT_TAG(json_archive, cryptonote::txout_to_key, "key");
-VARIANT_TAG(json_archive, cryptonote::transaction, "tx");
-VARIANT_TAG(json_archive, cryptonote::block, "block");
-
-VARIANT_TAG(debug_archive, cryptonote::txin_gen, "gen");
-VARIANT_TAG(debug_archive, cryptonote::txin_to_script, "script");
-VARIANT_TAG(debug_archive, cryptonote::txin_to_scripthash, "scripthash");
-VARIANT_TAG(debug_archive, cryptonote::txin_to_key, "key");
-VARIANT_TAG(debug_archive, cryptonote::txout_to_script, "script");
-VARIANT_TAG(debug_archive, cryptonote::txout_to_scripthash, "scripthash");
-VARIANT_TAG(debug_archive, cryptonote::txout_to_key, "key");
-VARIANT_TAG(debug_archive, cryptonote::transaction, "tx");
-VARIANT_TAG(debug_archive, cryptonote::block, "block");

src/cryptonote_core/cryptonote_format_utils.cpp

@@ -56,56 +56,6 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  bool generate_key_image_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki)
-  {
-    crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation);
-    bool r = crypto::generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation);
-    CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")");
-
-    r = crypto::derive_public_key(recv_derivation, real_output_index, ack.m_account_address.m_spend_public_key, in_ephemeral.pub);
-    CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to derive_public_key(" << recv_derivation << ", " << real_output_index <<  ", " << ack.m_account_address.m_spend_public_key << ")");
-
-    crypto::derive_secret_key(recv_derivation, real_output_index, ack.m_spend_secret_key, in_ephemeral.sec);
-
-    crypto::generate_key_image(in_ephemeral.pub, in_ephemeral.sec, ki);
-    return true;
-  }
-  //---------------------------------------------------------------
-  uint64_t power_integral(uint64_t a, uint64_t b)
-  {
-    if(b == 0)
-      return 1;
-    uint64_t total = a;
-    for(uint64_t i = 1; i != b; i++)
-      total *= a;
-    return total;
-  }
-  //---------------------------------------------------------------
-  bool get_tx_fee(const transaction& tx, uint64_t & fee)
-  {
-    uint64_t amount_in = 0;
-    uint64_t amount_out = 0;
-    BOOST_FOREACH(auto& in, tx.vin)
-    {
-      CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key), 0, "unexpected type id in transaction");
-      amount_in += boost::get<txin_to_key>(in).amount;
-    }
-    BOOST_FOREACH(auto& o, tx.vout)
-      amount_out += o.amount;
-
-    CHECK_AND_ASSERT_MES(amount_in >= amount_out, false, "transaction spend (" <<amount_in << ") more than it has (" << amount_out << ")");
-    fee = amount_in - amount_out;
-    return true;
-  }
-  //---------------------------------------------------------------
-  uint64_t get_tx_fee(const transaction& tx)
-  {
-    uint64_t r = 0;
-    if(!get_tx_fee(tx, r))
-      return 0;
-    return r;
-  }
-  //---------------------------------------------------------------
   bool parse_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<tx_extra_field>& tx_extra_fields)
   {
     tx_extra_fields.clear();
@@ -213,90 +163,6 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  bool get_inputs_money_amount(const transaction& tx, uint64_t& money)
-  {
-    money = 0;
-    BOOST_FOREACH(const auto& in, tx.vin)
-    {
-      CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false);
-      money += tokey_in.amount;
-    }
-    return true;
-  }
-  //---------------------------------------------------------------
-  uint64_t get_block_height(const block& b)
-  {
-    CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, 0, "wrong miner tx in block: " << get_block_hash(b) << ", b.miner_tx.vin.size() != 1");
-    CHECKED_GET_SPECIFIC_VARIANT(b.miner_tx.vin[0], const txin_gen, coinbase_in, 0);
-    return coinbase_in.height;
-  }
-  //---------------------------------------------------------------
-  bool check_inputs_types_supported(const transaction& tx)
-  {
-    BOOST_FOREACH(const auto& in, tx.vin)
-    {
-      CHECK_AND_ASSERT_MES(in.type() == typeid(txin_to_key), false, "wrong variant type: "
-        << in.type().name() << ", expected " << typeid(txin_to_key).name()
-        << ", in transaction id=" << get_transaction_hash(tx));
-
-    }
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool check_outs_valid(const transaction& tx)
-  {
-    BOOST_FOREACH(const tx_out& out, tx.vout)
-    {
-      CHECK_AND_ASSERT_MES(out.target.type() == typeid(txout_to_key), false, "wrong variant type: "
-        << out.target.type().name() << ", expected " << typeid(txout_to_key).name()
-        << ", in transaction id=" << get_transaction_hash(tx));
-
-      CHECK_AND_NO_ASSERT_MES(0 < out.amount, false, "zero amount ouput in transaction id=" << get_transaction_hash(tx));
-
-      if(!check_key(boost::get<txout_to_key>(out.target).key))
-        return false;
-    }
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool check_money_overflow(const transaction& tx)
-  {
-    return check_inputs_overflow(tx) && check_outs_overflow(tx);
-  }
-  //---------------------------------------------------------------
-  bool check_inputs_overflow(const transaction& tx)
-  {
-    uint64_t money = 0;
-    BOOST_FOREACH(const auto& in, tx.vin)
-    {
-      CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false);
-      if(money > tokey_in.amount + money)
-        return false;
-      money += tokey_in.amount;
-    }
-    return true;
-  }
-  //---------------------------------------------------------------
-  bool check_outs_overflow(const transaction& tx)
-  {
-    uint64_t money = 0;
-    BOOST_FOREACH(const auto& o, tx.vout)
-    {
-      if(money > o.amount + money)
-        return false;
-      money += o.amount;
-    }
-    return true;
-  }
-  //---------------------------------------------------------------
-  uint64_t get_outs_money_amount(const transaction& tx)
-  {
-    uint64_t outputs_amount = 0;
-    BOOST_FOREACH(const auto& o, tx.vout)
-      outputs_amount += o.amount;
-    return outputs_amount;
-  }
-  //---------------------------------------------------------------
   std::string short_hash_str(const crypto::hash& h)
   {
     std::string res = string_tools::pod_to_hex(h);
@@ -306,40 +172,6 @@ namespace cryptonote
     return res;
   }
   //---------------------------------------------------------------
-  bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::public_key& tx_pub_key, size_t output_index)
-  {
-    crypto::key_derivation derivation;
-    generate_key_derivation(tx_pub_key, acc.m_view_secret_key, derivation);
-    crypto::public_key pk;
-    derive_public_key(derivation, output_index, acc.m_account_address.m_spend_public_key, pk);
-    return pk == out_key.key;
-  }
-  //---------------------------------------------------------------
-  bool lookup_acc_outs(const account_keys& acc, const transaction& tx, std::vector<size_t>& outs, uint64_t& money_transfered)
-  {
-    crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx);
-    if(null_pkey == tx_pub_key)
-      return false;
-    return lookup_acc_outs(acc, tx, tx_pub_key, outs, money_transfered);
-  }
-  //---------------------------------------------------------------
-  bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector<size_t>& outs, uint64_t& money_transfered)
-  {
-    money_transfered = 0;
-    size_t i = 0;
-    BOOST_FOREACH(const tx_out& o,  tx.vout)
-    {
-      CHECK_AND_ASSERT_MES(o.target.type() ==  typeid(txout_to_key), false, "wrong type id in transaction out" );
-      if(is_out_to_acc(acc, boost::get<txout_to_key>(o.target), tx_pub_key, i))
-      {
-        outs.push_back(i);
-        money_transfered += o.amount;
-      }
-      i++;
-    }
-    return true;
-  }
-  //---------------------------------------------------------------
   void get_blob_hash(const blobdata& blob, crypto::hash& res)
   {
     cn_fast_hash(blob.data(), blob.size(), res);
@@ -388,8 +220,8 @@ namespace cryptonote
     {
       std::stringstream ss;
       binary_archive<true> ba(ss);
-      const size_t inputs = t.vin.size();
-      const size_t outputs = t.vout.size();
+      const size_t inputs = t.blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR ? t.vin_zephyr.size() : t.vin.size();
+      const size_t outputs = t.blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR ? t.vout_zephyr.size() : t.blob_type != BLOB_TYPE_CRYPTONOTE_XHV ? t.vout.size() : t.vout_xhv.size();
       bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs);
       CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures base");
       cryptonote::get_blob_hash(ss.str(), hashes[1]);
@@ -404,9 +236,22 @@ namespace cryptonote
     {
       std::stringstream ss;
       binary_archive<true> ba(ss);
-      const size_t inputs = t.vin.size();
-      const size_t outputs = t.vout.size();
-      const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0;
+      const size_t inputs = t.blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR ? t.vin_zephyr.size() : t.vin.size();
+      const size_t outputs = t.blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR ? t.vout_zephyr.size() : t.blob_type != BLOB_TYPE_CRYPTONOTE_XHV ? t.vout.size() : t.vout_xhv.size();
+      size_t mixin;
+      if (t.blob_type == BLOB_TYPE_CRYPTONOTE_ZEPHYR) {
+        mixin = t.vin_zephyr.empty() ? 0 : t.vin_zephyr[0].type() == typeid(txin_zephyr_key) ? boost::get<txin_zephyr_key>(t.vin_zephyr[0]).key_offsets.size() - 1 : 0;
+      } else if (t.blob_type == BLOB_TYPE_CRYPTONOTE_XHV) {
+        mixin = t.vin.empty() ? 0 :
+          t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 :
+          t.vin[0].type() == typeid(txin_offshore) ? boost::get<txin_offshore>(t.vin[0]).key_offsets.size() - 1 :
+          t.vin[0].type() == typeid(txin_onshore) ? boost::get<txin_onshore>(t.vin[0]).key_offsets.size() - 1 :
+          t.vin[0].type() == typeid(txin_xasset) ? boost::get<txin_xasset>(t.vin[0]).key_offsets.size() - 1 :
+          t.vin[0].type() == typeid(txin_haven_key) ? boost::get<txin_haven_key>(t.vin[0]).key_offsets.size() - 1 :
+          0;
+      } else {
+        mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0;
+      }
       bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin);
       CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable");
       cryptonote::get_blob_hash(ss.str(), hashes[2]);
@@ -429,7 +274,7 @@ namespace cryptonote
   //---------------------------------------------------------------
   bool get_block_hashing_blob(const block& b, blobdata& blob)
   {
-    if (b.blob_type == BLOB_TYPE_CRYPTONOTE_XTNC || b.blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO || b.blob_type == BLOB_TYPE_CRYPTONOTE_TUBE) {
+    if (b.blob_type == BLOB_TYPE_CRYPTONOTE_XTNC || b.blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO || b.blob_type == BLOB_TYPE_CRYPTONOTE_TUBE || b.blob_type == BLOB_TYPE_CRYPTONOTE_XTA) {
       blob = t_serializable_object_to_blob(b.major_version);
       blob.append(reinterpret_cast<const char*>(&b.minor_version), sizeof(b.minor_version));
       blob.append(reinterpret_cast<const char*>(&b.timestamp), sizeof(b.timestamp));
@@ -444,7 +289,7 @@ namespace cryptonote
     if (b.blob_type == BLOB_TYPE_CRYPTONOTE3) {
       blob.append(reinterpret_cast<const char*>(&b.uncle), sizeof(b.uncle));
     }
-    if (b.blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO || b.blob_type == BLOB_TYPE_CRYPTONOTE_TUBE) {
+    if (b.blob_type == BLOB_TYPE_CRYPTONOTE_CUCKOO || b.blob_type == BLOB_TYPE_CRYPTONOTE_TUBE || b.blob_type == BLOB_TYPE_CRYPTONOTE_XTA) {
       blob.append(reinterpret_cast<const char*>(&b.nonce8), sizeof(b.nonce8));
     }
     return true;
@@ -491,15 +336,6 @@ namespace cryptonote
     return get_object_hash(blob, res);
   }
   //---------------------------------------------------------------
-  bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height)
-  {
-    blobdata bd;
-    if(!get_block_hashing_blob(b, bd))
-      return false;
-    crypto::cn_slow_hash(bd.data(), bd.size(), res);
-    return true;
-  }
-  //---------------------------------------------------------------
   std::vector<uint64_t> relative_output_offsets_to_absolute(const std::vector<uint64_t>& off)
   {
     std::vector<uint64_t> res = off;
@@ -520,13 +356,6 @@ namespace cryptonote
     return res;
   }
   //---------------------------------------------------------------
-  crypto::hash get_block_longhash(const block& b, uint64_t height)
-  {
-    crypto::hash p = null_hash;
-    get_block_longhash(b, p, height);
-    return p;
-  }
-  //---------------------------------------------------------------
   bool get_bytecoin_block_longhash(const block& b, crypto::hash& res)
   {
     blobdata bd;
@@ -542,7 +371,7 @@ namespace cryptonote
     ss << b_blob;
     binary_archive<false> ba(ss);
     bool r = ::serialization::serialize(ba, b);
-    CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob");
+    CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob 1");
     return true;
   }
   //---------------------------------------------------------------

src/cryptonote_core/cryptonote_format_utils.h

@@ -63,12 +63,6 @@ namespace cryptonote
   bool get_payment_id_from_tx_extra_nonce(const blobdata& extra_nonce, crypto::hash& payment_id);
   bool append_mm_tag_to_extra(std::vector<uint8_t>& tx_extra, const tx_extra_merge_mining_tag& mm_tag);
   bool get_mm_tag_from_extra(const std::vector<uint8_t>& tx_extra, tx_extra_merge_mining_tag& mm_tag);
-  bool is_out_to_acc(const account_keys& acc, const txout_to_key& out_key, const crypto::public_key& tx_pub_key, size_t output_index);
-  bool lookup_acc_outs(const account_keys& acc, const transaction& tx, const crypto::public_key& tx_pub_key, std::vector<size_t>& outs, uint64_t& money_transfered);
-  bool lookup_acc_outs(const account_keys& acc, const transaction& tx, std::vector<size_t>& outs, uint64_t& money_transfered);
-  bool get_tx_fee(const transaction& tx, uint64_t & fee);
-  uint64_t get_tx_fee(const transaction& tx);
-  bool generate_key_image_helper(const account_keys& ack, const crypto::public_key& tx_public_key, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki);
   void get_blob_hash(const blobdata& blob, crypto::hash& res);
   crypto::hash get_blob_hash(const blobdata& blob);
   std::string short_hash_str(const crypto::hash& h);
@@ -82,19 +76,11 @@ namespace cryptonote
   bool get_block_hash(const block& b, crypto::hash& res);
   crypto::hash get_block_hash(const block& b);
   bool get_block_header_hash(const block& b, crypto::hash& res);
-  bool get_block_longhash(const block& b, crypto::hash& res, uint64_t height);
-  crypto::hash get_block_longhash(const block& b, uint64_t height);
   bool get_bytecoin_block_longhash(const block& blk, crypto::hash& res);
   bool parse_and_validate_block_from_blob(const blobdata& b_blob, block& b);
-  bool get_inputs_money_amount(const transaction& tx, uint64_t& money);
-  uint64_t get_outs_money_amount(const transaction& tx);
-  bool check_inputs_types_supported(const transaction& tx);
+  std::map<std::string, uint64_t> get_outs_money_amount(const transaction& tx);
   bool check_outs_valid(const transaction& tx);
 
-  bool check_money_overflow(const transaction& tx);
-  bool check_outs_overflow(const transaction& tx);
-  bool check_inputs_overflow(const transaction& tx);
-  uint64_t get_block_height(const block& b);
   std::vector<uint64_t> relative_output_offsets_to_absolute(const std::vector<uint64_t>& off);
   std::vector<uint64_t> absolute_output_offsets_to_relative(const std::vector<uint64_t>& off);
   //---------------------------------------------------------------
@@ -139,16 +125,6 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------
-  template <typename T>
-  std::string obj_to_json_str(T& obj)
-  {
-    std::stringstream ss;
-    json_archive<true> ar(ss, true);
-    bool r = ::serialization::serialize(ar, obj);
-    CHECK_AND_ASSERT_MES(r, "", "obj_to_json_str failed: serialization::serialize returned false");
-    return ss.str();
-  }
-  //---------------------------------------------------------------
   // 62387455827 -> 455827 + 7000000 + 80000000 + 300000000 + 2000000000 + 60000000000, where 455827 <= dust_threshold
   template<typename chunk_handler_t, typename dust_handler_t>
   void decompose_amount_into_digits(uint64_t amount, uint64_t dust_threshold, const chunk_handler_t& chunk_handler, const dust_handler_t& dust_handler)

src/cryptonote_core/tx_extra.h

@@ -32,12 +32,16 @@
 
 #define TX_EXTRA_PADDING_MAX_COUNT            255
 #define TX_EXTRA_NONCE_MAX_COUNT              255
+#define TX_EXTRA_OFFSHORE_MAX_COUNT           255
+#define TX_EXTRA_MEMO_MAX_COUNT               255
 
 #define TX_EXTRA_TAG_PADDING                  0x00
 #define TX_EXTRA_TAG_PUBKEY                   0x01
 #define TX_EXTRA_NONCE                        0x02
 #define TX_EXTRA_MERGE_MINING_TAG             0x03
 #define TX_EXTRA_TAG_ADDITIONAL_PUBKEYS       0x04
+#define TX_EXTRA_TAG_OFFSHORE                 0x17
+#define TX_EXTRA_TAG_MEMO                     0x18
 #define TX_EXTRA_TAG_SERVICE_NODE_REGISTER    0x70
 #define TX_EXTRA_TAG_SERVICE_NODE_DEREGISTER  0x71
 #define TX_EXTRA_TAG_SERVICE_NODE_WINNER      0x72
@@ -186,6 +190,25 @@ namespace cryptonote
     END_SERIALIZE()
   };
 
+  struct tx_extra_offshore
+  {
+    std::string data;
+
+    BEGIN_SERIALIZE()
+      FIELD(data)
+    END_SERIALIZE()
+  };
+
+  struct tx_extra_memo
+  {
+    // Actual memo data as string
+    std::string data;
+
+    BEGIN_SERIALIZE()
+      FIELD(data)
+    END_SERIALIZE()
+  };
+  
   struct tx_extra_service_node_winner
   {
     crypto::public_key m_service_node_key;
@@ -301,6 +324,8 @@ namespace cryptonote
                          tx_extra_merge_mining_tag,
                          tx_extra_additional_pub_keys,
                          tx_extra_mysterious_minergate,
+			 tx_extra_offshore,
+			 tx_extra_memo,
                          tx_extra_service_node_pubkey,
                          tx_extra_service_node_register,
                          tx_extra_service_node_contributor,
@@ -321,6 +346,8 @@ VARIANT_TAG(binary_archive, cryptonote::tx_extra_nonce,                    TX_EX
 VARIANT_TAG(binary_archive, cryptonote::tx_extra_merge_mining_tag,         TX_EXTRA_MERGE_MINING_TAG);
 VARIANT_TAG(binary_archive, cryptonote::tx_extra_additional_pub_keys,      TX_EXTRA_TAG_ADDITIONAL_PUBKEYS);
 VARIANT_TAG(binary_archive, cryptonote::tx_extra_mysterious_minergate,     TX_EXTRA_MYSTERIOUS_MINERGATE_TAG);
+VARIANT_TAG(binary_archive, cryptonote::tx_extra_offshore,                 TX_EXTRA_TAG_OFFSHORE);
+VARIANT_TAG(binary_archive, cryptonote::tx_extra_memo,                     TX_EXTRA_TAG_MEMO);
 VARIANT_TAG(binary_archive, cryptonote::tx_extra_service_node_register,    TX_EXTRA_TAG_SERVICE_NODE_REGISTER);
 VARIANT_TAG(binary_archive, cryptonote::tx_extra_service_node_deregister,  TX_EXTRA_TAG_SERVICE_NODE_DEREGISTER);
 VARIANT_TAG(binary_archive, cryptonote::tx_extra_service_node_contributor, TX_EXTRA_TAG_SERVICE_NODE_CONTRIBUTOR);

src/main.cc

@@ -145,7 +145,7 @@ NAN_METHOD(convert_blob) { // (parentBlockBuffer, cnBlobType)
 
     block b = AUTO_VAL_INIT(b);
     b.set_blob_type(blob_type);
-    if (!parse_and_validate_block_from_blob(input, b)) return THROW_ERROR_EXCEPTION("Failed to parse block");
+    if (!parse_and_validate_block_from_blob(input, b)) return THROW_ERROR_EXCEPTION("Failed to parse block 2");
 
     if (blob_type == BLOB_TYPE_FORKNOTE2) {
         block parent_block;
@@ -231,6 +231,12 @@ NAN_METHOD(construct_block_blob) { // (parentBlockTemplateBuffer, nonceBuffer, c
         for (int i = 0; i < 40; i++ ) b.cycle40.data[i] = cycle->Get(isolate->GetCurrentContext(), i).ToLocalChecked()->NumberValue(isolate->GetCurrentContext()).ToChecked();
     }
 
+    if (blob_type == BLOB_TYPE_CRYPTONOTE_XTA) {
+        if (info.Length() != 4) return THROW_ERROR_EXCEPTION("You must provide 4 arguments.");
+        Local<Array> cycle = Local<Array>::Cast(info[3]);
+        for (int i = 0; i < 48; i++ ) b.cycle48.data[i] = cycle->Get(isolate->GetCurrentContext(), i).ToLocalChecked()->NumberValue(isolate->GetCurrentContext()).ToChecked();
+    }
+
     if (!block_to_blob(b, output)) return THROW_ERROR_EXCEPTION("Failed to convert block to blob");
 
     v8::Local<v8::Value> returnValue = Nan::CopyBuffer((char*)output.data(), output.size()).ToLocalChecked();

src/offshore/asset_types.h

@@ -0,0 +1,143 @@
+// Copyright (c) 2021, Haven Protocol
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+#include <string>
+#include <vector>
+
+namespace offshore {
+
+  const std::vector<std::string> ASSET_TYPES = {"XHV", "XAG", "XAU", "XAUD", "XBTC", "XCAD", "XCHF", "XCNY", "XEUR", "XGBP", "XJPY", "XNOK", "XNZD", "XUSD"};
+
+  class asset_type_counts
+  {
+
+    public:
+
+      // Fields 
+      uint64_t XHV;
+      uint64_t XAG;
+      uint64_t XAU;
+      uint64_t XAUD;
+      uint64_t XBTC;
+      uint64_t XCAD;
+      uint64_t XCHF;
+      uint64_t XCNY;
+      uint64_t XEUR;
+      uint64_t XGBP;
+      uint64_t XJPY;
+      uint64_t XNOK;
+      uint64_t XNZD;
+      uint64_t XUSD;
+
+      asset_type_counts() noexcept
+        : XHV(0)
+        , XAG(0)
+        , XAU(0)
+        , XAUD(0)
+        , XBTC(0)
+        , XCAD(0)
+        , XCHF(0)
+        , XCNY(0)
+        , XEUR(0)
+        , XGBP(0)
+        , XJPY(0)
+        , XNOK(0)
+        , XNZD(0)
+        , XUSD(0)
+      {
+      }
+
+      uint64_t operator[](const std::string asset_type) const noexcept
+      {
+        if (asset_type == "XHV") {
+          return XHV;
+        } else if (asset_type == "XUSD") {
+          return XUSD;
+        } else if (asset_type == "XAG") {
+          return XAG;
+        } else if (asset_type == "XAU") {
+          return XAU;
+        } else if (asset_type == "XAUD") {
+          return XAUD;
+        } else if (asset_type == "XBTC") {
+          return XBTC;
+        } else if (asset_type == "XCAD") {
+          return XCAD;
+        } else if (asset_type == "XCHF") {
+          return XCHF;
+        } else if (asset_type == "XCNY") {
+          return XCNY;
+        } else if (asset_type == "XEUR") {
+          return XEUR;
+        } else if (asset_type == "XGBP") {
+          return XGBP;
+        } else if (asset_type == "XJPY") {
+          return XJPY;
+        } else if (asset_type == "XNOK") {
+          return XNOK;
+        } else if (asset_type == "XNZD") {
+          return XNZD;
+        }
+
+        return 0;
+      }
+
+      void add(const std::string asset_type, const uint64_t val)
+      {
+        if (asset_type == "XHV") {
+          XHV += val;
+        } else if (asset_type == "XUSD") {
+          XUSD += val;
+        } else if (asset_type == "XAG") {
+          XAG += val;
+        } else if (asset_type == "XAU") {
+          XAU += val;
+        } else if (asset_type == "XAUD") {
+          XAUD += val;
+        } else if (asset_type == "XBTC") {
+          XBTC += val;
+        } else if (asset_type == "XCAD") {
+          XCAD += val;
+        } else if (asset_type == "XCHF") {
+          XCHF += val;
+        } else if (asset_type == "XCNY") {
+          XCNY += val;
+        } else if (asset_type == "XEUR") {
+          XEUR += val;
+        } else if (asset_type == "XGBP") {
+          XGBP += val;
+        } else if (asset_type == "XJPY") {
+          XJPY += val;
+        } else if (asset_type == "XNOK") {
+          XNOK += val;
+        } else if (asset_type == "XNZD") {
+          XNZD += val;
+        }
+      }
+  };
+}

src/offshore/pricing_record.cpp

@@ -0,0 +1,452 @@
+// Copyright (c) 2019, Haven Protocol
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Portions of this code based upon code Copyright (c) 2019, The Monero Project
+
+#include "pricing_record.h"
+
+#include "serialization/keyvalue_serialization.h"
+#include "storages/portable_storage.h"
+
+#include "string_tools.h"
+namespace offshore
+{
+
+  namespace
+  {
+    struct pr_serialized
+    {
+      uint64_t xAG;
+      uint64_t xAU;
+      uint64_t xAUD;
+      uint64_t xBTC;
+      uint64_t xCAD;
+      uint64_t xCHF;
+      uint64_t xCNY;
+      uint64_t xEUR;
+      uint64_t xGBP;
+      uint64_t xJPY;
+      uint64_t xNOK;
+      uint64_t xNZD;
+      uint64_t xUSD;
+      uint64_t unused1;
+      uint64_t unused2;
+      uint64_t unused3;
+      uint64_t timestamp;
+      std::string signature;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(xAG)
+        KV_SERIALIZE(xAU)
+        KV_SERIALIZE(xAUD)
+        KV_SERIALIZE(xBTC)
+        KV_SERIALIZE(xCAD)
+        KV_SERIALIZE(xCHF)
+        KV_SERIALIZE(xCNY)
+        KV_SERIALIZE(xEUR)
+        KV_SERIALIZE(xGBP)
+        KV_SERIALIZE(xJPY)
+        KV_SERIALIZE(xNOK)
+        KV_SERIALIZE(xNZD)
+        KV_SERIALIZE(xUSD)
+        KV_SERIALIZE(unused1)
+        KV_SERIALIZE(unused2)
+        KV_SERIALIZE(unused3)
+        KV_SERIALIZE(timestamp)
+        KV_SERIALIZE(signature)
+      END_KV_SERIALIZE_MAP()
+    };
+  }
+  
+  pricing_record::pricing_record() noexcept
+    : xAG(0)
+    , xAU(0)
+    , xAUD(0)
+    , xBTC(0)
+    , xCAD(0)
+    , xCHF(0)
+    , xCNY(0)
+    , xEUR(0)
+    , xGBP(0)
+    , xJPY(0)
+    , xNOK(0)
+    , xNZD(0)
+    , xUSD(0)
+    , unused1(0)
+    , unused2(0)
+    , unused3(0)
+    , timestamp(0)
+  {
+    std::memset(signature, 0, sizeof(signature));
+  }
+
+  bool pricing_record::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent)
+  {
+    pr_serialized in{};
+    if (in._load(src, hparent))
+    {
+      // Copy everything into the local instance
+      xAG = in.xAG;
+      xAU = in.xAU;
+      xAUD = in.xAUD;
+      xBTC = in.xBTC;
+      xCAD = in.xCAD;
+      xCHF = in.xCHF;
+      xCNY = in.xCNY;
+      xEUR = in.xEUR;
+      xGBP = in.xGBP;
+      xJPY = in.xJPY;
+      xNOK = in.xNOK;
+      xNZD = in.xNZD;
+      xUSD = in.xUSD;
+      unused1 = in.unused1;
+      unused2 = in.unused2;
+      unused3 = in.unused3;
+      timestamp = in.timestamp;
+      for (unsigned int i = 0; i < in.signature.length(); i += 2) {
+        std::string byteString = in.signature.substr(i, 2);
+        signature[i>>1] = (char) strtol(byteString.c_str(), NULL, 16);
+      }
+      return true;
+    }
+
+    // Report error here?
+    return false;
+  }
+
+  bool pricing_record::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const
+  {
+    std::string sig_hex;
+    for (unsigned int i=0; i<64; i++) {
+      std::stringstream ss;
+      ss << std::hex << std::setw(2) << std::setfill('0') << (0xff & signature[i]);
+      sig_hex += ss.str();
+    }
+    const pr_serialized out{xAG,xAU,xAUD,xBTC,xCAD,xCHF,xCNY,xEUR,xGBP,xJPY,xNOK,xNZD,xUSD,unused1,unused2,unused3,timestamp,sig_hex};
+    return out.store(dest, hparent);
+  }
+
+  pricing_record::pricing_record(const pricing_record& orig) noexcept
+    : xAG(orig.xAG)
+    , xAU(orig.xAU)
+    , xAUD(orig.xAUD)
+    , xBTC(orig.xBTC)
+    , xCAD(orig.xCAD)
+    , xCHF(orig.xCHF)
+    , xCNY(orig.xCNY)
+    , xEUR(orig.xEUR)
+    , xGBP(orig.xGBP)
+    , xJPY(orig.xJPY)
+    , xNOK(orig.xNOK)
+    , xNZD(orig.xNZD)
+    , xUSD(orig.xUSD)
+    , unused1(orig.unused1)
+    , unused2(orig.unused2)
+    , unused3(orig.unused3)
+    , timestamp(orig.timestamp)
+  {
+    std::memcpy(signature, orig.signature, sizeof(signature));
+  }
+
+  pricing_record& pricing_record::operator=(const pricing_record& orig) noexcept
+  {
+    xAG = orig.xAG;
+    xAU = orig.xAU;
+    xAUD = orig.xAUD;
+    xBTC = orig.xBTC;
+    xCAD = orig.xCAD;
+    xCHF = orig.xCHF;
+    xCNY = orig.xCNY;
+    xEUR = orig.xEUR;
+    xGBP = orig.xGBP;
+    xJPY = orig.xJPY;
+    xNOK = orig.xNOK;
+    xNZD = orig.xNZD;
+    xUSD = orig.xUSD;
+    unused1 = orig.unused1;
+    unused2 = orig.unused2;
+    unused3 = orig.unused3;
+    timestamp = orig.timestamp;
+    ::memcpy(signature, orig.signature, sizeof(signature));
+    return *this;
+  }
+
+  uint64_t pricing_record::operator[](const std::string& asset_type) const
+  {
+    if (asset_type == "XHV") {
+      return 1000000000000;
+    } else if (asset_type == "XUSD") {
+      return unused1;
+    } else if (asset_type == "XAG") {
+      return xAG;
+    } else if (asset_type == "XAU") {
+      return xAU;
+    } else if (asset_type == "XAUD") {
+      return xAUD;
+    } else if (asset_type == "XBTC") {
+      return xBTC;
+    } else if (asset_type == "XCAD") {
+      return xCAD;
+    } else if (asset_type == "XCHF") {
+      return xCHF;
+    } else if (asset_type == "XCNY") {
+      return xCNY;
+    } else if (asset_type == "XEUR") {
+      return xEUR;
+    } else if (asset_type == "XGBP") {
+      return xGBP;
+    } else if (asset_type == "XJPY") {
+      return xJPY;
+    } else if (asset_type == "XNOK") {
+      return xNOK;
+    } else if (asset_type == "XNZD") {
+      return xNZD;
+    } else {
+     CHECK_AND_ASSERT_THROW_MES(false, "Asset type doesn't exist in pricing record!");
+    }
+  }
+  
+  bool pricing_record::equal(const pricing_record& other) const noexcept
+  {
+    return ((xAG == other.xAG) &&
+	    (xAU == other.xAU) &&
+	    (xAUD == other.xAUD) &&
+	    (xBTC == other.xBTC) &&
+	    (xCAD == other.xCAD) &&
+	    (xCHF == other.xCHF) &&
+	    (xCNY == other.xCNY) &&
+	    (xEUR == other.xEUR) &&
+	    (xGBP == other.xGBP) &&
+	    (xJPY == other.xJPY) &&
+	    (xNOK == other.xNOK) &&
+	    (xNZD == other.xNZD) &&
+	    (xUSD == other.xUSD) &&
+	    (unused1 == other.unused1) &&
+	    (unused2 == other.unused2) &&
+	    (unused3 == other.unused3) &&
+	    (timestamp == other.timestamp) &&
+	    !::memcmp(signature, other.signature, sizeof(signature)));
+  }
+
+  bool pricing_record::empty() const noexcept
+  {
+    const pricing_record empty_pr = offshore::pricing_record();
+    return (*this).equal(empty_pr);
+  }
+
+  bool pricing_record::verifySignature(const std::string& public_key) const 
+  {
+    CHECK_AND_ASSERT_THROW_MES(!public_key.empty(), "Pricing record verification failed. NULL public key. PK Size: " << public_key.size()); // TODO: is this necessary or the one below already covers this case, meannin it will produce empty pubkey?
+    
+    // extract the key
+    EVP_PKEY* pubkey;
+    BIO* bio = BIO_new_mem_buf(public_key.c_str(), public_key.size());
+    if (!bio) {
+      return false;
+    }
+    pubkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL);
+    BIO_free(bio);
+    CHECK_AND_ASSERT_THROW_MES(pubkey != NULL, "Pricing record verification failed. NULL public key.");
+
+
+    // Convert our internal 64-byte binary representation into 128-byte hex string
+    std::string sig_hex;
+    for (unsigned int i=0; i<64; i++) {
+      std::stringstream ss;
+      ss << std::hex << std::setw(2) << std::setfill('0') << (0xff & signature[i]);
+      sig_hex += ss.str();
+    }
+    
+    // Rebuild the OpenSSL format of the signature from the r+s values
+    std::string sig_rebuilt = "30";
+    std::string r_rebuilt = (signature[0] == 0) ? sig_hex.substr(2, 62) : sig_hex.substr(0,64);
+    if (signature[(signature[0] == 0) ? 1 : 0] & 0x80)
+      r_rebuilt = "00" + r_rebuilt;
+    std::string s_rebuilt = (signature[(signature[32] == 0) ? 33 : 32] == 0) ? sig_hex.substr(66, 62) : sig_hex.substr(64,64);
+    if (signature[(signature[32] == 0) ? 33 : 32] & 0x80)
+      s_rebuilt = "00" + s_rebuilt;
+    size_t sig_length = (r_rebuilt.length() + s_rebuilt.length() + 8) >> 1;
+    std::stringstream ss;
+    ss << std::hex << sig_length;
+    sig_rebuilt += std::string(2-ss.str().length(), '0') + ss.str();
+    ss.clear();
+    sig_rebuilt += "02";
+    size_t r_length = r_rebuilt.length() >> 1;
+    std::stringstream ss2;
+    ss2 << std::hex << r_length;
+    sig_rebuilt += std::string(2-ss2.str().length(), '0') + ss2.str();
+    ss2.clear();
+    sig_rebuilt += r_rebuilt;
+    sig_rebuilt += "02";
+    size_t s_length = s_rebuilt.length() >> 1;
+    std::stringstream ss3;
+    ss3 << std::hex << s_length;
+    sig_rebuilt += std::string(2-ss3.str().length(), '0') + ss3.str();
+    ss3.clear();
+    sig_rebuilt += s_rebuilt;
+
+    // Build the JSON string, so that we can verify the signature
+    std::ostringstream oss;
+    oss << "{\"xAG\":" << xAG;
+    oss << ",\"xAU\":" << xAU;
+    oss << ",\"xAUD\":" << xAUD;
+    oss << ",\"xBTC\":" << xBTC;
+    oss << ",\"xCAD\":" << xCAD;
+    oss << ",\"xCHF\":" << xCHF;
+    oss << ",\"xCNY\":" << xCNY;
+    oss << ",\"xEUR\":" << xEUR;
+    oss << ",\"xGBP\":" << xGBP;
+    oss << ",\"xJPY\":" << xJPY;
+    oss << ",\"xNOK\":" << xNOK;
+    oss << ",\"xNZD\":" << xNZD;
+    oss << ",\"xUSD\":" << xUSD;
+    oss << ",\"unused1\":" << unused1;
+    oss << ",\"unused2\":" << unused2;
+    oss << ",\"unused3\":" << unused3;
+    if (timestamp > 0)
+      oss << ",\"timestamp\":" << timestamp;
+    oss << "}";
+    std::string message = oss.str();    
+
+    // Convert signature from hex-encoded to binary
+    std::string compact;
+    for (unsigned int i = 0; i < sig_rebuilt.length(); i += 2) {
+      std::string byteString = sig_rebuilt.substr(i, 2);
+      char byte = (char) strtol(byteString.c_str(), NULL, 16);
+      compact += (byte);
+    }
+
+    // Create a verify digest from the message
+    EVP_MD_CTX *ctx = EVP_MD_CTX_create();
+    int ret = 0;
+    if (ctx) {
+      ret = EVP_DigestVerifyInit(ctx, NULL, EVP_sha256(), NULL, pubkey);
+      if (ret == 1) {
+        ret = EVP_DigestVerifyUpdate(ctx, message.data(), message.length());
+        if (ret == 1) {
+          ret = EVP_DigestVerifyFinal(ctx, (const unsigned char *)compact.data(), compact.length());
+        }
+      }
+    }
+
+    // Cleanup the context we created
+    EVP_MD_CTX_destroy(ctx);
+    // Cleanup the openssl stuff
+    EVP_PKEY_free(pubkey);
+    
+    if (ret == 1)
+      return true;
+
+    // Get the errors from OpenSSL
+    //ERR_print_errors_fp (stderr);
+  
+    return false;
+  }
+
+  void pricing_record::set_for_height_821428() {
+    const std::string pr_821428 = "9b3f6f2f8f0000003d620e1202000000be71be2555120000b8627010000000000000000000000000ea0885b2270d00000000000000000000f797ff9be00b0000ddbdb005270a0000fc90cfe02b01060000000000000000000000000000000000d0a28224000e000000d643be960e0000002e8bb6a40e000000f8a817f80d00002f5d27d45cdbfbac3d0f6577103f68de30895967d7562fbd56c161ae90130f54301b1ea9d5fd062f37dac75c3d47178bc6f149d21da1ff0e8430065cb762b93a";
+    this->xAG = 614976143259;
+    this->xAU = 8892867133;
+    this->xAUD = 20156914758078;
+    this->xBTC = 275800760;
+    this->xCAD = 0;
+    this->xCHF = 14464149948650;
+    this->xCNY = 0;
+    this->xEUR = 13059317798903;
+    this->xGBP = 11162715471325;
+    this->xJPY = 1690137827184892;
+    this->xNOK = 0;
+    this->xNZD = 0;
+    this->xUSD = 15393775330000;
+    this->unused1 = 16040600000000;
+    this->unused2 = 16100600000000;
+    this->unused3 = 15359200000000;
+    this->timestamp = 0;
+    std::string sig = "2f5d27d45cdbfbac3d0f6577103f68de30895967d7562fbd56c161ae90130f54301b1ea9d5fd062f37dac75c3d47178bc6f149d21da1ff0e8430065cb762b93a";
+    int j=0;
+    for (unsigned int i = 0; i < sig.size(); i += 2) {
+      std::string byteString = sig.substr(i, 2);
+      this->signature[j++] = (char) strtol(byteString.c_str(), NULL, 16);
+    }
+  }
+
+  // overload for pr validation for block
+  bool pricing_record::valid(uint32_t hf_version, uint64_t bl_timestamp, uint64_t last_bl_timestamp) const 
+  {
+    // check for empty pr 
+    if (hf_version >= HF_VERSION_XASSET_FEES_V2) {
+      if (this->empty())
+        return true;
+    } else {
+      unsigned char test_sig[64];
+      std::memset(test_sig, 0, sizeof(test_sig));
+      if (std::memcmp(test_sig, this->signature, sizeof(this->signature)) == 0) {
+        return true;
+      }
+    }
+
+    // Oracle public keys
+    std::string const mainnet_public_key = "-----BEGIN PUBLIC KEY-----\n"
+      "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE5YBxWx1AZCA9jTUk8Pr2uZ9jpfRt\n"
+      "KWv3Vo1/Gny+1vfaxsXhBQiG1KlHkafNGarzoL0WHW4ocqaaqF5iv8i35A==\n"
+      "-----END PUBLIC KEY-----\n";
+    std::string const testnet_public_key = "-----BEGIN PUBLIC KEY-----\n"
+      "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtWqvQh7OdXrdgXcDeBMRVfLWTW3F\n"
+      "wByeoVJFBfZymScJIJl46j66xG6ngnyj4ai4/QPFnSZ1I9jjMRlTWC4EPA==\n"
+      "-----END PUBLIC KEY-----\n";
+    std::string const stagenet_public_key = "-----BEGIN PUBLIC KEY-----\n"
+      "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtWqvQh7OdXrdgXcDeBMRVfLWTW3F\n"
+      "wByeoVJFBfZymScJIJl46j66xG6ngnyj4ai4/QPFnSZ1I9jjMRlTWC4EPA==\n"
+      "-----END PUBLIC KEY-----\n";
+    
+    // Comment out all but 1 of the following lines to select the correct Oracle PK
+    std::string const public_key = mainnet_public_key;
+    //std::string const public_key = testnet_public_key;
+    //std::string const public_key = stagenet_public_key;
+
+    // verify the signature
+    if (!verifySignature(public_key)) {
+      LOG_ERROR("Invalid pricing record signature.");
+      return false;
+    }
+  
+    // valiadte the timestmap
+    if (hf_version >= HF_VERSION_XASSET_FEES_V2) {
+      if (this->timestamp > bl_timestamp + PRICING_RECORD_VALID_TIME_DIFF_FROM_BLOCK) {
+        LOG_ERROR("Pricing record timestamp is too far in the future.");
+        return false;
+      }
+      if (this->timestamp <= last_bl_timestamp) {
+        LOG_ERROR("Pricing record timestamp is too old.");
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+}

src/offshore/pricing_record.h

@@ -0,0 +1,205 @@
+// Copyright (c) 2019, Haven Protocol
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Portions of this code based upon code Copyright (c) 2019, The Monero Project
+
+#pragma once
+#include "common/pod-class.h"
+
+#include <openssl/bio.h>
+#include <openssl/crypto.h>
+#include <openssl/ecdsa.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+#include <openssl/rsa.h>
+#include <openssl/ssl.h>
+
+#include <cstdint>
+#include <string>
+#include <cstring>
+
+#include "cryptonote_config.h"
+#include "crypto/hash.h"
+
+namespace epee
+{
+  namespace serialization
+  {
+    class portable_storage;
+    struct section;
+  }
+}
+
+namespace offshore
+{
+
+#pragma pack(push, 1)
+  POD_CLASS pricing_record_old {
+    double xAG;
+    double xAU;
+    double xAUD;
+    double xBTC;
+    double xCAN;
+    double xCHF;
+    double xCNY;
+    double xEUR;
+    double xGBP;
+    double xJPY;
+    double xNOK;
+    double xNZD;
+    double xUSD;
+    double unused1;
+    double unused2;
+    double unused3;
+    char signature[32];
+  };
+#pragma pack(pop)
+
+  class pricing_record
+  {
+
+    public:
+
+      // Fields 
+      uint64_t xAG;
+      uint64_t xAU;
+      uint64_t xAUD;
+      uint64_t xBTC;
+      uint64_t xCAD;
+      uint64_t xCHF;
+      uint64_t xCNY;
+      uint64_t xEUR;
+      uint64_t xGBP;
+      uint64_t xJPY;
+      uint64_t xNOK;
+      uint64_t xNZD;
+      uint64_t xUSD;
+      uint64_t unused1;
+      uint64_t unused2;
+      uint64_t unused3;
+      uint64_t timestamp;
+      unsigned char signature[64];
+
+      // Default c'tor
+      pricing_record() noexcept;
+      //! Load from epee p2p format
+      bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent);
+      //! Store in epee p2p format
+      bool store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const;
+      pricing_record(const pricing_record& orig) noexcept;
+      ~pricing_record() = default;
+      void set_for_height_821428();
+      bool equal(const pricing_record& other) const noexcept;
+      bool empty() const noexcept;
+      bool verifySignature(const std::string& public_key) const;
+      bool valid(uint32_t hf_version, uint64_t bl_timestamp, uint64_t last_bl_timestamp) const;
+
+      pricing_record& operator=(const pricing_record& orig) noexcept;
+      uint64_t operator[](const std::string& asset_type) const;
+  };
+
+  inline bool operator==(const pricing_record& a, const pricing_record& b) noexcept
+  {
+   return a.equal(b);
+  }
+  
+  inline bool operator!=(const pricing_record& a, const pricing_record& b) noexcept
+  {
+   return !a.equal(b);
+  }
+
+  // did not have a timestamp
+  class pricing_record_v1
+  {
+
+  public:
+    uint64_t xAG;
+    uint64_t xAU;
+    uint64_t xAUD;
+    uint64_t xBTC;
+    uint64_t xCAD;
+    uint64_t xCHF;
+    uint64_t xCNY;
+    uint64_t xEUR;
+    uint64_t xGBP;
+    uint64_t xJPY;
+    uint64_t xNOK;
+    uint64_t xNZD;
+    uint64_t xUSD;
+    uint64_t unused1;
+    uint64_t unused2;
+    uint64_t unused3;
+    unsigned char signature[64];
+
+    bool write_to_pr(offshore::pricing_record &pr)
+    {
+      pr.xAG = xAG;
+      pr.xAU = xAU;
+      pr.xAUD = xAUD;
+      pr.xBTC = xBTC;
+      pr.xCAD = xCAD;
+      pr.xCHF = xCHF;
+      pr.xCNY = xCNY;
+      pr.xEUR = xEUR;
+      pr.xGBP = xGBP;
+      pr.xJPY = xJPY;
+      pr.xNOK = xNOK;
+      pr.xNZD = xNZD;
+      pr.xUSD = xUSD;
+      pr.unused1 = unused1;
+      pr.unused2 = unused2;
+      pr.unused3 = unused3;
+      pr.timestamp = 0;
+      ::memcpy(pr.signature, signature, sizeof(pr.signature));
+      return true;
+    };
+
+    bool read_from_pr(offshore::pricing_record &pr)
+    {
+      xAG = pr.xAG;
+      xAU = pr.xAU;
+      xAUD = pr.xAUD;
+      xBTC = pr.xBTC;
+      xCAD = pr.xCAD;
+      xCHF = pr.xCHF;
+      xCNY = pr.xCNY;
+      xEUR = pr.xEUR;
+      xGBP = pr.xGBP;
+      xJPY = pr.xJPY;
+      xNOK = pr.xNOK;
+      xNZD = pr.xNZD;
+      xUSD = pr.xUSD;
+      unused1 = pr.unused1;
+      unused2 = pr.unused2;
+      unused3 = pr.unused3;
+      ::memcpy(signature, pr.signature, sizeof(signature));
+      return true;
+    };
+  };
+
+} // offshore

src/serialization/binary_archive.h

@@ -48,7 +48,7 @@ template <>
 struct binary_archive<false> : public binary_archive_base<std::istream, false>
 {
   explicit binary_archive(stream_type &s) : base_type(s) {
-    stream_type::streampos pos = stream_.tellg();
+    auto pos = stream_.tellg();
     stream_.seekg(0, std::ios_base::end);
     eof_pos_ = stream_.tellg();
     stream_.seekg(pos);

src/serialization/crypto.h

@@ -7,7 +7,6 @@
 #include <vector>
 
 #include "serialization.h"
-#include "debug_archive.h"
 #include "crypto/chacha8.h"
 #include "crypto/crypto.h"
 #include "crypto/hash.h"
@@ -55,18 +54,11 @@ BLOB_SERIALIZER(crypto::chacha8_iv);
 BLOB_SERIALIZER(crypto::hash);
 BLOB_SERIALIZER(crypto::cycle);
 BLOB_SERIALIZER(crypto::cycle40);
+BLOB_SERIALIZER(crypto::cycle48);
 BLOB_SERIALIZER(crypto::hash8);
 BLOB_SERIALIZER(crypto::public_key);
 BLOB_SERIALIZER(crypto::secret_key);
 BLOB_SERIALIZER(crypto::key_derivation);
 BLOB_SERIALIZER(crypto::key_image);
 BLOB_SERIALIZER(crypto::signature);
-VARIANT_TAG(debug_archive, crypto::cycle, "cycle");
-VARIANT_TAG(debug_archive, crypto::cycle40, "cycle40");
-VARIANT_TAG(debug_archive, crypto::hash, "hash");
-VARIANT_TAG(debug_archive, crypto::hash8, "hash8");
-VARIANT_TAG(debug_archive, crypto::public_key, "public_key");
-VARIANT_TAG(debug_archive, crypto::secret_key, "secret_key");
-VARIANT_TAG(debug_archive, crypto::key_derivation, "key_derivation");
-VARIANT_TAG(debug_archive, crypto::key_image, "key_image");
-VARIANT_TAG(debug_archive, crypto::signature, "signature");
+BLOB_SERIALIZER(crypto::view_tag);

src/serialization/debug_archive.h

@@ -1,28 +0,0 @@
-// Copyright (c) 2012-2013 The Cryptonote developers
-// Distributed under the MIT/X11 software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#pragma once
-
-#include "json_archive.h"
-#include "variant.h"
-
-template <bool W>
-struct debug_archive : public json_archive<W> {
-  typedef typename json_archive<W>::stream_type stream_type;
-
-  debug_archive(stream_type &s) : json_archive<W>(s) { }
-};
-
-template <class T>
-struct serializer<debug_archive<true>, T>
-{
-  static void serialize(debug_archive<true> &ar, T &v)
-  {
-    ar.begin_object();
-    ar.tag(variant_serialization_traits<debug_archive<true>, T>::get_tag());
-    serializer<json_archive<true>, T>::serialize(ar, v);
-    ar.end_object();
-    ar.stream() << std::endl;
-  }
-};

src/serialization/json_archive.h

@@ -1,145 +0,0 @@
-// Copyright (c) 2012-2013 The Cryptonote developers
-// Distributed under the MIT/X11 software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-/* json_archive.h
- *
- * JSON archive */
-
-#pragma once
-
-#include "serialization.h"
-#include <cassert>
-#include <iostream>
-#include <iomanip>
-
-template <class Stream, bool IsSaving>
-struct json_archive_base
-{
-  typedef Stream stream_type;
-  typedef json_archive_base<Stream, IsSaving> base_type;
-  typedef boost::mpl::bool_<IsSaving> is_saving;
-
-  typedef const char *variant_tag_type;
-
-  json_archive_base(stream_type &s, bool indent = false) : stream_(s), indent_(indent), object_begin(false), depth_(0) { }
-
-  void tag(const char *tag) {
-    if (!object_begin)
-      stream_ << ", ";
-    make_indent();
-    stream_ << '"' << tag << "\": ";
-    object_begin = false;
-  }
-
-  void begin_object()
-  {
-    stream_ << "{";
-    ++depth_;
-    object_begin = true;
-  }
-
-  void end_object()
-  {
-    --depth_;
-    make_indent();
-    stream_ << "}";
-  }
-
-  void begin_variant() { begin_object(); }
-  void end_variant() { end_object(); }
-  Stream &stream() { return stream_; }
-
-protected:
-  void make_indent()
-  {
-    if (indent_)
-    {
-      stream_ << '\n' << std::string(2 * depth_, ' ');
-    }
-  }
-
-protected:
-  stream_type &stream_;
-  bool indent_;
-  bool object_begin;
-  size_t depth_;
-};
-
-template <bool W>
-struct json_archive;
-
-template <>
-struct json_archive<true> : public json_archive_base<std::ostream, true>
-{
-  json_archive(stream_type &s, bool indent = false) : base_type(s, indent) { }
-
-  template<typename T>
-  static auto promote_to_printable_integer_type(T v) -> decltype(+v)
-  {
-    // Unary operator '+' performs integral promotion on type T [expr.unary.op].
-    // If T is signed or unsigned char, it's promoted to int and printed as number.
-    return +v;
-  }
-
-  template <class T>
-  void serialize_int(T v)
-  {
-    stream_ << std::dec << promote_to_printable_integer_type(v);
-  }
-
-  void serialize_blob(void *buf, size_t len, const char *delimiter="\"") {
-    begin_string(delimiter);
-    for (size_t i = 0; i < len; i++) {
-      unsigned char c = ((unsigned char *)buf)[i];
-      stream_ << std::hex << std::setw(2) << std::setfill('0') << (int)c;
-    }
-    end_string(delimiter);
-  }
-
-  template <class T>
-  void serialize_varint(T &v)
-  {
-    stream_ << std::dec << promote_to_printable_integer_type(v);
-  }
-
-  void begin_string(const char *delimiter="\"")
-  {
-    stream_ << delimiter;
-  }
-
-  void end_string(const char *delimiter="\"")
-  {
-    stream_ << delimiter;
-  }
-
-  void begin_array(size_t s=0)
-  {
-    inner_array_size_ = s;
-    ++depth_;
-    stream_ << "[ ";
-  }
-
-  void delimit_array()
-  {
-    stream_ << ", ";
-  }
-
-  void end_array()
-  {
-    --depth_;
-    if (0 < inner_array_size_)
-    {
-      make_indent();
-    }
-    stream_ << "]";
-  }
-
-  void write_variant_tag(const char *t)
-  {
-    tag(t);
-  }
-
-private:
-  size_t inner_array_size_;
-};

src/serialization/json_utils.h

@@ -1,21 +0,0 @@
-// Copyright (c) 2012-2013 The Cryptonote developers
-// Distributed under the MIT/X11 software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#pragma once
-
-#include <sstream>
-#include "json_archive.h"
-
-namespace serialization {
-
-template<class T>
-std::string dump_json(T &v)
-{
-  std::stringstream ostr;
-  json_archive<true> oar(ostr);
-  assert(serialization::serialize(oar, v));
-  return ostr.str();
-};
-
-} // namespace serialization

src/serialization/pricing_record.h

@@ -0,0 +1,96 @@
+// Copyright (c) 2019, Haven Protocol
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#include <vector>
+
+#include "serialization.h"
+#include "offshore/pricing_record.h"
+#include "cryptonote_config.h"
+
+// read
+template <template <bool> class Archive>
+bool do_serialize(Archive<false> &ar, offshore::pricing_record &pr, uint8_t version)
+{
+  if (version < HF_VERSION_XASSET_FEES_V2)
+  {
+    // very basic sanity check
+    if (ar.remaining_bytes() < sizeof(offshore::pricing_record_v1)) {
+      ar.stream().setstate(std::ios::failbit);
+      return false;
+    }
+
+    offshore::pricing_record_v1 pr_v1;
+    ar.serialize_blob(&pr_v1, sizeof(offshore::pricing_record_v1), "");
+    if (!ar.stream().good())
+      return false;
+
+    if (!pr_v1.write_to_pr(pr))
+      return false;
+  }
+  else
+  {
+    // very basic sanity check
+    if (ar.remaining_bytes() < sizeof(offshore::pricing_record)) {
+      ar.stream().setstate(std::ios::failbit);
+      return false;
+    }
+
+    ar.serialize_blob(&pr, sizeof(offshore::pricing_record), "");
+    if (!ar.stream().good())
+      return false;
+  }
+  return true;
+}
+
+// write
+template <template <bool> class Archive>
+bool do_serialize(Archive<true> &ar, offshore::pricing_record &pr, uint8_t version)
+{
+  ar.begin_string();
+  if (version < HF_VERSION_XASSET_FEES_V2)
+  {
+    offshore::pricing_record_v1 pr_v1;
+    if (!pr_v1.read_from_pr(pr))
+      return false;
+    ar.serialize_blob(&pr_v1, sizeof(offshore::pricing_record_v1), "");
+  }
+  else
+  {
+    ar.serialize_blob(&pr, sizeof(offshore::pricing_record), "");
+  }
+  if (!ar.stream().good())
+    return false;
+  ar.end_string();
+  return true;
+}
+
+BLOB_SERIALIZER(offshore::pricing_record);

src/serialization/serialization.h

@@ -109,6 +109,13 @@ inline bool do_serialize(Archive &ar, bool &v)
     ar.serialize_varint(f); \
     if (!ar.stream().good()) return false; \
   } while(0);
+#define VERSION_FIELD(v)                       \
+  uint32_t version = v;                                \
+  do {                                         \
+    ar.tag("version");                         \
+    ar.serialize_varint(version);              \
+    if (!ar.stream().good()) return false;             \
+  } while(0);
 
 namespace serialization {
   namespace detail

src/serialization/zephyr_pricing_record.h

@@ -0,0 +1,126 @@
+// Copyright (c) 2019, Haven Protocol
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// 
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+
+#pragma once
+
+#include <vector>
+
+#include "serialization.h"
+#include "zephyr_oracle/pricing_record.h"
+#include "cryptonote_config.h"
+
+// read
+template <template <bool> class Archive>
+bool do_serialize(Archive<false> &ar, zephyr_oracle::pricing_record &pr, uint8_t version)
+{
+  if (version < 3)
+  {
+    // very basic sanity check
+    if (ar.remaining_bytes() < sizeof(zephyr_oracle::pricing_record_v1)) {
+      return false;
+    }
+
+    zephyr_oracle::pricing_record_v1 pr_v1;
+    ar.serialize_blob(&pr_v1, sizeof(zephyr_oracle::pricing_record_v1), "");
+    if (!ar.good())
+      return false;
+
+    if (!pr_v1.write_to_pr(pr))
+      return false;
+  }
+  else
+  {
+    // very basic sanity check
+    if (ar.remaining_bytes() < sizeof(zephyr_oracle::pricing_record)) {
+      return false;
+    }
+
+    ar.serialize_blob(&pr, sizeof(zephyr_oracle::pricing_record), "");
+    if (!ar.good())
+      return false;
+  }
+
+  return true;
+}
+
+// write
+template <template <bool> class Archive>
+bool do_serialize(Archive<true> &ar, zephyr_oracle::pricing_record &pr, uint8_t version)
+{
+  ar.begin_string();
+
+  if (version < 3)
+  {
+    zephyr_oracle::pricing_record_v1 pr_v1;
+    if (!pr_v1.read_from_pr(pr))
+      return false;
+    ar.serialize_blob(&pr_v1, sizeof(zephyr_oracle::pricing_record_v1), "");
+  }
+  else
+  {
+    ar.serialize_blob(&pr, sizeof(zephyr_oracle::pricing_record), "");
+  }
+
+  if (!ar.good())
+    return false;
+  ar.end_string();
+  return true;
+}
+
+// read
+template <template <bool> class Archive>
+bool do_serialize(Archive<false> &ar, zephyr_oracle::pricing_record_v1 &pr, uint8_t version)
+{
+  // very basic sanity check
+  if (ar.remaining_bytes() < sizeof(zephyr_oracle::pricing_record_v1)) {
+    return false;
+  }
+
+  ar.serialize_blob(&pr, sizeof(zephyr_oracle::pricing_record_v1), "");
+  if (!ar.good())
+    return false;
+
+  return true;
+}
+
+// write
+template <template <bool> class Archive>
+bool do_serialize(Archive<true> &ar, zephyr_oracle::pricing_record_v1 &pr, uint8_t version)
+{
+  ar.begin_string();
+  ar.serialize_blob(&pr, sizeof(zephyr_oracle::pricing_record_v1), "");
+  if (!ar.good())
+    return false;
+  ar.end_string();
+  return true;
+}
+
+BLOB_SERIALIZER(zephyr_oracle::pricing_record);
+BLOB_SERIALIZER(zephyr_oracle::pricing_record_v1);

src/zephyr_oracle/asset_types.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2021, Haven Protocol
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+#include <string>
+#include <vector>
+
+namespace zephyr_oracle {
+
+  const std::vector<std::string> ASSET_TYPES = {"ZEPH", "ZEPHUSD", "ZEPHRSV"};
+
+  class asset_type_counts
+  {
+
+    public:
+
+      // Fields 
+      uint64_t ZEPH;
+      uint64_t ZEPHUSD;
+      uint64_t ZEPHRSV;
+
+      asset_type_counts() noexcept
+        : ZEPH(0)
+        , ZEPHUSD(0)
+        , ZEPHRSV(0)
+      {
+      }
+
+      uint64_t operator[](const std::string asset_type) const noexcept
+      {
+        if (asset_type == "ZEPH") {
+          return ZEPH;
+        } else if (asset_type == "ZEPHUSD") {
+          return ZEPHUSD;
+        } else if (asset_type == "ZEPHRSV") {
+          return ZEPHRSV;
+        }
+
+        return 0;
+      }
+
+      void add(const std::string asset_type, const uint64_t val)
+      {
+        if (asset_type == "ZEPH") {
+          ZEPH += val;
+        } else if (asset_type == "ZEPHUSD") {
+          ZEPHUSD += val;
+        } else if (asset_type == "ZEPHRSV") {
+          ZEPHRSV += val;
+        }
+      }
+  };
+}

src/zephyr_oracle/pricing_record.cpp

@@ -0,0 +1,259 @@
+// Copyright (c) 2019, Haven Protocol
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Portions of this code based upon code Copyright (c) 2019, The Monero Project
+
+#include "pricing_record.h"
+
+#include "serialization/keyvalue_serialization.h"
+#include "storages/portable_storage.h"
+
+#include "string_tools.h"
+namespace zephyr_oracle
+{
+
+  namespace
+  {
+    struct pr_serialized
+    {
+      uint64_t spot;
+      uint64_t moving_average;
+      uint64_t stable;
+      uint64_t stable_ma;
+      uint64_t reserve;
+      uint64_t reserve_ma;
+      uint64_t timestamp;
+      std::string signature;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(spot)
+        KV_SERIALIZE(moving_average)
+        KV_SERIALIZE(stable)
+        KV_SERIALIZE(stable_ma)
+        KV_SERIALIZE(reserve)
+        KV_SERIALIZE(reserve_ma)
+        KV_SERIALIZE(timestamp)
+        KV_SERIALIZE(signature)
+      END_KV_SERIALIZE_MAP()
+    };
+  }
+  
+  pricing_record::pricing_record() noexcept
+    : spot(0)
+    , moving_average(0)
+    , stable(0)
+    , stable_ma(0)
+    , reserve(0)
+    , reserve_ma(0)
+    , timestamp(0)
+  {
+    std::memset(signature, 0, sizeof(signature));
+  }
+
+  bool pricing_record::_load(epee::serialization::portable_storage& src, epee::serialization::section* hparent)
+  {
+    pr_serialized in{};
+    if (in._load(src, hparent))
+    {
+      // Copy everything into the local instance
+      spot = in.spot;
+      moving_average = in.moving_average;
+      stable = in.stable;
+      stable_ma = in.stable_ma;
+      reserve = in.reserve;
+      reserve_ma = in.reserve_ma;
+      timestamp = in.timestamp;
+      for (unsigned int i = 0; i < in.signature.length(); i += 2) {
+        std::string byteString = in.signature.substr(i, 2);
+        signature[i>>1] = (char) strtol(byteString.c_str(), NULL, 16);
+      }
+      return true;
+    }
+
+    // Report error here?
+    return false;
+  }
+
+  bool pricing_record::store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const
+  {
+    std::string sig_hex;
+    for (unsigned int i=0; i<64; i++) {
+      std::stringstream ss;
+      ss << std::hex << std::setw(2) << std::setfill('0') << (0xff & signature[i]);
+      sig_hex += ss.str();
+    }
+    const pr_serialized out{spot,moving_average,stable,stable_ma,reserve,reserve_ma,timestamp,sig_hex};
+    return out.store(dest, hparent);
+  }
+
+  pricing_record::pricing_record(const pricing_record& orig) noexcept
+    : spot(orig.spot)
+    , moving_average(orig.moving_average)
+    , stable(orig.stable)
+    , stable_ma(orig.stable_ma)
+    , reserve(orig.reserve)
+    , reserve_ma(orig.reserve_ma)
+    , timestamp(orig.timestamp)
+  {
+    std::memcpy(signature, orig.signature, sizeof(signature));
+  }
+
+  pricing_record& pricing_record::operator=(const pricing_record& orig) noexcept
+  {
+    spot = orig.spot;
+    moving_average = orig.moving_average;
+    stable = orig.stable;
+    stable_ma = orig.stable_ma;
+    reserve = orig.reserve;
+    reserve_ma = orig.reserve_ma;
+    timestamp = orig.timestamp;
+    ::memcpy(signature, orig.signature, sizeof(signature));
+    return *this;
+  }
+  
+  bool pricing_record::equal(const pricing_record& other) const noexcept
+  {
+    return ((spot == other.spot) &&
+      (moving_average == other.moving_average) &&
+      (stable == other.stable) &&
+      (stable_ma == other.stable_ma) &&
+      (reserve == other.reserve) &&
+      (reserve_ma == other.reserve_ma) &&
+	    (timestamp == other.timestamp) &&
+      !::memcmp(signature, other.signature, sizeof(signature)));
+  }
+
+  bool pricing_record::empty() const noexcept
+  {
+    const pricing_record empty_pr = zephyr_oracle::pricing_record();
+    return (*this).equal(empty_pr);
+  }
+
+  bool pricing_record::verifySignature(const std::string& public_key) const
+  {
+    CHECK_AND_ASSERT_THROW_MES(!public_key.empty(), "Pricing record verification failed. NULL public key. PK Size: " << public_key.size()); // TODO: is this necessary or the one below already covers this case, meannin it will produce empty pubkey?
+
+    // extract the key
+    EVP_PKEY* pubkey;
+    BIO* bio = BIO_new_mem_buf(public_key.c_str(), public_key.size());
+    if (!bio) {
+      return false;
+    }
+    pubkey = PEM_read_bio_PUBKEY(bio, NULL, NULL, NULL);
+    BIO_free(bio);
+    CHECK_AND_ASSERT_THROW_MES(pubkey != NULL, "Pricing record verification failed. NULL public key.");
+
+    // Convert our internal 64-byte binary representation into 128-byte hex string
+    std::string sig_hex;
+    for (unsigned int i=0; i<64; i++) {
+      std::stringstream ss;
+      ss << std::hex << std::setw(2) << std::setfill('0') << (0xff & signature[i]);
+      sig_hex += ss.str();
+    }
+
+    // Build the JSON string, so that we can verify the signature
+    std::ostringstream oss;
+    oss << "{\"spot\":" << spot;
+    oss << ",\"moving_average\":" << moving_average;
+    oss << ",\"timestamp\":" << timestamp;
+    oss << "}";
+    std::string message = oss.str();
+
+    // Create a verify digest from the message
+    EVP_MD_CTX *ctx = EVP_MD_CTX_create();
+    int ret = 0;
+    if (ctx) {
+      ret = EVP_DigestVerifyInit(ctx, NULL, EVP_sha256(), NULL, pubkey);
+      if (ret == 1) {
+        ret = EVP_DigestVerifyUpdate(ctx, message.data(), message.length());
+        if (ret == 1) {
+          ret = EVP_DigestVerifyFinal(ctx, (const unsigned char *)signature, 64);
+        }
+      }
+    }
+
+    // Cleanup the context we created
+    EVP_MD_CTX_destroy(ctx);
+    // Cleanup the openssl stuff
+    EVP_PKEY_free(pubkey);
+
+    if (ret == 1)
+      return true;
+
+    // Get the errors from OpenSSL
+    // ERR_print_errors_fp (stderr);
+
+    return false;
+  }
+
+  bool pricing_record::has_missing_rates() const noexcept
+  {
+    return (spot == 0) || (moving_average == 0) || (stable == 0) || (stable_ma == 0) || (reserve == 0) || (reserve_ma == 0);
+  }
+
+  // overload for pr validation for block
+  bool pricing_record::valid(uint32_t hf_version, uint64_t bl_timestamp, uint64_t last_bl_timestamp) const 
+  {
+    if (hf_version < 3) {
+      if (!this->empty())
+        return false;
+    }
+
+    if (this->empty())
+        return true;
+
+    if (this->has_missing_rates()) {
+      LOG_ERROR("Pricing record has missing rates.");
+      return false;
+    }
+
+    std::string const MAINNET_ORACLE_PUBLIC_KEY = "-----BEGIN PUBLIC KEY-----\n"
+    "MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAO5hVuc6ylYMbj3WhqOMoAcJ0SD4e3zW\n"
+    "edsUmhQeYwBkelAaFyxhX4ZotP+b/cFr2mX5iuND1znEnMZkyg+YmtkCAwEAAQ==\n"
+    "-----END PUBLIC KEY-----\n";
+
+    if (!verifySignature(MAINNET_ORACLE_PUBLIC_KEY)) {
+      LOG_ERROR("Invalid pricing record signature.");
+      return false;
+    }
+
+    // validate the timestmap
+    if (this->timestamp > bl_timestamp + PRICING_RECORD_VALID_TIME_DIFF_FROM_BLOCK) {
+      LOG_ERROR("Pricing record timestamp is too far in the future.");
+      return false;
+    }
+
+    if (this->timestamp <= last_bl_timestamp) {
+      LOG_ERROR("Pricing record timestamp: " << this->timestamp << ", block timestamp: " << bl_timestamp);
+      LOG_ERROR("Pricing record timestamp is too old.");
+      return false;
+    }
+
+    return true;
+  }
+}
+

src/zephyr_oracle/pricing_record.h

@@ -0,0 +1,140 @@
+// Copyright (c) 2019, Haven Protocol
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Portions of this code based upon code Copyright (c) 2019, The Monero Project
+
+#pragma once
+#include "common/pod-class.h"
+
+#include <openssl/bio.h>
+#include <openssl/crypto.h>
+#include <openssl/ecdsa.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+#include <openssl/rsa.h>
+#include <openssl/ssl.h>
+
+#include <cstdint>
+#include <string>
+#include <cstring>
+
+#include "cryptonote_config.h"
+#include "crypto/hash.h"
+
+namespace epee
+{
+  namespace serialization
+  {
+    class portable_storage;
+    struct section;
+  }
+}
+
+namespace zephyr_oracle
+{
+  #pragma pack(push, 1)
+  POD_CLASS pricing_record_pre {
+    uint64_t zEPHUSD;
+    uint64_t zEPHRSV;
+    uint64_t timestamp;
+  };
+  #pragma pack(pop)
+  class pricing_record
+  {
+
+    public:
+
+      // Fields 
+      uint64_t spot;
+      uint64_t moving_average;
+      uint64_t stable;
+      uint64_t stable_ma;
+      uint64_t reserve;
+      uint64_t reserve_ma;
+      uint64_t timestamp;
+      unsigned char signature[64];
+
+      // Default c'tor
+      pricing_record() noexcept;
+      //! Load from epee p2p format
+      bool _load(epee::serialization::portable_storage& src, epee::serialization::section* hparent);
+      //! Store in epee p2p format
+      bool store(epee::serialization::portable_storage& dest, epee::serialization::section* hparent) const;
+      pricing_record(const pricing_record& orig) noexcept;
+      ~pricing_record() = default;
+      bool equal(const pricing_record& other) const noexcept;
+      bool empty() const noexcept;
+      bool verifySignature(const std::string& public_key) const;
+      bool has_missing_rates() const noexcept;
+      bool valid(uint32_t hf_version, uint64_t bl_timestamp, uint64_t last_bl_timestamp) const;
+
+      pricing_record& operator=(const pricing_record& orig) noexcept;
+      uint64_t operator[](const std::string& asset_type) const;
+  };
+
+  inline bool operator==(const pricing_record& a, const pricing_record& b) noexcept
+  {
+   return a.equal(b);
+  }
+  
+  inline bool operator!=(const pricing_record& a, const pricing_record& b) noexcept
+  {
+   return !a.equal(b);
+  }
+
+  class pricing_record_v1
+  {
+
+  public:
+    uint64_t zEPHUSD;
+    uint64_t zEPHRSV;
+    uint64_t timestamp;
+
+    bool write_to_pr(zephyr_oracle::pricing_record &pr)
+    {
+      pr.spot = 0;
+      pr.moving_average = 0;
+      pr.stable = 0;
+      pr.stable_ma = 0;
+      pr.reserve = 0;
+      pr.reserve_ma = 0;
+      pr.timestamp = 0;
+      std::memset(pr.signature, 0, sizeof(zephyr_oracle::pricing_record::signature));
+      return true;
+    };
+
+    bool read_from_pr(zephyr_oracle::pricing_record &pr)
+    {
+      zEPHUSD = 0;
+      zEPHRSV = 0;
+      timestamp = 0;
+      return true;
+    };
+  };
+
+} // oracle