From e483a689e30708d5a953fc226712bbbc8e47b9e6 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com>
Date: Fri, 7 Jul 2017 15:48:13 -0700
Subject: [PATCH 1/6] Merge #17850: Convert addrdb/addrman to new serialization

https://github.com/bitcoin/bitcoin/pull/17850/commits/9250a087d2f450f37342010aea0a5d583eea508b

Partial #17850: Introduce new serialization macros without casts

https://github.com/bitcoin/bitcoin/pull/17850/commits/ca33451535dc29f64d37c49af8b22142d7716d0d
---
 src/addrdb.h    | 10 +---------
 src/addrman.h   | 14 +++++---------
 src/serialize.h | 24 ++++++++++++++++++++++++
 3 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/src/addrdb.h b/src/addrdb.h
index c42494f88f16..b467645d098a 100644
--- a/src/addrdb.h
+++ b/src/addrdb.h
@@ -43,15 +43,7 @@ class CBanEntry
         nCreateTime = nCreateTimeIn;
     }
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(this->nVersion);
-        READWRITE(nCreateTime);
-        READWRITE(nBanUntil);
-        READWRITE(banReason);
-    }
+    SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); }
 
     void SetNull()
     {
diff --git a/src/addrman.h b/src/addrman.h
index 48ee327e6994..7a4cd4ee647d 100644
--- a/src/addrman.h
+++ b/src/addrman.h
@@ -55,14 +55,10 @@ class CAddrInfo : public CAddress
 
 public:
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITEAS(CAddress, *this);
-        READWRITE(source);
-        READWRITE(nLastSuccess);
-        READWRITE(nAttempts);
+    SERIALIZE_METHODS(CAddrInfo, obj)
+    {
+        READWRITEAS(CAddress, obj);
+        READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts);
     }
 
     void Init()
@@ -315,7 +311,7 @@ class CAddrMan
      * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports
      * changes to the ADDRMAN_ parameters without breaking the on-disk structure.
      *
-     * We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has
+     * We don't use SERIALIZE_METHODS since the serialization and deserialization code has
      * very little in common.
      */
     template<typename Stream>
diff --git a/src/serialize.h b/src/serialize.h
index 23ec49d47ca6..63bee528a84f 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -203,6 +203,30 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
         SerializationOp(s, CSerActionUnserialize());                  \
     }
 
+/**
+ * Implement the Serialize and Unserialize methods by delegating to a single templated
+ * static method that takes the to-be-(de)serialized object as a parameter. This approach
+ * has the advantage that the constness of the object becomes a template parameter, and
+ * thus allows a single implementation that sees the object as const for serializing
+ * and non-const for deserializing, without casts.
+ */
+#define SERIALIZE_METHODS(cls, obj)                                                 \
+    template<typename Stream>                                                       \
+    void Serialize(Stream& s) const                                                 \
+    {                                                                               \
+        static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
+        SerializationOps(*this, s, CSerActionSerialize());                          \
+    }                                                                               \
+    template<typename Stream>                                                       \
+    void Unserialize(Stream& s)                                                     \
+    {                                                                               \
+        static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
+        SerializationOps(*this, s, CSerActionUnserialize());                        \
+    }                                                                               \
+    template<typename Stream, typename Type, typename Operation>                    \
+    static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
+
+
 #ifndef CHAR_EQUALS_INT8
 template<typename Stream> inline void Serialize(Stream& s, char a    ) { ser_writedata8(s, a); } // TODO Get rid of bare char
 #endif

From 46cb6e23dcdd6a5916c778f14336a968d051b546 Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com>
Date: Wed, 8 Jan 2020 08:56:19 -0800
Subject: [PATCH 2/6] Merge #17896: Convert chain to new serialization

https://github.com/bitcoin/bitcoin/pull/17896/commits/9b66083788581c264a097e26795561cb3eac455d

Partial #17896: Convert VARINT to the formatter/Using approach

https://github.com/bitcoin/bitcoin/pull/17896/commits/2f1b2f4ed044fe005e5a6c1b55e95822e83c16df

Partial #17896: Add a generic approach for (de)serialization of objects

https://github.com/bitcoin/bitcoin/pull/17896/commits/ca62563df341786d1d1809a037d8b592924e78c4
---
 src/chain.h     | 60 +++++++++++++++++++++----------------------------
 src/serialize.h | 53 +++++++++++++++++++++++++++++--------------
 2 files changed, 62 insertions(+), 51 deletions(-)

diff --git a/src/chain.h b/src/chain.h
index 89123185ab62..bb2763a1b1f5 100644
--- a/src/chain.h
+++ b/src/chain.h
@@ -39,17 +39,15 @@ class CBlockFileInfo
     uint64_t nTimeFirst;       //!< earliest time of block in file
     uint64_t nTimeLast;        //!< latest time of block in file
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(VARINT(nBlocks));
-        READWRITE(VARINT(nSize));
-        READWRITE(VARINT(nUndoSize));
-        READWRITE(VARINT(nHeightFirst));
-        READWRITE(VARINT(nHeightLast));
-        READWRITE(VARINT(nTimeFirst));
-        READWRITE(VARINT(nTimeLast));
+    SERIALIZE_METHODS(CBlockFileInfo, obj)
+    {
+        READWRITE(VARINT(obj.nBlocks));
+        READWRITE(VARINT(obj.nSize));
+        READWRITE(VARINT(obj.nUndoSize));
+        READWRITE(VARINT(obj.nHeightFirst));
+        READWRITE(VARINT(obj.nHeightLast));
+        READWRITE(VARINT(obj.nTimeFirst));
+        READWRITE(VARINT(obj.nTimeLast));
     }
 
      void SetNull() {
@@ -383,33 +381,27 @@ class CDiskBlockIndex : public CBlockIndex
         hashPrev = (pprev ? pprev->GetBlockHash() : uint256());
     }
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
+    SERIALIZE_METHODS(CDiskBlockIndex, obj)
+    {
         int _nVersion = s.GetVersion();
-        if (!(s.GetType() & SER_GETHASH))
-            READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
-
-        READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
-        READWRITE(VARINT(nStatus));
-        READWRITE(VARINT(nTx));
-        if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO))
-            READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
-        if (nStatus & BLOCK_HAVE_DATA)
-            READWRITE(VARINT(nDataPos));
-        if (nStatus & BLOCK_HAVE_UNDO)
-            READWRITE(VARINT(nUndoPos));
+        if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED));
+
+        READWRITE(VARINT(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED));
+        READWRITE(VARINT(obj.nStatus));
+        READWRITE(VARINT(obj.nTx));
+        if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED));
+        if (obj.nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(obj.nDataPos));
+        if (obj.nStatus & BLOCK_HAVE_UNDO) READWRITE(VARINT(obj.nUndoPos));
 
         // block hash
-        READWRITE(hash);
+        READWRITE(obj.hash);
         // block header
-        READWRITE(this->nVersion);
-        READWRITE(hashPrev);
-        READWRITE(hashMerkleRoot);
-        READWRITE(nTime);
-        READWRITE(nBits);
-        READWRITE(nNonce);
+        READWRITE(obj.nVersion);
+        READWRITE(obj.hashPrev);
+        READWRITE(obj.hashMerkleRoot);
+        READWRITE(obj.nTime);
+        READWRITE(obj.nBits);
+        READWRITE(obj.nNonce);
     }
 
     uint256 GetBlockHash() const
diff --git a/src/serialize.h b/src/serialize.h
index 63bee528a84f..c5d489ff972b 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -448,11 +448,37 @@ I ReadVarInt(Stream& is)
     }
 }
 
+/** Simple wrapper class to serialize objects using a formatter; used by Using(). */
+template<typename Formatter, typename T>
+class Wrapper
+{
+    static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
+protected:
+    T m_object;
+public:
+    explicit Wrapper(T obj) : m_object(obj) {}
+    template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
+    template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
+};
+
+/** Cause serialization/deserialization of an object to be done using a specified formatter class.
+ *
+ * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for
+ * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside
+ * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object).
+ *
+ * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is
+ * const during serialization, and non-const during deserialization, which maintains const
+ * correctness.
+ */
+template<typename Formatter, typename T>
+static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
+
 #define FIXEDBITSET(obj, size) CFixedBitSet(REF(obj), (size))
 #define DYNBITSET(obj) CDynamicBitSet(REF(obj))
 #define FIXEDVARINTSBITSET(obj, size) CFixedVarIntsBitSet(REF(obj), (size))
 #define AUTOBITSET(obj, size) CAutoBitSet(REF(obj), (size))
-#define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj))
+#define VARINT(obj, ...) Using<VarIntFormatter<__VA_ARGS__>>(obj)
 #define COMPACTSIZE(obj) CCompactSize(REF(obj))
 #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
 
@@ -612,22 +638,18 @@ class CAutoBitSet
     }
 };
 
-template<VarIntMode Mode, typename I>
-class CVarInt
+/** Serialization wrapper class for integers in VarInt format. */
+template<VarIntMode Mode=VarIntMode::DEFAULT>
+struct VarIntFormatter
 {
-protected:
-    I &n;
-public:
-    explicit CVarInt(I& nIn) : n(nIn) { }
-
-    template<typename Stream>
-    void Serialize(Stream &s) const {
-        WriteVarInt<Stream,Mode,I>(s, n);
+    template<typename Stream, typename I> void Ser(Stream &s, I v)
+    {
+        WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
     }
 
-    template<typename Stream>
-    void Unserialize(Stream& s) {
-        n = ReadVarInt<Stream,Mode,I>(s);
+    template<typename Stream, typename I> void Unser(Stream& s, I& v)
+    {
+        v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
     }
 };
 
@@ -716,9 +738,6 @@ class LimitedString
     }
 };
 
-template<VarIntMode Mode=VarIntMode::DEFAULT, typename I>
-CVarInt<Mode, I> WrapVarInt(I& n) { return CVarInt<Mode, I>{n}; }
-
 template<typename I>
 BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
 

From 0802bdfa373e32af2b974bde02d0455c1088ee1e Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com>
Date: Wed, 6 Sep 2017 11:25:03 -0700
Subject: [PATCH 3/6] Merge #18021: Convert undo.h to new serialization
 framework

https://github.com/bitcoin/bitcoin/pull/18021/commits/3c94b0039d2ca2a8c41fd6127ff5019a2afc304e

Partial #18021: Make std::vector and prevector reuse the VectorFormatter logic

https://github.com/bitcoin/bitcoin/pull/18021/commits/3cd8ab9d11e4c0ea47e56be4f6f2fdd48806796c

Partial #18021: Add custom vector-element formatter

https://github.com/bitcoin/bitcoin/pull/18021/commits/abf86243568af380c1384ac4e0bfcdcfd4dab085

Partial #18021: Add a constant for the maximum vector allocation (5 Mbyte)

37d800bea016d5cba5635db036f53a486614ed30
---
 src/serialize.h | 98 +++++++++++++++++++++++++++++++++----------------
 src/undo.h      | 68 +++++++---------------------------
 2 files changed, 79 insertions(+), 87 deletions(-)

diff --git a/src/serialize.h b/src/serialize.h
index c5d489ff972b..ef4f5ad9365f 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -29,6 +29,9 @@
 
 static const unsigned int MAX_SIZE = 0x02000000;
 
+/** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
+static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
+
 /**
  * Dummy data type to identify deserializing constructors.
  *
@@ -741,6 +744,53 @@ class LimitedString
 template<typename I>
 BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
 
+/** Formatter to serialize/deserialize vector elements using another formatter
+ *
+ * Example:
+ *   struct X {
+ *     std::vector<uint64_t> v;
+ *     SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
+ *   };
+ * will define a struct that contains a vector of uint64_t, which is serialized
+ * as a vector of VarInt-encoded integers.
+ *
+ * V is not required to be an std::vector type. It works for any class that
+ * exposes a value_type, size, reserve, push_back, and const iterators.
+ */
+template<class Formatter>
+struct VectorFormatter
+{
+    template<typename Stream, typename V>
+    void Ser(Stream& s, const V& v)
+    {
+        WriteCompactSize(s, v.size());
+        for (const typename V::value_type& elem : v) {
+            s << Using<Formatter>(elem);
+        }
+    }
+
+    template<typename Stream, typename V>
+    void Unser(Stream& s, V& v)
+    {
+        v.clear();
+        size_t size = ReadCompactSize(s);
+        size_t allocated = 0;
+        while (allocated < size) {
+            // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
+            // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
+            // X MiB of data to make us allocate X+5 Mib.
+            static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
+            allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
+            v.reserve(allocated);
+            while (v.size() < allocated) {
+                typename V::value_type val;
+                s >> Using<Formatter>(val);
+                v.push_back(std::move(val));
+            }
+        }
+    };
+};
+
 /**
  * Forward declarations
  */
@@ -863,8 +913,20 @@ inline void Unserialize(Stream& s, T& a )
     a = (T)b;
 }
 
+/** Default formatter. Serializes objects as themselves.
+ *
+ * The vector/prevector serialization code passes this to VectorFormatter
+ * to enable reusing that logic. It shouldn't be needed elsewhere.
+ */
+struct DefaultFormatter
+{
+    template<typename Stream, typename T>
+    static void Ser(Stream& s, const T& t) { Serialize(s, t); }
 
 
+    template<typename Stream, typename T>
+    static void Unser(Stream& s, T& t) { Unserialize(s, t); }
+};
 
 /**
  * string
@@ -902,9 +964,7 @@ void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
 template<typename Stream, unsigned int N, typename T, typename V>
 void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
 {
-    WriteCompactSize(os, v.size());
-    for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
-        ::Serialize(os, (*vi));
+    Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, unsigned int N, typename T>
@@ -933,19 +993,7 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
 template<typename Stream, unsigned int N, typename T, typename V>
 void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
 {
-    v.clear();
-    unsigned int nSize = ReadCompactSize(is);
-    unsigned int i = 0;
-    unsigned int nMid = 0;
-    while (nMid < nSize)
-    {
-        nMid += 5000000 / sizeof(T);
-        if (nMid > nSize)
-            nMid = nSize;
-        v.resize_uninitialized(nMid);
-        for (; i < nMid; ++i)
-            Unserialize(is, v[i]);
-    }
+    Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, unsigned int N, typename T>
@@ -970,9 +1018,7 @@ void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&
 template<typename Stream, typename T, typename A, typename V>
 void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
 {
-    WriteCompactSize(os, v.size());
-    for (typename std::vector<T, A>::const_iterator vi = v.begin(); vi != v.end(); ++vi)
-        ::Serialize(os, (*vi));
+    Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, typename T, typename A>
@@ -1001,19 +1047,7 @@ void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
 template<typename Stream, typename T, typename A, typename V>
 void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
 {
-    v.clear();
-    unsigned int nSize = ReadCompactSize(is);
-    unsigned int i = 0;
-    unsigned int nMid = 0;
-    while (nMid < nSize)
-    {
-        nMid += 5000000 / sizeof(T);
-        if (nMid > nSize)
-            nMid = nSize;
-        v.resize(nMid);
-        for (; i < nMid; i++)
-            Unserialize(is, v[i]);
-    }
+    Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
 }
 
 template<typename Stream, typename T, typename A>
diff --git a/src/undo.h b/src/undo.h
index 70ebc1e04f5b..36f881509843 100644
--- a/src/undo.h
+++ b/src/undo.h
@@ -12,57 +12,42 @@
 #include <primitives/transaction.h>
 #include <serialize.h>
 
-/** Undo information for a CTxIn
+/** Formatter for undo information for a CTxIn
  *
  *  Contains the prevout's CTxOut being spent, and its metadata as well
  *  (coinbase or not, height). The serialization contains a dummy value of
  *  zero. This is compatible with older versions which expect to see
  *  the transaction version there.
  */
-class TxInUndoSerializer
+struct TxInUndoFormatter
 {
-    const Coin* txout;
-
-public:
     template<typename Stream>
-    void Serialize(Stream &s) const {
-        ::Serialize(s, VARINT(txout->nHeight * 2 + (txout->fCoinBase ? 1u : 0u)));
-        if (txout->nHeight > 0) {
+    void Ser(Stream &s, const Coin& txout) {
+        ::Serialize(s, VARINT(txout.nHeight * 2 + (txout.fCoinBase ? 1u : 0u)));
+        if (txout.nHeight > 0) {
             // Required to maintain compatibility with older undo format.
             ::Serialize(s, (unsigned char)0);
         }
-        ::Serialize(s, CTxOutCompressor(REF(txout->out)));
+        ::Serialize(s, CTxOutCompressor(REF(txout.out)));
     }
 
-    explicit TxInUndoSerializer(const Coin* coin) : txout(coin) {}
-};
-
-class TxInUndoDeserializer
-{
-    Coin* txout;
-
-public:
     template<typename Stream>
-    void Unserialize(Stream &s) {
+    void Unser(Stream &s, Coin& txout) {
         unsigned int nCode = 0;
         ::Unserialize(s, VARINT(nCode));
-        txout->nHeight = nCode / 2;
-        txout->fCoinBase = nCode & 1;
-        if (txout->nHeight > 0) {
+        txout.nHeight = nCode / 2;
+        txout.fCoinBase = nCode & 1;
+        if (txout.nHeight > 0) {
             // Old versions stored the version number for the last spend of
             // a transaction's outputs. Non-final spends were indicated with
             // height = 0.
             unsigned int nVersionDummy;
             ::Unserialize(s, VARINT(nVersionDummy));
         }
-        ::Unserialize(s, CTxOutCompressor(REF(txout->out)));
+        ::Unserialize(s, CTxOutCompressor(REF(txout.out)));
     }
-
-    explicit TxInUndoDeserializer(Coin* coin) : txout(coin) {}
 };
 
-static const size_t MAX_INPUTS_PER_BLOCK = MaxBlockSize() / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION);
-
 /** Undo information for a CTransaction */
 class CTxUndo
 {
@@ -70,29 +55,7 @@ class CTxUndo
     // undo information for all txins
     std::vector<Coin> vprevout;
 
-    template <typename Stream>
-    void Serialize(Stream& s) const {
-        // TODO: avoid reimplementing vector serializer
-        uint64_t count = vprevout.size();
-        ::Serialize(s, COMPACTSIZE(REF(count)));
-        for (const auto& prevout : vprevout) {
-            ::Serialize(s, TxInUndoSerializer(&prevout));
-        }
-    }
-
-    template <typename Stream>
-    void Unserialize(Stream& s) {
-        // TODO: avoid reimplementing vector deserializer
-        uint64_t count = 0;
-        ::Unserialize(s, COMPACTSIZE(count));
-        if (count > MAX_INPUTS_PER_BLOCK) {
-            throw std::ios_base::failure("Too many input undo records");
-        }
-        vprevout.resize(count);
-        for (auto& prevout : vprevout) {
-            ::Unserialize(s, TxInUndoDeserializer(&prevout));
-        }
-    }
+    SERIALIZE_METHODS(CTxUndo, obj) { READWRITE(Using<VectorFormatter<TxInUndoFormatter>>(obj.vprevout)); }
 };
 
 /** Undo information for a CBlock */
@@ -101,12 +64,7 @@ class CBlockUndo
 public:
     std::vector<CTxUndo> vtxundo; // for all but the coinbase
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(vtxundo);
-    }
+    SERIALIZE_METHODS(CBlockUndo, obj) { READWRITE(obj.vtxundo); }
 };
 
 #endif // BITCOIN_UNDO_H

From 2f542a93a13a8c26d5a29decf094bebc315a176d Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com>
Date: Sat, 8 Jul 2017 16:43:34 -0700
Subject: [PATCH 4/6] Merge #12752: Move compressor utility functions out of
 class

https://github.com/bitcoin/bitcoin/commit/76a9aacd3fb75d5e0854e53bb3376f2ab603a561
---
 src/compressor.cpp          | 30 +++++++++++++++++++-----------
 src/compressor.h            | 31 ++++++++++---------------------
 src/test/compress_tests.cpp |  8 ++++----
 3 files changed, 33 insertions(+), 36 deletions(-)

diff --git a/src/compressor.cpp b/src/compressor.cpp
index a729e28f079f..b003a86ce7a4 100644
--- a/src/compressor.cpp
+++ b/src/compressor.cpp
@@ -9,7 +9,15 @@
 #include <pubkey.h>
 #include <script/standard.h>
 
-bool CScriptCompressor::IsToKeyID(CKeyID &hash) const
+/*
+ * These check for scripts for which a special case with a shorter encoding is defined.
+ * They are implemented separately from the CScript test, as these test for exact byte
+ * sequence correspondences, and are more strict. For example, IsToPubKey also verifies
+ * whether the public key is valid (as invalid ones cannot be represented in compressed
+ * form).
+ */
+
+static bool IsToKeyID(const CScript& script, CKeyID &hash)
 {
     if (script.size() == 25 && script[0] == OP_DUP && script[1] == OP_HASH160
                             && script[2] == 20 && script[23] == OP_EQUALVERIFY
@@ -20,7 +28,7 @@ bool CScriptCompressor::IsToKeyID(CKeyID &hash) const
     return false;
 }
 
-bool CScriptCompressor::IsToScriptID(CScriptID &hash) const
+static bool IsToScriptID(const CScript& script, CScriptID &hash)
 {
     if (script.size() == 23 && script[0] == OP_HASH160 && script[1] == 20
                             && script[22] == OP_EQUAL) {
@@ -30,7 +38,7 @@ bool CScriptCompressor::IsToScriptID(CScriptID &hash) const
     return false;
 }
 
-bool CScriptCompressor::IsToPubKey(CPubKey &pubkey) const
+static bool IsToPubKey(const CScript& script, CPubKey &pubkey)
 {
     if (script.size() == 35 && script[0] == 33 && script[34] == OP_CHECKSIG
                             && (script[1] == 0x02 || script[1] == 0x03)) {
@@ -45,24 +53,24 @@ bool CScriptCompressor::IsToPubKey(CPubKey &pubkey) const
     return false;
 }
 
-bool CScriptCompressor::Compress(std::vector<unsigned char> &out) const
+bool CompressScript(const CScript& script, std::vector<unsigned char> &out)
 {
     CKeyID keyID;
-    if (IsToKeyID(keyID)) {
+    if (IsToKeyID(script, keyID)) {
         out.resize(21);
         out[0] = 0x00;
         memcpy(&out[1], &keyID, 20);
         return true;
     }
     CScriptID scriptID;
-    if (IsToScriptID(scriptID)) {
+    if (IsToScriptID(script, scriptID)) {
         out.resize(21);
         out[0] = 0x01;
         memcpy(&out[1], &scriptID, 20);
         return true;
     }
     CPubKey pubkey;
-    if (IsToPubKey(pubkey)) {
+    if (IsToPubKey(script, pubkey)) {
         out.resize(33);
         memcpy(&out[1], &pubkey[1], 32);
         if (pubkey[0] == 0x02 || pubkey[0] == 0x03) {
@@ -76,7 +84,7 @@ bool CScriptCompressor::Compress(std::vector<unsigned char> &out) const
     return false;
 }
 
-unsigned int CScriptCompressor::GetSpecialSize(unsigned int nSize) const
+unsigned int GetSpecialScriptSize(unsigned int nSize)
 {
     if (nSize == 0 || nSize == 1)
         return 20;
@@ -85,7 +93,7 @@ unsigned int CScriptCompressor::GetSpecialSize(unsigned int nSize) const
     return 0;
 }
 
-bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigned char> &in)
+bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &in)
 {
     switch(nSize) {
     case 0x00:
@@ -139,7 +147,7 @@ bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector<unsigne
 // * if e==9, we only know the resulting number is not zero, so output 1 + 10*(n - 1) + 9
 // (this is decodable, as d is in [1-9] and e is in [0-9])
 
-uint64_t CTxOutCompressor::CompressAmount(uint64_t n)
+uint64_t CompressAmount(uint64_t n)
 {
     if (n == 0)
         return 0;
@@ -158,7 +166,7 @@ uint64_t CTxOutCompressor::CompressAmount(uint64_t n)
     }
 }
 
-uint64_t CTxOutCompressor::DecompressAmount(uint64_t x)
+uint64_t DecompressAmount(uint64_t x)
 {
     // x = 0  OR  x = 1+10*(9*n + d - 1) + e  OR  x = 1+10*(n - 1) + 9
     if (x == 0)
diff --git a/src/compressor.h b/src/compressor.h
index 2342c6d62d7c..28a79d26201f 100644
--- a/src/compressor.h
+++ b/src/compressor.h
@@ -15,6 +15,13 @@ class CKeyID;
 class CPubKey;
 class CScriptID;
 
+bool CompressScript(const CScript& script, std::vector<unsigned char> &out);
+unsigned int GetSpecialScriptSize(unsigned int nSize);
+bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out);
+
+uint64_t CompressAmount(uint64_t nAmount);
+uint64_t DecompressAmount(uint64_t nAmount);
+
 /** Compact serializer for scripts.
  *
  *  It detects common cases and encodes them much more efficiently.
@@ -38,28 +45,13 @@ class CScriptCompressor
     static const unsigned int nSpecialScripts = 6;
 
     CScript &script;
-protected:
-    /**
-     * These check for scripts for which a special case with a shorter encoding is defined.
-     * They are implemented separately from the CScript test, as these test for exact byte
-     * sequence correspondences, and are more strict. For example, IsToPubKey also verifies
-     * whether the public key is valid (as invalid ones cannot be represented in compressed
-     * form).
-     */
-    bool IsToKeyID(CKeyID &hash) const;
-    bool IsToScriptID(CScriptID &hash) const;
-    bool IsToPubKey(CPubKey &pubkey) const;
-
-    bool Compress(std::vector<unsigned char> &out) const;
-    unsigned int GetSpecialSize(unsigned int nSize) const;
-    bool Decompress(unsigned int nSize, const std::vector<unsigned char> &out);
 public:
     explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { }
 
     template<typename Stream>
     void Serialize(Stream &s) const {
         std::vector<unsigned char> compr;
-        if (Compress(compr)) {
+        if (CompressScript(script, compr)) {
             s << MakeSpan(compr);
             return;
         }
@@ -73,9 +65,9 @@ class CScriptCompressor
         unsigned int nSize = 0;
         s >> VARINT(nSize);
         if (nSize < nSpecialScripts) {
-            std::vector<unsigned char> vch(GetSpecialSize(nSize), 0x00);
+            std::vector<unsigned char> vch(GetSpecialScriptSize(nSize), 0x00);
             s >> MakeSpan(vch);
-            Decompress(nSize, vch);
+            DecompressScript(script, nSize, vch);
             return;
         }
         nSize -= nSpecialScripts;
@@ -97,9 +89,6 @@ class CTxOutCompressor
     CTxOut &txout;
 
 public:
-    static uint64_t CompressAmount(uint64_t nAmount);
-    static uint64_t DecompressAmount(uint64_t nAmount);
-
     explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { }
 
     ADD_SERIALIZE_METHODS;
diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp
index 0dbd3f981dd5..8521ff0b0ab1 100644
--- a/src/test/compress_tests.cpp
+++ b/src/test/compress_tests.cpp
@@ -25,16 +25,16 @@
 BOOST_FIXTURE_TEST_SUITE(compress_tests, BasicTestingSetup)
 
 bool static TestEncode(uint64_t in) {
-    return in == CTxOutCompressor::DecompressAmount(CTxOutCompressor::CompressAmount(in));
+    return in == DecompressAmount(CompressAmount(in));
 }
 
 bool static TestDecode(uint64_t in) {
-    return in == CTxOutCompressor::CompressAmount(CTxOutCompressor::DecompressAmount(in));
+    return in == CompressAmount(DecompressAmount(in));
 }
 
 bool static TestPair(uint64_t dec, uint64_t enc) {
-    return CTxOutCompressor::CompressAmount(dec) == enc &&
-           CTxOutCompressor::DecompressAmount(enc) == dec;
+    return CompressAmount(dec) == enc &&
+           DecompressAmount(enc) == dec;
 }
 
 BOOST_AUTO_TEST_CASE(compress_amounts)

From bfe59713591919aeb6a0a8c7e8016ef27c9340ba Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com>
Date: Sat, 18 Jan 2020 07:32:31 -0800
Subject: [PATCH 5/6] Merge #17957: Convert compression.h to new serialization
 framework

https://github.com/bitcoin/bitcoin/pull/17957/commits/4de934b9b5b4be1bac8fe205f4ee9a79e772dc34

Partial #17957: Add FORMATTER_METHODS, similar to SERIALIZE_METHODS, but for formatters

https://github.com/bitcoin/bitcoin/pull/17957/commits/ca34c5cba5fbb9b046b074a234f06ecf6ed5d610
---
 src/coins.h                  |  4 +--
 src/compressor.h             | 53 +++++++++++++-----------------------
 src/serialize.h              | 32 ++++++++++++++++++----
 src/test/test_dash_fuzzy.cpp |  2 +-
 src/txdb.cpp                 |  2 +-
 src/undo.h                   |  4 +--
 6 files changed, 52 insertions(+), 45 deletions(-)

diff --git a/src/coins.h b/src/coins.h
index bd472dacf787..2034477c7721 100644
--- a/src/coins.h
+++ b/src/coins.h
@@ -60,7 +60,7 @@ class Coin
         assert(!IsSpent());
         uint32_t code = nHeight * 2 + fCoinBase;
         ::Serialize(s, VARINT(code));
-        ::Serialize(s, CTxOutCompressor(REF(out)));
+        ::Serialize(s, Using<TxOutCompression>(out));
     }
 
     template<typename Stream>
@@ -69,7 +69,7 @@ class Coin
         ::Unserialize(s, VARINT(code));
         nHeight = code >> 1;
         fCoinBase = code & 1;
-        ::Unserialize(s, CTxOutCompressor(out));
+        ::Unserialize(s, Using<TxOutCompression>(out));
     }
 
     bool IsSpent() const {
diff --git a/src/compressor.h b/src/compressor.h
index 28a79d26201f..71784d612f80 100644
--- a/src/compressor.h
+++ b/src/compressor.h
@@ -11,10 +11,6 @@
 #include <serialize.h>
 #include <span.h>
 
-class CKeyID;
-class CPubKey;
-class CScriptID;
-
 bool CompressScript(const CScript& script, std::vector<unsigned char> &out);
 unsigned int GetSpecialScriptSize(unsigned int nSize);
 bool DecompressScript(CScript& script, unsigned int nSize, const std::vector<unsigned char> &out);
@@ -33,9 +29,8 @@ uint64_t DecompressAmount(uint64_t nAmount);
  *  Other scripts up to 121 bytes require 1 byte + script length. Above
  *  that, scripts up to 16505 bytes require 2 bytes + script length.
  */
-class CScriptCompressor
+struct ScriptCompression
 {
-private:
     /**
      * make this static for now (there are only 6 special scripts defined)
      * this can potentially be extended together with a new nVersion for
@@ -44,12 +39,8 @@ class CScriptCompressor
      */
     static const unsigned int nSpecialScripts = 6;
 
-    CScript &script;
-public:
-    explicit CScriptCompressor(CScript &scriptIn) : script(scriptIn) { }
-
     template<typename Stream>
-    void Serialize(Stream &s) const {
+    void Ser(Stream &s, const CScript& script) {
         std::vector<unsigned char> compr;
         if (CompressScript(script, compr)) {
             s << MakeSpan(compr);
@@ -61,7 +52,7 @@ class CScriptCompressor
     }
 
     template<typename Stream>
-    void Unserialize(Stream &s) {
+    void Unser(Stream &s, CScript& script) {
         unsigned int nSize = 0;
         s >> VARINT(nSize);
         if (nSize < nSpecialScripts) {
@@ -82,30 +73,24 @@ class CScriptCompressor
     }
 };
 
-/** wrapper for CTxOut that provides a more compact serialization */
-class CTxOutCompressor
+struct AmountCompression
 {
-private:
-    CTxOut &txout;
-
-public:
-    explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { }
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        if (!ser_action.ForRead()) {
-            uint64_t nVal = CompressAmount(txout.nValue);
-            READWRITE(VARINT(nVal));
-        } else {
-            uint64_t nVal = 0;
-            READWRITE(VARINT(nVal));
-            txout.nValue = DecompressAmount(nVal);
-        }
-        CScriptCompressor cscript(REF(txout.scriptPubKey));
-        READWRITE(cscript);
+    template<typename Stream, typename I> void Ser(Stream& s, I val)
+    {
+        s << VARINT(CompressAmount(val));
+    }
+    template<typename Stream, typename I> void Unser(Stream& s, I& val)
+    {
+        uint64_t v;
+        s >> VARINT(v);
+        val = DecompressAmount(v);
     }
 };
 
+/** wrapper for CTxOut that provides a more compact serialization */
+struct TxOutCompression
+{
+    FORMATTER_METHODS(CTxOut, obj) { READWRITE(Using<AmountCompression>(obj.nValue), Using<ScriptCompression>(obj.scriptPubKey)); }
+};
+
 #endif // BITCOIN_COMPRESSOR_H
diff --git a/src/serialize.h b/src/serialize.h
index ef4f5ad9365f..655e90c5aaab 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -206,6 +206,30 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
         SerializationOp(s, CSerActionUnserialize());                  \
     }
 
+/**
+ * Implement the Ser and Unser methods needed for implementing a formatter (see Using below).
+ *
+ * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic
+ * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when
+ * deserializing).
+ *
+ * Example use:
+ *   struct FooFormatter {
+ *     FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); }
+ *   }
+ *   would define a class FooFormatter that defines a serialization of Class objects consisting
+ *   of serializing its val1 member using the default serialization, and its val2 member using
+ *   VARINT serialization. That FooFormatter can then be used in statements like
+ *   READWRITE(Using<FooFormatter>(obj.bla)).
+ */
+#define FORMATTER_METHODS(cls, obj) \
+    template<typename Stream> \
+    static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
+    template<typename Stream> \
+    static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
+    template<typename Stream, typename Type, typename Operation> \
+    static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
+
 /**
  * Implement the Serialize and Unserialize methods by delegating to a single templated
  * static method that takes the to-be-(de)serialized object as a parameter. This approach
@@ -218,17 +242,15 @@ template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
     void Serialize(Stream& s) const                                                 \
     {                                                                               \
         static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
-        SerializationOps(*this, s, CSerActionSerialize());                          \
+        Ser(s, *this);                                                              \
     }                                                                               \
     template<typename Stream>                                                       \
     void Unserialize(Stream& s)                                                     \
     {                                                                               \
         static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
-        SerializationOps(*this, s, CSerActionUnserialize());                        \
+        Unser(s, *this);                                                            \
     }                                                                               \
-    template<typename Stream, typename Type, typename Operation>                    \
-    static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
-
+    FORMATTER_METHODS(cls, obj)
 
 #ifndef CHAR_EQUALS_INT8
 template<typename Stream> inline void Serialize(Stream& s, char a    ) { ser_writedata8(s, a); } // TODO Get rid of bare char
diff --git a/src/test/test_dash_fuzzy.cpp b/src/test/test_dash_fuzzy.cpp
index e9d8de85ff5e..6647a9b937a6 100644
--- a/src/test/test_dash_fuzzy.cpp
+++ b/src/test/test_dash_fuzzy.cpp
@@ -241,7 +241,7 @@ int test_one_input(std::vector<uint8_t> buffer) {
         case CTXOUTCOMPRESSOR_DESERIALIZE:
         {
             CTxOut to;
-            CTxOutCompressor toc(to);
+            auto toc = Using<TxOutCompression>(to);
             try
             {
                 ds >> toc;
diff --git a/src/txdb.cpp b/src/txdb.cpp
index ca2b7f0a1d12..1beb79e834f3 100644
--- a/src/txdb.cpp
+++ b/src/txdb.cpp
@@ -502,7 +502,7 @@ class CCoins
         vout.assign(vAvail.size(), CTxOut());
         for (unsigned int i = 0; i < vAvail.size(); i++) {
             if (vAvail[i])
-                ::Unserialize(s, CTxOutCompressor(vout[i]));
+                ::Unserialize(s, Using<TxOutCompression>(vout[i]));
         }
         // coinbase height
         ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
diff --git a/src/undo.h b/src/undo.h
index 36f881509843..3fd3ebc1bc64 100644
--- a/src/undo.h
+++ b/src/undo.h
@@ -28,7 +28,7 @@ struct TxInUndoFormatter
             // Required to maintain compatibility with older undo format.
             ::Serialize(s, (unsigned char)0);
         }
-        ::Serialize(s, CTxOutCompressor(REF(txout.out)));
+        ::Serialize(s, Using<TxOutCompression>(REF(txout.out)));
     }
 
     template<typename Stream>
@@ -44,7 +44,7 @@ struct TxInUndoFormatter
             unsigned int nVersionDummy;
             ::Unserialize(s, VARINT(nVersionDummy));
         }
-        ::Unserialize(s, CTxOutCompressor(REF(txout.out)));
+        ::Unserialize(s, Using<TxOutCompression>(REF(txout.out)));
     }
 };
 

From 97caca3033c12d79afe782274a08ae0e440aab0a Mon Sep 17 00:00:00 2001
From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com>
Date: Mon, 3 Feb 2020 19:49:10 -0800
Subject: [PATCH 6/6] Merge #18112: Convert blockencodings.h to new
 serialization framework

https://github.com/bitcoin/bitcoin/pull/18112/commits/353f376277ad9b87e03c9ccbc1028c4b6d12e8ea

Partial #18112: Add CustomUintFormatter

https://github.com/bitcoin/bitcoin/pull/18112/commits/e574fff53eec4a27c83b765cb69e31d8399047ea

Partial #18112: Add DifferenceFormatter

https://github.com/bitcoin/bitcoin/pull/18112/commits/10633398f2dddf929d3f535aa48d138ad5e6c50f

Partial #18112: Make VectorFormatter support stateful formatters

https://github.com/bitcoin/bitcoin/pull/18112/commits/56dd9f04c701aa3ac340e95065bf83de20373c8b

Partial #18112: Convert CCompactSize to proper formatter

https://github.com/bitcoin/bitcoin/pull/18112/commits/3ca574cef0b4423f21b2c3efd8f5c9f71d52f219
---
 src/blockencodings.h | 140 +++++++++++--------------------------------
 src/prevector.h      |   9 ++-
 src/serialize.h      |  66 +++++++++++++-------
 3 files changed, 86 insertions(+), 129 deletions(-)

diff --git a/src/blockencodings.h b/src/blockencodings.h
index 61134c94b0e6..84d722adc1a2 100644
--- a/src/blockencodings.h
+++ b/src/blockencodings.h
@@ -11,18 +11,29 @@
 
 class CTxMemPool;
 
-// Dumb helper to handle CTransaction compression at serialize-time
-struct TransactionCompressor {
-private:
-    CTransactionRef& tx;
-public:
-    explicit TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {}
+// Transaction compression schemes for compact block relay can be introduced by writing
+// an actual formatter here.
+using TransactionCompression = DefaultFormatter;
 
-    ADD_SERIALIZE_METHODS;
+class DifferenceFormatter
+{
+    uint64_t m_shift = 0;
 
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(tx); //TODO: Compress tx encoding
+public:
+    template<typename Stream, typename I>
+    void Ser(Stream& s, I v)
+    {
+        if (v < m_shift || v >= std::numeric_limits<uint64_t>::max()) throw std::ios_base::failure("differential value overflow");
+        WriteCompactSize(s, v - m_shift);
+        m_shift = uint64_t(v) + 1;
+    }
+    template<typename Stream, typename I>
+    void Unser(Stream& s, I& v)
+    {
+        uint64_t n = ReadCompactSize(s);
+        m_shift += n;
+        if (m_shift < n || m_shift >= std::numeric_limits<uint64_t>::max() || m_shift < std::numeric_limits<I>::min() || m_shift > std::numeric_limits<I>::max()) throw std::ios_base::failure("differential value overflow");
+        v = I(m_shift++);
     }
 };
 
@@ -32,39 +43,9 @@ class BlockTransactionsRequest {
     uint256 blockhash;
     std::vector<uint16_t> indexes;
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(blockhash);
-        uint64_t indexes_size = (uint64_t)indexes.size();
-        READWRITE(COMPACTSIZE(indexes_size));
-        if (ser_action.ForRead()) {
-            size_t i = 0;
-            while (indexes.size() < indexes_size) {
-                indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size));
-                for (; i < indexes.size(); i++) {
-                    uint64_t index = 0;
-                    READWRITE(COMPACTSIZE(index));
-                    if (index > std::numeric_limits<uint16_t>::max())
-                        throw std::ios_base::failure("index overflowed 16 bits");
-                    indexes[i] = index;
-                }
-            }
-
-            int32_t offset = 0;
-            for (size_t j = 0; j < indexes.size(); j++) {
-                if (int32_t(indexes[j]) + offset > std::numeric_limits<uint16_t>::max())
-                    throw std::ios_base::failure("indexes overflowed 16 bits");
-                indexes[j] = indexes[j] + offset;
-                offset = int32_t(indexes[j]) + 1;
-            }
-        } else {
-            for (size_t i = 0; i < indexes.size(); i++) {
-                uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
-                READWRITE(COMPACTSIZE(index));
-            }
-        }
+    SERIALIZE_METHODS(BlockTransactionsRequest, obj)
+    {
+        READWRITE(obj.blockhash, Using<VectorFormatter<DifferenceFormatter>>(obj.indexes));
     }
 };
 
@@ -78,24 +59,9 @@ class BlockTransactions {
     explicit BlockTransactions(const BlockTransactionsRequest& req) :
         blockhash(req.blockhash), txn(req.indexes.size()) {}
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(blockhash);
-        uint64_t txn_size = (uint64_t)txn.size();
-        READWRITE(COMPACTSIZE(txn_size));
-        if (ser_action.ForRead()) {
-            size_t i = 0;
-            while (txn.size() < txn_size) {
-                txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size));
-                for (; i < txn.size(); i++)
-                    READWRITE(TransactionCompressor(txn[i]));
-            }
-        } else {
-            for (size_t i = 0; i < txn.size(); i++)
-                READWRITE(TransactionCompressor(txn[i]));
-        }
+    SERIALIZE_METHODS(BlockTransactions, obj)
+    {
+        READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn));
     }
 };
 
@@ -106,17 +72,7 @@ struct PrefilledTransaction {
     uint16_t index;
     CTransactionRef tx;
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        uint64_t idx = index;
-        READWRITE(COMPACTSIZE(idx));
-        if (idx > std::numeric_limits<uint16_t>::max())
-            throw std::ios_base::failure("index overflowed 16-bits");
-        index = idx;
-        READWRITE(TransactionCompressor(tx));
-    }
+    SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); }
 };
 
 typedef enum ReadStatus_t
@@ -154,43 +110,15 @@ class CBlockHeaderAndShortTxIDs {
 
     size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); }
 
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(header);
-        READWRITE(nonce);
-
-        uint64_t shorttxids_size = (uint64_t)shorttxids.size();
-        READWRITE(COMPACTSIZE(shorttxids_size));
+    SERIALIZE_METHODS(CBlockHeaderAndShortTxIDs, obj)
+    {
+        READWRITE(obj.header, obj.nonce, Using<VectorFormatter<CustomUintFormatter<SHORTTXIDS_LENGTH>>>(obj.shorttxids), obj.prefilledtxn);
         if (ser_action.ForRead()) {
-            size_t i = 0;
-            while (shorttxids.size() < shorttxids_size) {
-                shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size));
-                for (; i < shorttxids.size(); i++) {
-                    uint32_t lsb = 0; uint16_t msb = 0;
-                    READWRITE(lsb);
-                    READWRITE(msb);
-                    shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb);
-                    static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
-                }
-            }
-        } else {
-            for (size_t i = 0; i < shorttxids.size(); i++) {
-                uint32_t lsb = shorttxids[i] & 0xffffffff;
-                uint16_t msb = (shorttxids[i] >> 32) & 0xffff;
-                READWRITE(lsb);
-                READWRITE(msb);
+            if (obj.BlockTxCount() > std::numeric_limits<uint16_t>::max()) {
+                throw std::ios_base::failure("indexes overflowed 16 bits");
             }
+            obj.FillShortTxIDSelector();
         }
-
-        READWRITE(prefilledtxn);
-
-        if (BlockTxCount() > std::numeric_limits<uint16_t>::max())
-            throw std::ios_base::failure("indexes overflowed 16 bits");
-
-        if (ser_action.ForRead())
-            FillShortTxIDSelector();
     }
 };
 
diff --git a/src/prevector.h b/src/prevector.h
index fa2bf62272f0..fbbc86fc5716 100644
--- a/src/prevector.h
+++ b/src/prevector.h
@@ -452,15 +452,20 @@ class prevector {
         return first;
     }
 
-    void push_back(const T& value) {
+    template<typename... Args>
+    void emplace_back(Args&&... args) {
         size_type new_size = size() + 1;
         if (capacity() < new_size) {
             change_capacity(new_size + (new_size >> 1));
         }
-        new(item_ptr(size())) T(value);
+        new(item_ptr(size())) T(std::forward<Args>(args)...);
         _size++;
     }
 
+    void push_back(const T& value) {
+        emplace_back(value);
+    }
+
     void pop_back() {
         erase(end() - 1, end());
     }
diff --git a/src/serialize.h b/src/serialize.h
index 655e90c5aaab..d7ace2ff920f 100644
--- a/src/serialize.h
+++ b/src/serialize.h
@@ -504,7 +504,7 @@ static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&
 #define FIXEDVARINTSBITSET(obj, size) CFixedVarIntsBitSet(REF(obj), (size))
 #define AUTOBITSET(obj, size) CAutoBitSet(REF(obj), (size))
 #define VARINT(obj, ...) Using<VarIntFormatter<__VA_ARGS__>>(obj)
-#define COMPACTSIZE(obj) CCompactSize(REF(obj))
+#define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
 #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
 
 class CFixedBitSet
@@ -678,6 +678,28 @@ struct VarIntFormatter
     }
 };
 
+template<int Bytes>
+struct CustomUintFormatter
+{
+    static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
+    static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
+
+    template <typename Stream, typename I> void Ser(Stream& s, I v)
+    {
+        if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
+        uint64_t raw = htole64(v);
+        s.write((const char*)&raw, Bytes);
+    }
+
+    template <typename Stream, typename I> void Unser(Stream& s, I& v)
+    {
+        static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small");
+        uint64_t raw = 0;
+        s.read((char*)&raw, Bytes);
+        v = le64toh(raw);
+    }
+};
+
 /** Serialization wrapper class for big-endian integers.
  *
  * Use this wrapper around integer types that are stored in memory in native
@@ -712,25 +734,26 @@ class BigEndian
     }
 };
 
-class CCompactSize
+/** Formatter for integers in CompactSize format. */
+struct CompactSizeFormatter
 {
-protected:
-    uint64_t &n;
-public:
-    explicit CCompactSize(uint64_t& nIn) : n(nIn) { }
-
-    unsigned int GetSerializeSize() const {
-        return GetSizeOfCompactSize(n);
+    template<typename Stream, typename I>
+    void Unser(Stream& s, I& v)
+    {
+        uint64_t n = ReadCompactSize<Stream>(s);
+        if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
+            throw std::ios_base::failure("CompactSize exceeds limit of type");
+        }
+        v = n;
     }
 
-    template<typename Stream>
-    void Serialize(Stream &s) const {
-        WriteCompactSize<Stream>(s, n);
-    }
+    template<typename Stream, typename I>
+    void Ser(Stream& s, I v)
+    {
+        static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
+        static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
 
-    template<typename Stream>
-    void Unserialize(Stream& s) {
-        n = ReadCompactSize<Stream>(s);
+        WriteCompactSize<Stream>(s, v);
     }
 };
 
@@ -777,7 +800,7 @@ BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
  * as a vector of VarInt-encoded integers.
  *
  * V is not required to be an std::vector type. It works for any class that
- * exposes a value_type, size, reserve, push_back, and const iterators.
+ * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
  */
 template<class Formatter>
 struct VectorFormatter
@@ -785,15 +808,17 @@ struct VectorFormatter
     template<typename Stream, typename V>
     void Ser(Stream& s, const V& v)
     {
+        Formatter formatter;
         WriteCompactSize(s, v.size());
         for (const typename V::value_type& elem : v) {
-            s << Using<Formatter>(elem);
+            formatter.Ser(s, elem);
         }
     }
 
     template<typename Stream, typename V>
     void Unser(Stream& s, V& v)
     {
+        Formatter formatter;
         v.clear();
         size_t size = ReadCompactSize(s);
         size_t allocated = 0;
@@ -805,9 +830,8 @@ struct VectorFormatter
             allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
             v.reserve(allocated);
             while (v.size() < allocated) {
-                typename V::value_type val;
-                s >> Using<Formatter>(val);
-                v.push_back(std::move(val));
+                v.emplace_back();
+                formatter.Unser(s, v.back());
             }
         }
     };