From faf9dc145589b13ab91763a2dce1ee824c293801 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2020 13:38:13 +0100 Subject: [PATCH 1/7] bitcoin: Compute block hash while parsing This avoids having to re-serialize the block header just to compute the hash. It also frees us from having to carry around all the details in the header and we can hand around a minimal version. --- bitcoin/block.c | 2 +- bitcoin/block.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bitcoin/block.c b/bitcoin/block.c index dc478291d8df..ee929b2f2bcb 100644 --- a/bitcoin/block.c +++ b/bitcoin/block.c @@ -60,7 +60,7 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, b->hdr.nonce = pull_le32(&p, &len); sha256_le32(&shactx, b->hdr.nonce); } - sha256_double_done(&shactx, &b->hdr.hash); + sha256_double_done(&shactx, &b->hdr.hash.shad); num = pull_varint(&p, &len); b->tx = tal_arr(b, struct bitcoin_tx *, num); diff --git a/bitcoin/block.h b/bitcoin/block.h index cf0c913b506b..9ac3592beba6 100644 --- a/bitcoin/block.h +++ b/bitcoin/block.h @@ -23,7 +23,7 @@ struct bitcoin_block_hdr { le32 timestamp; le32 target; le32 nonce; - struct sha256_double hash; + struct bitcoin_blkid hash; }; struct elements_block_proof { From 71857ee11eb2ef6f14921116b6362c6a3b37125e Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2020 15:06:02 +0100 Subject: [PATCH 2/7] sha256: Add primitive to add a varint-encoded int to a hash We're about to push a couple of varints as part of the dynafed blockparsing, so we might as well make it easier for us. --- bitcoin/block.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bitcoin/block.c b/bitcoin/block.c index ee929b2f2bcb..7415f7bad647 100644 --- a/bitcoin/block.c +++ b/bitcoin/block.c @@ -4,6 +4,14 @@ #include #include +static void sha256_varint(struct sha256_ctx *ctx, u64 val) +{ + u8 vt[VARINT_MAX_LEN]; + size_t vtlen; + vtlen = varint_put(vt, val); + sha256_update(ctx, vt, vtlen); +} + /* Encoding is ... */ struct bitcoin_block * bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, @@ -81,8 +89,6 @@ void bitcoin_block_blkid(const struct bitcoin_block *b, struct bitcoin_blkid *out) { struct sha256_ctx shactx; - u8 vt[VARINT_MAX_LEN]; - size_t vtlen; sha256_init(&shactx); sha256_le32(&shactx, b->hdr.version); @@ -94,8 +100,7 @@ void bitcoin_block_blkid(const struct bitcoin_block *b, size_t clen = tal_bytelen(b->elements_hdr->proof.challenge); sha256_le32(&shactx, b->elements_hdr->block_height); - vtlen = varint_put(vt, clen); - sha256_update(&shactx, vt, vtlen); + sha256_varint(&shactx, clen); sha256_update(&shactx, b->elements_hdr->proof.challenge, clen); /* The solution is skipped, since that'd create a circular * dependency apparently */ From b69ee8d42bd63cafda5a1e5cb7fb1d2bd6d6627a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 23 Jan 2020 16:00:09 +0100 Subject: [PATCH 3/7] elements: Construct the block header hash while we parse it --- bitcoin/block.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bitcoin/block.c b/bitcoin/block.c index 7415f7bad647..292881adc71e 100644 --- a/bitcoin/block.c +++ b/bitcoin/block.c @@ -52,8 +52,11 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, if (is_elements(chainparams)) { b->elements_hdr = tal(b, struct elements_block_hdr); b->elements_hdr->block_height = pull_le32(&p, &len); + sha256_le32(&shactx, b->elements_hdr->block_height); size_t challenge_len = pull_varint(&p, &len); + sha256_varint(&shactx, challenge_len); + sha256_update(&shactx, p, challenge_len); b->elements_hdr->proof.challenge = tal_arr(b->elements_hdr, u8, challenge_len); pull(&p, &len, b->elements_hdr->proof.challenge, challenge_len); From 8b30481310a941f4f66be6630146979b5dd4daef Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 24 Jan 2020 21:02:07 +0100 Subject: [PATCH 4/7] bitcoin: Use the block hash we computed while deserializing it Since we now compute the hash while deserializing the block header we can now just use it, no reason to serialize the header just to hash it again. This also allows us to throw away all the added dynafed fields in the next commit instead of having to carry them around. --- bitcoin/block.c | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/bitcoin/block.c b/bitcoin/block.c index 292881adc71e..059d0a584ddc 100644 --- a/bitcoin/block.c +++ b/bitcoin/block.c @@ -91,27 +91,7 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, void bitcoin_block_blkid(const struct bitcoin_block *b, struct bitcoin_blkid *out) { - struct sha256_ctx shactx; - - sha256_init(&shactx); - sha256_le32(&shactx, b->hdr.version); - sha256_update(&shactx, &b->hdr.prev_hash, sizeof(b->hdr.prev_hash)); - sha256_update(&shactx, &b->hdr.merkle_hash, sizeof(b->hdr.merkle_hash)); - sha256_le32(&shactx, b->hdr.timestamp); - - if (is_elements(chainparams)) { - size_t clen = tal_bytelen(b->elements_hdr->proof.challenge); - sha256_le32(&shactx, b->elements_hdr->block_height); - - sha256_varint(&shactx, clen); - sha256_update(&shactx, b->elements_hdr->proof.challenge, clen); - /* The solution is skipped, since that'd create a circular - * dependency apparently */ - } else { - sha256_le32(&shactx, b->hdr.target); - sha256_le32(&shactx, b->hdr.nonce); - } - sha256_double_done(&shactx, &out->shad); + *out = b->hdr.hash; } /* We do the same hex-reversing crud as txids. */ From 2e092191eb9c7df8ae47ed34e8d55a81b192ab96 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 24 Jan 2020 22:19:04 +0100 Subject: [PATCH 5/7] elements: Implement dynafed block format support Changelog-Added: elements: Added support for the dynafed block header format and elementsd >=0.18.1 --- bitcoin/block.c | 103 +++++++++++++++++++++++++++++++++++++++++++----- bitcoin/block.h | 6 +++ 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/bitcoin/block.c b/bitcoin/block.c index 059d0a584ddc..de652dfc3bbf 100644 --- a/bitcoin/block.c +++ b/bitcoin/block.c @@ -12,6 +12,80 @@ static void sha256_varint(struct sha256_ctx *ctx, u64 val) sha256_update(ctx, vt, vtlen); } +static void bitcoin_block_pull_dynafed_params(const u8 **cursor, size_t *len, struct sha256_ctx *shactx) +{ + u8 type; + u64 l1, l2; + pull(cursor, len, &type, 1); + sha256_update(shactx, &type, 1); + switch ((enum dynafed_params_type)type) { + case DYNAFED_PARAMS_NULL: + break; + case DYNAFED_PARAMS_COMPACT: + /* "scriptPubKey" used for block signing */ + l1 = pull_varint(cursor, len); + sha256_varint(shactx, l1); + sha256_update(shactx, *cursor, l1); + pull(cursor, len, NULL, l1); + + /* signblock_witness_limit */ + sha256_update(shactx, *cursor, 4); + pull(cursor, len, NULL, 4); + + /* Skip elided_root */ + sha256_update(shactx, *cursor, 32); + pull(cursor, len, NULL, 32); + break; + + case DYNAFED_PARAMS_FULL: + /* "scriptPubKey" used for block signing */ + l1 = pull_varint(cursor, len); + sha256_varint(shactx, l1); + sha256_update(shactx, *cursor, l1); + pull(cursor, len, NULL, l1); + + /* signblock_witness_limit */ + sha256_update(shactx, *cursor, 4); + pull(cursor, len, NULL, 4); + + /* fedpeg_program */ + l1 = pull_varint(cursor, len); + sha256_varint(shactx, l1); + sha256_update(shactx, *cursor, l1); + pull(cursor, len, NULL, l1); + + /* fedpegscript */ + l1 = pull_varint(cursor, len); + sha256_varint(shactx, l1); + sha256_update(shactx, *cursor, l1); + pull(cursor, len, NULL, l1); + + /* extension space */ + l2 = pull_varint(cursor, len); + sha256_varint(shactx, l2); + for (size_t i = 0; i < l2; i++) { + l1 = pull_varint(cursor, len); + sha256_varint(shactx, l1); + sha256_update(shactx, *cursor, l1); + pull(cursor, len, NULL, l1); + } + break; + } +} + +static void bitcoin_block_pull_dynafed_details(const u8 **cursor, size_t *len, struct sha256_ctx *shactx) +{ + bitcoin_block_pull_dynafed_params(cursor, len, shactx); + bitcoin_block_pull_dynafed_params(cursor, len, shactx); + + /* Consume the signblock_witness */ + u64 numwitnesses = pull_varint(cursor, len); + for (size_t i=0; i ... */ struct bitcoin_block * bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, @@ -22,6 +96,7 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, const u8 *p; size_t len, i, num; struct sha256_ctx shactx; + bool is_dynafed; if (hexlen && hex[hexlen-1] == '\n') hexlen--; @@ -50,19 +125,29 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, sha256_le32(&shactx, b->hdr.timestamp); if (is_elements(chainparams)) { + /* A dynafed block is signalled by setting the MSB of the version. */ + is_dynafed = (b->hdr.version >> 31 == 1); b->elements_hdr = tal(b, struct elements_block_hdr); b->elements_hdr->block_height = pull_le32(&p, &len); sha256_le32(&shactx, b->elements_hdr->block_height); - size_t challenge_len = pull_varint(&p, &len); - sha256_varint(&shactx, challenge_len); - sha256_update(&shactx, p, challenge_len); - b->elements_hdr->proof.challenge = tal_arr(b->elements_hdr, u8, challenge_len); - pull(&p, &len, b->elements_hdr->proof.challenge, challenge_len); - - size_t solution_len = pull_varint(&p, &len); - b->elements_hdr->proof.solution = tal_arr(b->elements_hdr, u8, solution_len); - pull(&p, &len, b->elements_hdr->proof.solution, solution_len); + if (is_dynafed) { + bitcoin_block_pull_dynafed_details(&p, &len, &shactx); + } else { + size_t challenge_len = pull_varint(&p, &len); + sha256_varint(&shactx, challenge_len); + sha256_update(&shactx, p, challenge_len); + b->elements_hdr->proof.challenge = + tal_arr(b->elements_hdr, u8, challenge_len); + pull(&p, &len, b->elements_hdr->proof.challenge, + challenge_len); + + size_t solution_len = pull_varint(&p, &len); + b->elements_hdr->proof.solution = + tal_arr(b->elements_hdr, u8, solution_len); + pull(&p, &len, b->elements_hdr->proof.solution, + solution_len); + } } else { b->hdr.target = pull_le32(&p, &len); diff --git a/bitcoin/block.h b/bitcoin/block.h index 9ac3592beba6..8654ed5a1235 100644 --- a/bitcoin/block.h +++ b/bitcoin/block.h @@ -10,6 +10,12 @@ struct chainparams; +enum dynafed_params_type { + DYNAFED_PARAMS_NULL, + DYNAFED_PARAMS_COMPACT, + DYNAFED_PARAMS_FULL, +}; + struct bitcoin_blkid { struct sha256_double shad; }; From a94881a5d888912f863645cb54e69c136b3b6acc Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sat, 25 Jan 2020 13:14:34 +0100 Subject: [PATCH 6/7] cleanup: Remove block header fields we don't use --- bitcoin/block.c | 33 ++++++++++++++++----------------- bitcoin/block.h | 11 ----------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/bitcoin/block.c b/bitcoin/block.c index de652dfc3bbf..424f4f52becb 100644 --- a/bitcoin/block.c +++ b/bitcoin/block.c @@ -94,9 +94,10 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, struct bitcoin_block *b; u8 *linear_tx; const u8 *p; - size_t len, i, num; + size_t len, i, num, templen; struct sha256_ctx shactx; bool is_dynafed; + u32 height; if (hexlen && hex[hexlen-1] == '\n') hexlen--; @@ -127,26 +128,24 @@ bitcoin_block_from_hex(const tal_t *ctx, const struct chainparams *chainparams, if (is_elements(chainparams)) { /* A dynafed block is signalled by setting the MSB of the version. */ is_dynafed = (b->hdr.version >> 31 == 1); - b->elements_hdr = tal(b, struct elements_block_hdr); - b->elements_hdr->block_height = pull_le32(&p, &len); - sha256_le32(&shactx, b->elements_hdr->block_height); + + /* elements_header.height */ + height = pull_le32(&p, &len); + sha256_le32(&shactx, height); if (is_dynafed) { bitcoin_block_pull_dynafed_details(&p, &len, &shactx); } else { - size_t challenge_len = pull_varint(&p, &len); - sha256_varint(&shactx, challenge_len); - sha256_update(&shactx, p, challenge_len); - b->elements_hdr->proof.challenge = - tal_arr(b->elements_hdr, u8, challenge_len); - pull(&p, &len, b->elements_hdr->proof.challenge, - challenge_len); - - size_t solution_len = pull_varint(&p, &len); - b->elements_hdr->proof.solution = - tal_arr(b->elements_hdr, u8, solution_len); - pull(&p, &len, b->elements_hdr->proof.solution, - solution_len); + /* elemens_header.challenge */ + templen = pull_varint(&p, &len); + sha256_varint(&shactx, templen); + sha256_update(&shactx, p, templen); + pull(&p, &len, NULL, templen); + + /* elements_header.solution. Not hashed since it'd be + * a circular dependency. */ + templen = pull_varint(&p, &len); + pull(&p, &len, NULL, templen); } } else { diff --git a/bitcoin/block.h b/bitcoin/block.h index 8654ed5a1235..7f82b8aa55ce 100644 --- a/bitcoin/block.h +++ b/bitcoin/block.h @@ -32,19 +32,8 @@ struct bitcoin_block_hdr { struct bitcoin_blkid hash; }; -struct elements_block_proof { - u8 *challenge; - u8 *solution; -}; - -struct elements_block_hdr { - u32 block_height; - struct elements_block_proof proof; -}; - struct bitcoin_block { struct bitcoin_block_hdr hdr; - struct elements_block_hdr *elements_hdr; /* tal_count shows now many */ struct bitcoin_tx **tx; }; From fa8248aa9e1686d71eed6047d5273bb1b6bf80f5 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Sat, 25 Jan 2020 13:30:59 +0100 Subject: [PATCH 7/7] elementsd: Do not use `generate` for elementsd >= 0.17.0 The `generate` has been deprecated since 0.16 and has been removed in 0.18.0 so we better use `generatetoaddress` instead, which is already what we do with `bitcoind`. So we remove the override here. --- contrib/pyln-testing/pyln/testing/utils.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/contrib/pyln-testing/pyln/testing/utils.py b/contrib/pyln-testing/pyln/testing/utils.py index 3071d57441c7..759e48556061 100644 --- a/contrib/pyln-testing/pyln/testing/utils.py +++ b/contrib/pyln-testing/pyln/testing/utils.py @@ -443,17 +443,6 @@ def __init__(self, bitcoin_dir="/tmp/bitcoind-test", rpcport=None): self.rpc = SimpleBitcoinProxy(btc_conf_file=self.conf_file) self.prefix = 'elementsd' - def generate_block(self, numblocks=1, wait_for_mempool=0): - if wait_for_mempool: - if isinstance(wait_for_mempool, str): - wait_for_mempool = [wait_for_mempool] - if isinstance(wait_for_mempool, list): - wait_for(lambda: all(txid in self.rpc.getrawmempool() for txid in wait_for_mempool)) - else: - wait_for(lambda: len(self.rpc.getrawmempool()) >= wait_for_mempool) - # As of 0.16, generate() is removed; use generatetoaddress. - return self.rpc.generate(numblocks) - def getnewaddress(self): """Need to get an address and then make it unconfidential """