UTXO high watermark in db

bitcoin/chainparams.c

@@ -12,6 +12,8 @@ const struct chainparams networks[] = {
      .cli = "bitcoin-cli",
      .cli_args = NULL,
      .dust_limit = 546,
+     /* "Lightning Charge Powers Developers & Blockstream Store" */
+     .when_lightning_became_cool = 504500,
      .testnet = false},
     {.index = 1,
      .network_name = "regtest",
@@ -21,6 +23,7 @@ const struct chainparams networks[] = {
      .cli = "bitcoin-cli",
      .cli_args = "-regtest",
      .dust_limit = 546,
+     .when_lightning_became_cool = 1,
      .testnet = true},
     {.index = 2,
      .network_name = "testnet",
@@ -39,6 +42,7 @@ const struct chainparams networks[] = {
      .cli = "litecoin-cli",
      .cli_args = NULL,
      .dust_limit = 100000,
+     .when_lightning_became_cool = 1,
      .testnet = false},
     {.index = 4,
      .network_name = "litecoin-testnet",
@@ -48,6 +52,7 @@ const struct chainparams networks[] = {
      .cli = "litecoin-cli",
      .cli_args = "-testnet",
      .dust_limit = 100000,
+     .when_lightning_became_cool = 1,
      .testnet = true}
 };
 

bitcoin/chainparams.h

@@ -15,6 +15,7 @@ struct chainparams {
 	const char *cli;
 	const char *cli_args;
 	const u64 dust_limit;
+	const u32 when_lightning_became_cool;
 
 	/* Whether this is a test network or not */
 	const bool testnet;

lightningd/chaintopology.c

@@ -348,6 +348,10 @@ static void updates_complete(struct chain_topology *topo)
 		/* Maybe need to rebroadcast. */
 		rebroadcast_txs(topo, NULL);
 
+		/* We've processed these UTXOs */
+		db_set_intvar(topo->bitcoind->ld->wallet->db,
+			      "last_processed_block", topo->tip->height);
+
 		topo->prev_tip = topo->tip;
 	}
 
@@ -466,12 +470,13 @@ static void get_init_block(struct bitcoind *bitcoind,
 static void get_init_blockhash(struct bitcoind *bitcoind, u32 blockcount,
 			       struct chain_topology *topo)
 {
-	/* This happens if first_blocknum is UINTMAX-1 */
+	/* This can happen if bitcoind still syncing, or first_blocknum is MAX */
 	if (blockcount < topo->first_blocknum)
 		topo->first_blocknum = blockcount;
 
-	/* FIXME: Because we don't handle our root disappearing, we go
-	 * 100 blocks back */
+	/* For fork protection (esp. because we don't handle our own first
+	 * block getting reorged out), we always go 100 blocks further back
+	 * than we need. */
 	if (topo->first_blocknum < 100)
 		topo->first_blocknum = 0;
 	else
@@ -710,14 +715,13 @@ struct chain_topology *new_topology(struct lightningd *ld, struct log *log)
 
 void setup_topology(struct chain_topology *topo,
 		    struct timers *timers,
-		    struct timerel poll_time, u32 first_channel_block)
+		    struct timerel poll_time, u32 first_blocknum)
 {
 	memset(&topo->feerate, 0, sizeof(topo->feerate));
 	topo->timers = timers;
 	topo->poll_time = poll_time;
-	/* Start one before the block we are interested in (as we won't
-	 * get notifications on txs in that block). */
-	topo->first_blocknum = first_channel_block - 1;
+
+	topo->first_blocknum = first_blocknum;
 
 	/* Make sure bitcoind is started, and ready */
 	wait_for_bitcoind(topo->bitcoind);

lightningd/lightningd.c

@@ -239,7 +239,7 @@ int main(int argc, char *argv[])
 	struct log_book *log_book;
 	struct lightningd *ld;
 	bool newdir;
-	u32 peer_first_blocknum;
+	u32 first_blocknum;
 
 	err_set_progname(argv[0]);
 
@@ -322,15 +322,18 @@ int main(int argc, char *argv[])
 	if (!wallet_htlcs_reconnect(ld->wallet, &ld->htlcs_in, &ld->htlcs_out))
 		fatal("could not reconnect htlcs loaded from wallet, wallet may be inconsistent.");
 
-	peer_first_blocknum = wallet_channels_first_blocknum(ld->wallet);
+	/* Worst case, scan back to the first lightning deployment */
+	first_blocknum = wallet_first_blocknum(ld->wallet,
+					       get_chainparams(ld)
+					       ->when_lightning_became_cool);
 
 	db_commit_transaction(ld->wallet->db);
 
 	/* Initialize block topology (does its own transaction) */
 	setup_topology(ld->topology,
 		       &ld->timers,
 		       ld->config.poll_time,
-		       peer_first_blocknum);
+		       first_blocknum);
 
 	/* Create RPC socket (if any) */
 	setup_jsonrpc(ld, ld->rpc_filename);

lightningd/test/run-find_my_path.c

@@ -87,12 +87,12 @@ struct txfilter *txfilter_new(const tal_t *ctx UNNEEDED)
 /* Generated stub for version */
 const char *version(void)
 { fprintf(stderr, "version called!\n"); abort(); }
-/* Generated stub for wallet_channels_first_blocknum */
-u32 wallet_channels_first_blocknum(struct wallet *w UNNEEDED)
-{ fprintf(stderr, "wallet_channels_first_blocknum called!\n"); abort(); }
 /* Generated stub for wallet_channels_load_active */
 bool wallet_channels_load_active(const tal_t *ctx UNNEEDED, struct wallet *w UNNEEDED)
 { fprintf(stderr, "wallet_channels_load_active called!\n"); abort(); }
+/* Generated stub for wallet_first_blocknum */
+u32 wallet_first_blocknum(struct wallet *w UNNEEDED, u32 first_possible UNNEEDED)
+{ fprintf(stderr, "wallet_first_blocknum called!\n"); abort(); }
 /* Generated stub for wallet_htlcs_load_for_channel */
 bool wallet_htlcs_load_for_channel(struct wallet *wallet UNNEEDED,
 				   struct channel *chan UNNEEDED,

tests/test_lightningd.py

@@ -2823,6 +2823,25 @@ def test_lockin_between_restart(self):
         l1.daemon.wait_for_log(' to CHANNELD_NORMAL')
         l2.daemon.wait_for_log(' to CHANNELD_NORMAL')
 
+    def test_funding_while_offline(self):
+        l1 = self.node_factory.get_node()
+        addr = l1.rpc.newaddr()['address']
+
+        # l1 goes down.
+        l1.stop()
+
+        # We send funds
+        bitcoind.rpc.sendtoaddress(addr, (10**6 + 1000000) / 10**8)
+
+        # Now 120 blocks go by...
+        bitcoind.generate_block(120)
+
+        # Restart
+        l1.daemon.start()
+        sync_blockheight([l1])
+
+        assert len(l1.rpc.listfunds()['outputs']) == 1
+
     def test_addfunds_from_block(self):
         """Send funds to the daemon without telling it explicitly
         """

wallet/wallet.c

@@ -667,22 +667,63 @@ bool wallet_channels_load_active(const tal_t *ctx, struct wallet *w)
 	return ok;
 }
 
-u32 wallet_channels_first_blocknum(struct wallet *w)
+/* Upgrade of db (or initial create): do we have anything to scan for? */
+static bool wallet_ever_used(struct wallet *w)
+{
+	sqlite3_stmt *stmt;
+	bool channel_utxos;
+
+	/* If we ever handed out an address. */
+	if (db_get_intvar(w->db, "bip32_max_index", 0) != 0)
+		return true;
+
+	/* Or if they do a unilateral close, the output to us provides a UTXO. */
+	stmt = db_query(__func__, w->db,
+			"SELECT COUNT(*) FROM outputs WHERE commitment_point IS NOT NULL;");
+	sqlite3_step(stmt);
+	channel_utxos = (sqlite3_column_int(stmt, 0) != 0);
+	sqlite3_finalize(stmt);
+
+	return channel_utxos;
+}
+
+/* We want the earlier of either:
+ * 1. The first channel we're still watching (it might have closed),
+ * 2. The last block we scanned for UTXO (might have new incoming payments)
+ *
+ * chaintopology actually subtracts another 100 blocks to make sure we
+ * catch chain forks.
+ */
+u32 wallet_first_blocknum(struct wallet *w, u32 first_possible)
 {
 	int err;
-	u32 first_blocknum;
+	u32 first_channel, first_utxo;
 	sqlite3_stmt *stmt =
 	    db_query(__func__, w->db,
 		     "SELECT MIN(first_blocknum) FROM channels;");
 
+	/* If we ever opened a channel, this will give us the first block. */
 	err = sqlite3_step(stmt);
 	if (err == SQLITE_ROW && sqlite3_column_type(stmt, 0) != SQLITE_NULL)
-		first_blocknum = sqlite3_column_int(stmt, 0);
+		first_channel = sqlite3_column_int(stmt, 0);
 	else
-		first_blocknum = UINT32_MAX;
-
+		first_channel = UINT32_MAX;
 	sqlite3_finalize(stmt);
-	return first_blocknum;
+
+	first_utxo = db_get_intvar(w->db, "last_processed_block", 0);
+	if (first_utxo == 0) {
+		/* Don't know?  New db, or upgraded. */
+		if (wallet_ever_used(w))
+			/* Be conservative */
+			first_utxo = first_possible;
+		else
+			first_utxo = UINT32_MAX;
+	}
+
+	if (first_utxo < first_channel)
+		return first_utxo;
+	else
+		return first_channel;
 }
 
 void wallet_channel_config_save(struct wallet *w, struct channel_config *cc)

wallet/wallet.h

@@ -260,13 +260,12 @@ bool wallet_peer_by_nodeid(struct wallet *w, const struct pubkey *nodeid,
 bool wallet_channels_load_active(const tal_t *ctx, struct wallet *w);
 
 /**
- * wallet_channels_first_blocknum - get first block we're interested in.
+ * wallet_first_blocknum - get first block we're interested in.
  *
  * @w: wallet to load from.
- *
- * Returns UINT32_MAX if nothing interesting.
+ * @first_possible: when c-lightning may have been active from
  */
-u32 wallet_channels_first_blocknum(struct wallet *w);
+u32 wallet_first_blocknum(struct wallet *w, u32 first_possible);
 
 /**
  * wallet_extract_owned_outputs - given a tx, extract all of our outputs