Fix Shutdown for Archive (#301)

* use actorpool, custom smol executor

* cleanup

* migrations

* simplify type arguments for all structs

* finish cleaning up type arguments

* use coil master

* migrations for coil changes

* dont continue on empty

* some tweaks/cleanup

* troubleshooting shutdown

* adding test for missing_storage query

* paginate storage restoration

* update substrate/polkadot

* create function for converting hex to integer in sql

* add runtime features for polkadot-archive

* use coil from crates

* add offline sql

* make tests pass

* cancel storage restoration on Ctrl-C

* cleanup

* cleanup

* query format

* sqlx-data for offline compilation

* troubleshoot shutdown

* lints & paginate query

* offline sqlx

* fix shutdown

* make pagination a stream

* cleanup

* update locked dependencies

* remove tasks vector in Actors struct

* bump xtra to 0.5

* fix comment on tick_interval

* spacing

* update lockfiles

* fix lint

* use coil on crates

* fix changes during merge

* fix spawn & kill listener in Drop

* add back deny dead_code

* remove debug log

* fix tests

* clippy lints

* moar clippy lints

Author: insipx

CHANGELOG.md

@@ -6,14 +6,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+### Changed
+- Use `async-std` instead of `smol` for async tasks. ([#301](https://github.com/paritytech/substrate-archive/pull/301))
+
+### Removed
+- **BREAKING** `max_tasks` from builder. Max tasks will always be the number of CPUS on the system. ([#301](https://github.com/paritytech/substrate-archive/pull/301))
+- **BREAKING** `task_workers` from builder. Task workers will always be number of CPUS on the system.([#301](https://github.com/paritytech/substrate-archive/pull/301))
+
+### Fixed
+- archive no longer hangs on Ctrl-C. ([#301](https://github.com/paritytech/substrate-archive/pull/301))
+
 ## [v0.5.2] - 2021-06-02
 ### Added
 - Test for tracing enabled wasm-blobs `v0.9.0`, `v0.9.1`, `v0.9.2`, `v0.9.3` ([#284](https://github.com/paritytech/substrate-archive/pull/284)) ([cd6a446](https://github.com/paritytech/substrate-archive/commit/cd6a446bc66002d1945cbdf0c1b39957218f90fd))
 - Unit testing CI workflow ([#288](https://github.com/paritytech/substrate-archive/pull/288)) ([482af68](https://github.com/paritytech/substrate-archive/commit/482af68fff515a7e3a34ee0c512d735790193cd6))
 
 ### Changed
 - Clarify release checklist ([#279](https://github.com/paritytech/substrate-archive/pull/279)) ([9abef6e](https://github.com/paritytech/substrate-archive/commit/9abef6e2bdda4c1492b6e232ec38c8c0d59a3749)) && ([#288](https://github.com/paritytech/substrate-archive/pull/288)) ([482af68](https://github.com/paritytech/substrate-archive/commit/482af68fff515a7e3a34ee0c512d735790193cd6))
-- Update dependencies to match runtime `0.9.3`. 
+- Update dependencies to match runtime `0.9.3`.
 - Refactor tracing to work with the latest tracing changes in substrate ([#273](https://github.com/paritytech/substrate-archive/pull/273)) ([b322ded](https://github.com/paritytech/substrate-archive/commit/b322ded5cf683270da6d21478e80c9f4dba706dc))
 
 ### Fixed

Cargo.lock

@@ -57,10 +57,6 @@ task_workers = 8
 # Optional, default: 20 seconds
 task_timeout = 20
 
-# Maximum number of tasks to queue in the threadpool.
-# Optional, default: 64
-max_tasks = 64
-
 # Maximium number of blocks to load and insert into database at a time.
 # Useful for controlling memory usage.
 # Optional, defaults: 100,000

bin/node-template-archive/archive.toml

@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Removed
+- `max_tasks` from toml configuration file. ([#301](https://github.com/paritytech/substrate-archive/pull/301))
+- `task_workers` from toml configuration file. ([#301](https://github.com/paritytech/substrate-archive/pull/301))
 
 ## [0.2.4]
 ### Changed

bin/polkadot-archive/CHANGELOG.md

@@ -53,18 +53,10 @@ std = "DEBUG"
 #
 # Changing these may lead to unexpected results.
 [control]
-# Number of threads to dedicate for executing tasks
-# Optional, default: the number of logical system threads
-task_workers = 8
-
 # Timeout to wait for a task to start execution.
 # Optional, default: 20 seconds
 task_timeout = 20
 
-# Maximum number of tasks to queue in the threadpool.
-# Optional, default: 64
-max_tasks = 64
-
 # Maximium number of blocks to load and insert into database at a time.
 # Useful for controlling memory usage.
 # Optional, defaults: 100,000

bin/polkadot-archive/Cargo.lock

@@ -148,8 +148,8 @@ where
 }
 
 pub fn runtime_api<Block, Runtime, Dispatch, D: ReadOnlyDb + 'static>(
-	db: Arc<D>,
 	config: RuntimeConfig,
+	backend: Arc<ReadOnlyBackend<Block, D>>,
 ) -> Result<TArchiveClient<Block, Runtime, Dispatch, D>, BackendError>
 where
 	Block: BlockT,
@@ -162,8 +162,6 @@ where
 	Dispatch: NativeExecutionDispatch + 'static,
 	<Runtime::RuntimeApi as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
 {
-	let backend = Arc::new(ReadOnlyBackend::new(db, true, config.storage_mode));
-
 	let executor = NativeExecutor::<Dispatch>::new(config.exec_method.into(), config.wasm_pages, config.block_workers);
 	let executor =
 		LocalCallExecutor::new(backend.clone(), executor, Box::new(TaskExecutor::new()), config.try_into()?)?;