From 7fc39eb07d3d0bbeb8123ec480a21faab21bb2f2 Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:31:24 +0000 Subject: [PATCH 1/9] chore: update dependencies --- crates/base/Cargo.toml | 7 ------- crates/event_worker/Cargo.toml | 3 ++- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/base/Cargo.toml b/crates/base/Cargo.toml index df6799596..185c258c8 100644 --- a/crates/base/Cargo.toml +++ b/crates/base/Cargo.toml @@ -131,13 +131,6 @@ sb_os = { version = "0.1.0", path = "../sb_os" } sb_node = { version = "0.1.0", path = "../node" } sb_ai = { version = "0.1.0", path = "../sb_ai" } -anyhow.workspace = true -bytes.workspace = true -hyper = { workspace = true, features = ["full"] } -http_v02.workspace = true -log.workspace = true -serde = { workspace = true, features = ["derive"] } -tokio.workspace = true url.workspace = true [features] diff --git a/crates/event_worker/Cargo.toml b/crates/event_worker/Cargo.toml index 1f74419e8..b76996493 100644 --- a/crates/event_worker/Cargo.toml +++ b/crates/event_worker/Cargo.toml @@ -20,4 +20,5 @@ serde.workspace = true anyhow.workspace = true tokio.workspace = true log.workspace = true -tracing.workspace = true \ No newline at end of file +tracing.workspace = true +enum-as-inner.workspace = true \ No newline at end of file From 49c940d95e5c3127873ca035f2f76ed18c466bb6 Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:31:30 +0000 Subject: [PATCH 2/9] chore: update `Cargo.lock` --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 605368528..3a8a2e966 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2682,6 +2682,7 @@ dependencies = [ "anyhow", "base_mem_check", "deno_core", + "enum-as-inner", "log", "serde", "tokio", From 7a2c310a31241d4c479aa05aabc938cc1fa5fb45 Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:32:55 +0000 Subject: [PATCH 3/9] chore: enable deno workspace feature --- crates/base/test_cases/deno.json | 8 ++++++++ deno.json | 9 ++++++--- examples/deno.json | 7 +++++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 crates/base/test_cases/deno.json create mode 100644 examples/deno.json diff --git a/crates/base/test_cases/deno.json b/crates/base/test_cases/deno.json new file mode 100644 index 000000000..dae3c075b --- /dev/null +++ b/crates/base/test_cases/deno.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "types": [ + "../../../types/global.d.ts" + ] + } +} + \ No newline at end of file diff --git a/deno.json b/deno.json index 04dc868d9..6cfc199d0 100644 --- a/deno.json +++ b/deno.json @@ -1,13 +1,16 @@ { + "workspace": [ + "./examples", + "./crates/base/test_cases" + ], "fmt": { "useTabs": true, "lineWidth": 100, "indentWidth": 4, "singleQuote": true, - "proseWrap": "preserve", - "include": ["examples/**/*.ts"] + "proseWrap": "preserve" }, "imports": { "npm:@meowmeow/foobar": "npm:is-odd" } -} +} \ No newline at end of file diff --git a/examples/deno.json b/examples/deno.json new file mode 100644 index 000000000..740ea20db --- /dev/null +++ b/examples/deno.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": { + "types": [ + "../types/global.d.ts" + ] + } +} From 1281624bb9fcb091b6ba8f871e1f85f9a505fc3d Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:33:08 +0000 Subject: [PATCH 4/9] chore: update `types/global.d.ts` --- types/global.d.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/types/global.d.ts b/types/global.d.ts index b29b9c0e7..ac02f3656 100644 --- a/types/global.d.ts +++ b/types/global.d.ts @@ -1,3 +1,13 @@ +declare type BeforeunloadReason = "cpu" | "memory" | "wall_clock" | "early_drop" | "termination"; +declare interface BeforeunloadEvent extends CustomEvent { } + +declare interface WindowEventMap { + "load": Event; + "unload": Event; + "beforeunload": BeforeunloadEvent; + "drain": Event; +} + type DecoratorType = "tc39" | "typescript" | "typescript_with_metadata"; interface JsxImportBaseConfig { @@ -79,7 +89,7 @@ interface MemInfo { declare namespace EdgeRuntime { export namespace ai { - function tryCleanupUnusedSession(): Promise; + function tryCleanupUnusedSession(): Promise; } class UserWorker { @@ -89,6 +99,7 @@ declare namespace EdgeRuntime { static create(opts: UserWorkerCreateOptions): Promise; } + export function scheduleTermination(): void; export function waitUntil(promise: Promise): Promise; export function getRuntimeMetrics(): Promise; export function applySupabaseTag(src: Request, dest: Request): void; From 804703b5466fb654e68b404b3fb7f1a3d0607a53 Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:33:45 +0000 Subject: [PATCH 5/9] chore: update sample main script --- examples/main/index.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/examples/main/index.ts b/examples/main/index.ts index e91840924..92ea9424c 100644 --- a/examples/main/index.ts +++ b/examples/main/index.ts @@ -5,6 +5,10 @@ import { handleRegistryRequest } from './registry/mod.ts'; console.log('main function started'); +addEventListener('beforeunload', () => { + console.log('main worker exiting'); +}); + // log system memory usage every 30s // setInterval(() => console.log(EdgeRuntime.systemMemoryInfo()), 30 * 1000); @@ -99,7 +103,7 @@ Deno.serve(async (req: Request) => { const createWorker = async () => { const memoryLimitMb = 150; - const workerTimeoutMs = 5 * 60 * 1000; + const workerTimeoutMs = 100 * 1000; const noModuleCache = false; // you can provide an import map inline @@ -125,8 +129,8 @@ Deno.serve(async (req: Request) => { // or load module source from an inline module // const maybeModuleCode = 'Deno.serve((req) => new Response("Hello from Module Code"));'; // - const cpuTimeSoftLimitMs = 10000; - const cpuTimeHardLimitMs = 20000; + const cpuTimeSoftLimitMs = 1000; + const cpuTimeHardLimitMs = 2000; return await EdgeRuntime.userWorkers.create({ servicePath, @@ -191,4 +195,4 @@ Deno.serve(async (req: Request) => { }; return callWorker(); -}); +}); \ No newline at end of file From 01d6a5c70607a452037879dec8eb38b8d976785f Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:35:38 +0000 Subject: [PATCH 6/9] chore: update integration test samples --- .../ai-ort-rust-backend/main/index.ts | 6 +- crates/base/test_cases/jsx-2/index.ts | 99 ++++++++-------- crates/base/test_cases/jsx/index.ts | 109 ++++++++++-------- .../test_cases/oak-v12-file-upload/index.ts | 7 +- 4 files changed, 124 insertions(+), 97 deletions(-) diff --git a/crates/base/test_cases/ai-ort-rust-backend/main/index.ts b/crates/base/test_cases/ai-ort-rust-backend/main/index.ts index aac3fa06b..4ef97d860 100644 --- a/crates/base/test_cases/ai-ort-rust-backend/main/index.ts +++ b/crates/base/test_cases/ai-ort-rust-backend/main/index.ts @@ -1,6 +1,6 @@ import * as path from "jsr:@std/path"; -setInterval(async () => { +const handle = setInterval(async () => { try { const cleanupCount = await EdgeRuntime.ai.tryCleanupUnusedSession(); if (cleanupCount == 0) { @@ -12,6 +12,10 @@ setInterval(async () => { } }, 100); +addEventListener("beforeunload", () => { + clearInterval(handle); +}); + Deno.serve(async (req: Request) => { console.log(req.url); const url = new URL(req.url); diff --git a/crates/base/test_cases/jsx-2/index.ts b/crates/base/test_cases/jsx-2/index.ts index 39231560d..f6b5b3142 100644 --- a/crates/base/test_cases/jsx-2/index.ts +++ b/crates/base/test_cases/jsx-2/index.ts @@ -2,54 +2,63 @@ import { serve } from 'https://deno.land/std@0.131.0/http/server.ts'; console.log('main function started'); -serve(async (req: Request) => { - const url = new URL(req.url); - const { pathname } = url; - const path_parts = pathname.split('/'); - const service_name = path_parts[1]; - - if (!service_name || service_name === '') { - const error = { msg: 'missing function name in request' }; - return new Response( - JSON.stringify(error), - { status: 400, headers: { 'Content-Type': 'application/json' } }, - ); - } +const controller = new AbortController(); + +serve( + (req: Request) => { + const url = new URL(req.url); + const { pathname } = url; + const path_parts = pathname.split('/'); + const service_name = path_parts[1]; - const servicePath = `./test_cases/${service_name}`; - console.error(`serving the request with ${servicePath}`); - - const createWorker = async () => { - const memoryLimitMb = 150; - const workerTimeoutMs = 1 * 60 * 1000; - const noModuleCache = false; - const importMapPath = null; - const envVarsObj = Deno.env.toObject(); - const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]); - - return await EdgeRuntime.userWorkers.create({ - servicePath, - memoryLimitMb, - workerTimeoutMs, - noModuleCache, - importMapPath, - envVars - }); - }; - - const callWorker = async () => { - try { - const worker = await createWorker(); - return await worker.fetch(req); - } catch (e) { - console.error(e); - const error = { msg: e.toString() }; + if (!service_name || service_name === '') { + const error = { msg: 'missing function name in request' }; return new Response( JSON.stringify(error), - { status: 500, headers: { 'Content-Type': 'application/json' } }, + { status: 400, headers: { 'Content-Type': 'application/json' } }, ); } - }; - return callWorker(); -}); + const servicePath = `./test_cases/${service_name}`; + console.error(`serving the request with ${servicePath}`); + + const createWorker = async () => { + const memoryLimitMb = 150; + const workerTimeoutMs = 1 * 60 * 1000; + const noModuleCache = false; + const importMapPath = null; + const envVarsObj = Deno.env.toObject(); + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]); + + return await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars + }); + }; + + const callWorker = async () => { + try { + const worker = await createWorker(); + return await worker.fetch(req); + } catch (e) { + console.error(e); + const error = { msg: e.toString() }; + return new Response( + JSON.stringify(error), + { status: 500, headers: { 'Content-Type': 'application/json' } }, + ); + } + }; + + return callWorker(); + }, + { + signal: controller.signal + } +); + +addEventListener('beforeunload', () => controller.abort()); diff --git a/crates/base/test_cases/jsx/index.ts b/crates/base/test_cases/jsx/index.ts index 0c72466dd..2e4ed4a6f 100644 --- a/crates/base/test_cases/jsx/index.ts +++ b/crates/base/test_cases/jsx/index.ts @@ -2,59 +2,68 @@ import { serve } from 'https://deno.land/std@0.131.0/http/server.ts'; console.log('main function started'); -serve(async (req: Request) => { - const url = new URL(req.url); - const { pathname } = url; - const path_parts = pathname.split('/'); - const service_name = path_parts[1]; - - if (!service_name || service_name === '') { - const error = { msg: 'missing function name in request' }; - return new Response( - JSON.stringify(error), - { status: 400, headers: { 'Content-Type': 'application/json' } }, - ); - } +const controller = new AbortController(); - const servicePath = `./test_cases/${service_name}`; - console.error(`serving the request with ${servicePath}`); - - const createWorker = async () => { - const memoryLimitMb = 150; - const workerTimeoutMs = 1 * 60 * 1000; - const noModuleCache = false; - const importMapPath = null; - const envVarsObj = Deno.env.toObject(); - const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]); - - return await EdgeRuntime.userWorkers.create({ - servicePath, - memoryLimitMb, - workerTimeoutMs, - noModuleCache, - importMapPath, - envVars, - jsxImportSourceConfig: { - defaultSpecifier: "https://esm.sh/preact", - module: "jsx-runtime", - baseUrl: servicePath - } - }); - }; - - const callWorker = async () => { - try { - const worker = await createWorker(); - return await worker.fetch(req); - } catch (e) { - console.error(e); - const error = { msg: e.toString() }; +serve( + (req: Request) => { + const url = new URL(req.url); + const { pathname } = url; + const path_parts = pathname.split('/'); + const service_name = path_parts[1]; + + if (!service_name || service_name === '') { + const error = { msg: 'missing function name in request' }; return new Response( JSON.stringify(error), - { status: 500, headers: { 'Content-Type': 'application/json' } }, + { status: 400, headers: { 'Content-Type': 'application/json' } }, ); } - }; - return callWorker(); -}); + const servicePath = `./test_cases/${service_name}`; + console.error(`serving the request with ${servicePath}`); + + const createWorker = async () => { + const memoryLimitMb = 150; + const workerTimeoutMs = 1 * 60 * 1000; + const noModuleCache = false; + const importMapPath = null; + const envVarsObj = Deno.env.toObject(); + const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]); + + return await EdgeRuntime.userWorkers.create({ + servicePath, + memoryLimitMb, + workerTimeoutMs, + noModuleCache, + importMapPath, + envVars, + jsxImportSourceConfig: { + defaultSpecifier: "https://esm.sh/preact", + module: "jsx-runtime", + baseUrl: servicePath + } + }); + }; + + const callWorker = async () => { + try { + const worker = await createWorker(); + return await worker.fetch(req); + } catch (e) { + console.error(e); + const error = { msg: e.toString() }; + return new Response( + JSON.stringify(error), + { status: 500, headers: { 'Content-Type': 'application/json' } }, + ); + } + }; + + return callWorker(); + }, + { + signal: controller.signal + } +); + +addEventListener('beforeunload', () => controller.abort()); diff --git a/crates/base/test_cases/oak-v12-file-upload/index.ts b/crates/base/test_cases/oak-v12-file-upload/index.ts index 20bad3f12..45390d27f 100644 --- a/crates/base/test_cases/oak-v12-file-upload/index.ts +++ b/crates/base/test_cases/oak-v12-file-upload/index.ts @@ -3,6 +3,7 @@ import { Application, Router } from 'https://deno.land/x/oak@v12.3.0/mod.ts'; const MB = 1024 * 1024; const router = new Router(); +const controller = new AbortController(); router .post('/file-upload', async (ctx) => { @@ -32,4 +33,8 @@ const app = new Application(); app.use(router.routes()); app.use(router.allowedMethods()); -await app.listen(); +addEventListener('beforeunload', () => controller.abort()); + +await app.listen({ + signal: controller.signal +}); From 635a9e0476127cde4e21f41051303370f1a08b64 Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:38:02 +0000 Subject: [PATCH 7/9] refactor: retouch major crates --- crates/base/src/commands.rs | 6 +- crates/base/src/deno_runtime.rs | 659 +++++++----- crates/base/src/lib.rs | 2 +- crates/base/src/macros/test_macros.rs | 2 +- .../implementation/default_handler.rs | 85 -- .../base/src/rt_worker/implementation/mod.rs | 1 - crates/base/src/rt_worker/mod.rs | 6 - crates/base/src/rt_worker/supervisor/mod.rs | 212 ---- crates/base/src/rt_worker/utils.rs | 59 -- crates/base/src/rt_worker/worker.rs | 384 ------- crates/base/src/rt_worker/worker_ctx.rs | 945 ------------------ crates/base/src/server.rs | 162 +-- crates/base/src/utils/test_utils.rs | 129 ++- crates/base/src/worker/driver/managed.rs | 179 ++++ crates/base/src/worker/driver/mod.rs | 153 +++ crates/base/src/worker/driver/user.rs | 176 ++++ crates/base/src/worker/mod.rs | 27 + .../worker_pool.rs => worker/pool.rs} | 137 ++- crates/base/src/worker/supervisor/mod.rs | 419 ++++++++ .../supervisor/strategy_per_request.rs | 47 +- .../supervisor/strategy_per_worker.rs | 93 +- .../base/src/worker/supervisor/v8_handler.rs | 115 +++ crates/base/src/worker/termination_token.rs | 48 + crates/base/src/worker/utils.rs | 80 ++ crates/base/src/worker/worker_inner.rs | 328 ++++++ .../src/worker/worker_surface_creation.rs | 783 +++++++++++++++ crates/base/tests/integration_tests.rs | 169 ++-- crates/base_rt/src/lib.rs | 19 +- crates/cli/src/main.rs | 8 +- crates/event_worker/events.rs | 10 +- crates/event_worker/js_interceptors.rs | 10 +- crates/sb_core/js/async_hook.js | 26 +- crates/sb_core/js/bootstrap.js | 52 +- crates/sb_core/js/edge_runtime.js | 60 ++ crates/sb_core/js/http.js | 40 +- crates/sb_core/js/main_worker.js | 22 - crates/sb_core/lib.rs | 16 +- crates/sb_core/net.rs | 40 +- 38 files changed, 3429 insertions(+), 2280 deletions(-) delete mode 100644 crates/base/src/rt_worker/implementation/default_handler.rs delete mode 100644 crates/base/src/rt_worker/implementation/mod.rs delete mode 100644 crates/base/src/rt_worker/mod.rs delete mode 100644 crates/base/src/rt_worker/supervisor/mod.rs delete mode 100644 crates/base/src/rt_worker/utils.rs delete mode 100644 crates/base/src/rt_worker/worker.rs delete mode 100644 crates/base/src/rt_worker/worker_ctx.rs create mode 100644 crates/base/src/worker/driver/managed.rs create mode 100644 crates/base/src/worker/driver/mod.rs create mode 100644 crates/base/src/worker/driver/user.rs create mode 100644 crates/base/src/worker/mod.rs rename crates/base/src/{rt_worker/worker_pool.rs => worker/pool.rs} (81%) create mode 100644 crates/base/src/worker/supervisor/mod.rs rename crates/base/src/{rt_worker => worker}/supervisor/strategy_per_request.rs (84%) rename crates/base/src/{rt_worker => worker}/supervisor/strategy_per_worker.rs (80%) create mode 100644 crates/base/src/worker/supervisor/v8_handler.rs create mode 100644 crates/base/src/worker/termination_token.rs create mode 100644 crates/base/src/worker/utils.rs create mode 100644 crates/base/src/worker/worker_inner.rs create mode 100644 crates/base/src/worker/worker_surface_creation.rs create mode 100644 crates/sb_core/js/edge_runtime.js delete mode 100644 crates/sb_core/js/main_worker.js diff --git a/crates/base/src/commands.rs b/crates/base/src/commands.rs index 11eaccca4..df6465fe5 100644 --- a/crates/base/src/commands.rs +++ b/crates/base/src/commands.rs @@ -1,7 +1,7 @@ use crate::{ inspector_server::Inspector, - rt_worker::{worker_ctx::TerminationToken, worker_pool::WorkerPoolPolicy}, - server::{Server, ServerFlags, ServerHealth, Tls, WorkerEntrypoints}, + server::{Server, ServerFlags, ServerHealth, SignumOrExitCode, Tls, WorkerEntrypoints}, + worker::{pool::WorkerPoolPolicy, TerminationToken}, InspectorOption, }; use anyhow::Error; @@ -26,7 +26,7 @@ pub async fn start_server( inspector_option: Option, jsx_specifier: Option, jsx_module: Option, -) -> Result, Error> { +) -> Result, Error> { let mut server = Server::new( ip, port, diff --git a/crates/base/src/deno_runtime.rs b/crates/base/src/deno_runtime.rs index eaee239af..ae545ef31 100644 --- a/crates/base/src/deno_runtime.rs +++ b/crates/base/src/deno_runtime.rs @@ -1,20 +1,20 @@ use crate::inspector_server::Inspector; -use crate::rt_worker::supervisor::{CPUUsage, CPUUsageMetrics}; -use crate::rt_worker::worker::DuplexStreamEntry; -use crate::server::ServerFlags; use crate::utils::json; use crate::utils::path::find_up; use crate::utils::units::{bytes_to_display, mib_to_bytes, percentage_value}; +use crate::worker::supervisor::{CPUUsage, CPUUsageMetrics}; +use crate::worker::{DuplexStreamEntry, Worker}; use anyhow::{anyhow, bail, Context, Error}; use arc_swap::ArcSwapOption; use base_mem_check::{MemCheckState, WorkerHeapStatistics}; -use base_rt::DenoRuntimeDropToken; use base_rt::{get_current_cpu_time_ns, BlockingScopeCPUUsage}; +use base_rt::{DenoRuntimeDropToken, DropToken}; use cooked_waker::{IntoWaker, WakeRef}; use ctor::ctor; use deno_cache::SqliteBackedCache; use deno_core::error::{AnyError, JsError}; +use deno_core::unsync::AtomicFlag; use deno_core::url::Url; use deno_core::v8::{self, GCCallbackFlags, GCType, HeapStatistics, Isolate}; use deno_core::{ @@ -32,7 +32,6 @@ use log::error; use once_cell::sync::{Lazy, OnceCell}; use sb_core::http::sb_core_http; use sb_core::http_start::sb_core_http_start; -use sb_core::util::sync::AtomicFlag; use sb_fs::prefix_fs::PrefixFs; use sb_fs::s3_fs::S3Fs; use sb_fs::static_fs::StaticFs; @@ -261,6 +260,7 @@ struct DispatchEventFunctions { dispatch_load_event_fn_global: v8::Global, dispatch_beforeunload_event_fn_global: v8::Global, dispatch_unload_event_fn_global: v8::Global, + dispatch_drain_event_fn_global: v8::Global, } #[derive(IntoStaticStr, Debug, Clone, Copy)] @@ -270,22 +270,114 @@ pub enum WillTerminateReason { Memory, WallClock, EarlyDrop, + Termination, +} + +#[derive(Debug, Clone, Default)] +pub struct RuntimeState { + pub evaluating_mod: Arc, + pub event_loop_completed: Arc, + pub terminated: Arc, + pub found_inspector_session: Arc, +} + +impl RuntimeState { + pub fn is_evaluating_mod(&self) -> bool { + self.evaluating_mod.is_raised() + } + + pub fn is_event_loop_completed(&self) -> bool { + self.event_loop_completed.is_raised() + } + + pub fn is_terminated(&self) -> bool { + self.terminated.is_raised() + } + + pub fn is_found_inspector_session(&self) -> bool { + self.found_inspector_session.is_raised() + } +} + +#[derive(Debug)] +pub struct RunOptions { + wait_termination_request_token: bool, + duplex_stream_rx: mpsc::UnboundedReceiver, + maybe_cpu_usage_metrics_tx: Option>, +} + +pub struct RunOptionsBuilder { + wait_termination_request_token: bool, + duplex_stream_rx: Option>, + maybe_cpu_usage_metrics_tx: Option>, +} + +impl Default for RunOptionsBuilder { + fn default() -> Self { + Self { + wait_termination_request_token: true, + duplex_stream_rx: None, + maybe_cpu_usage_metrics_tx: None, + } + } +} + +impl RunOptionsBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn wait_termination_request_token(mut self, val: bool) -> Self { + self.wait_termination_request_token = val; + self + } + + pub fn stream_rx(mut self, val: mpsc::UnboundedReceiver) -> Self { + self.duplex_stream_rx = Some(val); + self + } + + pub fn cpu_usage_metrics_tx( + mut self, + val: Option>, + ) -> Self { + self.maybe_cpu_usage_metrics_tx = val; + self + } + + pub fn build(self) -> Result { + let Self { + wait_termination_request_token, + duplex_stream_rx, + maybe_cpu_usage_metrics_tx, + } = self; + + // TODO(Nyannyacha): Make this as optional. + let Some(duplex_stream_rx) = duplex_stream_rx else { + return Err(anyhow!("stream_rx can't be empty")); + }; + + Ok(RunOptions { + wait_termination_request_token, + duplex_stream_rx, + maybe_cpu_usage_metrics_tx, + }) + } } pub struct DenoRuntime { + pub runtime_state: Arc, pub js_runtime: ManuallyDrop, + pub drop_token: CancellationToken, + pub(crate) termination_request_token: CancellationToken, + pub env_vars: HashMap, // TODO: does this need to be pub? pub conf: WorkerRuntimeOpts, pub s3_fs: Option, - pub(crate) termination_request_token: CancellationToken, - - pub(crate) is_terminated: Arc, - pub(crate) is_found_inspector_session: Arc, - main_module_id: ModuleId, - maybe_inspector: Option, + worker: Worker, promise_metrics: PromiseMetrics, mem_check: Arc, @@ -330,11 +422,12 @@ where { #[allow(clippy::unnecessary_literal_unwrap)] #[allow(clippy::arc_with_non_send_sync)] - pub async fn new( - opts: WorkerContextInitOpts, - maybe_inspector: Option, - flags: Arc, - ) -> Result { + pub(crate) async fn new(mut worker: Worker) -> Result { + let init_opts = worker.init_opts.take(); + let flags = worker.flags.clone(); + + debug_assert!(init_opts.is_some(), "init_opts must not be None"); + let WorkerContextInitOpts { mut conf, service_path, @@ -350,12 +443,14 @@ where maybe_s3_fs_config, maybe_tmp_fs_config, .. - } = opts; + } = init_opts.unwrap(); // TODO(Nyannyacha): Make sure `service_path` is an absolute path first. let drop_token = CancellationToken::default(); + let termination_request_token = CancellationToken::default(); let promise_metrics = PromiseMetrics::default(); + let runtime_state = Arc::::default(); let base_dir_path = std::env::current_dir().map(|p| p.join(&service_path))?; let Ok(mut main_module_url) = Url::from_directory_path(&base_dir_path) else { @@ -518,7 +613,7 @@ where }); } - let has_inspector = maybe_inspector.is_some(); + let has_inspector = worker.inspector.is_some(); let need_source_map = user_context .get("sourceMap") .and_then(serde_json::Value::as_bool) @@ -690,7 +785,7 @@ where let runtime_options = RuntimeOptions { extensions, is_main: true, - inspector: maybe_inspector.is_some(), + inspector: has_inspector, create_params, get_error_class_fn: Some(&get_error_class_name), shared_array_buffer_store: None, @@ -716,38 +811,23 @@ where .to_object(scope) .unwrap(); - let dispatch_load_event_fn_str = - v8::String::new_external_onebyte_static(scope, b"dispatchLoadEvent").unwrap(); - let dispatch_load_event_fn = bootstrap_ns - .get(scope, dispatch_load_event_fn_str.into()) - .unwrap(); - let dispatch_load_event_fn = - v8::Local::::try_from(dispatch_load_event_fn).unwrap(); - let dispatch_beforeunload_event_fn_str = - v8::String::new_external_onebyte_static(scope, b"dispatchBeforeUnloadEvent") + macro_rules! get_global { + ($name:expr) => {{ + let dispatch_fn_str = + v8::String::new_external_onebyte_static(scope, $name).unwrap(); + let dispatch_fn = v8::Local::::try_from( + bootstrap_ns.get(scope, dispatch_fn_str.into()).unwrap(), + ) .unwrap(); - let dispatch_beforeunload_event_fn = bootstrap_ns - .get(scope, dispatch_beforeunload_event_fn_str.into()) - .unwrap(); - let dispatch_beforeunload_event_fn = - v8::Local::::try_from(dispatch_beforeunload_event_fn).unwrap(); - let dispatch_unload_event_fn_str = - v8::String::new_external_onebyte_static(scope, b"dispatchUnloadEvent").unwrap(); - let dispatch_unload_event_fn = bootstrap_ns - .get(scope, dispatch_unload_event_fn_str.into()) - .unwrap(); - let dispatch_unload_event_fn = - v8::Local::::try_from(dispatch_unload_event_fn).unwrap(); - - let dispatch_load_event_fn_global = v8::Global::new(scope, dispatch_load_event_fn); - let dispatch_beforeunload_event_fn_global = - v8::Global::new(scope, dispatch_beforeunload_event_fn); - let dispatch_unload_event_fn_global = v8::Global::new(scope, dispatch_unload_event_fn); + v8::Global::new(scope, dispatch_fn) + }}; + } DispatchEventFunctions { - dispatch_load_event_fn_global, - dispatch_beforeunload_event_fn_global, - dispatch_unload_event_fn_global, + dispatch_load_event_fn_global: get_global!(b"dispatchLoadEvent"), + dispatch_beforeunload_event_fn_global: get_global!(b"dispatchBeforeUnloadEvent"), + dispatch_unload_event_fn_global: get_global!(b"dispatchUnloadEvent"), + dispatch_drain_event_fn_global: get_global!(b"dispatchDrainEvent"), } }; @@ -758,6 +838,7 @@ where op_state.put(dispatch_fns); op_state.put(promise_metrics.clone()); + op_state.put(runtime_state.clone()); op_state.put(GlobalMainContext(main_context)); } @@ -770,7 +851,7 @@ where op_state.put(sb_env::EnvVars::default()); } - if let Some(inspector) = maybe_inspector.clone() { + if let Some(inspector) = worker.inspector.as_ref() { inspector.server.register_inspector( main_module_url.to_string(), &mut js_runtime, @@ -799,10 +880,19 @@ where option_env!("GIT_V_TAG"), )); + let tokens = { + let op_state = js_runtime.op_state(); + let resource_table = &mut op_state.borrow_mut().resource_table; + serde_json::json!({ + "terminationRequestToken": resource_table.add(DropToken(termination_request_token.clone())) + }) + }; + let extra_context = { let mut context = serde_json::json!(RuntimeContext::get_extra_context()); json::merge_object(&mut context, &serde_json::Value::Object(user_context)); + json::merge_object(&mut context, &tokens); context }; @@ -875,7 +965,7 @@ where } op_state.put(sb_env::EnvVars(env_vars)); - op_state.put(DenoRuntimeDropToken(drop_token.clone())); + op_state.put(DenoRuntimeDropToken(DropToken(drop_token.clone()))); } let main_module_id = { @@ -913,19 +1003,18 @@ where } Ok(Self { - drop_token, + runtime_state, js_runtime: ManuallyDrop::new(js_runtime), + + drop_token, + termination_request_token, + env_vars, conf, s3_fs: maybe_s3_fs, - termination_request_token: CancellationToken::new(), - - is_terminated: Arc::default(), - is_found_inspector_session: Arc::default(), - main_module_id, - maybe_inspector, + worker, promise_metrics, mem_check, @@ -938,12 +1027,13 @@ where }) } - pub async fn run( - &mut self, - duplex_stream_rx: mpsc::UnboundedReceiver, - maybe_cpu_usage_metrics_tx: Option>, - name: Option, - ) -> (Result<(), Error>, i64) { + pub async fn run(&mut self, options: RunOptions) -> (Result<(), Error>, i64) { + let RunOptions { + wait_termination_request_token, + duplex_stream_rx, + maybe_cpu_usage_metrics_tx, + } = options; + { let op_state_rc = self.js_runtime.op_state(); let mut op_state = op_state_rc.borrow_mut(); @@ -957,7 +1047,7 @@ where } } - let _terminate_guard = scopeguard::guard(self.is_terminated.clone(), |v| { + let _terminate_guard = scopeguard::guard(self.runtime_state.terminated.clone(), |v| { v.raise(); }); @@ -966,20 +1056,22 @@ where let current_thread_id = std::thread::current().id(); let mut accumulated_cpu_time_ns = 0i64; - let span = debug_span!("runtime", ?name, thread_id = ?current_thread_id); + let span = debug_span!("runtime", thread_id = ?current_thread_id); let inspector = self.inspector(); let mut mod_result_rx = unsafe { self.js_runtime.v8_isolate().enter(); if inspector.is_some() { - let is_terminated = self.is_terminated.clone(); - let mut this = scopeguard::guard_on_unwind(&mut *self, |this| { - this.js_runtime.v8_isolate().exit(); - is_terminated.raise(); + let state = self.runtime_state.clone(); + let mut this = scopeguard::guard_on_unwind(&mut *self, { + |this| { + this.js_runtime.v8_isolate().exit(); + state.terminated.raise(); + } }); { - let _guard = scopeguard::guard(this.is_found_inspector_session.clone(), |v| { + let _guard = scopeguard::guard(state.found_inspector_session.clone(), |v| { v.raise(); }); @@ -993,7 +1085,7 @@ where if this.termination_request_token.is_cancelled() { this.js_runtime.v8_isolate().exit(); - is_terminated.raise(); + state.terminated.raise(); return (Ok(()), 0i64); } } @@ -1019,9 +1111,17 @@ where } { + let evaluating_mod = + scopeguard::guard(self.runtime_state.evaluating_mod.clone(), |v| { + v.lower(); + }); + + evaluating_mod.raise(); + let event_loop_fut = self .run_event_loop( current_thread_id, + wait_termination_request_token, &maybe_cpu_usage_metrics_tx, &mut accumulated_cpu_time_ns, ) @@ -1050,77 +1150,86 @@ where if let Err(err) = mod_result { return (Err(err), get_accumulated_cpu_time_ms!()); } + if self.runtime_state.is_event_loop_completed() + && self.promise_metrics.have_all_promises_been_resolved() + { + return (Ok(()), get_accumulated_cpu_time_ms!()); + } let mut this = self.get_v8_tls_guard(); - if let Err(err) = with_cpu_metrics_guard( - current_thread_id, - this.js_runtime.op_state(), - &maybe_cpu_usage_metrics_tx, - &mut accumulated_cpu_time_ns, - || MaybeDenoRuntime::DenoRuntime(*this).dispatch_load_event(), - ) { - return (Err(err), get_accumulated_cpu_time_ms!()); + if !this.termination_request_token.is_cancelled() { + if let Err(err) = with_cpu_metrics_guard( + current_thread_id, + this.js_runtime.op_state(), + &maybe_cpu_usage_metrics_tx, + &mut accumulated_cpu_time_ns, + || MaybeDenoRuntime::DenoRuntime(*this).dispatch_load_event(), + ) { + return (Err(err), get_accumulated_cpu_time_ms!()); + } } } + self.runtime_state.event_loop_completed.lower(); + if let Err(err) = self .run_event_loop( current_thread_id, + wait_termination_request_token, &maybe_cpu_usage_metrics_tx, &mut accumulated_cpu_time_ns, ) .instrument(span) .await { + return ( + Err(anyhow!("event loop error: {}", err)), + get_accumulated_cpu_time_ms!(), + ); + } + + if !self.conf.is_user_worker() { let mut this = self.get_v8_tls_guard(); - let _ = with_cpu_metrics_guard( + let mut this = this.get_v8_termination_guard(); + + if let Err(err) = with_cpu_metrics_guard( current_thread_id, this.js_runtime.op_state(), &maybe_cpu_usage_metrics_tx, &mut accumulated_cpu_time_ns, || MaybeDenoRuntime::DenoRuntime(&mut this).dispatch_unload_event(), - ); + ) { + return (Err(err), get_accumulated_cpu_time_ms!()); + } // TODO(Nyannyacha): Here we also need to trigger the event for node platform (i.e; exit) - - return ( - Err(anyhow!("event loop error: {}", err)), - get_accumulated_cpu_time_ms!(), - ); - } - - let mut this = self.get_v8_tls_guard(); - - if let Err(err) = with_cpu_metrics_guard( - current_thread_id, - this.js_runtime.op_state(), - &maybe_cpu_usage_metrics_tx, - &mut accumulated_cpu_time_ns, - || MaybeDenoRuntime::DenoRuntime(&mut this).dispatch_unload_event(), - ) { - return (Err(err), get_accumulated_cpu_time_ms!()); } - // TODO(Nyannyacha): Here we also need to trigger the event for node platform (i.e; exit) - (Ok(()), get_accumulated_cpu_time_ms!()) } fn run_event_loop<'l>( &'l mut self, #[allow(unused_variables)] current_thread_id: ThreadId, + wait_termination_request_token: bool, maybe_cpu_usage_metrics_tx: &'l Option>, accumulated_cpu_time_ns: &'l mut i64, ) -> impl Future> + 'l { let has_inspector = self.inspector().is_some(); let is_user_worker = self.conf.is_user_worker(); let global_waker = self.waker.clone(); - let termination_request_token = self.termination_request_token.clone(); + + let mut termination_request_fut = self + .termination_request_token + .clone() + .cancelled_owned() + .boxed(); let beforeunload_cpu_threshold = self.beforeunload_cpu_threshold.clone(); let beforeunload_mem_threshold = self.beforeunload_mem_threshold.clone(); + let state = self.runtime_state.clone(); let mem_check_state = is_user_worker.then(|| self.mem_check.clone()); let mut poll_sem = None::; @@ -1233,24 +1342,34 @@ where } } - // NOTE(Nyannyacha): If tasks are empty or V8 is not evaluating the - // function, and so V8 is no longer inside its loop, it turns out - // that requesting termination does not work; thus, we need another - // way to escape from the polling loop so the supervisor can - // terminate the runtime. if need_pool_event_loop && poll_result.is_pending() - && termination_request_token.is_cancelled() + && termination_request_fut.poll_unpin(cx).is_ready() { return Poll::Ready(Ok(())); } - poll_result + match poll_result { + Poll::Pending => Poll::Pending, + Poll::Ready(err @ Err(_)) => Poll::Ready(err), + Poll::Ready(Ok(())) => { + if !state.is_event_loop_completed() { + state.event_loop_completed.raise(); + } + if wait_termination_request_token + && !termination_request_fut.poll_unpin(cx).is_ready() + { + return Poll::Pending; + } + + Poll::Ready(Ok(())) + } + } }) } pub fn inspector(&self) -> Option { - self.maybe_inspector.clone() + self.worker.inspector.clone() } pub fn promise_metrics(&self) -> PromiseMetrics { @@ -1289,8 +1408,8 @@ where #[instrument(level = "debug", skip(self))] fn wait_for_inspector_session(&mut self) { - debug!(has_inspector = self.maybe_inspector.is_some()); - if let Some(inspector) = self.maybe_inspector.as_ref() { + debug!(has_inspector = self.worker.inspector.is_some()); + if let Some(inspector) = self.worker.inspector.as_ref() { debug!(addr = %inspector.server.host, server.inspector = ?inspector.option); let inspector_impl = self.js_runtime.inspector(); let mut inspector_impl_ref = inspector_impl.borrow_mut(); @@ -1328,6 +1447,31 @@ where guard } + + fn get_v8_termination_guard<'l>( + &'l mut self, + ) -> scopeguard::ScopeGuard< + &'l mut DenoRuntime, + impl FnOnce(&'l mut DenoRuntime) + 'l, + > { + let was_terminating_execution = self.js_runtime.v8_isolate().is_execution_terminating(); + let mut guard = scopeguard::guard(self, move |v| { + if was_terminating_execution { + v.js_runtime.v8_isolate().terminate_execution(); + } + + v.js_runtime + .v8_isolate() + .set_microtasks_policy(v8::MicrotasksPolicy::Auto); + }); + + guard.js_runtime.v8_isolate().cancel_terminate_execution(); + guard + .js_runtime + .v8_isolate() + .set_microtasks_policy(v8::MicrotasksPolicy::Explicit); + guard + } } type TerminateExecutionIfCancelledReturnType = @@ -1516,6 +1660,17 @@ where |_| Ok(()), ) } + + /// Dispatches "drain" event to the JavaScript runtime. + /// + /// Does not poll event loop, and thus not await any of the "drain" event handlers. + pub fn dispatch_drain_event(&mut self) -> Result<(), AnyError> { + self.dispatch_event_with_callback( + |fns| &fns.dispatch_drain_event_fn_global, + |_| vec![], + |_| Ok(()), + ) + } } pub fn import_meta_resolve_callback( @@ -1657,7 +1812,7 @@ extern "C" fn mem_check_gc_prologue_callback_fn( #[cfg(test)] mod test { use crate::deno_runtime::DenoRuntime; - use crate::rt_worker::worker::DuplexStreamEntry; + use crate::worker::{DuplexStreamEntry, WorkerBuilder}; use anyhow::Context; use deno_config::JsxImportSourceConfig; use deno_core::error::AnyError; @@ -1687,6 +1842,7 @@ mod test { use url::Url; use super::GetRuntimeContext; + use super::RunOptionsBuilder; impl DenoRuntime { fn to_value_mut(&mut self, global_value: &v8::Global) -> Result @@ -1757,42 +1913,45 @@ mod test { let (worker_pool_tx, _) = mpsc::unbounded_channel::(); DenoRuntime::new( - WorkerContextInitOpts { - maybe_eszip: eszip, - service_path: path - .map(PathBuf::from) - .unwrap_or(PathBuf::from("./test_cases/main")), - - conf: { - if let Some(conf) = worker_runtime_conf { - conf - } else { - WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }) - } - }, + WorkerBuilder::new( + WorkerContextInitOpts { + maybe_eszip: eszip, + service_path: path + .map(PathBuf::from) + .unwrap_or(PathBuf::from("./test_cases/main")), + + conf: { + if let Some(conf) = worker_runtime_conf { + conf + } else { + WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }) + } + }, - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, - no_module_cache: false, - env_vars: env_vars.unwrap_or_default(), + no_module_cache: false, + env_vars: env_vars.unwrap_or_default(), - static_patterns, - maybe_jsx_import_source_config: jsx_import_source_config, + static_patterns, + maybe_jsx_import_source_config: jsx_import_source_config, - timing: None, - import_map_path: None, + timing: None, + import_map_path: None, - maybe_s3_fs_config: s3_fs_config, - maybe_tmp_fs_config: tmp_fs_config, - }, - None, - Arc::default(), + maybe_s3_fs_config: s3_fs_config, + maybe_tmp_fs_config: tmp_fs_config, + }, + Arc::default(), + ) + .build() + .unwrap(), ) .await .unwrap() @@ -1873,33 +2032,36 @@ mod test { let (worker_pool_tx, _) = mpsc::unbounded_channel::(); DenoRuntime::<()>::new( - WorkerContextInitOpts { - service_path: PathBuf::from("./test_cases/"), - no_module_cache: false, - import_map_path: None, - env_vars: Default::default(), - timing: None, - maybe_eszip: None, - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: Some(FastString::from(String::from( - "Deno.serve((req) => new Response('Hello World'));", - ))), - conf: { - WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }) - }, - static_patterns: vec![], + WorkerBuilder::new( + WorkerContextInitOpts { + service_path: PathBuf::from("./test_cases/"), + no_module_cache: false, + import_map_path: None, + env_vars: Default::default(), + timing: None, + maybe_eszip: None, + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: Some(FastString::from(String::from( + "Deno.serve((req) => new Response('Hello World'));", + ))), + conf: { + WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }) + }, + static_patterns: vec![], - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }, - None, - Arc::default(), + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }, + Arc::default(), + ) + .build() + .unwrap(), ) .await .expect("It should not panic"); @@ -1923,31 +2085,34 @@ mod test { let eszip_code = bin_eszip.into_bytes(); let runtime = DenoRuntime::<()>::new( - WorkerContextInitOpts { - service_path: PathBuf::from("./test_cases/"), - no_module_cache: false, - import_map_path: None, - env_vars: Default::default(), - timing: None, - maybe_eszip: Some(EszipPayloadKind::VecKind(eszip_code)), - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, - conf: { - WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }) - }, - static_patterns: vec![], + WorkerBuilder::new( + WorkerContextInitOpts { + service_path: PathBuf::from("./test_cases/"), + no_module_cache: false, + import_map_path: None, + env_vars: Default::default(), + timing: None, + maybe_eszip: Some(EszipPayloadKind::VecKind(eszip_code)), + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, + conf: { + WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }) + }, + static_patterns: vec![], - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }, - None, - Arc::default(), + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }, + Arc::default(), + ) + .build() + .unwrap(), ) .await; @@ -1988,33 +2153,35 @@ mod test { .unwrap(); let eszip_code = binary_eszip.into_bytes(); - let runtime = DenoRuntime::<()>::new( - WorkerContextInitOpts { - service_path, - no_module_cache: false, - import_map_path: None, - env_vars: Default::default(), - timing: None, - maybe_eszip: Some(EszipPayloadKind::VecKind(eszip_code)), - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, - conf: { - WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }) - }, - static_patterns: vec![], + WorkerBuilder::new( + WorkerContextInitOpts { + service_path, + no_module_cache: false, + import_map_path: None, + env_vars: Default::default(), + timing: None, + maybe_eszip: Some(EszipPayloadKind::VecKind(eszip_code)), + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, + conf: { + WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }) + }, + static_patterns: vec![], - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }, - None, - Arc::default(), + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }, + Arc::default(), + ) + .build() + .unwrap(), ) .await; @@ -2466,7 +2633,15 @@ mod test { .await; let (_tx, duplex_stream_rx) = mpsc::unbounded_channel::(); - let (result, _) = user_rt.run(duplex_stream_rx, None, None).await; + let (result, _) = user_rt + .run( + RunOptionsBuilder::new() + .wait_termination_request_token(false) + .stream_rx(duplex_stream_rx) + .build() + .unwrap(), + ) + .await; assert!(result.is_ok(), "expected no errors"); @@ -2483,7 +2658,15 @@ mod test { .await; let (_tx, duplex_stream_rx) = mpsc::unbounded_channel::(); - let (result, _) = user_rt.run(duplex_stream_rx, None, None).await; + let (result, _) = user_rt + .run( + RunOptionsBuilder::new() + .wait_termination_request_token(false) + .stream_rx(duplex_stream_rx) + .build() + .unwrap(), + ) + .await; match result { Err(err) => { @@ -2517,13 +2700,21 @@ mod test { let handle = user_rt.js_runtime.v8_isolate().thread_safe_handle(); user_rt.add_memory_limit_callback(move |_| { - handle.terminate_execution(); + assert!(handle.terminate_execution()); waker.wake(); callback_tx.send(()).unwrap(); }); let wait_fut = async move { - let (result, _) = user_rt.run(duplex_stream_rx, None, None).await; + let (result, _) = user_rt + .run( + RunOptionsBuilder::new() + .wait_termination_request_token(false) + .stream_rx(duplex_stream_rx) + .build() + .unwrap(), + ) + .await; assert!(result .unwrap_err() @@ -2626,7 +2817,17 @@ mod test { let (_tx, duplex_stream_rx) = mpsc::unbounded_channel(); - user_rt.run(duplex_stream_rx, None, None).await.0.unwrap(); + user_rt + .run( + RunOptionsBuilder::new() + .wait_termination_request_token(false) + .stream_rx(duplex_stream_rx) + .build() + .unwrap(), + ) + .await + .0 + .unwrap(); } #[tokio::test] @@ -2644,6 +2845,16 @@ mod test { let (_tx, duplex_stream_rx) = mpsc::unbounded_channel(); - user_rt.run(duplex_stream_rx, None, None).await.0.unwrap(); + user_rt + .run( + RunOptionsBuilder::new() + .wait_termination_request_token(false) + .stream_rx(duplex_stream_rx) + .build() + .unwrap(), + ) + .await + .0 + .unwrap(); } } diff --git a/crates/base/src/lib.rs b/crates/base/src/lib.rs index 2f25b8a8b..a2cc76818 100644 --- a/crates/base/src/lib.rs +++ b/crates/base/src/lib.rs @@ -3,10 +3,10 @@ extern crate core; pub mod commands; pub mod deno_runtime; pub mod macros; -pub mod rt_worker; pub mod server; pub mod snapshot; pub mod utils; +pub mod worker; mod inspector_server; mod timeout; diff --git a/crates/base/src/macros/test_macros.rs b/crates/base/src/macros/test_macros.rs index 7919b4c47..efc80f0af 100644 --- a/crates/base/src/macros/test_macros.rs +++ b/crates/base/src/macros/test_macros.rs @@ -198,10 +198,10 @@ pub mod __private { use crate::server::ServerEvent; pub use crate::commands::start_server; - pub use crate::rt_worker::worker_ctx::TerminationToken; pub use crate::server::ServerFlags; pub use crate::server::ServerHealth; pub use crate::server::Tls; + pub use crate::worker::TerminationToken; pub use futures_util; pub use reqwest_v011; diff --git a/crates/base/src/rt_worker/implementation/default_handler.rs b/crates/base/src/rt_worker/implementation/default_handler.rs deleted file mode 100644 index 0c8cad75d..000000000 --- a/crates/base/src/rt_worker/implementation/default_handler.rs +++ /dev/null @@ -1,85 +0,0 @@ -use crate::deno_runtime::DenoRuntime; -use crate::rt_worker::supervisor::CPUUsageMetrics; -use crate::rt_worker::worker::{DuplexStreamEntry, HandleCreationType, Worker, WorkerHandler}; -use anyhow::Error; -use event_worker::events::{ - BootFailureEvent, EventLoopCompletedEvent, UncaughtExceptionEvent, WorkerEvents, -}; -use log::error; -use std::any::Any; -use std::time::Duration; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio::sync::oneshot::Receiver; - -impl WorkerHandler for Worker { - fn handle_error(&self, error: Error) -> Result { - log::error!("{}", format!("{error:#}")); - Ok(WorkerEvents::BootFailure(BootFailureEvent { - msg: format!("{error:#}"), - })) - } - - fn handle_creation<'r>( - &self, - created_rt: &'r mut DenoRuntime, - duplex_stream_rx: UnboundedReceiver, - termination_event_rx: Receiver, - maybe_cpu_usage_metrics_tx: Option>, - name: Option, - ) -> HandleCreationType<'r> { - let run_worker_rt = async move { - match created_rt - .run(duplex_stream_rx, maybe_cpu_usage_metrics_tx, name) - .await - { - // if the error is execution terminated, check termination event reason - (Err(err), cpu_usage_ms) => { - let err_string = err.to_string(); - - if err_string.ends_with("execution terminated") { - Ok(termination_event_rx - .await - .unwrap() - .with_cpu_time_used(cpu_usage_ms as usize)) - } else { - error!( - "runtime has escaped from the event loop unexpectedly: {}", - err_string.as_str() - ); - - Ok(WorkerEvents::UncaughtException(UncaughtExceptionEvent { - exception: err_string, - cpu_time_used: cpu_usage_ms as usize, - })) - } - } - - (Ok(()), cpu_usage_ms) => { - // NOTE(Nyannyacha): If a supervisor unconditionally - // requests the isolate to terminate, it might not come to - // this branch, so it might be removed in the future. - if created_rt.termination_request_token.is_cancelled() { - static EVENT_RECV_DEADLINE_DUR: Duration = Duration::from_secs(5); - - if let Ok(Ok(ev)) = - tokio::time::timeout(EVENT_RECV_DEADLINE_DUR, termination_event_rx) - .await - { - return Ok(ev.with_cpu_time_used(cpu_usage_ms as usize)); - } - } - - Ok(WorkerEvents::EventLoopCompleted(EventLoopCompletedEvent { - cpu_time_used: cpu_usage_ms as usize, - })) - } - } - }; - - Box::pin(run_worker_rt) - } - - fn as_any(&self) -> &dyn Any { - self - } -} diff --git a/crates/base/src/rt_worker/implementation/mod.rs b/crates/base/src/rt_worker/implementation/mod.rs deleted file mode 100644 index 5d0af6f13..000000000 --- a/crates/base/src/rt_worker/implementation/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod default_handler; diff --git a/crates/base/src/rt_worker/mod.rs b/crates/base/src/rt_worker/mod.rs deleted file mode 100644 index 0eb5d9db4..000000000 --- a/crates/base/src/rt_worker/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod implementation; -pub mod supervisor; -pub mod utils; -pub mod worker; -pub mod worker_ctx; -pub mod worker_pool; diff --git a/crates/base/src/rt_worker/supervisor/mod.rs b/crates/base/src/rt_worker/supervisor/mod.rs deleted file mode 100644 index 15e520d92..000000000 --- a/crates/base/src/rt_worker/supervisor/mod.rs +++ /dev/null @@ -1,212 +0,0 @@ -pub mod strategy_per_request; -pub mod strategy_per_worker; - -use std::{future::pending, sync::Arc, time::Duration}; - -use cpu_timer::{CPUAlarmVal, CPUTimer}; -use deno_core::v8; -use enum_as_inner::EnumAsInner; -use futures_util::task::AtomicWaker; -use log::{error, warn}; -use sb_core::PromiseMetrics; -use sb_workers::context::{Timing, UserWorkerMsgs, UserWorkerRuntimeOpts}; -use tokio::sync::{ - mpsc::{self, UnboundedReceiver}, - oneshot, -}; -use tokio_util::sync::CancellationToken; -use tracing::{debug, instrument}; -use uuid::Uuid; - -use crate::{ - deno_runtime::{MaybeDenoRuntime, WillTerminateReason}, - server::ServerFlags, - utils::units::percentage_value, -}; - -use super::{worker_ctx::TerminationToken, worker_pool::SupervisorPolicy}; - -#[repr(C)] -pub struct IsolateMemoryStats { - pub used_heap_size: usize, - pub external_memory: usize, -} - -#[derive(Clone, Copy)] -pub struct CPUTimerParam { - soft_limit_ms: u64, - hard_limit_ms: u64, -} - -impl CPUTimerParam { - pub fn new(soft_limit_ms: u64, hard_limit_ms: u64) -> Self { - Self { - soft_limit_ms, - hard_limit_ms, - } - } - - pub fn get_cpu_timer( - &self, - policy: SupervisorPolicy, - ) -> Option<(CPUTimer, UnboundedReceiver<()>)> { - let (cpu_alarms_tx, cpu_alarms_rx) = mpsc::unbounded_channel::<()>(); - - if self.is_disabled() { - return None; - } - - Some(( - CPUTimer::start( - if policy.is_per_worker() { - self.soft_limit_ms - } else { - self.hard_limit_ms - }, - if policy.is_per_request() { - 0 - } else { - self.hard_limit_ms - }, - CPUAlarmVal { cpu_alarms_tx }, - ) - .ok()?, - cpu_alarms_rx, - )) - } - - pub fn limits(&self) -> (u64, u64) { - (self.soft_limit_ms, self.hard_limit_ms) - } - - pub fn is_disabled(&self) -> bool { - self.soft_limit_ms == 0 && self.hard_limit_ms == 0 - } -} - -pub struct Tokens { - pub termination: Option, - pub supervise: CancellationToken, -} - -pub struct Arguments { - pub key: Uuid, - pub runtime_opts: UserWorkerRuntimeOpts, - pub cpu_timer: Option<(CPUTimer, mpsc::UnboundedReceiver<()>)>, - pub cpu_usage_metrics_rx: Option>, - pub cpu_timer_param: CPUTimerParam, - pub supervisor_policy: SupervisorPolicy, - pub promise_metrics: PromiseMetrics, - pub timing: Option, - pub memory_limit_rx: mpsc::UnboundedReceiver<()>, - pub pool_msg_tx: Option>, - pub isolate_memory_usage_tx: oneshot::Sender, - pub thread_safe_handle: v8::IsolateHandle, - pub waker: Arc, - pub tokens: Tokens, - pub flags: Arc, -} - -pub struct CPUUsage { - pub accumulated: i64, - pub diff: i64, -} - -#[derive(EnumAsInner)] -pub enum CPUUsageMetrics { - Enter(std::thread::ThreadId), - Leave(CPUUsage), -} - -async fn wait_cpu_alarm(maybe_alarm: Option<&mut UnboundedReceiver<()>>) -> Option<()> { - match maybe_alarm { - Some(alarm) => Some(alarm.recv().await?), - None => None, - } -} - -async fn create_wall_clock_beforeunload_alert(wall_clock_limit_ms: u64, pct: Option) { - let dur = pct - .and_then(|it| percentage_value(wall_clock_limit_ms, it)) - .map(Duration::from_millis); - - if let Some(dur) = dur { - tokio::time::sleep(dur).await; - } else { - pending::<()>().await; - unreachable!() - } -} - -#[repr(C)] -pub struct V8HandleTerminationData { - pub should_terminate: bool, - pub isolate_memory_usage_tx: Option>, -} - -pub extern "C" fn v8_handle_termination(isolate: &mut v8::Isolate, data: *mut std::ffi::c_void) { - let mut data = unsafe { Box::from_raw(data as *mut V8HandleTerminationData) }; - - // log memory usage - let mut heap_stats = v8::HeapStatistics::default(); - - isolate.get_heap_statistics(&mut heap_stats); - - let usage = IsolateMemoryStats { - used_heap_size: heap_stats.used_heap_size(), - external_memory: heap_stats.external_memory(), - }; - - if let Some(usage_tx) = data.isolate_memory_usage_tx.take() { - if usage_tx.send(usage).is_err() { - error!("failed to send isolate memory usage - receiver may have been dropped"); - } - } - - if data.should_terminate { - isolate.terminate_execution(); - } -} - -extern "C" fn v8_handle_wall_clock_beforeunload( - isolate: &mut v8::Isolate, - _data: *mut std::ffi::c_void, -) { - if let Err(err) = MaybeDenoRuntime::<()>::Isolate(isolate) - .dispatch_beforeunload_event(WillTerminateReason::WallClock) - { - error!( - "found an error while dispatching the beforeunload event: {}", - err - ); - } -} - -#[repr(C)] -pub struct V8HandleEarlyRetireData { - token: CancellationToken, -} - -extern "C" fn v8_handle_early_drop_beforeunload( - isolate: &mut v8::Isolate, - data: *mut std::ffi::c_void, -) { - let data = unsafe { Box::from_raw(data as *mut V8HandleEarlyRetireData) }; - - if let Err(err) = MaybeDenoRuntime::<()>::Isolate(isolate) - .dispatch_beforeunload_event(WillTerminateReason::EarlyDrop) - { - error!( - "found an error while dispatching the beforeunload event: {}", - err - ); - } else { - data.token.cancel(); - } -} - -#[instrument(level = "debug", skip_all)] -extern "C" fn v8_handle_early_retire(isolate: &mut v8::Isolate, _data: *mut std::ffi::c_void) { - isolate.low_memory_notification(); - debug!("sent low mem notification"); -} diff --git a/crates/base/src/rt_worker/utils.rs b/crates/base/src/rt_worker/utils.rs deleted file mode 100644 index fadadaf4e..000000000 --- a/crates/base/src/rt_worker/utils.rs +++ /dev/null @@ -1,59 +0,0 @@ -use event_worker::events::{EventMetadata, WorkerEventWithMetadata, WorkerEvents}; -use sb_workers::context::{UserWorkerMsgs, WorkerRuntimeOpts}; -use tokio::sync::mpsc::{self, UnboundedSender}; -use tokio_util::sync::CancellationToken; -use uuid::Uuid; - -type WorkerCoreConfig = ( - Option, - Option>, - Option>, - Option, - String, -); - -// TODO: Refactor this -pub fn parse_worker_conf(conf: &WorkerRuntimeOpts) -> WorkerCoreConfig { - let worker_core: WorkerCoreConfig = match conf { - WorkerRuntimeOpts::UserWorker(worker_opts) => ( - worker_opts.key, - worker_opts.pool_msg_tx.clone(), - worker_opts.events_msg_tx.clone(), - worker_opts.cancel.clone(), - worker_opts - .key - .map(|k| format!("sb-iso-{:?}", k)) - .unwrap_or("isolate-worker-unknown".to_string()), - ), - WorkerRuntimeOpts::MainWorker(_) => (None, None, None, None, "main-worker".to_string()), - WorkerRuntimeOpts::EventsWorker(_) => (None, None, None, None, "events-worker".to_string()), - }; - - worker_core -} - -pub fn get_event_metadata(conf: &WorkerRuntimeOpts) -> EventMetadata { - let mut event_metadata = EventMetadata { - service_path: None, - execution_id: None, - }; - if conf.is_user_worker() { - let conf = conf.as_user_worker().unwrap(); - event_metadata = EventMetadata { - service_path: conf.service_path.clone(), - execution_id: conf.key, - }; - } - - event_metadata -} - -pub fn send_event_if_event_worker_available( - maybe_event_worker: Option<&mpsc::UnboundedSender>, - event: WorkerEvents, - metadata: EventMetadata, -) { - if let Some(event_worker) = maybe_event_worker { - let _ = event_worker.send(WorkerEventWithMetadata { event, metadata }); - } -} diff --git a/crates/base/src/rt_worker/worker.rs b/crates/base/src/rt_worker/worker.rs deleted file mode 100644 index fad42ba66..000000000 --- a/crates/base/src/rt_worker/worker.rs +++ /dev/null @@ -1,384 +0,0 @@ -use crate::deno_runtime::DenoRuntime; -use crate::inspector_server::Inspector; -use crate::rt_worker::supervisor; -use crate::rt_worker::utils::{ - get_event_metadata, parse_worker_conf, send_event_if_event_worker_available, -}; -use crate::rt_worker::worker_ctx::create_supervisor; -use crate::server::ServerFlags; - -use anyhow::Error; -use base_mem_check::MemCheckState; -use base_rt::error::CloneableError; -use event_worker::events::{ - EventLoopCompletedEvent, EventMetadata, ShutdownEvent, ShutdownReason, UncaughtExceptionEvent, - WorkerEventWithMetadata, WorkerEvents, WorkerMemoryUsed, -}; -use futures_util::FutureExt; -use log::{debug, error}; -use sb_core::{MetricSource, RuntimeMetricSource, WorkerMetricSource}; -use sb_workers::context::{UserWorkerMsgs, WorkerContextInitOpts, WorkerExit, WorkerExitStatus}; -use std::any::Any; -use std::future::{pending, Future}; -use std::pin::Pin; -use std::sync::Arc; -use std::time::Duration; -use tokio::io; -use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; -use tokio::sync::oneshot::{self, Receiver, Sender}; -use tokio::time::Instant; -use tokio_util::sync::CancellationToken; -use uuid::Uuid; - -use super::supervisor::CPUUsageMetrics; -use super::worker_ctx::TerminationToken; -use super::worker_pool::SupervisorPolicy; - -#[derive(Clone)] -pub struct Worker { - pub worker_boot_start_time: Instant, - pub events_msg_tx: Option>, - pub pool_msg_tx: Option>, - pub cancel: Option, - pub event_metadata: EventMetadata, - pub worker_key: Option, - pub inspector: Option, - pub supervisor_policy: SupervisorPolicy, - pub worker_name: String, -} - -pub type HandleCreationType<'r> = Pin> + 'r>>; -pub type DuplexStreamEntry = (io::DuplexStream, Option); - -pub trait WorkerHandler: Send { - fn handle_error(&self, error: Error) -> Result; - fn handle_creation<'r>( - &self, - created_rt: &'r mut DenoRuntime, - duplex_stream_rx: UnboundedReceiver, - termination_event_rx: Receiver, - maybe_cpu_metrics_tx: Option>, - name: Option, - ) -> HandleCreationType<'r>; - fn as_any(&self) -> &dyn Any; -} - -impl Worker { - pub fn new(init_opts: &WorkerContextInitOpts) -> Result { - let (worker_key, pool_msg_tx, events_msg_tx, cancel, worker_name) = - parse_worker_conf(&init_opts.conf); - - let event_metadata = get_event_metadata(&init_opts.conf); - let worker_boot_start_time = Instant::now(); - - Ok(Self { - worker_boot_start_time, - events_msg_tx, - pool_msg_tx, - cancel, - event_metadata, - worker_key, - supervisor_policy: SupervisorPolicy::default(), - inspector: None, - worker_name, - }) - } - - pub fn set_inspector(&mut self, inspector: Inspector) { - self.inspector = Some(inspector); - } - - pub fn set_supervisor_policy(&mut self, supervisor_policy: Option) { - self.supervisor_policy = supervisor_policy.unwrap_or_default(); - } - - #[allow(clippy::too_many_arguments)] - pub fn start( - self: Arc, - mut opts: WorkerContextInitOpts, - duplex_stream_pair: ( - UnboundedSender, - UnboundedReceiver, - ), - booter_signal: Sender>, - exit: WorkerExit, - termination_token: Option, - inspector: Option, - flags: Arc, - ) { - let worker_name = self.worker_name.clone(); - let worker_key = self.worker_key; - let event_metadata = self.event_metadata.clone(); - let supervisor_policy = self.supervisor_policy; - - let (duplex_stream_tx, duplex_stream_rx) = duplex_stream_pair; - let events_msg_tx = self.events_msg_tx.clone(); - let pool_msg_tx = self.pool_msg_tx.clone(); - - let method_cloner = self.clone(); - let timing = opts.timing.take(); - let worker_kind = opts.conf.to_worker_kind(); - - let cancel = self.cancel.clone(); - let rt = if worker_kind.is_user_worker() { - &base_rt::USER_WORKER_RT - } else { - &base_rt::PRIMARY_WORKER_RT - }; - - let _worker_handle = rt.spawn_pinned(move || { - tokio::task::spawn_local(async move { - let (maybe_cpu_usage_metrics_tx, maybe_cpu_usage_metrics_rx) = worker_kind - .is_user_worker() - .then(unbounded_channel::) - .unzip(); - - let permit = DenoRuntime::acquire().await; - let result = match DenoRuntime::new(opts, inspector, flags.clone()).await { - Ok(new_runtime) => { - let mut runtime = scopeguard::guard(new_runtime, |mut runtime| unsafe { - runtime.js_runtime.v8_isolate().enter(); - }); - - unsafe { - runtime.js_runtime.v8_isolate().exit(); - } - - drop(permit); - - let metric_src = { - let metric_src = - WorkerMetricSource::from_js_runtime(&mut runtime.js_runtime); - if let Some(opts) = runtime.conf.as_main_worker().cloned() { - let state = runtime.js_runtime.op_state(); - let mut state_mut = state.borrow_mut(); - let metric_src = RuntimeMetricSource::new( - metric_src.clone(), - opts.event_worker_metric_src - .and_then(|it| it.into_worker().ok()), - opts.shared_metric_src, - ); - - state_mut.put(metric_src.clone()); - MetricSource::Runtime(metric_src) - } else { - MetricSource::Worker(metric_src) - } - }; - - let _ = booter_signal.send(Ok(metric_src)); - - // CPU TIMER - let (termination_event_tx, termination_event_rx) = - oneshot::channel::(); - - let _cpu_timer; - let mut supervise_cancel_token = None; - - let termination_fut = if worker_kind.is_user_worker() { - // cputimer is returned from supervisor and assigned here to keep it in scope. - let Ok((maybe_timer, cancel_token)) = create_supervisor( - worker_key.unwrap_or(Uuid::nil()), - &mut runtime, - supervisor_policy, - termination_event_tx, - pool_msg_tx.clone(), - maybe_cpu_usage_metrics_rx, - cancel, - timing, - termination_token.clone(), - flags, - ) else { - return; - }; - - _cpu_timer = maybe_timer; - supervise_cancel_token = Some(cancel_token); - - pending().boxed() - } else if let Some(token) = termination_token.clone() { - let is_terminated = runtime.is_terminated.clone(); - let termination_request_token = - runtime.termination_request_token.clone(); - - let (waker, thread_safe_handle) = { - let js_runtime = &mut runtime.js_runtime; - ( - js_runtime.op_state().borrow().waker.clone(), - js_runtime.v8_isolate().thread_safe_handle(), - ) - }; - - let maybe_event_worker_ctx = - runtime.conf.as_events_worker().map(|it| { - Duration::from_secs( - it.event_worker_exit_deadline_sec.unwrap_or(10), - ) - }); - - base_rt::SUPERVISOR_RT - .spawn(async move { - token.inbound.cancelled().await; - - let mut already_terminated = false; - if let Some(dur) = maybe_event_worker_ctx { - already_terminated = tokio::time::timeout(dur, async { - while !is_terminated.is_raised() { - waker.wake(); - tokio::task::yield_now().await; - } - }) - .await - .is_ok(); - } - - if !already_terminated { - termination_request_token.cancel(); - - let data_ptr_mut = Box::into_raw(Box::new( - supervisor::V8HandleTerminationData { - should_terminate: true, - isolate_memory_usage_tx: None, - }, - )); - - if !thread_safe_handle.request_interrupt( - supervisor::v8_handle_termination, - data_ptr_mut as *mut std::ffi::c_void, - ) { - drop(unsafe { Box::from_raw(data_ptr_mut) }); - } - - while !is_terminated.is_raised() { - waker.wake(); - tokio::task::yield_now().await; - } - } - - let _ = termination_event_tx.send(WorkerEvents::Shutdown( - ShutdownEvent { - reason: ShutdownReason::TerminationRequested, - cpu_time_used: 0, - memory_used: WorkerMemoryUsed { - total: 0, - heap: 0, - external: 0, - mem_check_captured: MemCheckState::default(), - }, - }, - )); - }) - .boxed() - } else { - pending().boxed() - }; - - let _guard = scopeguard::guard((), |_| { - worker_key.and_then(|worker_key_unwrapped| { - pool_msg_tx.map(|tx| { - if let Err(err) = - tx.send(UserWorkerMsgs::Shutdown(worker_key_unwrapped)) - { - error!( - "failed to send the shutdown signal to user worker pool: {:?}", - err - ); - } - }) - }); - }); - - let result = { - let supervise_cancel_token = - scopeguard::guard_on_unwind(supervise_cancel_token, |token| { - if let Some(token) = token { - token.cancel(); - } - }); - - let result = method_cloner - .handle_creation( - &mut runtime, - duplex_stream_rx, - termination_event_rx, - maybe_cpu_usage_metrics_tx, - Some(worker_name), - ) - .await; - - let maybe_uncaught_exception_event = match result.as_ref() { - Ok(WorkerEvents::UncaughtException(ev)) => Some(ev.clone()), - Err(err) => Some(UncaughtExceptionEvent { - cpu_time_used: 0, - exception: err.to_string(), - }), - - _ => None, - }; - - if let Some(ev) = maybe_uncaught_exception_event { - exit.set(WorkerExitStatus::WithUncaughtException(ev)).await; - - if let Some(token) = supervise_cancel_token.as_ref() { - token.cancel(); - } - } - - result - }; - - if let Some(token) = termination_token.as_ref() { - if !worker_kind.is_user_worker() { - let _ = termination_fut.await; - token.outbound.cancel(); - } - } - if let Some(fs) = runtime.s3_fs.as_ref() { - fs.flush_background_tasks().await; - } - - result - } - - Err(err) => { - drop(permit); - - let err = CloneableError::from(err.context("worker boot error")); - let _ = booter_signal.send(Err(err.clone().into())); - - method_cloner.handle_error(err.into()) - } - }; - - drop(duplex_stream_tx); - - match result { - Ok(event) => { - match event { - WorkerEvents::Shutdown(ShutdownEvent { cpu_time_used, .. }) - | WorkerEvents::UncaughtException(UncaughtExceptionEvent { - cpu_time_used, - .. - }) - | WorkerEvents::EventLoopCompleted(EventLoopCompletedEvent { - cpu_time_used, - .. - }) => { - debug!("CPU time used: {:?}ms", cpu_time_used); - } - - _ => {} - }; - - send_event_if_event_worker_available( - events_msg_tx.as_ref(), - event, - event_metadata.clone(), - ); - } - Err(err) => error!("unexpected worker error {}", err), - }; - }) - }); - } -} diff --git a/crates/base/src/rt_worker/worker_ctx.rs b/crates/base/src/rt_worker/worker_ctx.rs deleted file mode 100644 index 5e5f3c395..000000000 --- a/crates/base/src/rt_worker/worker_ctx.rs +++ /dev/null @@ -1,945 +0,0 @@ -use super::utils::send_event_if_event_worker_available; -use crate::deno_runtime::DenoRuntime; -use crate::inspector_server::Inspector; -use crate::server::ServerFlags; -use crate::timeout::{self, CancelOnWriteTimeout, ReadTimeoutStream}; - -use crate::rt_worker::worker::Worker; -use crate::rt_worker::worker_pool::WorkerPool; -use anyhow::{anyhow, bail, Error}; -use base_mem_check::MemCheckState; -use cpu_timer::CPUTimer; -use deno_config::JsxImportSourceConfig; -use deno_core::unsync::AtomicFlag; -use deno_core::{InspectorSessionProxy, LocalInspectorSession}; -use event_worker::events::{ - BootEvent, ShutdownEvent, WorkerEventWithMetadata, WorkerEvents, WorkerMemoryUsed, -}; -use futures_util::pin_mut; -use http_utils::io::Upgraded2; -use http_utils::utils::{emit_status_code, get_upgrade_type}; -use http_v02::StatusCode; -use hyper_v014::client::conn::http1; -use hyper_v014::upgrade::OnUpgrade; -use hyper_v014::{Body, Request, Response}; -use log::{debug, error}; -use once_cell::sync::Lazy; -use sb_core::{MetricSource, SharedMetricSource}; -use sb_graph::{DecoratorType, EszipPayloadKind}; -use sb_workers::context::{ - EventWorkerRuntimeOpts, MainWorkerRuntimeOpts, Timing, UserWorkerMsgs, WorkerContextInitOpts, - WorkerExit, WorkerKind, WorkerRequestMsg, WorkerRuntimeOpts, -}; -use sb_workers::errors::WorkerError; -use std::future::pending; -use std::io::ErrorKind; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; -use tokio::io::{self, copy_bidirectional}; -use tokio::net::TcpStream; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; -use tokio::sync::{mpsc, oneshot, Mutex}; -use tokio::time::sleep; -use tokio_rustls::server::TlsStream; -use tokio_util::sync::CancellationToken; -use tracing::warn; -use uuid::Uuid; - -use super::supervisor::{self, CPUTimerParam, CPUUsageMetrics}; -use super::worker::DuplexStreamEntry; -use super::worker_pool::{SupervisorPolicy, WorkerPoolPolicy}; - -#[derive(Clone)] -pub struct TerminationToken { - pub inbound: CancellationToken, - pub outbound: CancellationToken, -} - -impl std::fmt::Debug for TerminationToken { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TerminationToken").finish() - } -} - -impl Default for TerminationToken { - fn default() -> Self { - Self::new() - } -} - -impl TerminationToken { - pub fn new() -> Self { - Self { - inbound: CancellationToken::default(), - outbound: CancellationToken::default(), - } - } - - pub fn child_token(&self) -> Self { - Self { - inbound: self.inbound.child_token(), - outbound: self.outbound.child_token(), - } - } - - pub fn cancel(&self) { - self.inbound.cancel(); - } - - pub async fn cancel_and_wait(&self) { - if self.outbound.is_cancelled() { - return; - } - - self.cancel(); - self.outbound.cancelled().await; - } -} - -async fn handle_request( - flags: Arc, - worker_kind: WorkerKind, - duplex_stream_tx: mpsc::UnboundedSender, - msg: WorkerRequestMsg, -) -> Result<(), Error> { - let request_idle_timeout_ms = flags.request_idle_timeout_ms; - let request_buf_size = flags.request_buffer_size.unwrap_or_else(|| { - const KIB: usize = 1024; - static CHECK: Lazy = Lazy::new(AtomicFlag::default); - - if !CHECK.is_raised() { - CHECK.raise(); - warn!("request buffer size is not specified, so it will be set to 1 KiB"); - } - - KIB as u64 - }); - - let (ours, theirs) = io::duplex(request_buf_size as usize); - let WorkerRequestMsg { - mut req, - res_tx, - conn_token, - } = msg; - - let _ = duplex_stream_tx.send((theirs, conn_token.clone())); - let req_upgrade_type = get_upgrade_type(req.headers()); - let req_upgrade = req_upgrade_type - .clone() - .and_then(|it| Some(it).zip(req.extensions_mut().remove::())); - - // send the HTTP request to the worker over duplex stream - let (mut request_sender, connection) = - http1::Builder::new().writev(true).handshake(ours).await?; - - let (upgrade_tx, upgrade_rx) = oneshot::channel(); - - // spawn a task to poll the connection and drive the HTTP state - tokio::task::spawn({ - async move { - match connection.without_shutdown().await { - Err(e) => { - error!( - "error in {} worker connection: {}", - worker_kind, - e.message() - ); - } - - Ok(parts) => { - if let Some((requested, req_upgrade)) = req_upgrade { - if let Ok((Some(accepted), status)) = upgrade_rx.await { - if status == StatusCode::SWITCHING_PROTOCOLS && accepted == requested { - tokio::spawn(relay_upgraded_request_and_response( - req_upgrade, - parts, - request_idle_timeout_ms, - )); - - return; - } - }; - } - - if let Some(token) = conn_token { - token.cancelled_owned().await; - } - } - } - } - }); - - tokio::task::yield_now().await; - - let maybe_cancel_fut = async move { - if let Some(timeout_ms) = request_idle_timeout_ms { - sleep(Duration::from_millis(timeout_ms)).await; - } else { - pending::<()>().await; - unreachable!() - } - }; - - let res = tokio::select! { - resp = request_sender.send_request(req) => resp, - _ = maybe_cancel_fut => { - Ok(emit_status_code(http_v02::StatusCode::GATEWAY_TIMEOUT, None, false)) - } - }; - - let Ok(res) = res else { - drop(res_tx.send(res)); - return Ok(()); - }; - - if let Some(requested) = req_upgrade_type { - let res_upgrade_type = get_upgrade_type(res.headers()); - let _ = upgrade_tx.send((res_upgrade_type.clone(), res.status())); - - match res_upgrade_type { - Some(accepted) if accepted == requested => {} - _ => { - drop(res_tx.send(Ok(emit_status_code(StatusCode::BAD_GATEWAY, None, true)))); - return Ok(()); - } - } - } - - if let Some(timeout_ms) = flags.request_idle_timeout_ms { - let headers = res.headers(); - let is_streamed_response = !headers.contains_key(http_v02::header::CONTENT_LENGTH); - - if is_streamed_response { - let duration = Duration::from_millis(timeout_ms); - let (parts, body) = res.into_parts(); - - drop(res_tx.send(Ok(Response::from_parts( - parts, - Body::wrap_stream(CancelOnWriteTimeout::new(body, duration)), - )))); - - return Ok(()); - } - } - - drop(res_tx.send(Ok(res))); - Ok(()) -} - -async fn relay_upgraded_request_and_response( - downstream: OnUpgrade, - parts: http1::Parts, - maybe_idle_timeout: Option, -) { - let upstream = Upgraded2::new(parts.io, parts.read_buf); - let mut upstream = if let Some(timeout_ms) = maybe_idle_timeout { - ReadTimeoutStream::with_timeout(upstream, Duration::from_millis(timeout_ms)) - } else { - ReadTimeoutStream::with_bypass(upstream) - }; - - let mut downstream = downstream.await.expect("failed to upgrade request"); - - match copy_bidirectional(&mut upstream, &mut downstream).await { - Ok(_) => {} - Err(err) if matches!(err.kind(), ErrorKind::TimedOut | ErrorKind::BrokenPipe) => {} - Err(err) if matches!(err.kind(), ErrorKind::UnexpectedEof) => { - let Ok(_) = downstream.downcast::>>() else { - // TODO(Nyannyacha): It would be better if we send - // `close_notify` before shutdown an upstream if downstream is a - // TLS stream. - - // INVARIANT: `UnexpectedEof` due to shutdown `DuplexStream` is - // only expected to occur in the context of `TlsStream`. - panic!("unhandleable unexpected eof"); - }; - } - - value => { - unreachable!("coping between upgraded connections failed: {:?}", value); - } - } - - // XXX(Nyannyacha): Here you might want to emit the event metadata. -} - -#[allow(clippy::too_many_arguments)] -pub fn create_supervisor( - key: Uuid, - worker_runtime: &mut DenoRuntime, - supervisor_policy: SupervisorPolicy, - termination_event_tx: oneshot::Sender, - pool_msg_tx: Option>, - cpu_usage_metrics_rx: Option>, - cancel: Option, - timing: Option, - termination_token: Option, - flags: Arc, -) -> Result<(Option, CancellationToken), Error> { - let (memory_limit_tx, memory_limit_rx) = mpsc::unbounded_channel(); - let (waker, thread_safe_handle) = { - let js_runtime = &mut worker_runtime.js_runtime; - ( - js_runtime.op_state().borrow().waker.clone(), - js_runtime.v8_isolate().thread_safe_handle(), - ) - }; - - // we assert supervisor is only run for user workers - let conf = worker_runtime.conf.as_user_worker().unwrap().clone(); - let mem_check_state = worker_runtime.mem_check_state(); - let termination_request_token = worker_runtime.termination_request_token.clone(); - - let giveup_process_requests_token = cancel.clone(); - let supervise_cancel_token = CancellationToken::new(); - let tokens = supervisor::Tokens { - termination: termination_token, - supervise: supervise_cancel_token.clone(), - }; - - let maybe_inspector_params = worker_runtime.inspector().map(|_| { - ( - worker_runtime - .js_runtime - .inspector() - .borrow_mut() - .get_session_sender(), - worker_runtime.is_terminated.clone(), - worker_runtime.is_found_inspector_session.clone(), - ) - }); - - let send_memory_limit_fn = move |kind: &'static str| { - debug!("memory limit triggered: isolate: {:?}, kind: {}", key, kind); - - if memory_limit_tx.send(()).is_err() { - error!( - "failed to send memory limit reached notification(isolate may already be terminating): isolate: {:?}, kind: {}", - key, kind - ); - } - }; - - worker_runtime.add_memory_limit_callback({ - let send_fn = send_memory_limit_fn.clone(); - move |_| { - send_fn("mem_check"); - } - }); - - worker_runtime.js_runtime.add_near_heap_limit_callback({ - let send_fn = send_memory_limit_fn; - move |current, _| { - send_fn("v8"); - - // give an allowance on current limit (until the isolate is - // terminated) we do this so that oom won't end up killing the - // edge-runtime process - current * (conf.low_memory_multiplier as usize) - } - }); - - // Note: CPU timer must be started in the same thread as the worker runtime - - let cpu_timer_param = - CPUTimerParam::new(conf.cpu_time_soft_limit_ms, conf.cpu_time_hard_limit_ms); - - let (maybe_cpu_timer, maybe_cpu_alarms_rx) = - cpu_timer_param.get_cpu_timer(supervisor_policy).unzip(); - - drop({ - let _rt_guard = base_rt::SUPERVISOR_RT.enter(); - let maybe_cpu_timer_inner = maybe_cpu_timer.clone(); - let supervise_cancel_token_inner = supervise_cancel_token.clone(); - let promise_metrics = worker_runtime.promise_metrics(); - - tokio::spawn(async move { - let (isolate_memory_usage_tx, isolate_memory_usage_rx) = - oneshot::channel::(); - - let args = supervisor::Arguments { - key, - runtime_opts: conf.clone(), - cpu_timer: maybe_cpu_timer_inner.zip(maybe_cpu_alarms_rx), - cpu_usage_metrics_rx, - cpu_timer_param, - supervisor_policy, - promise_metrics, - timing, - memory_limit_rx, - pool_msg_tx, - isolate_memory_usage_tx, - thread_safe_handle, - waker: waker.clone(), - tokens, - flags, - }; - - let (reason, cpu_usage_ms) = { - use supervisor::*; - match supervisor_policy { - SupervisorPolicy::PerWorker => strategy_per_worker::supervise(args).await, - SupervisorPolicy::PerRequest { oneshot, .. } => { - strategy_per_request::supervise(args, oneshot).await - } - } - }; - - // NOTE: Sending a signal to the pooler that it is the user worker going - // disposed down and will not accept awaiting subsequent requests, so - // they must be re-polled again. - if let Some(cancel) = giveup_process_requests_token.as_ref() { - cancel.cancel(); - } - - if let Some((session_tx, is_terminated, is_found)) = maybe_inspector_params { - use deno_core::futures::channel::mpsc; - use deno_core::serde_json::Value; - - let termination_request_token = termination_request_token.clone(); - - base_rt::SUPERVISOR_RT - .spawn_blocking(move || { - let wait_inspector_disconnect_fut = async move { - let ls = tokio::task::LocalSet::new(); - ls.run_until(async move { - if is_terminated.is_raised() - || termination_request_token.is_cancelled() - { - return; - } - - termination_request_token.cancel(); - - if is_found.is_raised() { - return; - } - - let (outbound_tx, outbound_rx) = mpsc::unbounded(); - let (inbound_tx, inbound_rx) = mpsc::unbounded(); - - if session_tx - .unbounded_send(InspectorSessionProxy { - tx: outbound_tx, - rx: inbound_rx, - }) - .is_err() - { - return; - } - - let session = Arc::new(Mutex::new(LocalInspectorSession::new( - inbound_tx, - outbound_rx, - ))); - - let send_msg_fn = { - |msg| { - let is_terminated = is_terminated.clone(); - let session = session.clone(); - async move { - let mut session = session.lock().await; - let mut int = - tokio::time::interval(Duration::from_millis(61)); - - let fut = session.post_message(msg, None::); - - pin_mut!(fut); - - loop { - tokio::select! { - _ = int.tick() => { - if is_terminated.is_raised() { - break - } - } - - res = &mut fut => { - res.unwrap(); - break - } - } - } - } - } - }; - - send_msg_fn("Debugger.enable").await; - send_msg_fn("Runtime.runIfWaitingForDebugger").await; - }) - .await; - }; - - base_rt::SUPERVISOR_RT.block_on(wait_inspector_disconnect_fut); - }) - .await - .unwrap(); - } - - // NOTE: If we issue a hard CPU time limit, It's OK because it is - // still possible the worker's context is in the v8 event loop. The - // interrupt callback would be invoked from the V8 engine - // gracefully. But some case doesn't. - // - // Such as the worker going to a retired state due to the soft CPU - // time limit but not hitting the hard CPU time limit. In this case, - // we must wake up the worker's event loop manually. Otherwise, the - // supervisor has to wait until the wall clock future that we placed - // out on the runtime side is times out. - waker.wake(); - - let memory_used = match isolate_memory_usage_rx.await { - Ok(v) => WorkerMemoryUsed { - total: v.used_heap_size + v.external_memory, - heap: v.used_heap_size, - external: v.external_memory, - mem_check_captured: tokio::task::spawn_blocking(move || { - *mem_check_state.read().unwrap() - }) - .await - .unwrap(), - }, - - Err(_) => { - if !supervise_cancel_token_inner.is_cancelled() { - error!("isolate memory usage sender dropped"); - } - - WorkerMemoryUsed { - total: 0, - heap: 0, - external: 0, - mem_check_captured: MemCheckState::default(), - } - } - }; - - if !termination_request_token.is_cancelled() { - termination_request_token.cancel(); - waker.wake(); - } - - // send termination reason - let termination_event = WorkerEvents::Shutdown(ShutdownEvent { - reason, - memory_used, - cpu_time_used: cpu_usage_ms as usize, - }); - - let _ = termination_event_tx.send(termination_event); - }) - }); - - Ok((maybe_cpu_timer, supervise_cancel_token)) -} - -pub struct CreateWorkerArgs( - WorkerContextInitOpts, - Option, - Option, -); - -impl From for CreateWorkerArgs { - fn from(val: WorkerContextInitOpts) -> Self { - CreateWorkerArgs(val, None, None) - } -} - -impl From<(WorkerContextInitOpts, SupervisorPolicy)> for CreateWorkerArgs { - fn from(val: (WorkerContextInitOpts, SupervisorPolicy)) -> Self { - CreateWorkerArgs(val.0, Some(val.1), None) - } -} - -impl>> From<(WorkerContextInitOpts, T)> for CreateWorkerArgs { - fn from(val: (WorkerContextInitOpts, T)) -> Self { - CreateWorkerArgs(val.0, None, val.1.into()) - } -} - -impl - From<( - WorkerContextInitOpts, - SupervisorPolicy, - Option, - )> for CreateWorkerArgs -{ - fn from( - val: ( - WorkerContextInitOpts, - SupervisorPolicy, - Option, - ), - ) -> Self { - CreateWorkerArgs(val.0, Some(val.1), val.2) - } -} - -impl CreateWorkerArgs { - pub fn with_supervisor_policy(mut self, policy: SupervisorPolicy) -> Self { - self.1 = Some(policy); - self - } - - pub fn with_termination_token(mut self, token: TerminationToken) -> Self { - self.2 = Some(token); - self - } -} - -#[derive(Debug, Clone)] -pub struct WorkerCtx { - pub metric: MetricSource, - pub msg_tx: mpsc::UnboundedSender, - pub exit: WorkerExit, -} - -pub async fn create_worker>( - flags: Arc, - init_opts: Opt, - inspector: Option, -) -> Result { - let (duplex_stream_tx, duplex_stream_rx) = mpsc::unbounded_channel::(); - let (worker_boot_result_tx, worker_boot_result_rx) = - oneshot::channel::>(); - - let CreateWorkerArgs(worker_init_opts, maybe_supervisor_policy, maybe_termination_token) = - init_opts.into(); - - let worker_kind = worker_init_opts.conf.to_worker_kind(); - let exit = WorkerExit::default(); - let mut worker = Worker::new(&worker_init_opts)?; - - if worker_kind.is_user_worker() { - worker.set_supervisor_policy(maybe_supervisor_policy); - } - - let worker = Arc::new(worker); - - worker.clone().start( - worker_init_opts, - (duplex_stream_tx.clone(), duplex_stream_rx), - worker_boot_result_tx, - exit.clone(), - maybe_termination_token.clone(), - inspector, - flags.clone(), - ); - - // create an async task waiting for requests for worker - let (worker_req_tx, mut worker_req_rx) = mpsc::unbounded_channel::(); - - let worker_req_handle: tokio::task::JoinHandle> = tokio::task::spawn({ - let stream_tx = duplex_stream_tx; - async move { - while let Some(msg) = worker_req_rx.recv().await { - tokio::task::spawn({ - let flags = flags.clone(); - let stream_tx_inner = stream_tx.clone(); - async move { - if let Err(err) = - handle_request(flags, worker_kind, stream_tx_inner, msg).await - { - error!("worker failed to handle request: {:?}", err); - } - } - }); - } - - Ok(()) - } - }); - - // wait for worker to be successfully booted - match worker_boot_result_rx.await? { - Ok(metric) => { - let elapsed = worker.worker_boot_start_time.elapsed().as_millis(); - - send_event_if_event_worker_available( - worker.events_msg_tx.as_ref(), - WorkerEvents::Boot(BootEvent { - boot_time: elapsed as usize, - }), - worker.event_metadata.clone(), - ); - - Ok(WorkerCtx { - metric, - msg_tx: worker_req_tx, - exit, - }) - } - - Err(err) => { - worker_req_handle.abort(); - - if let Some(token) = maybe_termination_token.as_ref() { - token.outbound.cancel(); - } - - Err(err) - } - } -} - -pub async fn send_user_worker_request( - worker_request_msg_tx: mpsc::UnboundedSender, - req: Request, - cancel: CancellationToken, - exit: WorkerExit, - conn_token: Option, -) -> Result, Error> { - let (res_tx, res_rx) = oneshot::channel::, hyper_v014::Error>>(); - let msg = WorkerRequestMsg { - req, - res_tx, - conn_token, - }; - - // send the message to worker - worker_request_msg_tx.send(msg)?; - - // wait for the response back from the worker - let res = tokio::select! { - () = cancel.cancelled() => { - bail!(exit - .error() - .await - .unwrap_or(anyhow!(WorkerError::RequestCancelledBySupervisor))) - } - - res = res_rx => res, - }?; - - match res { - Ok(v) => { - // send the response back to the caller - Ok(v) - } - - Err(err) => { - if let Some(actual_error) = exit.error().await { - return Err(actual_error); - } - - Err(err.into()) - } - } -} - -// Todo: Fix -#[allow(clippy::too_many_arguments)] -pub async fn create_main_worker( - flags: Arc, - main_worker_path: PathBuf, - import_map_path: Option, - no_module_cache: bool, - runtime_opts: MainWorkerRuntimeOpts, - maybe_entrypoint: Option, - maybe_decorator: Option, - termination_token: Option, - inspector: Option, - jsx: Option, -) -> Result, Error> { - let mut service_path = main_worker_path.clone(); - let mut maybe_eszip = None; - if let Some(ext) = main_worker_path.extension() { - if ext == "eszip" { - service_path = main_worker_path.parent().unwrap().to_path_buf(); - maybe_eszip = Some(EszipPayloadKind::VecKind(std::fs::read(main_worker_path)?)); - } - } - - let ctx = create_worker( - flags, - ( - WorkerContextInitOpts { - service_path, - import_map_path, - no_module_cache, - timing: None, - maybe_eszip, - maybe_entrypoint, - maybe_decorator, - maybe_module_code: None, - conf: WorkerRuntimeOpts::MainWorker(runtime_opts), - env_vars: std::env::vars().collect(), - static_patterns: vec![], - - maybe_jsx_import_source_config: jsx, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }, - termination_token, - ), - inspector, - ) - .await - .map_err(|err| anyhow!("main worker boot error: {}", err))?; - - Ok(ctx.msg_tx) -} - -pub async fn create_events_worker( - flags: Arc, - events_worker_path: PathBuf, - import_map_path: Option, - maybe_entrypoint: Option, - maybe_decorator: Option, - termination_token: Option, -) -> Result<(WorkerCtx, mpsc::UnboundedSender), Error> { - let (events_tx, events_rx) = mpsc::unbounded_channel::(); - - let no_module_cache = flags.no_module_cache; - let event_worker_exit_deadline_sec = flags.event_worker_exit_deadline_sec; - - let mut service_path = events_worker_path.clone(); - let mut maybe_eszip = None; - - if let Some(ext) = events_worker_path.extension() { - if ext == "eszip" { - service_path = events_worker_path.parent().unwrap().to_path_buf(); - maybe_eszip = Some(EszipPayloadKind::VecKind(std::fs::read( - events_worker_path, - )?)); - } - } - - let ctx = create_worker( - flags, - ( - WorkerContextInitOpts { - service_path, - no_module_cache, - import_map_path, - env_vars: std::env::vars().collect(), - timing: None, - maybe_eszip, - maybe_entrypoint, - maybe_decorator, - maybe_module_code: None, - conf: WorkerRuntimeOpts::EventsWorker(EventWorkerRuntimeOpts { - events_msg_rx: Some(events_rx), - event_worker_exit_deadline_sec: Some(event_worker_exit_deadline_sec), - }), - static_patterns: vec![], - - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }, - termination_token, - ), - None, - ) - .await - .map_err(|err| anyhow!("events worker boot error: {}", err))?; - - Ok((ctx, events_tx)) -} - -pub async fn create_user_worker_pool( - flags: Arc, - policy: WorkerPoolPolicy, - worker_event_sender: Option>, - termination_token: Option, - static_patterns: Vec, - inspector: Option, - jsx: Option, -) -> Result<(SharedMetricSource, mpsc::UnboundedSender), Error> { - let metric_src = SharedMetricSource::default(); - let (user_worker_msgs_tx, mut user_worker_msgs_rx) = - mpsc::unbounded_channel::(); - - let user_worker_msgs_tx_clone = user_worker_msgs_tx.clone(); - - let _handle: tokio::task::JoinHandle> = tokio::spawn({ - let metric_src_inner = metric_src.clone(); - async move { - let token = termination_token.as_ref(); - let mut termination_requested = false; - let mut worker_pool = WorkerPool::new( - flags, - policy, - metric_src_inner, - worker_event_sender, - user_worker_msgs_tx_clone, - inspector, - ); - - // Note: Keep this loop non-blocking. Spawn a task to run blocking calls. - // Handle errors within tasks and log them - do not bubble up errors. - loop { - tokio::select! { - _ = async { - if let Some(token) = token { - token.inbound.cancelled().await; - } else { - pending::<()>().await; - } - }, if !termination_requested => { - termination_requested = true; - - if worker_pool.user_workers.is_empty() { - if let Some(token) = token { - token.outbound.cancel(); - } - - break; - } - } - - msg = user_worker_msgs_rx.recv() => { - match msg { - None => break, - Some(UserWorkerMsgs::Create(worker_options, tx)) => { - worker_pool.create_user_worker(WorkerContextInitOpts { - static_patterns: static_patterns.clone(), - maybe_jsx_import_source_config: { - if worker_options.maybe_jsx_import_source_config.is_some() { - worker_options.maybe_jsx_import_source_config - } else { - jsx.clone() - } - }, - ..worker_options - }, tx, token.map(TerminationToken::child_token)); - } - - Some(UserWorkerMsgs::Created(key, profile)) => { - worker_pool.add_user_worker(key, profile); - } - - Some(UserWorkerMsgs::SendRequest(key, req, res_tx, conn_token)) => { - worker_pool.send_request(&key, req, res_tx, conn_token); - } - - Some(UserWorkerMsgs::Idle(key)) => { - worker_pool.idle(&key); - } - - Some(UserWorkerMsgs::Shutdown(key)) => { - worker_pool.shutdown(&key); - - if termination_requested && worker_pool.user_workers.is_empty() { - if let Some(token) = token { - token.outbound.cancel(); - } - - break; - } - } - } - } - } - } - - worker_pool.worker_event_sender.take(); - - Ok(()) - } - }); - - Ok((metric_src, user_worker_msgs_tx)) -} diff --git a/crates/base/src/server.rs b/crates/base/src/server.rs index fa6b2b930..1a84e1280 100644 --- a/crates/base/src/server.rs +++ b/crates/base/src/server.rs @@ -1,11 +1,10 @@ use crate::inspector_server::Inspector; -use crate::rt_worker::worker_ctx::{ - create_events_worker, create_main_worker, create_user_worker_pool, TerminationToken, -}; -use crate::rt_worker::worker_pool::WorkerPoolPolicy; +use crate::worker::pool::WorkerPoolPolicy; +use crate::worker::{self, TerminationToken}; use crate::InspectorOption; use anyhow::{anyhow, bail, Context, Error}; use deno_config::JsxImportSourceConfig; +use either::Either::{self, Left, Right}; use enum_as_inner::EnumAsInner; use futures_util::future::{poll_fn, BoxFuture}; use futures_util::{FutureExt, Stream}; @@ -15,12 +14,11 @@ use rustls_pemfile::read_one_from_slice; use rustls_pemfile::Item; use sb_core::SharedMetricSource; use sb_graph::DecoratorType; -use sb_workers::context::{MainWorkerRuntimeOpts, WorkerRequestMsg}; +use sb_workers::context::WorkerRequestMsg; use std::future::{pending, Future}; use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::SocketAddr; -use std::path::Path; use std::pin::Pin; use std::str; use std::str::FromStr; @@ -329,11 +327,13 @@ impl Tls { } } +pub type SignumOrExitCode = Either; + pub struct Server { ip: Ipv4Addr, port: u16, tls: Option, - main_worker_req_tx: mpsc::UnboundedSender, + main_worker_surface: worker::MainWorkerSurface, callback_tx: Option>, termination_tokens: TerminationTokens, flags: Arc, @@ -347,7 +347,7 @@ impl Server { port: u16, tls: Option, main_service_path: String, - maybe_events_service_path: Option, + maybe_event_service_path: Option, maybe_decorator: Option, maybe_user_worker_policy: Option, import_map_path: Option, @@ -360,31 +360,26 @@ impl Server { jsx_specifier: Option, jsx_module: Option, ) -> Result { - let mut worker_events_tx = None; - let flags = Arc::new(flags); - let maybe_events_entrypoint = entrypoints.events; + let maybe_event_entrypoint = entrypoints.events; let maybe_main_entrypoint = entrypoints.main; let termination_tokens = - TerminationTokens::new(termination_token, maybe_events_service_path.is_some()); - - // Create Event Worker - let event_worker_metric_src = if let Some(events_service_path) = maybe_events_service_path { - let events_path = Path::new(&events_service_path); - let events_path_buf = events_path.to_path_buf(); - - let (ctx, sender) = create_events_worker( - flags.clone(), - events_path_buf, - import_map_path.clone(), - maybe_events_entrypoint, - maybe_decorator, - Some(termination_tokens.event.clone().unwrap()), - ) - .await?; - - worker_events_tx = Some(sender); - Some(ctx.metric) + TerminationTokens::new(termination_token, maybe_event_service_path.is_some()); + + // create an event worker + let event_worker_surface = if let Some(service_path) = maybe_event_service_path { + let mut builder = worker::EventWorkerSurfaceBuilder::new(&service_path); + + builder + .set_server_flags(Some(Left(flags.clone()))) + .set_termination_token(Some(termination_tokens.event.clone().unwrap())); + + builder + .set_import_map_path(import_map_path.as_deref()) + .set_entrypoint(maybe_event_entrypoint.as_deref()) + .set_decorator(maybe_decorator); + + Some(builder.build().await?) } else { None }; @@ -396,11 +391,13 @@ impl Server { base_url: Url::from_file_path(std::env::current_dir().unwrap()).unwrap(), }); - // Create a user worker pool - let (shared_metric_src, worker_pool_tx) = create_user_worker_pool( + // create a user worker pool + let (shared_metric_src, worker_pool_tx) = worker::create_user_worker_pool( flags.clone(), maybe_user_worker_policy.unwrap_or_default(), - worker_events_tx, + event_worker_surface + .as_ref() + .map(|it| it.event_message_sender()), Some(termination_tokens.pool.clone()), static_patterns, inspector.clone(), @@ -409,39 +406,40 @@ impl Server { .await?; // create main worker - let main_worker_path = Path::new(&main_service_path).to_path_buf(); - let main_worker_req_tx = create_main_worker( - flags.clone(), - main_worker_path, - import_map_path.clone(), - flags.no_module_cache, - MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: Some(shared_metric_src.clone()), - event_worker_metric_src, - }, - maybe_main_entrypoint, - maybe_decorator, - Some(termination_tokens.main.clone()), + let main_worker_surface = { + let mut builder = worker::MainWorkerSurfaceBuilder::new(&main_service_path); + + builder + .set_server_flags(Some(Left(flags.clone()))) + .set_termination_token(Some(termination_tokens.main.clone())); + if flags.allow_main_inspector { - inspector.map(|it| Inspector { + builder.set_inspector(inspector.map(|it| Inspector { option: InspectorOption::Inspect(it.option.socket_addr()), server: it.server, - }) - } else { - None - }, - jsx_config, - ) - .await?; + })); + } - let ip = Ipv4Addr::from_str(ip)?; + builder + .set_import_map_path(import_map_path.as_deref()) + .set_entrypoint(maybe_main_entrypoint.as_deref()) + .set_decorator(maybe_decorator) + .set_no_module_cache(Some(flags.no_module_cache)) + .set_jsx_import_source_config(jsx_config) + .set_worker_pool_sender(Some(worker_pool_tx)) + .set_shared_metric_source(Some(shared_metric_src.clone())) + .set_event_worker_metric_source( + event_worker_surface.as_ref().map(|it| it.metric.clone()), + ); + + builder.build().await? + }; Ok(Self { - ip, + ip: Ipv4Addr::from_str(ip)?, port, tls, - main_worker_req_tx, + main_worker_surface, callback_tx, termination_tokens, flags, @@ -453,7 +451,7 @@ impl Server { self.termination_tokens.terminate().await; } - pub async fn listen(&mut self) -> Result, Error> { + pub async fn listen(&mut self) -> Result, Error> { let addr = SocketAddr::new(IpAddr::V4(self.ip), self.port); let non_secure_listener = TcpListener::bind(&addr).await?; let mut secure_listener = if let Some(tls) = self.tls.take() { @@ -470,9 +468,8 @@ impl Server { let termination_tokens = &self.termination_tokens; let input_termination_token = termination_tokens.input.as_ref(); - let mut ret = None::; + let mut ret = None::; let mut can_receive_event = false; - let mut interrupted = false; let (event_tx, event_rx) = mpsc::unbounded_channel(); debug!( @@ -493,6 +490,7 @@ impl Server { let event_tx = can_receive_event.then_some(event_tx.clone()); let graceful_exit_token = CancellationToken::new(); + let main_worker_cancel_token = self.main_worker_surface.cancel.clone(); let ServerFlags { tcp_nodelay, @@ -505,11 +503,23 @@ impl Server { let request_read_timeout_dur = request_read_timeout_ms.map(Duration::from_millis); let mut terminate_signal_fut = get_termination_signal(); + #[derive(Default, Clone, Copy, EnumAsInner)] + enum LoopState { + #[default] + Normal, + Interrupted, + MainWorkerDestroyed, + } + + let mut loop_state = LoopState::Normal; loop { - let main_worker_req_tx = self.main_worker_req_tx.clone(); + let main_worker_req_tx = self.main_worker_surface.msg_tx.clone(); + let main_worker_cancel_fut = main_worker_cancel_token.cancelled(); let event_tx = event_tx.clone(); let metric_src = metric_src.clone(); + pin!(main_worker_cancel_fut); + tokio::select! { msg = non_secure_listener.accept() => { match msg { @@ -576,21 +586,39 @@ impl Server { break; } + _ = &mut main_worker_cancel_fut => { + error!("main worker has been destroyed"); + loop_state = LoopState::MainWorkerDestroyed; + break; + } + signum = &mut terminate_signal_fut => { info!("shutdown signal received: {}", signum); - ret = Some(signum); + ret = Some(Left(signum)); break; } _ = signal::ctrl_c() => { info!("interrupt signal received"); - interrupted = true; + loop_state = LoopState::Interrupted; break; } } } - if !interrupted && graceful_exit_deadline_sec > 0 { + if loop_state.is_main_worker_destroyed() { + ret = Some(Right(std::process::ExitCode::FAILURE)); + + let Some(err) = self.main_worker_surface.exit.error().await else { + return Ok(ret); + }; + + error!("{}", format!("{err:?}")); + + return Ok(ret); + } + + if !loop_state.is_interrupted() && graceful_exit_deadline_sec > 0 { static REQ_METRIC_CHECK_SLEEP_DUR: Duration = Duration::from_millis(10); let wait_fut = async move { @@ -673,7 +701,9 @@ impl Server { error!("received interrupt signal while waiting workers"); } } - } else if !interrupted && metric_src.received_requests() != metric_src.handled_requests() { + } else if !loop_state.is_interrupted() + && metric_src.received_requests() != metric_src.handled_requests() + { warn!("runtime exits immediately since the graceful exit feature has been disabled"); } diff --git a/crates/base/src/utils/test_utils.rs b/crates/base/src/utils/test_utils.rs index b3e5bae00..14f26684a 100644 --- a/crates/base/src/utils/test_utils.rs +++ b/crates/base/src/utils/test_utils.rs @@ -9,14 +9,16 @@ use std::{ }; use crate::{ - rt_worker::{ - worker_ctx::{create_user_worker_pool, create_worker, CreateWorkerArgs, TerminationToken}, - worker_pool::{SupervisorPolicy, WorkerPoolPolicy}, - }, server::ServerFlags, + worker::{ + self, + pool::{SupervisorPolicy, WorkerPoolPolicy}, + TerminationToken, + }, }; use anyhow::{bail, Context, Error}; +use either::Either::Right; use event_worker::events::WorkerEventWithMetadata; use futures_util::{future::BoxFuture, Future, FutureExt}; use http_v02::{Request, Response}; @@ -117,17 +119,17 @@ impl Future for RequestScopeGuard { } } -pub trait WorkerContextInitOptsForTesting { - fn with_policy(self, policy: SupervisorPolicy) -> CreateWorkerArgs - where - Self: Sized; -} +// pub trait WorkerContextInitOptsForTesting { +// fn with_policy(self, policy: SupervisorPolicy) -> CreateWorkerArgs +// where +// Self: Sized; +// } -impl WorkerContextInitOptsForTesting for WorkerContextInitOpts { - fn with_policy(self, policy: SupervisorPolicy) -> CreateWorkerArgs { - (self, policy).into() - } -} +// impl WorkerContextInitOptsForTesting for WorkerContextInitOpts { +// fn with_policy(self, policy: SupervisorPolicy) -> CreateWorkerArgs { +// (self, policy).into() +// } +// } pub struct TestBedBuilder { main_service_path: PathBuf, @@ -217,7 +219,7 @@ impl TestBedBuilder { let ((_, worker_pool_tx), pool_termination_token) = { let token = TerminationToken::new(); ( - create_user_worker_pool( + worker::create_user_worker_pool( Arc::new(self.flags), self.worker_pool_policy .unwrap_or_else(test_user_worker_pool_policy), @@ -233,41 +235,39 @@ impl TestBedBuilder { ) }; - let main_worker_init_opts = WorkerContextInitOpts { - service_path: self.main_service_path, - no_module_cache: false, - import_map_path: None, - env_vars: std::env::vars().collect(), - timing: None, - maybe_eszip: None, - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, - conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }), - static_patterns: vec![], - - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }; - let main_termination_token = TerminationToken::new(); - let ctx = create_worker( - Arc::new(self.flags), - (main_worker_init_opts, main_termination_token.clone()), - None, - ) - .await - .unwrap(); + let main_worker_surface = worker::WorkerSurfaceBuilder::new() + .sever_flags(Right(self.flags)) + .termination_token(main_termination_token.clone()) + .init_opts(WorkerContextInitOpts { + service_path: self.main_service_path, + no_module_cache: false, + import_map_path: None, + env_vars: std::env::vars().collect(), + timing: None, + maybe_eszip: None, + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, + conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }), + static_patterns: vec![], + + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }) + .build() + .await + .unwrap(); TestBed { pool_termination_token, main_termination_token, - main_worker_msg_tx: ctx.msg_tx, + main_worker_surface, } } } @@ -275,7 +275,7 @@ impl TestBedBuilder { pub struct TestBed { pool_termination_token: TerminationToken, main_termination_token: TerminationToken, - main_worker_msg_tx: mpsc::UnboundedSender, + main_worker_surface: worker::WorkerSurface, } impl TestBed { @@ -291,7 +291,7 @@ impl TestBed { let req: Request = request_factory_fn(http_v02::request::Builder::new())?; - let _ = self.main_worker_msg_tx.send(WorkerRequestMsg { + let _ = self.main_worker_surface.msg_tx.send(WorkerRequestMsg { req, res_tx, conn_token: Some(conn_token.clone()), @@ -323,7 +323,7 @@ impl TestBed { pub async fn create_test_user_worker>( opts: Opt, -) -> Result<(mpsc::UnboundedSender, RequestScope), Error> { +) -> Result<(worker::WorkerSurface, RequestScope), Error> { let CreateTestUserWorkerArgs(mut opts, maybe_policy) = opts.into(); let (req_start_tx, req_start_rx) = mpsc::unbounded_channel(); let (req_end_tx, req_end_rx) = mpsc::unbounded_channel(); @@ -336,26 +336,23 @@ pub async fn create_test_user_worker>( ..Default::default() }); - Ok({ - let ctx = create_worker( - Arc::default(), - opts.with_policy(policy) - .with_termination_token(termination_token.clone()), - None, - ) + let worker_surface = worker::WorkerSurfaceBuilder::new() + .init_opts(opts) + .policy(policy) + .termination_token(termination_token.clone()) + .build() .await?; - ( - ctx.msg_tx, - RequestScope { - policy, - req_start_tx, - req_end_tx, - termination_token, - conn_token: CancellationToken::new(), - }, - ) - }) + Ok(( + worker_surface, + RequestScope { + policy, + req_start_tx, + req_end_tx, + termination_token, + conn_token: CancellationToken::new(), + }, + )) } pub fn test_user_worker_pool_policy() -> WorkerPoolPolicy { diff --git a/crates/base/src/worker/driver/managed.rs b/crates/base/src/worker/driver/managed.rs new file mode 100644 index 000000000..26e84c7b4 --- /dev/null +++ b/crates/base/src/worker/driver/managed.rs @@ -0,0 +1,179 @@ +use std::future::Future; +use std::sync::Arc; +use std::time::Duration; + +use crate::deno_runtime::{DenoRuntime, RunOptionsBuilder, WillTerminateReason}; +use crate::worker::supervisor::{self, v8_handle_beforeunload}; +use crate::worker::{DuplexStreamEntry, WorkerCx}; +use anyhow::Error; +use base_mem_check::MemCheckState; +use event_worker::events::{ + ShutdownEvent, ShutdownReason, UncaughtExceptionEvent, WorkerEvents, WorkerMemoryUsed, +}; +use futures_util::FutureExt; +use log::error; +use tokio::sync::mpsc; +use tokio::sync::Mutex; +use tokio::task::JoinError; +use tokio::time::timeout; +use tracing::warn; + +use super::{BaseCx, WorkerDriver}; + +const SUPERVISE_DEADLINE_SEC: Duration = Duration::from_secs(15); + +#[derive(Clone)] +pub(crate) struct Managed { + inner: Arc, + cx: Arc>, +} + +impl Managed { + pub fn new(inner: Arc) -> Self { + Managed { + inner, + cx: Arc::default(), + } + } +} + +impl WorkerDriver for Managed { + fn on_created<'l>( + &self, + runtime: &'l mut DenoRuntime, + ) -> impl Future> + 'l { + let Managed { inner, cx } = self.clone(); + + async move { + let mut cx = cx.lock().await; + let network_receiver = cx.take_network_receiver()?; + let termination_event_receiver = cx.take_termination_event_receiver()?; + let termination_token = inner.termination_token.clone(); + + drop(cx); + + let result = runtime + .run( + RunOptionsBuilder::new() + .stream_rx(network_receiver) + .wait_termination_request_token(false) + .build() + .unwrap(), + ) + .await; + + if let Some(token) = termination_token { + token.cancel(); + } + + match result { + // if the error is execution terminated, check termination event reason + (Err(err), cpu_usage_ms) => { + let err_string = err.to_string(); + + if err_string.ends_with("execution terminated") { + Ok(termination_event_receiver + .await + .unwrap() + .with_cpu_time_used(cpu_usage_ms as usize)) + } else { + error!( + "runtime has escaped from the event loop unexpectedly: {}", + err_string.as_str() + ); + + Ok(WorkerEvents::UncaughtException(UncaughtExceptionEvent { + exception: err_string, + cpu_time_used: cpu_usage_ms as usize, + })) + } + } + + (Ok(()), cpu_usage_ms) => Ok(termination_event_receiver + .await + .unwrap() + .with_cpu_time_used(cpu_usage_ms as usize)), + } + } + } + + fn supervise( + &self, + runtime: &mut DenoRuntime, + ) -> Option> + 'static> { + let Managed { inner, cx } = self.clone(); + let mut cx = cx.try_lock().ok()?; + let runtime_drop_token = runtime.drop_token.clone(); + let termination_token = inner.termination_token.clone()?; + let termination_event_sender = match cx.take_termination_event_sender() { + Ok(v) => v, + Err(err) => { + tracing::error!(?err); + return None; + } + }; + + let (waker, thread_safe_handle) = { + let js_runtime = &mut runtime.js_runtime; + ( + js_runtime.op_state().borrow().waker.clone(), + js_runtime.v8_isolate().thread_safe_handle(), + ) + }; + + let wait_fut = async move { + termination_token.inbound.cancelled().await; + + let _ = termination_event_sender.send(WorkerEvents::Shutdown(ShutdownEvent { + reason: ShutdownReason::TerminationRequested, + cpu_time_used: 0, + memory_used: WorkerMemoryUsed { + total: 0, + heap: 0, + external: 0, + mem_check_captured: MemCheckState::default(), + }, + })); + + let data_ptr_mut = Box::into_raw(Box::new(supervisor::V8HandleBeforeunloadData { + reason: WillTerminateReason::Termination, + })); + + if thread_safe_handle.request_interrupt(v8_handle_beforeunload, data_ptr_mut as *mut _) + { + waker.wake(); + } else { + drop(unsafe { Box::from_raw(data_ptr_mut) }); + } + + if (timeout(SUPERVISE_DEADLINE_SEC, runtime_drop_token.cancelled()).await).is_err() { + warn!( + concat!( + "termination job is running for over {} seconds ", + "(Press Control-C to terminate the job immediately)" + ), + SUPERVISE_DEADLINE_SEC.as_secs() + ); + + tokio::select! { + _ = runtime_drop_token.cancelled() => {}, + Ok(_) = tokio::signal::ctrl_c() => { + warn!("interrupt signal received"); + } + } + } + + termination_token.outbound.cancel(); + }; + + base_rt::SUPERVISOR_RT.spawn(wait_fut).boxed().into() + } + + fn runtime_handle(&self) -> &'static tokio_util::task::LocalPoolHandle { + &base_rt::PRIMARY_WORKER_RT + } + + async fn network_sender(&self) -> mpsc::UnboundedSender { + self.cx.lock().await.get_network_sender() + } +} diff --git a/crates/base/src/worker/driver/mod.rs b/crates/base/src/worker/driver/mod.rs new file mode 100644 index 000000000..d7597bde8 --- /dev/null +++ b/crates/base/src/worker/driver/mod.rs @@ -0,0 +1,153 @@ +use std::{future::Future, sync::Arc}; + +use anyhow::{Context, Error}; +use event_worker::events::{BootFailureEvent, WorkerEvents}; +use futures_util::FutureExt; +use sb_workers::context::{WorkerContextInitOpts, WorkerKind}; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinError, +}; +use tokio_util::task::LocalPoolHandle; + +use crate::deno_runtime::DenoRuntime; +use crate::worker::{DuplexStreamEntry, WorkerCx}; + +mod managed; +mod user; + +struct BaseCx { + network_tx: mpsc::UnboundedSender, + network_rx: Option>, + termination_event_tx: Option>, + termination_event_rx: Option>, +} + +impl Default for BaseCx { + fn default() -> Self { + let (network_tx, network_rx) = mpsc::unbounded_channel(); + let (termination_event_tx, termination_event_rx) = oneshot::channel(); + + Self { + network_tx, + network_rx: Some(network_rx), + termination_event_tx: Some(termination_event_tx), + termination_event_rx: Some(termination_event_rx), + } + } +} + +impl BaseCx { + fn get_network_sender(&self) -> mpsc::UnboundedSender { + self.network_tx.clone() + } + + fn take_network_receiver( + &mut self, + ) -> Result, Error> { + self.network_rx + .take() + .context("network_rx already been consumed") + } + + fn take_termination_event_sender(&mut self) -> Result, Error> { + self.termination_event_tx + .take() + .context("termination_event_tx already been consumed") + } + + fn take_termination_event_receiver( + &mut self, + ) -> Result, Error> { + self.termination_event_rx + .take() + .context("termination_event_rx already been consumed") + } +} + +#[derive(Clone)] +pub(crate) enum WorkerDriverImpl { + User(user::User), + Managed(managed::Managed), +} + +impl WorkerDriver for WorkerDriverImpl { + async fn on_boot_error(&self, error: Error) -> Result { + match self { + Self::User(user) => user.on_boot_error(error).await, + Self::Managed(managed) => managed.on_boot_error(error).await, + } + } + + fn on_created<'l>( + &self, + runtime: &'l mut DenoRuntime, + ) -> impl Future> + 'l { + let this = self.clone(); + async move { + match this { + Self::User(user) => user.on_created(runtime).await, + Self::Managed(managed) => managed.on_created(runtime).await, + } + } + } + + fn supervise( + &self, + runtime: &mut DenoRuntime, + ) -> Option> + 'static> { + match self { + Self::User(user) => user.supervise(runtime).map(FutureExt::boxed), + Self::Managed(managed) => managed.supervise(runtime).map(FutureExt::boxed), + } + } + + fn runtime_handle(&self) -> &'static LocalPoolHandle { + match self { + Self::User(user) => user.runtime_handle(), + Self::Managed(managed) => managed.runtime_handle(), + } + } + + async fn network_sender(&self) -> mpsc::UnboundedSender { + match self { + Self::User(user) => user.network_sender().await, + Self::Managed(managed) => managed.network_sender().await, + } + } +} + +impl WorkerDriverImpl { + pub fn new(init_opts: &mut WorkerContextInitOpts, inner: Arc) -> Self { + match init_opts.conf.to_worker_kind() { + WorkerKind::UserWorker => Self::User(user::User::new(init_opts, inner)), + WorkerKind::MainWorker | WorkerKind::EventsWorker => { + Self::Managed(managed::Managed::new(inner)) + } + } + } +} + +pub(super) trait WorkerDriver: Send { + fn on_boot_error(&self, error: Error) -> impl Future> { + async move { + log::error!("{}", format!("{error:#}")); + Ok(WorkerEvents::BootFailure(BootFailureEvent { + msg: format!("{error:#}"), + })) + } + } + + fn on_created<'l>( + &self, + runtime: &'l mut DenoRuntime, + ) -> impl Future> + 'l; + + fn supervise( + &self, + runtime: &mut DenoRuntime, + ) -> Option> + 'static>; + + fn runtime_handle(&self) -> &'static LocalPoolHandle; + fn network_sender(&self) -> impl Future>; +} diff --git a/crates/base/src/worker/driver/user.rs b/crates/base/src/worker/driver/user.rs new file mode 100644 index 000000000..18249dc59 --- /dev/null +++ b/crates/base/src/worker/driver/user.rs @@ -0,0 +1,176 @@ +use std::future::Future; +use std::sync::Arc; + +use crate::deno_runtime::{DenoRuntime, RunOptionsBuilder}; +use crate::worker::supervisor::{create_supervisor, CPUUsageMetrics}; +use crate::worker::{DuplexStreamEntry, WorkerCx}; + +use anyhow::Error; +use event_worker::events::{UncaughtExceptionEvent, WorkerEvents}; +use sb_workers::context::{Timing, WorkerContextInitOpts}; +use tokio::sync::mpsc; +use tokio::sync::Mutex; +use tokio::task::JoinError; +use tracing::debug; +use uuid::Uuid; + +use super::{BaseCx, WorkerDriver}; + +#[derive(Clone)] +pub(crate) struct User { + inner: Arc, + cx: Arc>, +} + +struct UserCx { + inner: BaseCx, + timing: Option, + cpu_usage_metrics_tx: Option>, +} + +impl std::ops::Deref for UserCx { + type Target = BaseCx; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for UserCx { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl User { + pub fn new(init_opts: &mut WorkerContextInitOpts, inner: Arc) -> Self { + Self { + inner, + cx: Arc::new(Mutex::new(UserCx { + inner: BaseCx::default(), + timing: init_opts.timing.take(), + cpu_usage_metrics_tx: None, + })), + } + } +} + +impl WorkerDriver for User { + fn on_created<'l>( + &self, + runtime: &'l mut DenoRuntime, + ) -> impl Future> + 'l { + let User { cx, .. } = self.clone(); + + async move { + let mut cx = cx.lock().await; + let network_receiver = cx.take_network_receiver()?; + let cpu_usage_metrics_receiver = cx.cpu_usage_metrics_tx.take(); + let termination_event_receiver = cx.take_termination_event_receiver()?; + + drop(cx); + + match runtime + .run( + RunOptionsBuilder::new() + .stream_rx(network_receiver) + .cpu_usage_metrics_tx(cpu_usage_metrics_receiver) + .wait_termination_request_token(true) + .build() + .unwrap(), + ) + .await + { + // if the error is execution terminated, check termination event reason + (Err(err), cpu_usage_ms) => { + let err_string = err.to_string(); + + if err_string.ends_with("execution terminated") { + Ok(termination_event_receiver + .await + .unwrap() + .with_cpu_time_used(cpu_usage_ms as usize)) + } else { + log::error!( + "runtime has escaped from the event loop unexpectedly: {}", + err_string.as_str() + ); + + Ok(WorkerEvents::UncaughtException(UncaughtExceptionEvent { + exception: err_string, + cpu_time_used: cpu_usage_ms as usize, + })) + } + } + + (Ok(()), cpu_usage_ms) => Ok(termination_event_receiver + .await + .unwrap() + .with_cpu_time_used(cpu_usage_ms as usize)), + } + } + } + + fn supervise( + &self, + runtime: &mut DenoRuntime, + ) -> Option> + 'static> { + let User { inner, cx } = self.clone(); + let mut cx = cx.try_lock().ok()?; + + let UserCx { + timing, + cpu_usage_metrics_tx, + inner: base, + } = &mut *cx; + + let termination_event_sender = match base.take_termination_event_sender() { + Ok(v) => v, + Err(err) => { + tracing::error!(?err); + return None; + } + }; + + let mut s3_fs = runtime.s3_fs.clone(); + debug!(use_s3_fs = s3_fs.is_some()); + + let (cpu_tx, cpu_rx) = mpsc::unbounded_channel(); + let Ok((maybe_timer, cancel_token)) = create_supervisor( + inner.worker_key.unwrap_or(Uuid::nil()), + runtime, + inner.supervisor_policy, + termination_event_sender, + inner.pool_msg_tx.clone(), + Some(cpu_rx), + inner.cancel.clone(), + timing.take(), + inner.termination_token.clone(), + inner.flags.clone(), + ) else { + return None; + }; + + *cpu_usage_metrics_tx = Some(cpu_tx); + + async move { + let _cpu_timer = maybe_timer; + let _guard = cancel_token.drop_guard(); + + if let Some(fs) = s3_fs.take() { + fs.flush_background_tasks().await; + } + + Ok(()) + } + .into() + } + + fn runtime_handle(&self) -> &'static tokio_util::task::LocalPoolHandle { + &base_rt::USER_WORKER_RT + } + + async fn network_sender(&self) -> mpsc::UnboundedSender { + self.cx.lock().await.get_network_sender() + } +} diff --git a/crates/base/src/worker/mod.rs b/crates/base/src/worker/mod.rs new file mode 100644 index 000000000..98701a30f --- /dev/null +++ b/crates/base/src/worker/mod.rs @@ -0,0 +1,27 @@ +pub mod driver; +pub mod pool; +pub mod supervisor; +pub mod utils; + +mod worker_inner; + +mod termination_token; +mod worker_surface_creation; + +pub use termination_token::TerminationToken; + +pub(crate) use worker_inner::DuplexStreamEntry; +pub(crate) use worker_inner::Worker; +pub(crate) use worker_inner::WorkerBuilder; +pub(crate) use worker_inner::WorkerCx; + +pub use worker_inner::WorkerSurface; + +pub use worker_surface_creation::EventWorkerSurface; +pub use worker_surface_creation::MainWorkerSurface; + +pub use worker_surface_creation::EventWorkerSurfaceBuilder; +pub use worker_surface_creation::MainWorkerSurfaceBuilder; +pub use worker_surface_creation::WorkerSurfaceBuilder; + +pub use pool::create_user_worker_pool; diff --git a/crates/base/src/rt_worker/worker_pool.rs b/crates/base/src/worker/pool.rs similarity index 81% rename from crates/base/src/rt_worker/worker_pool.rs rename to crates/base/src/worker/pool.rs index 39c877e5d..45d6b2245 100644 --- a/crates/base/src/rt_worker/worker_pool.rs +++ b/crates/base/src/worker/pool.rs @@ -1,7 +1,10 @@ use crate::inspector_server::Inspector; -use crate::rt_worker::worker_ctx::{create_worker, send_user_worker_request}; use crate::server::ServerFlags; +use crate::worker::WorkerSurfaceBuilder; + use anyhow::{anyhow, bail, Context, Error}; +use deno_config::JsxImportSourceConfig; +use either::Either::Left; use enum_as_inner::EnumAsInner; use event_worker::events::WorkerEventWithMetadata; use http_v02::Request; @@ -16,6 +19,7 @@ use sb_workers::context::{ use sb_workers::errors::WorkerError; use std::collections::{HashMap, HashSet}; use std::convert::Infallible; +use std::future::pending; use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -26,7 +30,8 @@ use tokio::sync::{mpsc, Notify, OwnedSemaphorePermit, Semaphore, TryAcquireError use tokio_util::sync::CancellationToken; use uuid::Uuid; -use super::worker_ctx::TerminationToken; +use super::termination_token::TerminationToken; +use super::utils::send_user_worker_request; #[derive(Debug, Clone, Copy, EnumAsInner)] pub enum SupervisorPolicy { @@ -425,21 +430,24 @@ impl WorkerPool { worker_options.conf = WorkerRuntimeOpts::UserWorker(user_worker_rt_opts); - match create_worker( - flags, - (worker_options, supervisor_policy, termination_token.clone()), - inspector, - ) - .await - { - Ok(ctx) => { + let mut builder = WorkerSurfaceBuilder::new() + .init_opts(worker_options) + .policy(supervisor_policy) + .sever_flags(Left(flags)); + + builder + .set_termination_token(termination_token.clone()) + .set_inspector(inspector); + + match builder.build().await { + Ok(surface) => { let profile = UserWorkerProfile { - worker_request_msg_tx: ctx.msg_tx, + worker_request_msg_tx: surface.msg_tx, timing_tx_pair: (req_start_timing_tx, req_end_timing_tx), service_path, permit: permit.map(Arc::new), status: status.clone(), - exit: ctx.exit, + exit: surface.exit, cancel, }; @@ -657,3 +665,108 @@ impl WorkerPool { } } } + +pub async fn create_user_worker_pool( + flags: Arc, + policy: WorkerPoolPolicy, + worker_event_sender: Option>, + termination_token: Option, + static_patterns: Vec, + inspector: Option, + jsx: Option, +) -> Result<(SharedMetricSource, mpsc::UnboundedSender), Error> { + let metric_src = SharedMetricSource::default(); + let (user_worker_msgs_tx, mut user_worker_msgs_rx) = + mpsc::unbounded_channel::(); + + let user_worker_msgs_tx_clone = user_worker_msgs_tx.clone(); + + let _handle: tokio::task::JoinHandle> = tokio::spawn({ + let metric_src_inner = metric_src.clone(); + async move { + let token = termination_token.as_ref(); + let mut termination_requested = false; + let mut worker_pool = WorkerPool::new( + flags, + policy, + metric_src_inner, + worker_event_sender, + user_worker_msgs_tx_clone, + inspector, + ); + + // Note: Keep this loop non-blocking. Spawn a task to run blocking calls. + // Handle errors within tasks and log them - do not bubble up errors. + loop { + tokio::select! { + _ = async { + if let Some(token) = token { + token.inbound.cancelled().await; + } else { + pending::<()>().await; + } + }, if !termination_requested => { + termination_requested = true; + + if worker_pool.user_workers.is_empty() { + if let Some(token) = token { + token.outbound.cancel(); + } + + break; + } + } + + msg = user_worker_msgs_rx.recv() => { + match msg { + None => break, + Some(UserWorkerMsgs::Create(worker_options, tx)) => { + worker_pool.create_user_worker(WorkerContextInitOpts { + static_patterns: static_patterns.clone(), + maybe_jsx_import_source_config: { + if worker_options.maybe_jsx_import_source_config.is_some() { + worker_options.maybe_jsx_import_source_config + } else { + jsx.clone() + } + }, + ..worker_options + }, tx, token.map(TerminationToken::child_token)); + } + + Some(UserWorkerMsgs::Created(key, profile)) => { + worker_pool.add_user_worker(key, profile); + } + + Some(UserWorkerMsgs::SendRequest(key, req, res_tx, conn_token)) => { + worker_pool.send_request(&key, req, res_tx, conn_token); + } + + Some(UserWorkerMsgs::Idle(key)) => { + worker_pool.idle(&key); + } + + Some(UserWorkerMsgs::Shutdown(key)) => { + worker_pool.shutdown(&key); + + if termination_requested && worker_pool.user_workers.is_empty() { + if let Some(token) = token { + token.outbound.cancel(); + } + + break; + } + } + } + } + } + } + + worker_pool.worker_event_sender.take(); + + Ok(()) + } + }); + + Ok((metric_src, user_worker_msgs_tx)) +} diff --git a/crates/base/src/worker/supervisor/mod.rs b/crates/base/src/worker/supervisor/mod.rs new file mode 100644 index 000000000..cc933ecd7 --- /dev/null +++ b/crates/base/src/worker/supervisor/mod.rs @@ -0,0 +1,419 @@ +use std::{future::pending, sync::Arc, time::Duration}; + +use anyhow::anyhow; +use base_mem_check::MemCheckState; +use cpu_timer::{CPUAlarmVal, CPUTimer}; +use deno_core::{v8, InspectorSessionProxy, LocalInspectorSession}; +use enum_as_inner::EnumAsInner; +use event_worker::events::{ShutdownEvent, WorkerEvents, WorkerMemoryUsed}; +use futures_util::{pin_mut, task::AtomicWaker}; +use sb_core::PromiseMetrics; +use sb_workers::context::{Timing, UserWorkerMsgs, UserWorkerRuntimeOpts}; +use tokio::sync::{ + mpsc::{self, UnboundedReceiver}, + oneshot, Mutex, +}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::{ + deno_runtime::{DenoRuntime, RuntimeState}, + server::ServerFlags, + utils::units::percentage_value, +}; + +use super::{pool::SupervisorPolicy, termination_token::TerminationToken}; + +pub mod strategy_per_request; +pub mod strategy_per_worker; + +pub mod v8_handler; + +pub use v8_handler::*; + +#[repr(C)] +pub struct IsolateMemoryStats { + pub used_heap_size: usize, + pub external_memory: usize, +} + +#[derive(Clone, Copy)] +pub struct CPUTimerParam { + soft_limit_ms: u64, + hard_limit_ms: u64, +} + +impl CPUTimerParam { + pub fn new(soft_limit_ms: u64, hard_limit_ms: u64) -> Self { + Self { + soft_limit_ms, + hard_limit_ms, + } + } + + pub fn get_cpu_timer( + &self, + policy: SupervisorPolicy, + ) -> Option<(CPUTimer, UnboundedReceiver<()>)> { + let (cpu_alarms_tx, cpu_alarms_rx) = mpsc::unbounded_channel::<()>(); + + if self.is_disabled() { + return None; + } + + Some(( + CPUTimer::start( + if policy.is_per_worker() { + self.soft_limit_ms + } else { + self.hard_limit_ms + }, + if policy.is_per_request() { + 0 + } else { + self.hard_limit_ms + }, + CPUAlarmVal { cpu_alarms_tx }, + ) + .ok()?, + cpu_alarms_rx, + )) + } + + pub fn limits(&self) -> (u64, u64) { + (self.soft_limit_ms, self.hard_limit_ms) + } + + pub fn is_disabled(&self) -> bool { + self.soft_limit_ms == 0 && self.hard_limit_ms == 0 + } +} + +pub struct Tokens { + pub termination: Option, + pub supervise: CancellationToken, +} + +pub struct Arguments { + pub key: Uuid, + pub runtime_opts: UserWorkerRuntimeOpts, + pub cpu_timer: Option<(CPUTimer, mpsc::UnboundedReceiver<()>)>, + pub cpu_usage_metrics_rx: Option>, + pub cpu_timer_param: CPUTimerParam, + pub supervisor_policy: SupervisorPolicy, + pub runtime_state: Arc, + pub promise_metrics: PromiseMetrics, + pub timing: Option, + pub memory_limit_rx: mpsc::UnboundedReceiver<()>, + pub pool_msg_tx: Option>, + pub isolate_memory_usage_tx: oneshot::Sender, + pub thread_safe_handle: v8::IsolateHandle, + pub waker: Arc, + pub tokens: Tokens, + pub flags: Arc, +} + +pub struct CPUUsage { + pub accumulated: i64, + pub diff: i64, +} + +#[derive(EnumAsInner)] +pub enum CPUUsageMetrics { + Enter(std::thread::ThreadId), + Leave(CPUUsage), +} + +async fn wait_cpu_alarm(maybe_alarm: Option<&mut UnboundedReceiver<()>>) -> Option<()> { + match maybe_alarm { + Some(alarm) => Some(alarm.recv().await?), + None => None, + } +} + +async fn create_wall_clock_beforeunload_alert(wall_clock_limit_ms: u64, pct: Option) { + let dur = pct + .and_then(|it| percentage_value(wall_clock_limit_ms, it)) + .map(Duration::from_millis); + + if let Some(dur) = dur { + tokio::time::sleep(dur).await; + } else { + pending::<()>().await; + unreachable!() + } +} + +#[allow(clippy::too_many_arguments)] +pub fn create_supervisor( + key: Uuid, + runtime: &mut DenoRuntime, + policy: SupervisorPolicy, + termination_event_tx: oneshot::Sender, + pool_msg_tx: Option>, + cpu_usage_metrics_rx: Option>, + cancel: Option, + timing: Option, + termination_token: Option, + flags: Arc, +) -> Result<(Option, CancellationToken), anyhow::Error> { + let (memory_limit_tx, memory_limit_rx) = mpsc::unbounded_channel(); + let (waker, thread_safe_handle) = { + let js_runtime = &mut runtime.js_runtime; + ( + js_runtime.op_state().borrow().waker.clone(), + js_runtime.v8_isolate().thread_safe_handle(), + ) + }; + + // we assert supervisor is only run for user workers + let conf = runtime.conf.as_user_worker().unwrap().clone(); + let mem_check_state = runtime.mem_check_state(); + let termination_request_token = runtime.termination_request_token.clone(); + let runtime_drop_token = runtime.drop_token.clone(); + + let giveup_process_requests_token = cancel.clone(); + let supervise_cancel_token = CancellationToken::new(); + let tokens = Tokens { + termination: termination_token.clone(), + supervise: supervise_cancel_token.clone(), + }; + + let maybe_inspector_params = runtime.inspector().map(|_| { + ( + runtime + .js_runtime + .inspector() + .borrow_mut() + .get_session_sender(), + runtime.runtime_state.clone(), + ) + }); + + let send_memory_limit_fn = move |kind: &'static str| { + log::debug!("memory limit triggered: isolate: {:?}, kind: {}", key, kind); + + if memory_limit_tx.send(()).is_err() { + log::error!( + "failed to send memory limit reached notification(isolate may already be terminating): isolate: {:?}, kind: {}", + key, kind + ); + } + }; + + runtime.add_memory_limit_callback({ + let send_fn = send_memory_limit_fn.clone(); + move |_| { + send_fn("mem_check"); + } + }); + + runtime.js_runtime.add_near_heap_limit_callback({ + let send_fn = send_memory_limit_fn; + move |current, _| { + send_fn("v8"); + + // give an allowance on current limit (until the isolate is + // terminated) we do this so that oom won't end up killing the + // edge-runtime process + current * (conf.low_memory_multiplier as usize) + } + }); + + // Note: CPU timer must be started in the same thread as the worker runtime + + let cpu_timer_param = + CPUTimerParam::new(conf.cpu_time_soft_limit_ms, conf.cpu_time_hard_limit_ms); + + let (maybe_cpu_timer, maybe_cpu_alarms_rx) = cpu_timer_param.get_cpu_timer(policy).unzip(); + + drop({ + let _rt_guard = base_rt::SUPERVISOR_RT.enter(); + let maybe_cpu_timer_inner = maybe_cpu_timer.clone(); + let supervise_cancel_token_inner = supervise_cancel_token.clone(); + let runtime_state = runtime.runtime_state.clone(); + let promise_metrics = runtime.promise_metrics(); + + tokio::spawn(async move { + let (isolate_memory_usage_tx, isolate_memory_usage_rx) = + oneshot::channel::(); + + let args = Arguments { + key, + runtime_opts: conf.clone(), + cpu_timer: maybe_cpu_timer_inner.zip(maybe_cpu_alarms_rx), + cpu_usage_metrics_rx, + cpu_timer_param, + supervisor_policy: policy, + runtime_state, + promise_metrics, + timing, + memory_limit_rx, + pool_msg_tx, + isolate_memory_usage_tx, + thread_safe_handle, + waker: waker.clone(), + tokens, + flags, + }; + + let (reason, cpu_usage_ms) = { + match policy { + SupervisorPolicy::PerWorker => strategy_per_worker::supervise(args).await, + SupervisorPolicy::PerRequest { oneshot, .. } => { + strategy_per_request::supervise(args, oneshot).await + } + } + }; + + // NOTE: Sending a signal to the pooler that it is the user worker going + // disposed down and will not accept awaiting subsequent requests, so + // they must be re-polled again. + if let Some(cancel) = giveup_process_requests_token.as_ref() { + cancel.cancel(); + } + + if let Some((session_tx, state)) = maybe_inspector_params { + use deno_core::futures::channel::mpsc; + use deno_core::serde_json::Value; + + let termination_request_token = termination_request_token.clone(); + + base_rt::SUPERVISOR_RT + .spawn_blocking(move || { + let wait_inspector_disconnect_fut = async move { + let ls = tokio::task::LocalSet::new(); + ls.run_until(async move { + if state.is_terminated() || termination_request_token.is_cancelled() + { + return; + } + + termination_request_token.cancel(); + + if state.is_found_inspector_session() { + return; + } + + let (outbound_tx, outbound_rx) = mpsc::unbounded(); + let (inbound_tx, inbound_rx) = mpsc::unbounded(); + + if session_tx + .unbounded_send(InspectorSessionProxy { + tx: outbound_tx, + rx: inbound_rx, + }) + .is_err() + { + return; + } + + let session = Arc::new(Mutex::new(LocalInspectorSession::new( + inbound_tx, + outbound_rx, + ))); + + let send_msg_fn = { + |msg| { + let state = state.clone(); + let session = session.clone(); + async move { + let mut session = session.lock().await; + let mut int = + tokio::time::interval(Duration::from_millis(61)); + + let fut = session.post_message(msg, None::); + + pin_mut!(fut); + + loop { + tokio::select! { + _ = int.tick() => { + if state.is_terminated() { + break + } + } + + res = &mut fut => { + res.unwrap(); + break + } + } + } + } + } + }; + + send_msg_fn("Debugger.enable").await; + send_msg_fn("Runtime.runIfWaitingForDebugger").await; + }) + .await; + }; + + base_rt::SUPERVISOR_RT.block_on(wait_inspector_disconnect_fut); + }) + .await + .unwrap(); + } + + // NOTE: If we issue a hard CPU time limit, It's OK because it is + // still possible the worker's context is in the v8 event loop. The + // interrupt callback would be invoked from the V8 engine + // gracefully. But some case doesn't. + // + // Such as the worker going to a retired state due to the soft CPU + // time limit but not hitting the hard CPU time limit. In this case, + // we must wake up the worker's event loop manually. Otherwise, the + // supervisor has to wait until the wall clock future that we placed + // out on the runtime side is times out. + waker.wake(); + + let memory_report = tokio::select! { + report = isolate_memory_usage_rx => report.map_err(anyhow::Error::from), + _ = runtime_drop_token.cancelled() => Err(anyhow!("termination requested")) + }; + + let memory_used = match memory_report { + Ok(v) => WorkerMemoryUsed { + total: v.used_heap_size + v.external_memory, + heap: v.used_heap_size, + external: v.external_memory, + mem_check_captured: tokio::task::spawn_blocking(move || { + *mem_check_state.read().unwrap() + }) + .await + .unwrap(), + }, + + Err(_) => { + if !supervise_cancel_token_inner.is_cancelled() { + log::warn!("isolate memory usage sender dropped"); + } + + WorkerMemoryUsed { + total: 0, + heap: 0, + external: 0, + mem_check_captured: MemCheckState::default(), + } + } + }; + + if !termination_request_token.is_cancelled() { + termination_request_token.cancel(); + waker.wake(); + } + + // send termination reason + let termination_event = WorkerEvents::Shutdown(ShutdownEvent { + reason, + memory_used, + cpu_time_used: cpu_usage_ms as usize, + }); + + let _ = termination_event_tx.send(termination_event); + }) + }); + + Ok((maybe_cpu_timer, supervise_cancel_token)) +} diff --git a/crates/base/src/rt_worker/supervisor/strategy_per_request.rs b/crates/base/src/worker/supervisor/strategy_per_request.rs similarity index 84% rename from crates/base/src/rt_worker/supervisor/strategy_per_request.rs rename to crates/base/src/worker/supervisor/strategy_per_request.rs index 6383edc2b..c6ea63c49 100644 --- a/crates/base/src/rt_worker/supervisor/strategy_per_request.rs +++ b/crates/base/src/worker/supervisor/strategy_per_request.rs @@ -4,14 +4,16 @@ use std::{future::pending, sync::atomic::Ordering, time::Duration}; use std::thread::ThreadId; use event_worker::events::ShutdownReason; -use log::error; use sb_workers::context::{Timing, TimingStatus, UserWorkerMsgs}; use tokio::time::Instant; -use crate::rt_worker::supervisor::{ - create_wall_clock_beforeunload_alert, v8_handle_early_retire, v8_handle_termination, - v8_handle_wall_clock_beforeunload, wait_cpu_alarm, CPUUsage, CPUUsageMetrics, Tokens, - V8HandleTerminationData, +use crate::{ + deno_runtime::WillTerminateReason, + worker::supervisor::{ + create_wall_clock_beforeunload_alert, v8_handle_beforeunload, v8_handle_early_retire, + v8_handle_termination, wait_cpu_alarm, CPUUsage, CPUUsageMetrics, Tokens, + V8HandleBeforeunloadData, V8HandleTerminationData, + }, }; use super::Arguments; @@ -47,11 +49,11 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) let (_, hard_limit_ms) = cpu_timer_param.limits(); let _guard = scopeguard::guard(is_retired, |v| { + v.raise(); + if thread_safe_handle.request_interrupt(v8_handle_early_retire, std::ptr::null_mut()) { waker.wake(); } - - v.raise(); }); #[cfg(debug_assertions)] @@ -118,7 +120,7 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) if !cpu_timer_param.is_disabled() { if let Some(Err(err)) = cpu_timer.as_ref().map(|it| it.reset()) { - error!("can't reset cpu timer: {}", err); + log::error!("can't reset cpu timer: {}", err); } } } @@ -132,12 +134,12 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) if !cpu_timer_param.is_disabled() { if cpu_usage_ms >= hard_limit_ms as i64 { - error!("CPU time limit reached: isolate: {:?}", key); + log::error!("CPU time limit reached: isolate: {:?}", key); complete_reason = Some(ShutdownReason::CPUTime); } if let Some(Err(err)) = cpu_timer.as_ref().map(|it| it.reset()) { - error!("can't reset cpu timer: {}", err); + log::error!("can't reset cpu timer: {}", err); } } } @@ -146,7 +148,7 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) Some(_) = wait_cpu_alarm(cpu_alarms_rx.as_mut()) => { if is_worker_entered && req_start_ack { - error!("CPU time limit reached: isolate: {:?}", key); + log::error!("CPU time limit reached: isolate: {:?}", key); complete_reason = Some(ShutdownReason::CPUTime); } } @@ -160,7 +162,7 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) if let Some(cpu_timer) = cpu_timer.as_ref() { if let Err(ex) = cpu_timer.reset() { - error!("cannot reset the cpu timer: {}", ex); + log::error!("cannot reset the cpu timer: {}", ex); } } @@ -187,7 +189,7 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) continue; } else { - error!("wall clock duraiton reached: isolate: {:?}", key); + log::error!("wall clock duraiton reached: isolate: {:?}", key); complete_reason = Some(ShutdownReason::WallClockTime); } } @@ -195,18 +197,23 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) _ = &mut wall_clock_beforeunload_alert, if !is_wall_clock_limit_disabled && !is_wall_clock_beforeunload_armed => { - if thread_safe_handle.request_interrupt( - v8_handle_wall_clock_beforeunload, - std::ptr::null_mut() - ) { + let data_ptr_mut = Box::into_raw(Box::new(V8HandleBeforeunloadData { + reason: WillTerminateReason::WallClock + })); + + if thread_safe_handle + .request_interrupt(v8_handle_beforeunload, data_ptr_mut as *mut _) + { waker.wake(); + } else { + drop(unsafe { Box::from_raw(data_ptr_mut)}); } is_wall_clock_beforeunload_armed = true; } Some(_) = memory_limit_rx.recv() => { - error!("memory limit reached for the worker: isolate: {:?}", key); + log::error!("memory limit reached for the worker: isolate: {:?}", key); complete_reason = Some(ShutdownReason::Memory); } } @@ -220,7 +227,7 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) if let Some(tx) = pool_msg_tx.clone() { if tx.send(UserWorkerMsgs::Idle(key)).is_err() { - error!("failed to send idle msg to pool: {:?}", key); + log::error!("failed to send idle msg to pool: {:?}", key); } } @@ -234,7 +241,7 @@ pub async fn supervise(args: Arguments, oneshot: bool) -> (ShutdownReason, i64) })); if !thread_safe_handle - .request_interrupt(v8_handle_termination, data_ptr_mut as *mut std::ffi::c_void) + .request_interrupt(v8_handle_termination, data_ptr_mut as *mut _) { drop(unsafe { Box::from_raw(data_ptr_mut) }); } diff --git a/crates/base/src/rt_worker/supervisor/strategy_per_worker.rs b/crates/base/src/worker/supervisor/strategy_per_worker.rs similarity index 80% rename from crates/base/src/rt_worker/supervisor/strategy_per_worker.rs rename to crates/base/src/worker/supervisor/strategy_per_worker.rs index e349ef6ad..a56afea81 100644 --- a/crates/base/src/rt_worker/supervisor/strategy_per_worker.rs +++ b/crates/base/src/worker/supervisor/strategy_per_worker.rs @@ -8,10 +8,13 @@ use log::{error, info}; use sb_workers::context::{Timing, TimingStatus, UserWorkerMsgs}; use tokio_util::sync::CancellationToken; -use crate::rt_worker::supervisor::{ - create_wall_clock_beforeunload_alert, v8_handle_early_drop_beforeunload, - v8_handle_early_retire, v8_handle_wall_clock_beforeunload, wait_cpu_alarm, CPUUsage, Tokens, - V8HandleEarlyRetireData, +use crate::{ + deno_runtime::WillTerminateReason, + worker::supervisor::{ + create_wall_clock_beforeunload_alert, v8_handle_beforeunload, v8_handle_drain, + v8_handle_early_drop_beforeunload, v8_handle_early_retire, wait_cpu_alarm, CPUUsage, + Tokens, V8HandleBeforeunloadData, V8HandleEarlyDropData, + }, }; use super::{v8_handle_termination, Arguments, CPUUsageMetrics, V8HandleTerminationData}; @@ -20,6 +23,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { let Arguments { key, runtime_opts, + runtime_state, promise_metrics, timing, mut memory_limit_rx, @@ -46,7 +50,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { let (cpu_timer, mut cpu_alarms_rx) = cpu_timer.unzip(); let (soft_limit_ms, hard_limit_ms) = cpu_timer_param.limits(); - let guard = scopeguard::guard(is_retired, |v| { + let guard = scopeguard::guard(is_retired.clone(), |v| { v.raise(); }); @@ -60,7 +64,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { let mut is_wall_clock_beforeunload_armed = false; let mut is_cpu_time_soft_limit_reached = false; let mut is_waiting_for_termination = false; - let mut have_all_reqs_been_acknowledged = false; + let mut have_all_reqs_been_acknowledged: bool; let mut cpu_usage_metrics_rx = cpu_usage_metrics_rx.unwrap(); let mut cpu_usage_ms = 0i64; @@ -91,13 +95,13 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { ); let early_retire_fn = || { - if thread_safe_handle.request_interrupt(v8_handle_early_retire, std::ptr::null_mut()) { - waker.wake(); - } - // we should raise a retire signal because subsequent incoming requests are unlikely to get // enough wall clock time or cpu time guard.raise(); + + if thread_safe_handle.request_interrupt(v8_handle_early_retire, std::ptr::null_mut()) { + waker.wake(); + } }; let early_drop_token = CancellationToken::new(); @@ -106,24 +110,30 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { let mut dispatch_early_drop_beforeunload_fn = Some({ let token = early_drop_token.clone(); || { - let data_ptr_mut = Box::into_raw(Box::new(V8HandleEarlyRetireData { token })); + let data_ptr_mut = Box::into_raw(Box::new(V8HandleEarlyDropData { token })); if !thread_safe_handle.request_interrupt( v8_handle_early_drop_beforeunload, data_ptr_mut as *mut std::ffi::c_void, ) { - drop(unsafe { Box::from_raw(data_ptr_mut) }); + unsafe { Box::from_raw(data_ptr_mut) }.token.cancel(); } else { waker.wake(); } } }); + let mut dispatch_drain_fn = Some(|| { + if thread_safe_handle.request_interrupt(v8_handle_drain, std::ptr::null_mut()) { + waker.wake(); + } + }); + let terminate_fn = { let thread_safe_handle = thread_safe_handle.clone(); - move || { + move |should_terminate: bool| { let data_ptr_mut = Box::into_raw(Box::new(V8HandleTerminationData { - should_terminate: true, + should_terminate, isolate_memory_usage_tx: Some(isolate_memory_usage_tx), })); @@ -152,10 +162,29 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { } }, if !is_waiting_for_termination => { is_waiting_for_termination = true; + + early_retire_fn(); + if let Some(func) = dispatch_drain_fn.take() { + func(); + } if promise_metrics.have_all_promises_been_resolved() { if let Some(func) = dispatch_early_drop_beforeunload_fn.take() { func(); } + } else { + let is_retired = is_retired.clone(); + let waker = waker.clone(); + + drop(tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + loop { + interval.tick().await; + waker.wake(); + if is_retired.is_raised() { + break; + } + } + })); } } @@ -185,6 +214,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { is_worker_entered = false; cpu_usage_ms = accumulated / 1_000_000; + have_all_reqs_been_acknowledged = req_ack_count == demand.load(Ordering::Acquire); if !cpu_timer_param.is_disabled() { if cpu_usage_ms >= hard_limit_ms as i64 { @@ -195,7 +225,6 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { error!("CPU time soft limit reached: isolate: {:?}", key); is_cpu_time_soft_limit_reached = true; - have_all_reqs_been_acknowledged = req_ack_count == demand.load(Ordering::Acquire); if have_all_reqs_been_acknowledged && promise_metrics.have_all_promises_been_resolved() @@ -204,14 +233,15 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { func(); } } + } + } - } else if is_cpu_time_soft_limit_reached + if (is_cpu_time_soft_limit_reached || is_waiting_for_termination) && have_all_reqs_been_acknowledged && promise_metrics.have_all_promises_been_resolved() - { - if let Some(func) = dispatch_early_drop_beforeunload_fn.take() { - func(); - } + { + if let Some(func) = dispatch_early_drop_beforeunload_fn.take() { + func(); } } } @@ -253,6 +283,12 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { } } + if have_all_reqs_been_acknowledged && guard.is_raised() { + if let Some(func) = dispatch_drain_fn.take() { + func(); + } + } + if !is_cpu_time_soft_limit_reached || !have_all_reqs_been_acknowledged || !promise_metrics.have_all_promises_been_resolved() @@ -284,11 +320,16 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { _ = &mut wall_clock_beforeunload_alert, if !is_wall_clock_limit_disabled && !is_wall_clock_beforeunload_armed => { - if thread_safe_handle.request_interrupt( - v8_handle_wall_clock_beforeunload, - std::ptr::null_mut() - ) { + let data_ptr_mut = Box::into_raw(Box::new(V8HandleBeforeunloadData { + reason: WillTerminateReason::WallClock + })); + + if thread_safe_handle + .request_interrupt(v8_handle_beforeunload, data_ptr_mut as *mut _) + { waker.wake(); + } else { + drop(unsafe { Box::from_raw(data_ptr_mut) }); } is_wall_clock_beforeunload_armed = true; @@ -307,7 +348,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { match complete_reason.take() { Some(ShutdownReason::EarlyDrop) => { - terminate_fn(); + terminate_fn(runtime_state.is_evaluating_mod()); return ( if is_waiting_for_termination { ShutdownReason::TerminationRequested @@ -319,7 +360,7 @@ pub async fn supervise(args: Arguments) -> (ShutdownReason, i64) { } Some(result) => { - terminate_fn(); + terminate_fn(true); return (result, cpu_usage_ms); } None => continue, diff --git a/crates/base/src/worker/supervisor/v8_handler.rs b/crates/base/src/worker/supervisor/v8_handler.rs new file mode 100644 index 000000000..2c536f03e --- /dev/null +++ b/crates/base/src/worker/supervisor/v8_handler.rs @@ -0,0 +1,115 @@ +//! # Warning +//! +//! Do not directly call v8 functions that are likely to execute deno ops within the interrupted +//! context. This may cause a panic due to the reentrancy check. +//! +//! If you need to call a v8 function that has side effects that might be calling the deno ops, you +//! can safely call it through [`V8TaskSpawner`]. + +use deno_core::{v8, JsRuntime, V8TaskSpawner}; +use tokio::sync::oneshot; +use tokio_util::sync::CancellationToken; +use tracing::{debug, instrument}; + +use crate::deno_runtime::{MaybeDenoRuntime, WillTerminateReason}; + +use super::IsolateMemoryStats; + +#[repr(C)] +pub struct V8HandleTerminationData { + pub should_terminate: bool, + pub isolate_memory_usage_tx: Option>, +} + +pub extern "C" fn v8_handle_termination(isolate: &mut v8::Isolate, data: *mut std::ffi::c_void) { + let mut data = unsafe { Box::from_raw(data as *mut V8HandleTerminationData) }; + + // log memory usage + let mut heap_stats = v8::HeapStatistics::default(); + + isolate.get_heap_statistics(&mut heap_stats); + + let usage = IsolateMemoryStats { + used_heap_size: heap_stats.used_heap_size(), + external_memory: heap_stats.external_memory(), + }; + + if let Some(usage_tx) = data.isolate_memory_usage_tx.take() { + if usage_tx.send(usage).is_err() { + log::error!("failed to send isolate memory usage - receiver may have been dropped"); + } + } + + if data.should_terminate { + isolate.terminate_execution(); + } +} + +#[repr(C)] +pub struct V8HandleBeforeunloadData { + pub reason: WillTerminateReason, +} + +pub extern "C" fn v8_handle_beforeunload(isolate: &mut v8::Isolate, data: *mut std::ffi::c_void) { + let data = unsafe { Box::from_raw(data as *mut V8HandleBeforeunloadData) }; + + JsRuntime::op_state_from(isolate) + .borrow() + .borrow::() + .spawn(move |scope| { + if let Err(err) = + MaybeDenoRuntime::<()>::Isolate(scope).dispatch_beforeunload_event(data.reason) + { + log::error!( + "found an error while dispatching the beforeunload event: {}", + err + ); + } + }); +} + +#[repr(C)] +pub struct V8HandleEarlyDropData { + pub token: CancellationToken, +} + +pub extern "C" fn v8_handle_early_drop_beforeunload( + isolate: &mut v8::Isolate, + data: *mut std::ffi::c_void, +) { + let data = unsafe { Box::from_raw(data as *mut V8HandleEarlyDropData) }; + + JsRuntime::op_state_from(isolate) + .borrow() + .borrow::() + .spawn(move |scope| { + if let Err(err) = MaybeDenoRuntime::<()>::Isolate(scope) + .dispatch_beforeunload_event(WillTerminateReason::EarlyDrop) + { + log::error!( + "found an error while dispatching the beforeunload event: {}", + err + ); + } else { + data.token.cancel(); + } + }); +} + +#[instrument(level = "debug", skip_all)] +pub extern "C" fn v8_handle_early_retire(isolate: &mut v8::Isolate, _data: *mut std::ffi::c_void) { + isolate.low_memory_notification(); + debug!("sent low mem notification"); +} + +#[instrument(level = "debug", skip_all)] +pub extern "C" fn v8_handle_drain(isolate: &mut v8::Isolate, _data: *mut std::ffi::c_void) { + JsRuntime::op_state_from(isolate) + .borrow() + .borrow::() + .spawn(move |scope| { + if let Err(err) = MaybeDenoRuntime::<()>::Isolate(scope).dispatch_drain_event() { + log::error!("found an error while dispatching the drain event: {}", err); + } + }); +} diff --git a/crates/base/src/worker/termination_token.rs b/crates/base/src/worker/termination_token.rs new file mode 100644 index 000000000..7f9427cb4 --- /dev/null +++ b/crates/base/src/worker/termination_token.rs @@ -0,0 +1,48 @@ +use tokio_util::sync::CancellationToken; + +#[derive(Clone)] +pub struct TerminationToken { + pub inbound: CancellationToken, + pub outbound: CancellationToken, +} + +impl std::fmt::Debug for TerminationToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TerminationToken").finish() + } +} + +impl Default for TerminationToken { + fn default() -> Self { + Self::new() + } +} + +impl TerminationToken { + pub fn new() -> Self { + Self { + inbound: CancellationToken::default(), + outbound: CancellationToken::default(), + } + } + + pub fn child_token(&self) -> Self { + Self { + inbound: self.inbound.child_token(), + outbound: self.outbound.child_token(), + } + } + + pub fn cancel(&self) { + self.inbound.cancel(); + } + + pub async fn cancel_and_wait(&self) { + if self.outbound.is_cancelled() { + return; + } + + self.cancel(); + self.outbound.cancelled().await; + } +} diff --git a/crates/base/src/worker/utils.rs b/crates/base/src/worker/utils.rs new file mode 100644 index 000000000..9f39fef63 --- /dev/null +++ b/crates/base/src/worker/utils.rs @@ -0,0 +1,80 @@ +use anyhow::{anyhow, bail}; +use event_worker::events::{EventMetadata, WorkerEventWithMetadata, WorkerEvents}; +use hyper_v014::{Body, Request, Response}; +use sb_workers::{ + context::{WorkerExit, WorkerRequestMsg, WorkerRuntimeOpts}, + errors::WorkerError, +}; +use tokio::sync::{mpsc, oneshot}; +use tokio_util::sync::CancellationToken; + +pub fn get_event_metadata(conf: &WorkerRuntimeOpts) -> EventMetadata { + let mut event_metadata = EventMetadata { + service_path: None, + execution_id: None, + }; + if conf.is_user_worker() { + let conf = conf.as_user_worker().unwrap(); + event_metadata = EventMetadata { + service_path: conf.service_path.clone(), + execution_id: conf.key, + }; + } + + event_metadata +} + +pub async fn send_user_worker_request( + worker_request_msg_tx: mpsc::UnboundedSender, + req: Request, + cancel: CancellationToken, + exit: WorkerExit, + conn_token: Option, +) -> Result, anyhow::Error> { + let (res_tx, res_rx) = oneshot::channel::, hyper_v014::Error>>(); + let msg = WorkerRequestMsg { + req, + res_tx, + conn_token, + }; + + // send the message to worker + worker_request_msg_tx.send(msg)?; + + // wait for the response back from the worker + let res = tokio::select! { + () = cancel.cancelled() => { + bail!(exit + .error() + .await + .unwrap_or(anyhow!(WorkerError::RequestCancelledBySupervisor))) + } + + res = res_rx => res, + }?; + + match res { + Ok(v) => { + // send the response back to the caller + Ok(v) + } + + Err(err) => { + if let Some(actual_error) = exit.error().await { + return Err(actual_error); + } + + Err(err.into()) + } + } +} + +pub fn send_event_if_event_worker_available( + maybe_event_worker: Option<&mpsc::UnboundedSender>, + event: WorkerEvents, + metadata: EventMetadata, +) { + if let Some(event_worker) = maybe_event_worker { + let _ = event_worker.send(WorkerEventWithMetadata { event, metadata }); + } +} diff --git a/crates/base/src/worker/worker_inner.rs b/crates/base/src/worker/worker_inner.rs new file mode 100644 index 000000000..af42fd717 --- /dev/null +++ b/crates/base/src/worker/worker_inner.rs @@ -0,0 +1,328 @@ +use crate::deno_runtime::DenoRuntime; +use crate::inspector_server::Inspector; +use crate::server::ServerFlags; +use crate::worker::utils::{get_event_metadata, send_event_if_event_worker_available}; + +use anyhow::Error; +use base_rt::error::CloneableError; +use deno_core::unsync::MaskFutureAsSend; +use event_worker::events::{ + EventMetadata, ShutdownEvent, UncaughtExceptionEvent, WorkerEventWithMetadata, WorkerEvents, +}; +use futures_util::FutureExt; +use log::{debug, error}; +use sb_core::{MetricSource, RuntimeMetricSource, WorkerMetricSource}; +use sb_workers::context::{ + UserWorkerMsgs, WorkerContextInitOpts, WorkerExit, WorkerExitStatus, WorkerKind, + WorkerRequestMsg, +}; +use std::future::ready; +use std::sync::Arc; +use tokio::io; +use tokio::sync::{mpsc, oneshot}; +use tokio::time::Instant; +use tokio_util::sync::CancellationToken; +use tracing::{debug_span, Instrument}; +use uuid::Uuid; + +use super::driver::{WorkerDriver, WorkerDriverImpl}; +use super::pool::SupervisorPolicy; +use super::termination_token::TerminationToken; + +pub type DuplexStreamEntry = (io::DuplexStream, Option); + +pub struct WorkerCx { + pub flags: Arc, + pub worker_boot_start_time: Instant, + pub events_msg_tx: Option>, + pub pool_msg_tx: Option>, + pub cancel: Option, + pub event_metadata: EventMetadata, + pub worker_key: Option, + pub inspector: Option, + pub supervisor_policy: SupervisorPolicy, + pub worker_name: String, + pub worker_kind: WorkerKind, + pub termination_token: Option, +} + +pub struct WorkerBuilder { + init_opts: WorkerContextInitOpts, + flags: Arc, + inspector: Option, + supervisor_policy: Option, + termination_token: Option, + worker_naming_fn: Option) -> String + Send>>, +} + +impl WorkerBuilder { + pub fn new(init_opts: WorkerContextInitOpts, flags: Arc) -> Self { + Self { + init_opts, + flags, + inspector: None, + supervisor_policy: None, + termination_token: None, + worker_naming_fn: None, + } + } + + pub fn inspector(mut self, value: Inspector) -> Self { + self.inspector = Some(value); + self + } + + pub fn supervisor_policy(mut self, value: SupervisorPolicy) -> Self { + self.supervisor_policy = Some(value); + self + } + + pub fn termination_token(mut self, value: TerminationToken) -> Self { + self.termination_token = Some(value); + self + } + + pub fn worker_naming_fn(mut self, value: F) -> Self + where + F: Fn(Option) -> String + Send + 'static, + { + self.worker_naming_fn = Some(Box::new(value) as _); + self + } + + pub fn set_inspector(&mut self, value: Option) -> &mut Self { + self.inspector = value; + self + } + + pub fn set_supervisor_policy(&mut self, value: Option) -> &mut Self { + self.supervisor_policy = value; + self + } + + pub fn set_termination_token(&mut self, value: Option) -> &mut Self { + self.termination_token = value; + self + } + + pub fn set_worker_naming_fn(&mut self, value: Option) -> &mut Self + where + F: Fn(Option) -> String + Send + 'static, + { + self.worker_naming_fn = value.map(|it| Box::new(it) as _); + self + } + + pub(crate) fn build(self) -> Result { + let Self { + mut init_opts, + flags, + inspector, + supervisor_policy, + termination_token, + worker_naming_fn, + } = self; + + let conf = &init_opts.conf; + let worker_kind = conf.to_worker_kind(); + let worker_naming_fn = worker_naming_fn.unwrap_or(Box::new(|uuid| match worker_kind { + WorkerKind::MainWorker => "main-worker".to_string(), + WorkerKind::EventsWorker => "events-workers".to_string(), + WorkerKind::UserWorker => uuid + .map(|it| format!("sb-iso-{:?}", it)) + .unwrap_or("isolate-worker-unknown".to_string()), + })); + + let worker_key = conf.as_user_worker().and_then(|it| it.key); + let worker_name = worker_naming_fn(worker_key); + let worker_cancel_token = conf.as_user_worker().and_then(|it| it.cancel.clone()); + let worker_pool_msg_tx = conf.as_user_worker().and_then(|it| it.pool_msg_tx.clone()); + let worker_events_msg_tx = conf + .as_user_worker() + .and_then(|it| it.events_msg_tx.clone()); + + let cx = Arc::new(WorkerCx { + flags, + worker_boot_start_time: Instant::now(), + events_msg_tx: worker_events_msg_tx, + pool_msg_tx: worker_pool_msg_tx, + cancel: worker_cancel_token, + event_metadata: get_event_metadata(conf), + worker_key, + supervisor_policy: supervisor_policy.unwrap_or_default(), + inspector, + worker_name, + worker_kind, + termination_token, + }); + + let imp = WorkerDriverImpl::new(&mut init_opts, cx.clone()); + + Ok(Worker { + imp, + cx, + init_opts: Some(init_opts), + }) + } +} + +pub(crate) struct Worker { + pub(crate) imp: WorkerDriverImpl, + pub(crate) cx: Arc, + pub(crate) init_opts: Option, +} + +impl std::ops::Deref for Worker { + type Target = WorkerCx; + + fn deref(&self) -> &Self::Target { + &self.cx + } +} + +impl Worker { + pub fn start( + self, + booter_signal: oneshot::Sender>, + exit: WorkerExit, + ) { + let worker_name = self.worker_name.clone(); + let worker_key = self.worker_key; + let event_metadata = self.event_metadata.clone(); + + let events_msg_tx = self.events_msg_tx.clone(); + let pool_msg_tx = self.pool_msg_tx.clone(); + + let imp = self.imp.clone(); + let cx = self.cx.clone(); + let worker_kind = cx.worker_kind; + + let rt = imp.runtime_handle(); + let worker_fut = async move { + let permit = DenoRuntime::acquire().await; + let new_runtime = match DenoRuntime::new(self).await { + Ok(v) => v, + Err(err) => { + drop(permit); + + let err = CloneableError::from(err.context("worker boot error")); + let _ = booter_signal.send(Err(err.clone().into())); + + return Some(imp.on_boot_error(err.into()).await); + } + }; + + let mut runtime = scopeguard::guard(new_runtime, |mut runtime| unsafe { + runtime.js_runtime.v8_isolate().enter(); + }); + + unsafe { + runtime.js_runtime.v8_isolate().exit(); + } + + drop(permit); + + let metric_src = { + let metric_src = WorkerMetricSource::from_js_runtime(&mut runtime.js_runtime); + + if let Some(opts) = runtime.conf.as_main_worker().cloned() { + let state = runtime.js_runtime.op_state(); + let mut state_mut = state.borrow_mut(); + let metric_src = RuntimeMetricSource::new( + metric_src.clone(), + opts.event_worker_metric_src + .and_then(|it| it.into_worker().ok()), + opts.shared_metric_src, + ); + + state_mut.put(metric_src.clone()); + MetricSource::Runtime(metric_src) + } else { + MetricSource::Worker(metric_src) + } + }; + + let _ = booter_signal.send(Ok((metric_src, runtime.drop_token.clone()))); + let supervise_fut = match imp.clone().supervise(&mut runtime) { + Some(v) => v.boxed(), + None if worker_kind.is_user_worker() => return None, + None => ready(Ok(())).boxed(), + }; + + let _guard = scopeguard::guard((), |_| { + if let Some((key, tx)) = worker_key.zip(pool_msg_tx) { + if let Err(err) = tx.send(UserWorkerMsgs::Shutdown(key)) { + error!( + "failed to send the shutdown signal to user worker pool: {:?}", + err + ); + } + } + }); + + let result = imp.on_created(&mut runtime).await; + let maybe_uncaught_exception_event = match result.as_ref() { + Ok(WorkerEvents::UncaughtException(ev)) => Some(ev.clone()), + Err(err) => Some(UncaughtExceptionEvent { + cpu_time_used: 0, + exception: err.to_string(), + }), + + _ => None, + }; + + if let Some(ev) = maybe_uncaught_exception_event { + exit.set(WorkerExitStatus::WithUncaughtException(ev)).await; + } + + drop(runtime); + let _ = supervise_fut.await; + + Some(result) + }; + + let worker_fut = async move { + let Some(result) = worker_fut.await else { + return; + }; + + match result { + Ok(event) => { + match event { + WorkerEvents::Shutdown(ShutdownEvent { cpu_time_used, .. }) + | WorkerEvents::UncaughtException(UncaughtExceptionEvent { + cpu_time_used, + .. + }) => { + debug!("CPU time used: {:?}ms", cpu_time_used); + } + + _ => {} + }; + + send_event_if_event_worker_available( + events_msg_tx.as_ref(), + event, + event_metadata.clone(), + ); + } + + Err(err) => error!("unexpected worker error {}", err), + }; + } + .instrument(debug_span!("worker", name = worker_name.as_str(), kind = %worker_kind)); + + drop(rt.spawn_pinned({ + let worker_fut = unsafe { MaskFutureAsSend::new(worker_fut) }; + move || tokio::task::spawn_local(worker_fut) + })); + } +} + +#[derive(Debug, Clone)] +pub struct WorkerSurface { + pub metric: MetricSource, + pub msg_tx: mpsc::UnboundedSender, + pub exit: WorkerExit, + pub cancel: CancellationToken, +} diff --git a/crates/base/src/worker/worker_surface_creation.rs b/crates/base/src/worker/worker_surface_creation.rs new file mode 100644 index 000000000..a949fbcf8 --- /dev/null +++ b/crates/base/src/worker/worker_surface_creation.rs @@ -0,0 +1,783 @@ +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::Context; +use deno_config::JsxImportSourceConfig; +use either::Either; +use event_worker::events::{BootEvent, WorkerEventWithMetadata, WorkerEvents}; +use sb_core::{MetricSource, SharedMetricSource}; +use sb_graph::{DecoratorType, EszipPayloadKind}; +use sb_workers::context::{ + EventWorkerRuntimeOpts, MainWorkerRuntimeOpts, UserWorkerMsgs, WorkerContextInitOpts, + WorkerExit, WorkerRequestMsg, WorkerRuntimeOpts, +}; +use tokio::sync::{mpsc, oneshot}; +use tokio_util::sync::CancellationToken; + +use crate::{inspector_server::Inspector, server::ServerFlags}; + +use super::{ + driver::WorkerDriver, pool::SupervisorPolicy, termination_token::TerminationToken, + utils::send_event_if_event_worker_available, WorkerBuilder, WorkerSurface, +}; + +mod request { + use std::{future::pending, io::ErrorKind, sync::Arc, time::Duration}; + + use deno_core::unsync::AtomicFlag; + use http_utils::{ + io::Upgraded2, + utils::{emit_status_code, get_upgrade_type}, + }; + use http_v02::StatusCode; + use hyper_v014::{client::conn::http1, upgrade::OnUpgrade, Body, Response}; + use once_cell::sync::Lazy; + use sb_workers::context::{WorkerKind, WorkerRequestMsg}; + use tokio::{ + io, + net::TcpStream, + sync::{mpsc, oneshot}, + time::sleep, + }; + use tokio_rustls::server::TlsStream; + use tracing::warn; + + use crate::{ + server::ServerFlags, + timeout::{self, CancelOnWriteTimeout, ReadTimeoutStream}, + worker::DuplexStreamEntry, + }; + + pub(super) async fn handle_request( + flags: Arc, + worker_kind: WorkerKind, + duplex_stream_tx: mpsc::UnboundedSender, + msg: WorkerRequestMsg, + ) -> Result<(), anyhow::Error> { + let request_idle_timeout_ms = flags.request_idle_timeout_ms; + let request_buf_size = flags.request_buffer_size.unwrap_or_else(|| { + const KIB: usize = 1024; + static CHECK: Lazy = Lazy::new(AtomicFlag::default); + + if !CHECK.is_raised() { + CHECK.raise(); + warn!("request buffer size is not specified, so it will be set to 1 KiB"); + } + + KIB as u64 + }); + + let (ours, theirs) = io::duplex(request_buf_size as usize); + let WorkerRequestMsg { + mut req, + res_tx, + conn_token, + } = msg; + + let _ = duplex_stream_tx.send((theirs, conn_token.clone())); + let req_upgrade_type = get_upgrade_type(req.headers()); + let req_upgrade = req_upgrade_type + .clone() + .and_then(|it| Some(it).zip(req.extensions_mut().remove::())); + + // send the HTTP request to the worker over duplex stream + let (mut request_sender, connection) = + http1::Builder::new().writev(true).handshake(ours).await?; + + let (upgrade_tx, upgrade_rx) = oneshot::channel(); + + // spawn a task to poll the connection and drive the HTTP state + tokio::task::spawn({ + async move { + match connection.without_shutdown().await { + Err(e) => { + log::error!( + "error in {} worker connection: {}", + worker_kind, + e.message() + ); + } + + Ok(parts) => { + if let Some((requested, req_upgrade)) = req_upgrade { + if let Ok((Some(accepted), status)) = upgrade_rx.await { + if status == StatusCode::SWITCHING_PROTOCOLS + && accepted == requested + { + tokio::spawn(relay_upgraded_request_and_response( + req_upgrade, + parts, + request_idle_timeout_ms, + )); + + return; + } + }; + } + + if let Some(token) = conn_token { + token.cancelled_owned().await; + } + } + } + } + }); + + tokio::task::yield_now().await; + + let maybe_cancel_fut = async move { + if let Some(timeout_ms) = request_idle_timeout_ms { + sleep(Duration::from_millis(timeout_ms)).await; + } else { + pending::<()>().await; + unreachable!() + } + }; + + let res = tokio::select! { + resp = request_sender.send_request(req) => resp, + _ = maybe_cancel_fut => { + Ok(emit_status_code(http_v02::StatusCode::GATEWAY_TIMEOUT, None, false)) + } + }; + + let Ok(res) = res else { + drop(res_tx.send(res)); + return Ok(()); + }; + + if let Some(requested) = req_upgrade_type { + let res_upgrade_type = get_upgrade_type(res.headers()); + let _ = upgrade_tx.send((res_upgrade_type.clone(), res.status())); + + match res_upgrade_type { + Some(accepted) if accepted == requested => {} + _ => { + drop(res_tx.send(Ok(emit_status_code(StatusCode::BAD_GATEWAY, None, true)))); + return Ok(()); + } + } + } + + if let Some(timeout_ms) = flags.request_idle_timeout_ms { + let headers = res.headers(); + let is_streamed_response = !headers.contains_key(http_v02::header::CONTENT_LENGTH); + + if is_streamed_response { + let duration = Duration::from_millis(timeout_ms); + let (parts, body) = res.into_parts(); + + drop(res_tx.send(Ok(Response::from_parts( + parts, + Body::wrap_stream(CancelOnWriteTimeout::new(body, duration)), + )))); + + return Ok(()); + } + } + + drop(res_tx.send(Ok(res))); + Ok(()) + } + + async fn relay_upgraded_request_and_response( + downstream: OnUpgrade, + parts: http1::Parts, + maybe_idle_timeout: Option, + ) { + let upstream = Upgraded2::new(parts.io, parts.read_buf); + let mut upstream = if let Some(timeout_ms) = maybe_idle_timeout { + ReadTimeoutStream::with_timeout(upstream, Duration::from_millis(timeout_ms)) + } else { + ReadTimeoutStream::with_bypass(upstream) + }; + + let mut downstream = downstream.await.expect("failed to upgrade request"); + + match io::copy_bidirectional(&mut upstream, &mut downstream).await { + Ok(_) => {} + Err(err) if matches!(err.kind(), ErrorKind::TimedOut | ErrorKind::BrokenPipe) => {} + Err(err) if matches!(err.kind(), ErrorKind::UnexpectedEof) => { + let Ok(_) = downstream.downcast::>>() else { + // TODO(Nyannyacha): It would be better if we send + // `close_notify` before shutdown an upstream if downstream is a + // TLS stream. + + // INVARIANT: `UnexpectedEof` due to shutdown `DuplexStream` is + // only expected to occur in the context of `TlsStream`. + panic!("unhandleable unexpected eof"); + }; + } + + value => { + unreachable!("coping between upgraded connections failed: {:?}", value); + } + } + + // XXX(Nyannyacha): Here you might want to emit the event metadata. + } +} + +pub type WorkerBuilderHook = + Box Result<(), anyhow::Error> + Send>; + +pub struct WorkerSurfaceBuilder { + init_opts: Option, + flags: Option>, + policy: Option, + termination_token: Option, + inspector: Option, + worker_builder_hook: Option, +} + +impl Default for WorkerSurfaceBuilder { + fn default() -> Self { + Self::new() + } +} + +impl WorkerSurfaceBuilder { + pub fn new() -> Self { + Self { + init_opts: None, + flags: None, + policy: None, + termination_token: None, + inspector: None, + worker_builder_hook: None, + } + } + + pub fn init_opts(mut self, value: WorkerContextInitOpts) -> Self { + self.init_opts = Some(value); + self + } + + pub fn sever_flags(mut self, value: Either, ServerFlags>) -> Self { + self.flags = Some(value.map_right(Arc::new).into_inner()); + self + } + + pub fn policy(mut self, value: SupervisorPolicy) -> Self { + self.policy = Some(value); + self + } + + pub fn termination_token(mut self, value: TerminationToken) -> Self { + self.termination_token = Some(value); + self + } + + pub fn inspector(mut self, value: Inspector) -> Self { + self.inspector = Some(value); + self + } + + pub fn worker_builder_hook(mut self, value: F) -> Self + where + F: FnOnce(&mut WorkerBuilder) -> Result<(), anyhow::Error> + Send + 'static, + { + self.worker_builder_hook = Some(Box::new(value) as _); + self + } + + pub fn set_init_opts(&mut self, value: Option) -> &mut Self { + self.init_opts = value; + self + } + + pub fn set_server_flags( + &mut self, + value: Option, ServerFlags>>, + ) -> &mut Self { + self.flags = value.map(|it| it.map_right(Arc::new).into_inner()); + self + } + + pub fn set_policy(&mut self, value: Option) -> &mut Self { + self.policy = value; + self + } + + pub fn set_termination_token(&mut self, value: Option) -> &mut Self { + self.termination_token = value; + self + } + + pub fn set_inspector(&mut self, value: Option) -> &mut Self { + self.inspector = value; + self + } + + pub fn set_worker_builder_hook(&mut self, value: Option) -> &mut Self + where + F: FnOnce(&mut WorkerBuilder) -> Result<(), anyhow::Error> + Send + 'static, + { + self.worker_builder_hook = value.map(|it| Box::new(it) as _); + self + } + + pub async fn build(self) -> Result { + let Self { + init_opts, + flags, + policy, + termination_token, + inspector, + worker_builder_hook, + } = self; + + let (worker_boot_result_tx, worker_boot_result_rx) = + oneshot::channel::>(); + + let flags = flags.unwrap_or_default(); + let init_opts = init_opts.context("init_opts must be specified")?; + let worker_kind = init_opts.conf.to_worker_kind(); + let exit = WorkerExit::default(); + let mut worker_builder = WorkerBuilder::new(init_opts, flags.clone()); + + worker_builder + .set_inspector(inspector) + .set_supervisor_policy(worker_kind.is_user_worker().then_some(policy).flatten()) + .set_termination_token(termination_token.clone()); + + if let Some(hook) = worker_builder_hook { + hook(&mut worker_builder)?; + } + + let worker = worker_builder.build()?; + let cx = worker.cx.clone(); + let network_sender = worker.imp.network_sender().await; + + worker.start(worker_boot_result_tx, exit.clone()); + + // create an async task waiting for requests for worker + let (worker_req_tx, mut worker_req_rx) = mpsc::unbounded_channel::(); + let worker_req_handle = tokio::task::spawn({ + async move { + while let Some(msg) = worker_req_rx.recv().await { + tokio::task::spawn({ + let flags = flags.clone(); + let network_sender = network_sender.clone(); + + async move { + if let Err(err) = + request::handle_request(flags, worker_kind, network_sender, msg) + .await + { + log::error!("worker failed to handle request: {:?}", err); + } + } + }); + } + } + }); + + // wait for worker to be successfully booted + match worker_boot_result_rx.await? { + Ok((metric, cancel)) => { + let elapsed = cx.worker_boot_start_time.elapsed().as_millis(); + + send_event_if_event_worker_available( + cx.events_msg_tx.as_ref(), + WorkerEvents::Boot(BootEvent { + boot_time: elapsed as usize, + }), + cx.event_metadata.clone(), + ); + + Ok(WorkerSurface { + metric, + msg_tx: worker_req_tx, + exit, + cancel, + }) + } + + Err(err) => { + worker_req_handle.abort(); + + if let Some(token) = termination_token.as_ref() { + token.outbound.cancel(); + } + + Err(err) + } + } + } +} + +pub struct MainWorkerSurface(WorkerSurface); + +impl std::ops::Deref for MainWorkerSurface { + type Target = WorkerSurface; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for MainWorkerSurface { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +pub struct MainWorkerSurfaceBuilder { + inner: WorkerSurfaceBuilder, + + main_worker_path: PathBuf, + no_module_cache: Option, + import_map_path: Option, + entrypoint: Option, + decorator: Option, + jsx: Option, + + worker_pool_tx: Option>, + shared_metric_src: Option, + event_worker_metric_src: Option, +} + +impl std::ops::Deref for MainWorkerSurfaceBuilder { + type Target = WorkerSurfaceBuilder; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for MainWorkerSurfaceBuilder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl MainWorkerSurfaceBuilder { + pub fn new

(main_worker_path: P) -> Self + where + P: AsRef, + { + Self { + inner: WorkerSurfaceBuilder::new(), + + main_worker_path: main_worker_path.as_ref().to_path_buf(), + no_module_cache: None, + import_map_path: None, + entrypoint: None, + decorator: None, + jsx: None, + + worker_pool_tx: None, + shared_metric_src: None, + event_worker_metric_src: None, + } + } + + pub fn no_module_cache(mut self, value: bool) -> Self { + self.no_module_cache = Some(value); + self + } + + pub fn import_map_path(mut self, value: &str) -> Self { + self.import_map_path = Some(value.to_string()); + self + } + + pub fn entrypoint(mut self, value: &str) -> Self { + self.entrypoint = Some(value.to_string()); + self + } + + pub fn decorator(mut self, value: DecoratorType) -> Self { + self.decorator = Some(value); + self + } + + pub fn jsx_import_source_config(mut self, value: JsxImportSourceConfig) -> Self { + self.jsx = Some(value); + self + } + + pub fn worker_pool_sender(mut self, value: mpsc::UnboundedSender) -> Self { + self.worker_pool_tx = Some(value); + self + } + + pub fn shared_metric_source(mut self, value: SharedMetricSource) -> Self { + self.shared_metric_src = Some(value); + self + } + + pub fn event_worker_metric_source(mut self, value: MetricSource) -> Self { + self.event_worker_metric_src = Some(value); + self + } + + pub fn set_no_module_cache(&mut self, value: Option) -> &mut Self { + self.no_module_cache = value; + self + } + + pub fn set_import_map_path(&mut self, value: Option<&str>) -> &mut Self { + self.import_map_path = value.map(str::to_string); + self + } + + pub fn set_entrypoint(&mut self, value: Option<&str>) -> &mut Self { + self.entrypoint = value.map(str::to_string); + self + } + + pub fn set_decorator(&mut self, value: Option) -> &mut Self { + self.decorator = value; + self + } + + pub fn set_jsx_import_source_config( + &mut self, + value: Option, + ) -> &mut Self { + self.jsx = value; + self + } + + pub fn set_worker_pool_sender( + &mut self, + value: Option>, + ) -> &mut Self { + self.worker_pool_tx = value; + self + } + + pub fn set_shared_metric_source(&mut self, value: Option) -> &mut Self { + self.shared_metric_src = value; + self + } + + pub fn set_event_worker_metric_source(&mut self, value: Option) -> &mut Self { + self.event_worker_metric_src = value; + self + } + + pub async fn build(self) -> Result { + let Self { + mut inner, + main_worker_path, + no_module_cache, + import_map_path, + entrypoint, + decorator, + jsx, + worker_pool_tx, + shared_metric_src, + event_worker_metric_src, + } = self; + + let flags = inner.flags.as_ref().cloned().unwrap_or_default(); + + let mut service_path = main_worker_path.clone(); + let mut maybe_eszip = None; + + if let Some(ext) = main_worker_path.extension() { + if ext == "eszip" { + service_path = main_worker_path.parent().unwrap().to_path_buf(); + maybe_eszip = Some(EszipPayloadKind::VecKind(std::fs::read(main_worker_path)?)); + } + } + + inner.set_init_opts(Some(WorkerContextInitOpts { + service_path, + import_map_path, + no_module_cache: no_module_cache.unwrap_or(flags.no_module_cache), + + timing: None, + maybe_eszip, + maybe_entrypoint: entrypoint, + maybe_decorator: decorator, + maybe_module_code: None, + conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx: worker_pool_tx.context("worker_pool_sender must be specified")?, + shared_metric_src, + event_worker_metric_src, + }), + env_vars: std::env::vars().collect(), + static_patterns: vec![], + + maybe_jsx_import_source_config: jsx, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + })); + + Ok(MainWorkerSurface( + inner + .build() + .await + .map_err(|err| err.context("main worker boot error"))?, + )) + } +} + +pub struct EventWorkerSurface { + inner: WorkerSurface, + event_msg_tx: mpsc::UnboundedSender, +} + +impl std::ops::Deref for EventWorkerSurface { + type Target = WorkerSurface; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for EventWorkerSurface { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl EventWorkerSurface { + pub fn event_message_sender(&self) -> mpsc::UnboundedSender { + self.event_msg_tx.clone() + } +} + +pub struct EventWorkerSurfaceBuilder { + inner: WorkerSurfaceBuilder, + + event_worker_path: PathBuf, + no_module_cache: Option, + import_map_path: Option, + entrypoint: Option, + decorator: Option, +} + +impl std::ops::Deref for EventWorkerSurfaceBuilder { + type Target = WorkerSurfaceBuilder; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl std::ops::DerefMut for EventWorkerSurfaceBuilder { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner + } +} + +impl EventWorkerSurfaceBuilder { + pub fn new

(event_worker_path: P) -> Self + where + P: AsRef, + { + Self { + inner: WorkerSurfaceBuilder::new(), + + event_worker_path: event_worker_path.as_ref().to_path_buf(), + no_module_cache: None, + import_map_path: None, + entrypoint: None, + decorator: None, + } + } + + pub fn no_module_cache(mut self, value: bool) -> Self { + self.no_module_cache = Some(value); + self + } + + pub fn import_map_path(mut self, value: &str) -> Self { + self.import_map_path = Some(value.to_string()); + self + } + + pub fn entrypoint(mut self, value: &str) -> Self { + self.entrypoint = Some(value.to_string()); + self + } + + pub fn decorator(mut self, value: DecoratorType) -> Self { + self.decorator = Some(value); + self + } + + pub fn set_no_module_cache(&mut self, value: Option) -> &mut Self { + self.no_module_cache = value; + self + } + + pub fn set_import_map_path(&mut self, value: Option<&str>) -> &mut Self { + self.import_map_path = value.map(str::to_string); + self + } + + pub fn set_entrypoint(&mut self, value: Option<&str>) -> &mut Self { + self.entrypoint = value.map(str::to_string); + self + } + + pub fn set_decorator(&mut self, value: Option) -> &mut Self { + self.decorator = value; + self + } + + pub async fn build(self) -> Result { + let Self { + mut inner, + event_worker_path, + no_module_cache, + import_map_path, + entrypoint, + decorator, + } = self; + + let (event_msg_tx, event_msg_rx) = mpsc::unbounded_channel::(); + + let flags = inner.flags.as_ref().cloned().unwrap_or_default(); + let event_worker_exit_deadline_sec = flags.event_worker_exit_deadline_sec; + + let mut service_path = event_worker_path.clone(); + let mut maybe_eszip = None; + + if let Some(ext) = event_worker_path.extension() { + if ext == "eszip" { + service_path = event_worker_path.parent().unwrap().to_path_buf(); + maybe_eszip = Some(EszipPayloadKind::VecKind(std::fs::read(event_worker_path)?)); + } + } + + inner.set_init_opts(Some(WorkerContextInitOpts { + service_path, + no_module_cache: no_module_cache.unwrap_or(flags.no_module_cache), + + import_map_path, + env_vars: std::env::vars().collect(), + timing: None, + maybe_eszip, + maybe_entrypoint: entrypoint, + maybe_decorator: decorator, + maybe_module_code: None, + conf: WorkerRuntimeOpts::EventsWorker(EventWorkerRuntimeOpts { + events_msg_rx: Some(event_msg_rx), + event_worker_exit_deadline_sec: Some(event_worker_exit_deadline_sec), + }), + static_patterns: vec![], + + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + })); + + Ok(EventWorkerSurface { + inner: inner + .build() + .await + .map_err(|err| err.context("event worker boot error"))?, + + event_msg_tx, + }) + } +} diff --git a/crates/base/tests/integration_tests.rs b/crates/base/tests/integration_tests.rs index 3b2c04cc8..cd58e8c4e 100644 --- a/crates/base/tests/integration_tests.rs +++ b/crates/base/tests/integration_tests.rs @@ -21,15 +21,17 @@ use std::{ use anyhow::Context; use async_tungstenite::WebSocketStream; -use base::utils::test_utils::{ - self, create_test_user_worker, test_user_runtime_opts, test_user_worker_pool_policy, - TestBedBuilder, -}; use base::{ integration_test, integration_test_listen_fut, integration_test_with_server_flag, - rt_worker::worker_ctx::{create_user_worker_pool, create_worker, TerminationToken}, server::{Server, ServerEvent, ServerFlags, ServerHealth, Tls}, - DecoratorType, + worker, DecoratorType, +}; +use base::{ + utils::test_utils::{ + self, create_test_user_worker, test_user_runtime_opts, test_user_worker_pool_policy, + TestBedBuilder, + }, + worker::TerminationToken, }; use deno_core::serde_json::{self, json}; use futures_util::{future::BoxFuture, Future, FutureExt, SinkExt, StreamExt}; @@ -183,7 +185,7 @@ async fn test_not_trigger_pku_sigsegv_due_to_jit_compilation_non_cli() { let main_termination_token = TerminationToken::new(); // create a user worker pool - let (_, worker_pool_tx) = create_user_worker_pool( + let (_, worker_pool_tx) = worker::create_user_worker_pool( Arc::default(), test_utils::test_user_worker_pool_policy(), None, @@ -195,29 +197,30 @@ async fn test_not_trigger_pku_sigsegv_due_to_jit_compilation_non_cli() { .await .unwrap(); - let opts = WorkerContextInitOpts { - service_path: "./test_cases/slow_resp".into(), - no_module_cache: false, - import_map_path: None, - env_vars: HashMap::new(), - timing: None, - maybe_eszip: None, - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, - conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }), - static_patterns: vec![], - - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }; + let surface = worker::WorkerSurfaceBuilder::new() + .init_opts(WorkerContextInitOpts { + service_path: "./test_cases/slow_resp".into(), + no_module_cache: false, + import_map_path: None, + env_vars: HashMap::new(), + timing: None, + maybe_eszip: None, + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, + conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }), + static_patterns: vec![], - let ctx = create_worker(Arc::default(), (opts, main_termination_token.clone()), None) + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }) + .termination_token(main_termination_token.clone()) + .build() .await .unwrap(); @@ -236,7 +239,7 @@ async fn test_not_trigger_pku_sigsegv_due_to_jit_compilation_non_cli() { conn_token: Some(conn_token.clone()), }; - let _ = ctx.msg_tx.send(msg); + let _ = surface.msg_tx.send(msg); let res = res_rx.await.unwrap().unwrap(); assert!(res.status().as_u16() == 200); @@ -344,7 +347,7 @@ async fn test_main_worker_boot_error() { let main_termination_token = TerminationToken::new(); // create a user worker pool - let (_, worker_pool_tx) = create_user_worker_pool( + let (_, worker_pool_tx) = worker::create_user_worker_pool( Arc::default(), test_user_worker_pool_policy(), None, @@ -356,29 +359,31 @@ async fn test_main_worker_boot_error() { .await .unwrap(); - let opts = WorkerContextInitOpts { - service_path: "./test_cases/main".into(), - no_module_cache: false, - import_map_path: Some("./non-existing-import-map.json".to_string()), - env_vars: HashMap::new(), - timing: None, - maybe_eszip: None, - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, - conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }), - static_patterns: vec![], - - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }; + let result = worker::WorkerSurfaceBuilder::new() + .init_opts(WorkerContextInitOpts { + service_path: "./test_cases/main".into(), + no_module_cache: false, + import_map_path: Some("./non-existing-import-map.json".to_string()), + env_vars: HashMap::new(), + timing: None, + maybe_eszip: None, + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, + conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }), + static_patterns: vec![], - let result = create_worker(Arc::default(), (opts, main_termination_token.clone()), None).await; + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }) + .termination_token(main_termination_token.clone()) + .build() + .await; assert!(result.is_err()); assert!(result @@ -472,7 +477,7 @@ async fn test_main_worker_user_worker_mod_evaluate_exception() { let main_termination_token = TerminationToken::new(); // create a user worker pool - let (_, worker_pool_tx) = create_user_worker_pool( + let (_, worker_pool_tx) = worker::create_user_worker_pool( Arc::default(), test_user_worker_pool_policy(), None, @@ -484,29 +489,30 @@ async fn test_main_worker_user_worker_mod_evaluate_exception() { .await .unwrap(); - let opts = WorkerContextInitOpts { - service_path: "./test_cases/main".into(), - no_module_cache: false, - import_map_path: None, - env_vars: HashMap::new(), - timing: None, - maybe_eszip: None, - maybe_entrypoint: None, - maybe_decorator: None, - maybe_module_code: None, - conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { - worker_pool_tx, - shared_metric_src: None, - event_worker_metric_src: None, - }), - static_patterns: vec![], - - maybe_jsx_import_source_config: None, - maybe_s3_fs_config: None, - maybe_tmp_fs_config: None, - }; + let surface = worker::WorkerSurfaceBuilder::new() + .init_opts(WorkerContextInitOpts { + service_path: "./test_cases/main".into(), + no_module_cache: false, + import_map_path: None, + env_vars: HashMap::new(), + timing: None, + maybe_eszip: None, + maybe_entrypoint: None, + maybe_decorator: None, + maybe_module_code: None, + conf: WorkerRuntimeOpts::MainWorker(MainWorkerRuntimeOpts { + worker_pool_tx, + shared_metric_src: None, + event_worker_metric_src: None, + }), + static_patterns: vec![], - let ctx = create_worker(Arc::default(), (opts, main_termination_token.clone()), None) + maybe_jsx_import_source_config: None, + maybe_s3_fs_config: None, + maybe_tmp_fs_config: None, + }) + .termination_token(main_termination_token.clone()) + .build() .await .unwrap(); @@ -525,7 +531,7 @@ async fn test_main_worker_user_worker_mod_evaluate_exception() { conn_token: Some(conn_token.clone()), }; - let _ = ctx.msg_tx.send(msg); + let _ = surface.msg_tx.send(msg); let res = res_rx.await.unwrap().unwrap(); assert!(res.status().as_u16() == 500); @@ -1895,6 +1901,15 @@ async fn test_request_idle_timeout_no_streamed_response_non_secure() { test_request_idle_timeout_no_streamed_response(new_localhost_tls(false)).await; } +#[tokio::test] +#[serial] +#[ignore = "running too much tests"] +async fn test_request_idle_timeout_no_streamed_response_non_secure_1000() { + for _ in 0..1000 { + test_request_idle_timeout_no_streamed_response(new_localhost_tls(false)).await; + } +} + #[tokio::test] #[serial] async fn test_request_idle_timeout_no_streamed_response_secure() { diff --git a/crates/base_rt/src/lib.rs b/crates/base_rt/src/lib.rs index e75160d8a..ff7638a66 100644 --- a/crates/base_rt/src/lib.rs +++ b/crates/base_rt/src/lib.rs @@ -7,10 +7,10 @@ use std::{ }; use cpu_timer::get_thread_time; -use deno_core::{anyhow::Context, error::AnyError, OpState, V8CrossThreadTaskSpawner}; +use deno_core::{anyhow::Context, error::AnyError, OpState, Resource, V8CrossThreadTaskSpawner}; use once_cell::sync::Lazy; use tokio::{runtime::Handle, sync::oneshot}; -use tokio_util::sync::CancellationToken; +use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; use tracing::{debug, debug_span, Instrument}; pub mod error; @@ -71,13 +71,24 @@ pub static USER_WORKER_RT: Lazy = Lazy::new(| }); #[derive(Clone)] -pub struct DenoRuntimeDropToken(pub CancellationToken); +pub struct DropToken(pub CancellationToken); + +impl Resource for DropToken {} + +#[derive(Clone)] +pub struct DenoRuntimeDropToken(pub DropToken); impl std::ops::Deref for DenoRuntimeDropToken { type Target = CancellationToken; fn deref(&self) -> &Self::Target { - &self.0 + &self.0 .0 + } +} + +impl DenoRuntimeDropToken { + pub fn cancelled_owned(self) -> WaitForCancellationFutureOwned { + self.0 .0.cancelled_owned() } } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index ae47ac13b..c330839a8 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -7,10 +7,10 @@ mod logger; use anyhow::{anyhow, bail, Error}; use base::commands::start_server; -use base::rt_worker::worker_pool::{SupervisorPolicy, WorkerPoolPolicy}; use base::server::{ServerFlags, Tls, WorkerEntrypoints}; use base::utils::path::find_up; use base::utils::units::percentage_value; +use base::worker::pool::{SupervisorPolicy, WorkerPoolPolicy}; use base::{CacheSetting, DecoratorType, InspectorOption}; use clap::ArgMatches; use deno_core::url::Url; @@ -208,7 +208,7 @@ fn main() -> Result { beforeunload_memory_pct: maybe_beforeunload_memory_pct, }; - let maybe_received_signum = start_server( + let maybe_received_signum_or_exit_code = start_server( ip.as_str(), port, maybe_tls, @@ -254,8 +254,8 @@ fn main() -> Result { ) .await?; - maybe_received_signum - .map(|it| ExitCode::from(it as u8)) + maybe_received_signum_or_exit_code + .map(|it| it.map_left(|it| ExitCode::from(it as u8)).into_inner()) .unwrap_or_default() } diff --git a/crates/event_worker/events.rs b/crates/event_worker/events.rs index adb0c0728..987bf8dd1 100644 --- a/crates/event_worker/events.rs +++ b/crates/event_worker/events.rs @@ -1,4 +1,5 @@ use base_mem_check::MemCheckState; +use enum_as_inner::EnumAsInner; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -21,6 +22,7 @@ pub struct WorkerMemoryUsed { #[derive(Serialize, Deserialize, Debug)] pub enum ShutdownReason { + EventLoopCompleted, WallClockTime, CPUTime, Memory, @@ -41,11 +43,6 @@ pub struct UncaughtExceptionEvent { pub cpu_time_used: usize, } -#[derive(Serialize, Deserialize, Debug)] -pub struct EventLoopCompletedEvent { - pub cpu_time_used: usize, -} - #[derive(Serialize, Deserialize, Debug)] pub struct LogEvent { pub msg: String, @@ -60,13 +57,12 @@ pub enum LogLevel { Error, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, EnumAsInner)] pub enum WorkerEvents { Boot(BootEvent), BootFailure(BootFailureEvent), UncaughtException(UncaughtExceptionEvent), Shutdown(ShutdownEvent), - EventLoopCompleted(EventLoopCompletedEvent), Log(LogEvent), } diff --git a/crates/event_worker/js_interceptors.rs b/crates/event_worker/js_interceptors.rs index 5d44e5fed..18e4543e1 100644 --- a/crates/event_worker/js_interceptors.rs +++ b/crates/event_worker/js_interceptors.rs @@ -3,9 +3,8 @@ use crate::WorkerEventWithMetadata; use deno_core::error::AnyError; use deno_core::op2; use deno_core::OpState; -use log::error; use tokio::sync::mpsc; -use tracing::trace; +use tracing::{event, trace}; #[op2(fast)] fn op_user_worker_log( @@ -36,7 +35,12 @@ fn op_user_worker_log( trace!(?metadata); tx.send(metadata)?; } else { - error!("[{:?}] {}", level, msg.to_string()); + match level { + LogLevel::Debug => event!(tracing::Level::DEBUG, "{msg}"), + LogLevel::Info => event!(tracing::Level::INFO, "{msg}"), + LogLevel::Warning => event!(tracing::Level::WARN, "{msg}"), + LogLevel::Error => event!(tracing::Level::ERROR, "{msg}"), + } } Ok(()) diff --git a/crates/sb_core/js/async_hook.js b/crates/sb_core/js/async_hook.js index e0b461f6c..3c542630f 100644 --- a/crates/sb_core/js/async_hook.js +++ b/crates/sb_core/js/async_hook.js @@ -1,23 +1,31 @@ -import { core, primordials } from 'ext:core/mod.js'; +import { core, primordials } from "ext:core/mod.js"; const ops = core.ops; -const { - Promise -} = primordials; +const { Promise } = primordials; -let COUNTER = 0; -const PROMISES = new Map(); +const PROMISES = new Set(); -function waitUntil(maybePromise) { +function waitUntilInner(maybePromise) { if (maybePromise instanceof Promise) { ops.op_tap_promise_metrics("init"); - PROMISES.set(maybePromise, ++COUNTER); + PROMISES.add(maybePromise); } return maybePromise; } -function installPromiseHook() { +function waitUntil(maybePromise) { + return waitUntilInner(maybePromise); +} + +/** + * @param {"user" | "main" | "event"} kind + */ +function installPromiseHook(kind) { + if (kind !== "user") { + return; + } + core.setPromiseHooks( null, null, diff --git a/crates/sb_core/js/bootstrap.js b/crates/sb_core/js/bootstrap.js index 099cf3fac..5f57c028b 100644 --- a/crates/sb_core/js/bootstrap.js +++ b/crates/sb_core/js/bootstrap.js @@ -32,18 +32,19 @@ import * as eventSource from 'ext:deno_fetch/27_eventsource.js'; import * as WebGPU from 'ext:deno_webgpu/00_init.js'; import * as WebGPUSurface from 'ext:deno_webgpu/02_surface.js'; -import * as MainWorker from 'ext:sb_core_main_js/js/main_worker.js'; +import 'ext:sb_ai/js/onnxruntime/cache_adapter.js'; import { SUPABASE_ENV } from 'ext:sb_env/env.js'; import { USER_WORKER_API as ai } from 'ext:sb_ai/js/ai.js'; -import 'ext:sb_ai/js/onnxruntime/cache_adapter.js'; -import { waitUntil, installPromiseHook } from 'ext:sb_core_main_js/js/async_hook.js'; -import { registerErrors } from 'ext:sb_core_main_js/js/errors.js'; +import { SupabaseEventListener } from 'ext:sb_user_event_worker/event_worker.js'; +import { installEdgeRuntimeNamespace } from 'ext:sb_core_main_js/js/edge_runtime.js'; import { promiseRejectMacrotaskCallback } from 'ext:sb_core_main_js/js/promises.js'; +import { installPromiseHook } from 'ext:sb_core_main_js/js/async_hook.js'; +import { registerErrors } from 'ext:sb_core_main_js/js/errors.js'; import { denoOverrides, fsVars } from 'ext:sb_core_main_js/js/denoOverrides.js'; import { registerDeclarativeServer } from 'ext:sb_core_main_js/js/00_serve.js'; -import { SupabaseEventListener } from 'ext:sb_user_event_worker/event_worker.js'; + import { formatException, getterOnly, @@ -60,8 +61,6 @@ import { setUserAgent, } from 'ext:sb_core_main_js/js/navigator.js'; -import 'ext:sb_ai/js/onnxruntime/cache_adapter.js'; - let globalThis_; const ops = core.ops; @@ -365,10 +364,10 @@ ObjectAssign(internals, { warnOnDeprecatedApi }); function runtimeStart(target) { // core.setMacrotaskCallback(timers.handleTimerMacrotask); // core.setMacrotaskCallback(promiseRejectMacrotaskCallback); + core.setWasmStreamingCallback(fetch.handleWasmStreaming); ops.op_set_format_exception_callback(formatException); core.setBuildInfo(target); - installPromiseHook(); // deno-lint-ignore prefer-primordials Error.prepareStackTrace = core.prepareStackTrace; @@ -430,6 +429,11 @@ function dispatchUnloadEvent() { globalThis_.dispatchEvent(new Event("unload")); } +function dispatchDrainEvent() { + internals.drain = true; + globalThis_.dispatchEvent(new Event("drain")); +} + // Notification that the core received an unhandled promise rejection that is about to // terminate the runtime. If we can handle it, attempt to do so. function processUnhandledPromiseRejection(promise, reason) { @@ -495,6 +499,7 @@ globalThis.bootstrapSBEdge = (opts, ctx) => { "beforeunload", "unload", "unhandledrejection", + "drain", ]; eventHandlers.forEach((handlerName) => event.defineEventHandler(globalThis, handlerName)); @@ -526,14 +531,21 @@ globalThis.bootstrapSBEdge = (opts, ctx) => { flags } = opts; - deprecatedApiWarningDisabled = flags['SHOULD_DISABLE_DEPRECATED_API_WARNING']; verboseDeprecatedApiWarning = flags['SHOULD_USE_VERBOSE_DEPRECATED_API_WARNING']; bootstrapMockFnThrowError = ctx?.shouldBootstrapMockFnThrowError ?? false; runtimeStart(target); - ObjectAssign(internals, { bootstrapArgs: { opts, ctx } }); + ObjectAssign(internals, { + bootstrapArgs: { opts }, + worker: { kind }, + __ctx: ctx, + }); + + installPromiseHook(kind); + installEdgeRuntimeNamespace(kind, ctx.terminationRequestToken); + ObjectDefineProperty(globalThis, 'SUPABASE_VERSION', readOnly(String(version.runtime))); ObjectDefineProperty(globalThis, 'DENO_VERSION', readOnly(version.deno)); @@ -561,14 +573,6 @@ globalThis.bootstrapSBEdge = (opts, ctx) => { ); setLanguage('en'); - Object.defineProperty(globalThis, 'Supabase', { - get() { - return { - ai, - }; - }, - }); - // Find declarative fetch handler core.addMainModuleHandler(main => { if (ObjectHasOwn(main, 'default')) { @@ -618,15 +622,6 @@ globalThis.bootstrapSBEdge = (opts, ctx) => { /// DISABLE SHARED MEMORY INSTALL MEM CHECK TIMING if (kind === 'user') { - ObjectDefineProperties(globalThis, { - EdgeRuntime: { - value: { - waitUntil, - }, - configurable: true, - }, - }); - // override console if (!inspector) { ObjectDefineProperties(globalThis, { @@ -685,8 +680,6 @@ globalThis.bootstrapSBEdge = (opts, ctx) => { } if (kind === 'event') { - // Event Manager should have the same as the `main` except it can't create workers (that would be catastrophic) - delete globalThis.EdgeRuntime; ObjectDefineProperties(globalThis, { EventManager: getterOnly(() => SupabaseEventListener), }); @@ -713,6 +706,7 @@ globalThis.bootstrap = { dispatchBeforeUnloadEvent, // dispatchProcessExitEvent, // dispatchProcessBeforeExitEvent, + dispatchDrainEvent, }; core.setUnhandledPromiseRejectionHandler(processUnhandledPromiseRejection); diff --git a/crates/sb_core/js/edge_runtime.js b/crates/sb_core/js/edge_runtime.js new file mode 100644 index 000000000..e811798b3 --- /dev/null +++ b/crates/sb_core/js/edge_runtime.js @@ -0,0 +1,60 @@ +import { core, primordials } from "ext:core/mod.js"; + +import { MAIN_WORKER_API as ai } from "ext:sb_ai/js/ai.js"; +import { SUPABASE_USER_WORKERS } from "ext:sb_user_workers/user_workers.js"; +import { applySupabaseTag } from "ext:sb_core_main_js/js/http.js"; +import { waitUntil } from "ext:sb_core_main_js/js/async_hook.js"; + +const ops = core.ops; +const { ObjectDefineProperty } = primordials; + +/** + * @param {"user" | "main" | "event"} kind + * @param {number} terminationRequestTokenRid + */ +function installEdgeRuntimeNamespace(kind, terminationRequestTokenRid) { + let props = { + scheduleTermination: () => ops.op_cancel_drop_token(terminationRequestTokenRid) + }; + + switch (kind) { + case "main": + props = { + ai, + userWorkers: SUPABASE_USER_WORKERS, + getRuntimeMetrics: () => /* async */ ops.op_runtime_metrics(), + applySupabaseTag: (src, dest) => applySupabaseTag(src, dest), + systemMemoryInfo: () => ops.op_system_memory_info(), + raiseSegfault: () => ops.op_raise_segfault(), + ...props, + }; + break; + + case "event": + props = { + ...props + }; + break; + + case "user": + props = { + waitUntil + }; + break; + } + + if (props === void 0) { + return; + } + + ObjectDefineProperty(globalThis, "EdgeRuntime", { + get() { + return props; + }, + configurable: true, + }); +} + +export { + installEdgeRuntimeNamespace, +} diff --git a/crates/sb_core/js/http.js b/crates/sb_core/js/http.js index 347326a7d..7dbcdb819 100644 --- a/crates/sb_core/js/http.js +++ b/crates/sb_core/js/http.js @@ -8,13 +8,17 @@ import { upgradeWebSocket } from "ext:deno_http/02_websocket.ts"; const ops = core.ops; -const { internalRidSymbol } = core; +const { BadResourcePrototype, internalRidSymbol } = core; const { ObjectPrototypeIsPrototypeOf } = primordials; const HttpConnPrototypeNextRequest = HttpConn.prototype.nextRequest; const HttpConnPrototypeClose = HttpConn.prototype.close; const kSupabaseTag = Symbol("kSupabaseTag"); + +let ACTIVE_REQUESTS = 0; + +const HTTP_CONNS = new Set(); const RAW_UPGRADE_RESPONSE_SENTINEL = fromInnerResponse( newInnerResponse(101), "immutable", @@ -74,11 +78,14 @@ function serveHttp(conn) { httpConn.close = () => { if (!closed) { closed = true; + HTTP_CONNS.delete(httpConn); core.tryClose(watcherRid); HttpConnPrototypeClose.call(httpConn); } }; + HTTP_CONNS.add(httpConn); + return httpConn; } @@ -115,12 +122,15 @@ function serve(args1, args2) { try { for await (const requestEvent of currentHttpConn) { + ACTIVE_REQUESTS++; // NOTE: Respond to the request. Note we do not await this async // method to allow the connection to handle multiple requests in // the case of h2. // // [1]: https://deno.land/std@0.131.0/http/server.ts?source=#L338 - respond(requestEvent, currentHttpConn, options); + respond(requestEvent, currentHttpConn, options).then(() => { + ACTIVE_REQUESTS--; + }); } } catch { // connection has been closed @@ -140,10 +150,34 @@ function serve(args1, args2) { } })(); + const kind = internals.worker.kind; + const shutdownEventName = kind === "user" ? "drain" : "beforeunload"; + const handleShutdownEvent = () => { + shutdown(); + }; + const shutdown = () => { - // TODO: Not currently supported + removeEventListener(shutdownEventName, handleShutdownEvent); + + try { + listener.close(); + } catch (error) { + if ( + ObjectPrototypeIsPrototypeOf(BadResourcePrototype, error) + ) { + return; + } + + throw error; + } + + for (const httpConn of HTTP_CONNS) { + closeHttpConn(httpConn); + } }; + addEventListener(shutdownEventName, handleShutdownEvent, { once: true }); + return { finished, shutdown, diff --git a/crates/sb_core/js/main_worker.js b/crates/sb_core/js/main_worker.js deleted file mode 100644 index cb8d74af8..000000000 --- a/crates/sb_core/js/main_worker.js +++ /dev/null @@ -1,22 +0,0 @@ -import { core, primordials } from 'ext:core/mod.js'; - -import { MAIN_WORKER_API as ai } from 'ext:sb_ai/js/ai.js'; -import { SUPABASE_USER_WORKERS } from 'ext:sb_user_workers/user_workers.js'; -import { applySupabaseTag } from 'ext:sb_core_main_js/js/http.js'; - -const ops = core.ops; -const { ObjectDefineProperty } = primordials; - -ObjectDefineProperty(globalThis, 'EdgeRuntime', { - get() { - return { - ai, - userWorkers: SUPABASE_USER_WORKERS, - getRuntimeMetrics: () => /* async */ ops.op_runtime_metrics(), - applySupabaseTag: (src, dest) => applySupabaseTag(src, dest), - systemMemoryInfo: () => ops.op_system_memory_info(), - raiseSegfault: () => ops.op_raise_segfault(), - }; - }, - configurable: true, -}); diff --git a/crates/sb_core/lib.rs b/crates/sb_core/lib.rs index d3dcb8bba..afe8d6e2f 100644 --- a/crates/sb_core/lib.rs +++ b/crates/sb_core/lib.rs @@ -4,10 +4,11 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use base_mem_check::WorkerHeapStatistics; +use base_rt::DropToken; use deno_core::error::AnyError; -use deno_core::v8; use deno_core::OpState; use deno_core::{op2, JsRuntime}; +use deno_core::{v8, ResourceId}; use enum_as_inner::EnumAsInner; use futures::task::AtomicWaker; use futures::FutureExt; @@ -397,6 +398,14 @@ fn op_tap_promise_metrics(state: &mut OpState, #[string] kind: &str) { debug!(?metrics); } +#[op2(fast)] +fn op_cancel_drop_token(state: &mut OpState, #[smi] rid: ResourceId) -> Result<(), AnyError> { + let token = state.resource_table.get::(rid)?; + + token.0.cancel(); + Ok(()) +} + #[op2] #[serde] pub fn op_bootstrap_unstable_args(_state: &mut OpState) -> Vec { @@ -406,7 +415,7 @@ pub fn op_bootstrap_unstable_args(_state: &mut OpState) -> Vec { deno_core::extension!( sb_core_main_js, ops = [ - /*op_is_terminal,*/ + // op_is_terminal, op_stdin_set_raw, op_console_size, op_read_line_prompt, @@ -418,9 +427,11 @@ deno_core::extension!( op_bootstrap_unstable_args, op_raise_segfault, op_tap_promise_metrics, + op_cancel_drop_token, ], esm_entry_point = "ext:sb_core_main_js/js/bootstrap.js", esm = [ + "js/edge_runtime.js", "js/async_hook.js", "js/permissions.js", "js/errors.js", @@ -430,7 +441,6 @@ deno_core::extension!( "js/denoOverrides.js", "js/navigator.js", "js/bootstrap.js", - "js/main_worker.js", "js/00_serve.js", "js/01_http.js" ] diff --git a/crates/sb_core/net.rs b/crates/sb_core/net.rs index 3275ab8b2..dcbd91125 100644 --- a/crates/sb_core/net.rs +++ b/crates/sb_core/net.rs @@ -76,13 +76,24 @@ impl Resource for TokioDuplexResource { } } +#[derive(Debug, Clone, Default)] +struct ListenMarker(CancellationToken); + +impl Drop for ListenMarker { + fn drop(&mut self) { + self.0.cancel(); + } +} + +impl Resource for ListenMarker {} + #[op2] #[serde] -pub fn op_net_listen(_state: &mut OpState) -> Result<(ResourceId, IpAddr), AnyError> { +pub fn op_net_listen(state: &mut OpState) -> Result<(ResourceId, IpAddr), AnyError> { // this is a noop // TODO: customize to match the service ip and port Ok(( - 0, + state.resource_table.add(ListenMarker::default()), IpAddr { hostname: "0.0.0.0".to_string(), port: 9999, @@ -94,7 +105,15 @@ pub fn op_net_listen(_state: &mut OpState) -> Result<(ResourceId, IpAddr), AnyEr #[serde] pub async fn op_net_accept( state: Rc>, + #[smi] rid: ResourceId, ) -> Result<(ResourceId, IpAddr, IpAddr), AnyError> { + let accept_token = state + .borrow() + .resource_table + .get::(rid)? + .0 + .clone(); + // we do not want to keep the op_state locked, // so we take the channel receiver from it and release op state. // we need to add it back later after processing a message. @@ -112,11 +131,10 @@ pub async fn op_net_accept( ) }; - if rx.is_none() { + let Some(rx) = rx else { return Err(bad_resource("duplex stream receiver is already used")); - } + }; - let rx = rx.unwrap(); let mut rx = scopeguard::guard(rx, { let state = state.clone(); move |value| { @@ -127,8 +145,14 @@ pub async fn op_net_accept( } }); - let Some((stream, conn_token)) = rx.recv().await else { - return Err(bad_resource("duplex stream channel is closed")); + let (stream, conn_token) = match tokio::select! { + ret = rx.recv() => { ret } + _ = accept_token.cancelled() => { None } + } { + Some(ret) => ret, + None => { + return Err(bad_resource("duplex stream channel is closed")); + } }; let resource = TokioDuplexResource::new(stream); @@ -148,7 +172,7 @@ pub async fn op_net_accept( async move { let _lt_track = span!(Level::DEBUG, "lt_track", id); tokio::select! { - _ = runtime_token.0.cancelled_owned() => { + _ = runtime_token.cancelled_owned() => { if !token.is_cancelled() { token.cancel(); } From 0529aaedee773cd1cec871abb39fcefa1eb5b478 Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:40:37 +0000 Subject: [PATCH 8/9] stamp: polishing --- crates/base/src/utils/test_utils.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/crates/base/src/utils/test_utils.rs b/crates/base/src/utils/test_utils.rs index 14f26684a..bfb7b2adc 100644 --- a/crates/base/src/utils/test_utils.rs +++ b/crates/base/src/utils/test_utils.rs @@ -119,18 +119,6 @@ impl Future for RequestScopeGuard { } } -// pub trait WorkerContextInitOptsForTesting { -// fn with_policy(self, policy: SupervisorPolicy) -> CreateWorkerArgs -// where -// Self: Sized; -// } - -// impl WorkerContextInitOptsForTesting for WorkerContextInitOpts { -// fn with_policy(self, policy: SupervisorPolicy) -> CreateWorkerArgs { -// (self, policy).into() -// } -// } - pub struct TestBedBuilder { main_service_path: PathBuf, worker_pool_policy: Option, From d3ab51e3f3d09302a8b6e473715971c794d072bc Mon Sep 17 00:00:00 2001 From: Nyannyacha Date: Tue, 17 Dec 2024 02:44:47 +0000 Subject: [PATCH 9/9] stamp: polishing --- examples/main/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/main/index.ts b/examples/main/index.ts index 92ea9424c..2f60592c1 100644 --- a/examples/main/index.ts +++ b/examples/main/index.ts @@ -103,7 +103,7 @@ Deno.serve(async (req: Request) => { const createWorker = async () => { const memoryLimitMb = 150; - const workerTimeoutMs = 100 * 1000; + const workerTimeoutMs = 5 * 60 * 1000; const noModuleCache = false; // you can provide an import map inline @@ -129,8 +129,8 @@ Deno.serve(async (req: Request) => { // or load module source from an inline module // const maybeModuleCode = 'Deno.serve((req) => new Response("Hello from Module Code"));'; // - const cpuTimeSoftLimitMs = 1000; - const cpuTimeHardLimitMs = 2000; + const cpuTimeSoftLimitMs = 10000; + const cpuTimeHardLimitMs = 20000; return await EdgeRuntime.userWorkers.create({ servicePath, @@ -195,4 +195,4 @@ Deno.serve(async (req: Request) => { }; return callWorker(); -}); \ No newline at end of file +});