feat: revamp actor build endpoint, js builds -> tar

packages/api/actor/src/route/builds.rs

@@ -1,9 +1,8 @@
 use std::collections::HashMap;
 
 use api_helper::{anchor::WatchIndexQuery, ctx::Ctx};
-use proto::backend;
 use rivet_api::models;
-use rivet_convert::ApiTryInto;
+use rivet_convert::{ApiInto, ApiTryInto};
 use rivet_operation::prelude::*;
 use serde::Deserialize;
 use serde_json::json;
@@ -190,8 +189,7 @@ pub async fn patch_tags(
 	let CheckOutput { env_id, .. } = ctx.auth().check(ctx.op_ctx(), &query, false).await?;
 
 	let tags = unwrap_with!(body.tags, API_BAD_BODY, error = "missing field `tags`");
-	let tags = serde_json::from_value::<HashMap<String, Option<String>>>(tags)
-		.map_err(|err| err_code!(API_BAD_BODY, error = err))?;
+	let tags = serde_json::from_value(tags).map_err(|err| err_code!(API_BAD_BODY, error = err))?;
 
 	let build_res = ctx
 		.op(build::ops::get::Input {
@@ -246,44 +244,48 @@ pub async fn create_build(
 
 	let (kind, image_tag) = match body.kind {
 		Option::None | Some(models::ActorBuildKind::DockerImage) => (
-			backend::build::BuildKind::DockerImage,
+			build::types::BuildKind::DockerImage,
 			unwrap_with!(
 				body.image_tag,
 				API_BAD_BODY,
 				error = "missing field `image_tag`"
 			),
 		),
 		Some(models::ActorBuildKind::OciBundle) => (
-			backend::build::BuildKind::OciBundle,
+			build::types::BuildKind::OciBundle,
 			// HACK(RVT-4125): Generate nonexistent image tag
 			body.image_tag
 				.unwrap_or_else(|| format!("nonexistent:{}", Uuid::new_v4())),
 		),
 		Some(models::ActorBuildKind::Javascript) => (
-			backend::build::BuildKind::JavaScript,
+			build::types::BuildKind::JavaScript,
 			// HACK(RVT-4125): Generate nonexistent image tag
 			body.image_tag
 				.unwrap_or_else(|| format!("nonexistent:{}", Uuid::new_v4())),
 		),
 	};
 
-	let compression = match body.compression {
-		Option::None | Some(models::ActorBuildCompression::None) => {
-			backend::build::BuildCompression::None
-		}
-		Some(models::ActorBuildCompression::Lz4) => backend::build::BuildCompression::Lz4,
-	};
-
-	let create_res = op!([ctx] build_create {
-		env_id: Some(env_id.into()),
-		display_name: body.name,
-		image_tag: Some(image_tag),
-		image_file: Some((*body.image_file).api_try_into()?),
-		kind: kind as i32,
-		compression: compression as i32,
-	})
-	.await?;
-	let build_id = unwrap_ref!(create_res.build_id).as_uuid();
+	let create_res = ctx
+		.op(build::ops::create::Input {
+			game_id: None,
+			env_id: Some(env_id),
+			tags: body
+				.tags
+				.map(serde_json::from_value)
+				.transpose()?
+				.unwrap_or_default(),
+			display_name: body.name,
+			content: build::ops::create::Content::New {
+				image_file: (*body.image_file).api_try_into()?,
+				image_tag,
+			},
+			kind,
+			compression: body
+				.compression
+				.map(ApiInto::api_into)
+				.unwrap_or(build::types::BuildCompression::None),
+		})
+		.await?;
 
 	let cluster_res = ctx
 		.op(cluster::ops::get_for_game::Input {
@@ -320,38 +322,18 @@ pub async fn create_build(
 	if !prewarm_datacenter_ids.is_empty() {
 		ctx.op(build::ops::prewarm_ats::Input {
 			datacenter_ids: prewarm_datacenter_ids,
-			build_ids: vec![build_id],
+			build_ids: vec![create_res.build_id],
 		})
 		.await?;
 	}
 
-	let image_presigned_request = if !multipart_upload {
-		Some(Box::new(
-			unwrap!(create_res.image_presigned_requests.first())
-				.clone()
-				.api_try_into()?,
-		))
-	} else {
-		None
-	};
-
-	let image_presigned_requests = if multipart_upload {
-		Some(
-			create_res
-				.image_presigned_requests
-				.iter()
-				.cloned()
-				.map(ApiTryInto::api_try_into)
-				.collect::<GlobalResult<Vec<_>>>()?,
-		)
-	} else {
-		None
-	};
-
 	Ok(models::ActorPrepareBuildResponse {
-		build: build_id,
-		image_presigned_request,
-		image_presigned_requests,
+		build: create_res.build_id,
+		presigned_requests: create_res
+			.presigned_requests
+			.into_iter()
+			.map(ApiTryInto::api_try_into)
+			.collect::<GlobalResult<Vec<_>>>()?,
 	})
 }
 
@@ -394,15 +376,31 @@ pub async fn create_build_deprecated(
 			}),
 			multipart_upload: body.multipart_upload,
 			name: body.name,
+			tags: None,
 			prewarm_regions,
 		},
 		global,
 	)
 	.await?;
+
+	let multipart_upload = body.multipart_upload.unwrap_or(false);
+
+	let (image_presigned_request, image_presigned_requests) = if !multipart_upload {
+		(
+			Some(Box::new(unwrap!(build_res
+				.presigned_requests
+				.into_iter()
+				.next()))),
+			None,
+		)
+	} else {
+		(None, Some(build_res.presigned_requests))
+	};
+
 	Ok(models::ServersCreateBuildResponse {
 		build: build_res.build,
-		image_presigned_request: build_res.image_presigned_request,
-		image_presigned_requests: build_res.image_presigned_requests,
+		image_presigned_request,
+		image_presigned_requests,
 	})
 }
 

packages/infra/client/isolate-v8-runner/src/isolate.rs

@@ -142,10 +142,13 @@ pub async fn run_inner(
 
 	tracing::info!(?actor_id, "Isolate KV initialized");
 
-	// Load script into a static module loader. No dynamic scripts can be loaded this way.
-	let script_content = fs::read_to_string(actor_path.join("index.js"))
+	// Should match the path from `Actor::download_image` in manager/src/actor/setup.rs. index.js might not
+	// exist but thats up to the user to bundle it correctly.
+	let script_content = fs::read_to_string(actor_path.join("js-bundle").join("index.js"))
 		.await
 		.with_context(|| format!("failed to load {}", actor_path.join("index.js").display()))?;
+
+	// Load script into a static module loader. No dynamic scripts can be loaded this way.
 	let main_module = ModuleSpecifier::from_file_path(Path::new("/index.js"))
 		.map_err(|_| anyhow!("invalid file name"))?;
 	let loader = StaticModuleLoader::new([(main_module.clone(), script_content)]);

packages/infra/client/manager/src/actor/setup.rs

@@ -20,7 +20,6 @@ impl Actor {
 		tracing::info!(actor_id=?self.actor_id, "downloading artifact");
 
 		let actor_path = ctx.actor_path(self.actor_id);
-		let oci_bundle_path = actor_path.join("oci-bundle");
 
 		let mut stream = reqwest::get(&self.config.image.artifact_url)
 			.await?
@@ -82,10 +81,16 @@ impl Actor {
 					}
 				}
 			}
-			protocol::ImageKind::OciBundle => {
+			protocol::ImageKind::OciBundle | protocol::ImageKind::JavaScript => {
 				tracing::info!(actor_id=?self.actor_id, "decompressing and unarchiving artifact");
 
-				fs::create_dir(&oci_bundle_path).await?;
+				let bundle_path = match self.config.image.kind {
+					protocol::ImageKind::OciBundle => actor_path.join("oci-bundle"),
+					protocol::ImageKind::JavaScript => actor_path.join("js-bundle"),
+					_ => unreachable!(),
+				};
+
+				fs::create_dir(&bundle_path).await?;
 
 				// Spawn the lz4 process
 				let mut lz4_child = Command::new("lz4")
@@ -98,7 +103,7 @@ impl Actor {
 				let mut tar_child = Command::new("tar")
 					.arg("-x")
 					.arg("-C")
-					.arg(&oci_bundle_path)
+					.arg(&bundle_path)
 					.stdin(Stdio::piped())
 					.spawn()?;
 
@@ -147,68 +152,14 @@ impl Actor {
 						let cmd_out = tar_child.wait_with_output().await?;
 						ensure!(
 							cmd_out.status.success(),
-							"failed `lz4` command\n{}",
+							"failed `tar` command\n{}",
 							std::str::from_utf8(&cmd_out.stderr)?
 						);
 
 						Ok(())
 					},
 				)?;
 			}
-			protocol::ImageKind::JavaScript => {
-				let script_path = actor_path.join("index.js");
-
-				match self.config.image.compression {
-					protocol::ImageCompression::None => {
-						tracing::info!(actor_id=?self.actor_id, "saving artifact to file");
-
-						let mut output_file = File::create(&script_path).await?;
-
-						// Write from stream to file
-						while let Some(chunk) = stream.next().await {
-							output_file.write_all(&chunk?).await?;
-						}
-					}
-					protocol::ImageCompression::Lz4 => {
-						tracing::info!(actor_id=?self.actor_id, "decompressing artifact");
-
-						// Spawn the lz4 process
-						let mut lz4_child = Command::new("lz4")
-							.arg("-d")
-							.arg("-")
-							.arg(&script_path)
-							.stdin(Stdio::piped())
-							.spawn()?;
-
-						// Take the stdin of lz4
-						let mut lz4_stdin = lz4_child.stdin.take().context("lz4 stdin")?;
-
-						tokio::try_join!(
-							// Pipe the response body to lz4 stdin
-							async move {
-								while let Some(chunk) = stream.next().await {
-									let data = chunk?;
-									lz4_stdin.write_all(&data).await?;
-								}
-								lz4_stdin.shutdown().await?;
-
-								anyhow::Ok(())
-							},
-							// Wait for child process
-							async {
-								let cmd_out = lz4_child.wait_with_output().await?;
-								ensure!(
-									cmd_out.status.success(),
-									"failed `lz4` command\n{}",
-									std::str::from_utf8(&cmd_out.stderr)?
-								);
-
-								Ok(())
-							},
-						)?;
-					}
-				}
-			}
 		}
 
 		Ok(())

packages/infra/client/manager/tests/common.rs

@@ -341,14 +341,17 @@ pub async fn build_binaries(gen_path: &Path) {
 	assert!(status.success());
 
 	// Js image
-	tokio::fs::copy(
-		Path::new(env!("CARGO_MANIFEST_DIR"))
-			.join("tests")
-			.join("echo.js"),
-		js_image_path(gen_path),
-	)
-	.await
-	.unwrap();
+	let status = Command::new("tar")
+		.arg("-czf")
+		.arg(js_image_path(gen_path))
+		.arg("-C")
+		.arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"))
+		.arg("echo.js")
+		.status()
+		.await
+		.unwrap();
+
+	assert!(status.success());
 
 	build_runner(gen_path, "container").await;
 	build_runner(gen_path, "isolate-v8").await;

packages/services/build/Cargo.toml

@@ -19,23 +19,26 @@ nomad-client = "0.0.9"
 nomad-util.workspace = true
 rand = "0.8"
 reqwest = { version = "0.11", features = ["json"] }
+rivet-api.workspace = true
+rivet-convert.workspace = true
 rivet-metrics.workspace = true
 rivet-operation.workspace = true
 rivet-runtime.workspace = true
 s3-util.workspace = true
 serde = { version = "1.0.198", features = ["derive"] }
 serde_json = "1.0"
 ssh2 = "0.9.4"
-trust-dns-resolver = { version = "0.23.2", features = ["dns-over-native-tls"] }
 strum = { version = "0.26", features = ["derive"] }
 util-job.workspace = true
 
 cluster.workspace = true
+game-get.workspace = true
+game-namespace-get.workspace = true
 ip-info.workspace = true
 linode.workspace = true
 token-create.workspace = true
 upload-get.workspace = true
-rivet-config.workspace = true
+upload-prepare.workspace = true
 
 [dependencies.sqlx]
 workspace = true