From c59b7187063b33c07abce806bd372ca0701b542f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Th=C3=B6mmes?= Date: Tue, 29 Nov 2022 19:07:35 +0100 Subject: [PATCH 1/4] SERVERLESS-2327 Map environment into a context parameter This translates our current environment variable based approach into passing a second "context" parameter to the function. The context somewhat resembles Lambda's context parameter but notably keeps the notion of an "activation_id" as that's still an entity we use in our docs and API. --- core/python3Action/lib/launcher.py | 24 +++++++++- core/python3Action/lib/prelauncher.py | 24 +++++++++- .../PythonAdvancedTests.scala | 44 ++++++++++++++++++- 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/core/python3Action/lib/launcher.py b/core/python3Action/lib/launcher.py index e6c5e31a..29222ada 100755 --- a/core/python3Action/lib/launcher.py +++ b/core/python3Action/lib/launcher.py @@ -19,7 +19,7 @@ from sys import stdout from sys import stderr from os import fdopen -import sys, os, json, traceback, warnings +import sys, os, json, traceback, time log_sentinel="XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX\n" @@ -45,6 +45,26 @@ # now import the action as process input/output from main__ import main as main +class Context: + def __init__(self, env): + self.function_name = env["__OW_ACTION_NAME"] + self.function_version = env["__OW_ACTION_VERSION"] + self.activation_id = env["__OW_ACTIVATION_ID"] + self.deadline = int(os.environ["__OW_DEADLINE"]) + + def get_remaining_time_in_millis(self): + epoch_now_in_ms = int(time.time() * 1000) + delta_ms = self.deadline - epoch_now_in_ms + return delta_ms if delta_ms > 0 else 0 + +def fun(payload, env): + # Compatibility: Supports "old" context-less functions. + if main.__code__.co_argcount == 1: + return main(payload) + + # Lambda-like "new-style" function. + return main(payload, Context(env)) + out = fdopen(3, "wb") if os.getenv("__OW_WAIT_FOR_ACK", "") != "": out.write(json.dumps({"ok": True}, ensure_ascii=False).encode('utf-8')) @@ -64,7 +84,7 @@ env["__OW_%s" % key.upper()]= args[key] res = {} try: - res = main(payload) + res = fun(payload, env) except Exception as ex: print(traceback.format_exc(), file=stderr) res = {"error": str(ex)} diff --git a/core/python3Action/lib/prelauncher.py b/core/python3Action/lib/prelauncher.py index c02e41e3..6bc62b8b 100755 --- a/core/python3Action/lib/prelauncher.py +++ b/core/python3Action/lib/prelauncher.py @@ -21,7 +21,7 @@ from sys import stdout from sys import stderr from os import fdopen -import sys, os, json, traceback, base64, io, zipfile +import sys, os, json, traceback, base64, io, zipfile, time log_sentinel="XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX\n" def write_sentinels(): @@ -97,6 +97,26 @@ def cannot_start(msg): except Exception as ex: cannot_start("Invalid action: %s\n" % str(ex)) +class Context: + def __init__(self, env): + self.function_name = env["__OW_ACTION_NAME"] + self.function_version = env["__OW_ACTION_VERSION"] + self.activation_id = env["__OW_ACTIVATION_ID"] + self.deadline = int(os.environ["__OW_DEADLINE"]) + + def get_remaining_time_in_millis(self): + epoch_now_in_ms = int(time.time() * 1000) + delta_ms = self.deadline - epoch_now_in_ms + return delta_ms if delta_ms > 0 else 0 + +def fun(payload, env): + # Compatibility: Supports "old" context-less functions. + if main.__code__.co_argcount == 1: + return main(payload) + + # Lambda-like "new-style" function. + return main(payload, Context(env)) + # Acknowledge the initialization. write_result({"ok": True}) @@ -113,7 +133,7 @@ def cannot_start(msg): os.environ["__OW_%s" % key.upper()]= args[key] res = {} try: - res = main(payload) + res = fun(payload, os.environ) except Exception as ex: print(traceback.format_exc(), file=stderr) res = {"error": str(ex)} diff --git a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala index d4bd9d9a..0a0e0b00 100644 --- a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala +++ b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala @@ -16,7 +16,8 @@ */ package runtime.actionContainers -import spray.json.{JsObject, JsString} +import spray.json._ +import spray.json.DefaultJsonProtocol._ trait PythonAdvancedTests { this: PythonBasicTests => @@ -91,4 +92,45 @@ trait PythonAdvancedTests { e shouldBe empty }) } + + Map( + "prelaunched" -> Map.empty[String, String], + "non-prelaunched" -> Map("OW_INIT_IN_ACTIONLOOP" -> ""), + ).foreach { case (name, env) => + it should s"support a function with a lambda-like signature $name" in { + val (out, err) = withActionContainer(env) { c => + val code = + """ + |def main(event, context): + | return { + | "remaining_time": context.get_remaining_time_in_millis(), + | "activation_id": context.activation_id, + | "function_name": context.function_name, + | "function_version": context.function_version + | } + """.stripMargin + + val (initCode, _) = c.init(initPayload(code)) + initCode should be(200) + + val (runCode, out) = c.run(runPayload( + JsObject(), + Some(JsObject( + "deadline" -> "0".toJson, + "activation_id" -> "testid".toJson, + "action_name" -> "testfunction".toJson, + "action_version" -> "0.0.1".toJson + )) + )) + runCode should be(200) + + out shouldBe Some(JsObject( + "remaining_time" -> 0.toJson, // This being 0 proofs that the function exists and is callable at least. + "activation_id" -> "testid".toJson, + "function_name" -> "testfunction".toJson, + "function_version" -> "0.0.1".toJson + )) + } + } + } } From d75a5839b08a79007346aa6ee34fb739df3dee9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Th=C3=B6mmes?= Date: Thu, 1 Dec 2022 08:15:46 +0100 Subject: [PATCH 2/4] Narrow down test for remaining time --- .../runtime/actionContainers/PythonAdvancedTests.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala index 0a0e0b00..3822ee62 100644 --- a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala +++ b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala @@ -18,6 +18,7 @@ package runtime.actionContainers import spray.json._ import spray.json.DefaultJsonProtocol._ +import java.time.Instant trait PythonAdvancedTests { this: PythonBasicTests => @@ -116,7 +117,7 @@ trait PythonAdvancedTests { val (runCode, out) = c.run(runPayload( JsObject(), Some(JsObject( - "deadline" -> "0".toJson, + "deadline" -> Instant.now.plusSeconds(10).toEpochMilli.toString.toJson, "activation_id" -> "testid".toJson, "action_name" -> "testfunction".toJson, "action_version" -> "0.0.1".toJson @@ -124,8 +125,10 @@ trait PythonAdvancedTests { )) runCode should be(200) + val remainingTime = out.get.fields("remaining_time").convertTo[Int] + remainingTime should be > 9500 // We give the test 500ms of slack to invoke the function to avoid flakes. out shouldBe Some(JsObject( - "remaining_time" -> 0.toJson, // This being 0 proofs that the function exists and is callable at least. + "remaining_time" -> remainingTime.toJson, "activation_id" -> "testid".toJson, "function_name" -> "testfunction".toJson, "function_version" -> "0.0.1".toJson From 6141936ffbdea46a99b5a24d8a8d9a9cb1d60132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Th=C3=B6mmes?= Date: Thu, 1 Dec 2022 12:30:12 +0100 Subject: [PATCH 3/4] Add request_id to map tid --- core/python3Action/lib/launcher.py | 1 + core/python3Action/lib/prelauncher.py | 1 + .../runtime/actionContainers/PythonAdvancedTests.scala | 7 +++++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/python3Action/lib/launcher.py b/core/python3Action/lib/launcher.py index 29222ada..e9de1223 100755 --- a/core/python3Action/lib/launcher.py +++ b/core/python3Action/lib/launcher.py @@ -50,6 +50,7 @@ def __init__(self, env): self.function_name = env["__OW_ACTION_NAME"] self.function_version = env["__OW_ACTION_VERSION"] self.activation_id = env["__OW_ACTIVATION_ID"] + self.request_id = env["__OW_TRANSACTION_ID"] self.deadline = int(os.environ["__OW_DEADLINE"]) def get_remaining_time_in_millis(self): diff --git a/core/python3Action/lib/prelauncher.py b/core/python3Action/lib/prelauncher.py index 6bc62b8b..44c7a2c4 100755 --- a/core/python3Action/lib/prelauncher.py +++ b/core/python3Action/lib/prelauncher.py @@ -102,6 +102,7 @@ def __init__(self, env): self.function_name = env["__OW_ACTION_NAME"] self.function_version = env["__OW_ACTION_VERSION"] self.activation_id = env["__OW_ACTIVATION_ID"] + self.request_id = env["__OW_TRANSACTION_ID"] self.deadline = int(os.environ["__OW_DEADLINE"]) def get_remaining_time_in_millis(self): diff --git a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala index 3822ee62..ddf2aa40 100644 --- a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala +++ b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala @@ -106,6 +106,7 @@ trait PythonAdvancedTests { | return { | "remaining_time": context.get_remaining_time_in_millis(), | "activation_id": context.activation_id, + | "request_id": context.request_id, | "function_name": context.function_name, | "function_version": context.function_version | } @@ -118,7 +119,8 @@ trait PythonAdvancedTests { JsObject(), Some(JsObject( "deadline" -> Instant.now.plusSeconds(10).toEpochMilli.toString.toJson, - "activation_id" -> "testid".toJson, + "activation_id" -> "testaid".toJson, + "transaction_id" -> "testtid".toJson, "action_name" -> "testfunction".toJson, "action_version" -> "0.0.1".toJson )) @@ -129,7 +131,8 @@ trait PythonAdvancedTests { remainingTime should be > 9500 // We give the test 500ms of slack to invoke the function to avoid flakes. out shouldBe Some(JsObject( "remaining_time" -> remainingTime.toJson, - "activation_id" -> "testid".toJson, + "activation_id" -> "testaid".toJson, + "request_id" -> "testtid".toJson, "function_name" -> "testfunction".toJson, "function_version" -> "0.0.1".toJson )) From 791118e4b358122e36b013bae9bb50157d01f44e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20Th=C3=B6mmes?= Date: Thu, 1 Dec 2022 15:52:43 +0100 Subject: [PATCH 4/4] Add namespace, api_host and api_key --- core/python3Action/lib/launcher.py | 5 ++++- core/python3Action/lib/prelauncher.py | 5 ++++- .../actionContainers/PythonAdvancedTests.scala | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/core/python3Action/lib/launcher.py b/core/python3Action/lib/launcher.py index e9de1223..7b6d9a61 100755 --- a/core/python3Action/lib/launcher.py +++ b/core/python3Action/lib/launcher.py @@ -51,7 +51,10 @@ def __init__(self, env): self.function_version = env["__OW_ACTION_VERSION"] self.activation_id = env["__OW_ACTIVATION_ID"] self.request_id = env["__OW_TRANSACTION_ID"] - self.deadline = int(os.environ["__OW_DEADLINE"]) + self.deadline = int(env["__OW_DEADLINE"]) + self.api_host = env["__OW_API_HOST"] + self.api_key = env.get("__OW_AUTH_KEY", "") + self.namespace = env["__OW_NAMESPACE"] def get_remaining_time_in_millis(self): epoch_now_in_ms = int(time.time() * 1000) diff --git a/core/python3Action/lib/prelauncher.py b/core/python3Action/lib/prelauncher.py index 44c7a2c4..e4bfb3dd 100755 --- a/core/python3Action/lib/prelauncher.py +++ b/core/python3Action/lib/prelauncher.py @@ -103,7 +103,10 @@ def __init__(self, env): self.function_version = env["__OW_ACTION_VERSION"] self.activation_id = env["__OW_ACTIVATION_ID"] self.request_id = env["__OW_TRANSACTION_ID"] - self.deadline = int(os.environ["__OW_DEADLINE"]) + self.deadline = int(env["__OW_DEADLINE"]) + self.api_host = env["__OW_API_HOST"] + self.api_key = env.get("__OW_AUTH_KEY", "") + self.namespace = env["__OW_NAMESPACE"] def get_remaining_time_in_millis(self): epoch_now_in_ms = int(time.time() * 1000) diff --git a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala index ddf2aa40..007485ef 100644 --- a/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala +++ b/tests/src/test/scala/runtime/actionContainers/PythonAdvancedTests.scala @@ -99,7 +99,7 @@ trait PythonAdvancedTests { "non-prelaunched" -> Map("OW_INIT_IN_ACTIONLOOP" -> ""), ).foreach { case (name, env) => it should s"support a function with a lambda-like signature $name" in { - val (out, err) = withActionContainer(env) { c => + val (out, err) = withActionContainer(env + ("__OW_API_HOST" -> "testhost")) { c => val code = """ |def main(event, context): @@ -108,7 +108,10 @@ trait PythonAdvancedTests { | "activation_id": context.activation_id, | "request_id": context.request_id, | "function_name": context.function_name, - | "function_version": context.function_version + | "function_version": context.function_version, + | "api_host": context.api_host, + | "api_key": context.api_key, + | "namespace": context.namespace | } """.stripMargin @@ -122,7 +125,9 @@ trait PythonAdvancedTests { "activation_id" -> "testaid".toJson, "transaction_id" -> "testtid".toJson, "action_name" -> "testfunction".toJson, - "action_version" -> "0.0.1".toJson + "action_version" -> "0.0.1".toJson, + "namespace" -> "testnamespace".toJson, + "auth_key" -> "testkey".toJson )) )) runCode should be(200) @@ -134,7 +139,10 @@ trait PythonAdvancedTests { "activation_id" -> "testaid".toJson, "request_id" -> "testtid".toJson, "function_name" -> "testfunction".toJson, - "function_version" -> "0.0.1".toJson + "function_version" -> "0.0.1".toJson, + "api_host" -> "testhost".toJson, + "api_key" -> "testkey".toJson, + "namespace" -> "testnamespace".toJson )) } }