From d6cfa0fe2b6c2358f6f58482488375a1781c7147 Mon Sep 17 00:00:00 2001 From: Brian Holt Date: Mon, 8 Nov 2021 15:31:02 -0600 Subject: [PATCH 1/5] add tracing support with Natchez --- build.sbt | 8 +++- core/src/main/scala/feral/IOSetup.scala | 12 ++++-- .../lambda/ApiGatewayProxyHttp4sLambda.scala | 4 +- .../CloudFormationCustomResource.scala | 8 ++-- .../scala/feral/lambda/IOLambdaPlatform.scala | 24 ++++++++--- .../scala/feral/lambda/IOLambdaPlatform.scala | 37 +++++++++-------- .../main/scala/feral/lambda/IOLambda.scala | 7 ++-- .../main/scala/feral/lambda/XRayTracing.scala | 40 +++++++++++++++++++ 8 files changed, 104 insertions(+), 36 deletions(-) create mode 100644 lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala diff --git a/build.sbt b/build.sbt index 86096ef4..5cce4bc6 100644 --- a/build.sbt +++ b/build.sbt @@ -38,6 +38,7 @@ val catsEffectVersion = "3.2.9" val circeVersion = "0.14.1" val fs2Version = "3.2.2" val http4sVersion = "0.23.6" +val natchezVersion = "0.1.5" lazy val root = project @@ -62,7 +63,9 @@ lazy val core = crossProject(JSPlatform, JVMPlatform) .settings( name := "feral-core", libraryDependencies ++= Seq( - "org.typelevel" %%% "cats-effect" % catsEffectVersion + "org.typelevel" %%% "cats-effect" % catsEffectVersion, + "org.tpolecat" %%% "natchez-core" % natchezVersion, + "org.tpolecat" %%% "natchez-noop" % natchezVersion, // TODO Rob needs to publish this for SJS ) ) @@ -71,7 +74,8 @@ lazy val lambda = crossProject(JSPlatform, JVMPlatform) .settings( name := "feral-lambda", libraryDependencies ++= Seq( - "io.circe" %%% "circe-core" % circeVersion + "io.circe" %%% "circe-core" % circeVersion, +// "org.tpolecat" %%% "natchez-xray" % natchezVersion, // TODO uncomment when this is published ) ) .jsSettings( diff --git a/core/src/main/scala/feral/IOSetup.scala b/core/src/main/scala/feral/IOSetup.scala index eda4dbe8..48285b3c 100644 --- a/core/src/main/scala/feral/IOSetup.scala +++ b/core/src/main/scala/feral/IOSetup.scala @@ -17,10 +17,12 @@ package feral import cats.effect.unsafe.IORuntime -import cats.effect.kernel.Resource -import cats.effect.IO -import cats.effect.kernel.Deferred +import cats.effect.{Deferred, IO, Resource} import cats.syntax.all._ +import natchez.Span +import natchez.noop.NoopSpan + +import scala.annotation.nowarn private[feral] trait IOSetup { @@ -29,6 +31,10 @@ private[feral] trait IOSetup { protected type Setup protected def setup: Resource[IO, Setup] = Resource.pure(null.asInstanceOf[Setup]) + @nowarn(value = "msg=parameter value name in method traceRootSpan is never used") + protected def traceRootSpan(name: String): Resource[IO, Span[IO]] = + Resource.pure(NoopSpan[IO]()) + private[feral] final lazy val setupMemo: IO[Setup] = { val deferred = Deferred.unsafe[IO, Either[Throwable, Setup]] setup diff --git a/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala b/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala index 06d1bce9..60591ead 100644 --- a/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala +++ b/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala @@ -22,6 +22,7 @@ import cats.effect.SyncIO import feral.lambda.events.ApiGatewayProxyEventV2 import feral.lambda.events.ApiGatewayProxyStructuredResultV2 import fs2.Stream +import natchez.Trace import org.http4s.Charset import org.http4s.Header import org.http4s.Headers @@ -47,7 +48,8 @@ abstract class ApiGatewayProxyHttp4sLambda override final def apply( event: ApiGatewayProxyEventV2, context: Context, - routes: HttpRoutes[IO]): IO[Some[ApiGatewayProxyStructuredResultV2]] = + routes: HttpRoutes[IO])( + implicit T: Trace[IO]): IO[Some[ApiGatewayProxyStructuredResultV2]] = for { method <- IO.fromEither(Method.fromString(event.requestContext.http.method)) uri <- IO.fromEither(Uri.fromString(event.rawPath)) diff --git a/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala b/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala index 04cda301..d5b79f78 100644 --- a/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala +++ b/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala @@ -16,17 +16,17 @@ package feral.lambda.cloudformation -import cats.effect._ -import cats.effect.kernel.Resource +import cats.effect.{IO, Resource} import cats.syntax.all._ import feral.lambda.cloudformation.CloudFormationCustomResourceHandler.stackTraceLines import feral.lambda.cloudformation.CloudFormationRequestType._ import feral.lambda.{Context, IOLambda} import io.circe._ import io.circe.syntax._ +import natchez.Trace import org.http4s.Method.POST -import org.http4s.client.Client import org.http4s.circe._ +import org.http4s.client.Client import org.http4s.client.dsl.io._ import org.http4s.ember.client.EmberClientBuilder @@ -59,7 +59,7 @@ abstract class CloudFormationCustomResourceHandler[Input: Decoder, Output: Encod override def apply( event: CloudFormationCustomResourceRequest[Input], context: Context, - setup: Setup): IO[Option[Unit]] = + setup: Setup)(implicit T: Trace[IO]): IO[Option[Unit]] = (event.RequestType match { case CreateRequest => setup._2.createResource(event, context) case UpdateRequest => setup._2.updateResource(event, context) diff --git a/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala b/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala index d71bbeb9..eadb38dd 100644 --- a/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala +++ b/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala @@ -16,8 +16,10 @@ package feral.lambda -import cats.effect.IO +import cats.effect.{IO, Resource} +import cats.syntax.all._ import io.circe.scalajs._ +import natchez.Trace.ioTrace import scala.scalajs.js import scala.scalajs.js.JSConverters._ @@ -26,11 +28,21 @@ import scala.scalajs.js.| private[lambda] trait IOLambdaPlatform[Event, Result] { this: IOLambda[Event, Result] => + private def ioHandler(event: js.Any, context: facade.Context): IO[js.Any | Unit] = + Resource + .pure(Context.fromJS(context)) + .flatMap { (context: Context) => + traceRootSpan(context.functionName).evalMap(ioTrace).evalMap { implicit trace => + for { + setup <- setupMemo + event <- IO.fromEither(decodeJs[Event](event)) + result <- apply(event, context, setup) + } yield result.map(_.asJsAny).orUndefined + } + } + .use(_.pure[IO]) + // @JSExportTopLevel("handler") // TODO final def handler(event: js.Any, context: facade.Context): js.Promise[js.Any | Unit] = - (for { - setup <- setupMemo - event <- IO.fromEither(decodeJs[Event](event)) - result <- apply(event, Context.fromJS(context), setup) - } yield result.map(_.asJsAny).orUndefined).unsafeToPromise()(runtime) + ioHandler(event, context).unsafeToPromise()(runtime) } diff --git a/lambda/jvm/src/main/scala/feral/lambda/IOLambdaPlatform.scala b/lambda/jvm/src/main/scala/feral/lambda/IOLambdaPlatform.scala index 73a7fdcd..8355bb36 100644 --- a/lambda/jvm/src/main/scala/feral/lambda/IOLambdaPlatform.scala +++ b/lambda/jvm/src/main/scala/feral/lambda/IOLambdaPlatform.scala @@ -21,6 +21,7 @@ import cats.effect.IO import cats.effect.kernel.Resource import com.amazonaws.services.lambda.{runtime => lambdaRuntime} import io.circe +import natchez.Trace.ioTrace import java.io.InputStream import java.io.OutputStream @@ -34,23 +35,25 @@ private[lambda] abstract class IOLambdaPlatform[Event, Result] output: OutputStream, context: lambdaRuntime.Context): Unit = Resource - .eval { - for { - setup <- setupMemo - event <- fs2 - .io - .readInputStream(IO.pure(input), 8192, closeAfterUse = false) - .through(circe.fs2.byteStreamParser) - .through(circe.fs2.decoder[IO, Event]) - .head - .compile - .lastOrError - context <- IO(Context.fromJava(context)) - _ <- OptionT(apply(event, context, setup)).foldF(IO.unit) { result => - // TODO can circe write directly to output? - IO(output.write(encoder(result).noSpaces.getBytes(StandardCharsets.UTF_8))) - } - } yield () + .eval(IO(Context.fromJava(context))) + .flatMap { (context: Context) => + traceRootSpan(context.functionName).evalMap(ioTrace).evalMap { implicit trace => + for { + setup <- setupMemo + event <- fs2 + .io + .readInputStream(IO.pure(input), 8192, closeAfterUse = false) + .through(circe.fs2.byteStreamParser) + .through(circe.fs2.decoder[IO, Event]) + .head + .compile + .lastOrError + _ <- OptionT(apply(event, context, setup)).foldF(IO.unit) { result => + // TODO can circe write directly to output? + IO(output.write(encoder(result).noSpaces.getBytes(StandardCharsets.UTF_8))) + } + } yield () + } } .onFinalize(IO(input.close())) .onFinalize(IO(output.close())) diff --git a/lambda/shared/src/main/scala/feral/lambda/IOLambda.scala b/lambda/shared/src/main/scala/feral/lambda/IOLambda.scala index 81d22e81..d5020979 100644 --- a/lambda/shared/src/main/scala/feral/lambda/IOLambda.scala +++ b/lambda/shared/src/main/scala/feral/lambda/IOLambda.scala @@ -18,8 +18,8 @@ package feral package lambda import cats.effect.IO -import io.circe.Decoder -import io.circe.Encoder +import io.circe.{Decoder, Encoder} +import natchez.Trace abstract class IOLambda[Event, Result]( implicit private[lambda] val decoder: Decoder[Event], @@ -27,5 +27,6 @@ abstract class IOLambda[Event, Result]( ) extends IOLambdaPlatform[Event, Result] with IOSetup { - def apply(event: Event, context: Context, setup: Setup): IO[Option[Result]] + def apply(event: Event, context: Context, setup: Setup)( + implicit T: Trace[IO]): IO[Option[Result]] } diff --git a/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala b/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala new file mode 100644 index 00000000..c97c1c72 --- /dev/null +++ b/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala @@ -0,0 +1,40 @@ +/* + * Copyright 2021 Typelevel + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* TODO this file won't compile until these PRs are merged & published: + * https://github.com/christiankjaer/natchez/pull/1 + * https://github.com/tpolecat/natchez/pull/427 + */ + +package feral.lambda + +import cats.effect.{IO, Resource} +import feral.IOSetup +import natchez.xray.{XRay, XRayEnvironment} +import natchez.{EntryPoint, Span} + +trait XRayTracing extends IOSetup { + private def traceEntryPoint: Resource[IO, EntryPoint[IO]] = + Resource.eval(XRayEnvironment[IO].daemonAddress).flatMap { + case Some(addr) => XRay.entryPoint(addr) + case None => XRay.entryPoint() + } + + override protected def traceRootSpan(name: String): Resource[IO, Span[IO]] = + Resource.eval(XRayEnvironment[IO].kernelFromEnvironment).flatMap { kernel => + traceEntryPoint.flatMap(_.continueOrElseRoot(name, kernel)) + } +} From 2404de8f73cae1de3790ae8eca73b554443c3a59 Mon Sep 17 00:00:00 2001 From: Brian Holt Date: Mon, 8 Nov 2021 18:46:42 -0600 Subject: [PATCH 2/5] add Trace[IO] to CloudFormationCustomResource --- .../CloudFormationCustomResource.scala | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala b/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala index d5b79f78..69f92f12 100644 --- a/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala +++ b/lambda-cloudformation-custom-resource/src/main/scala/feral/lambda/cloudformation/CloudFormationCustomResource.scala @@ -33,15 +33,12 @@ import org.http4s.ember.client.EmberClientBuilder import java.io.{PrintWriter, StringWriter} trait CloudFormationCustomResource[F[_], Input, Output] { - def createResource( - event: CloudFormationCustomResourceRequest[Input], - context: Context): F[HandlerResponse[Output]] - def updateResource( - event: CloudFormationCustomResourceRequest[Input], - context: Context): F[HandlerResponse[Output]] - def deleteResource( - event: CloudFormationCustomResourceRequest[Input], - context: Context): F[HandlerResponse[Output]] + def createResource(event: CloudFormationCustomResourceRequest[Input], context: Context)( + implicit T: Trace[F]): F[HandlerResponse[Output]] + def updateResource(event: CloudFormationCustomResourceRequest[Input], context: Context)( + implicit T: Trace[F]): F[HandlerResponse[Output]] + def deleteResource(event: CloudFormationCustomResourceRequest[Input], context: Context)( + implicit T: Trace[F]): F[HandlerResponse[Output]] } abstract class CloudFormationCustomResourceHandler[Input: Decoder, Output: Encoder] From 2fe058991c92d124fc8589e7337486f058ddcff5 Mon Sep 17 00:00:00 2001 From: Brian Holt Date: Mon, 8 Nov 2021 18:47:14 -0600 Subject: [PATCH 3/5] clean up tracing resource in JS IOLambdaPlatform --- .../main/scala/feral/lambda/IOLambdaPlatform.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala b/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala index eadb38dd..29f7c86b 100644 --- a/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala +++ b/lambda/js/src/main/scala/feral/lambda/IOLambdaPlatform.scala @@ -30,17 +30,16 @@ private[lambda] trait IOLambdaPlatform[Event, Result] { private def ioHandler(event: js.Any, context: facade.Context): IO[js.Any | Unit] = Resource - .pure(Context.fromJS(context)) - .flatMap { (context: Context) => - traceRootSpan(context.functionName).evalMap(ioTrace).evalMap { implicit trace => + .pure[IO, Context](Context.fromJS(context)) + .mproduct(context => traceRootSpan(context.functionName).evalMap(ioTrace)) + .use { + case (context, trace) => for { setup <- setupMemo event <- IO.fromEither(decodeJs[Event](event)) - result <- apply(event, context, setup) + result <- apply(event, context, setup)(trace) } yield result.map(_.asJsAny).orUndefined - } } - .use(_.pure[IO]) // @JSExportTopLevel("handler") // TODO final def handler(event: js.Any, context: facade.Context): js.Promise[js.Any | Unit] = From c1f8e4821f1ca1cc3ba89cade78f683e5957875c Mon Sep 17 00:00:00 2001 From: Brian Holt Date: Tue, 9 Nov 2021 11:46:17 -0600 Subject: [PATCH 4/5] use Natchez http4s middleware to trace ApiGatewayProxyHttp4sLambda --- build.sbt | 3 ++- .../lambda/ApiGatewayProxyHttp4sLambda.scala | 26 +++++++------------ 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/build.sbt b/build.sbt index 5cce4bc6..b1104627 100644 --- a/build.sbt +++ b/build.sbt @@ -108,7 +108,8 @@ lazy val lambdaApiGatewayProxyHttp4s = crossProject(JSPlatform, JVMPlatform) .settings( name := "feral-lambda-api-gateway-proxy-http4s", libraryDependencies ++= Seq( - "org.http4s" %%% "http4s-core" % http4sVersion + "org.http4s" %%% "http4s-core" % http4sVersion, + "org.tpolecat" %% "natchez-http4s" % "0.1.3", ) ) .dependsOn(lambda, lambdaEvents) diff --git a/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala b/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala index 60591ead..2df3dd0a 100644 --- a/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala +++ b/lambda-api-gateway-proxy-http4s/src/main/scala/feral/lambda/ApiGatewayProxyHttp4sLambda.scala @@ -16,23 +16,13 @@ package feral.lambda -import cats.effect.IO -import cats.effect.Resource -import cats.effect.SyncIO -import feral.lambda.events.ApiGatewayProxyEventV2 -import feral.lambda.events.ApiGatewayProxyStructuredResultV2 +import cats.effect.{IO, Resource, SyncIO} +import feral.lambda.events.{ApiGatewayProxyEventV2, ApiGatewayProxyStructuredResultV2} import fs2.Stream -import natchez.Trace -import org.http4s.Charset -import org.http4s.Header -import org.http4s.Headers -import org.http4s.HttpRoutes -import org.http4s.Method -import org.http4s.Request -import org.http4s.Response -import org.http4s.Uri -import org.typelevel.vault.Key -import org.typelevel.vault.Vault +import natchez.http4s.NatchezMiddleware +import natchez.{Span, Trace} +import org.http4s.{Charset, Header, Headers, HttpRoutes, Method, Request, Response, Uri} +import org.typelevel.vault.{Key, Vault} abstract class ApiGatewayProxyHttp4sLambda extends IOLambda[ApiGatewayProxyEventV2, ApiGatewayProxyStructuredResultV2] { @@ -45,6 +35,8 @@ abstract class ApiGatewayProxyHttp4sLambda protected type Setup = HttpRoutes[IO] protected override final val setup: Resource[IO, HttpRoutes[IO]] = routes + override protected def traceRootSpan(name: String): Resource[IO, Span[IO]] = super.traceRootSpan(name) + override final def apply( event: ApiGatewayProxyEventV2, context: Context, @@ -65,7 +57,7 @@ abstract class ApiGatewayProxyHttp4sLambda headers = headers, body = requestBody, attributes = Vault.empty.insert(ContextKey, context).insert(EventKey, event)) - response <- routes(request).getOrElse(Response.notFound[IO]) + response <- NatchezMiddleware.server(routes).run(request).getOrElse(Response.notFound[IO]) isBase64Encoded = !response.charset.contains(Charset.`UTF-8`) responseBody <- (if (isBase64Encoded) response.body.through(fs2.text.base64.encode) From 4244696c714ca1ce2e991ac1f649bff08e725c29 Mon Sep 17 00:00:00 2001 From: Brian Holt Date: Tue, 9 Nov 2021 11:48:05 -0600 Subject: [PATCH 5/5] incorporate pending upstream natchez-xray changes --- .../src/main/scala/feral/lambda/XRayTracing.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala b/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala index c97c1c72..babcf206 100644 --- a/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala +++ b/lambda/shared/src/main/scala/feral/lambda/XRayTracing.scala @@ -21,6 +21,7 @@ package feral.lambda +import cats.effect.std.Random import cats.effect.{IO, Resource} import feral.IOSetup import natchez.xray.{XRay, XRayEnvironment} @@ -28,10 +29,12 @@ import natchez.{EntryPoint, Span} trait XRayTracing extends IOSetup { private def traceEntryPoint: Resource[IO, EntryPoint[IO]] = - Resource.eval(XRayEnvironment[IO].daemonAddress).flatMap { - case Some(addr) => XRay.entryPoint(addr) - case None => XRay.entryPoint() - } + Resource.eval(Random.scalaUtilRandom[IO]).flatMap { implicit random => + Resource.eval(XRayEnvironment[IO].daemonAddress).flatMap { + case Some(addr) => XRay.entryPoint[IO](addr) + case None => XRay.entryPoint[IO]() + } + } override protected def traceRootSpan(name: String): Resource[IO, Span[IO]] = Resource.eval(XRayEnvironment[IO].kernelFromEnvironment).flatMap { kernel =>