Skip to content

Add initial implementation of pdatagrcp #3231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bogdandrutu merged 2 commits into from
May 19, 2021
Merged

Add initial implementation of pdatagrcp #3231

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa27faf
Add initial implementation of pdatagrcp
bogdandrutu May 19, 2021
4a27e08
Add details for the TODOs left
bogdandrutu May 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 16 additions & 21 deletions exporter/otlpexporter/otlp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,7 @@ import (
"go.opentelemetry.io/collector/consumer/consumererror"
"go.opentelemetry.io/collector/consumer/pdata"
"go.opentelemetry.io/collector/exporter/exporterhelper"
"go.opentelemetry.io/collector/internal"
otlplogs "go.opentelemetry.io/collector/internal/data/protogen/collector/logs/v1"
otlpmetrics "go.opentelemetry.io/collector/internal/data/protogen/collector/metrics/v1"
otlptrace "go.opentelemetry.io/collector/internal/data/protogen/collector/trace/v1"
"go.opentelemetry.io/collector/internal/pdatagrpc"
)

type exporter struct {
Expand Down Expand Up @@ -66,33 +63,31 @@ func (e *exporter) shutdown(context.Context) error {
}

func (e *exporter) pushTraceData(ctx context.Context, td pdata.Traces) error {
if err := e.w.exportTrace(ctx, internal.TracesToOtlp(td.InternalRep())); err != nil {
if err := e.w.exportTrace(ctx, td); err != nil {
return fmt.Errorf("failed to push trace data via OTLP exporter: %w", err)
}
return nil
}

func (e *exporter) pushMetricsData(ctx context.Context, md pdata.Metrics) error {
req := internal.MetricsToOtlp(md.InternalRep())
if err := e.w.exportMetrics(ctx, req); err != nil {
if err := e.w.exportMetrics(ctx, md); err != nil {
return fmt.Errorf("failed to push metrics data via OTLP exporter: %w", err)
}
return nil
}

func (e *exporter) pushLogData(ctx context.Context, ld pdata.Logs) error {
request := internal.LogsToOtlp(ld.InternalRep())
if err := e.w.exportLogs(ctx, request); err != nil {
if err := e.w.exportLogs(ctx, ld); err != nil {
return fmt.Errorf("failed to push log data via OTLP exporter: %w", err)
}
return nil
}

type grpcSender struct {
// gRPC clients and connection.
traceExporter otlptrace.TraceServiceClient
metricExporter otlpmetrics.MetricsServiceClient
logExporter otlplogs.LogsServiceClient
traceExporter pdatagrpc.TracesClient
metricExporter pdatagrpc.MetricsClient
logExporter pdatagrpc.LogsClient
clientConn *grpc.ClientConn
metadata metadata.MD
callOptions []grpc.CallOption
Expand All @@ -110,9 +105,9 @@ func newGrpcSender(config *Config) (*grpcSender, error) {
}

gs := &grpcSender{
traceExporter: otlptrace.NewTraceServiceClient(clientConn),
metricExporter: otlpmetrics.NewMetricsServiceClient(clientConn),
logExporter: otlplogs.NewLogsServiceClient(clientConn),
traceExporter: pdatagrpc.NewTracesClient(clientConn),
metricExporter: pdatagrpc.NewMetricsClient(clientConn),
logExporter: pdatagrpc.NewLogsClient(clientConn),
clientConn: clientConn,
metadata: metadata.New(config.GRPCClientSettings.Headers),
callOptions: []grpc.CallOption{
Expand All @@ -126,18 +121,18 @@ func (gs *grpcSender) stop() error {
return gs.clientConn.Close()
}

func (gs *grpcSender) exportTrace(ctx context.Context, request *otlptrace.ExportTraceServiceRequest) error {
_, err := gs.traceExporter.Export(gs.enhanceContext(ctx), request, gs.callOptions...)
func (gs *grpcSender) exportTrace(ctx context.Context, td pdata.Traces) error {
_, err := gs.traceExporter.Export(gs.enhanceContext(ctx), td, gs.callOptions...)
return processError(err)
}

func (gs *grpcSender) exportMetrics(ctx context.Context, request *otlpmetrics.ExportMetricsServiceRequest) error {
_, err := gs.metricExporter.Export(gs.enhanceContext(ctx), request, gs.callOptions...)
func (gs *grpcSender) exportMetrics(ctx context.Context, md pdata.Metrics) error {
_, err := gs.metricExporter.Export(gs.enhanceContext(ctx), md, gs.callOptions...)
return processError(err)
}

func (gs *grpcSender) exportLogs(ctx context.Context, request *otlplogs.ExportLogsServiceRequest) error {
_, err := gs.logExporter.Export(gs.enhanceContext(ctx), request, gs.callOptions...)
func (gs *grpcSender) exportLogs(ctx context.Context, ld pdata.Logs) error {
_, err := gs.logExporter.Export(gs.enhanceContext(ctx), ld, gs.callOptions...)
return processError(err)
}

Expand Down
91 changes: 31 additions & 60 deletions exporter/otlpexporter/otlp_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,7 @@ import (
"go.opentelemetry.io/collector/config/configgrpc"
"go.opentelemetry.io/collector/config/configtls"
"go.opentelemetry.io/collector/consumer/pdata"
"go.opentelemetry.io/collector/internal"
otlplogs "go.opentelemetry.io/collector/internal/data/protogen/collector/logs/v1"
otlpmetrics "go.opentelemetry.io/collector/internal/data/protogen/collector/metrics/v1"
otlptraces "go.opentelemetry.io/collector/internal/data/protogen/collector/trace/v1"
"go.opentelemetry.io/collector/internal/pdatagrpc"
"go.opentelemetry.io/collector/internal/testdata"
"go.opentelemetry.io/collector/obsreport"
)
Expand All @@ -57,29 +54,20 @@ func (r *mockReceiver) GetMetadata() metadata.MD {

type mockTracesReceiver struct {
mockReceiver
lastRequest *otlptraces.ExportTraceServiceRequest
lastRequest pdata.Traces
}

func (r *mockTracesReceiver) Export(
ctx context.Context,
req *otlptraces.ExportTraceServiceRequest,
) (*otlptraces.ExportTraceServiceResponse, error) {
func (r *mockTracesReceiver) Export(ctx context.Context, td pdata.Traces) (interface{}, error) {
atomic.AddInt32(&r.requestCount, 1)
spanCount := 0
for _, rs := range req.ResourceSpans {
for _, ils := range rs.InstrumentationLibrarySpans {
spanCount += len(ils.Spans)
}
}
atomic.AddInt32(&r.totalItems, int32(spanCount))
atomic.AddInt32(&r.totalItems, int32(td.SpanCount()))
r.mux.Lock()
defer r.mux.Unlock()
r.lastRequest = req
r.lastRequest = td
r.metadata, _ = metadata.FromIncomingContext(ctx)
return &otlptraces.ExportTraceServiceResponse{}, nil
return nil, nil
}

func (r *mockTracesReceiver) GetLastRequest() *otlptraces.ExportTraceServiceRequest {
func (r *mockTracesReceiver) GetLastRequest() pdata.Traces {
r.mux.Lock()
defer r.mux.Unlock()
return r.lastRequest
Expand All @@ -93,7 +81,7 @@ func otlpTracesReceiverOnGRPCServer(ln net.Listener) *mockTracesReceiver {
}

// Now run it as a gRPC server
otlptraces.RegisterTraceServiceServer(rcv.srv, rcv)
pdatagrpc.RegisterTracesServer(rcv.srv, rcv)
go func() {
_ = rcv.srv.Serve(ln)
}()
Expand All @@ -103,29 +91,20 @@ func otlpTracesReceiverOnGRPCServer(ln net.Listener) *mockTracesReceiver {

type mockLogsReceiver struct {
mockReceiver
lastRequest *otlplogs.ExportLogsServiceRequest
lastRequest pdata.Logs
}

func (r *mockLogsReceiver) Export(
ctx context.Context,
req *otlplogs.ExportLogsServiceRequest,
) (*otlplogs.ExportLogsServiceResponse, error) {
func (r *mockLogsReceiver) Export(ctx context.Context, ld pdata.Logs) (interface{}, error) {
atomic.AddInt32(&r.requestCount, 1)
recordCount := 0
for _, rs := range req.ResourceLogs {
for _, il := range rs.InstrumentationLibraryLogs {
recordCount += len(il.Logs)
}
}
atomic.AddInt32(&r.totalItems, int32(recordCount))
atomic.AddInt32(&r.totalItems, int32(ld.LogRecordCount()))
r.mux.Lock()
defer r.mux.Unlock()
r.lastRequest = req
r.lastRequest = ld
r.metadata, _ = metadata.FromIncomingContext(ctx)
return &otlplogs.ExportLogsServiceResponse{}, nil
return nil, nil
}

func (r *mockLogsReceiver) GetLastRequest() *otlplogs.ExportLogsServiceRequest {
func (r *mockLogsReceiver) GetLastRequest() pdata.Logs {
r.mux.Lock()
defer r.mux.Unlock()
return r.lastRequest
Expand All @@ -139,7 +118,7 @@ func otlpLogsReceiverOnGRPCServer(ln net.Listener) *mockLogsReceiver {
}

// Now run it as a gRPC server
otlplogs.RegisterLogsServiceServer(rcv.srv, rcv)
pdatagrpc.RegisterLogsServer(rcv.srv, rcv)
go func() {
_ = rcv.srv.Serve(ln)
}()
Expand All @@ -149,24 +128,21 @@ func otlpLogsReceiverOnGRPCServer(ln net.Listener) *mockLogsReceiver {

type mockMetricsReceiver struct {
mockReceiver
lastRequest *otlpmetrics.ExportMetricsServiceRequest
lastRequest pdata.Metrics
}

func (r *mockMetricsReceiver) Export(
ctx context.Context,
req *otlpmetrics.ExportMetricsServiceRequest,
) (*otlpmetrics.ExportMetricsServiceResponse, error) {
func (r *mockMetricsReceiver) Export(ctx context.Context, md pdata.Metrics) (interface{}, error) {
atomic.AddInt32(&r.requestCount, 1)
_, recordCount := pdata.MetricsFromInternalRep(internal.MetricsFromOtlp(req)).MetricAndDataPointCount()
_, recordCount := md.MetricAndDataPointCount()
atomic.AddInt32(&r.totalItems, int32(recordCount))
r.mux.Lock()
defer r.mux.Unlock()
r.lastRequest = req
r.lastRequest = md
r.metadata, _ = metadata.FromIncomingContext(ctx)
return &otlpmetrics.ExportMetricsServiceResponse{}, nil
return nil, nil
}

func (r *mockMetricsReceiver) GetLastRequest() *otlpmetrics.ExportMetricsServiceRequest {
func (r *mockMetricsReceiver) GetLastRequest() pdata.Metrics {
r.mux.Lock()
defer r.mux.Unlock()
return r.lastRequest
Expand All @@ -180,7 +156,7 @@ func otlpMetricsReceiverOnGRPCServer(ln net.Listener) *mockMetricsReceiver {
}

// Now run it as a gRPC server
otlpmetrics.RegisterMetricsServiceServer(rcv.srv, rcv)
pdatagrpc.RegisterMetricsServer(rcv.srv, rcv)
go func() {
_ = rcv.srv.Serve(ln)
}()
Expand Down Expand Up @@ -238,8 +214,6 @@ func TestSendTraces(t *testing.T) {
// A trace with 2 spans.
td = testdata.GenerateTracesTwoSpansSameResource()

expectedOTLPReq := internal.TracesToOtlp(td.Clone().InternalRep())

err = exp.ConsumeTraces(context.Background(), td)
assert.NoError(t, err)

Expand All @@ -253,7 +227,7 @@ func TestSendTraces(t *testing.T) {
// Verify received span.
assert.EqualValues(t, 2, atomic.LoadInt32(&rcv.totalItems))
assert.EqualValues(t, 2, atomic.LoadInt32(&rcv.requestCount))
assert.EqualValues(t, expectedOTLPReq, rcv.GetLastRequest())
assert.EqualValues(t, td, rcv.GetLastRequest())

require.EqualValues(t, rcv.GetMetadata().Get("header"), expectedHeader)
}
Expand Down Expand Up @@ -308,8 +282,6 @@ func TestSendMetrics(t *testing.T) {
// A trace with 2 spans.
md = testdata.GenerateMetricsTwoMetrics()

expectedOTLPReq := internal.MetricsToOtlp(md.Clone().InternalRep())

err = exp.ConsumeMetrics(context.Background(), md)
assert.NoError(t, err)

Expand All @@ -323,7 +295,7 @@ func TestSendMetrics(t *testing.T) {
// Verify received metrics.
assert.EqualValues(t, 2, atomic.LoadInt32(&rcv.requestCount))
assert.EqualValues(t, 4, atomic.LoadInt32(&rcv.totalItems))
assert.EqualValues(t, expectedOTLPReq, rcv.GetLastRequest())
assert.EqualValues(t, md, rcv.GetLastRequest())

require.EqualValues(t, rcv.GetMetadata().Get("header"), expectedHeader)
}
Expand Down Expand Up @@ -447,7 +419,7 @@ func startServerAndMakeRequest(t *testing.T, exp component.TracesExporter, td pd
assert.EqualValues(t, 0, atomic.LoadInt32(&rcv.requestCount))

// Clone the request and store as expected.
expectedOTLPReq := internal.TracesToOtlp(td.Clone().InternalRep())
expectedData := td.Clone()

// Resend the request, this should succeed.
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
Expand All @@ -461,7 +433,7 @@ func startServerAndMakeRequest(t *testing.T, exp component.TracesExporter, td pd

// Verify received span.
assert.EqualValues(t, 2, atomic.LoadInt32(&rcv.totalItems))
assert.EqualValues(t, expectedOTLPReq, rcv.GetLastRequest())
assert.EqualValues(t, expectedData, rcv.GetLastRequest())
}

func TestSendLogData(t *testing.T) {
Expand Down Expand Up @@ -497,8 +469,8 @@ func TestSendLogData(t *testing.T) {
assert.EqualValues(t, 0, atomic.LoadInt32(&rcv.requestCount))

// Send empty request.
td := pdata.NewLogs()
assert.NoError(t, exp.ConsumeLogs(context.Background(), td))
ld := pdata.NewLogs()
assert.NoError(t, exp.ConsumeLogs(context.Background(), ld))

// Wait until it is received.
assert.Eventually(t, func() bool {
Expand All @@ -509,10 +481,9 @@ func TestSendLogData(t *testing.T) {
assert.EqualValues(t, 0, atomic.LoadInt32(&rcv.totalItems))

// A request with 2 log entries.
td = testdata.GenerateLogsTwoLogRecordsSameResource()
expectedOTLPReq := internal.LogsToOtlp(td.Clone().InternalRep())
ld = testdata.GenerateLogsTwoLogRecordsSameResource()

err = exp.ConsumeLogs(context.Background(), td)
err = exp.ConsumeLogs(context.Background(), ld)
assert.NoError(t, err)

// Wait until it is received.
Expand All @@ -523,5 +494,5 @@ func TestSendLogData(t *testing.T) {
// Verify received logs.
assert.EqualValues(t, 2, atomic.LoadInt32(&rcv.requestCount))
assert.EqualValues(t, 2, atomic.LoadInt32(&rcv.totalItems))
assert.EqualValues(t, expectedOTLPReq, rcv.GetLastRequest())
assert.EqualValues(t, ld, rcv.GetLastRequest())
}
74 changes: 74 additions & 0 deletions internal/pdatagrpc/logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pdatagrpc

import (
"context"

"google.golang.org/grpc"

"go.opentelemetry.io/collector/consumer/pdata"
"go.opentelemetry.io/collector/internal"
otlpcollectorlogs "go.opentelemetry.io/collector/internal/data/protogen/collector/logs/v1"
)

// TODO: Consider to add `LogsRequest` and `LogsResponse`

// LogsClient is the client API for OTLP-GRPC Logs service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
type LogsClient interface {
Copy link
Copy Markdown
Member

@tigrannajaryan tigrannajaryan May 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need an interface? I assume there will be only one implementation ever. Should we simply, delete the interface and just have the struct?

Copy link
Copy Markdown
Member Author

@bogdandrutu bogdandrutu May 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can either expose the struct or the interface. I think exposing the interface is more future proof, because we can offer alternative implementations in the future. Not a big thing though, so if you feel strong I can change.

Copy link
Copy Markdown
Member

@tigrannajaryan tigrannajaryan May 19, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a strong opinion. Works either way.

// Export pdata.Logs to the server.
//
// For performance reasons, it is recommended to keep this RPC
// alive for the entire life of the application.
Export(ctx context.Context, in pdata.Logs, opts ...grpc.CallOption) (interface{}, error)
}

type logsClient struct {
rawClient otlpcollectorlogs.LogsServiceClient
}

// NewLogsClient returns a new LogsClient connected using the given connection.
func NewLogsClient(cc *grpc.ClientConn) LogsClient {
return &logsClient{rawClient: otlpcollectorlogs.NewLogsServiceClient(cc)}
}

func (c *logsClient) Export(ctx context.Context, in pdata.Logs, opts ...grpc.CallOption) (interface{}, error) {
return c.rawClient.Export(ctx, internal.LogsToOtlp(in.InternalRep()), opts...)
}

// LogsServer is the server API for OTLP gRPC LogsService service.
type LogsServer interface {
// Export is called every time a new request is received.
//
// For performance reasons, it is recommended to keep this RPC
// alive for the entire life of the application.
Export(context.Context, pdata.Logs) (interface{}, error)
}

// RegisterLogsServer registers the LogsServer to the grpc.Server.
func RegisterLogsServer(s *grpc.Server, srv LogsServer) {
otlpcollectorlogs.RegisterLogsServiceServer(s, &rawLogsServer{srv: srv})
}

type rawLogsServer struct {
srv LogsServer
}

func (s rawLogsServer) Export(ctx context.Context, request *otlpcollectorlogs.ExportLogsServiceRequest) (*otlpcollectorlogs.ExportLogsServiceResponse, error) {
_, err := s.srv.Export(ctx, pdata.LogsFromInternalRep(internal.LogsFromOtlp(request)))
return &otlpcollectorlogs.ExportLogsServiceResponse{}, err
}
Loading