api.proto

// api - bitdrift's client/server API definitions
// Copyright Bitdrift, Inc. All rights reserved.
//
// Use of this source code and APIs are governed by a source available license that can be found in
// the LICENSE file or at:
// https://polyformproject.org/wp-content/uploads/2020/06/PolyForm-Shield-1.0.0.txt

syntax = "proto3";

package bitdrift_public.protobuf.client.v1;

import "bitdrift_public/protobuf/bdtail/v1/bdtail_config.proto";
import "bitdrift_public/protobuf/client/v1/feature_flag.proto";
import "bitdrift_public/protobuf/client/v1/metric.proto";
import "bitdrift_public/protobuf/client/v1/runtime.proto";
import "bitdrift_public/protobuf/config/v1/config.proto";
import "bitdrift_public/protobuf/filter/v1/filter.proto";
import "bitdrift_public/protobuf/logging/v1/payload.proto";
import "bitdrift_public/protobuf/workflow/v1/workflow.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/timestamp.proto";
import "validate/validate.proto";

// File written to disk if the client has been placed in the "killed" state and told not to
// contact the server for some period of time.
message ClientKillFile {
  // This is the hash of the API key. If the API key hash changes, we will always ignore the
  // kill_until timestamp. This is to ease local development/onboarding when folks might be
  // confused about what API key to use.
  bytes api_key_hash = 1;

  // The time the client will be killed until. After this time it will try to contact the server
  // again.
  google.protobuf.Timestamp kill_until = 2;
}

// The initial request sent over the stream.
message HandshakeRequest {
  // A set of opaque metadata that identifies the connecting device. These will
  // remain static for the duration of this stream.
  // TODO(snowp): Support updating these throughout the session.
  // TODO(snowp): Improve the story for this when parameters change during a
  // logging period, e.g. if we capture multiple process restarts that may span
  // versions.
  map<string, logging.v1.Data> static_device_metadata = 1;

  // This was never used by the client. Reserve it for now to avoid implementing it. We can bring
  // it back later if needed.
  reserved "fields_for_all_logs";
  reserved 2;

  // If the client has an active configuration, previously obtained via a ConfigurationUpdate
  // message, it should indicate it in the handshake. This will allow the server to not have to
  // send out a redundant configuration update. If there is no active configuration this should
  // be empty.
  string configuration_version_nonce = 3;

  // If the client has an active runtime, previously obtained via a RuntimeUpdate
  // message, it should indicate it in the handshake. This will allow the server to not have to
  // send out a redundant runtime update. If there is no active runtime this should
  // be empty.
  string runtime_version_nonce = 4;

  reserved 5;

  // Reports the reason, if any, that the client was previously disconnected from the server. This
  // is useful for debugging reconnect loops.
  string previous_disconnect_reason = 6;

  // At the time of the handshake, whether the client is operating in sleep mode.
  bool sleep_mode = 7;

  // Opaque client state previously provided by the server in a HandshakeResponse. The client
  // should continue to send this on every handshake request until the server provides a new
  // value (which may be empty to clear existing state).
  optional bytes opaque_client_state = 8;
}

// Notifies the server about the intent to upload one or more batches of logs. The client is expected (but
// not required, for backwards compatibility) to notify the server about the intent to upload
// before uploading. The server is thusly able to influence whether the client
// actually uploads (e.g. the backend no longer wants more of these kind of logs)
// or delay the upload (e.g. we are hitting rate limits and are applying back-pressure).
message LogUploadIntentRequest {
  // The number of logs that the client wants to upload. Note that for a ListenerUpload this is
  // an approximation of all logs across all batches.
  uint32 log_count = 1;

  // The size of the intended upload, in bytes. Note that for a ListenerUpload this is
  // an approximation of the size of all logs across all batches.
  uint32 byte_count = 2;

  // The buffer these logs are uploaded from.
  string buffer_id = 3;

  // The client generated UUID for this intent. This allows the server to enforce idempotence
  // during intent negotiation as well as allowing for parallel intent requests to be responded
  // to out of order.
  string intent_uuid = 4;

  // The session ID of the log that caused the intent negotiation. This allows correlating the intent with
  // a specific user session.
  string session_id = 6;

  message WorkflowActionUpload {
    // The listener(s) which triggered this upload.
    repeated string workflow_action_ids = 1;
  }

  message ExplicitSessionCapture {
    // An opaque identifier that is used to identify the reason for the session capture.
    string id = 1 [(validate.rules).string = {min_len: 1}];
  }

  oneof intent_type {
    WorkflowActionUpload workflow_action_upload = 5;

    // Session capture was explicitly triggered by the client.
    ExplicitSessionCapture explicit_session_capture = 7;
  }
}

message LogUploadIntentResponse {
  // The UUID of the intent being negotiated.
  string intent_uuid = 1;

  message UploadImmediately {}

  message Drop {
    // If the upload intent was a ListenerUpload the entire upload should be canceled.
  }

  oneof decision {
    // The log should be uploaded immediately.
    UploadImmediately upload_immediately = 2;

    // The candidate batch should be dropped.
    Drop drop = 3;
  }
}

// A single log upload payload.
message LogUploadRequest {
  // A UUID (v4) associated with this upload. This should be generated by the
  // client. Retries should use the same upload_id.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // A repeated set of flatbuffer encoding of a number of log lines. Each log line is of type
  // Log defined in buffer_log.fbs. This is deprecated. New clients send protobuf encoded logs
  // in the 'logs' field.
  repeated bytes legacy_flatbuffer_logs = 2;

  // Logs encoded in bitdrift_public.protobuf.logging.v1.Log format.
  repeated bytes proto_logs = 5;

  // The UUID (v4) of the buffer that is producing the logs.
  string buffer_uuid = 3 [(validate.rules).string = {min_len: 1}];

  // Whether the server should not respond to this upload. This is used for high
  // throughput, low value logs where the client does not need to know whether
  // the upload succeeded or failed.
  bool ackless = 4;
}

// Empty message to maintain a application layer keep alive mechanism.
message PingRequest {
  // At the time of the ping, whether the client is operating in sleep mode.
  bool sleep_mode = 1;
}

// Configuration update response from client to server.
message ConfigurationUpdateAck {
  message Nack {
    // The version nonce of the configuration update that failed.
    string version_nonce = 1;

    // Error details of the failure.
    string error_details = 2;
  }

  // The version nonce that the client is actually using. This version nonce was sent in a
  // ConfigurationUpdate message. If no configuration has been applied (NACK only) this will be
  // empty.
  string last_applied_version_nonce = 1;

  // If a configuration update failed, the client should respond to the server and let it know.
  // It should continue to use the last good config. This message is empty on success, and the
  // client should respond with last_applied_version_nonce being equal to the update that was
  // applied.
  Nack nack = 2;
}

// A multiplexed request sent over the bitdrift API. Upon stream creation, the
// client will issue a single handshake request, then await a handshake
// response. Once the handshake has completed, the client may issue any number
// of non-handshake requests; corresponding responses may come in any order.
message ApiRequest {
  oneof request_type {
    option (validate.required) = true;

    HandshakeRequest handshake = 1;
    LogUploadIntentRequest log_upload_intent = 7;
    LogUploadRequest log_upload = 2;
    StatsUploadRequest stats_upload = 6;
    PingRequest ping = 3;
    ConfigurationUpdateAck configuration_update_ack = 4;
    ConfigurationUpdateAck runtime_update_ack = 5;
    SankeyPathUploadRequest sankey_path_upload = 10;
    SankeyIntentRequest sankey_intent = 11;
    UploadArtifactRequest artifact_upload = 12;
    UploadArtifactIntentRequest artifact_intent = 13;
    DebugDataRequest debug_data = 14;
  }

  reserved 8;
  reserved 9;
}

// A request to upload a Sankey diagram path.
message SankeyPathUploadRequest {
  // Upload UUID used to provide idempotence and to correlate a response with this request.
  string upload_uuid = 4 [(validate.rules).string = {min_len: 1}];

  // A single node in the Sankey diagram. This differs from workflow states. Each node corresponds
  // to a single transition from a workflow origin state to a workflow target state.
  message Node {
    // The value extracted from the matched log.
    string extracted_value = 1 [(validate.rules).string = {min_len: 1}];
  }

  // Sankey diagram ID.
  string id = 1 [(validate.rules).string = {min_len: 1}];
  // The identifier that represents a traversed state's path registered for a diagram. Two diagram paths
  // within the same diagram can have the same ID only if their nodes are identical.
  // Conflicts in diagram path IDs between different diagrams are possible.
  string path_id = 2 [(validate.rules).string = {min_len: 1}];
  // The list of traversed diagram nodes.
  repeated Node nodes = 3 [(validate.rules).repeated = {min_items: 1}];
}

// A request to ask whether to upload a Sankey path.
message SankeyIntentRequest {
  // The UUID of the intent being negotiated. This is used to correlate the response with the request.
  string intent_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // The ID of the path that is being considered for upload.
  string path_id = 2 [(validate.rules).string = {min_len: 1}];

  // The ID of the diagram that the path was discovered in.
  string sankey_diagram_id = 3 [(validate.rules).string = {min_len: 1}];
}

message UploadArtifactIntentRequest {
  // The UUID of the intent being negotiated. This is used to correlate the response with the request.
  string intent_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // The type of the artifact being considered for upload.
  string type_id = 2 [(validate.rules).string = {min_len: 1}];

  reserved 3;

  // The metadata associated with the artifact. The contents within this map depends on the type of the artfact but will generally contain information that can help the server make a decision about whether to accept the upload or not. For example, for issue reports this may contain the "fields" associated with the issue report.
  map<string, logging.v1.Data> metadata = 6;

  // A client-generated ID that uniquely identifies the artifact. This is used to correlate the artifact
  // with logs that reference it.
  string artifact_id = 4 [(validate.rules).string = {min_len: 1}];

  // The timestamp associated with the artifact being uploaded. This allows us to possibly reject the upload of very old artifacts.
  google.protobuf.Timestamp time = 5 [(validate.rules).message = {required: true}];

  // The session ID associated with the artifact. This allows correlating the intent with a specific user session.
  // This may not be set for all kinds of artifacts, such as state snapshots.
  optional string session_id = 7;
}

message UploadArtifactIntentResponse {
  // The UUID of the intent being negotiated. This is used to correlate the response with the request.
  string intent_uuid = 1 [(validate.rules).string = {min_len: 1}];

  message UploadImmediately {}

  message Drop {}

  oneof decision {
    // The artifact should be uploaded immediately.
    UploadImmediately upload_immediately = 3;

    // The candidate artifact should be dropped.
    Drop drop = 4;
  }
}

message UploadArtifactRequest {
  // Upload UUID used to provide idempotence and to correlate a response with this request.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // The type of the artifact being uploaded.
  string type_id = 2 [(validate.rules).string = {min_len: 1}];

  // The artifact to upload. This is a binary blob that is interpreted by the server based on the type_id.
  bytes contents = 3;

  // A client-generated ID that uniquely identifies the artifact being uploaded. This is used to correlate
  // the artifact with logs that reference it.
  string artifact_id = 4 [(validate.rules).string = {min_len: 1}];

  // An optional set of key-value data indicating the state of the device at the time of artifact emission. For example,
  // this may capture information about the device at the time of a crash.
  map<string, logging.v1.Data> state_metadata = 5;

  // The timestamp associated with this upload. This allows us to possibly reject the upload of very old artifacts.
  google.protobuf.Timestamp time = 6 [(validate.rules).message = {required: true}];

  // The session ID associated with the artifact if applicable.
  string session_id = 7;

  // The set of feature flags that were active at the time of artifact emission.
  repeated FeatureFlag feature_flags = 8;
}

message UploadArtifactResponse {
  // The UUID corresponding to the upload request.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // Optional error message which indicates that artifact upload failed.
  string error = 2;
}

// The response sent as part of stream establishment.
message HandshakeResponse {
  message StreamSettings {
    // How often the client should ping the server. This informs the client how
    // often a ping request should be issued over the newly created stream.
    //
    // If this is not set, the client should not issue ping requests.
    google.protobuf.Duration ping_interval = 1;
  }

  StreamSettings stream_settings = 1;

  // A flag set that describes the status of configuration updates based on the provided
  // configuration nonces.
  // 0x1: Runtime is up to date and no further update message will be supplied.
  // 0x2: Configuration is up to date and no further update message will be supplied.
  uint32 configuration_update_status = 2;

  // Opaque client state that should be echoed back to the server on every future handshake
  // request. If unset, the client should continue to send any existing opaque state, or none
  // if it has none. Sending an explicit empty value will clear any existing state.
  optional bytes opaque_client_state_to_echo = 3;
}

// A general indication of rate limiting from server to client.
message RateLimited {
  // Optional minimum retry backoff duration that the client should adhere to.
  google.protobuf.Duration retry_after = 1;
}

// A response to an upload request, intended to ack or nack the success of the
// upload. Upon failure, the client might choose to retry this upload.
message LogUploadResponse {
  // The upload UUID provided in the upload request corresponding to this
  // response.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // If set, indicates that the log upload failed. This will be set to a value
  // helpful for debugging the failure.
  string error = 2;

  // If any logs were dropped due to validation errors, the count will be supplied here. This does
  // not count as a total failure and 'error' will not be populated. Debugging information will be
  // available on the server.
  uint32 logs_dropped = 3;

  // If set the log upload was blocked due to rate limiting. Further information is available in
  // the `error` field.
  RateLimited rate_limited = 4;
}

message StatsUploadRequest {
  enum UploadReason {
    // Default value. Should not be used.
    UPLOAD_REASON_UNSPECIFIED = 0;

    // Periodic upload of stats.
    UPLOAD_REASON_PERIODIC = 1;

    // Upload triggered by a specific event, e.g., buffer flush.
    UPLOAD_REASON_EVENT_TRIGGERED = 2;
  }

  // Upload UUID used to provide idempotence and to correlate a response with this request.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  message Snapshot {
    oneof snapshot_type {
      option (validate.required) = true;

      MetricsList metrics = 1;
    }

    message Aggregated {
      // The point in time where the first set of stats in this aggregation period was aggregated.
      // This allows the server to get some understanding how old the stats being uploaded are.
      google.protobuf.Timestamp period_start = 4 [(validate.rules).message = {required: true}];

      // When the aggregation was closed. If specified, the server can decide to handle variable
      // size aggregation windows by averaging the data over the period or some other heuristic.
      google.protobuf.Timestamp period_end = 5;
    }

    // To support different kinds of snapshots, we support providing information about when the data was
    // collected.
    oneof occurred_at {
      option (validate.required) = true;

      // The snapshot data is aggregated over an indefinite period. This supports metrics where we
      // care more about the total data (e.g. counts) than understanding precisely when the data was
      // recorded.
      Aggregated aggregated = 2;
    }

    // A map of metric ID to any cardinality overflows that occurred during this snapshot interval.
    map<string, uint64> metric_id_overflows = 3;

    // Workflow ID to debug data that occurred during this snapshot interval.
    map<string, DebugDataRequest.WorkflowDebugData> workflow_debug_data = 4;
  }

  // A collection of stats snapshots to be recorded by the backend.
  repeated Snapshot snapshot = 2 [(validate.rules).repeated = {min_items: 1}];

  // The point in time when the client initialized the process of uploading collected snapshots.
  // Used by the server to detect clients with skewed clocks.
  // The idea is that upon receiving the stats payload, the server's current time should be close to this
  // value.
  google.protobuf.Timestamp sent_at = 3;

  // The reason for this stats upload.
  UploadReason upload_reason = 4;
}

message StatsUploadResponse {
  // The UUID corresponding to the upload request.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // Optional error message which indicates that stats upload failed and/or some metrics were
  // dropped.
  string error = 2;

  // If any metrics were dropped due to validation errors, the count will be supplied here. Error
  // will be populated with debugging information to help understand the error.
  uint32 metrics_dropped = 3;
}

// Response to a client ping.
message PongResponse {}

// Configuration update from server to client.
message ConfigurationUpdate {
  // A complete configuration snapshot. The client should reconfigure to match the supplied
  // configuration.
  message StateOfTheWorld {
    // Replaced with `workflow_list` and not available for clients with config_version 6+.
    reserved "mll_list";
    reserved 2;

    // The list of buffers.
    config.v1.BufferConfigList buffer_config_list = 3;

    // The workflows configuration.
    workflow.v1.WorkflowsConfiguration workflows_configuration = 4;

    // The list of active bdtail sessions.
    bdtail.v1.BdTailConfigurations bdtail_configuration = 6;

    reserved "insights_configuration";
    reserved 7;

    // The filters configuration.
    filter.v1.FiltersConfiguration filters_configuration = 8;

    // The list of workflows to debug. If the workflow is already deployed, debug data will be
    // generated for it alongside the deployed workflow. If the workflow is not deployed, the
    // workflow will be executed in debug mode, however actions will *not* be executed (metrics,
    // flushes, and so on).
    // NOTE: "already deployed" implies that the workflow will be found in the
    // workflows_configuration field. The server will not de-dup for simplification purposes.
    workflow.v1.WorkflowsConfiguration debug_workflows = 9;
  }

  // A version nonce that can be used for both tracking the last applied update as well as for
  // responding with a NACK message.
  string version_nonce = 1;

  oneof update_type {
    // SoTW is the only currently supported configuration type.
    StateOfTheWorld state_of_the_world = 2;
  }
}

// Runtime update from server to client.
message RuntimeUpdate {
  // A version nonce that can be used for both tracking the last applied update as well as for
  // responding with a NACK message.
  string version_nonce = 1;

  // The runtime instance to use.
  Runtime runtime = 2;
}

// In order to support clients that can't easily implement gRPC (e.g., iOS URLSession), instead of
// closing a stream with trailers, this frame will be sent followed by stream closure. This allows
// an easier way for the client to debug and handle errors.
message ErrorShutdown {
  // The status that would have been sent in trailers.
  int32 grpc_status = 1;

  // The message that would have been sent in trailers.
  string grpc_message = 2;

  // Optional rate limiting that the client should adhere to.
  RateLimited rate_limited = 3;
}

// Called by the server to tell the client to flush a set of owned buffers. When this command is
// received by the client it should proceed with normal behavior as if it had decided to flush
// the buffers locally (e.g., rate limiting, intents, etc.).
message FlushBuffers {
  // The list of buffers to flush. If the list is empty, all buffers should be flushed. In the case
  // of a server side listener trigger, every occurrence of ActionFlushBuffer in the trigger_actions
  // field will populate a buffer in this list.
  repeated string buffer_id_list = 1;
}

// The response to Sankey diagram path upload request.
message SankeyPathUploadResponse {
  // The UUID corresponding to the upload request.
  string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

  // Optional error message which indicates that sankey upload failed and/or some metrics were
  // dropped.
  string error = 2;
}

// The response to Sankey diagram intent request.
message SankeyIntentResponse {
  string intent_uuid = 1 [(validate.rules).string = {min_len: 1}];

  reserved 2;
  reserved 'decision';

  message UploadImmediately {}

  message Drop {}

  oneof decision {
    // The candidate sankey path should be uploaded immediately.
    UploadImmediately upload_immediately = 3;

    // The candidate sankey path should be dropped.
    Drop drop = 4;
  }
}

// Debug data that is periodically sent when instructed via the DebugControlResponse message.
message DebugDataRequest {
  message WorkflowTransitionDebugData {
    oneof transition_type {
      // The index of the transition that was taken to leave this state.
      uint32 transition_index = 1;

      // Whether this transition was a timeout transition.
      bool timeout_transition = 2;
    }

    // The number of times this state/transition has been transitioned out of.
    uint64 transition_count = 3;

    // The last time this state/transition was transitioned out of.
    google.protobuf.Timestamp last_transition_time = 4;
  }

  message WorkflowStateDebugData {
    repeated WorkflowTransitionDebugData transitions = 1;
  }

  message WorkflowDebugData {
    // The state debug data for each state in the workflow that has been transitioned. This is a map
    // of state ID to data.
    map<string, WorkflowStateDebugData> states = 1;

    // This is incremented every time the workflow starts or resets. Effectively it will be
    // incremented when the workflow is delivered, and every time it is reset to the initial state.
    // We reuse the existing WorkflowTransitionDebugData message and leave the transition_type
    // unset.
    WorkflowTransitionDebugData start_reset = 2;
  }

  // If instructed to debug workflows, this will contain the debug data for each workflow being
  // debugged. This is map of workflow ID to data.
  map<string, WorkflowDebugData> workflow_debug_data = 1;
}

// A multiplexed response sent over the bitdrift API.
message ApiResponse {
  oneof response_type {
    option (validate.required) = true;

    HandshakeResponse handshake = 1;
    LogUploadResponse log_upload = 2;
    LogUploadIntentResponse log_upload_intent = 8;
    StatsUploadResponse stats_upload = 7;
    PongResponse pong = 3;
    ConfigurationUpdate configuration_update = 4;
    RuntimeUpdate runtime_update = 5;
    ErrorShutdown error_shutdown = 6;
    FlushBuffers flush_buffers = 9;
    SankeyPathUploadResponse sankey_diagram_upload = 12;
    SankeyIntentResponse sankey_intent_response = 13;
    UploadArtifactResponse artifact_upload = 14;
    UploadArtifactIntentResponse artifact_intent = 15;
  }

  reserved 10;
  reserved 11;
}

service ApiService {
  rpc Mux(stream ApiRequest) returns (stream ApiResponse);
}