Document LLM Observability LLM provider integration endpoints

examples/v2/llm-observability/CreateLLMObsIntegrationInference.rb

@@ -0,0 +1,105 @@
+# Run an LLM inference returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.create_llm_obs_integration_inference".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+
+body = DatadogAPIClient::V2::LLMObsIntegrationInferenceRequest.new({
+  anthropic_metadata: DatadogAPIClient::V2::LLMObsAnthropicMetadata.new({
+    effort: DatadogAPIClient::V2::LLMObsAnthropicEffort::MEDIUM,
+    thinking: DatadogAPIClient::V2::LLMObsAnthropicThinkingConfig.new({
+      budget_tokens: 1024,
+      type: DatadogAPIClient::V2::LLMObsAnthropicThinkingType::ENABLED,
+    }),
+  }),
+  azure_openai_metadata: DatadogAPIClient::V2::LLMObsAzureOpenAIMetadata.new({
+    deployment_id: "my-gpt4-deployment",
+    model_version: "0613",
+    resource_name: "my-azure-resource",
+  }),
+  bedrock_metadata: DatadogAPIClient::V2::LLMObsBedrockMetadata.new({
+    region: "us-east-1",
+  }),
+  frequency_penalty: 0.0,
+  json_schema: '{"type":"object","properties":{"answer":{"type":"string"}}}',
+  max_completion_tokens: 1024,
+  max_tokens: 1024,
+  messages: [
+    DatadogAPIClient::V2::LLMObsInferenceMessage.new({
+      content: "What is the capital of France?",
+      contents: [
+        DatadogAPIClient::V2::LLMObsInferenceContent.new({
+          type: "text",
+          value: DatadogAPIClient::V2::LLMObsInferenceContentValue.new({
+            text: "Hello, how can I help you?",
+            tool_call: DatadogAPIClient::V2::LLMObsInferenceToolCall.new({
+              arguments: {
+                "location": "San Francisco",
+              },
+              name: "get_weather",
+              tool_id: "call_abc123",
+              type: "function",
+            }),
+            tool_call_result: DatadogAPIClient::V2::LLMObsInferenceToolResult.new({
+              name: "get_weather",
+              result: "The weather in San Francisco is 68°F and sunny.",
+              tool_id: "call_abc123",
+              type: "function",
+            }),
+          }),
+        }),
+      ],
+      id: "msg_001",
+      role: "user",
+      tool_calls: [
+        DatadogAPIClient::V2::LLMObsInferenceToolCall.new({
+          arguments: {
+            "location": "San Francisco",
+          },
+          name: "get_weather",
+          tool_id: "call_abc123",
+          type: "function",
+        }),
+      ],
+      tool_results: [
+        DatadogAPIClient::V2::LLMObsInferenceToolResult.new({
+          name: "get_weather",
+          result: "The weather in San Francisco is 68°F and sunny.",
+          tool_id: "call_abc123",
+          type: "function",
+        }),
+      ],
+    }),
+  ],
+  model_id: "gpt-4o",
+  openai_metadata: DatadogAPIClient::V2::LLMObsOpenAIMetadata.new({
+    reasoning_effort: DatadogAPIClient::V2::LLMObsOpenAIReasoningEffort::MEDIUM,
+    reasoning_summary: DatadogAPIClient::V2::LLMObsOpenAIReasoningSummary::AUTO,
+  }),
+  presence_penalty: 0.0,
+  temperature: 0.7,
+  tools: [
+    DatadogAPIClient::V2::LLMObsInferenceTool.new({
+      function: DatadogAPIClient::V2::LLMObsInferenceFunction.new({
+        description: "Get the current weather for a location.",
+        name: "get_weather",
+        parameters: {
+          "properties": "{'location': {'type': 'string'}}", "type": "object",
+        },
+      }),
+      type: "function",
+    }),
+  ],
+  top_k: 50,
+  top_p: 1.0,
+  vertex_ai_metadata: DatadogAPIClient::V2::LLMObsVertexAIMetadata.new({
+    location: "us-central1",
+    project: "my-gcp-project",
+    project_ids: [
+      "my-gcp-project",
+    ],
+  }),
+})
+p api_instance.create_llm_obs_integration_inference(LLMObsIntegrationName::OPENAI, "account_id", body)

examples/v2/llm-observability/ListLLMObsIntegrationAccounts.rb

@@ -0,0 +1,8 @@
+# List LLM integration accounts returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.list_llm_obs_integration_accounts".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+p api_instance.list_llm_obs_integration_accounts(LLMObsIntegrationName::OPENAI)

examples/v2/llm-observability/ListLLMObsIntegrationModels.rb

@@ -0,0 +1,8 @@
+# List LLM integration models returns "OK" response
+
+require "datadog_api_client"
+DatadogAPIClient.configure do |config|
+  config.unstable_operations["v2.list_llm_obs_integration_models".to_sym] = true
+end
+api_instance = DatadogAPIClient::V2::LLMObservabilityAPI.new
+p api_instance.list_llm_obs_integration_models(LLMObsIntegrationName::OPENAI, "account_id")

features/scenarios_model_mapping.rb

@@ -984,6 +984,18 @@
             "experiment_id" => "String",
             "body" => "LLMObsExperimentEventsRequest",
     },
+    "v2.ListLLMObsIntegrationAccounts" => {
+            "integration" => "LLMObsIntegrationName",
+    },
+    "v2.CreateLLMObsIntegrationInference" => {
+            "integration" => "LLMObsIntegrationName",
+            "account_id" => "String",
+            "body" => "LLMObsIntegrationInferenceRequest",
+    },
+    "v2.ListLLMObsIntegrationModels" => {
+            "integration" => "LLMObsIntegrationName",
+            "account_id" => "String",
+    },
     "v2.ListLLMObsProjects" => {
             "filter_id" => "String",
             "filter_name" => "String",

features/v2/llm_observability.feature

@@ -588,6 +588,40 @@ Feature: LLM Observability
     When the request is sent
     Then the response status is 200 OK
 
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List LLM integration accounts returns "Bad Request" response
+    Given operation "ListLLMObsIntegrationAccounts" enabled
+    And new "ListLLMObsIntegrationAccounts" request
+    And request contains "integration" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List LLM integration accounts returns "OK" response
+    Given operation "ListLLMObsIntegrationAccounts" enabled
+    And new "ListLLMObsIntegrationAccounts" request
+    And request contains "integration" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 200 OK
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List LLM integration models returns "Bad Request" response
+    Given operation "ListLLMObsIntegrationModels" enabled
+    And new "ListLLMObsIntegrationModels" request
+    And request contains "integration" parameter from "REPLACE.ME"
+    And request contains "account_id" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: List LLM integration models returns "OK" response
+    Given operation "ListLLMObsIntegrationModels" enabled
+    And new "ListLLMObsIntegrationModels" request
+    And request contains "integration" parameter from "REPLACE.ME"
+    And request contains "account_id" parameter from "REPLACE.ME"
+    When the request is sent
+    Then the response status is 200 OK
+
   @generated @skip @team:DataDog/ml-observability
   Scenario: List events for an LLM Observability experiment returns "Bad Request" response
     Given operation "ListLLMObsExperimentEvents" enabled
@@ -639,6 +673,26 @@ Feature: LLM Observability
     When the request is sent
     Then the response status is 404 Not Found
 
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Run an LLM inference returns "Bad Request" response
+    Given operation "CreateLLMObsIntegrationInference" enabled
+    And new "CreateLLMObsIntegrationInference" request
+    And request contains "integration" parameter from "REPLACE.ME"
+    And request contains "account_id" parameter from "REPLACE.ME"
+    And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}}
+    When the request is sent
+    Then the response status is 400 Bad Request
+
+  @generated @skip @team:DataDog/ml-observability
+  Scenario: Run an LLM inference returns "OK" response
+    Given operation "CreateLLMObsIntegrationInference" enabled
+    And new "CreateLLMObsIntegrationInference" request
+    And request contains "integration" parameter from "REPLACE.ME"
+    And request contains "account_id" parameter from "REPLACE.ME"
+    And body with value {"anthropic_metadata": {"effort": "medium", "thinking": {"budget_tokens": 1024, "type": "enabled"}}, "azure_openai_metadata": {"deployment_id": "my-gpt4-deployment", "model_version": "0613", "resource_name": "my-azure-resource"}, "bedrock_metadata": {"region": "us-east-1"}, "frequency_penalty": 0.0, "json_schema": "{\"type\":\"object\",\"properties\":{\"answer\":{\"type\":\"string\"}}}", "max_completion_tokens": 1024, "max_tokens": 1024, "messages": [{"content": "What is the capital of France?", "contents": [{"type": "text", "value": {"text": "Hello, how can I help you?", "tool_call": {"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}}}], "id": "msg_001", "role": "user", "tool_calls": [{"arguments": {"location": "San Francisco"}, "name": "get_weather", "tool_id": "call_abc123", "type": "function"}], "tool_results": [{"name": "get_weather", "result": "The weather in San Francisco is 68\u00b0F and sunny.", "tool_id": "call_abc123", "type": "function"}]}], "model_id": "gpt-4o", "openai_metadata": {"reasoning_effort": "medium", "reasoning_summary": "auto"}, "presence_penalty": 0.0, "temperature": 0.7, "tools": [{"function": {"description": "Get the current weather for a location.", "name": "get_weather", "parameters": {"properties": {"location": {"type": "string"}}, "type": "object"}}, "type": "function"}], "top_k": 50, "top_p": 1.0, "vertex_ai_metadata": {"location": "us-central1", "project": "my-gcp-project", "project_ids": ["my-gcp-project"]}}
+    When the request is sent
+    Then the response status is 200 OK
+
   @generated @skip @team:DataDog/ml-observability
   Scenario: Search LLM Observability experimentation entities returns "Bad Request" response
     Given operation "SearchLLMObsExperimentation" enabled

features/v2/undo.json

@@ -3797,6 +3797,24 @@
       "type": "unsafe"
     }
   },
+  "ListLLMObsIntegrationAccounts": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
+  "CreateLLMObsIntegrationInference": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
+  "ListLLMObsIntegrationModels": {
+    "tag": "LLM Observability",
+    "undo": {
+      "type": "safe"
+    }
+  },
   "ListLLMObsProjects": {
     "tag": "LLM Observability",
     "undo": {

lib/datadog_api_client/configuration.rb

@@ -214,6 +214,7 @@ def initialize
             "v2.create_llm_obs_dataset_records": false,
             "v2.create_llm_obs_experiment": false,
             "v2.create_llm_obs_experiment_events": false,
+            "v2.create_llm_obs_integration_inference": false,
             "v2.create_llm_obs_project": false,
             "v2.delete_llm_obs_annotation_queue": false,
             "v2.delete_llm_obs_annotation_queue_interactions": false,
@@ -231,6 +232,8 @@ def initialize
             "v2.list_llm_obs_datasets": false,
             "v2.list_llm_obs_experiment_events": false,
             "v2.list_llm_obs_experiments": false,
+            "v2.list_llm_obs_integration_accounts": false,
+            "v2.list_llm_obs_integration_models": false,
             "v2.list_llm_obs_projects": false,
             "v2.list_llm_obs_spans": false,
             "v2.search_llm_obs_experimentation": false,

lib/datadog_api_client/inflector.rb

@@ -3719,6 +3719,12 @@ def overrides
           "v2.llm_obs_annotation_queue_update_data_request" => "LLMObsAnnotationQueueUpdateDataRequest",
           "v2.llm_obs_annotation_queue_update_request" => "LLMObsAnnotationQueueUpdateRequest",
           "v2.llm_obs_annotation_schema" => "LLMObsAnnotationSchema",
+          "v2.llm_obs_anthropic_effort" => "LLMObsAnthropicEffort",
+          "v2.llm_obs_anthropic_metadata" => "LLMObsAnthropicMetadata",
+          "v2.llm_obs_anthropic_thinking_config" => "LLMObsAnthropicThinkingConfig",
+          "v2.llm_obs_anthropic_thinking_type" => "LLMObsAnthropicThinkingType",
+          "v2.llm_obs_azure_open_ai_metadata" => "LLMObsAzureOpenAIMetadata",
+          "v2.llm_obs_bedrock_metadata" => "LLMObsBedrockMetadata",
           "v2.llm_obs_content_block" => "LLMObsContentBlock",
           "v2.llm_obs_content_block_header_level" => "LLMObsContentBlockHeaderLevel",
           "v2.llm_obs_content_block_llm_obs_trace_interaction_type" => "LLMObsContentBlockLLMObsTraceInteractionType",
@@ -3860,10 +3866,29 @@ def overrides
           "v2.llm_obs_experiment_update_data_attributes_request" => "LLMObsExperimentUpdateDataAttributesRequest",
           "v2.llm_obs_experiment_update_data_request" => "LLMObsExperimentUpdateDataRequest",
           "v2.llm_obs_experiment_update_request" => "LLMObsExperimentUpdateRequest",
+          "v2.llm_obs_inference_code" => "LLMObsInferenceCode",
+          "v2.llm_obs_inference_content" => "LLMObsInferenceContent",
+          "v2.llm_obs_inference_content_value" => "LLMObsInferenceContentValue",
+          "v2.llm_obs_inference_error_response" => "LLMObsInferenceErrorResponse",
+          "v2.llm_obs_inference_function" => "LLMObsInferenceFunction",
+          "v2.llm_obs_inference_message" => "LLMObsInferenceMessage",
+          "v2.llm_obs_inference_run_result" => "LLMObsInferenceRunResult",
+          "v2.llm_obs_inference_tool" => "LLMObsInferenceTool",
+          "v2.llm_obs_inference_tool_call" => "LLMObsInferenceToolCall",
+          "v2.llm_obs_inference_tool_result" => "LLMObsInferenceToolResult",
+          "v2.llm_obs_integration_account" => "LLMObsIntegrationAccount",
+          "v2.llm_obs_integration_inference_request" => "LLMObsIntegrationInferenceRequest",
+          "v2.llm_obs_integration_inference_response" => "LLMObsIntegrationInferenceResponse",
+          "v2.llm_obs_integration_model" => "LLMObsIntegrationModel",
+          "v2.llm_obs_integration_name" => "LLMObsIntegrationName",
+          "v2.llm_obs_internal_reasoning" => "LLMObsInternalReasoning",
           "v2.llm_obs_label_schema" => "LLMObsLabelSchema",
           "v2.llm_obs_label_schema_type" => "LLMObsLabelSchemaType",
           "v2.llm_obs_metric_assessment" => "LLMObsMetricAssessment",
           "v2.llm_obs_metric_score_type" => "LLMObsMetricScoreType",
+          "v2.llm_obs_open_ai_metadata" => "LLMObsOpenAIMetadata",
+          "v2.llm_obs_open_ai_reasoning_effort" => "LLMObsOpenAIReasoningEffort",
+          "v2.llm_obs_open_ai_reasoning_summary" => "LLMObsOpenAIReasoningSummary",
           "v2.llm_obs_project_data_attributes_request" => "LLMObsProjectDataAttributesRequest",
           "v2.llm_obs_project_data_attributes_response" => "LLMObsProjectDataAttributesResponse",
           "v2.llm_obs_project_data_request" => "LLMObsProjectDataRequest",
@@ -3900,6 +3925,7 @@ def overrides
           "v2.llm_obs_trace_interaction_item" => "LLMObsTraceInteractionItem",
           "v2.llm_obs_trace_interaction_response_item" => "LLMObsTraceInteractionResponseItem",
           "v2.llm_obs_trace_interaction_type" => "LLMObsTraceInteractionType",
+          "v2.llm_obs_vertex_ai_metadata" => "LLMObsVertexAIMetadata",
           "v2.log" => "Log",
           "v2.log_attributes" => "LogAttributes",
           "v2.logs_aggregate_bucket" => "LogsAggregateBucket",