tests: Prepared and executed the testing strategy #46
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| # Compiled binary | ||
| /gomodel | ||
| /recordapi | ||
| /bin | ||
|
|
||
| # Environment variables | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,275 @@ | ||
| // Package main provides a CLI tool to record real API responses for contract tests. | ||
| // Usage: | ||
| // | ||
| // OPENAI_API_KEY=sk-xxx go run ./cmd/recordapi \ | ||
| // -provider=openai \ | ||
| // -endpoint=chat \ | ||
| // -output=tests/contract/testdata/openai/chat_completion.json | ||
| package main | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
||
| // Provider configurations | ||
| var providerConfigs = map[string]struct { | ||
| baseURL string | ||
| envKey string | ||
| authHeader string | ||
| contentType string | ||
| }{ | ||
| "openai": { | ||
| baseURL: "https://api.openai.com", | ||
| envKey: "OPENAI_API_KEY", | ||
| authHeader: "Authorization", | ||
| contentType: "application/json", | ||
| }, | ||
| "anthropic": { | ||
| baseURL: "https://api.anthropic.com", | ||
| envKey: "ANTHROPIC_API_KEY", | ||
| authHeader: "x-api-key", | ||
| contentType: "application/json", | ||
| }, | ||
| "gemini": { | ||
| baseURL: "https://generativelanguage.googleapis.com/v1beta/openai", | ||
| envKey: "GEMINI_API_KEY", | ||
| authHeader: "Authorization", | ||
| contentType: "application/json", | ||
| }, | ||
| "groq": { | ||
| baseURL: "https://api.groq.com/openai", | ||
| envKey: "GROQ_API_KEY", | ||
| authHeader: "Authorization", | ||
| contentType: "application/json", | ||
| }, | ||
| "xai": { | ||
| baseURL: "https://api.x.ai", | ||
| envKey: "XAI_API_KEY", | ||
| authHeader: "Authorization", | ||
| contentType: "application/json", | ||
| }, | ||
| } | ||
|
|
||
| // Endpoint configurations | ||
| var endpointConfigs = map[string]struct { | ||
| path string | ||
| method string | ||
| requestBody map[string]interface{} | ||
| }{ | ||
| "chat": { | ||
| path: "/v1/chat/completions", | ||
| method: http.MethodPost, | ||
| requestBody: map[string]interface{}{ | ||
| "model": "gpt-4o-mini", | ||
| "messages": []map[string]string{ | ||
| {"role": "user", "content": "Say 'Hello, World!' and nothing else."}, | ||
| }, | ||
| "max_tokens": 50, | ||
| }, | ||
| }, | ||
| "chat_stream": { | ||
| path: "/v1/chat/completions", | ||
| method: http.MethodPost, | ||
| requestBody: map[string]interface{}{ | ||
| "model": "gpt-4o-mini", | ||
| "messages": []map[string]string{ | ||
| {"role": "user", "content": "Say 'Hello, World!' and nothing else."}, | ||
| }, | ||
| "max_tokens": 50, | ||
| "stream": true, | ||
| }, | ||
| }, | ||
| "models": { | ||
| path: "/v1/models", | ||
| method: http.MethodGet, | ||
| }, | ||
| } | ||
|
SantiagoDePolonia marked this conversation as resolved.
|
||
|
|
||
| func main() { | ||
| provider := flag.String("provider", "openai", "Provider to test (openai, anthropic, gemini, groq, xai)") | ||
| endpoint := flag.String("endpoint", "chat", "Endpoint to test (chat, chat_stream, models)") | ||
| output := flag.String("output", "", "Output file path (required)") | ||
| model := flag.String("model", "", "Override model in request") | ||
| flag.Parse() | ||
|
|
||
| if *output == "" { | ||
| fmt.Fprintln(os.Stderr, "Error: -output flag is required") | ||
| flag.Usage() | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| pConfig, ok := providerConfigs[*provider] | ||
| if !ok { | ||
| fmt.Fprintf(os.Stderr, "Error: unknown provider %q\n", *provider) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| eConfig, ok := endpointConfigs[*endpoint] | ||
| if !ok { | ||
| fmt.Fprintf(os.Stderr, "Error: unknown endpoint %q\n", *endpoint) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| apiKey := os.Getenv(pConfig.envKey) | ||
| if apiKey == "" { | ||
| fmt.Fprintf(os.Stderr, "Error: %s environment variable is required\n", pConfig.envKey) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| // Build request body | ||
| var bodyReader io.Reader | ||
| if eConfig.requestBody != nil { | ||
| reqBody := eConfig.requestBody | ||
|
|
||
| // Override model if specified | ||
| if *model != "" { | ||
| reqBody["model"] = *model | ||
| } | ||
|
|
||
| // Adjust request for different providers | ||
| if *provider == "anthropic" { | ||
| reqBody = adjustForAnthropic(reqBody) | ||
| } | ||
|
|
||
| bodyBytes, err := json.Marshal(reqBody) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error marshaling request body: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
| bodyReader = bytes.NewReader(bodyBytes) | ||
| } | ||
|
SantiagoDePolonia marked this conversation as resolved.
|
||
|
|
||
| // Build URL | ||
| url := pConfig.baseURL + eConfig.path | ||
|
|
||
| // Create request | ||
| req, err := http.NewRequest(eConfig.method, url, bodyReader) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error creating request: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| req.Header.Set("Content-Type", pConfig.contentType) | ||
|
|
||
| // Add auth header (except for Gemini which uses query param) | ||
| if pConfig.authHeader != "" { | ||
| if pConfig.authHeader == "Authorization" { | ||
| req.Header.Set(pConfig.authHeader, "Bearer "+apiKey) | ||
| } else { | ||
| req.Header.Set(pConfig.authHeader, apiKey) | ||
| } | ||
| } | ||
|
|
||
| // Add Anthropic-specific headers | ||
| if *provider == "anthropic" { | ||
| req.Header.Set("anthropic-version", "2023-06-01") | ||
| } | ||
|
|
||
| // Send request | ||
| client := &http.Client{Timeout: 60 * time.Second} | ||
| fmt.Printf("Sending request to %s %s...\n", eConfig.method, url) | ||
|
|
||
| resp, err := client.Do(req) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error sending request: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| fmt.Printf("Response status: %d %s\n", resp.StatusCode, resp.Status) | ||
|
|
||
| // Read response body | ||
| body, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error reading response: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| // Handle streaming responses differently | ||
| if strings.HasSuffix(*endpoint, "_stream") { | ||
| if err := writeStreamOutput(*output, body); err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error writing output: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
| fmt.Printf("Streaming response saved to %s\n", *output) | ||
| return | ||
| } | ||
|
Comment on lines
+190
to
+205
Contributor
There was a problem hiding this comment. 🧹 Nitpick | 🔵 Trivial Stream SSE responses directly to disk instead of For streaming endpoints, buffering the entire response defeats the purpose and can spike memory. Pipe ♻️ Proposed refactor (stream to file)- // Read response body
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error reading response: %v\n", err)
- os.Exit(1)
- }
-
// Handle streaming responses differently
if strings.HasSuffix(*endpoint, "_stream") {
- if err := writeStreamOutput(*output, body); err != nil {
+ if err := writeStreamOutput(*output, resp.Body); err != nil {
fmt.Fprintf(os.Stderr, "Error writing output: %v\n", err)
os.Exit(1)
}
fmt.Printf("Streaming response saved to %s\n", *output)
return
}
+
+ // Read response body
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Error reading response: %v\n", err)
+ os.Exit(1)
+ }
@@
-func writeStreamOutput(path string, data []byte) error {
- // For streaming responses, save as-is (SSE format)
- return writeOutput(path, data)
+func writeStreamOutput(path string, r io.Reader) error {
+ dir := filepath.Dir(path)
+ if err := os.MkdirAll(dir, 0755); err != nil {
+ return fmt.Errorf("failed to create directory: %w", err)
+ }
+ f, err := os.Create(path)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = f.Close() }()
+ _, err = io.Copy(f, r)
+ return err
}Also applies to: 271-274 🤖 Prompt for AI Agents |
||
|
|
||
| // Pretty print JSON | ||
| var prettyJSON bytes.Buffer | ||
| if err := json.Indent(&prettyJSON, body, "", " "); err != nil { | ||
| // If it's not valid JSON, write raw | ||
| if err := writeOutput(*output, body); err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error writing output: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
| fmt.Printf("Raw response saved to %s\n", *output) | ||
|
SantiagoDePolonia marked this conversation as resolved.
|
||
| return | ||
| } | ||
|
|
||
| if err := writeOutput(*output, prettyJSON.Bytes()); err != nil { | ||
| fmt.Fprintf(os.Stderr, "Error writing output: %v\n", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| fmt.Printf("Response saved to %s\n", *output) | ||
|
|
||
| // Print response summary | ||
| var respMap map[string]interface{} | ||
| if err := json.Unmarshal(body, &respMap); err == nil { | ||
| if id, ok := respMap["id"].(string); ok { | ||
| fmt.Printf("Response ID: %s\n", id) | ||
| } | ||
| if model, ok := respMap["model"].(string); ok { | ||
| fmt.Printf("Model: %s\n", model) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // adjustForAnthropic converts OpenAI-style request to Anthropic format | ||
| func adjustForAnthropic(req map[string]interface{}) map[string]interface{} { | ||
| result := make(map[string]interface{}) | ||
|
|
||
| // Copy model | ||
| if model, ok := req["model"].(string); ok { | ||
| result["model"] = model | ||
| } | ||
|
|
||
| // Convert max_tokens | ||
| if maxTokens, ok := req["max_tokens"].(int); ok { | ||
| result["max_tokens"] = maxTokens | ||
| } else { | ||
| result["max_tokens"] = 1024 // Default for Anthropic | ||
| } | ||
|
|
||
| // Convert messages | ||
| if messages, ok := req["messages"].([]map[string]string); ok { | ||
| result["messages"] = messages | ||
| } | ||
|
|
||
| return result | ||
| } | ||
|
|
||
| // writeOutput writes data to the output file, creating directories as needed. | ||
| func writeOutput(path string, data []byte) error { | ||
| dir := filepath.Dir(path) | ||
| if err := os.MkdirAll(dir, 0755); err != nil { | ||
| return fmt.Errorf("failed to create directory: %w", err) | ||
| } | ||
| return os.WriteFile(path, data, 0644) | ||
| } | ||
|
|
||
| // writeStreamOutput writes streaming response data to a text file. | ||
| func writeStreamOutput(path string, data []byte) error { | ||
| // For streaming responses, save as-is (SSE format) | ||
| return writeOutput(path, data) | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.