Skip to content

Commit 8cbe058

Browse files
mirpomirpo
authored
Merge pull request #37 from mirpo/feat/move_code_v2
refactor old code
2 parents 9c5dc42 + 6230a88 commit 8cbe058

19 files changed

Lines changed: 1083 additions & 392 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,5 +199,8 @@ Options:
199199
| [Fine-tuning Data](./examples/v1/7.%20fine-tuning%20dataset/README.md) | Training dataset creation | Ollama |
200200
| [Vision Models](./examples/v1/8.%20hugginface%20images%20and%20qwen2.5vl%20or%20gemma3/README.md) | Image analysis | Ollama, LM Studio |
201201
| [OpenAI](./examples/v1/9.%20openai-example/README.md) | Cloud provider usage | OpenAI |
202-
| [Gemini](./examples/v1/10.%20openrouter-example/README.md) | Cloud provider usage | Gemini |
202+
| [OpenRouter](./examples/v1/10.%20openrouter-example/README.md) | Cloud provider usage | OpenRouter |
203+
| [Gemini](./examples/v1/11.%20gemini-example/README.md) | Cloud provider usage | Gemini |
203204
| [CV Processing Pipeline](./examples/v1/12.%20cv-processing-pipeline/README.md) | Multi-step CV processing | Ollama |
205+
| [Retry Configuration](./examples/v1/13.%20retry%20configuration%20example/README.md) | Retry logic configuration | Ollama |
206+
| [Recipe with Nested Fields](./examples/v1/14.%20recipe%20generation%20with%20nested%20fields/README.md) | Nested field access demo | Ollama |

‎config/config.go‎

Lines changed: 5 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
package config
22

33
import (
4-
"time"
5-
64
"github.com/mirpo/datamatic/jsonschema"
75
"github.com/mirpo/datamatic/llm"
6+
"github.com/mirpo/datamatic/retry"
87
)
98

109
const (
@@ -20,7 +19,7 @@ func NewConfig() *Config {
2019
HTTPTimeout: 300,
2120
ValidateResponse: true,
2221
SkipCliWarning: false,
23-
RetryConfig: NewDefaultRetryConfig(),
22+
RetryConfig: retry.NewDefaultConfig(),
2423
}
2524
}
2625

@@ -32,9 +31,9 @@ type Config struct {
3231
HTTPTimeout int
3332
ValidateResponse bool
3433
SkipCliWarning bool
35-
Version string `yaml:"version"`
36-
Steps []Step `yaml:"steps"`
37-
RetryConfig RetryConfig `yaml:"retryConfig"`
34+
Version string `yaml:"version"`
35+
Steps []Step `yaml:"steps"`
36+
RetryConfig retry.Config `yaml:"retryConfig"`
3837
}
3938

4039
type StepType string
@@ -69,38 +68,6 @@ type ModelConfig struct {
6968
MaxTokens *int `yaml:"maxTokens"`
7069
}
7170

72-
type RetryConfig struct {
73-
MaxAttempts int `yaml:"maxAttempts"`
74-
InitialDelay time.Duration `yaml:"initialDelay"`
75-
MaxDelay time.Duration `yaml:"maxDelay"`
76-
BackoffMultiplier float64 `yaml:"backoffMultiplier"`
77-
Enabled bool `yaml:"enabled"`
78-
}
79-
80-
func NewDefaultRetryConfig() RetryConfig {
81-
return RetryConfig{
82-
MaxAttempts: 3,
83-
InitialDelay: 1 * time.Second,
84-
MaxDelay: 10 * time.Second,
85-
BackoffMultiplier: 2.0,
86-
Enabled: true,
87-
}
88-
}
89-
90-
func (s *Step) GetProviderConfig(httpTimeout int) llm.ProviderConfig {
91-
providerConfig := llm.ProviderConfig{
92-
BaseURL: s.ModelConfig.BaseURL,
93-
ProviderType: s.ModelConfig.ModelProvider,
94-
ModelName: s.ModelConfig.ModelName,
95-
AuthToken: "token",
96-
HTTPTimeout: httpTimeout,
97-
Temperature: s.ModelConfig.Temperature,
98-
MaxTokens: s.ModelConfig.MaxTokens,
99-
}
100-
101-
return providerConfig
102-
}
103-
10471
func (c *Config) GetStepByName(name string) *Step {
10572
for _, step := range c.Steps {
10673
if step.Name == name {

‎config/config_test.go‎

Lines changed: 0 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,9 @@ import (
44
"testing"
55
"time"
66

7-
"github.com/mirpo/datamatic/llm"
87
"github.com/stretchr/testify/assert"
98
)
109

11-
func floatPtr(f float64) *float64 {
12-
return &f
13-
}
14-
15-
func intPtr(i int) *int {
16-
return &i
17-
}
18-
1910
func TestNewConfig(t *testing.T) {
2011
cfg := NewConfig()
2112

@@ -34,89 +25,6 @@ func TestNewConfig(t *testing.T) {
3425
assert.Equal(t, 2.0, cfg.RetryConfig.BackoffMultiplier)
3526
}
3627

37-
func TestGetProviderConfig(t *testing.T) {
38-
tests := []struct {
39-
name string
40-
step Step
41-
httpTimeout int
42-
expected llm.ProviderConfig
43-
}{
44-
{
45-
name: "Full config with temperature and max tokens",
46-
step: Step{
47-
ModelConfig: ModelConfig{
48-
BaseURL: "http://example.com/api",
49-
ModelProvider: llm.ProviderOllama,
50-
ModelName: "llama3",
51-
Temperature: floatPtr(0.7),
52-
MaxTokens: intPtr(1000),
53-
},
54-
},
55-
httpTimeout: 30,
56-
expected: llm.ProviderConfig{
57-
ProviderType: llm.ProviderOllama,
58-
BaseURL: "http://example.com/api",
59-
ModelName: "llama3",
60-
AuthToken: "token",
61-
Temperature: floatPtr(0.7),
62-
MaxTokens: intPtr(1000),
63-
HTTPTimeout: 30,
64-
},
65-
},
66-
{
67-
name: "Config without temperature and max tokens",
68-
step: Step{
69-
ModelConfig: ModelConfig{
70-
BaseURL: "http://another-api.com",
71-
ModelProvider: llm.ProviderOllama,
72-
ModelName: "command",
73-
Temperature: nil,
74-
MaxTokens: nil,
75-
},
76-
},
77-
httpTimeout: 60,
78-
expected: llm.ProviderConfig{
79-
ProviderType: llm.ProviderOllama,
80-
BaseURL: "http://another-api.com",
81-
ModelName: "command",
82-
AuthToken: "token",
83-
Temperature: nil,
84-
MaxTokens: nil,
85-
HTTPTimeout: 60,
86-
},
87-
},
88-
{
89-
name: "Different http timeout",
90-
step: Step{
91-
ModelConfig: ModelConfig{
92-
BaseURL: "http://api3.org",
93-
ModelProvider: llm.ProviderLmStudio,
94-
ModelName: "gpt-3.5-turbo",
95-
Temperature: floatPtr(0.1),
96-
MaxTokens: intPtr(500),
97-
},
98-
},
99-
httpTimeout: 10,
100-
expected: llm.ProviderConfig{
101-
ProviderType: llm.ProviderLmStudio,
102-
BaseURL: "http://api3.org",
103-
ModelName: "gpt-3.5-turbo",
104-
AuthToken: "token",
105-
Temperature: floatPtr(0.1),
106-
MaxTokens: intPtr(500),
107-
HTTPTimeout: 10,
108-
},
109-
},
110-
}
111-
112-
for _, tt := range tests {
113-
t.Run(tt.name, func(t *testing.T) {
114-
actual := tt.step.GetProviderConfig(tt.httpTimeout)
115-
assert.Equal(t, tt.expected, actual, "ProviderConfig should match expected")
116-
})
117-
}
118-
}
119-
12028
func TestGetStepByName(t *testing.T) {
12129
step1 := Step{Name: "step1"}
12230
step2 := Step{Name: "step2"}
@@ -133,19 +41,3 @@ func TestGetStepByName(t *testing.T) {
13341
assert.Nil(t, step)
13442
})
13543
}
136-
137-
func TestRetryConfig(t *testing.T) {
138-
cfg := RetryConfig{
139-
MaxAttempts: 5,
140-
InitialDelay: 2 * time.Second,
141-
MaxDelay: 30 * time.Second,
142-
BackoffMultiplier: 1.5,
143-
Enabled: false,
144-
}
145-
146-
assert.False(t, cfg.Enabled)
147-
assert.Equal(t, 5, cfg.MaxAttempts)
148-
assert.Equal(t, 2*time.Second, cfg.InitialDelay)
149-
assert.Equal(t, 30*time.Second, cfg.MaxDelay)
150-
assert.Equal(t, 1.5, cfg.BackoffMultiplier)
151-
}

‎config/validate.go‎

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"strings"
99

1010
"github.com/mirpo/datamatic/promptbuilder"
11+
"github.com/mirpo/datamatic/retry"
1112
)
1213

1314
func validateVersion(version string) error {
@@ -57,6 +58,26 @@ func validateModelConfig(step ModelConfig) error {
5758
return nil
5859
}
5960

61+
func validateRetryConfig(cfg retry.Config) error {
62+
if cfg.MaxAttempts <= 0 {
63+
return errors.New("maxAttempts must be greater than 0")
64+
}
65+
66+
if cfg.InitialDelay <= 0 {
67+
return errors.New("initialDelay must be greater than 0")
68+
}
69+
70+
if cfg.MaxDelay < cfg.InitialDelay {
71+
return errors.New("maxDelay must be greater than or equal to initialDelay")
72+
}
73+
74+
if cfg.BackoffMultiplier < 1.0 {
75+
return errors.New("backoffMultiplier must be greater than or equal to 1.0")
76+
}
77+
78+
return nil
79+
}
80+
6081
func validateMaxResults(step *Step, stepNames map[string]bool) error {
6182
switch v := step.MaxResults.(type) {
6283
case nil, int:
@@ -86,6 +107,15 @@ func (c *Config) Validate() error {
86107
return err
87108
}
88109

110+
// If retryConfig is not set in YAML (has zero values), use defaults
111+
if c.RetryConfig.MaxAttempts == 0 {
112+
c.RetryConfig = retry.NewDefaultConfig()
113+
}
114+
115+
if err := validateRetryConfig(c.RetryConfig); err != nil {
116+
return fmt.Errorf("retry config validation failed: %w", err)
117+
}
118+
89119
stepNames := map[string]bool{}
90120
cliCalls := []string{}
91121

@@ -127,20 +157,23 @@ func (c *Config) Validate() error {
127157
return fmt.Errorf("placeholder has a references to unknown or not previous steps, step: %s, placeholder: %+v", step.Name, val)
128158
}
129159

130-
// JSON key
160+
// JSON key - supports nested paths like "user.profile.name"
131161
if len(val.Key) > 0 {
132-
if strings.Contains(val.Key, ".") {
133-
return fmt.Errorf("placeholders currently support only one level of nesting, step: %s, placeholder: %+v", step.Name, val)
134-
}
135-
136162
refStep := c.GetStepByName(val.Step)
137163
if refStep.Type == PromptStepType {
138164
if !refStep.JSONSchema.HasSchemaDefinition() {
139165
return fmt.Errorf("step %s must have JSON schema, key: %s", val.Step, val.Key)
140166
}
141167

142-
if !refStep.JSONSchema.HasRequiredProperty(val.Key) {
143-
return fmt.Errorf("'%s' key must be defined in step %s in JSON schema as a property and required", val.Key, val.Step)
168+
if strings.Contains(val.Key, ".") {
169+
if !refStep.JSONSchema.HasFieldPath(val.Key) {
170+
return fmt.Errorf("field path '%s' not found in step %s JSON schema", val.Key, val.Step)
171+
}
172+
} else {
173+
// For single field, use existing validation
174+
if !refStep.JSONSchema.HasRequiredProperty(val.Key) {
175+
return fmt.Errorf("'%s' key must be defined in step %s in JSON schema as a property and required", val.Key, val.Step)
176+
}
144177
}
145178
}
146179
}

0 commit comments

Comments
 (0)