netclaw-dev · Aaronontheweb · Apr 12, 2026 · Apr 12, 2026 · Apr 12, 2026 · Apr 12, 2026
diff --git a/evals/run-evals.sh b/evals/run-evals.sh
@@ -334,6 +334,18 @@ CREATE TABLE IF NOT EXISTS eval_results (
     details     TEXT,
     PRIMARY KEY (run_id, case_name, run_number)
 );
+CREATE TABLE IF NOT EXISTS eval_metrics (
+    run_id            TEXT NOT NULL REFERENCES eval_runs(run_id),
+    category          TEXT NOT NULL,
+    case_name         TEXT NOT NULL,
+    run_number        INTEGER NOT NULL,
+    input_tokens      INTEGER,
+    output_tokens     INTEGER,
+    cached_tokens     INTEGER,
+    prompt_ms         REAL,
+    predicted_tok_s   REAL,
+    PRIMARY KEY (run_id, case_name, run_number)
+);
 SQL
 }
 
@@ -349,6 +361,73 @@ store_result() {
          VALUES ('$RUN_ID', '$esc_category', '$case_name', $run_number, '$esc_prompt', $passed, '$esc_details');"
 }
 
+## Parses the [usage] line from stdout and stores performance metrics.
+## Called after each run_prompt / store_result pair.
+store_metrics() {
+    [[ -z "$RESULTS_DB" ]] && return
+    [[ ! -f "$STDOUT_FILE" ]] && return
+
+    local case_name="$1" run_number="$2"
+    local usage_line
+    usage_line=$(grep -o '\[usage\].*' "$STDOUT_FILE" 2>/dev/null | tail -1) || return 0
+
+    # Parse fields from: [usage] in=X out=Y total=Z cached=C prompt_ms=P tok_s=T
+    local input_tokens output_tokens cached_tokens prompt_ms tok_s
+    input_tokens=$(echo "$usage_line" | grep -oP 'in=\K[0-9]+' || echo "")
+    output_tokens=$(echo "$usage_line" | grep -oP 'out=\K[0-9]+' || echo "")
+    cached_tokens=$(echo "$usage_line" | grep -oP 'cached=\K[0-9]+' || echo "")
+    prompt_ms=$(echo "$usage_line" | grep -oP 'prompt_ms=\K[0-9.]+' || echo "")
+    tok_s=$(echo "$usage_line" | grep -oP 'tok_s=\K[0-9.]+' || echo "")
+
+    # Skip if no metrics found
+    [[ -z "$input_tokens" && -z "$cached_tokens" && -z "$prompt_ms" ]] && return 0
+
+    local esc_category="${CURRENT_CATEGORY//\'/\'\'}"
+    sqlite3 "$RESULTS_DB" \
+        "INSERT INTO eval_metrics (run_id, category, case_name, run_number, input_tokens, output_tokens, cached_tokens, prompt_ms, predicted_tok_s)
+         VALUES ('$RUN_ID', '$esc_category', '$case_name', $run_number,
+                 ${input_tokens:-NULL}, ${output_tokens:-NULL}, ${cached_tokens:-NULL},
+                 ${prompt_ms:-NULL}, ${tok_s:-NULL});"
+}
+
+print_metrics_summary() {
+    [[ -z "$RESULTS_DB" ]] && return
+
+    local count
+    count=$(sqlite3 "$RESULTS_DB" "SELECT COUNT(*) FROM eval_metrics WHERE run_id='$RUN_ID' AND prompt_ms IS NOT NULL;" 2>/dev/null || echo "0")
+    [[ "$count" == "0" ]] && return
+
+    echo ""
+    echo "── Performance Metrics ──"
+    sqlite3 -header -column "$RESULTS_DB" <<SQL
+SELECT
+    category,
+    COUNT(*) as prompts,
+    CAST(ROUND(AVG(input_tokens)) AS INTEGER) as avg_input,
+    CAST(ROUND(AVG(output_tokens)) AS INTEGER) as avg_output,
+    CAST(ROUND(AVG(cached_tokens)) AS INTEGER) as avg_cached,
+    ROUND(AVG(prompt_ms), 1) as avg_prompt_ms,
+    ROUND(AVG(predicted_tok_s), 1) as avg_tok_s
+FROM eval_metrics
+WHERE run_id='$RUN_ID' AND input_tokens IS NOT NULL
+GROUP BY category;
+SQL
+
+    echo ""
+    sqlite3 -header -column "$RESULTS_DB" <<SQL
+SELECT
+    'overall' as scope,
+    COUNT(*) as prompts,
+    CAST(ROUND(AVG(prompt_ms)) AS INTEGER) as avg_prompt_ms,
+    CAST(ROUND(MIN(prompt_ms)) AS INTEGER) as min_prompt_ms,
+    CAST(ROUND(MAX(prompt_ms)) AS INTEGER) as max_prompt_ms,
+    ROUND(AVG(predicted_tok_s), 1) as avg_tok_s,
+    CAST(ROUND(AVG(cached_tokens)) AS INTEGER) as avg_cached
+FROM eval_metrics
+WHERE run_id='$RUN_ID' AND prompt_ms IS NOT NULL;
+SQL
+}
+
 finalize_db() {
     [[ -z "$RESULTS_DB" ]] && return
     local score
@@ -615,6 +694,7 @@ run_case() {
         fi
 
         store_result "$case_name" "$run" "$prompt" "$passed" "$details"
+        store_metrics "$case_name" "$run"
     done
 
     local score
@@ -794,6 +874,8 @@ main() {
     echo "─────────────────────────────────────────────────"
     echo "Overall: $PASSED_CASES/$TOTAL_CASES cases passed ($overall_score%)"
 
+    print_metrics_summary
+
     if [[ -n "$RESULTS_DB" ]]; then
         echo "Results: $RESULTS_DB (run_id: $RUN_ID)"
     fi

diff --git a/src/Netclaw.Actors/Protocol/SessionOutput.cs b/src/Netclaw.Actors/Protocol/SessionOutput.cs
@@ -114,6 +114,21 @@ public sealed record UsageOutput : SessionOutput
     /// are unavailable.
     /// </summary>
     public double? UsagePercent { get; init; }
+
+    // ── Server-side timing (llama.cpp timings object) ──
+
+    /// <summary>
+    /// Server-side prompt processing (prefill) time in milliseconds.
+    /// Sourced from llama.cpp <c>timings.prompt_ms</c>. Null when the
+    /// provider does not report timing data.
+    /// </summary>
+    public double? PromptMs { get; init; }
+
+    /// <summary>
+    /// Server-side output generation throughput in tokens per second.
+    /// Sourced from llama.cpp <c>timings.predicted_per_second</c>.
+    /// </summary>
+    public double? PredictedPerSecond { get; init; }
 }
 
 /// <summary>

diff --git a/src/Netclaw.Actors/Protocol/SessionOutputDto.cs b/src/Netclaw.Actors/Protocol/SessionOutputDto.cs
@@ -67,8 +67,12 @@ public sealed record SessionOutputDto
     public long? InputTokens { get; init; }
     public long? OutputTokens { get; init; }
     public long? TotalTokens { get; init; }
+    public long? CachedInputTokens { get; init; }
+    public long? ReasoningTokens { get; init; }
     public int? ContextWindowTokens { get; init; }
     public double? UsagePercent { get; init; }
+    public double? PromptMs { get; init; }
+    public double? PredictedPerSecond { get; init; }
 
     // Turn Completed
     public int? TurnNumber { get; init; }

diff --git a/src/Netclaw.Actors/Protocol/SessionOutputDtoMapper.cs b/src/Netclaw.Actors/Protocol/SessionOutputDtoMapper.cs
@@ -68,8 +68,12 @@ public static class SessionOutputDtoMapper
             InputTokens = msg.InputTokens,
             OutputTokens = msg.OutputTokens,
             TotalTokens = msg.TotalTokens,
+            CachedInputTokens = msg.CachedInputTokens,
+            ReasoningTokens = msg.ReasoningTokens,
             ContextWindowTokens = msg.ContextWindowTokens,
-            UsagePercent = msg.UsagePercent
+            UsagePercent = msg.UsagePercent,
+            PromptMs = msg.PromptMs,
+            PredictedPerSecond = msg.PredictedPerSecond,
         },
 
         TurnCompleted msg => new SessionOutputDto
@@ -233,8 +237,12 @@ public static SessionOutput FromDto(SessionOutputDto dto)
                 InputTokens = dto.InputTokens,
                 OutputTokens = dto.OutputTokens,
                 TotalTokens = dto.TotalTokens,
+                CachedInputTokens = dto.CachedInputTokens,
+                ReasoningTokens = dto.ReasoningTokens,
                 ContextWindowTokens = dto.ContextWindowTokens ?? 0,
-                UsagePercent = dto.UsagePercent
+                UsagePercent = dto.UsagePercent,
+                PromptMs = dto.PromptMs,
+                PredictedPerSecond = dto.PredictedPerSecond,
             },
             SessionOutputTypes.TurnCompleted => new TurnCompleted
             {

diff --git a/src/Netclaw.Actors/Sessions/LlmSessionActor.cs b/src/Netclaw.Actors/Sessions/LlmSessionActor.cs
@@ -2503,6 +2503,15 @@ private void EmitUsageOutput(UsageDetails usage)
             ? (double)usage.InputTokenCount.Value / contextWindow
             : null;
 
+        // Decode llama.cpp server-side timing from UsageDetails.AdditionalCounts.
+        // Canonical encoding lives in Netclaw.Providers.SelfHosted.OpenAiCompatibleChatClient
+        // (PromptUsKey, PredictedTokPerSecX100Key). Keep these strings in sync.
+        var additional = usage.AdditionalCounts;
+        double? promptMs = additional is not null && additional.TryGetValue("prompt_us", out var pUs)
+            ? pUs / 1000.0 : null;
+        double? predictedPerSec = additional is not null && additional.TryGetValue("predicted_tok_per_sec_x100", out var pps)
+            ? pps / 100.0 : null;
+
         EmitOutput(new UsageOutput
         {
             SessionId = _sessionId,
@@ -2512,7 +2521,9 @@ private void EmitUsageOutput(UsageDetails usage)
             CachedInputTokens = usage.CachedInputTokenCount,
             ReasoningTokens = usage.ReasoningTokenCount,
             ContextWindowTokens = contextWindow,
-            UsagePercent = usagePercent
+            UsagePercent = usagePercent,
+            PromptMs = promptMs,
+            PredictedPerSecond = predictedPerSec,
         }, OutputFilter.Usage);
     }
 

diff --git a/src/Netclaw.Cli/HeadlessChannel.cs b/src/Netclaw.Cli/HeadlessChannel.cs
@@ -1,3 +1,4 @@
+using System.Diagnostics;
 using System.Text;
 using System.Text.Json;
 using Microsoft.Extensions.AI;
@@ -37,6 +38,10 @@ public sealed class HeadlessChannel : IChannel
     private JsonUsage? _usage;
     private string? _resolvedSessionId;
 
+    // Client-side timing
+    private long _promptSentTicks;
+    private long _firstDeltaTicks;
+
     public Actors.Channels.ChannelType ChannelType => Actors.Channels.ChannelType.Headless;
     public string DisplayName => "Headless Prompt";
 
@@ -130,6 +135,8 @@ private async Task RunHeadlessAsync(CancellationToken stopping)
             logWriter!.WriteLine($"[{_timeProvider.GetUtcNow():o}] Headless session started: {sessionId}");
             logWriter.WriteLine($"[{_timeProvider.GetUtcNow():o}] PROMPT: {_prompt}");
 
+            _promptSentTicks = Stopwatch.GetTimestamp();
+
             await _daemonClient.SendAsync(new Netclaw.Actors.Channels.ChannelInput
             {
                 SenderId = "local-user",
@@ -185,6 +192,8 @@ private void HandleOutput(SessionOutput output, StreamWriter? log)
                 break;
 
             case TextDeltaOutput msg:
+                if (!_receivedTextDeltaInCurrentTurn && _promptSentTicks > 0)
+                    Interlocked.CompareExchange(ref _firstDeltaTicks, Stopwatch.GetTimestamp(), 0);
                 _receivedTextDeltaInCurrentTurn = true;
                 if (_jsonOutput)
                     _responseBuffer.Append(msg.Delta);
@@ -241,14 +250,16 @@ private void HandleOutput(SessionOutput output, StreamWriter? log)
                         OutputTokens = msg.OutputTokens,
                         TotalTokens = msg.TotalTokens,
                         CachedInputTokens = msg.CachedInputTokens,
-                        ReasoningTokens = msg.ReasoningTokens
+                        ReasoningTokens = msg.ReasoningTokens,
+                        PromptMs = msg.PromptMs,
+                        PredictedPerSecond = msg.PredictedPerSecond,
                     };
                 }
                 else
                 {
-                    Console.WriteLine($"[usage] in={msg.InputTokens} out={msg.OutputTokens} total={msg.TotalTokens}");
+                    Console.WriteLine($"[usage] in={msg.InputTokens} out={msg.OutputTokens} total={msg.TotalTokens} cached={msg.CachedInputTokens} prompt_ms={msg.PromptMs} tok_s={msg.PredictedPerSecond}");
                 }
-                Log(log, $"USAGE: in={msg.InputTokens} out={msg.OutputTokens} total={msg.TotalTokens} cached={msg.CachedInputTokens} reasoning={msg.ReasoningTokens} context_window={msg.ContextWindowTokens}");
+                Log(log, $"USAGE: in={msg.InputTokens} out={msg.OutputTokens} total={msg.TotalTokens} cached={msg.CachedInputTokens} reasoning={msg.ReasoningTokens} context_window={msg.ContextWindowTokens} prompt_ms={msg.PromptMs} predicted_tok_s={msg.PredictedPerSecond}");
                 break;
 
             case ErrorOutput msg:
@@ -311,12 +322,23 @@ private void HandleOutput(SessionOutput output, StreamWriter? log)
 
     private void WriteJsonEnvelope()
     {
+        // Client-side timing
+        var now = Stopwatch.GetTimestamp();
+        double? ttftMs = _firstDeltaTicks > 0 && _promptSentTicks > 0
+            ? Stopwatch.GetElapsedTime(_promptSentTicks, _firstDeltaTicks).TotalMilliseconds
+            : null;
+        double? totalMs = _promptSentTicks > 0
+            ? Stopwatch.GetElapsedTime(_promptSentTicks, now).TotalMilliseconds
+            : null;
+
         var envelope = new JsonEnvelope
         {
             SessionId = _resolvedSessionId!,
             Response = _responseBuffer.ToString(),
             ToolCalls = _toolCalls.Count > 0 ? _toolCalls : null,
-            Usage = _usage
+            Usage = _usage,
+            TtftMs = ttftMs.HasValue ? Math.Round(ttftMs.Value, 1) : null,
+            TotalMs = totalMs.HasValue ? Math.Round(totalMs.Value, 1) : null,
         };
 
         Console.WriteLine(JsonSerializer.Serialize(envelope, s_jsonOptions));
@@ -350,6 +372,8 @@ private sealed class JsonEnvelope
         public required string Response { get; init; }
         public List<JsonToolCall>? ToolCalls { get; init; }
         public JsonUsage? Usage { get; init; }
+        public double? TtftMs { get; init; }
+        public double? TotalMs { get; init; }
     }
 
     private sealed class JsonToolCall
@@ -366,5 +390,7 @@ private sealed class JsonUsage
         public long? TotalTokens { get; init; }
         public long? CachedInputTokens { get; init; }
         public long? ReasoningTokens { get; init; }
+        public double? PromptMs { get; init; }
+        public double? PredictedPerSecond { get; init; }
     }
 }