Workflow Execution Trace — Implementation Plan

export interface LLMCallUpdate {
  type: "llm_call";
  node_id: string;
  node_name?: string | null;
  provider: string;
  model: string;
  messages: Array<{ role: string; content: unknown }>;
  response: unknown;
  tool_calls?: Array<{ id: string; name: string; args: unknown }> | null;
  tokens_input?: number | null;
  tokens_output?: number | null;
  cost?: number | null;
  duration_ms: number;
  error?: string | null;
  timestamp: string;
}

private _emitMessage: ((msg: unknown) => void) | null = null;

setMessageEmitter(fn: (msg: unknown) => void): void {
  this._emitMessage = fn;
}

protected emitMessage(msg: unknown): void {
  if (this._emitMessage) this._emitMessage(msg);
}

async generateMessageTraced(args: Parameters<this["generateMessage"]>[0]): Promise<Message> {
  const startTime = Date.now();
  const tracer = getTracer();

  const doCall = async (): Promise<Message> => {
    if (!tracer) return this.generateMessage(args);
    return tracer.startActiveSpan(`llm.chat ${this.provider}/${args.model}`, async (span) => {
      span.setAttributes({
        "llm.provider": this.provider,
        "llm.model": args.model,
        "llm.request.message_count": args.messages.length,
        "llm.request.tools_count": args.tools?.length ?? 0,
        "llm.request.max_tokens": args.maxTokens ?? 0,
        "llm.request.stream": false,
      });
      try {
        const result = await this.generateMessage(args);
        const content = typeof result.content === "string" ? result.content : JSON.stringify(result.content);
        span.setAttributes({
          "llm.response.role": result.role,
          "llm.response.content": content.slice(0, 2000),
          "llm.response.tool_calls_count": result.toolCalls?.length ?? 0,
        });
        span.setStatus({ code: SpanStatusCode.OK });
        return result;
      } catch (err) {
        span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
        span.recordException(err as Error);
        throw err;
      } finally {
        span.end();
      }
    });
  };

  let result: Message | undefined;
  let error: string | undefined;
  try {
    result = await doCall();
    return result;
  } catch (err) {
    error = String(err);
    throw err;
  } finally {
    this.emitMessage({
      type: "llm_call",
      node_id: "",
      provider: this.provider,
      model: args.model,
      messages: args.messages.map((m) => ({ role: m.role, content: m.content })),
      response: result?.content ?? null,
      tool_calls: result?.toolCalls?.map((tc) => ({ id: tc.id, name: tc.name, args: tc.args })) ?? null,
      tokens_input: null,
      tokens_output: null,
      cost: null,
      duration_ms: Date.now() - startTime,
      error: error ?? null,
      timestamp: new Date(startTime).toISOString(),
    });
  }
}

async *generateMessagesTraced(args: Parameters<this["generateMessages"]>[0]): AsyncGenerator<ProviderStreamItem> {
  const startTime = Date.now();
  log.debug("LLM call", { provider: this.provider, model: args.model });
  const tracer = getTracer();

  let fullResponse = "";
  let collectedToolCalls: Array<{ id: string; name: string; args: unknown }> = [];
  let error: string | undefined;

  try {
    const source = tracer
      ? this._tracedStream(args, tracer)
      : this.generateMessages(args);

    for await (const item of source) {
      // Accumulate text content from chunks
      if ("type" in item && (item as { type: string }).type === "chunk") {
        const chunk = item as { content?: string };
        if (chunk.content) fullResponse += chunk.content;
      }
      // Collect tool calls
      if ("id" in item && "name" in item && "args" in item) {
        const tc = item as { id: string; name: string; args: unknown };
        collectedToolCalls.push({ id: tc.id, name: tc.name, args: tc.args });
      }
      yield item;
    }
    log.debug("LLM call complete", { provider: this.provider, model: args.model });
  } catch (err) {
    error = String(err);
    throw err;
  } finally {
    this.emitMessage({
      type: "llm_call",
      node_id: "",
      provider: this.provider,
      model: args.model,
      messages: args.messages.map((m) => ({ role: m.role, content: m.content })),
      response: fullResponse || null,
      tool_calls: collectedToolCalls.length > 0 ? collectedToolCalls : null,
      tokens_input: null,
      tokens_output: null,
      cost: null,
      duration_ms: Date.now() - startTime,
      error: error ?? null,
      timestamp: new Date(startTime).toISOString(),
    });
  }
}

/** Internal: wrap generateMessages with OTel span */
private async *_tracedStream(
  args: Parameters<this["generateMessages"]>[0],
  tracer: ReturnType<typeof getTracer> & object,
): AsyncGenerator<ProviderStreamItem> {
  const span = tracer.startSpan(`llm.stream ${this.provider}/${args.model}`);
  span.setAttributes({
    "llm.provider": this.provider,
    "llm.model": args.model,
    "llm.request.message_count": args.messages.length,
    "llm.request.tools_count": args.tools?.length ?? 0,
    "llm.request.max_tokens": args.maxTokens ?? 0,
    "llm.request.stream": true,
  });
  let chunkCount = 0;
  try {
    for await (const item of this.generateMessages(args)) {
      chunkCount++;
      yield item;
    }
    span.setAttributes({ "llm.response.chunk_count": chunkCount });
    span.setStatus({ code: SpanStatusCode.OK });
  } catch (err) {
    span.setStatus({ code: SpanStatusCode.ERROR, message: String(err) });
    span.recordException(err as Error);
    throw err;
  } finally {
    span.end();
  }
}

resolved.setMessageEmitter((msg) => this.postMessage(msg as ProcessingMessage));

import { create } from "zustand";

export type TraceEventType =
  | "node_start"
  | "node_complete"
  | "node_error"
  | "llm_call"
  | "tool_call"
  | "tool_result"
  | "edge_active"
  | "output";

export interface TraceEvent {
  id: string;
  timestamp: string;
  relativeMs: number;
  type: TraceEventType;
  nodeId?: string;
  nodeName?: string;
  nodeType?: string;
  summary: string;
  detail: unknown;
}

const MAX_EVENTS = 10_000;

interface TraceStoreState {
  events: TraceEvent[];
  runStartTime: string | null;
  isRecording: boolean;
  startRun: (timestamp: string) => void;
  append: (event: TraceEvent) => void;
  clear: () => void;
  exportJSON: () => string;
}

let nextId = 0;
export function traceEventId(): string {
  return `te-${++nextId}`;
}

const useTraceStore = create<TraceStoreState>((set, get) => ({
  events: [],
  runStartTime: null,
  isRecording: false,

  startRun: (timestamp: string) =>
    set({ events: [], runStartTime: timestamp, isRecording: true }),

  append: (event: TraceEvent) =>
    set((state) => {
      if (!state.isRecording) return state;
      const events =
        state.events.length >= MAX_EVENTS
          ? [...state.events.slice(1), event]
          : [...state.events, event];
      return { events };
    }),

  clear: () => set({ events: [], runStartTime: null, isRecording: false }),

  exportJSON: () => {
    const { events, runStartTime } = get();
    return JSON.stringify({ runStartTime, events }, null, 2);
  },
}));

export default useTraceStore;

import useTraceStore, { traceEventId } from "./TraceStore";
import type { TraceEvent, TraceEventType } from "./TraceStore";

const traceAppend = useTraceStore.getState().append;
const traceStart = useTraceStore.getState().startRun;
const traceRunStart = useTraceStore.getState().runStartTime;

const relativeMs = (ts?: string): number => {
  if (!traceRunStart) return 0;
  const start = new Date(traceRunStart).getTime();
  const now = ts ? new Date(ts).getTime() : Date.now();
  return Math.max(0, now - start);
};

const now = new Date().toISOString();

// --- Trace event capture ---
if (data.type === "job_update") {
  const ju = data as JobUpdate;
  if (ju.status === "running") {
    traceStart(now);
  } else if (["completed", "failed", "cancelled", "error"].includes(ju.status)) {
    useTraceStore.setState({ isRecording: false });
  }
}

if (data.type === "node_update") {
  const nu = data as NodeUpdate;
  let traceType: TraceEventType | null = null;
  let summary = "";
  if (nu.status === "running") {
    traceType = "node_start";
    summary = `${nu.node_name || nu.node_id} started`;
  } else if (nu.status === "completed") {
    traceType = "node_complete";
    summary = `${nu.node_name || nu.node_id} completed`;
  } else if (nu.status === "error" || nu.status === "failed") {
    traceType = "node_error";
    summary = `${nu.node_name || nu.node_id} error: ${nu.error || "unknown"}`;
  }
  if (traceType) {
    traceAppend({
      id: traceEventId(),
      timestamp: now,
      relativeMs: relativeMs(),
      type: traceType,
      nodeId: nu.node_id,
      nodeName: nu.node_name ?? undefined,
      nodeType: nu.node_type ?? undefined,
      summary,
      detail: nu,
    });
  }
}

if (data.type === "llm_call") {
  const lc = data as any;
  const tokens = lc.tokens_output ? `${lc.tokens_output} tokens` : "";
  const dur = lc.duration_ms ? `${lc.duration_ms}ms` : "";
  const cost = lc.cost ? `$${lc.cost.toFixed(4)}` : "";
  const parts = [tokens, dur, cost].filter(Boolean).join(", ");
  traceAppend({
    id: traceEventId(),
    timestamp: lc.timestamp || now,
    relativeMs: relativeMs(lc.timestamp),
    type: "llm_call",
    nodeId: lc.node_id ?? undefined,
    nodeName: lc.node_name ?? undefined,
    summary: `${lc.provider}/${lc.model}${parts ? ` → ${parts}` : ""}`,
    detail: lc,
  });
}

if (data.type === "tool_call") {
  const tc = data as ToolCallUpdate;
  traceAppend({
    id: traceEventId(),
    timestamp: now,
    relativeMs: relativeMs(),
    type: "tool_call",
    nodeId: tc.node_id,
    summary: `Tool call: ${tc.name}`,
    detail: tc,
  });
}

if (data.type === "tool_result") {
  const tr = data as ToolResultUpdate;
  traceAppend({
    id: traceEventId(),
    timestamp: now,
    relativeMs: relativeMs(),
    type: "tool_result",
    nodeId: tr.node_id,
    summary: `Tool result: ${tr.name}`,
    detail: tr,
  });
}

if (data.type === "output_update") {
  const ou = data as OutputUpdate;
  traceAppend({
    id: traceEventId(),
    timestamp: now,
    relativeMs: relativeMs(),
    type: "output",
    nodeId: ou.node_id,
    summary: `Output: ${ou.output_name}`,
    detail: ou,
  });
}

if (data.type === "edge_update") {
  const eu = data as EdgeUpdate;
  if (eu.status === "active") {
    traceAppend({
      id: traceEventId(),
      timestamp: now,
      relativeMs: relativeMs(),
      type: "edge_active",
      summary: `Edge active: ${eu.edge_id}`,
      detail: eu,
    });
  }
}

export type BottomPanelView = "terminal" | "trace";

import TracePanel from "./TracePanel";
import TimelineIcon from "@mui/icons-material/Timeline";

useCombo(["Control", "Shift", "T"], () => handlePanelToggle("trace"), false);

const handleTerminalToggle = useCallback(() => {
  handlePanelToggle("terminal");
}, [handlePanelToggle]);

const handleTraceToggle = useCallback(() => {
  handlePanelToggle("trace");
}, [handlePanelToggle]);