Watch: evaluate live traces

An online policy that scores OpenTelemetry traces as they arrive.

Source: examples/watch. Read the guide for the concepts behind it.

harness-cost.yaml

# Agent harness templates (4 of 4): cost and steps per execution. What each
# execution spent, from the cost its trace reports (usage.cost_usd; an
# execution whose trace has none is skipped for cost, not scored zero), and
# whether it stayed within a step budget, with an alert when the hourly mean
# cost climbs.
#
#   evalsi policy apply -f examples/watch/harness-cost.yaml --server $URL
apiVersion: evals.si/v1alpha1
kind: OnlineEvalPolicy
metadata:
  name: harness-cost
  labels: {evals.si/project: studio}
spec:
  selector: '"workflow" in labels && labels["stage"] == "production"'
  stages:
    - evaluators:
        - {ref: builtin/cost}
        - {ref: builtin/step-budget, params: {max_steps: 30}}
  window: 1h
  alerts:
    - {metric: cost, above: 0.25, min_samples: 20}   # USD per execution

harness-default.yaml

# Agent harness templates (1 of 4): the default policy. Every execution of
# every deployed workflow, scored with no judge and no cost, from its first
# trace: nothing is applied per workflow. The bootstrap Job can create it at
# install (bootstrap.policies, docs/guides/integrate-an-agent-harness.md).
#
# The workflows label their traces (OTEL_RESOURCE_ATTRIBUTES or a trace
# source's traceLabels): evalsi.label.workflow, evalsi.label.version and
# evalsi.label.stage=production. Copy the file and change the project.
#
#   evalsi policy apply -f examples/watch/harness-default.yaml --server $URL
apiVersion: evals.si/v1alpha1
kind: OnlineEvalPolicy
metadata:
  name: harness-default
  labels: {evals.si/project: studio}
spec:
  selector: '"workflow" in labels && labels["stage"] == "production"'
  stages:
    - evaluators:
        - {ref: builtin/tool-errors}      # share of tool calls that failed
        - {ref: builtin/loop-detection}   # the same call repeated
        - {ref: builtin/latency}
        - {ref: builtin/token-usage}
  window: 1h
  # Executions whose tools failed become the next deploy gate's test set.
  promote:
    when: 'scores["tool-errors"] > 0.0'
    dataset: harness-regressions

harness-quality.yaml

# Agent harness templates (3 of 4): task quality, judged. A sample of each
# workflow's executions (and every one that errored) is graded by your judge,
# only after cheap checks pass, so a looping agent is not billed for. Needs a
# judge: set default_judge in the server's config, or name one under spec
# (judge: <name>). There is no default judge, and the server refuses this
# policy until it has one.
#
#   evalsi policy apply -f examples/watch/harness-quality.yaml --server $URL
apiVersion: evals.si/v1alpha1
kind: OnlineEvalPolicy
metadata:
  name: harness-quality
  labels: {evals.si/project: studio}
spec:
  selector: '"workflow" in labels && labels["stage"] == "production"'
  # judge: claude                     # one of the server's judges
  sampling:
    rate: 0.1
    always: ["error"]
  stages:
    - evaluators:
        - {ref: builtin/loop-detection}
    - when: 'scores["loop-detection"] == 1.0'
      evaluators:
        - {ref: builtin/goal-completion}   # did the trajectory reach the user's goal
        - ref: builtin/llm-judge
          name: answer-quality
          params:
            rubric: >-
              The final answer does what the user asked, is correct, and says
              so when it could not finish.
  window: 1h
  alerts:
    - {metric: answer-quality, below: 0.7, min_samples: 20}
  promote:
    when: 'scores["answer-quality"] < 0.5'
    dataset: harness-regressions

harness-safety.yaml

# Agent harness templates (2 of 4): safety on every execution. PII and secrets
# in a workflow's output, checked by pattern (no judge), on every trace, with
# an alert as soon as a few leak in an hour.
#
#   evalsi policy apply -f examples/watch/harness-safety.yaml --server $URL
apiVersion: evals.si/v1alpha1
kind: OnlineEvalPolicy
metadata:
  name: harness-safety
  labels: {evals.si/project: studio}
spec:
  selector: '"workflow" in labels && labels["stage"] == "production"'
  stages:
    - evaluators:
        - {ref: builtin/pii-leak}       # passes when the output has no PII
        - {ref: builtin/secret-leak}    # passes when it has no credentials
  window: 1h
  alerts:
    # Pass rate below 99%: more than one execution in a hundred leaked.
    - {metric: pii-leak, below: 0.99, min_samples: 20}
    - {metric: secret-leak, below: 0.99, min_samples: 20}

support-policy.yaml

# An online evaluation policy. Apply it to a running server:
#   evalsi policy apply -f examples/watch/support-policy.yaml --server http://localhost:8080
# Then point any OpenTelemetry-instrumented agent (or agentgateway) at the
# server's OTLP endpoint (main port, or 4317/4318 when enabled in evalsi.yaml).
apiVersion: evals.si/v1alpha1
kind: OnlineEvalPolicy
metadata:
  name: support-agent
  labels: {evals.si/project: support}   # on Kubernetes, default: the namespace
spec:
  # CEL over the trace: service, name, duration_ms, error, steps, tools,
  # models, attributes, resource.
  selector: service == "support-agent"
  sampling:
    rate: 0.1                                   # 10% of matching traces...
    always: ["error", "duration_ms > 20000"]    # ...plus every failure or slow turn
  stages:
    # Cheap checks on every sampled trace.
    - evaluators:
        - {ref: builtin/latency}
        - {ref: builtin/json-valid, name: valid-json}
    # The judge only when the cheap checks pass (scores: metric -> number).
    - when: 'scores["valid-json"] == 1.0'
      evaluators:
        - ref: builtin/llm-judge
          params: {rubric: helpfulness}
  window: 300s
  alerts:
    - metric: llm-judge
      below: 0.7
      min_samples: 20
      webhook: https://hooks.example.com/evalsi
  promote:
    when: 'scores["llm-judge"] < 0.5'
    dataset: support-regressions                # <datasets_dir>/promoted/support-regressions.jsonl