Model Routing Under Pressure: Designing Latency-Aware Fallback Strategies for Production Agentic Pipelines

// TypeScript: Model router with circuit breaker and latency tracking
import { CircuitBreaker, CircuitState } from './circuit-breaker';
import { SlidingWindowMetrics } from './metrics';

interface ModelEndpoint {
  id: string;
  client: (payload: ModelPayload) => Promise<ModelResponse>;
  sloThresholdMs: number;
}

const primary: ModelEndpoint = {
  id: 'gpt-4o',
  client: gpt4oClient,
  sloThresholdMs: 4000,
};

const backup: ModelEndpoint = {
  id: 'claude-haiku',
  client: claudeHaikuClient,
  sloThresholdMs: 8000,
};

const metrics = new SlidingWindowMetrics({ windowSize: 20 });
const breaker = new CircuitBreaker({
  openThresholdPercentile: 95,
  sloMs: primary.sloThresholdMs,
  halfOpenProbeAfterMs: 30_000,
});

export async function routedModelCall(
  payload: ModelPayload,
  emit: (event: RoutingEvent) => void
): Promise<ModelResponse> {
  const endpoint = breaker.state === CircuitState.Open ? backup : primary;
  const start = Date.now();

  try {
    const response = await endpoint.client(payload);
    const latencyMs = Date.now() - start;

    metrics.record(endpoint.id, latencyMs);
    breaker.recordSuccess(latencyMs);

    emit({ type: 'model_routed', endpoint: endpoint.id, latencyMs, circuitState: breaker.state });

    // Check if we should trip the circuit on next call
    if (endpoint.id === primary.id && metrics.p95(primary.id) > primary.sloThresholdMs) {
      breaker.trip();
      emit({ type: 'circuit_tripped', reason: 'p95_slo_breach', p95Ms: metrics.p95(primary.id) });
    }

    return response;
  } catch (err) {
    breaker.recordFailure();
    emit({ type: 'model_error', endpoint: endpoint.id, error: String(err) });
    throw err;
  }
}

// TypeScript: Task capability tier classification and routing tag emission
type CapabilityTier = 'frontier' | 'regional' | 'local';
type RoutingCondition = 'cloud-healthy' | 'cloud-degraded' | 'offline';

interface TaskManifest {
  taskType: string;
  minimumTier: CapabilityTier;
  degradedFallbackAllowed: boolean;
}

const taskRegistry: Record<string, TaskManifest> = {
  'summarize_incident_report': { taskType: 'summarize_incident_report', minimumTier: 'frontier', degradedFallbackAllowed: true },
  'extract_sensor_fields': { taskType: 'extract_sensor_fields', minimumTier: 'regional', degradedFallbackAllowed: true },
  'classify_alert_severity': { taskType: 'classify_alert_severity', minimumTier: 'local', degradedFallbackAllowed: false },
};

export function resolveRoutingPlan(
  taskType: string,
  networkCondition: RoutingCondition,
  primaryLatencyP95Ms: number,
  sloMs: number
): { endpoint: CapabilityTier; routingTag: string; requiresReview: boolean } {
  const manifest = taskRegistry[taskType];
  const primaryHealthy = networkCondition === 'cloud-healthy' && primaryLatencyP95Ms <= sloMs;

  if (primaryHealthy && manifest.minimumTier === 'frontier') {
    return { endpoint: 'frontier', routingTag: 'cloud-tier1-healthy', requiresReview: false };
  }

  if (!primaryHealthy && manifest.degradedFallbackAllowed) {
    const fallback = networkCondition === 'offline' ? 'local' : 'regional';
    return {
      endpoint: fallback,
      routingTag: `${fallback}-degraded`,
      requiresReview: manifest.minimumTier === 'frontier', // frontier tasks routed to lower tier need review
    };
  }

  // Cannot satisfy task requirements under current conditions
  throw new Error(`Cannot route task '${taskType}' under condition '${networkCondition}' — no valid fallback tier`);
}

// TypeScript: OpenAI-compatible client abstraction for local and cloud model tiers
import OpenAI from 'openai';

const modelClients: Record<string, OpenAI> = {
  frontier: new OpenAI({ apiKey: process.env.OPENAI_API_KEY }),
  regional: new OpenAI({
    apiKey: process.env.ANTHROPIC_API_KEY,
    baseURL: 'https://api.anthropic.com/v1',
  }),
  local: new OpenAI({
    apiKey: 'not-required',
    baseURL: 'http://localhost:11434/v1', // Ollama OpenAI-compatible endpoint
  }),
};

const modelNames: Record<string, string> = {
  frontier: 'gpt-4o',
  regional: 'claude-haiku-4-5',
  local: 'qwen3:8b',
};

export async function callModelTier(
  tier: 'frontier' | 'regional' | 'local',
  messages: OpenAI.ChatCompletionMessageParam[],
  schema: object
): Promise<{ content: string; tier: string; latencyMs: number }> {
  const client = modelClients[tier];
  const model = modelNames[tier];
  const start = Date.now();

  const response = await client.chat.completions.create({
    model,
    messages,
    response_format: { type: 'json_schema', json_schema: { name: 'output', schema, strict: true } },
  });

  return {
    content: response.choices[0].message.content ?? '',
    tier,
    latencyMs: Date.now() - start,
  };
}

# Kubernetes sidecar container spec for stateless routing process
# Sidecar reads circuit state from external store — no local state
apiVersion: v1
kind: Pod
metadata:
  name: agent-worker
spec:
  initContainers:
  - name: routing-sidecar
    image: your-org/model-router:1.4.2
    restartPolicy: Always  # Sidecar lifecycle — stays up with main container
    env:
    - name: REDIS_URL
      valueFrom:
        secretKeyRef:
          name: routing-secrets
          key: redis-url
    - name: PRIMARY_SLO_MS
      value: "4000"
    - name: BACKUP_MODEL
      value: "claude-haiku-4-5"
    ports:
    - containerPort: 8080
      name: router-api
    resources:
      requests:
        memory: "128Mi"
        cpu: "100m"
      limits:
        memory: "256Mi"
  containers:
  - name: agent-process
    image: your-org/agent-worker:2.1.0
    env:
    - name: MODEL_ROUTER_URL
      value: "http://localhost:8080"
    resources:
      requests:
        memory: "512Mi"
        cpu: "500m"

// TypeScript: Minimal spec template for delegating a coding task to an AI agent
// Forces the engineer to make implicit requirements explicit before delegation

interface CodingTaskSpec {
  taskId: string;
  objective: string;                   // What it should do — one sentence
  inputs: Record<string, string>;      // Variable name → type + description
  outputs: Record<string, string>;     // Return type + description
  constraints: string[];               // Must-not-do list (security, perf, style)
  verificationCriteria: string[];      // How the engineer will know it's correct
  outOfScope: string[];                // Explicit boundary — prevents scope creep
}

const exampleSpec: CodingTaskSpec = {
  taskId: 'routing-circuit-breaker-v1',
  objective: 'Implement a circuit breaker that tracks P95 latency over a sliding window and exposes open/closed/half-open state',
  inputs: {
    windowSize: 'number — count of recent calls to include in window',
    sloThresholdMs: 'number — P95 latency above this value trips the breaker',
  },
  outputs: {
    state: 'CircuitState enum — current circuit state',
    record: '(latencyMs: number) => void — record a new call latency',
    trip: '() => void — manually open the circuit',
  },
  constraints: [
    'No external dependencies — pure TypeScript only',
    'Thread-safe for single-process use only — no distributed locking',
    'No console.log — emit events via callback instead',
  ],
  verificationCriteria: [
    'State transitions to Open when P95 of last N calls exceeds sloThresholdMs',
    'State transitions to HalfOpen after halfOpenDelayMs without new failures',
    'Record latency of 0 never throws — returns gracefully',
  ],
  outOfScope: [
    'Retry logic — handled at call site',
    'Persistence — caller manages state externalization',
  ],
};