{
  "metadata": {
    "name": "The LLMOps Stack: OpenTelemetry, Self-Host and Pricing-Model Comparison 2026",
    "description": "A structured comparison of 19 production LLM-operations tools across four stack layers (gateway/routing, observability/tracing, evaluation/testing, guardrails/safety) on five interoperability and cost dimensions: OpenTelemetry-native support, self-host availability, open-source licensing, pricing model, and whether evaluation and guardrails are built in. Compiled vendor-neutral with conflict disclosure; each cell read from the tool's own documentation, repository, or pricing page.",
    "toolsCompared": 19,
    "layers": [
      "Gateway / Routing",
      "Observability / Tracing",
      "Evaluation / Testing",
      "Guardrails / Safety"
    ],
    "dimensions": [
      "otel_native",
      "self_host",
      "open_source",
      "pricing_model",
      "eval_guardrails_built_in"
    ],
    "datePublished": "2026-06-17",
    "dateModified": "2026-06-17",
    "license": "CC-BY-4.0",
    "licenseUrl": "https://creativecommons.org/licenses/by/4.0/",
    "canonicalUrl": "https://nesyona.com/research/llmops-stack-comparison-2026/",
    "methodology": "Each cell read from the vendor's own public documentation, GitHub repository, or pricing page as of June 2026. OpenTelemetry support marked 'yes' (native) only where confirmed from the tool's own docs; 'partial' denotes tier-limited, proxy-based, ingest-only, or unconfirmed-native. Vendor performance claims are not encoded here. Rankings and the four-layer model were fixed before any monetization check; no paid placement. Defunct (HumanLoop) and acquired (Protect AI) products are excluded from the live comparison.",
    "creator": "Vincent Wesley Couey",
    "orcid": "0009-0005-6869-308X",
    "publisher": "Nesyona",
    "definedTerm": "The LLMOps Stack -- a four-layer reference model for production LLM tooling, ordered along the request path, in which the strongest tools span multiple layers and OpenTelemetry is the interoperability standard that connects them without lock-in.",
    "dataProvenance": "SOURCED: every matrix cell read from the vendor's own public documentation, GitHub repository, or pricing page as of June 2026 (OpenTelemetry support marked native only where confirmed in the tool's own docs; 'partial' where qualified). DERIVED: none material — this is a sourced capability/pricing matrix, not a modeled estimate."
  },
  "data": [
    {
      "tool": "Portkey",
      "layer": "Gateway",
      "spans": [
        "Observability",
        "Guardrails"
      ],
      "otel_native": "yes",
      "self_host": "yes",
      "open_source": "Apache-2.0 core",
      "pricing_model": "usage (per log)",
      "eval_guardrails_built_in": "guardrails + observability"
    },
    {
      "tool": "LiteLLM",
      "layer": "Gateway",
      "spans": [],
      "otel_native": "yes",
      "self_host": "yes",
      "open_source": "OSS + Enterprise",
      "pricing_model": "free OSS / Enterprise quote",
      "eval_guardrails_built_in": "routing only"
    },
    {
      "tool": "Cloudflare AI Gateway",
      "layer": "Gateway",
      "spans": [],
      "otel_native": "partial",
      "self_host": "no (cloud only)",
      "open_source": "no",
      "pricing_model": "free (with Workers)",
      "eval_guardrails_built_in": "no"
    },
    {
      "tool": "Kong AI Gateway",
      "layer": "Gateway",
      "spans": [],
      "otel_native": "yes",
      "self_host": "yes",
      "open_source": "core OSS, AI plugins paid",
      "pricing_model": "Enterprise license",
      "eval_guardrails_built_in": "policy-level"
    },
    {
      "tool": "OpenRouter",
      "layer": "Gateway (aggregator)",
      "spans": [],
      "otel_native": "partial",
      "self_host": "no (cloud only)",
      "open_source": "no",
      "pricing_model": "flat 5.5% on credits",
      "eval_guardrails_built_in": "no"
    },
    {
      "tool": "AIMLAPI",
      "layer": "Gateway (aggregator)",
      "spans": [],
      "otel_native": "partial",
      "self_host": "no (cloud only)",
      "open_source": "no",
      "pricing_model": "usage (pay-as-you-go)",
      "eval_guardrails_built_in": "no"
    },
    {
      "tool": "Langfuse",
      "layer": "Observability",
      "spans": [
        "Evaluation"
      ],
      "otel_native": "yes (v3)",
      "self_host": "yes",
      "open_source": "MIT",
      "pricing_model": "usage (per unit)",
      "eval_guardrails_built_in": "eval + prompt management"
    },
    {
      "tool": "LangSmith",
      "layer": "Observability",
      "spans": [
        "Evaluation"
      ],
      "otel_native": "partial (late)",
      "self_host": "partial (enterprise)",
      "open_source": "no",
      "pricing_model": "seat + per-trace",
      "eval_guardrails_built_in": "eval"
    },
    {
      "tool": "Helicone",
      "layer": "Observability",
      "spans": [
        "Gateway"
      ],
      "otel_native": "partial",
      "self_host": "yes",
      "open_source": "OSS",
      "pricing_model": "usage (per request)",
      "eval_guardrails_built_in": "light eval"
    },
    {
      "tool": "Arize Phoenix",
      "layer": "Observability",
      "spans": [
        "Evaluation"
      ],
      "otel_native": "yes (OpenInference)",
      "self_host": "yes",
      "open_source": "OSS",
      "pricing_model": "free OSS / usage",
      "eval_guardrails_built_in": "eval"
    },
    {
      "tool": "Traceloop / OpenLLMetry",
      "layer": "Observability",
      "spans": [],
      "otel_native": "yes (reference)",
      "self_host": "yes (library)",
      "open_source": "Apache-2.0",
      "pricing_model": "free library / platform",
      "eval_guardrails_built_in": "platform only"
    },
    {
      "tool": "Datadog LLM Observability",
      "layer": "Observability",
      "spans": [],
      "otel_native": "yes (GenAI convention)",
      "self_host": "no (cloud only)",
      "open_source": "no",
      "pricing_model": "usage (per span)",
      "eval_guardrails_built_in": "eval"
    },
    {
      "tool": "Maxim",
      "layer": "Evaluation",
      "spans": [
        "Observability",
        "Gateway"
      ],
      "otel_native": "yes (Bifrost)",
      "self_host": "partial (enterprise)",
      "open_source": "Bifrost Apache-2.0",
      "pricing_model": "usage / Enterprise",
      "eval_guardrails_built_in": "observability + guardrails"
    },
    {
      "tool": "Braintrust",
      "layer": "Evaluation",
      "spans": [
        "Observability"
      ],
      "otel_native": "partial (ingests OTLP)",
      "self_host": "partial (enterprise)",
      "open_source": "no",
      "pricing_model": "usage / Enterprise",
      "eval_guardrails_built_in": "observability"
    },
    {
      "tool": "Promptfoo",
      "layer": "Evaluation",
      "spans": [],
      "otel_native": "no (not the focus)",
      "self_host": "yes (local-first)",
      "open_source": "OSS",
      "pricing_model": "free OSS / Enterprise",
      "eval_guardrails_built_in": "red-team"
    },
    {
      "tool": "DeepEval / Confident AI",
      "layer": "Evaluation",
      "spans": [
        "Observability"
      ],
      "otel_native": "yes",
      "self_host": "partial (OSS library only)",
      "open_source": "DeepEval OSS",
      "pricing_model": "usage (cloud)",
      "eval_guardrails_built_in": "observability (cloud)"
    },
    {
      "tool": "Prediction Guard",
      "layer": "Guardrails",
      "spans": [
        "Inference"
      ],
      "otel_native": "yes (events)",
      "self_host": "no (hosted)",
      "open_source": "no",
      "pricing_model": "usage / Enterprise",
      "eval_guardrails_built_in": "eval-style checks"
    },
    {
      "tool": "Guardrails AI",
      "layer": "Guardrails",
      "spans": [],
      "otel_native": "yes (telemetry)",
      "self_host": "yes",
      "open_source": "Apache-2.0",
      "pricing_model": "free OSS / Pro",
      "eval_guardrails_built_in": "validation"
    },
    {
      "tool": "NeMo Guardrails",
      "layer": "Guardrails",
      "spans": [],
      "otel_native": "no",
      "self_host": "yes",
      "open_source": "Apache-2.0",
      "pricing_model": "free OSS",
      "eval_guardrails_built_in": "rails only"
    }
  ]
}