sentry-setup-ai-monitoring

npm install @sentry/nextjs@latest  # or appropriate package

pip show sentry-sdk | grep Version

pip install --upgrade "sentry-sdk[openai]"  # include detected extras

AI Agent Monitoring requires tracing to be enabled. I'll add tracesSampleRate to your Sentry configuration.

I detected the following AI SDK(s) in your project:
- [SDK NAME] (version X.X.X)

Sentry has automatic integration support for this SDK. I'll configure the appropriate integration.

I didn't detect any AI SDKs with automatic Sentry integration support in your project.

Your options:
1. Manual Instrumentation - I can help you set up custom spans for your AI calls
2. Install a supported SDK - If you're planning to add one of the supported SDKs

Would you like to proceed with manual instrumentation? If so, please describe where your AI/LLM calls are located.

import * as Sentry from "@sentry/nextjs"; // or @sentry/node, @sentry/react
 
Sentry.init({
  dsn: "YOUR_DSN_HERE",
  tracesSampleRate: 1.0,
 
  integrations: [
    Sentry.openAIIntegration({
      recordInputs: true,  // Capture prompts/messages
      recordOutputs: true, // Capture responses
    }),
  ],
});

import OpenAI from "openai";
 
const openai = new OpenAI();
const response = await openai.chat.completions.create({
  model: "gpt-4o",
  messages: [{ role: "user", content: "Hello!" }],
});
// Automatically captured by Sentry

import * as Sentry from "@sentry/nextjs";
 
Sentry.init({
  dsn: "YOUR_DSN_HERE",
  tracesSampleRate: 1.0,
 
  integrations: [
    Sentry.anthropicAIIntegration({
      recordInputs: true,
      recordOutputs: true,
    }),
  ],
});

import * as Sentry from "@sentry/nextjs";
 
Sentry.init({
  dsn: "YOUR_DSN_HERE",
  tracesSampleRate: 1.0,
  integrations: [Sentry.vercelAIIntegration()],
});

import { generateText } from "ai";
import { openai } from "@ai-sdk/openai";
 
const result = await generateText({
  model: openai("gpt-4o"),
  prompt: "Hello!",
  experimental_telemetry: {
    isEnabled: true,
    recordInputs: true,
    recordOutputs: true,
    functionId: "my-generation", // Optional but recommended
  },
});

Sentry.vercelAIIntegration({
  recordInputs: true,
  recordOutputs: true,
  force: false, // Force enable even if AI module not detected
})

import * as Sentry from "@sentry/nextjs";
 
Sentry.init({
  dsn: "YOUR_DSN_HERE",
  tracesSampleRate: 1.0,
 
  integrations: [
    Sentry.langChainIntegration({
      recordInputs: true,
      recordOutputs: true,
    }),
  ],
});

import * as Sentry from "@sentry/nextjs";
 
Sentry.init({
  dsn: "YOUR_DSN_HERE",
  tracesSampleRate: 1.0,
 
  integrations: [
    Sentry.langGraphIntegration({
      recordInputs: true,
      recordOutputs: true,
    }),
  ],
});

import * as Sentry from "@sentry/nextjs";
 
Sentry.init({
  dsn: "YOUR_DSN_HERE",
  tracesSampleRate: 1.0,
 
  integrations: [
    Sentry.googleGenAIIntegration({
      recordInputs: true,
      recordOutputs: true,
    }),
  ],
});

pip install "sentry-sdk[openai]"

import sentry_sdk
from sentry_sdk.integrations.openai import OpenAIIntegration
 
sentry_sdk.init(
    dsn="YOUR_DSN_HERE",
    traces_sample_rate=1.0,
    send_default_pii=True,  # Required for input/output capture
    integrations=[
        OpenAIIntegration(
            include_prompts=True,  # Capture inputs/outputs
            tiktoken_encoding_name="cl100k_base",  # For streaming token count
        ),
    ],
)

from openai import OpenAI
 
client = OpenAI()
 
with sentry_sdk.start_transaction(name="AI inference"):
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": "Hello!"}]
    )

pip install "sentry-sdk[anthropic]"

import sentry_sdk
from sentry_sdk.integrations.anthropic import AnthropicIntegration
 
sentry_sdk.init(
    dsn="YOUR_DSN_HERE",
    traces_sample_rate=1.0,
    send_default_pii=True,
    integrations=[
        AnthropicIntegration(
            include_prompts=True,
        ),
    ],
)

from anthropic import Anthropic
 
client = Anthropic()
 
with sentry_sdk.start_transaction(name="Claude inference"):
    response = client.messages.create(
        model="claude-3-5-sonnet-20240620",
        max_tokens=1024,
        messages=[{"role": "user", "content": "Hello!"}]
    )

pip install "sentry-sdk[langchain]"

import sentry_sdk
from sentry_sdk.integrations.langchain import LangchainIntegration
 
sentry_sdk.init(
    dsn="YOUR_DSN_HERE",
    traces_sample_rate=1.0,
    send_default_pii=True,  # Required for LLM input/output capture
    integrations=[
        LangchainIntegration(
            include_prompts=True,
        ),
    ],
)

with sentry_sdk.start_transaction(name="LangChain agent"):
    agent_executor = AgentExecutor(agent=agent, tools=tools)
    result = agent_executor.invoke({"input": "your prompt"})

pip install "sentry-sdk[huggingface_hub]"

import sentry_sdk
from sentry_sdk.integrations.huggingface_hub import HuggingfaceHubIntegration
 
sentry_sdk.init(
    dsn="YOUR_DSN_HERE",
    traces_sample_rate=1.0,
    send_default_pii=True,
    integrations=[
        HuggingfaceHubIntegration(
            include_prompts=True,
        ),
    ],
)

I'll help you set up manual AI instrumentation. To do this effectively, I need to know:

1. Where are your AI/LLM calls located? (file paths)
2. What AI provider/model are you using?
3. What operations do you want to track?
   - LLM requests (prompts, completions)
   - Agent executions
   - Tool calls
   - Agent handoffs

import * as Sentry from "@sentry/nextjs";
 
async function callLLM(prompt, model = "custom-model") {
  return await Sentry.startSpan(
    {
      op: "gen_ai.request",
      name: `LLM request ${model}`,
      attributes: {
        "gen_ai.request.model": model,
        "gen_ai.request.temperature": 0.7,
        "gen_ai.request.max_tokens": 1000,
      },
    },
    async (span) => {
      // Record input messages
      span.setAttribute(
        "gen_ai.request.messages",
        JSON.stringify([{ role: "user", content: prompt }])
      );
 
      const startTime = performance.now();
 
      // Your actual LLM call here
      const result = await yourLLMClient.complete(prompt);
 
      // Record output and metrics
      span.setAttribute("gen_ai.response.text", result.text);
      span.setAttribute("gen_ai.usage.input_tokens", result.inputTokens || 0);
      span.setAttribute("gen_ai.usage.output_tokens", result.outputTokens || 0);
 
      return result;
    }
  );
}

async function runAgent(task) {
  return await Sentry.startSpan(
    {
      op: "gen_ai.invoke_agent",
      name: "Execute AI Agent",
      attributes: {
        "gen_ai.agent.name": "my-agent",
        "gen_ai.agent.available_tools": JSON.stringify(["search", "calculate"]),
      },
    },
    async (span) => {
      span.setAttribute("gen_ai.agent.input", task);
 
      // Agent execution logic (may include multiple LLM calls and tool uses)
      const result = await agent.execute(task);
 
      span.setAttribute("gen_ai.agent.output", JSON.stringify(result));
      span.setAttribute("gen_ai.usage.total_tokens", result.totalTokens || 0);
 
      return result;
    }
  );
}

async function executeTool(toolName, toolInput) {
  return await Sentry.startSpan(
    {
      op: "gen_ai.execute_tool",
      name: `Tool: ${toolName}`,
      attributes: {
        "gen_ai.tool.name": toolName,
        "gen_ai.tool.description": getToolDescription(toolName),
      },
    },
    async (span) => {
      span.setAttribute("gen_ai.tool.input", JSON.stringify(toolInput));
 
      const result = await tools[toolName](toolInput);
 
      span.setAttribute("gen_ai.tool.output", JSON.stringify(result));
 
      return result;
    }
  );
}

async function handoffToAgent(fromAgent, toAgent, context) {
  return await Sentry.startSpan(
    {
      op: "gen_ai.handoff",
      name: `Handoff: ${fromAgent} -> ${toAgent}`,
      attributes: {
        "gen_ai.handoff.from_agent": fromAgent,
        "gen_ai.handoff.to_agent": toAgent,
      },
    },
    async (span) => {
      span.setAttribute("gen_ai.handoff.context", JSON.stringify(context));
 
      // Perform handoff
      const result = await agents[toAgent].receive(context);
 
      return result;
    }
  );
}

import sentry_sdk
import json
 
def call_llm(prompt, model="custom-model"):
    with sentry_sdk.start_span(
        op="gen_ai.request",
        name=f"LLM request {model}",
    ) as span:
        span.set_data("gen_ai.request.model", model)
        span.set_data("gen_ai.request.messages", json.dumps([
            {"role": "user", "content": prompt}
        ]))
 
        # Your actual LLM call
        result = your_llm_client.complete(prompt)
 
        span.set_data("gen_ai.response.text", result.text)
        span.set_data("gen_ai.usage.input_tokens", result.input_tokens)
        span.set_data("gen_ai.usage.output_tokens", result.output_tokens)
 
        return result

Sentry.init({
  sendDefaultPii: true,
  // OR configure per-integration:
  integrations: [
    Sentry.openAIIntegration({
      recordInputs: true,
      recordOutputs: true,
    }),
  ],
});

sentry_sdk.init(
    send_default_pii=True,
    # OR configure per-integration:
    integrations=[
        OpenAIIntegration(include_prompts=True),
    ],
)

// Trigger a test LLM call
const response = await openai.chat.completions.create({
  model: "gpt-4o-mini",
  messages: [{ role: "user", content: "Say 'Sentry test successful'" }],
});
console.log("Test complete:", response.choices[0].message.content);

with sentry_sdk.start_transaction(name="AI Test"):
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": "Say 'Sentry test'"}]
    )
    print("Test complete:", response.choices[0].message.content)

## Sentry AI Agent Monitoring Setup Complete
 
### Detection Results:
- [ ] AI SDK(s) detected: [LIST DETECTED SDKS]
- [ ] Sentry SDK version verified
- [ ] Tracing enabled
 
### Configuration Applied:
- [ ] Integration(s) added to Sentry.init()
- [ ] Input/output recording configured
- [ ] PII settings reviewed
 
### For Vercel AI SDK (if applicable):
- [ ] Edge runtime integration added
- [ ] experimental_telemetry enabled in function calls
 
### Next Steps:
1. Make an AI/LLM call to verify
2. Check Sentry > Traces > AI Spans
3. Review AI Agents Dashboard for metrics