Manual Instrumentation

This guide shows how to instrument an agent built from scratch, without relying on a framework integration. You will add:

a top-level run span
manual step spans for LLM calls
manual tool-call spans for tool execution
run-level metadata for filtering
error handling for failed runs and child spans

Prerequisites

You have a Lemma API key and project ID.
You can run a plain Node.js or Python app.
Your agent loop is code you control directly (no framework wrapper).

Instrument the Agent

Install and configure tracing

TypeScript
Python

npm install @uselemma/tracing @opentelemetry/api openai

// tracer.ts
import { registerOTel } from "@uselemma/tracing";

registerOTel({
  apiKey: process.env.LEMMA_API_KEY,
  projectId: process.env.LEMMA_PROJECT_ID,
});

pip install uselemma-tracing opentelemetry-api openai

import os
# tracer.py
from uselemma_tracing import register_otel

register_otel(
    api_key=os.getenv("LEMMA_API_KEY"),
    project_id=os.getenv("LEMMA_PROJECT_ID"),
)

Wrap your agent as a run

Use wrapAgent / wrap_agent to create the top-level run span (ai.agent.run).

TypeScript
Python

import "./tracer";
import { wrapAgent } from "@uselemma/tracing";

const runAgent = wrapAgent("scratch-agent", async ({ span, onComplete, recordError }, input) => {
  try {
    span.setAttribute("lemma.user_id", input.userId);
    span.setAttribute("lemma.session_id", input.sessionId);
    span.setAttribute("lemma.feature", "support_chat");

    const output = await executeAgentLoop(input.message);
    onComplete(output);
    return output;
  } catch (error) {
    recordError(error);
    throw error;
  }
});

import tracer  # Ensure tracing is registered before agent code runs.
from uselemma_tracing import wrap_agent, TraceContext

async def agent_logic(ctx: TraceContext, input_data: dict):
    try:
        ctx.span.set_attribute("lemma.user_id", input_data["user_id"])
        ctx.span.set_attribute("lemma.session_id", input_data["session_id"])
        ctx.span.set_attribute("lemma.feature", "support_chat")

        output = await execute_agent_loop(input_data["message"])
        ctx.on_complete(output)
        return output
    except Exception as err:
        ctx.record_error(err)
        raise

run_agent = wrap_agent("scratch-agent", agent_logic)

Add a step span for each LLM call

A step is a child span inside the run that captures one LLM request/response.

TypeScript
Python

import { trace } from "@opentelemetry/api";
import OpenAI from "openai";

const tracer = trace.getTracer("scratch-agent");
const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });

async function llmStep(userMessage: string) {
  return tracer.startActiveSpan("llm.step.generate", async (stepSpan) => {
    try {
      const response = await client.chat.completions.create({
        model: "gpt-4o",
        messages: [{ role: "user", content: userMessage }],
      });

      const text = response.choices[0]?.message?.content ?? "";
      stepSpan.setAttribute("llm.model.requested", "gpt-4o");
      stepSpan.setAttribute("llm.tokens.completion", response.usage?.completion_tokens ?? 0);
      stepSpan.setAttribute("llm.response", text);
      return text;
    } catch (error) {
      stepSpan.recordException(error as Error);
      stepSpan.setAttribute("step.status", "error");
      throw error;
    } finally {
      stepSpan.end();
    }
  });
}

from opentelemetry import trace
from openai import AsyncOpenAI

tracer = trace.get_tracer("scratch-agent")
client = AsyncOpenAI()

async def llm_step(user_message: str):
    with tracer.start_as_current_span("llm.step.generate") as step_span:
        try:
            response = await client.chat.completions.create(
                model="gpt-4o",
                messages=[{"role": "user", "content": user_message}],
            )
            text = response.choices[0].message.content or ""
            step_span.set_attribute("llm.model.requested", "gpt-4o")
            step_span.set_attribute("llm.tokens.completion", response.usage.completion_tokens if response.usage else 0)
            step_span.set_attribute("llm.response", text)
            return text
        except Exception as err:
            step_span.record_exception(err)
            step_span.set_attribute("step.status", "error")
            raise

Add tool-call spans around tools

A tool call is another child span nested under the active run.

TypeScript
Python

import { trace } from "@opentelemetry/api";

const tracer = trace.getTracer("scratch-agent");

async function weatherTool(city: string) {
  return tracer.startActiveSpan("tool.call", async (toolSpan) => {
    toolSpan.setAttribute("tool.name", "get_weather");
    toolSpan.setAttribute("tool.args", JSON.stringify({ city }));
    try {
      const result = await getWeather(city);
      toolSpan.setAttribute("tool.result", JSON.stringify(result));
      return result;
    } catch (error) {
      toolSpan.recordException(error as Error);
      toolSpan.setAttribute("tool.status", "error");
      throw error;
    } finally {
      toolSpan.end();
    }
  });
}

from opentelemetry import trace
import json

tracer = trace.get_tracer("scratch-agent")

async def weather_tool(city: str):
    with tracer.start_as_current_span("tool.call") as tool_span:
        tool_span.set_attribute("tool.name", "get_weather")
        tool_span.set_attribute("tool.args", json.dumps({"city": city}))
        try:
            result = await get_weather(city)
            tool_span.set_attribute("tool.result", json.dumps(result))
            return result
        except Exception as err:
            tool_span.record_exception(err)
            tool_span.set_attribute("tool.status", "error")
            raise

Wire everything into one agent loop

Below is a minimal sequence:

run starts with wrapAgent / wrap_agent
step span records LLM decision
tool-call span records tool execution
step span records final LLM response
run ends with onComplete / on_complete

If any span fails, record the error on that span and rethrow so the run reflects the failure.

Run and verify in Lemma

Execute one agent request.
Capture the returned runId / run_id.
In Lemma, verify:
- top-level ai.agent.run exists
- llm.step.* spans are nested under the run
- tool.call spans appear with tool.name, args, and result
- custom metadata (lemma.user_id, lemma.session_id) is filterable

Troubleshooting checklist

No runs visible: ensure registerOTel / register_otel runs before your app logic.
Run appears but no child spans: make sure step/tool spans are created inside the wrapped function.
Run never closes: ensure onComplete / on_complete is reached, or that errors are rethrown.
Missing metadata filters: verify attributes are set on the run span, not on unrelated spans.

Guides

​Prerequisites

​Instrument the Agent

​Troubleshooting checklist

Prerequisites

Instrument the Agent

Troubleshooting checklist