wandb-primary

import os, wandb
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
api = wandb.Api(timeout=120)
 
sweeps = list(api.project(project, entity=entity).sweeps(per_page=50))
rows = []
for sweep in sweeps:
    config = sweep.config or {}
    metric = config.get("metric") or {}
    rows.append({
        "id": sweep.id,
        "state": sweep.state,
        "method": config.get("method"),
        "metric": metric.get("name"),
        "goal": metric.get("goal"),
        "run_count": len(sweep.runs),
    })
 
print(f"sweep_count={len(rows)}")
print(f"total_sweep_runs={sum(r['run_count'] for r in rows)}")
for r in rows:
    print(r)

import os, wandb
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
path = f"{entity}/{project}"
api = wandb.Api(timeout=120)
 
runs = api.runs(
    path,
    filters={"state": "finished"},
    order="+created_at",
    per_page=100,
    include_sweeps=False,
)
rows = []
for run in runs:
    user = getattr(run, "user", None)
    rows.append({
        "created_at": run.created_at,
        "name": run.display_name or run.name,
        "id": run.id,
        "username": getattr(user, "username", None),
        "user_name": getattr(user, "name", None),
    })
 
print(f"finished_count={len(rows)}")
for r in rows:
    print(r)

import weave, os, logging
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
from weave.trace_server.interface.query import Query
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
client = weave.init(f"{entity}/{project}")
pid = f"{entity}/{project}"
 
# Total calls/traces
stats = client.server.calls_query_stats(CallsQueryStatsReq(project_id=pid))
print(f"Total calls: {stats.count}")
 
# Root traces only
root_stats = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, filter={"trace_roots_only": True}
))
print(f"Root traces: {root_stats.count}")
 
# Count by op name
for op in ["Evaluation.evaluate", "my_op.turn"]:
    op_ref = f"weave:///{entity}/{project}/op/{op}:*"
    s = client.server.calls_query_stats(CallsQueryStatsReq(
        project_id=pid,
        filter={"op_names": [op_ref]},
    ))
    print(f"  {op}: {s.count}")
 
# Count calls whose op_name contains a substring, e.g. scorer calls.
score_query = Query(**{"$expr": {"$contains": {
    "input": {"$getField": "op_name"},
    "substr": {"$literal": ".score"},
    "case_insensitive": True,
}}})
score_stats = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, query=score_query
))
print(f"Scorer calls (.score): {score_stats.count}")
 
# Count a named op substring such as create_embeddings.
embedding_query = Query(**{"$expr": {"$contains": {
    "input": {"$getField": "op_name"},
    "substr": {"$literal": "create_embeddings"},
    "case_insensitive": True,
}}})
embedding_stats = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, query=embedding_query
))
print(f"create_embeddings calls: {embedding_stats.count}")
 
# Error/exception calls. Include descendant_error when the prompt says
# "error status or exception"; those are traces whose children failed.
error_query = Query(**{"$expr": {"$or": [
    {"$eq": [{"$getField": "summary.weave.status"}, {"$literal": "error"}]},
    {"$eq": [
        {"$getField": "summary.weave.status"},
        {"$literal": "descendant_error"},
    ]},
    {"$not": [{"$eq": [{"$getField": "exception"}, {"$literal": None}]}]},
]}})
error_stats = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, query=error_query
))
root_error_stats = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, filter={"trace_roots_only": True}, query=error_query
))
print(f"Error/exception calls: {error_stats.count}")
print(f"Root error/exception calls: {root_error_stats.count}")
print(f"Non-error calls: {stats.count - error_stats.count}")

import os, statistics, weave, logging, sys
from collections import Counter
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
from weave.trace_server.interface.query import Query
sys.path.insert(0, "skills/wandb-primary/scripts")
from weave_helpers import unwrap
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
 
query = Query(**{"$expr": {"$contains": {
    "input": {"$getField": "op_name"},
    "substr": {"$literal": "create_embeddings"},
    "case_insensitive": True,
}}})
total = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, query=query
)).count
 
sizes = []
for call in client.get_calls(query=query, limit=total, columns=["inputs"]):
    inputs = unwrap(call.inputs)
    texts = inputs.get("texts") or inputs.get("input") or []
    if isinstance(texts, str):
        sizes.append(1)
    else:
        sizes.append(len(texts))
 
dist = Counter(sizes)
print(f"create_embeddings calls: {total}")
print(f"typical texts per call: {dist.most_common(1)[0][0] if dist else 0}")
print(f"distribution: {dict(sorted(dist.items()))}")
print(f"mean texts per call: {statistics.mean(sizes) if sizes else 0:.4f}")

import os, weave, logging
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import FeedbackQueryReq
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
 
limit = 1000
offset = 0
total = 0
while True:
    res = client.server.feedback_query(FeedbackQueryReq(
        project_id=pid,
        fields=["id"],
        limit=limit,
        offset=offset,
    ))
    rows = (
        getattr(res, "result", None)
        or getattr(res, "feedback", None)
        or getattr(res, "rows", None)
        or []
    )
    n = len(rows)
    total += n
    if n < limit:
        break
    offset += limit
 
print(f"Feedback records: {total}")

import os, weave, logging
from collections import Counter
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
root_filter = {"trace_roots_only": True}
 
root_count = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, filter=root_filter
)).count
 
def short_op(op_name: str) -> str:
    tail = op_name.split("/op/")[-1]
    return tail.rsplit(":", 1)[0]
 
counts = Counter()
for call in client.get_calls(
    filter=root_filter,
    limit=root_count,
    columns=["op_name"],
):
    counts[short_op(call.op_name)] += 1
 
for name, count in counts.most_common():
    print(f"{name}\t{count}")

import os, weave, logging
from collections import Counter
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
total = client.server.calls_query_stats(CallsQueryStatsReq(project_id=pid)).count
 
def short_op(op_name: str) -> str:
    return op_name.split("/op/")[-1].rsplit(":", 1)[0]
 
counts = Counter()
for call in client.get_calls(limit=total, columns=["op_name"]):
    counts[short_op(call.op_name)] += 1
 
print(f"Unique ops: {len(counts)}")
for name, count in counts.most_common():
    print(f"{name}\t{count}")

import os, weave, logging
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
total = client.server.calls_query_stats(CallsQueryStatsReq(project_id=pid)).count
 
threshold_s = 60
long_count = 0
scanned = 0
for call in client.get_calls(
    limit=total,
    columns=["started_at", "ended_at"],
):
    scanned += 1
    if call.started_at and call.ended_at:
        duration_s = (call.ended_at - call.started_at).total_seconds()
        if duration_s > threshold_s:
            long_count += 1
 
print(f"Scanned calls: {scanned}")
print(f"Duration > {threshold_s}s: {long_count}")

import os, weave, logging
from collections import Counter
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from weave_helpers import unwrap
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
total = client.server.calls_query_stats(CallsQueryStatsReq(project_id=pid)).count
 
def collect_models(obj, out):
    obj = unwrap(obj)
    if isinstance(obj, dict):
        for k, v in obj.items():
            if k == "model" and isinstance(v, str):
                out.append(v)
            collect_models(v, out)
    elif isinstance(obj, list):
        for item in obj:
            collect_models(item, out)
 
models = Counter()
for call in client.get_calls(
    limit=total,
    columns=["inputs", "output", "summary"],
):
    found = []
    collect_models(call.inputs, found)
    collect_models(call.output, found)
    usage = unwrap(call.summary).get("usage", {}) if call.summary else {}
    for model_name in usage:
        if isinstance(model_name, str):
            found.append(model_name)
    for model_name in set(found):
        models[model_name] += 1
 
for name, count in models.most_common():
    print(f"{name}\t{count}")

import os, weave, logging
from collections import Counter
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.interface.query import Query
import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from weave_helpers import unwrap
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
client = weave.init(f"{entity}/{project}")
 
embedding_query = Query(**{"$expr": {"$contains": {
    "input": {"$getField": "op_name"},
    "substr": {"$literal": "create_embeddings"},
    "case_insensitive": True,
}}})
 
dims = Counter()
models = Counter()
no_output = 0
for call in client.get_calls(
    query=embedding_query,
    limit=100000,
    columns=["inputs", "output"],
):
    inputs = unwrap(call.inputs) or {}
    model = inputs.get("model") if isinstance(inputs, dict) else None
    models[model or "<missing>"] += 1
 
    output = unwrap(call.output)
    found = False
    if isinstance(output, list):
        for item in output:
            if isinstance(item, list) and item and isinstance(item[0], (int, float)):
                dims[len(item)] += 1
                found = True
    if not found:
        no_output += 1
 
print("embedding_models")
for name, count in models.most_common():
    print(f"{name}\t{count}")
print("embedding_dimensions")
for dim, count in dims.most_common():
    print(f"{dim}\t{count}")
print(f"no_embedding_output\t{no_output}")

import os, weave, logging
from collections import Counter
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
pid = f"{entity}/{project}"
client = weave.init(pid)
total = client.server.calls_query_stats(CallsQueryStatsReq(project_id=pid)).count
 
def short_op(op_name: str) -> str:
    return op_name.split("/op/")[-1].rsplit(":", 1)[0]
 
scorers = Counter()
for call in client.get_calls(limit=total, columns=["op_name"]):
    name = short_op(call.op_name)
    if name.endswith("_scorer") or name.endswith(".score"):
        scorers[name] += 1
 
for name, count in scorers.most_common():
    print(f"{name}\t{count}")

import wandb, weave, os, logging
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace_server.trace_server_interface import CallsQueryStatsReq
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
path = f"{entity}/{project}"
 
# --- Runs ---
api = wandb.Api(timeout=120)
total_runs = len(api.runs(path, per_page=1, include_sweeps=False, lazy=True))
finished = len(api.runs(path, filters={"state": "finished"}, per_page=1, include_sweeps=False, lazy=True))
recent = api.runs(path, order="-created_at", per_page=5)[:5]
 
print(f"=== Runs ({total_runs} total, {finished} finished) ===")
for r in recent:
    print(f"  {r.name} [{r.state}] {r.created_at[:10]}")
 
# --- Weave Traces ---
client = weave.init(path)
pid = f"{entity}/{project}"
root_stats = client.server.calls_query_stats(CallsQueryStatsReq(
    project_id=pid, filter={"trace_roots_only": True}
))
print(f"\n=== Weave Traces ({root_stats.count} root traces) ===")
 
recent_calls = list(client.get_calls(
    sort_by=[{"field": "started_at", "direction": "desc"}],
    limit=5,
    columns=["op_name", "started_at", "display_name"],
))
for c in recent_calls:
    name = c.display_name or c.op_name.split("/")[-1].split(":")[0]
    started = c.started_at.strftime("%Y-%m-%d %H:%M") if c.started_at else "?"
    print(f"  {name} @ {started}")

import wandb, os
api = wandb.Api(timeout=120)
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
 
run = api.run(f"{path}/RUN_ID")
print(f"Name: {run.name}")
print(f"State: {run.state}")
print(f"Created: {run.created_at}")
print(f"Tags: {run.tags}")
print(f"Last step: {run.lastHistoryStep}")
 
# Key metrics (replace with actual keys from probe or user request)
for k in ["loss", "val_loss", "accuracy"]:
    v = run.summary_metrics.get(k)
    if v is not None:
        print(f"  {k}: {v}")

import wandb, os
api = wandb.Api(timeout=120)
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
 
for atype in api.artifact_types(project=path):
    collections = list(api.artifact_collections(path, atype.name, per_page=1000))
    print(f"{atype.name}: {len(collections)} collections")
    for col in collections[:10]:
        try:
            n_versions = len(col.artifacts(per_page=50))
        except Exception:
            n_versions = "?"
        print(f"  {col.name}  ({n_versions} versions)")

import wandb, os
api = wandb.Api(timeout=120)
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
 
art = api.artifact(f"{path}/ARTIFACT_NAME:latest")  # or :v3
print(f"name={art.name} type={art.type} size_bytes={art.size} aliases={art.aliases}")
md = art.metadata or {}
print(f"metadata_keys={list(md)[:20]}")
 
entries = sorted(art.manifest.entries.values(), key=lambda e: e.path)
print(f"file_count={len(entries)}")
for e in entries[:25]:
    print(f"  {e.path}  ({e.size} bytes)")
 
# Read ONE small structured file without downloading the whole artifact:
# p = art.get_entry("metrics.jsonl").download()  # local path to just that file
# import pandas as pd; df = pd.read_json(p, lines=True); print(df.describe())

import wandb, os, pandas as pd
api = wandb.Api(timeout=120)
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
 
runs = api.runs(path, filters={"state": "finished"}, per_page=100)
rows = []
for run in runs:
    sys_df = run.history(stream="system", samples=500)
    if sys_df.empty or "system.gpu.0.gpu" not in sys_df.columns:
        rows.append({"run": run.name, "gpu_mean": None, "gpu_min": None, "gpu_max": None})
        continue
    gpu = sys_df["system.gpu.0.gpu"].dropna()
    rows.append({
        "run": run.name,
        "gpu_mean": round(gpu.mean(), 1),
        "gpu_min": round(gpu.min(), 1),
        "gpu_max": round(gpu.max(), 1),
        "low_util_pct": round(100 * (gpu < 30).sum() / len(gpu), 1) if len(gpu) else None,
    })
 
df = pd.DataFrame(rows)
print(df.to_string(index=False))

import wandb, os, numpy as np, pandas as pd
api = wandb.Api(timeout=120)
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
metric = "train/loss"  # discover the real key first (probe_project / inspect a run)
 
runs = api.runs(path, filters={"state": "finished"}, per_page=100)[:40]
rows = []
for run in runs:
    df = run.history(samples=300, keys=[metric])  # never omit keys on large runs
    series = df[metric].dropna() if metric in getattr(df, "columns", []) else pd.Series(dtype=float)
    arr = series.to_numpy(dtype=float)
    finite = arr[np.isfinite(arr)]
    diffs = np.abs(np.diff(finite)) if finite.size > 2 else np.array([])
    spike_threshold = 5 * (np.median(diffs) or 1.0)
    rows.append({
        "run": run.display_name or run.name,
        "id": run.id,
        "points": int(arr.size),
        "nan_or_inf": int((~np.isfinite(arr)).sum()),
        "min": round(float(finite.min()), 5) if finite.size else None,
        "final": round(float(finite[-1]), 5) if finite.size else None,
        "spikes": int((diffs > spike_threshold).sum()),
    })
 
out = pd.DataFrame(rows).sort_values("min", na_position="last")
print(out.to_string(index=False))

import wandb, os, sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from wandb_helpers import get_api, compare_configs
 
api = get_api()
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
 
run_a = api.run(f"{path}/RUN_A_ID")
run_b = api.run(f"{path}/RUN_B_ID")
 
# Config diff
diffs = compare_configs(run_a, run_b)
if diffs:
    print("Config differences:")
    for d in diffs:
        print(f"  {d['key']}: {d[run_a.name]} -> {d[run_b.name]}")
else:
    print("Configs are identical")
 
# Metric comparison
print("\nMetrics:")
for k in ["loss", "val_loss", "accuracy"]:
    a = run_a.summary_metrics.get(k, "N/A")
    b = run_b.summary_metrics.get(k, "N/A")
    print(f"  {k}: {a} vs {b}")

import wandb, os, numpy as np, pandas as pd
from collections import defaultdict
api = wandb.Api(timeout=120)
path = f"{os.environ['WANDB_ENTITY']}/{os.environ['WANDB_PROJECT']}"
metric = "accuracy"   # discover the real key first
axis = "optimizer"    # a config key; or use run.group / run.job_type
 
runs = api.runs(path, filters={"state": "finished"}, per_page=200)[:200]
buckets = defaultdict(list)
for run in runs:
    key = run.config.get(axis, "<missing>")  # or: run.group / run.job_type
    value = run.summary_metrics.get(metric)
    if value is not None:
        buckets[str(key)].append(float(value))
 
rows = [
    {axis: key, "n": len(vals), "mean": round(np.mean(vals), 4),
     "min": round(np.min(vals), 4), "max": round(np.max(vals), 4)}
    for key, vals in buckets.items()
]
out = pd.DataFrame(rows).sort_values("mean", ascending=False)
print(out.to_string(index=False))

import weave, os, sys, logging
logging.getLogger("weave").setLevel(logging.ERROR)
from weave.trace.weave_client import CallsFilter
sys.path.insert(0, "skills/wandb-primary/scripts")
from weave_helpers import unwrap, eval_results_to_dicts, results_summary
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
client = weave.init(f"{entity}/{project}")
 
# Get latest eval
op_ref = f"weave:///{entity}/{project}/op/Evaluation.evaluate:*"
evals = list(client.get_calls(
    filter=CallsFilter(op_names=[op_ref]),
    sort_by=[{"field": "started_at", "direction": "desc"}],
    limit=1,
))
 
if not evals:
    print("No evaluations found")
else:
    ec = evals[0]
    print(f"Eval: {ec.display_name or 'unnamed'} @ {ec.started_at}")
 
    # Get predict_and_score children
    pas_ref = f"weave:///{entity}/{project}/op/Evaluation.predict_and_score:*"
    pas = list(client.get_calls(
        filter=CallsFilter(op_names=[pas_ref], parent_ids=[ec.id])
    ))
    results = eval_results_to_dicts(pas, agent_name=ec.display_name or "agent")
    print(results_summary(results))

import weave, os, logging
logging.getLogger("weave").setLevel(logging.ERROR)
sys.path.insert(0, "skills/wandb-primary/scripts")
from weave_helpers import unwrap, get_token_usage
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
client = weave.init(f"{entity}/{project}")
 
calls = list(client.get_calls(
    sort_by=[{"field": "started_at", "direction": "desc"}],
    limit=10,
))
 
for c in calls:
    name = c.display_name or c.op_name.split("/")[-1].split(":")[0]
    started = c.started_at.strftime("%Y-%m-%d %H:%M") if c.started_at else "?"
    duration = ""
    if c.started_at and c.ended_at:
        duration = f" ({(c.ended_at - c.started_at).total_seconds():.1f}s)"
    status = c.summary.get("weave", {}).get("status", "?") if c.summary else "?"
    tokens = get_token_usage(c)
    tok_str = f" [{tokens['total_tokens']} tok]" if tokens['total_tokens'] else ""
    print(f"  {name} [{status}] {started}{duration}{tok_str}")

import os
 
import wandb_workspaces.reports.v2 as wr
 
entity = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
 
runset = wr.Runset(entity=entity, project=project, name="All runs")
plots = wr.PanelGrid(
    runsets=[runset],
    panels=[
        wr.LinePlot(title="Loss", x="_step", y=["LOSS_KEY"]),
        wr.BarPlot(title="Accuracy", metrics=["ACC_KEY"], orientation="v"),
    ],
)
 
report = wr.Report(
    entity=entity,
    project=project,
    title="Project Analysis",
    description="Auto-generated summary",
    width="fixed",
    blocks=[
        wr.H1("Project Analysis"),
        wr.P("Auto-generated summary from W&B API."),
        plots,
    ],
)
report.save(draft=True)
print(f"Report saved: {report.url}")

import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from report_helpers import save_report_verified
 
result = save_report_verified(report)  # draft=True by default
print(result["answer"])                # answer=... verified=True/False url=...

wandb launch-agent --queue QUEUE --entity ENTITY  # requires WANDB_API_KEY in the environment

python skills/wandb-primary/scripts/launch_helpers.py create-queue ENTITY \
  --queue wandb-launch-k8s \
  --namespace wandb-launch \
  --gpus 1 --cpu 8 --memory 80Gi

helm upgrade --install wandb-launch launch-agent \
  --repo https://charts.wandb.ai \
  --namespace wandb-launch \
  --create-namespace \
  --set agent.apiKey="$WANDB_SERVICE_ACCOUNT_API_KEY" \
  --set namespace=wandb-launch \
  --set "additionalTargetNamespaces={wandb-launch}" \
  --set-file launchConfig=<(cat <<YAML
entity: ${WANDB_ENTITY}
queues:
  - wandb-launch-k8s
max_jobs: 10
builder:
  type: noop
verbosity: 1
YAML
)

python skills/wandb-primary/scripts/launch_helpers.py list-queues ENTITY

python skills/wandb-primary/scripts/launch_helpers.py relaunch RUN_URL \
  --queue QUEUE \
  --config '{"epochs": 100}'

import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from launch_helpers import submit_code_artifact_job
 
status = submit_code_artifact_job(
    code_files=["train.py"],
    entrypoint="python train.py",
    entity="ENTITY",
    project="PROJECT",
    queue_name="QUEUE",
    job_name="JOB_NAME",
)
print(status["run_url"], status["queue_item_id"])

import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from launch_helpers import download_code_artifact, create_and_launch_modified_job
 
info = download_code_artifact("ENTITY/PROJECT/JOB_NAME:latest")
# edit files in info["code_dir"]; if using apply_patch, use headerless V4A
# diffs without standard unified-diff file headers like --- or +++.
status = create_and_launch_modified_job(
    code_dir=info["code_dir"],
    entrypoint=info["entrypoint"],
    entity=info["entity"],
    project=info["project"],
    queue_name="QUEUE",
    job_name="JOB_NAME",
    base_image=info["base_image"],
)
print(status["run_url"], status["queue_item_id"])

python skills/wandb-primary/scripts/launch_helpers.py check ENTITY PROJECT QUEUE QUEUE_ITEM_ID

https://wandb.ai/ENTITY/launch/QUEUE_ID
https://wandb.ai/ENTITY/launch/QUEUE_ID/agents
https://wandb.ai/ENTITY/launch/QUEUE_ID/config
https://wandb.ai/ENTITY/launch/agents/AGENT_ID
https://wandb.ai/ENTITY/launch/agents/AGENT_ID/logs
https://wandb.ai/RUN_ENTITY/RUN_PROJECT/runs/RUN_ID
https://wandb.ai/RUN_ENTITY/RUN_PROJECT/runs/RUN_ID/logs
https://wandb.ai/JOB_ENTITY/JOB_PROJECT/jobs/JOB_COLLECTION_ID/version_details/ALIAS

kubectl logs -n wandb deploy/launch-agent-wandb-launch --since=1h | rg 'QUEUE_ITEM_ID|RUN_ID|JOB_NAME'
kubectl get jobs,pods -n NAMESPACE --sort-by=.metadata.creationTimestamp
kubectl describe pod -n NAMESPACE POD_NAME
kubectl logs -n NAMESPACE POD_NAME --tail=200

import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
from launch_helpers import inspect_job_artifact, download_code_artifact
 
inspect_job_artifact("ENTITY/PROJECT/JOB_NAME:latest")
info = download_code_artifact("ENTITY/PROJECT/JOB_NAME:latest")
print(info["code_dir"], info["files"], info["entrypoint"], info["base_image"])

import sys
sys.path.insert(0, "skills/wandb-primary/scripts")
 
# Weave helpers (traces, evals, GenAI)
from weave_helpers import (
    unwrap,                  # Recursively convert Weave types -> plain Python
    get_token_usage,         # Extract token counts from a call's summary
    eval_results_to_dicts,   # predict_and_score calls -> list of result dicts
    pivot_solve_rate,        # Build task-level pivot table across agents
    results_summary,         # Print compact eval summary
    eval_health,             # Extract status/counts from Evaluation.evaluate calls
    eval_efficiency,         # Compute tokens-per-success across eval calls
    safe_eval_root_summary,  # Compact aggregate evidence from one Evaluation.evaluate call
    safe_eval_child_summary, # Count predict_and_score rows first; sample full payloads (capped)
    safe_project_eval_summary,  # Project eval landscape without predict_and_score payload scans
)
 
# W&B helpers (training runs, metrics) — large-project optimized
from wandb_helpers import (
    get_api,             # Create API with safe timeout (default 120s)
    probe_project,       # Discover project scale, metrics, config, artifacts BEFORE querying
    fetch_runs,          # FAST: Direct GraphQL with selective metrics (17x faster)
    runs_to_dataframe,   # Legacy: iterate run objects (slower, use fetch_runs instead)
    diagnose_run,        # Quick diagnostic summary (configurable metric keys)
    compare_configs,     # Side-by-side config diff between two runs
    scan_history,        # Exact history scan with explicit metric keys
)
 
# Launch helpers (job submission, run reproduction, queue management)
from launch_helpers import (
    parse_run_url,                       # Extract (entity, project, run_id) from a W&B URL
    list_queues,                         # List launch queues with active agents and resource defaults
    get_job_artifact,                    # Check if a run has a job artifact
    inspect_job_artifact,                # Download + inspect a job artifact's metadata
    download_code_artifact,              # Download source code from a job artifact
    create_and_launch_modified_job,      # Upload modified code + launch in one call
    relaunch_run,                        # Re-run with config overrides (no code change)
    launch_job_artifact,                 # Launch directly from an artifact path
    submit_code_artifact_job,            # Create job artifact and enqueue in one call
    check_launch,                        # Check queue item, run URL, run state, issues
    create_queue,                        # Create a K8s launch queue
    inspect_queue,                       # Print queue details
)
 
# Report helpers (save/edit W&B Reports with read-back verification)
from report_helpers import (
    save_report_verified,   # save a report (draft by default) then re-read to confirm it landed
    edit_report_verified,   # load via from_url, mutate in place, save + verify
)

python skills/wandb-primary/scripts/wandb_run_ops.py --help
python skills/wandb-primary/scripts/artifact_helpers.py --help
python skills/wandb-primary/scripts/workspace_ops.py --help
python skills/wandb-primary/scripts/collection_overview_helpers.py --help
python skills/wandb-primary/scripts/weave_agent_ops.py --help

# WRONG — hardcoded metric name
rows = fetch_runs(api, path, metric_keys=["loss", "accuracy"])
 
# RIGHT — discovered via probe_project or user's request
info = probe_project(api, path)
print("Metrics:", info["sample_metric_keys"])
print("Config keys:", info["sample_config_keys"])
print("Artifacts:", info["artifact_names"])  # {name: type} — know what's been logged
print("Weave traces:", info["weave_trace_count"], "| Top ops:", info.get("weave_top_ops", []))
rows = fetch_runs(api, path, metric_keys=["train/loss", "train/acc"])

from weave_helpers import unwrap
import json
 
output = unwrap(call.output)
print(json.dumps(output, indent=2, default=str))

import os
entity  = os.environ["WANDB_ENTITY"]
project = os.environ["WANDB_PROJECT"]
path = f"{entity}/{project}"

import wandb
api = wandb.Api(timeout=120)
path = f"{entity}/{project}"
 
total = len(api.runs(path, per_page=1, include_sweeps=False, lazy=True))
finished = len(api.runs(path, filters={"state": "finished"}, per_page=1, include_sweeps=False, lazy=True))

import wandb
from wandb_graphql.language import parser as gql_parser
 
api = wandb.Api(timeout=120)
doc = gql_parser.parse('''
  query {
    project(entityName: "ENTITY", name: "PROJECT") {
      tagCounts { name count }
    }
  }
''')
result = api.client.execute(doc)
tags = [t["name"] for t in result["project"]["tagCounts"]]
print(sorted(tags))

import wandb
from wandb_graphql.language import parser as gql_parser
 
api = wandb.Api(timeout=120)
doc = gql_parser.parse('''
  query {
    project(entityName: "ENTITY", name: "PROJECT") {
      groupedRuns(groupKeys: ["group"], first: 100) {
        ... on GroupedRunConnection {
          edges {
            node { group totalRuns }
          }
        }
      }
    }
  }
''')
result = api.client.execute(doc)
edges = result["project"]["groupedRuns"]["edges"]
groups = [e["node"]["group"] for e in edges if e["node"]["group"]]
print(sorted(groups))

import pandas as pd
from wandb_helpers import get_api, fetch_runs
 
api = get_api()
path = f"{entity}/{project}"
 
rows = fetch_runs(
    api, path,
    metric_keys=["LOSS_KEY", "ACC_KEY"],
    filters={"state": "finished"},
    limit=100,
)
df = pd.DataFrame(rows)
print(df.describe())

Evaluation.evaluate (root)
  +-- Evaluation.predict_and_score (one per dataset row x trials)
  |     +-- model.predict (the actual model call)
  |     +-- scorer_1.score
  |     +-- scorer_2.score
  +-- Evaluation.summarize

from weave_helpers import get_token_usage
 
usage = get_token_usage(call)
print(f"Tokens: {usage['total_tokens']} (in={usage['input_tokens']}, out={usage['output_tokens']})")

import wandb_workspaces.reports.v2 as wr
 
runset = wr.Runset(entity=entity, project=project, name="All runs")
plots = wr.PanelGrid(
    runsets=[runset],
    panels=[
        wr.LinePlot(title="Loss", x="_step", y=["LOSS_KEY"]),
        wr.BarPlot(title="Accuracy", metrics=["ACC_KEY"], orientation="v"),
    ],
)
 
report = wr.Report(
    entity=entity, project=project,
    title="Project analysis",
    description="Summary of recent runs",
    width="fixed",
    blocks=[
        wr.H1("Project analysis"),
        wr.P("Auto-generated summary from W&B API."),
        plots,
    ],
)
report.save(draft=True)

import logging
logging.getLogger("weave").setLevel(logging.ERROR)