Open-Weight Models & Local Inference

# script_id: day_091_open_weight_local/fit_check
# Pick a starting Ollama tag for this machine's RAM.
# Install: none (stdlib only) · Env: none · Services: none
# Usage: python fit_check.py [RAM_GB]   (pass RAM_GB on Windows, where sysconf is missing)
import os
import sys

# (min RAM GB, tag, download GB). Ollama registry sizes as of 2026-11; verify with `ollama pull`
FIT_TABLE = [
    (32, "gpt-oss:20b", 13.8),
    (16, "qwen3:8b", 5.2),
    (8, "qwen3:4b-instruct-2507-q4_K_M", 2.5),
    (0, "qwen3:1.7b", 1.4),  # under 8 GB: the small fallback
]


def total_ram_gb() -> float:
    """Physical RAM via POSIX sysconf (macOS and Linux)."""
    return os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1024**3


def pick_tag(ram_gb: float) -> tuple[str, float]:
    return next((tag, size) for min_gb, tag, size in FIT_TABLE if ram_gb >= min_gb)


ram = float(sys.argv[1]) if len(sys.argv) > 1 else total_ram_gb()
tag, size = pick_tag(ram)
print(f"RAM {ram:.0f} GB -> start with {tag} ({size} GB download)")
print(f"next: ollama pull {tag} && export ATLAS_LOCAL_MODEL={tag}")

RAM 16 GB -> start with qwen3:8b (5.2 GB download)
next: ollama pull qwen3:8b && export ATLAS_LOCAL_MODEL=qwen3:8b

# macOS / Linux (macOS can also use the Ollama.dmg app or `brew install ollama`)
curl -fsSL https://ollama.com/install.sh | sh
# Windows (PowerShell): irm https://ollama.com/install.ps1 | iex

ollama pull qwen3:4b-instruct-2507-q4_K_M          # download once (2.5 GB)
ollama pull nomic-embed-text  # the local embedding model used later (0.27 GB)
ollama run qwen3:4b-instruct-2507-q4_K_M "Say hi"  # one-off prompt from the terminal
ollama list                   # what's on disk
ollama ps                     # what's loaded in memory right now, and until when

# script_id: day_091_open_weight_local/first_local_call
# One Orbit question to a local model through the native ollama library. $0.
# Install: uv add "ollama>=0.6.3" · Env: ATLAS_LOCAL_MODEL (optional)
# Services: Ollama on localhost:11434, plus `ollama pull qwen3:4b-instruct-2507-q4_K_M`
import os

import ollama

LOCAL_MODEL = os.environ.get("ATLAS_LOCAL_MODEL", "qwen3:4b-instruct-2507-q4_K_M")

resp = ollama.chat(
    model=LOCAL_MODEL,
    messages=[{"role": "user", "content": "In the Orbit jobs library, what does error ORB-1101 mean?"}],
)
print(resp.message.content)
# Durations come back in nanoseconds; eval_* covers generated tokens (including any "thinking").
tok_s = resp.eval_count / (resp.eval_duration / 1e9)
print(f"\n{resp.eval_count} tokens at {tok_s:.1f} tokens/s · load {resp.load_duration / 1e9:.1f} s")

ORB-1101 is a configuration error raised when Orbit can't find a valid broker URL ...

412 tokens at 38.2 tokens/s · load 6.8 s

# script_id: day_091_open_weight_local/warm_vs_cold
# Time a cold call, a warm call, then unload and time a cold call again. $0.
# Install: uv add "ollama>=0.6.3" · Env: ATLAS_LOCAL_MODEL
# Services: Ollama on localhost:11434 with the tag pulled
import os
import time

import ollama

LOCAL_MODEL = os.environ.get("ATLAS_LOCAL_MODEL", "qwen3:4b-instruct-2507-q4_K_M")
MSG = [{"role": "user", "content": "Reply with the single word: ready"}]


def timed(label: str) -> None:
    start = time.perf_counter()
    r = ollama.chat(model=LOCAL_MODEL, messages=MSG)
    print(f"{label:<10} {time.perf_counter() - start:5.1f} s total, {r.load_duration / 1e9:4.1f} s loading")


def unload() -> None:
    # An empty messages list with keep_alive=0 unloads the model (Ollama API docs)
    ollama.chat(model=LOCAL_MODEL, messages=[], keep_alive=0)


unload()
timed("cold")
timed("warm")
unload()
timed("cold again")

cold         9.4 s total,  6.9 s loading
warm         1.6 s total,  0.0 s loading
cold again   8.8 s total,  6.5 s loading

# script_id: day_091_open_weight_local/base_url_swap
# The same Responses call against local Ollama and OpenAI. Only config changes.
# Install: uv add "openai>=3.46,<4"
# Env: ATLAS_LOCAL_BASE_URL, ATLAS_LOCAL_MODEL; OPENAI_API_KEY + ATLAS_MODEL for the openai backend
# Services: Ollama on localhost:11434 for the local backend
# Usage: python base_url_swap.py [local] [openai]   (no args = both)
import os
import sys
import time
from typing import Literal

from openai import APIConnectionError, NotFoundError, OpenAI

MODEL = os.environ.get("ATLAS_MODEL", "gpt-6-luna")
LOCAL_BASE_URL = os.environ.get("ATLAS_LOCAL_BASE_URL", "http://localhost:11434/v1/")
LOCAL_MODEL = os.environ.get("ATLAS_LOCAL_MODEL", "qwen3:4b-instruct-2507-q4_K_M")
Backend = Literal["openai", "local"]


def make_client(backend: Backend = "openai") -> tuple[OpenAI, str]:
    """Return (client, model id). Callers pass both along and never branch on the backend."""
    if backend == "local":
        # The SDK refuses to start without a key; Ollama ignores it. No retries: a stopped
        # daemon won't come back in 2 seconds, so fail fast with a clear message instead.
        return OpenAI(base_url=LOCAL_BASE_URL, api_key="ollama", max_retries=0), LOCAL_MODEL
    return OpenAI(), MODEL  # reads OPENAI_API_KEY


QUESTION = "In one sentence: what is a job lease in a background-jobs library?"
for backend in sys.argv[1:] or ["local", "openai"]:
    client, model = make_client(backend)
    start = time.perf_counter()
    try:
        resp = client.responses.create(model=model, input=QUESTION, max_output_tokens=300,
                                       reasoning={"effort": "none"})  # Ollama: thinking off
    except APIConnectionError:
        print(f"{backend:>6}: server unreachable. Start the Ollama app or run `ollama serve`.")
        continue
    except NotFoundError:
        print(f"{backend:>6}: model {model!r} not found. Local? Run `ollama pull {model}`.")
        continue
    print(f"{backend:>6} {time.perf_counter() - start:5.1f} s  {resp.output_text.strip()[:110]}")

 local   7.9 s  A job lease is a time-limited claim a worker holds on a job; if it isn't renewed or acknowledged ...
openai   1.2 s  A job lease is a time-limited lock a worker takes on a job so no other worker runs it, which ...

# script_id: day_091_open_weight_local/dimension_mismatch
# Why vectors from different embedding models can't share an index. Toy vectors, no services.
# Install: uv add numpy · Env: none · Services: none
import numpy as np

rng = np.random.default_rng(0)
docs_cloud = rng.normal(size=(3, 1536))  # stand-in: 3 chunks embedded by a 1,536-d cloud model
query_local = rng.normal(size=768)       # stand-in: a query embedded by a 768-d local model
try:
    docs_cloud @ query_local
except ValueError as e:
    print("mixed dimensions ->", e)


def cosine(m: np.ndarray, v: np.ndarray) -> np.ndarray:  # built on Day 23
    return (m @ v) / (np.linalg.norm(m, axis=1) * np.linalg.norm(v))


# Same size is no rescue. Simulate "model B" as a random rotation of model A's space:
# B keeps the same geometry internally, but its coordinates mean something else.
docs_a = rng.normal(size=(3, 768))
rotation, _ = np.linalg.qr(rng.normal(size=(768, 768)))
docs_b = docs_a @ rotation                          # the same 3 texts as model B embeds them
query_a = docs_a[1] + 0.5 * rng.normal(size=768)    # a query about chunk 1, embedded by A
print("query A vs index A:", np.round(cosine(docs_a, query_a), 2), "<- chunk 1 wins")
print("query A vs index B:", np.round(cosine(docs_b, query_a), 2), "<- no clear winner")

mixed dimensions -> matmul: Input operand 1 has a mismatch in its core dimension 0, with gufunc signature (n?,k),(k,m?)->(n?,m?) (size 768 is different from 1536)
query A vs index A: [0.   0.89 0.01] <- chunk 1 wins
query A vs index B: [ 0.02 -0.03 -0.05] <- no clear winner

# script_id: day_091_open_weight_local/local_embeddings
# Batch-embed 3 Orbit chunks locally with the native ollama library, then rank them for a query.
# Install: uv add "ollama>=0.6.3" numpy · Env: ORBIT_DATA, ATLAS_LOCAL_EMBED_MODEL
# Services: Ollama on localhost:11434, plus `ollama pull nomic-embed-text`
import os
import re
from pathlib import Path

import numpy as np
import ollama

DATA_DIR = Path(os.environ.get("ORBIT_DATA", "datasets/orbit"))
LOCAL_EMBED_MODEL = os.environ.get("ATLAS_LOCAL_EMBED_MODEL", "nomic-embed-text")


def embed_local(texts: list[str]) -> np.ndarray:
    """One batched call (input=list); unit-length rows, so a dot product is the cosine."""
    m = np.asarray(ollama.embed(model=LOCAL_EMBED_MODEL, input=texts).embeddings, dtype=np.float32)
    return m / np.linalg.norm(m, axis=1, keepdims=True)


body = (DATA_DIR / "docs" / "leases-and-heartbeats.md").read_text(encoding="utf-8").split("---\n", 2)[2]
chunks = [p.strip() for p in re.split(r"(?m)^(?=## )", body) if p.strip()][:3]  # Day 25, compact
# nomic-embed-text's model card asks for task prefixes; drop them for other embedding models.
docs = embed_local([f"search_document: {c}" for c in chunks])
query = embed_local(["search_query: why did my job run twice?"])[0]
print(f"dim={docs.shape[1]}  (a 1,536-d cloud index can't hold these)")
for score, c in sorted(zip(docs @ query, chunks), reverse=True):
    print(f"{score:.3f}  {c.splitlines()[0][:60]}")

dim=768  (a 1,536-d cloud index can't hold these)
0.712  ## What a lease is
0.655  ## Heartbeats
0.583  # Leases and heartbeats

# script_id: day_091_open_weight_local/local_rag
# Compact R2 (vector-only) that runs on either backend, then scores it. Local run: $0.
# Install: uv add "openai>=3.46,<4" numpy pydantic
# Env: ORBIT_DATA; ATLAS_LOCAL_* for local; OPENAI_API_KEY, ATLAS_MODEL, ATLAS_EMBED_MODEL for openai
# Services: Ollama on localhost:11434 with ATLAS_LOCAL_MODEL and ATLAS_LOCAL_EMBED_MODEL pulled
# Usage: python local_rag.py --backend local [--limit 10]
import argparse
import hashlib
import json
import math
import os
import re
import statistics
import time
from pathlib import Path
from typing import Literal

import numpy as np
from openai import OpenAI
from pydantic import BaseModel, ValidationError

DATA_DIR = Path(os.environ.get("ORBIT_DATA", "datasets/orbit"))
MODEL = os.environ.get("ATLAS_MODEL", "gpt-6-luna")
EMBED_MODEL = os.environ.get("ATLAS_EMBED_MODEL", "text-embedding-3-small")
LOCAL_BASE_URL = os.environ.get("ATLAS_LOCAL_BASE_URL", "http://localhost:11434/v1/")
LOCAL_MODEL = os.environ.get("ATLAS_LOCAL_MODEL", "qwen3:4b-instruct-2507-q4_K_M")
LOCAL_EMBED_MODEL = os.environ.get("ATLAS_LOCAL_EMBED_MODEL", "nomic-embed-text")
Backend = Literal["openai", "local"]
EMBEDDERS: dict[str, str] = {"openai": EMBED_MODEL, "local": LOCAL_EMBED_MODEL}


def make_client(backend: Backend = "openai") -> tuple[OpenAI, str]:  # from base_url_swap above
    if backend == "local":
        return OpenAI(base_url=LOCAL_BASE_URL, api_key="ollama", max_retries=0), LOCAL_MODEL
    return OpenAI(), MODEL


class Citation(BaseModel):  # built on Day 28
    doc_id: str
    quote: str


class Answer(BaseModel):  # built on Day 28
    text: str
    citations: list[Citation]

# script_id: day_091_open_weight_local/local_rag
PREFIXES = {"nomic-embed-text": ("search_query: ", "search_document: ")}  # from its model card


def load_chunks(folder: Path) -> list[tuple[str, str]]:
    """Built on Day 25, compact: one (doc_id, text) chunk per H2 section of each page."""
    chunks: list[tuple[str, str]] = []
    for path in sorted(folder.glob("*.md")):
        _, head, body = path.read_text(encoding="utf-8").split("---\n", 2)
        doc_id = re.search(r"(?m)^id:\s*(\S+)", head).group(1)
        chunks += [(doc_id, p.strip()) for p in re.split(r"(?m)^(?=## )", body) if p.strip()]
    return chunks


def embed(client: OpenAI, model: str, texts: list[str], *, kind: Literal["query", "doc"]) -> np.ndarray:
    q_prefix, d_prefix = PREFIXES.get(model.split(":")[0], ("", ""))
    texts = [(q_prefix if kind == "query" else d_prefix) + t for t in texts]
    rows: list[list[float]] = []
    for i in range(0, len(texts), 64):  # batches of 64 texts per request
        rows += [d.embedding for d in client.embeddings.create(model=model, input=texts[i:i + 64]).data]
    m = np.asarray(rows, dtype=np.float32)
    return m / np.linalg.norm(m, axis=1, keepdims=True)  # unit length: dot product = cosine


class Index:
    """Built on Day 26, as an in-memory stand-in: one index per embedding model, cached to .npz."""

    def __init__(self, client: OpenAI, embed_model: str, chunks: list[tuple[str, str]]) -> None:
        self.client, self.embed_model, self.chunks = client, embed_model, chunks
        digest = hashlib.sha256("\n".join(t for _, t in chunks).encode()).hexdigest()[:12]
        safe_name = re.sub(r"[^\w.-]", "_", embed_model)
        cache = Path("indexes") / f"{safe_name}-{digest}.npz"
        if cache.exists():
            self.matrix = np.load(cache)["m"]
        else:
            self.matrix = embed(client, embed_model, [t for _, t in chunks], kind="doc")
            cache.parent.mkdir(exist_ok=True)
            np.savez(cache, m=self.matrix)

    def search(self, query: str, k: int = 5) -> list[tuple[str, str, float]]:
        q = embed(self.client, self.embed_model, [query], kind="query")[0]
        sims = self.matrix @ q  # raises ValueError if someone mixed embedders
        return [(*self.chunks[i], float(sims[i])) for i in np.argsort(-sims)[:k]]

# script_id: day_091_open_weight_local/local_rag
INSTRUCTIONS = """Answer questions about Orbit using ONLY the sources below.
Reply with JSON only: {"text": "...", "citations": [{"doc_id": "...", "quote": "..."}]}
Each quote is at most 25 words copied exactly from that source. If the sources don't
answer the question, reply {"text": "I don't know.", "citations": []}"""
PRICES = {"gpt-6-luna": (0.10, 0.50)}  # USD per 1M in/out, as of 2026-11; verify at the pricing page
SPEND = {"usd": 0.0}  # local models aren't in PRICES, so they cost $0 here


def parse_answer(raw: str) -> Answer:
    match = re.search(r"\{.*\}", raw, re.S)  # small models often wrap JSON in ``` fences
    try:
        return Answer.model_validate_json(match.group(0) if match else "")  # "" fails validation
    except ValidationError:
        return Answer(text="I don't know.", citations=[])


def ask(question: str, *, client: OpenAI, model: str, index: Index, k: int = 5) -> Answer:
    """Built on Day 28: retrieve, answer with citations, keep only citations we retrieved."""
    hits = index.search(question, k)
    context = "\n\n".join(f"[{doc_id}]\n{text}" for doc_id, text, _ in hits)
    resp = client.responses.create(model=model, instructions=INSTRUCTIONS,
                                   input=f"{context}\n\nQuestion: {question}",
                                   reasoning={"effort": "none"}, max_output_tokens=800)
    if resp.usage and model in PRICES:
        p_in, p_out = PRICES[model]
        SPEND["usd"] += (resp.usage.input_tokens * p_in + resp.usage.output_tokens * p_out) / 1e6
    answer = parse_answer(resp.output_text)
    retrieved = {doc_id for doc_id, _, _ in hits}
    answer.citations = [c for c in answer.citations if c.doc_id in retrieved]
    return answer

# script_id: day_091_open_weight_local/local_rag
def read_jsonl(name: str, limit: int) -> list[dict]:
    lines = (DATA_DIR / "eval" / name).read_text(encoding="utf-8").splitlines()
    return [json.loads(line) for line in lines if line.strip()][:limit]


def pct(xs: list[float], p: float) -> float:
    """Nearest-rank percentile (built on Day 42)."""
    s = sorted(xs)
    return s[max(0, math.ceil(p / 100 * len(s)) - 1)]


def evaluate(backend: Backend, limit: int) -> dict:
    client, model = make_client(backend)
    index = Index(client, EMBEDDERS[backend], load_chunks(DATA_DIR / "docs"))
    recalls = []
    for q in read_jsonl("retrieval_queries.jsonl", limit):
        ranked = list(dict.fromkeys(d for d, _, _ in index.search(q["query"], k=20)))  # pages, once
        relevant = set(q["relevant_doc_ids"])
        recalls.append(len(set(ranked[:5]) & relevant) / len(relevant))  # Day 29's recall@5
    ask("warm-up", client=client, model=model, index=index)  # keep the cold load out of p95
    hits, secs = [], []
    for item in read_jsonl("qa_gold.jsonl", limit):
        start = time.perf_counter()
        answer = ask(item["question"], client=client, model=model, index=index)
        secs.append(time.perf_counter() - start)
        hits.append(any(c.doc_id in item["citations"] for c in answer.citations))
    return {"backend": backend, "model": model, "embed": EMBEDDERS[backend],
            "recall@5": statistics.mean(recalls), "citation_hit": statistics.mean(hits),
            "p50_s": pct(secs, 50), "p95_s": pct(secs, 95), "usd": SPEND["usd"], "n": len(secs)}

# script_id: day_091_open_weight_local/local_rag
if __name__ == "__main__":
    ap = argparse.ArgumentParser()
    ap.add_argument("--backend", choices=["local", "openai"], default="local")
    ap.add_argument("--limit", type=int, default=40, help="cases per eval file (10 = quick run)")
    args = ap.parse_args()
    row = evaluate(args.backend, args.limit)
    out = Path("reports/local_vs_cloud.jsonl")
    out.parent.mkdir(exist_ok=True)
    with out.open("a", encoding="utf-8") as f:
        f.write(json.dumps(row) + "\n")
    print("| backend | model | embed | recall@5 | citation hit | p50 s | p95 s | $ |")
    print("|---|---|---|---|---|---|---|---|")
    for r in map(json.loads, out.read_text(encoding="utf-8").splitlines()):
        print(f"| {r['backend']} | {r['model']} | {r['embed']} | {r['recall@5']:.2f} "
              f"| {r['citation_hit']:.2f} | {r['p50_s']:.1f} | {r['p95_s']:.1f} | {r['usd']:.3f} |")

| backend | model | embed | recall@5 | citation hit | p50 s | p95 s | $ |
|---|---|---|---|---|---|---|---|
| local | qwen3:4b-instruct-2507-q4_K_M | nomic-embed-text | 0.71 | 0.63 | 6.8 | 14.9 | 0.000 |
| openai | gpt-6-luna | text-embedding-3-small | 0.79 | 0.87 | 1.9 | 3.6 | 0.006 |