#!/usr/bin/env python3
"""Epiphani AI organizer (pure stdlib — runs under the system /usr/bin/python3).

Turns a folder of timestamp-named .aac recordings into a readable "diary of the day":
transcribe each clip (speaker-diarized via gpt-4o-transcribe-diarize), analyze it
(latest gpt flagship) into a title / who-with / summary / category, then group clips
into conversations per day. Writes:
  <rec>/.organized/<file>.json   per-clip sidecar (idempotent; re-runs skip done clips)
  <rec>/diary.json               grouped structure the app's Diary view renders
  <rec>/Diary.html               a standalone browser-openable diary (same data)

No third-party packages: every OpenAI call goes over stdlib urllib, so the shipped app
needs only the system Python + ffmpeg (Homebrew). The proven prototype this is adapted
from lives at hvn/claude_scripts/2026-06-16-organize-recordings.py.

Env contract (the Swift app sets these):
  OPENAI_API_KEY     required; without it the app still records, organization just waits
  EPIPHANI_REC_DIR   recordings folder (default ~/Recordings/Epiphani)
  EPIPHANI_CAP_SEC   transcribe at most the first N seconds/clip (default 600)

CLI:  organize.py [REC_DIR]      # one idempotent sweep; prints a final summary line
"""
import datetime
import html
import json
import logging
import mimetypes
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request

logger = logging.getLogger("epiphani.organize")

REC_DIR = os.path.expanduser(
    os.environ.get("EPIPHANI_REC_DIR") or "~/Recordings/Epiphani")
ORG_DIR = os.path.join(REC_DIR, ".organized")
DIARY_JSON = os.path.join(REC_DIR, "diary.json")
DIARY_HTML = os.path.join(REC_DIR, "Diary.html")
LOCK = os.path.join(ORG_DIR, ".lock")

SILENCE_DB = -65.0            # mean_volume below this == silent; skip the cloud
CAP_SEC = int(os.environ.get("EPIPHANI_CAP_SEC") or 600)
FRESH_SEC = 90               # skip a clip whose mtime is this recent — it may still be recording
TRANSCRIBE_MODEL = "gpt-4o-transcribe-diarize"
API = "https://api.openai.com/v1"

# Two INDEPENDENT engine dials, not one combined switch (App Atlas P0-1, 2026-07-26): transcription
# and analysis/summarization can each be "cloud" (OpenAI) or "local" (on-device: Parakeet for
# transcription, a local LLM via Ollama for analysis) regardless of the other. Mixing is a real,
# meaningful privacy choice (e.g. transcribe locally so audio never leaves the Mac, but still let
# OpenAI write the sharper summary from the transcript text alone). Neither local path ever touches
# the network; if the on-device stack it needs is unavailable it fails VISIBLY rather than silently
# round-tripping to the cloud. EPIPHANI_ORG_ENGINE is read as a fallback for either dial that isn't
# set, so an older build/env that only ever set the combined var keeps working unchanged.
_LEGACY_ENGINE = (os.environ.get("EPIPHANI_ORG_ENGINE") or "cloud").lower()
TRANSCRIBE_ENGINE = (os.environ.get("EPIPHANI_TRANSCRIBE_ENGINE") or _LEGACY_ENGINE).lower()
ANALYZE_ENGINE = (os.environ.get("EPIPHANI_ANALYZE_ENGINE") or _LEGACY_ENGINE).lower()
PYENV_PY = os.path.expanduser("~/.pyenv/versions/3.12.8/bin/python")   # runs local_transcribe (parakeet_mlx)
# On-device speaker labels for the local path (FluidAudio CoreML via local_diarize.py; council
# recapmuse-diarization-engine 2026-07-31). "auto" = on whenever the diarizer probe passes, so the
# local Diary gains "who said what" without configuration; "on"/"off" force it either way.
DIARIZE = (os.environ.get("EPIPHANI_DIARIZE") or "auto").lower()
_HERE = os.path.dirname(os.path.abspath(__file__))
if _HERE not in sys.path:
    sys.path.insert(0, _HERE)
try:
    import local_ai                    # on-device LLM (Ollama), same-dir sibling; pure stdlib
except Exception:                      # noqa: BLE001
    local_ai = None

# Vertical edition (RecapMuse build spec 02). Selects an edition-specific analysis schema so a
# recording's Diary entry carries the fields that edition sells. Works on EITHER engine (cloud or local).
# "none" = the generalist app. Billing/SKU gating for these editions is a separate step (a STOP: it
# needs a purchase flow), so this is the feature layer, gated by the app's edition setting.
EDITION = (os.environ.get("EPIPHANI_EDITION") or "none").lower()   # none|counsel|session|recall
EDITION_SYS = {
    # Sales: talk-ratio + objections (feeds "Copy for CRM", crm_export.py).
    "recall": ("\n\nSALES EDITION — ALSO include in the JSON: \"talk_ratio\" (number 0..1, the fraction "
               "of words spoken by the user / primary speaker vs. everyone else, estimated from the "
               "transcript), and \"objections\" (array of concerns or objections the other party raised; "
               "empty if none)."),
    # Clinical: a SOAP note. Explicitly DRAFT — a clinician must review before it is relied on.
    "session": ("\n\nCLINICAL EDITION — ALSO include a \"soap\" object: {\"subjective\":..., "
                "\"objective\":..., \"assessment\":..., \"plan\":...} drafting the session as a SOAP note "
                "(empty strings where not applicable). This is DRAFT documentation for a clinician to review, "
                "never final."),
    # Legal: privilege framing + the matter tag (echoed from the recording's matter field when provided).
    "counsel": ("\n\nLEGAL EDITION — ALSO include: \"privileged\" (boolean; treat as attorney-client "
                "privileged context) and \"matter\" (echo the matter tag from the input if present, else \"\")."),
}
# Extra keys each edition adds, so the schema normalizer preserves them.
EDITION_KEYS = {"recall": ("talk_ratio", "objections"), "session": ("soap",),
                "counsel": ("privileged", "matter")}


def _analyze_sys():
    """The analysis system prompt for the active edition (base schema + edition-specific fields)."""
    return ANALYZE_SYS + EDITION_SYS.get(EDITION, "")

ANALYZE_SYS = (
    "You organize a person's day from audio their Mac recorded (calls, videos, the person "
    "talking near the mic, ambient room sound). Given ONE recording's diarized transcript, return "
    "STRICT JSON so a busy person can scan their day at a glance. Infer from content; be concrete.\n"
    "Fields:\n"
    "- title: short, specific, human (max ~8 words). e.g. 'Call reviewing the cap-table scenarios', "
    "'Solo dictation to Claude about the recorder app', 'Quiet stretch, no conversation'. Never a timestamp.\n"
    "- summary: 1-2 sentences on what happened or was said.\n"
    "- category: one of [phone_call, video_call, screen_share, in_person, dictation_or_ai, "
    "media_playback, ambient_or_silent, test_or_tone, unknown].\n"
    "- who_with: who the person talks WITH, best guess (names if spoken, else roles): 'A colleague "
    "(screen-share)', 'Meg', 'Meg and Lily', 'Claude (AI assistant)', 'No one / working solo', 'Unclear'.\n"
    "- participants: array, one short label per distinct diarization speaker (A/B/...): "
    "['Jason (likely the user)', 'Colleague B'].\n"
    "- topics: array of 1-4 short tags.\n"
    "- action_items: array of clear to-dos mentioned (empty if none).\n"
    "- is_meaningful: boolean. true if real content worth keeping; false for silence/test tones/noise.\n"
    "Return ONLY the JSON object."
)

DAY_SYS = (
    "You write a one-day diary from a list of audio recordings a person made. Group the clips into "
    "distinct CONVERSATIONS/sessions (merge clips that are clearly the same continuous interaction by "
    "time and topic; keep unrelated ones separate). Return STRICT JSON:\n"
    "{\"day_summary\": \"2-3 sentence narrative of the person's day from these recordings\", "
    "\"conversations\": [{\"title\": \"...\", \"time_range\": \"9:08am-9:20am\", \"who_with\": \"...\", "
    "\"summary\": \"1-2 sentences\", \"files\": [\"<file names that belong to this conversation>\"]}]}\n"
    "Order conversations by start time. Use 12-hour times. Return ONLY the JSON object."
)


# ----------------------------------------------------------------------------- HTTP (stdlib)
def _req(path, key, data=None, headers=None, timeout=120):
    url = path if path.startswith("http") else API + path
    h = {"Authorization": "Bearer " + key}
    if headers:
        h.update(headers)
    r = urllib.request.Request(url, data=data, headers=h)
    with urllib.request.urlopen(r, timeout=timeout) as resp:
        return json.loads(resp.read().decode("utf-8"))


def _post_json(path, key, payload, timeout=120):
    body = json.dumps(payload).encode("utf-8")
    return _req(path, key, body, {"Content-Type": "application/json"}, timeout)


def _post_multipart(path, key, fields, file_field, file_path, timeout=300):
    """POST multipart/form-data with stdlib only (no `requests`)."""
    boundary = "----epiphani" + os.urandom(16).hex()
    ctype = mimetypes.guess_type(file_path)[0] or "application/octet-stream"
    pre = []
    for name, val in fields.items():
        pre.append("--" + boundary)
        pre.append('Content-Disposition: form-data; name="%s"' % name)
        pre.append("")
        pre.append(str(val))
    pre.append("--" + boundary)
    pre.append('Content-Disposition: form-data; name="%s"; filename="%s"'
               % (file_field, os.path.basename(file_path)))
    pre.append("Content-Type: " + ctype)
    pre.append("")
    pre.append("")
    head = "\r\n".join(pre).encode("utf-8")
    tail = ("\r\n--" + boundary + "--\r\n").encode("utf-8")
    with open(file_path, "rb") as fh:
        body = head + fh.read() + tail
    h = {"Content-Type": "multipart/form-data; boundary=" + boundary}
    return _req(path, key, body, h, timeout)


def _err_text(e):
    if isinstance(e, urllib.error.HTTPError):
        try:
            return "HTTP %s: %s" % (e.code, e.read().decode("utf-8")[:300])
        except Exception:
            return "HTTP %s" % e.code
    return str(e)


# ----------------------------------------------------------------------------- model pick
def _pick_fast(key):
    try:
        ids = [m["id"] for m in _req("/models", key, timeout=30).get("data", [])
               if m.get("id", "").startswith("gpt-")]
    except Exception as e:
        logger.warning("models.list failed (%s); defaulting model", _err_text(e))
        return "gpt-5.5"
    skip = ("-mini", "-nano", "-codex", "-chat-latest", "-search", "-audio", "-realtime",
            "-image", "-tts", "-transcribe", "-pro", "-embedding", "-instruct", "-turbo")
    cand = [i for i in ids if not any(s in i for s in skip)
            and not re.search(r"-\d{4}-\d{2}-\d{2}$", i)]

    def rank(i):
        try:
            return tuple(int(x) for x in i.replace("gpt-", "").split("."))
        except ValueError:
            return (0,)

    return sorted(cand, key=rank)[-1] if cand else "gpt-5.5"


# ----------------------------------------------------------------------------- ffmpeg
def _ffmpeg():
    for p in ("/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"):
        if os.path.exists(p):
            return p
    from shutil import which
    return which("ffmpeg")


FF = _ffmpeg()


def _probe(path):
    out = subprocess.run([FF, "-hide_banner", "-nostdin", "-i", path, "-af", "volumedetect", "-f", "null", "-"],
                         capture_output=True, text=True).stderr
    m = re.search(r"Duration: (\d+):(\d+):(\d+)", out)
    secs = (int(m.group(1)) * 3600 + int(m.group(2)) * 60 + int(m.group(3))) if m else 0
    mv = re.search(r"mean_volume: (-?[0-9.]+)", out)
    return secs, (float(mv.group(1)) if mv else -99.0)


def _started_at(fname):
    m = re.match(r"(\d{4})-(\d{2})-(\d{2})_(\d{2})-(\d{2})-(\d{2})", fname)
    return datetime.datetime(*[int(x) for x in m.groups()]) if m else None


# ----------------------------------------------------------------------------- transcribe / analyze
def _transcribe(key, path, secs):
    src = "/tmp/epiphani_org.m4a"
    args = [FF, "-y", "-hide_banner", "-loglevel", "error", "-i", path]
    if secs > CAP_SEC:
        args += ["-t", str(CAP_SEC)]
    args += ["-c", "copy", src]
    subprocess.run(args, check=True)
    resp = _post_multipart(
        "/audio/transcriptions", key,
        {"model": TRANSCRIBE_MODEL, "response_format": "diarized_json",
         "chunking_strategy": "auto"},   # required by diarization models for longer audio
        "file", src)
    segs = resp.get("segments") or []
    speakers, lines, cur, buf = [], [], None, []
    for s in segs:
        spk = s.get("speaker") or "?"
        txt = (s.get("text") or "").strip()
        if spk not in speakers:
            speakers.append(spk)
        if spk != cur and buf:
            lines.append("Speaker %s: %s" % (cur, " ".join(buf)))
            buf = []
        cur = spk
        if txt:
            buf.append(txt)
    if buf:
        lines.append("Speaker %s: %s" % (cur, " ".join(buf)))
    transcript = "\n".join(lines) if lines else (resp.get("text") or "").strip()
    return transcript, speakers


def _analyze(key, model, fname, when, secs, transcript):
    user = "Recording: %s\nStarted: %s\nDuration: %ss\n\nDiarized transcript:\n%s" % (
        fname, when, secs, transcript[:8000])
    r = _post_json("/chat/completions", key, {
        "model": model, "response_format": {"type": "json_object"},
        "messages": [{"role": "system", "content": _analyze_sys()},
                     {"role": "user", "content": user}]})
    return json.loads(r["choices"][0]["message"]["content"])


# ----------------------------------------------------------------------------- on-device (local) path
def _local_transcriber_ready():
    """(ok, reason). True only if the on-device transcriber (Parakeet/mlx-whisper) is ready. Checked
    independently of the local LLM, since transcription and analysis are now separate dials."""
    try:
        out = subprocess.run([PYENV_PY, os.path.join(_HERE, "local_transcribe.py"), "--check"],
                             capture_output=True, text=True, timeout=60)
        chk = json.loads(out.stdout or "{}")
    except Exception as e:  # noqa: BLE001
        return False, "on-device transcriber check failed: %s" % (str(e)[:160])
    if not chk.get("ok"):
        return False, chk.get("error") or "on-device transcriber not ready"
    return True, "ready"


def _local_diarizer_ready():
    """(ok, reason). True only if the on-device diarizer (diarize-cli + CoreML models) is ready."""
    try:
        out = subprocess.run([PYENV_PY, os.path.join(_HERE, "local_diarize.py"), "--check"],
                             capture_output=True, text=True, timeout=60)
        chk = json.loads(out.stdout or "{}")
    except Exception as e:  # noqa: BLE001
        return False, "on-device diarizer check failed: %s" % (str(e)[:160])
    if not chk.get("ok"):
        why = []
        if not chk.get("cli"):
            why.append("diarize-cli missing")
        if not chk.get("models"):
            why.append("diarizer models not downloaded")
        return False, "; ".join(why) or "on-device diarizer not ready"
    return True, "ready"


def _local_llm_ready():
    """(ok, reason). True only if a local LLM (Ollama) is installed, for the analysis/summarization
    dial. Checked independently of the transcriber."""
    if local_ai is None:
        return False, "local_ai module not importable"
    if not local_ai.installed_models():
        return False, "no local LLM installed (add one in Settings > AI Engines, e.g. Llama 3.2)"
    return True, "ready"


def _local_available():
    """(ok, reason). True only if BOTH on-device transcription and a local LLM are ready. Kept for
    any caller that still wants the old combined check (e.g. a single "can we go fully local at
    all?" probe); the real per-dial gates below use the two functions above independently."""
    ok, why = _local_transcriber_ready()
    if not ok:
        return False, why
    return _local_llm_ready()


def _transcribe_local_batch(paths, diarize=False):
    """On-device ASR for many clips in ONE model-load. Returns {abspath: entry} where entry carries
    at least {"transcript"} and, when diarization ran, {"speakers", "diarized"}. Never uses the
    network. Raises RuntimeError (a visible failure) if the on-device stack is unavailable: it does
    NOT fall back to the cloud, by design (see the module + spec integrity note)."""
    if not paths:
        return {}
    env = {**os.environ, "EPIPHANI_CAP_SEC": str(CAP_SEC)}
    args = [PYENV_PY, os.path.join(_HERE, "local_transcribe.py")]
    if diarize:
        args.append("--diarize")
    out = subprocess.run(args + list(paths), capture_output=True, text=True, timeout=3600, env=env)
    try:
        data = json.loads(out.stdout or "{}")
    except ValueError:
        raise RuntimeError("on-device transcriber returned no JSON: %s" % (out.stderr or "")[:200])
    if not data.get("ok"):
        raise RuntimeError(data.get("error") or "on-device transcription unavailable")
    return {p: r for p, r in (data.get("results") or {}).items() if r.get("ok")}


def _normalize_ai(ai, transcript):
    """Coerce a local LLM's JSON into the exact schema the Diary renderer expects, filling defaults so a
    partial/loose response (e.g. a coder model that omits is_meaningful) never silently drops a clip."""
    ai = ai if isinstance(ai, dict) else {}
    out = {
        "title": (str(ai.get("title") or "Recorded conversation")).strip()[:120],
        "summary": str(ai.get("summary") or ""),
        "category": ai.get("category") or "unknown",
        "who_with": str(ai.get("who_with") or ""),
        "participants": ai.get("participants") or [],
        "topics": ai.get("topics") or [],
        "action_items": ai.get("action_items") or [],
        "is_meaningful": bool(ai.get("is_meaningful", True)) if transcript.strip() else False,
    }
    for k in ("participants", "topics", "action_items"):
        if not isinstance(out[k], list):
            out[k] = [str(out[k])] if out[k] else []
    for k in EDITION_KEYS.get(EDITION, ()):     # carry edition-specific fields through (Recall/Session/Counsel)
        if k in ai:
            out[k] = ai[k]
    return out


def _redact_speaker_names(transcript, speakers):
    """Replace enrolled REAL names with generic labels before any transcript goes to a cloud
    analyzer. Voiceprint names are on-device identity data: they may render in the local Diary,
    but they never ride a network call (GPT privacy review 2026-07-31, finding 1). Generic
    'Speaker X' labels pass through untouched."""
    named = [s for s in (speakers or []) if not s.startswith("Speaker ")]
    if not named:
        return transcript
    taken = {s for s in speakers if s.startswith("Speaker ")}
    letters = (c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ" if ("Speaker " + c) not in taken)
    out = transcript
    for name in named:
        out = out.replace("%s:" % name, "%s:" % ("Speaker " + next(letters, "?")))
    return out


def _analyze_local(fname, when, secs, transcript):
    """On-device per-clip analysis via a local LLM (Ollama). Same JSON schema as the cloud _analyze."""
    if local_ai is None:
        raise RuntimeError("local_ai module unavailable")
    user = "Recording: %s\nStarted: %s\nDuration: %ss\n\nTranscript:\n%s" % (
        fname, when, secs, transcript[:8000])
    obj, _model = local_ai.chat_json(_analyze_sys(), user)
    return _normalize_ai(obj, transcript)


_EMPTY_AI = {"title": "Empty recording", "is_meaningful": False, "category": "ambient_or_silent",
             "who_with": "", "summary": "No audio captured.", "participants": [], "topics": [], "action_items": []}
_SILENT_AI = {"title": "Silent stretch", "is_meaningful": False, "category": "ambient_or_silent",
              "who_with": "No one / working solo", "summary": "Recording was effectively silent.",
              "participants": [], "topics": [], "action_items": []}
_NOSPEECH_AI = {"title": "Sound but no speech", "is_meaningful": False, "category": "test_or_tone",
                "who_with": "", "summary": "Audio but no discernible speech.",
                "participants": [], "topics": [], "action_items": []}


def _organize_clip(fname, key=None, model=None, local_transcript=None,
                    transcribe_engine=None, analyze_engine=None):
    """Return the sidecar meta for one clip. Transcription and analysis are INDEPENDENT dials (App
    Atlas P0-1): each is "cloud" (OpenAI) or "local" (on-device), and any of the four combinations
    is valid — e.g. transcribe locally (audio never leaves the Mac) but let OpenAI write the
    summary from the transcript text alone. `local_transcript` is the pre-batched on-device
    transcript, used only when transcribe_engine=="local"."""
    transcribe_engine = transcribe_engine or TRANSCRIBE_ENGINE
    analyze_engine = analyze_engine or ANALYZE_ENGINE
    path = os.path.join(REC_DIR, fname)
    when = _started_at(fname)
    meta = {"file": fname, "started_at": when.isoformat() if when else None,
            "engine": {"transcribe": transcribe_engine, "analyze": analyze_engine}}
    size = os.path.getsize(path)
    if size < 1024:
        meta.update(status="empty", duration_sec=0, ai=dict(_EMPTY_AI))
        return meta
    secs, mean_db = _probe(path)
    meta.update(duration_sec=secs, mean_db=mean_db)
    if mean_db < SILENCE_DB:
        meta.update(status="silent", ai=dict(_SILENT_AI))
        return meta
    try:
        if transcribe_engine == "local":
            # already produced on-device, no network; dict entries carry diarized speaker labels
            entry = local_transcript if isinstance(local_transcript, dict) else {"transcript": local_transcript}
            transcript = (entry.get("transcript") or "").strip()
            speakers = entry.get("speakers") or []
        else:
            transcript, speakers = _transcribe(key, path, secs)
        if len(transcript.strip()) < 4:
            meta.update(status="no_speech", transcript="", speakers=speakers, ai=dict(_NOSPEECH_AI))
            return meta
        if analyze_engine == "local":
            ai = _analyze_local(fname, when, secs, transcript)
        else:
            ai = _analyze(key, model, fname, when, secs,
                          _redact_speaker_names(transcript, speakers))
        meta.update(status="transcribed", transcript=transcript, speakers=speakers, ai=ai)
    except Exception as e:
        logger.warning("clip %s failed: %s", fname, _err_text(e))
        meta.update(status="error", error=_err_text(e)[:200],
                    ai={"title": "Could not process", "is_meaningful": False, "category": "unknown",
                        "who_with": "", "summary": _err_text(e)[:120], "participants": [],
                        "topics": [], "action_items": []})
    return meta


# ----------------------------------------------------------------------------- lock (avoid double runs)
def _acquire_lock():
    try:
        if os.path.exists(LOCK) and (time.time() - os.path.getmtime(LOCK)) < 1800:
            return False
    except OSError:
        pass
    try:
        with open(LOCK, "w") as fh:
            fh.write(str(os.getpid()))
        return True
    except OSError:
        return False


def _release_lock():
    try:
        os.remove(LOCK)
    except OSError:
        pass


# ----------------------------------------------------------------------------- sweep
def run():
    key = os.environ.get("OPENAI_API_KEY")
    if not FF:
        print("organize: ffmpeg not found; cannot transcribe. Install ffmpeg (brew install ffmpeg).")
        return 0
    if TRANSCRIBE_ENGINE == "local":
        ok, why = _local_transcriber_ready()
        if not ok:                       # fail VISIBLY; the on-device path NEVER falls back to cloud
            print("organize[local]: on-device transcriber not ready: %s" % why)
            return 0
    if ANALYZE_ENGINE == "local":
        ok, why = _local_llm_ready()
        if not ok:
            print("organize[local]: on-device summarizer not ready: %s" % why)
            return 0
    needs_key = (TRANSCRIBE_ENGINE != "local") or (ANALYZE_ENGINE != "local")
    if needs_key:
        if not key:
            print("organize: no OPENAI_API_KEY set; skipping (the app still records).")
            return 0
    else:
        key = None                       # fully local: must not carry a cloud key at all
    os.makedirs(ORG_DIR, exist_ok=True)
    if not _acquire_lock():
        print("organize: another sweep is running; skipping.")
        return 0
    try:
        files = sorted(f for f in os.listdir(REC_DIR) if f.endswith(".aac"))
        now = time.time()
        todo = []
        for fname in files:
            if os.path.exists(os.path.join(ORG_DIR, fname + ".json")):
                continue
            try:                                # skip a clip still being written (mtime fresh)
                if now - os.path.getmtime(os.path.join(REC_DIR, fname)) < FRESH_SEC:
                    continue
            except OSError:
                continue
            todo.append(fname)
        # Idle sweep is free: nothing new + a diary already on disk => no API calls at all.
        if not todo and os.path.exists(DIARY_JSON):
            print("organize: nothing new to organize.")
            return 0
        model, tmap = None, {}
        if TRANSCRIBE_ENGINE == "local":
            diarize_on = DIARIZE == "on"
            if DIARIZE == "auto":
                diarize_on = _local_diarizer_ready()[0]
            elif DIARIZE == "on":
                ok, why = _local_diarizer_ready()
                if not ok:                      # additive feature: say so visibly, keep transcribing
                    print("organize[local]: speaker labels unavailable: %s" % why)
                    diarize_on = False
            try:                                # one model-load transcribes every new clip on-device
                tmap = _transcribe_local_batch([os.path.join(REC_DIR, f) for f in todo],
                                               diarize=diarize_on)
            except Exception as e:
                print("organize[local]: on-device transcription failed: %s" % _err_text(e))
                return 0                        # visible failure; no cloud fallback
        if ANALYZE_ENGINE != "local":
            model = _pick_fast(key)
        logger.info("engine transcribe=%s analyze=%s%s", TRANSCRIBE_ENGINE, ANALYZE_ENGINE,
                    (" | analysis model %s" % model) if model else "")
        for fname in todo:
            meta = _organize_clip(fname, key=key, model=model,
                                   local_transcript=tmap.get(os.path.join(REC_DIR, fname)),
                                   transcribe_engine=TRANSCRIBE_ENGINE, analyze_engine=ANALYZE_ENGINE)
            json.dump(meta, open(os.path.join(ORG_DIR, fname + ".json"), "w"), indent=2, default=str)
            logger.info("  %s -> %s", fname, meta.get("ai", {}).get("title"))
        metas = []                              # rebuild the diary from ALL sidecars
        for fname in files:
            sc = os.path.join(ORG_DIR, fname + ".json")
            if os.path.exists(sc):
                try:
                    metas.append(json.load(open(sc)))
                except Exception:
                    pass
        diary = _build_diary(metas, key=key, model=model)
        json.dump(diary, open(DIARY_JSON, "w"), indent=2, default=str)
        _write_html(diary)
        print("organize: %d clip(s) newly organized; %d total; diary updated." % (len(todo), len(metas)))
        return 0
    finally:
        _release_lock()


# ----------------------------------------------------------------------------- diary grouping
_CAT_ICON = {"phone_call": "\U0001F4DE", "video_call": "\U0001F4F9", "screen_share": "\U0001F5A5",
             "in_person": "\U0001F465", "dictation_or_ai": "\U0001F916", "media_playback": "\U0001F3AC",
             "ambient_or_silent": "\U0001F907", "test_or_tone": "\U0001F50A", "unknown": "\U0001F4AC"}


def _fmt_time(iso):
    t = datetime.datetime.fromisoformat(iso)
    h = t.hour % 12 or 12
    return "%d:%02d%s" % (h, t.minute, "am" if t.hour < 12 else "pm")


GROUP_GAP_MIN = 12   # consecutive meaningful clips within this many minutes = one conversation (fallback)


def _fallback_group(clips):
    """Deterministic time-gap grouping: no model call, so it keeps the on-device path fully offline
    AND rescues the cloud path when its grouping request fails (previously such clips were wrongly
    counted as silent). `clips` arrive sorted by started_at."""
    groups, cur, prev = [], [], None
    for c in clips:
        try:
            t = datetime.datetime.fromisoformat(c["started_at"])
        except (TypeError, ValueError):
            t = prev
        if prev is not None and t is not None and (t - prev).total_seconds() > GROUP_GAP_MIN * 60:
            groups.append(cur); cur = []
        cur.append(c); prev = t or prev
    if cur:
        groups.append(cur)
    return [{"title": g[0]["ai"].get("title"), "files": [c["file"] for c in g]} for g in groups]


def _augment_conv_editions(conv, members):
    """Roll edition-specific per-clip fields up to the conversation: the Recall edition's talk-ratio +
    objections, and the Counsel edition's `matter` tag (see EDITION_SCHEMAS). No-op when clips don't
    carry them, so the generic Diary is unaffected."""
    objections, oseen = [], set()
    ratios = []
    matters = []
    for mm in members:
        ai = mm.get("ai") or {}
        mt = str(ai.get("matter") or "").strip()
        if mt:
            matters.append(mt)
        for ob in (ai.get("objections") or []):
            ol = str(ob).strip()
            if ol and ol.lower() not in oseen:
                oseen.add(ol.lower())
                objections.append(ol)
        tr = ai.get("talk_ratio")
        if isinstance(tr, (int, float)):
            ratios.append(float(tr))
    if objections:
        conv["objections"] = objections
    if ratios:
        conv["talk_ratio"] = round(sum(ratios) / len(ratios), 2)   # avg self-talk fraction across clips
    if matters:
        # The clips of one conversation can disagree (the tag is echoed per clip, and a clip may miss
        # it); take the most-echoed spelling, ties going to the one that appeared first, so the
        # conversation carries ONE matter rather than whichever clip happened to be last.
        best, best_n = matters[0], 0
        for cand in dict.fromkeys(matters):          # first-appearance order, deduped
            n = sum(1 for m in matters if m.lower() == cand.lower())
            if n > best_n:
                best, best_n = cand, n
        conv["matter"] = best


def _build_diary(metas, key=None, model=None):
    by_day = {}
    for m in metas:
        if not m.get("started_at"):
            continue
        by_day.setdefault(m["started_at"][:10], []).append(m)

    days = []
    for day in sorted(by_day, reverse=True):
        clips = sorted(by_day[day], key=lambda x: x["started_at"])
        by_name = {c["file"]: c for c in clips}
        meaningful = [c for c in clips if c.get("ai", {}).get("is_meaningful")]
        convs, day_summary = [], ""
        if meaningful:
            listing = "\n".join(
                "- %s | %s | %s | %s | %s :: %s" % (
                    c["file"], c["started_at"][11:16], c["ai"].get("category"),
                    c["ai"].get("who_with"), c["ai"].get("title"), c["ai"].get("summary"))
                for c in meaningful)
            try:
                if ANALYZE_ENGINE == "local":
                    obj, _m = local_ai.chat_json(          # on-device grouping, no network
                        DAY_SYS, "Date %s. Clips:\n%s" % (day, listing))
                else:
                    r = _post_json("/chat/completions", key, {
                        "model": model, "response_format": {"type": "json_object"},
                        "messages": [{"role": "system", "content": DAY_SYS},
                                     {"role": "user", "content": "Date %s. Clips:\n%s" % (day, listing)}]})
                    obj = json.loads(r["choices"][0]["message"]["content"])
                day_summary = obj.get("day_summary", "")
                raw_convs = obj.get("conversations", []) or _fallback_group(meaningful)
            except Exception as e:
                logger.warning("day %s grouping failed: %s", day, _err_text(e))
                raw_convs = _fallback_group(meaningful)   # deterministic, offline-safe
        else:
            raw_convs = []

        used = set()
        for cv in raw_convs:
            members = [by_name[f] for f in cv.get("files", []) if f in by_name]
            if not members:
                continue
            used.update(m["file"] for m in members)
            cat = members[0]["ai"].get("category", "unknown")
            actions, seen = [], set()
            topics, tseen = [], set()
            for mm in members:
                for a in (mm["ai"].get("action_items") or []):
                    if a and a not in seen:
                        seen.add(a)
                        actions.append(a)
                for tp in (mm["ai"].get("topics") or []):     # roll clip topics up to the conversation
                    tl = str(tp).strip()
                    if tl and tl.lower() not in tseen:
                        tseen.add(tl.lower())
                        topics.append(tl)
            conv = {
                "title": cv.get("title") or members[0]["ai"].get("title"),
                "time_range": cv.get("time_range") or _fmt_time(members[0]["started_at"]),
                "who_with": cv.get("who_with") or members[0]["ai"].get("who_with"),
                "summary": cv.get("summary") or members[0]["ai"].get("summary"),
                "category": cat,
                "icon": _CAT_ICON.get(cat, _CAT_ICON["unknown"]),
                "action_items": actions,
                "topics": topics,
                "clips": [{"time": _fmt_time(m["started_at"]), "title": m["ai"].get("title"),
                           "dur_min": (m.get("duration_sec", 0) or 0) // 60, "file": m["file"]}
                          for m in members],
            }
            _augment_conv_editions(conv, members)             # edition extras (Recall talk-ratio/objections)
            convs.append(conv)
        quiet = len([c for c in clips if c["file"] not in used
                     and not c.get("ai", {}).get("is_meaningful")])
        d = datetime.datetime.fromisoformat(day)
        days.append({"date": day, "heading": d.strftime("%A, %B %-d, %Y"),
                     "day_summary": day_summary, "conversations": convs, "quiet_count": quiet})
    return {"generated_at": datetime.datetime.now().isoformat(timespec="seconds"), "days": days}


# ----------------------------------------------------------------------------- standalone HTML (browser fallback)
def _esc(s):
    return html.escape(str(s or ""))


def _write_html(diary):
    blocks = []
    for day in diary["days"]:
        rows = []
        if day["day_summary"]:
            rows.append("<div class='daysum'>%s</div>" % _esc(day["day_summary"]))
        for cv in day["conversations"]:
            clips = "".join(
                "<div class='clip'><span class='ct'>%s</span><span class='cd'>%s</span>"
                "<span class='cdur'>%sm</span></div>" % (_esc(c["time"]), _esc(c["title"]), c["dur_min"])
                for c in cv["clips"])
            acts = ""
            if cv["action_items"]:
                acts = "<div class='acts'>" + "".join(
                    "<span class='act'>%s</span>" % _esc(a) for a in cv["action_items"]) + "</div>"
            rows.append(
                "<div class='conv'><div class='chead'><span class='cicon'>%s</span><div>"
                "<div class='ctitle'>%s</div><div class='cmeta'>%s &middot; %s</div></div></div>"
                "<div class='csum'>%s</div>%s%s</div>" % (
                    cv["icon"], _esc(cv["title"]), _esc(cv["time_range"]), _esc(cv["who_with"]),
                    _esc(cv["summary"]), acts, clips))
        if day["quiet_count"]:
            rows.append("<div class='quiet'>%d silent / test clip(s) hidden</div>" % day["quiet_count"])
        blocks.append("<section class='day'><h2>%s</h2>%s</section>" % (_esc(day["heading"]), "".join(rows)))
    open(DIARY_HTML, "w").write(_PAGE.replace("{{BODY}}", "\n".join(blocks) or "<p>No recordings yet.</p>"))


_PAGE = """<!DOCTYPE html><html><head><meta charset="utf-8"><title>Epiphani Diary</title>
<style>
 :root{--bg:#F4EFE4;--ink:#2A2D30;--muted:#8A8270;--line:#E4D9C2;--gold:#C8A872;--goldink:#B8860B;--card:#fff}
 *{box-sizing:border-box}body{margin:0;font-family:-apple-system,system-ui,sans-serif;color:var(--ink);
   background:radial-gradient(130% 100% at 50% -8%,#FBF7EE,#F4EFE4 70%);padding:34px 18px 60px}
 .wrap{max-width:720px;margin:0 auto}
 h1{font-family:Georgia,serif;font-weight:600;font-size:30px;margin:0 0 4px}h1 b{color:var(--goldink)}
 .lede{color:var(--muted);font-size:13px;margin:0 0 26px}
 .day{margin-bottom:30px}
 .day h2{font-size:14px;letter-spacing:.5px;text-transform:uppercase;color:var(--muted);
   border-bottom:1px solid var(--line);padding-bottom:7px;margin:0 0 12px}
 .daysum{font-size:14.5px;line-height:1.55;color:#4A4034;background:rgba(200,168,114,.1);
   border-left:3px solid var(--gold);border-radius:6px;padding:11px 14px;margin-bottom:16px}
 .conv{background:var(--card);border:1px solid var(--line);border-radius:13px;padding:15px 16px;margin-bottom:12px;
   box-shadow:0 1px 12px rgba(39,42,45,.05)}
 .chead{display:flex;gap:11px;align-items:flex-start}
 .cicon{font-size:21px;line-height:1.2}
 .ctitle{font-size:16px;font-weight:600}
 .cmeta{font-size:12px;color:var(--muted);margin-top:1px}
 .csum{font-size:13.5px;line-height:1.5;color:#4A4034;margin:9px 0 4px}
 .acts{display:flex;flex-wrap:wrap;gap:6px;margin:8px 0 4px}
 .act{font-size:11.5px;color:#6A5326;background:rgba(200,168,114,.18);border:1px solid var(--line);
   border-radius:20px;padding:3px 9px}
 .clip{display:flex;gap:10px;align-items:center;font-size:12.5px;padding:4px 0;border-top:1px solid #F1EAD9}
 .ct{color:var(--goldink);font-variant-numeric:tabular-nums;flex:0 0 58px}
 .cd{flex:1;color:#5A5142;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
 .cdur{color:var(--muted);flex:0 0 auto}
 .quiet{font-size:12px;color:var(--muted);font-style:italic;padding:6px 2px}
</style></head><body><div class="wrap">
 <h1>Your <b>Diary</b></h1>
 <p class="lede">Your recordings, transcribed and organized by what they actually are. Auto-generated by Epiphani.</p>
 {{BODY}}
</div></body></html>"""


if __name__ == "__main__":
    logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stderr)
    argv = sys.argv[1:]
    if "--engine" in argv:               # CLI override of BOTH dials at once (for testing)
        i = argv.index("--engine")
        TRANSCRIBE_ENGINE = ANALYZE_ENGINE = (argv[i + 1] if i + 1 < len(argv) else "cloud").lower()
        del argv[i:i + 2]
    if "--transcribe-engine" in argv:    # CLI override of just the transcription dial
        i = argv.index("--transcribe-engine")
        TRANSCRIBE_ENGINE = (argv[i + 1] if i + 1 < len(argv) else "cloud").lower()
        del argv[i:i + 2]
    if "--analyze-engine" in argv:       # CLI override of just the analysis/summarization dial
        i = argv.index("--analyze-engine")
        ANALYZE_ENGINE = (argv[i + 1] if i + 1 < len(argv) else "cloud").lower()
        del argv[i:i + 2]
    pos = [a for a in argv if not a.startswith("--")]
    if pos:                              # first positional = recordings dir
        REC_DIR = os.path.expanduser(pos[0])
        ORG_DIR = os.path.join(REC_DIR, ".organized")
        DIARY_JSON = os.path.join(REC_DIR, "diary.json")
        DIARY_HTML = os.path.join(REC_DIR, "Diary.html")
        LOCK = os.path.join(ORG_DIR, ".lock")
    sys.exit(run())
