"""Markdown report generation."""

from __future__ import annotations

import json
from collections import defaultdict
from datetime import datetime, timezone
from pathlib import Path

from .config import Config
from .db import Database
from .models import Draft, Rating, normalize_category

STAR = {1: "\u2581", 2: "\u2583", 3: "\u2585", 4: "\u2587", 5: "\u2588"}


def _bar(score: int) -> str:
    return STAR.get(score, "?")


def _score_str(rating: Rating) -> str:
    return f"{rating.composite_score:.1f}"


class Reporter:
    def __init__(self, config: Config | None = None, db: Database | None = None):
        self.config = config or Config.load()
        self.db = db or Database(self.config)
        self.output_dir = Path(self.config.data_dir) / "reports"
        self.output_dir.mkdir(parents=True, exist_ok=True)

    def overview(self) -> str:
        """Generate a sortable overview table of all rated drafts."""
        pairs = self.db.drafts_with_ratings()
        total = self.db.count_drafts()
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

        lines = [
            f"# IETF AI/Agent Draft Overview",
            f"*Generated {now} — {len(pairs)} rated / {total} tracked drafts*\n",
            "| Score | Draft | Date | N | M | O | Mom | R | Summary |",
            "|------:|-------|------|:-:|:-:|:-:|:---:|:-:|---------|",
        ]

        for draft, rating in pairs:
            name_link = f"[{draft.name}]({draft.datatracker_url})"
            lines.append(
                f"| {_score_str(rating)} | {name_link} | {draft.date} "
                f"| {_bar(rating.novelty)} | {_bar(rating.maturity)} "
                f"| {_bar(rating.overlap)} | {_bar(rating.momentum)} "
                f"| {_bar(rating.relevance)} | {rating.summary[:80]}... |"
            )

        lines.append("\n*N=Novelty, M=Maturity, O=Overlap, Mom=Momentum, R=Relevance (block height = score 1-5)*")

        report = "\n".join(lines)
        path = self.output_dir / "overview.md"
        path.write_text(report)
        return str(path)

    def landscape(self) -> str:
        """Generate a category-grouped landscape report."""
        pairs = self.db.drafts_with_ratings()
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

        # Group by category
        by_cat: dict[str, list[tuple[Draft, Rating]]] = {}
        for draft, rating in pairs:
            cats = rating.categories or ["Uncategorized"]
            for cat in cats:
                by_cat.setdefault(cat, []).append((draft, rating))

        lines = [
            f"# IETF AI/Agent Draft Landscape",
            f"*Generated {now}*\n",
        ]

        for cat in sorted(by_cat.keys()):
            items = by_cat[cat]
            items.sort(key=lambda x: x[1].composite_score, reverse=True)
            lines.append(f"\n## {cat} ({len(items)} drafts)\n")
            for draft, rating in items:
                lines.append(
                    f"- **[{draft.name}]({draft.datatracker_url})** "
                    f"(score: {_score_str(rating)}) — {rating.summary[:100]}"
                )

        report = "\n".join(lines)
        path = self.output_dir / "landscape.md"
        path.write_text(report)
        return str(path)

    def draft_detail(self, draft_name: str) -> str:
        """Generate a detailed report for a single draft."""
        draft = self.db.get_draft(draft_name)
        if draft is None:
            return ""

        rating = self.db.get_rating(draft_name)

        lines = [
            f"# {draft.title}",
            f"**{draft.name}** rev {draft.rev}\n",
            f"- **Date:** {draft.date}",
            f"- **Pages:** {draft.pages or '?'}",
            f"- **Group:** {draft.group or 'individual'}",
            f"- **Datatracker:** {draft.datatracker_url}",
            f"- **Text:** {draft.text_url}\n",
            f"## Abstract\n{draft.abstract}\n",
        ]

        if rating:
            lines.extend([
                f"## AI Assessment (score: {_score_str(rating)})\n",
                f"**Summary:** {rating.summary}\n",
                f"| Dimension | Score | Notes |",
                f"|-----------|:-----:|-------|",
                f"| Novelty | {rating.novelty}/5 | {rating.novelty_note} |",
                f"| Maturity | {rating.maturity}/5 | {rating.maturity_note} |",
                f"| Overlap | {rating.overlap}/5 | {rating.overlap_note} |",
                f"| Momentum | {rating.momentum}/5 | {rating.momentum_note} |",
                f"| Relevance | {rating.relevance}/5 | {rating.relevance_note} |",
                f"\n**Categories:** {', '.join(rating.categories) if rating.categories else 'none'}",
            ])
        else:
            lines.append("*Not yet rated — run `ietf analyze` to generate a rating.*")

        report = "\n".join(lines)
        path = self.output_dir / f"{draft_name}.md"
        path.write_text(report)
        return str(path)

    def digest(self, since_days: int = 7) -> str:
        """Generate a digest of recently fetched/updated drafts."""
        from datetime import timedelta
        cutoff = (datetime.now(timezone.utc) - timedelta(days=since_days)).isoformat()
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

        # Get recent drafts by fetched_at
        all_drafts = self.db.list_drafts(limit=500, order_by="fetched_at DESC")
        recent = [d for d in all_drafts if d.fetched_at and d.fetched_at >= cutoff]

        lines = [
            f"# Weekly Digest — IETF AI/Agent Drafts",
            f"*Generated {now} — showing drafts fetched in last {since_days} days*\n",
            f"**{len(recent)} drafts** in this period\n",
        ]

        if not recent:
            lines.append("No new drafts found. Run `ietf fetch` to update.")
        else:
            # Split into rated and unrated
            rated = []
            unrated = []
            for d in recent:
                r = self.db.get_rating(d.name)
                if r:
                    rated.append((d, r))
                else:
                    unrated.append(d)

            if rated:
                rated.sort(key=lambda x: x[1].composite_score, reverse=True)
                lines.append("## Top Rated New Drafts\n")
                for draft, rating in rated[:10]:
                    lines.append(
                        f"1. **[{draft.name}]({draft.datatracker_url})** "
                        f"(score: {_score_str(rating)}) — {rating.summary[:120]}"
                    )

            if unrated:
                lines.append(f"\n## Awaiting Analysis ({len(unrated)} drafts)\n")
                for d in unrated[:20]:
                    lines.append(f"- [{d.name}]({d.datatracker_url}) — {d.title}")

        report = "\n".join(lines)
        path = self.output_dir / "digest.md"
        path.write_text(report)
        return str(path)

    def timeline(self) -> str:
        """Generate a timeline report of draft submissions by month and category."""
        pairs = self.db.drafts_with_ratings(limit=500)
        all_drafts = self.db.list_drafts(limit=500, order_by="time ASC")
        total = len(all_drafts)
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

        # Group drafts by month
        by_month: dict[str, list[Draft]] = defaultdict(list)
        for d in all_drafts:
            month = d.time[:7] if d.time else "unknown"
            by_month[month].append(d)

        months = sorted(by_month.keys())

        # Build rating lookup by draft name
        rating_map: dict[str, Rating] = {}
        for draft, rating in pairs:
            rating_map[draft.name] = rating

        # Collect all categories
        all_cats: set[str] = set()
        for _, r in pairs:
            for c in r.categories:
                all_cats.add(c)
        cats = sorted(all_cats)

        # Category counts per month
        cat_by_month: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
        for d in all_drafts:
            month = d.time[:7] if d.time else "unknown"
            r = rating_map.get(d.name)
            if r:
                for c in r.categories:
                    cat_by_month[month][c] += 1

        lines = [
            "# IETF AI/Agent Drafts Timeline",
            f"*Generated {now} — {total} drafts across {len(months)} months*\n",
            "## Monthly Submission Volume\n",
            "```",
        ]

        max_count = max(len(ds) for ds in by_month.values()) if by_month else 1
        for month in months:
            count = len(by_month[month])
            bar_len = int(count / max_count * 40) if max_count else 0
            bar = "#" * bar_len
            lines.append(f"{month} | {bar:<40s} {count:>3}")

        lines.append("```\n")

        # Category breakdown table
        lines.append("## Category Breakdown by Month\n")
        header = "| Month   |" + " | ".join(f" {c[:12]:>12}" for c in cats) + " | Total |"
        sep = "|---------|" + " | ".join("-" * 13 for _ in cats) + " | -----:|"
        lines.append(header)
        lines.append(sep)

        for month in months:
            counts = [str(cat_by_month[month].get(c, 0)).rjust(13) for c in cats]
            total_m = len(by_month[month])
            lines.append(f"| {month} |" + " | ".join(counts) + f" | {total_m:>5} |")

        # Trends section
        lines.append("\n## Trends\n")
        if len(months) >= 4:
            mid = len(months) // 2
            early_months = months[:mid]
            late_months = months[mid:]

            early_cat: dict[str, int] = defaultdict(int)
            late_cat: dict[str, int] = defaultdict(int)
            for m in early_months:
                for c in cats:
                    early_cat[c] += cat_by_month[m].get(c, 0)
            for m in late_months:
                for c in cats:
                    late_cat[c] += cat_by_month[m].get(c, 0)

            growth = []
            for c in cats:
                e = early_cat[c]
                l = late_cat[c]
                if e > 0:
                    pct = ((l - e) / e) * 100
                    growth.append((c, pct, e, l))
                elif l > 0:
                    growth.append((c, float('inf'), e, l))

            growth.sort(key=lambda x: x[1], reverse=True)
            for c, pct, e, l in growth[:5]:
                if pct == float('inf'):
                    lines.append(f"- **{c}**: new (0 → {l} drafts)")
                else:
                    lines.append(f"- **{c}**: {pct:+.0f}% ({e} → {l} drafts, early vs late half)")
        else:
            lines.append("Not enough months for trend analysis.")

        report = "\n".join(lines)
        path = self.output_dir / "timeline.md"
        path.write_text(report)
        return str(path)

    def overlap_matrix(self, embedder) -> str:
        """Generate overlap matrix report from pairwise similarity."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

        names, matrix = embedder.similarity_matrix()
        n = len(names)

        # Build rating lookup
        pairs_data = self.db.drafts_with_ratings(limit=500)
        rating_map: dict[str, Rating] = {}
        draft_map: dict[str, Draft] = {}
        for draft, rating in pairs_data:
            rating_map[draft.name] = rating
            draft_map[draft.name] = draft

        # Top similar pairs (above 0.80, excluding self)
        sim_pairs: list[tuple[float, str, str]] = []
        for i in range(n):
            for j in range(i + 1, n):
                if matrix[i, j] >= 0.80:
                    sim_pairs.append((float(matrix[i, j]), names[i], names[j]))
        sim_pairs.sort(reverse=True)

        lines = [
            "# Overlap Matrix Report",
            f"*Generated {now} — {n}x{n} pairwise similarities*\n",
            f"## Top {min(50, len(sim_pairs))} Most Similar Pairs\n",
            "| Rank | Similarity | Draft A | Draft B |",
            "|-----:|-----------:|---------|---------|",
        ]

        for rank, (sim, a, b) in enumerate(sim_pairs[:50], 1):
            a_link = f"[{a}](https://datatracker.ietf.org/doc/{a}/)"
            b_link = f"[{b}](https://datatracker.ietf.org/doc/{b}/)"
            lines.append(f"| {rank} | {sim:.3f} | {a_link} | {b_link} |")

        # Per-category internal overlap
        cat_drafts: dict[str, list[int]] = defaultdict(list)
        name_idx = {name: i for i, name in enumerate(names)}
        for name in names:
            r = rating_map.get(name)
            if r:
                for c in r.categories:
                    if name in name_idx:
                        cat_drafts[c].append(name_idx[name])

        lines.extend([
            "\n## Per-Category Internal Overlap\n",
            "| Category | Drafts | Avg Pairwise Sim | Most Similar Pair |",
            "|----------|-------:|-----------------:|-------------------|",
        ])

        for cat in sorted(cat_drafts.keys()):
            indices = cat_drafts[cat]
            if len(indices) < 2:
                lines.append(f"| {cat} | {len(indices)} | — | — |")
                continue
            sims = []
            best_sim, best_a, best_b = 0.0, "", ""
            for ii in range(len(indices)):
                for jj in range(ii + 1, len(indices)):
                    s = float(matrix[indices[ii], indices[jj]])
                    sims.append(s)
                    if s > best_sim:
                        best_sim = s
                        best_a = names[indices[ii]]
                        best_b = names[indices[jj]]
            avg = sum(sims) / len(sims) if sims else 0
            short_a = best_a.replace("draft-", "")[:25]
            short_b = best_b.replace("draft-", "")[:25]
            lines.append(f"| {cat} | {len(indices)} | {avg:.3f} | {short_a} / {short_b} ({best_sim:.3f}) |")

        # Category cross-overlap matrix
        cat_names = sorted(cat_drafts.keys())
        if len(cat_names) > 1:
            lines.extend([
                "\n## Category Cross-Overlap\n",
                "Average similarity between drafts in different categories.\n",
                "| |" + " | ".join(c[:10] for c in cat_names) + " |",
                "|-|" + " | ".join("---:" for _ in cat_names) + " |",
            ])

            for ci, c1 in enumerate(cat_names):
                row = f"| **{c1[:12]}** |"
                for cj, c2 in enumerate(cat_names):
                    if ci > cj:
                        row += " |"
                        continue
                    idx1, idx2 = cat_drafts[c1], cat_drafts[c2]
                    sims = []
                    for i1 in idx1:
                        for i2 in idx2:
                            if i1 != i2:
                                sims.append(float(matrix[i1, i2]))
                    avg = sum(sims) / len(sims) if sims else 0
                    row += f" {avg:.2f} |"
                lines.append(row)

        # Most unique drafts
        lines.append("\n## Most Unique Drafts (max similarity < 0.70)\n")
        unique_drafts = []
        for i, name in enumerate(names):
            max_sim = 0.0
            best_match = ""
            for j in range(n):
                if i != j and matrix[i, j] > max_sim:
                    max_sim = float(matrix[i, j])
                    best_match = names[j]
            if max_sim < 0.70:
                d = draft_map.get(name)
                title = d.title[:60] if d else ""
                unique_drafts.append((name, max_sim, best_match, title))

        unique_drafts.sort(key=lambda x: x[1])
        if unique_drafts:
            for name, ms, bm, title in unique_drafts[:20]:
                lines.append(f"- **{name}** (max sim: {ms:.3f} with {bm}) — {title}")
        else:
            lines.append("No drafts with max similarity below 0.70.")

        report = "\n".join(lines)
        path = self.output_dir / "overlap-matrix.md"
        path.write_text(report)
        return str(path)

    def authors_report(self) -> str:
        """Generate author/organization network report."""
        from .orgs import normalize_org, detect_blocs, top_orgs_normalized, cross_org_normalized

        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        author_count = self.db.author_count()
        total_drafts = self.db.count_drafts()

        # Build rating lookup for category info
        pairs_data = self.db.drafts_with_ratings(limit=500)
        rating_map: dict[str, Rating] = {}
        for draft, rating in pairs_data:
            rating_map[draft.name] = rating

        lines = [
            "# Author & Organization Network",
            f"*Generated {now} — {author_count} unique authors across {total_drafts} drafts*\n",
        ]

        # ── Section 1: Top Authors ──
        top = self.db.top_authors(limit=30)
        lines.extend([
            "## Top Authors by Draft Count\n",
            "| # | Author | Organization | Drafts | Categories |",
            "|--:|--------|-------------|-------:|------------|",
        ])
        for rank, (name, aff, cnt, draft_names) in enumerate(top, 1):
            cats: set[str] = set()
            for dn in draft_names:
                r = rating_map.get(dn)
                if r:
                    cats.update(r.categories)
            cat_str = ", ".join(sorted(cats)[:3])
            lines.append(f"| {rank} | {name} | {normalize_org(aff)} | {cnt} | {cat_str} |")

        # ── Section 2: Top Organizations (normalized) ──
        orgs = top_orgs_normalized(self.db, limit=20)
        lines.extend([
            "\n## Top Organizations\n",
            "| # | Organization | Authors | Drafts |",
            "|--:|-------------|--------:|-------:|",
        ])
        for rank, (org, authors, drafts) in enumerate(orgs, 1):
            lines.append(f"| {rank} | {org} | {authors} | {drafts} |")

        # ── Section 3: Team Blocs ──
        blocs = detect_blocs(self.db, cohesion_threshold=0.70, min_size=2)
        bloc_pid_map: dict[int, int] = {}  # pid -> bloc index (for ALL blocs)

        # Register all bloc members for pair filtering (even small blocs)
        for i, bloc in enumerate(blocs):
            for pid, _, _ in bloc.members:
                bloc_pid_map[pid] = i

        # Only display blocs with >= 3 shared drafts (significant teams)
        display_blocs = [b for b in blocs if b.shared_drafts >= 3]
        if display_blocs:
            lines.extend([
                "\n## Team Blocs\n",
                "*Persistent co-author teams (>= 70% pairwise draft overlap, >= 3 shared drafts).*\n",
                "| # | Team | Members | Shared Drafts | Cohesion | Key Authors |",
                "|--:|------|--------:|-----:|--------:|------------|",
            ])
            for i, bloc in enumerate(display_blocs):
                names = ", ".join(
                    f"{n.split()[0][0]}. {n.split()[-1]}" if " " in n else n
                    for _, n, _ in bloc.members[:6]
                )
                if len(bloc.members) > 6:
                    names += f" +{len(bloc.members) - 6}"
                lines.append(
                    f"| {i + 1} | {bloc.label} | {len(bloc.members)} "
                    f"| {bloc.shared_drafts} | {bloc.cohesion:.0%} | {names} |"
                )

        # ── Section 4: Top Collaboration Pairs (cross-bloc) ──
        coauthors = self.db.coauthor_pairs()
        if coauthors:
            # Get pid mapping for filtering
            pid_lookup: dict[str, int] = {}
            for r in self.db.conn.execute("SELECT person_id, name FROM authors").fetchall():
                pid_lookup[r[1]] = r[0]

            lines.extend([
                "\n## Top Collaboration Pairs\n",
                "*Excluding intra-bloc pairs (see Team Blocs above).*\n",
                "| Author A | Author B | Shared Drafts |",
                "|----------|----------|-----:|",
            ])
            shown = 0
            for a, b, shared in coauthors:
                pid_a = pid_lookup.get(a)
                pid_b = pid_lookup.get(b)
                # Skip if both are in the same bloc
                if (pid_a is not None and pid_b is not None
                        and pid_a in bloc_pid_map and pid_b in bloc_pid_map
                        and bloc_pid_map[pid_a] == bloc_pid_map[pid_b]):
                    continue
                lines.append(f"| {a} | {b} | {shared} |")
                shown += 1
                if shown >= 20:
                    break

        # ── Section 5: Cross-Organization Collaboration (normalized) ──
        cross = cross_org_normalized(self.db, limit=15)
        if cross:
            lines.extend([
                "\n## Cross-Organization Collaboration\n",
                "| Org A | Org B | Shared Drafts |",
                "|-------|-------|-----:|",
            ])
            for org_a, org_b, shared in cross:
                lines.append(f"| {org_a} | {org_b} | {shared} |")

        report = "\n".join(lines)
        path = self.output_dir / "authors.md"
        path.write_text(report)
        return str(path)

    def ideas_report(self) -> str:
        """Generate report on extracted ideas across all drafts."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        all_ideas = self.db.all_ideas()

        # Build rating lookup for category info
        pairs_data = self.db.drafts_with_ratings(limit=500)
        rating_map: dict[str, Rating] = {}
        for draft, rating in pairs_data:
            rating_map[draft.name] = rating

        # Group ideas by normalized title for frequency analysis
        from difflib import SequenceMatcher
        idea_groups: list[dict] = []  # [{canonical, ideas: [idea], drafts: set}]

        for idea in all_ideas:
            title_lower = idea["title"].lower().strip()
            matched = False
            for group in idea_groups:
                ratio = SequenceMatcher(None, title_lower, group["canonical"]).ratio()
                if ratio >= 0.75:
                    group["ideas"].append(idea)
                    group["drafts"].add(idea["draft_name"])
                    matched = True
                    break
            if not matched:
                idea_groups.append({
                    "canonical": title_lower,
                    "title": idea["title"],
                    "ideas": [idea],
                    "drafts": {idea["draft_name"]},
                })

        idea_groups.sort(key=lambda g: len(g["drafts"]), reverse=True)
        drafts_with_ideas = len(set(i["draft_name"] for i in all_ideas))

        lines = [
            "# Technical Ideas Extracted from IETF AI/Agent Drafts",
            f"*Generated {now} — {len(all_ideas)} ideas from {drafts_with_ideas} drafts*\n",
        ]

        # Most common ideas (3+ drafts)
        common = [g for g in idea_groups if len(g["drafts"]) >= 3]
        if common:
            lines.extend([
                "## Most Common Ideas (appearing in 3+ drafts)\n",
                "| Idea | Appearances | Drafts |",
                "|------|------------:|--------|",
            ])
            for g in common:
                draft_list = ", ".join(sorted(g["drafts"])[:5])
                if len(g["drafts"]) > 5:
                    draft_list += f" +{len(g['drafts'])-5} more"
                lines.append(f"| {g['title']} | {len(g['drafts'])} | {draft_list} |")

        # Ideas appearing in 2 drafts
        two = [g for g in idea_groups if len(g["drafts"]) == 2]
        if two:
            lines.append(f"\n## Ideas Appearing in 2 Drafts ({len(two)} ideas)\n")
            for g in two[:30]:
                draft_list = ", ".join(sorted(g["drafts"]))
                lines.append(f"- **{g['title']}** — {draft_list}")

        # Unique ideas (only 1 draft) - just count and top examples
        unique = [g for g in idea_groups if len(g["drafts"]) == 1]
        lines.append(f"\n## Unique Ideas ({len(unique)} ideas appearing in only 1 draft)\n")
        for g in unique[:20]:
            idea = g["ideas"][0]
            lines.append(f"- **{g['title']}** ({idea['draft_name']}) — {idea['description'][:100]}")
        if len(unique) > 20:
            lines.append(f"\n*...and {len(unique) - 20} more unique ideas*")

        # By type
        by_type: dict[str, int] = defaultdict(int)
        for idea in all_ideas:
            by_type[idea.get("type", "other")] += 1
        if by_type:
            lines.extend(["\n## Ideas by Type\n"])
            for t, count in sorted(by_type.items(), key=lambda x: x[1], reverse=True):
                lines.append(f"- **{t or 'untyped'}**: {count}")

        report = "\n".join(lines)
        path = self.output_dir / "ideas.md"
        path.write_text(report)
        return str(path)

    def gaps_report(self) -> str:
        """Generate detailed gap analysis report with cross-references."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        gaps = self.db.all_gaps()
        total_drafts = self.db.count_drafts()
        all_ideas = self.db.all_ideas()

        # Build category -> (draft, rating) mapping (case-insensitive)
        pairs = self.db.drafts_with_ratings(limit=500)
        cat_drafts: dict[str, list[tuple[Draft, Rating]]] = defaultdict(list)
        for draft, rating in pairs:
            for c in rating.categories:
                cat_drafts[c.lower()].append((draft, rating))

        # Idea index
        idea_count_total = self.db.idea_count()

        lines = [
            "# Gap Analysis: IETF AI/Agent Draft Landscape",
            f"*Generated {now} — analyzing {total_drafts} drafts, "
            f"{idea_count_total} technical ideas*\n",
            "## Overview\n",
            f"This report identifies **{len(gaps)} gaps** — areas, problems, or "
            f"technical challenges not adequately addressed by the current {total_drafts} "
            f"IETF AI/agent drafts. Each gap is cross-referenced with related drafts "
            f"and extracted technical ideas to show partial coverage.\n",
        ]

        # Summary table
        severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
        gaps.sort(key=lambda g: severity_order.get(g["severity"], 4))

        by_sev: dict[str, int] = defaultdict(int)
        for g in gaps:
            by_sev[g["severity"]] += 1

        lines.extend([
            "| Severity | Count |",
            "|----------|------:|",
        ])
        for sev in ["critical", "high", "medium", "low"]:
            if by_sev[sev]:
                lines.append(f"| **{sev.upper()}** | {by_sev[sev]} |")
        lines.append("")

        # Safety deficit callout
        safety_count = len(cat_drafts.get("ai safety/alignment", []))
        protocol_count = len(cat_drafts.get("a2a protocols", []))
        netops_count = len(cat_drafts.get("autonomous netops", []))
        lines.extend([
            "### Safety Deficit\n",
            f"Only **{safety_count}** of {total_drafts} drafts address AI safety/alignment, "
            f"while **{protocol_count}** focus on A2A protocols and **{netops_count}** on "
            f"autonomous operations. The ratio of capability-building to safety is "
            f"roughly **{(protocol_count + netops_count) // max(safety_count, 1)}:1**.\n",
        ])

        # Detailed gap sections
        lines.append("---\n")

        for i, gap in enumerate(gaps, 1):
            sev = gap["severity"].upper()
            cat = gap["category"]
            cat_lower = cat.lower()
            related = cat_drafts.get(cat_lower, [])

            lines.extend([
                f"## {i}. {gap['topic']}",
                "",
                f"| | |",
                f"|---|---|",
                f"| **Severity** | {sev} |",
                f"| **Category** | {cat or 'cross-cutting'} |",
                f"| **Drafts in category** | {len(related)} |",
                "",
                f"{gap['description']}\n",
            ])

            if gap["evidence"]:
                lines.append(f"**Evidence:** {gap['evidence']}\n")

            # Related drafts (FTS search for gap topic keywords)
            keywords = [kw for kw in gap["topic"].replace("and", "").split() if len(kw) > 4]
            fts_results: list[Draft] = []
            seen: set[str] = set()
            for kw in keywords:
                try:
                    found = self.db.search_drafts('"' + kw + '"', limit=5)
                    for d in found:
                        if d.name not in seen:
                            seen.add(d.name)
                            fts_results.append(d)
                except Exception:
                    pass

            if fts_results or related:
                lines.append("### Related Drafts\n")

                if fts_results:
                    lines.append("**Keyword matches** (drafts mentioning gap topic):\n")
                    for d in fts_results[:6]:
                        r = self.db.get_rating(d.name)
                        score = f"score {r.composite_score:.1f}" if r else "unrated"
                        lines.append(
                            f"- [{d.name}]({d.datatracker_url}) ({score}) — {d.title[:80]}"
                        )
                    lines.append("")

                if related:
                    # Top-rated drafts in the same category
                    top_related = sorted(related, key=lambda x: x[1].composite_score, reverse=True)
                    lines.append(f"**Top-rated in {cat}** ({len(related)} drafts):\n")
                    for d, r in top_related[:5]:
                        lines.append(
                            f"- [{d.name}]({d.datatracker_url}) ({r.composite_score:.1f}) "
                            f"— {r.summary[:100]}"
                        )
                    lines.append("")

            # Related ideas
            gap_kws = set(kw.lower() for kw in gap["topic"].split() if len(kw) > 3)
            matching_ideas = []
            for idea in all_ideas:
                t = idea["title"].lower()
                d = idea["description"].lower()
                hits = sum(1 for kw in gap_kws if kw in t or kw in d)
                if hits >= 2:
                    matching_ideas.append((hits, idea))
            matching_ideas.sort(key=lambda x: x[0], reverse=True)

            if matching_ideas:
                lines.extend([
                    "### Partially Addressing Ideas\n",
                    f"{len(matching_ideas)} extracted ideas touch on this gap:\n",
                    "| Idea | Draft | Type |",
                    "|------|-------|------|",
                ])
                for _, idea in matching_ideas[:8]:
                    lines.append(
                        f"| {idea['title']} | {idea['draft_name']} | {idea.get('type', '')} |"
                    )
                if len(matching_ideas) > 8:
                    lines.append(f"\n*...and {len(matching_ideas) - 8} more*")
                lines.append("")
            else:
                lines.append("### Partially Addressing Ideas\n")
                lines.append("No directly related technical ideas found in current drafts — "
                             "this gap is entirely unaddressed.\n")

            lines.append("---\n")

        # Cross-cutting analysis
        lines.extend([
            "## Cross-Cutting Analysis\n",
            "### Gaps by Category\n",
            "| Category | Drafts | Gaps | Gap Topics |",
            "|----------|-------:|-----:|------------|",
        ])
        gap_by_cat: dict[str, list[str]] = defaultdict(list)
        for g in gaps:
            gap_by_cat[g["category"].lower()].append(g["topic"])

        all_cats = sorted(set(list(cat_drafts.keys()) + list(gap_by_cat.keys())))
        for cat in all_cats:
            draft_count = len(cat_drafts.get(cat, []))
            gap_topics = gap_by_cat.get(cat, [])
            if gap_topics:
                topics = "; ".join(gap_topics)
                lines.append(f"| {cat} | {draft_count} | {len(gap_topics)} | {topics} |")

        # Recommendations
        lines.extend([
            "\n## Recommendations\n",
            "Based on the gap analysis, the highest-impact areas for new standardization work:\n",
            "1. **Runtime behavior verification** — The most critical safety gap. "
            "Agents declare policies but nothing validates compliance at runtime.",
            "2. **Error recovery and rollback** — Autonomous operations need "
            "standardized failure handling before real deployment at scale.",
            "3. **Protocol interoperability layer** — 92 competing A2A protocols need "
            "a translation/negotiation framework to avoid fragmentation.",
            "4. **Dynamic trust systems** — Static certificates cannot scale to "
            "long-running agent ecosystems. Trust must be earned and revocable.",
            "5. **Human emergency override** — The 7:1 ratio of autonomous capability "
            "to human oversight drafts is concerning for production deployments.",
        ])

        report = "\n".join(lines)
        path = self.output_dir / "gaps.md"
        path.write_text(report)
        return str(path)

    def refs_report(self) -> str:
        """Generate cross-reference report — which RFCs/standards the agent ecosystem builds on."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        stats = self.db.ref_stats()
        total_drafts = self.db.count_drafts()

        # Build rating lookup for categories
        pairs_data = self.db.drafts_with_ratings(limit=500)
        rating_map: dict[str, Rating] = {}
        for draft, rating in pairs_data:
            rating_map[draft.name] = rating

        lines = [
            "# Cross-Reference Report: Standards Foundation of the AI Agent Ecosystem",
            f"*Generated {now} — {stats['drafts_with_refs']} of {total_drafts} drafts analyzed, "
            f"{stats['total_refs']} total references "
            f"({stats['rfc_refs']} RFC, {stats['draft_refs']} draft, {stats['bcp_refs']} BCP)*\n",
        ]

        # Well-known RFC names for context
        rfc_names = {
            "2119": "Key words (MUST/SHALL/MAY)",
            "8174": "Key words update",
            "8259": "JSON",
            "7519": "JWT",
            "6749": "OAuth 2.0",
            "7540": "HTTP/2",
            "9110": "HTTP Semantics",
            "7525": "TLS Recommendations",
            "8446": "TLS 1.3",
            "3986": "URIs",
            "7230": "HTTP/1.1 Message Syntax",
            "7231": "HTTP/1.1 Semantics",
            "8288": "Web Linking",
            "6125": "TLS Server Identity",
            "7515": "JWS",
            "7516": "JWE",
            "7517": "JWK",
            "7518": "JWA",
            "9449": "DPoP",
            "6750": "OAuth Bearer",
            "8725": "JWT Best Practices",
            "9396": "Rich Authorization Requests",
            "9101": "JAR (JWT-Secured Authorization)",
            "8414": "OAuth Server Metadata",
            "7591": "Dynamic Client Registration",
            "8705": "mTLS for OAuth",
            "9068": "JWT Access Tokens",
            "6819": "OAuth Threat Model",
            "9200": "ACE-OAuth",
            "9052": "COSE",
            "8392": "CWT",
            "7252": "CoAP",
        }

        # Top RFCs
        top_rfcs = self.db.top_referenced(ref_type="rfc", limit=40)
        if top_rfcs:
            lines.extend([
                "## Most-Referenced RFCs\n",
                "These are the standards building blocks of the IETF AI/agent ecosystem.\n",
                "| # | RFC | Name | Referenced By |",
                "|--:|-----|------|-------------:|",
            ])
            for rank, (rid, cnt, drafts) in enumerate(top_rfcs, 1):
                name = rfc_names.get(rid, "")
                lines.append(f"| {rank} | RFC {rid} | {name} | {cnt} drafts |")

            # Categorize the foundation
            lines.extend([
                "\n### Foundation Categories\n",
                "Grouping the most-referenced RFCs by function:\n",
            ])

            foundation_cats = {
                "Language/Conventions": ["2119", "8174"],
                "Data Formats": ["8259", "3986", "8288"],
                "HTTP": ["7540", "9110", "7230", "7231"],
                "Security/TLS": ["7525", "8446", "6125"],
                "OAuth/Identity": ["6749", "6750", "7519", "7515", "7516", "7517", "7518",
                                   "8725", "9449", "9396", "9101", "8414", "7591", "8705",
                                   "9068", "6819"],
                "IoT/Constrained": ["9200", "9052", "8392", "7252"],
            }

            rfc_set = {rid for rid, _, _ in top_rfcs}
            for fcat, rfcs in foundation_cats.items():
                hits = [r for r in rfcs if r in rfc_set]
                if hits:
                    total_refs = sum(cnt for rid, cnt, _ in top_rfcs if rid in hits)
                    rfc_list = ", ".join(f"RFC {r}" for r in hits)
                    lines.append(f"- **{fcat}** ({total_refs} total refs): {rfc_list}")

        # Top draft cross-references
        top_drafts = self.db.top_referenced(ref_type="draft", limit=20)
        if top_drafts:
            lines.extend([
                "\n## Most-Referenced Drafts\n",
                "These drafts are the most-cited within the AI/agent corpus itself.\n",
                "| # | Draft | Referenced By |",
                "|--:|-------|-------------:|",
            ])
            for rank, (rid, cnt, drafts) in enumerate(top_drafts, 1):
                lines.append(f"| {rank} | {rid} | {cnt} drafts |")

        # Drafts with most outgoing references
        ref_counts = self.db.ref_counts_by_draft()
        if ref_counts:
            lines.extend([
                "\n## Drafts With Most References (most well-connected)\n",
                "| # | Draft | RFCs | Drafts | BCPs | Total |",
                "|--:|-------|-----:|-------:|-----:|------:|",
            ])
            # Sort by total
            sorted_counts = sorted(ref_counts, key=lambda x: x[1] + x[2] + x[3], reverse=True)
            for rank, (dname, rfcs, drafts_cnt, bcps) in enumerate(sorted_counts[:20], 1):
                total = rfcs + drafts_cnt + bcps
                lines.append(f"| {rank} | {dname} | {rfcs} | {drafts_cnt} | {bcps} | {total} |")

        # Category analysis: which categories reference which RFCs most
        cat_rfc_counts: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
        for draft_name, rating in rating_map.items():
            draft_refs = self.db.get_refs_for_draft(draft_name)
            for rt, rid in draft_refs:
                if rt == "rfc":
                    for cat in rating.categories:
                        cat_rfc_counts[cat][rid] += 1

        if cat_rfc_counts:
            lines.extend([
                "\n## Category-Specific RFC Dependencies\n",
                "What each category builds on:\n",
            ])
            for cat in sorted(cat_rfc_counts.keys()):
                rfc_counts = cat_rfc_counts[cat]
                top_3 = sorted(rfc_counts.items(), key=lambda x: x[1], reverse=True)[:3]
                rfc_list = ", ".join(
                    f"RFC {rid} ({rfc_names.get(rid, '')})" if rfc_names.get(rid) else f"RFC {rid}"
                    for rid, _ in top_3
                )
                lines.append(f"- **{cat}**: {rfc_list}")

        report = "\n".join(lines)
        path = self.output_dir / "refs.md"
        path.write_text(report)
        return str(path)

    def trends_report(self) -> str:
        """Generate category trend analysis report with monthly breakdown and growth rates."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        pairs = self.db.drafts_with_ratings(limit=500)
        all_drafts = self.db.list_drafts(limit=500, order_by="time ASC")
        total = len(all_drafts)

        rating_map = {draft.name: rating for draft, rating in pairs}

        # Monthly counts per category
        monthly: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
        all_cats: set[str] = set()
        for d in all_drafts:
            month = d.time[:7] if d.time else "unknown"
            r = rating_map.get(d.name)
            if r:
                for c in r.categories:
                    monthly[month][c] += 1
                    all_cats.add(c)

        months = sorted(m for m in monthly.keys() if m != "unknown")
        cats = sorted(all_cats)

        lines = [
            "# Category Trend Analysis",
            f"*Generated {now} — {total} drafts, {len(months)} months, {len(cats)} categories*\n",
        ]

        # Growth summary
        recent_months = months[-3:] if len(months) >= 3 else months
        prev_months = months[-6:-3] if len(months) >= 6 else []

        lines.extend([
            "## Growth Summary\n",
            "| Category | Total | Last 3mo | Prev 3mo | Growth |",
            "|----------|------:|---------:|---------:|-------:|",
        ])

        cumulative: dict[str, int] = defaultdict(int)
        for month in months:
            for cat in cats:
                cumulative[cat] += monthly[month].get(cat, 0)

        for cat in cats:
            total_cat = cumulative[cat]
            recent = sum(monthly[m].get(cat, 0) for m in recent_months)
            prev = sum(monthly[m].get(cat, 0) for m in prev_months) if prev_months else 0
            if prev > 0:
                growth_str = f"{((recent - prev) / prev) * 100:+.0f}%"
            elif recent > 0:
                growth_str = "new"
            else:
                growth_str = "-"
            lines.append(f"| {cat} | {total_cat} | {recent} | {prev if prev_months else '-'} | {growth_str} |")

        # Monthly detail table
        lines.extend(["\n## Monthly Breakdown\n"])
        header = "| Month |" + " | ".join(f" {c[:15]}" for c in cats) + " | Total |"
        sep = "|-------|" + " | ".join("---:" for _ in cats) + " | -----:|"
        lines.append(header)
        lines.append(sep)

        for month in months:
            counts = [str(monthly[month].get(c, 0)) for c in cats]
            month_total = sum(monthly[month].values())
            lines.append(f"| {month} | " + " | ".join(counts) + f" | {month_total} |")

        # Half-over-half comparison
        if len(months) >= 4:
            mid = len(months) // 2
            early = months[:mid]
            late = months[mid:]

            lines.extend([
                "\n## Fastest Growing Categories (early vs late half)\n",
            ])

            growth_data = []
            for cat in cats:
                e = sum(monthly[m].get(cat, 0) for m in early)
                l = sum(monthly[m].get(cat, 0) for m in late)
                if e > 0:
                    pct = ((l - e) / e) * 100
                    growth_data.append((cat, pct, e, l))
                elif l > 0:
                    growth_data.append((cat, float("inf"), e, l))

            growth_data.sort(key=lambda x: x[1], reverse=True)
            for cat, pct, e, l in growth_data:
                if pct == float("inf"):
                    lines.append(f"- **{cat}**: new (0 -> {l} drafts)")
                else:
                    lines.append(f"- **{cat}**: {pct:+.0f}% ({e} -> {l} drafts)")

        report = "\n".join(lines)
        path = self.output_dir / "trends.md"
        path.write_text(report)
        return str(path)

    def idea_overlap_report(self, threshold: float = 0.75) -> str:
        """Generate cross-org idea overlap report."""
        from difflib import SequenceMatcher
        from .orgs import normalize_org

        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        all_ideas = self.db.all_ideas()
        total_drafts = self.db.count_drafts()

        # Build draft -> orgs mapping
        draft_orgs: dict[str, set[str]] = defaultdict(set)
        rows = self.db.conn.execute(
            """SELECT da.draft_name, a.affiliation
            FROM draft_authors da
            JOIN authors a ON da.person_id = a.person_id
            WHERE a.affiliation != ''"""
        ).fetchall()
        for r in rows:
            org = normalize_org(r["affiliation"])
            if org and org != "Independent":
                draft_orgs[r["draft_name"]].add(org)

        # Group similar ideas
        idea_groups: list[dict] = []
        for idea in all_ideas:
            title_lower = idea["title"].lower().strip()
            matched = False
            for group in idea_groups:
                ratio = SequenceMatcher(None, title_lower, group["canonical"]).ratio()
                if ratio >= threshold:
                    group["ideas"].append(idea)
                    group["drafts"].add(idea["draft_name"])
                    group["orgs"].update(draft_orgs.get(idea["draft_name"], set()))
                    matched = True
                    break
            if not matched:
                idea_groups.append({
                    "canonical": title_lower,
                    "title": idea["title"],
                    "ideas": [idea],
                    "drafts": {idea["draft_name"]},
                    "orgs": set(draft_orgs.get(idea["draft_name"], set())),
                })

        cross_org = [g for g in idea_groups if len(g["orgs"]) >= 2]
        cross_org.sort(key=lambda g: (-len(g["orgs"]), -len(g["drafts"])))

        lines = [
            "# Cross-Organization Idea Overlap",
            f"*Generated {now} — {len(all_ideas)} ideas from {total_drafts} drafts*\n",
            f"**{len(cross_org)}** ideas appear across 2+ organizations, "
            f"out of {len(idea_groups)} unique idea clusters.\n",
        ]

        if cross_org:
            # Top convergence points
            lines.extend([
                "## Top Convergence Points\n",
                "Ideas where the most organizations independently converge:\n",
                "| # | Idea | Orgs | Drafts | Organizations |",
                "|--:|------|-----:|-------:|---------------|",
            ])
            for rank, g in enumerate(cross_org[:30], 1):
                org_list = ", ".join(sorted(g["orgs"]))
                draft_list = ", ".join(sorted(g["drafts"])[:3])
                if len(g["drafts"]) > 3:
                    draft_list += f" +{len(g['drafts']) - 3}"
                lines.append(
                    f"| {rank} | {g['title']} | {len(g['orgs'])} | {len(g['drafts'])} | {org_list} |"
                )

            # Org-pair overlap heatmap
            org_pair_ideas: dict[tuple[str, str], list[str]] = defaultdict(list)
            for g in cross_org:
                orgs_sorted = sorted(g["orgs"])
                for i in range(len(orgs_sorted)):
                    for j in range(i + 1, len(orgs_sorted)):
                        key = (orgs_sorted[i], orgs_sorted[j])
                        org_pair_ideas[key].append(g["title"])

            top_pairs = sorted(org_pair_ideas.items(), key=lambda x: -len(x[1]))
            if top_pairs:
                lines.extend([
                    "\n## Most Overlapping Organization Pairs\n",
                    "| Org A | Org B | Shared Ideas | Examples |",
                    "|-------|-------|-------------:|----------|",
                ])
                for (org_a, org_b), ideas_list in top_pairs[:20]:
                    examples = "; ".join(ideas_list[:3])
                    if len(ideas_list) > 3:
                        examples += f" +{len(ideas_list) - 3}"
                    lines.append(f"| {org_a} | {org_b} | {len(ideas_list)} | {examples} |")

            # By idea type
            type_cross: dict[str, int] = defaultdict(int)
            for g in cross_org:
                for idea in g["ideas"]:
                    type_cross[idea.get("type", "other")] += 1
            if type_cross:
                lines.extend(["\n## Cross-Org Ideas by Type\n"])
                for t, count in sorted(type_cross.items(), key=lambda x: -x[1]):
                    lines.append(f"- **{t or 'untyped'}**: {count}")

        report = "\n".join(lines)
        path = self.output_dir / "idea-overlap.md"
        path.write_text(report)
        return str(path)

    def status_report(self) -> str:
        """Generate WG adoption status report — traction vs aspiration."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        all_status = self.db.draft_adoption_status()
        total = len(all_status)
        adopted = [s for s in all_status if s["wg_adopted"]]
        individual = [s for s in all_status if not s["wg_adopted"]]
        irtf = [s for s in all_status if s["stream"] == "irtf"]

        # Build rating/category lookup
        pairs = self.db.drafts_with_ratings(limit=500)
        rating_map: dict[str, Rating] = {}
        for draft, rating in pairs:
            rating_map[draft.name] = rating

        adopted_names = {s["name"] for s in adopted}

        lines = [
            "# WG Adoption Status Report",
            f"*Generated {now} — {total} drafts analyzed*\n",
            "## Overview\n",
            f"- **WG-adopted** (draft-ietf-*): **{len(adopted)}** ({len(adopted)/total*100:.1f}%)",
            f"- **IRTF** (draft-irtf-*): **{len(irtf)}**",
            f"- **Individual**: **{len(individual)}** ({len(individual)/total*100:.1f}%)\n",
        ]

        # WG breakdown
        wg_groups: dict[str, list[dict]] = defaultdict(list)
        for s in adopted:
            wg_groups[s["wg_name"]].append(s)

        lines.extend([
            "## Working Groups with AI/Agent Drafts\n",
            "| # | WG | Drafts | Draft Names |",
            "|--:|:---|-------:|-------------|",
        ])
        for rank, (wg_name, drafts) in enumerate(
            sorted(wg_groups.items(), key=lambda x: -len(x[1])), 1
        ):
            draft_names = ", ".join(d["name"] for d in drafts)
            lines.append(f"| {rank} | {wg_name} | {len(drafts)} | {draft_names} |")

        # Score comparison
        adopted_scores = [r.composite_score for d, r in pairs if d.name in adopted_names]
        individual_scores = [r.composite_score for d, r in pairs if d.name not in adopted_names]

        if adopted_scores and individual_scores:
            avg_adopted = sum(adopted_scores) / len(adopted_scores)
            avg_individual = sum(individual_scores) / len(individual_scores)
            lines.extend([
                "\n## Score Comparison: WG-Adopted vs Individual\n",
                "| Metric | WG-Adopted | Individual |",
                "|--------|----------:|----------:|",
                f"| Count (rated) | {len(adopted_scores)} | {len(individual_scores)} |",
                f"| Average Score | {avg_adopted:.2f} | {avg_individual:.2f} |",
                f"| Min Score | {min(adopted_scores):.2f} | {min(individual_scores):.2f} |",
                f"| Max Score | {max(adopted_scores):.2f} | {max(individual_scores):.2f} |",
            ])

            # Per-dimension comparison
            dims = ["novelty", "maturity", "overlap", "momentum", "relevance"]
            lines.extend([
                "\n### Per-Dimension Comparison\n",
                "| Dimension | WG-Adopted Avg | Individual Avg | Delta |",
                "|-----------|---------------:|---------------:|------:|",
            ])
            for dim in dims:
                adopted_dim = [getattr(r, dim) for d, r in pairs if d.name in adopted_names]
                individual_dim = [getattr(r, dim) for d, r in pairs if d.name not in adopted_names]
                if adopted_dim and individual_dim:
                    avg_a = sum(adopted_dim) / len(adopted_dim)
                    avg_i = sum(individual_dim) / len(individual_dim)
                    delta = avg_a - avg_i
                    lines.append(f"| {dim.capitalize()} | {avg_a:.2f} | {avg_i:.2f} | {delta:+.2f} |")

        # Category distribution comparison (normalize old verbose names)
        adopted_cats: dict[str, int] = defaultdict(int)
        individual_cats: dict[str, int] = defaultdict(int)
        for d, r in pairs:
            cat_dict = adopted_cats if d.name in adopted_names else individual_cats
            for c in r.categories:
                cat_dict[normalize_category(c)] += 1

        all_cats = sorted(set(list(adopted_cats.keys()) + list(individual_cats.keys())))
        if all_cats:
            lines.extend([
                "\n## Category Distribution: WG-Adopted vs Individual\n",
                "| Category | WG-Adopted | Individual | % WG |",
                "|----------|----------:|----------:|-----:|",
            ])
            for cat in all_cats:
                a = adopted_cats.get(cat, 0)
                i = individual_cats.get(cat, 0)
                pct = a / (a + i) * 100 if (a + i) > 0 else 0
                lines.append(f"| {cat} | {a} | {i} | {pct:.0f}% |")

        # Gap coverage
        gaps = self.db.all_gaps()
        if gaps:
            adopted_cat_set = {c.lower() for c in adopted_cats.keys()}
            lines.extend([
                "\n## Gap Coverage by WG-Adopted Work\n",
                "Do any WG-adopted drafts address the identified gaps?\n",
                "| Gap | Severity | WG Coverage |",
                "|-----|----------|-------------|",
            ])
            for g in gaps:
                cat_lower = g["category"].lower()
                has_wg = "Yes" if cat_lower in adopted_cat_set else "**No**"
                lines.append(f"| {g['topic']} | {g['severity'].upper()} | {has_wg} |")

        # List all WG-adopted drafts with scores
        lines.extend([
            "\n## All WG-Adopted Drafts\n",
            "| Draft | WG | Score | Categories | Title |",
            "|-------|:---|------:|------------|-------|",
        ])
        for s in sorted(adopted, key=lambda x: x["time"] or "", reverse=True):
            r = rating_map.get(s["name"])
            score = f"{r.composite_score:.1f}" if r else "-"
            cats = ", ".join(normalize_category(c) for c in r.categories[:2]) if r else ""
            lines.append(f"| {s['name']} | {s['wg_name']} | {score} | {cats} | {s['title'][:50]} |")

        report = "\n".join(lines)
        path = self.output_dir / "status.md"
        path.write_text(report)
        return str(path)

    def revisions_report(self) -> str:
        """Generate draft revision velocity report — who iterates vs fire-and-forget."""
        from .orgs import normalize_org

        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        all_revs = self.db.revision_velocity()
        total = len(all_revs)
        at_00 = sum(1 for r in all_revs if r["rev_int"] == 0)
        avg_rev = sum(r["rev_int"] for r in all_revs) / total if total else 0

        lines = [
            "# Draft Revision Velocity Report",
            f"*Generated {now} — {total} drafts analyzed*\n",
            "## Overview\n",
            f"- **Average revision**: {avg_rev:.2f}",
            f"- **At -00** (first draft only): **{at_00}** ({at_00/total*100:.1f}%)",
            f"- **Iterated** (rev >= 01): **{total - at_00}** ({(total-at_00)/total*100:.1f}%)",
            f"- **Highly iterated** (rev >= 05): **{sum(1 for r in all_revs if r['rev_int'] >= 5)}**\n",
        ]

        # Revision distribution
        rev_dist: dict[int, int] = defaultdict(int)
        for r in all_revs:
            rev_dist[r["rev_int"]] += 1

        lines.extend([
            "## Revision Distribution\n",
            "| Revision | Count | % |",
            "|:---------|------:|--:|",
        ])
        for rev in sorted(rev_dist.keys()):
            label = f"-{rev:02d}" if rev < 10 else f"-{rev}"
            cnt = rev_dist[rev]
            lines.append(f"| {label} | {cnt} | {cnt/total*100:.1f}% |")

        # Per-org analysis
        aff_rows = self.db.conn.execute(
            "SELECT da.draft_name, a.affiliation FROM draft_authors da "
            "JOIN authors a ON da.person_id = a.person_id "
            "WHERE a.affiliation != ''"
        ).fetchall()

        draft_rev = {r["name"]: r["rev_int"] for r in all_revs}
        org_drafts: dict[str, set[str]] = defaultdict(set)
        for row in aff_rows:
            norm = normalize_org(row["affiliation"])
            if norm:
                org_drafts[norm].add(row["draft_name"])

        org_stats = []
        for org_name, drafts in org_drafts.items():
            if len(drafts) < 3:
                continue
            revs = [draft_rev.get(d, 0) for d in drafts]
            n_00 = sum(1 for r in revs if r == 0)
            org_stats.append({
                "org": org_name,
                "drafts": len(drafts),
                "avg_rev": sum(revs) / len(revs),
                "at_00": n_00,
                "pct_00": n_00 / len(drafts) * 100,
                "max_rev": max(revs),
                "iterated": len(drafts) - n_00,
            })

        org_stats.sort(key=lambda x: -x["drafts"])

        lines.extend([
            "\n## Revision Velocity by Organization (>= 3 drafts)\n",
            "| # | Organization | Drafts | Avg Rev | At -00 | %-00 | Iterated | Max |",
            "|--:|:-------------|-------:|--------:|-------:|-----:|---------:|----:|",
        ])
        for rank, s in enumerate(org_stats, 1):
            lines.append(
                f"| {rank} | {s['org']} | {s['drafts']} | {s['avg_rev']:.2f} | "
                f"{s['at_00']} | {s['pct_00']:.0f}% | {s['iterated']} | {s['max_rev']} |"
            )

        # Fire-and-forget vs iterators
        high_00 = [s for s in org_stats if s["pct_00"] >= 70 and s["drafts"] >= 5]
        iterators = [s for s in org_stats if s["avg_rev"] >= 3.0 and s["drafts"] >= 3]

        if high_00:
            lines.extend([
                "\n## Fire-and-Forget Organizations (>= 70% at -00, >= 5 drafts)\n",
                "Organizations that submit many drafts but rarely iterate:\n",
            ])
            for s in sorted(high_00, key=lambda x: -x["pct_00"]):
                lines.append(f"- **{s['org']}**: {s['at_00']}/{s['drafts']} at -00 ({s['pct_00']:.0f}%), avg rev {s['avg_rev']:.2f}")

        if iterators:
            lines.extend([
                "\n## Active Iterators (avg revision >= 3.0)\n",
                "Organizations that actively revise and improve their drafts:\n",
            ])
            for s in sorted(iterators, key=lambda x: -x["avg_rev"]):
                lines.append(f"- **{s['org']}**: avg rev {s['avg_rev']:.1f}, max -{s['max_rev']:02d}, {s['drafts']} drafts")

        # Most-revised individual drafts
        top_revised = sorted(all_revs, key=lambda r: -r["rev_int"])[:20]
        lines.extend([
            "\n## Most-Revised Drafts (Top 20)\n",
            "| # | Rev | Name | Title |",
            "|--:|----:|:-----|:------|",
        ])
        for rank, r in enumerate(top_revised, 1):
            lines.append(f"| {rank} | -{r['rev_int']:02d} | {r['name']} | {r['title'][:50]} |")

        # Score comparison: iterated vs first-draft
        pairs = self.db.drafts_with_ratings(limit=500)
        if pairs:
            iterated_scores = [rat.composite_score for d, rat in pairs if draft_rev.get(d.name, 0) > 0]
            first_scores = [rat.composite_score for d, rat in pairs if draft_rev.get(d.name, 0) == 0]

            if iterated_scores and first_scores:
                lines.extend([
                    "\n## Score Comparison: Iterated vs First-Draft\n",
                    "| Metric | Iterated (rev > 00) | First-Draft (rev 00) |",
                    "|--------|--------------------:|---------------------:|",
                    f"| Count | {len(iterated_scores)} | {len(first_scores)} |",
                    f"| Average Score | {sum(iterated_scores)/len(iterated_scores):.2f} | {sum(first_scores)/len(first_scores):.2f} |",
                    f"| Max Score | {max(iterated_scores):.2f} | {max(first_scores):.2f} |",
                ])

        report = "\n".join(lines)
        path = self.output_dir / "revisions.md"
        path.write_text(report)
        return str(path)

    def centrality_report(self) -> str:
        """Generate author network centrality report."""
        import networkx as nx
        from .orgs import normalize_org

        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

        # Build co-authorship graph
        rows = self.db.conn.execute(
            """SELECT da1.person_id as p1, da2.person_id as p2, COUNT(*) as shared
            FROM draft_authors da1
            JOIN draft_authors da2 ON da1.draft_name = da2.draft_name
                AND da1.person_id < da2.person_id
            GROUP BY da1.person_id, da2.person_id"""
        ).fetchall()

        G = nx.Graph()
        for r in rows:
            G.add_edge(r[0], r[1], weight=r[2])

        persons = self.db.conn.execute(
            "SELECT person_id, name, affiliation FROM authors"
        ).fetchall()
        person_info = {r[0]: (r[1], normalize_org(r[2])) for r in persons}

        chinese_orgs = {
            "Huawei", "China Mobile", "China Telecom", "China Unicom",
            "ZTE Corporation", "Tsinghua University", "BUPT",
            "Pengcheng Laboratory", "CAICT", "AsiaInfo",
            "Zhongguancun Laboratory", "CNIC, CAS",
            "Tsinghua Shenzhen International Graduate School & Pengcheng Laboratory",
            "Huazhong University of Science and Technology",
        }

        # Edge classification
        cross_org = intra_org = cross_divide = 0
        for u, v in G.edges():
            _, org_u = person_info.get(u, ("?", ""))
            _, org_v = person_info.get(v, ("?", ""))
            if org_u and org_v:
                if org_u == org_v:
                    intra_org += 1
                else:
                    cross_org += 1
                    if (org_u in chinese_orgs) != (org_v in chinese_orgs):
                        cross_divide += 1

        total_edges = cross_org + intra_org
        avg_clustering = nx.average_clustering(G)
        components = list(nx.connected_components(G))
        bc = nx.betweenness_centrality(G)

        lines = [
            "# Author Network Centrality Report",
            f"*Generated {now} — {G.number_of_nodes()} authors, {G.number_of_edges()} co-authorship edges*\n",
            "## Network Overview\n",
            f"- **Authors (nodes)**: {G.number_of_nodes()}",
            f"- **Co-authorship edges**: {G.number_of_edges()}",
            f"- **Intra-org edges**: {intra_org} ({intra_org/total_edges*100:.1f}%)",
            f"- **Cross-org edges**: {cross_org} ({cross_org/total_edges*100:.1f}%)",
            f"- **Cross Chinese-Western edges**: {cross_divide}",
            f"- **Clustering coefficient**: {avg_clustering:.3f}",
            f"- **Connected components**: {len(components)}, largest: {len(max(components, key=len))}\n",
        ]

        # Top betweenness centrality
        top_bc = sorted(bc.items(), key=lambda x: -x[1])[:30]
        lines.extend([
            "## Top 30 Authors by Betweenness Centrality\n",
            "Betweenness centrality measures how often an author lies on the shortest path between "
            "other authors — high BC = key connector in the network.\n",
            "| # | Author | Organization | BC Score | Degree | CN Neighbors | Western Neighbors |",
            "|--:|:-------|:-------------|--------:|-------:|-------------:|------------------:|",
        ])
        for rank, (pid, score) in enumerate(top_bc, 1):
            name, org = person_info.get(pid, ("?", "?"))
            degree = G.degree(pid)
            cn = sum(1 for n in G.neighbors(pid) if person_info.get(n, ("", ""))[1] in chinese_orgs)
            west = sum(1 for n in G.neighbors(pid) if person_info.get(n, ("", ""))[1] not in chinese_orgs and person_info.get(n, ("", ""))[1])
            lines.append(f"| {rank} | {name} | {org} | {score:.4f} | {degree} | {cn} | {west} |")

        # Bridge-builders
        bridges = []
        for pid in G.nodes():
            name, org = person_info.get(pid, ("?", ""))
            cn = sum(1 for n in G.neighbors(pid) if person_info.get(n, ("", ""))[1] in chinese_orgs)
            west = sum(1 for n in G.neighbors(pid) if person_info.get(n, ("", ""))[1] not in chinese_orgs and person_info.get(n, ("", ""))[1])
            if cn > 0 and west > 0:
                bridges.append((pid, name, org, bc.get(pid, 0), cn, west))

        bridges.sort(key=lambda x: -x[3])
        lines.extend([
            f"\n## Cross-Divide Bridge-Builders ({len(bridges)} authors)\n",
            "Authors who co-author with people from both Chinese and Western organizations.\n",
            "| # | Author | Organization | BC Score | CN Neighbors | Western Neighbors |",
            "|--:|:-------|:-------------|--------:|-------------:|------------------:|",
        ])
        for rank, (pid, name, org, bc_score, cn, west) in enumerate(bridges[:20], 1):
            lines.append(f"| {rank} | {name} | {org} | {bc_score:.4f} | {cn} | {west} |")

        # Org-level centrality
        org_bc: dict[str, list[float]] = defaultdict(list)
        for pid, score in bc.items():
            _, org = person_info.get(pid, ("?", ""))
            if org:
                org_bc[org].append(score)

        org_avg_bc = [
            (org, sum(scores) / len(scores), max(scores), len(scores))
            for org, scores in org_bc.items()
            if len(scores) >= 3
        ]
        org_avg_bc.sort(key=lambda x: -x[1])

        lines.extend([
            "\n## Organization-Level Centrality (>= 3 authors)\n",
            "| # | Organization | Authors | Avg BC | Max BC |",
            "|--:|:-------------|--------:|-------:|-------:|",
        ])
        for rank, (org, avg, mx, cnt) in enumerate(org_avg_bc[:20], 1):
            lines.append(f"| {rank} | {org} | {cnt} | {avg:.4f} | {mx:.4f} |")

        report = "\n".join(lines)
        path = self.output_dir / "centrality.md"
        path.write_text(report)
        return str(path)

    def co_occurrence_report(self) -> str:
        """Generate category co-occurrence matrix report."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        pairs = self.db.drafts_with_ratings(limit=500)
        total = len(pairs)
        multi_cat = sum(1 for d, r in pairs if len(r.categories) > 1)

        # Build counts
        cat_counts: dict[str, int] = defaultdict(int)
        cooccur: dict[tuple[str, str], int] = defaultdict(int)
        for d, r in pairs:
            for c in r.categories:
                cat_counts[c] += 1
            for i, c1 in enumerate(r.categories):
                for c2 in r.categories[i + 1:]:
                    key = tuple(sorted([c1, c2]))
                    cooccur[key] += 1

        cats = sorted(cat_counts.keys(), key=lambda c: -cat_counts[c])

        lines = [
            "# Category Co-occurrence Report",
            f"*Generated {now} — {total} drafts, {multi_cat} ({multi_cat/total*100:.1f}%) multi-category*\n",
            "## Category Counts\n",
            "| Category | Count | % of Drafts |",
            "|:---------|------:|------------:|",
        ]
        for c in cats:
            lines.append(f"| {c} | {cat_counts[c]} | {cat_counts[c]/total*100:.1f}% |")

        # Co-occurrence matrix
        lines.extend([
            "\n## Co-occurrence Matrix\n",
            "Number of drafts assigned to both categories simultaneously.\n",
        ])

        # Build matrix header (abbreviated names)
        abbrev = {
            "A2A protocols": "A2A",
            "AI safety/alignment": "Safe",
            "ML traffic mgmt": "ML",
            "Autonomous netops": "Auto",
            "Agent identity/auth": "IdAuth",
            "Data formats/interop": "Data",
            "Policy/governance": "Policy",
            "Model serving/inference": "Model",
            "Agent discovery/reg": "Disc",
            "Human-agent interaction": "Human",
            "Other AI/agent": "Other",
        }

        header = "| |" + "|".join(f" {abbrev.get(c, c[:5])} " for c in cats) + "|"
        sep = "|:---|" + "|".join("---:" for _ in cats) + "|"
        lines.extend([header, sep])

        for c1 in cats:
            row = f"| **{abbrev.get(c1, c1[:5])}** |"
            for c2 in cats:
                if c1 == c2:
                    row += f" **{cat_counts[c1]}** |"
                else:
                    key = tuple(sorted([c1, c2]))
                    val = cooccur.get(key, 0)
                    row += f" {val if val > 0 else '-'} |"
            lines.append(row)

        # Top co-occurrences
        top_pairs = sorted(cooccur.items(), key=lambda x: -x[1])[:20]
        lines.extend([
            "\n## Top 20 Co-occurrences\n",
            "| # | Category A | Category B | Count | Jaccard Index |",
            "|--:|:-----------|:-----------|------:|--------------:|",
        ])
        for rank, ((c1, c2), n) in enumerate(top_pairs, 1):
            jaccard = n / (cat_counts[c1] + cat_counts[c2] - n)
            lines.append(f"| {rank} | {c1} | {c2} | {n} | {jaccard:.3f} |")

        # AI safety isolation analysis
        safety_cooccur = {k: v for k, v in cooccur.items() if "AI safety/alignment" in k}
        safety_total = cat_counts.get("AI safety/alignment", 0)
        lines.extend([
            f"\n## AI Safety/Alignment Coupling Analysis ({safety_total} drafts)\n",
            "Is AI safety structurally isolated from other categories?\n",
            "| Co-occurring Category | Count | % of Safety Drafts | Jaccard |",
            "|:----------------------|------:|-------------------:|--------:|",
        ])
        for (c1, c2), n in sorted(safety_cooccur.items(), key=lambda x: -x[1]):
            other = c2 if c1 == "AI safety/alignment" else c1
            other_total = cat_counts[other]
            jaccard = n / (safety_total + other_total - n)
            lines.append(f"| {other} | {n} | {n/safety_total*100:.1f}% | {jaccard:.3f} |")

        # Determine isolation verdict
        total_safety_links = sum(safety_cooccur.values())
        avg_links_per_safety = total_safety_links / safety_total if safety_total else 0
        lines.extend([
            f"\n**Verdict**: AI safety drafts have {total_safety_links} co-occurrence links across "
            f"{len(safety_cooccur)} categories ({avg_links_per_safety:.1f} avg links per safety draft). "
        ])
        if avg_links_per_safety > 1.5:
            lines.append("Safety is **well-coupled** with other categories, particularly policy/governance and identity/auth.")
        elif avg_links_per_safety > 0.8:
            lines.append("Safety is **moderately coupled** -- present but not deeply integrated.")
        else:
            lines.append("Safety is **structurally isolated** -- rarely co-occurs with other categories.")

        # Category clusters (categories that frequently appear together)
        lines.extend([
            "\n## Category Clusters\n",
            "Groups of categories that form natural clusters based on co-occurrence.\n",
        ])

        # Simple cluster detection: categories that co-occur more than expected
        for c in cats:
            strong_partners = []
            for (c1, c2), n in cooccur.items():
                if c not in (c1, c2):
                    continue
                other = c2 if c == c1 else c1
                expected = cat_counts[c] * cat_counts[other] / total
                lift = n / expected if expected > 0 else 0
                if lift > 1.5 and n >= 5:
                    strong_partners.append((other, n, lift))
            if strong_partners:
                strong_partners.sort(key=lambda x: -x[2])
                partner_str = ", ".join(f"{p} (lift {l:.1f}x)" for p, _, l in strong_partners[:3])
                lines.append(f"- **{c}** clusters with: {partner_str}")

        report = "\n".join(lines)
        path = self.output_dir / "co-occurrence.md"
        path.write_text(report)
        return str(path)

    def wg_report(self) -> str:
        """Generate working group analysis report."""
        now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
        summaries = self.db.wg_summary()
        spread = self.db.category_wg_spread()
        idea_overlaps = self.db.wg_idea_overlap()
        indiv_vs_wg = self.db.individual_vs_wg_categories()
        total = self.db.count_drafts()

        indiv_count = self.db.conn.execute(
            'SELECT COUNT(*) FROM drafts WHERE "group" = \'none\' OR "group" IS NULL'
        ).fetchone()[0]
        wg_count = total - indiv_count

        lines = [
            f"# Working Group Analysis",
            f"*Generated {now} — {total} drafts ({wg_count} WG-adopted, {indiv_count} individual)*\n",
        ]

        # WG summary table
        lines.extend([
            "## Working Group Overview\n",
            "| WG | Drafts | Ideas | Novelty | Maturity | Overlap | Momentum | Relevance |",
            "|:---|-------:|------:|--------:|---------:|--------:|---------:|----------:|",
        ])
        for s in summaries:
            lines.append(
                f"| **{s['wg']}** | {s['draft_count']} | {s['idea_count']} "
                f"| {s['avg_novelty']} | {s['avg_maturity']} | {s['avg_overlap']} "
                f"| {s['avg_momentum']} | {s['avg_relevance']} |"
            )

        # Category spread — where topics live across WGs
        multi_wg = [s for s in spread if s["wg_count"] >= 2
                    and not all(w["wg"] == "none" for w in s["wgs"])]
        if multi_wg:
            lines.extend([
                "\n## Cross-WG Category Spread\n",
                "Categories appearing in multiple WGs — potential coordination or alignment needed.\n",
                "| Category | WG Count | Total Drafts | WGs |",
                "|:---------|:--------:|-------------:|:----|",
            ])
            for s in multi_wg:
                real_wgs = [f"{w['wg']}({w['count']})" for w in s["wgs"] if w["wg"] != "none"]
                lines.append(
                    f"| {s['category']} | {s['wg_count']} | {s['total_drafts']} "
                    f"| {', '.join(real_wgs)} |"
                )

        # Idea overlap across WGs
        cross_wg_ideas = [o for o in idea_overlaps
                          if not all(w == "none" for w in o["wg_names"])]
        if cross_wg_ideas:
            lines.extend([
                "\n## Cross-WG Idea Overlap\n",
                "Same technical ideas appearing in different WGs — strongest signals for alignment.\n",
            ])
            for o in cross_wg_ideas[:30]:
                real_wgs = [w for w in o["wg_names"] if w != "none"]
                lines.append(f"### {o['idea_title']} ({len(real_wgs)} WGs: {', '.join(real_wgs)})\n")
                for entry in o["wgs"]:
                    if entry["wg"] != "none":
                        lines.append(f"- **[{entry['wg']}]** [{entry['draft_name']}]"
                                     f"(https://datatracker.ietf.org/doc/{entry['draft_name']}/) — "
                                     f"{entry['draft_title']}")
                lines.append("")

        # Individual vs WG comparison
        indiv = indiv_vs_wg["individual"]
        adopted = indiv_vs_wg["wg_adopted"]
        all_cats = sorted(set(list(indiv.keys()) + list(adopted.keys())))

        lines.extend([
            "\n## Individual vs WG-Adopted Distribution\n",
            "| Category | Individual | WG-Adopted | Assessment |",
            "|:---------|----------:|-----------:|:-----------|",
        ])
        consolidation_candidates = []
        for cat in all_cats:
            i_count = indiv.get(cat, 0)
            w_count = adopted.get(cat, 0)
            if i_count >= 5 and w_count == 0:
                assessment = "**Needs WG** — high individual activity, no WG home"
                consolidation_candidates.append((cat, i_count))
            elif i_count >= 3 and w_count >= 1:
                assessment = "WG exists — individual drafts could target it"
            elif w_count > i_count and i_count > 0:
                assessment = "WG leading"
            elif w_count == 0 and i_count > 0:
                assessment = "Individual only"
            else:
                assessment = "-"
            lines.append(f"| {cat} | {i_count} | {w_count} | {assessment} |")

        # Consolidation recommendations
        if consolidation_candidates:
            lines.extend([
                "\n## Consolidation Candidates\n",
                "Categories with significant individual draft activity but no WG — "
                "candidates for new WG charter or BoF.\n",
            ])
            for cat, count in sorted(consolidation_candidates, key=lambda x: x[1], reverse=True):
                lines.append(f"### {cat} ({count} individual drafts)\n")
                rows = self.db.conn.execute("""
                    SELECT d.name, d.title, r.summary FROM drafts d
                    JOIN ratings r ON d.name = r.draft_name
                    WHERE (d."group" = 'none' OR d."group" IS NULL)
                      AND r.categories LIKE ?
                    ORDER BY (r.novelty * 0.30 + r.relevance * 0.25 + r.maturity * 0.20
                              + r.momentum * 0.15 + (6 - r.overlap) * 0.10) DESC
                    LIMIT 8
                """, (f"%{cat}%",)).fetchall()
                for row in rows:
                    lines.append(
                        f"- [{row['name']}](https://datatracker.ietf.org/doc/{row['name']}/) — "
                        f"{row['title']}"
                    )
                lines.append("")

        # Submission targets
        lines.extend([
            "\n## Recommended Submission Targets\n",
            "For each category, the best WG to submit new work to.\n",
            "| Category | Best WG | Alternatives |",
            "|:---------|:--------|:-------------|",
        ])
        for s in spread:
            real_wgs = [w for w in s["wgs"] if w["wg"] != "none"]
            if not real_wgs:
                lines.append(f"| {s['category']} | *Individual submission* | - |")
            else:
                best = real_wgs[0]["wg"]
                alts = ", ".join(f"{w['wg']}({w['count']})" for w in real_wgs[1:3]) or "-"
                lines.append(f"| {s['category']} | **{best}** | {alts} |")

        report = "\n".join(lines)
        path = self.output_dir / "wg-analysis.md"
        path.write_text(report)
        return str(path)
