#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
harvest.py -- read what every producing script says about itself.

The corpus has ~77 scripts. Nobody can re-read them. But each one already
carries its own description: a module docstring, section titles passed to
rule(), the names of its checks, and in the later ones an explicit gap list.

This harvests that into one inventory, so the placement map in
docs/math-placement-map.md can be applied to material rather than to memory.

It reads source text only. It does NOT execute anything -- running 77 scripts
to find out what they are about would be both slow and a way to get a shell.

  python3 tools/harvest.py            -> docs/corpus-inventory.md
  python3 tools/harvest.py --gaps     -> just the gap register, to stdout
"""

import ast, os, re, sys, json

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OUT = os.path.join(ROOT, "docs", "corpus-inventory.md")

def books():
    for d in sorted(os.listdir(ROOT)):
        p = os.path.join(ROOT, d)
        if os.path.isdir(p) and (d.startswith("book") or d == "omega" or d == "tools"):
            yield d, p

def first_sentences(doc, n=3):
    """The first few non-empty, non-boilerplate lines of a docstring."""
    if not doc:
        return []
    out = []
    for ln in doc.strip().splitlines():
        t = ln.strip()
        if not t or set(t) <= set("-=_*"):
            continue
        if t.lower().startswith(("run:", "usage:", "run from", "python3")):
            continue
        out.append(t)
        if len(out) >= n:
            break
    return out

RULE_RE  = re.compile(r'rule\(\s*["\']([^"\']{4,90})["\']')
CHECK_RE = re.compile(r'check\(\s*["\']([^"\']{4,90})["\']')
SRC_RE   = re.compile(r'^\s*\[([A-Za-z][A-Za-z0-9]{0,3})\]\s{1,3}(\S.{0,70})', re.M)

def gaps_of(src):
    """Gap entries. Two shapes are in use across the corpus: a `gaps = [...]`
       list of tuples, and prose lines under an UNRESOLVED/OPEN heading."""
    found = []
    m = re.search(r'\ngaps\s*=\s*\[(.*?)\n\]', src, re.S)
    if m:
        for g in re.findall(r'\(\s*["\'](.+?)["\']\s*,', m.group(1), re.S):
            found.append(" ".join(g.split())[:150])
    for m2 in re.finditer(r'^\s*\("([^"]{15,200})",\s*$', src, re.M):
        t = " ".join(m2.group(1).split())[:150]
        if t not in found:
            found.append(t)
    return found

records = []
# os.walk, not os.listdir: wp125-verify.py's self-count caught that a
# non-recursive walk undercounts against `git ls-tree -r`, and these two tools
# measuring the same corpus must agree.
pairs = []
for book, path in books():
    for dirpath, _d, fs in os.walk(path):
        for f in sorted(fs):
            if f.endswith(".py"):
                pairs.append((book, os.path.relpath(os.path.join(dirpath, f), ROOT), os.path.join(dirpath, f)))
for book, rel, full in pairs:
    if True:
        f = rel
        try:
            src = open(full, encoding="utf-8", errors="ignore").read()
        except Exception as e:
            records.append(dict(book=book, file=f, path=rel, error=str(e))); continue
        doc = None
        try:
            doc = ast.get_docstring(ast.parse(src))
        except SyntaxError:
            pass
        records.append(dict(
            book=book, file=f, path=rel,
            lines=src.count("\n") + 1,
            summary=first_sentences(doc),
            sections=[" ".join(s.split()) for s in RULE_RE.findall(src) if s.strip()],
            checks=len(CHECK_RE.findall(src)),
            sources=[("[%s] %s" % (a, b.strip())) for a, b in SRC_RE.findall(src)][:6],
            gaps=gaps_of(src),
        ))

if "--gaps" in sys.argv:
    total = 0
    for r in records:
        if r.get("gaps"):
            print("\n## %s" % r["path"])
            for g in r["gaps"]:
                total += 1
                print("  - %s" % g)
    print("\n%d gap entries across %d scripts."
          % (total, sum(1 for r in records if r.get("gaps"))))
    sys.exit(0)

with_doc   = [r for r in records if r.get("summary")]
with_gaps  = [r for r in records if r.get("gaps")]
gap_count  = sum(len(r.get("gaps", [])) for r in records)
check_count= sum(r.get("checks", 0) for r in records)

L = []
L.append("# Corpus Inventory")
L.append("")
L.append("Generated by `tools/harvest.py` from source text. Not hand-written, and")
L.append("regenerating it is the only correct way to update it.")
L.append("")
L.append("| | |")
L.append("|---|---|")
L.append("| producing scripts | %d |" % len(records))
L.append("| with a module docstring | %d |" % len(with_doc))
L.append("| **without one** | **%d** |" % (len(records) - len(with_doc)))
L.append("| named checks | %d |" % check_count)
L.append("| scripts carrying an explicit gap list | %d |" % len(with_gaps))
L.append("| gap entries | %d |" % gap_count)
L.append("")
L.append("A script with no docstring cannot be placed by this tool, and is the")
L.append("first thing to fix -- not because documentation is virtuous but because")
L.append("an unplaceable script is one the corpus cannot reason about.")
L.append("")

for book, _ in books():
    rs = [r for r in records if r["book"] == book]
    if not rs:
        continue
    L.append("## %s  (%d scripts)" % (book, len(rs)))
    L.append("")
    for r in rs:
        L.append("### `%s`  ·  %d lines, %d checks" % (r["path"], r.get("lines", 0), r.get("checks", 0)))
        if r.get("summary"):
            for s in r["summary"]:
                L.append("> %s" % s)
        else:
            L.append("> **no docstring** — unplaceable until it says what it is for.")
        if r.get("sources"):
            L.append("")
            L.append("Sources: " + "; ".join(r["sources"]))
        if r.get("sections"):
            L.append("")
            L.append("Sections: " + " · ".join(r["sections"][:10]))
        if r.get("gaps"):
            L.append("")
            L.append("Open (%d):" % len(r["gaps"]))
            for g in r["gaps"][:8]:
                L.append("  - %s" % g)
        L.append("")

open(OUT, "w", encoding="utf-8").write("\n".join(L) + "\n")
json.dump(records, open(OUT.replace(".md", ".json"), "w", encoding="utf-8"), indent=1)
print("%d scripts, %d with docstrings, %d checks, %d gaps across %d scripts"
      % (len(records), len(with_doc), check_count, gap_count, len(with_gaps)))
print("wrote %s and .json" % os.path.relpath(OUT, ROOT))