#!/usr/bin/env python3
"""Boutique Realty Perth provenance records: normalization, hashing and verification.

Published with the manifest so anyone can reproduce the hashes. Standard library only.

Record text (spec brp-provenance-text/1):
  1. Take the page HTML (WordPress source or the rendered page).
  2. Keep the first <h1>, the first <p> after it (the summary), and the inner HTML of the
     first <div class="brp-prose"> (the article body).
  3. Convert block elements (p, h1-h6, li, dt, dd, tr, br) to line breaks, drop all other tags,
     decode HTML entities, apply Unicode NFC.
  4. Fold typographic punctuation to ASCII (WordPress curls quotes when it renders a page):
     ‘ ’ ′ → '   “ ” ″ → "   – — → -   … → ...   × → x   non-breaking space → space.
  5. Collapse runs of whitespace within a line to one space, trim each line, drop empty lines,
     join with "\n" and end with one "\n". Encode as UTF-8.
  text_sha256 = SHA-256 of those bytes.

Usage:
  python3 brp_provenance.py verify <dir-with-manifest.json-and-records.json> [--live]
"""
import hashlib
import html
import json
import pathlib
import re
import subprocess
import sys
import unicodedata
import urllib.request

SPEC = 'brp-provenance-text/1'
FOLD = str.maketrans({'\u2018': "'", '\u2019': "'", '\u2032': "'", '\u201c': '"', '\u201d': '"',
                      '\u2033': '"', '\u2013': '-', '\u2014': '-', '\u2026': '...', '\u00d7': 'x',
                      '\u00a0': ' '})
BLOCK = re.compile(r'</?(p|h[1-6]|li|dt|dd|tr|ul|ol|dl|table|div|section)\b[^>]*>|<br\s*/?>', re.I)


def _inner_div(h, start):
    """Inner HTML of the <div> whose opening tag starts at `start`, honoring nesting."""
    open_end = h.index('>', start) + 1
    depth = 1
    for m in re.finditer(r'<(/?)div\b', h[open_end:], re.I):
        depth += -1 if m.group(1) else 1
        if depth == 0:
            return h[open_end:open_end + m.start()]
    raise ValueError('unbalanced brp-prose div')


def extract(h):
    h1 = re.search(r'<h1\b[^>]*>.*?</h1>', h, re.S | re.I)
    if not h1:
        raise ValueError('no h1')
    lede = re.search(r'<p\b[^>]*>.*?</p>', h[h1.end():], re.S | re.I)
    prose = re.search(r'<div class="brp-prose"[^>]*>', h)
    if not prose:
        raise ValueError('no brp-prose')
    return h1.group(0) + '\n' + (lede.group(0) if lede else '') + '\n' + _inner_div(h, prose.start())


def normalize(fragment):
    t = re.sub(r'<!--.*?-->', '', fragment, flags=re.S)
    t = BLOCK.sub('\n', t)
    t = re.sub(r'<[^>]+>', '', t)
    t = unicodedata.normalize('NFC', html.unescape(t)).translate(FOLD)
    lines = [re.sub(r'\s+', ' ', ln).strip() for ln in t.split('\n')]
    return '\n'.join(ln for ln in lines if ln) + '\n'


def record_text(page_html):
    return normalize(extract(page_html))


def sha256(b):
    return hashlib.sha256(b).hexdigest()


def fetch(url):
    req = urllib.request.Request(url, headers={'User-Agent': 'brp-provenance-verify/1'})
    return urllib.request.urlopen(req, timeout=60).read().decode('utf-8')


def verify(d, live=False):
    d = pathlib.Path(d)
    m = json.loads((d / 'manifest.json').read_text())
    rb = (d / 'records.json').read_bytes()
    ok = True
    if sha256(rb) != m['records_sha256']:
        print('FAIL records.json hash does not match manifest'); ok = False
    recs = {r['url']: r for r in json.loads(rb)['records']}
    for it in m['items']:
        r = recs.get(it['url'])
        if r is None or sha256(r['text'].encode('utf-8')) != it['text_sha256']:
            print('FAIL', it['url']); ok = False
            continue
        if live:
            got = sha256(record_text(fetch(it['url'])).encode('utf-8'))
            print('LIVE', 'same' if got == it['text_sha256'] else 'changed since manifest', it['url'])
    print(('PASS' if ok else 'FAIL'), f'{len(m["items"])} records checked against manifest')
    ots = d / 'manifest.json.ots'
    if ots.exists():
        try:
            r = subprocess.run(['ots', 'verify', str(ots)], capture_output=True, text=True, timeout=120)
            out = (r.stdout + r.stderr).strip()
            print('OTS', out.splitlines()[-1] if out else r.returncode)
            if 'does not match' in out:
                ok = False
        except FileNotFoundError:
            print('OTS client not installed: pip install opentimestamps-client, then: ots verify manifest.json.ots')
    print('RESULT', 'PASS' if ok else 'FAIL')
    return ok


if __name__ == '__main__':
    if len(sys.argv) >= 3 and sys.argv[1] == 'verify':
        sys.exit(0 if verify(sys.argv[2], '--live' in sys.argv) else 1)
    print(__doc__)