"""Reproduce the article's CPV-72 sample with Python 3.10+ standard library.

Usage: python reproduce.py --output-dir tender-api-evaluation
Public data keeps its source terms. See the downloaded manifest and source links.
This script verifies file identity and selection, not source completeness.
"""

import argparse
import csv
import hashlib
import io
import json
from collections import Counter
from pathlib import Path
from urllib.request import Request, urlopen

BASE = "https://github.com/webtruffle/government-tenders-rfps/releases/download/2026-09-01/"
EXPECTED_CSV_SHA256 = "37ccad60bd020673c785d4c6c0b6b0af54efdbecde5a8dfc7df63efbc5b8d75c"
FIELDS = [
    "source", "source_id", "source_release_id", "source_url", "title",
    "buyer_name", "buyer_country", "record_stage", "status", "cpv_codes",
    "published_at", "source_updated_at", "deadline_at", "estimated_value",
    "currency", "document_urls", "first_seen_at", "last_seen_at",
]


def fetch(name):
    request = Request(BASE + name, headers={"User-Agent": "WebTruffle-Article-Evaluation/1.0"})
    with urlopen(request, timeout=90) as response:
        return response.read()


def make_sample(manifest_bytes, csv_bytes, output_dir):
    manifest = json.loads(manifest_bytes)
    declared = manifest["files"]["tenders.csv"]
    digest = hashlib.sha256(csv_bytes).hexdigest()
    if digest != EXPECTED_CSV_SHA256 or digest != declared["sha256"]:
        raise ValueError("CSV hash does not match the pinned article and release manifest")
    if len(csv_bytes) != declared["bytes"]:
        raise ValueError("CSV byte count does not match the manifest")
    reader = csv.DictReader(io.StringIO(csv_bytes.decode("utf-8-sig"), newline=""))
    missing = set(FIELDS) - set(reader.fieldnames or [])
    if missing:
        raise ValueError(f"Missing required columns: {sorted(missing)}")
    rows = list(reader)
    if len(rows) != manifest["record_count"]:
        raise ValueError("CSV row count does not match the manifest")
    selected = []
    for row in rows:
        codes = json.loads(row["cpv_codes"] or "[]")
        if not isinstance(codes, list) or any(not isinstance(c, str) for c in codes):
            raise ValueError("cpv_codes must contain a JSON array of strings")
        if any(code.startswith("72") for code in codes):
            selected.append(row)
    selected.sort(key=lambda row: (row["source"], row["source_id"], row["source_release_id"]))
    output_dir.mkdir(parents=True, exist_ok=True)
    with (output_dir / "sample.csv").open("w", encoding="utf-8", newline="") as stream:
        writer = csv.DictWriter(stream, fieldnames=FIELDS, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(selected)
    sample_bytes = (output_dir / "sample.csv").read_bytes()
    receipt = {
        "release_url": BASE.replace("/download/", "/tag/").rstrip("/"),
        "target_date": manifest["target_date"],
        "generated_at": manifest["generated_at"],
        "input_bytes": len(csv_bytes),
        "input_sha256": digest,
        "manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
        "input_rows": len(rows),
        "selection": "Any normalized cpv_codes element starts with 72; all record stages retained",
        "selected_rows": len(selected),
        "by_source": dict(sorted(Counter(row["source"] for row in selected).items())),
        "by_stage": dict(sorted(Counter(row["record_stage"] for row in selected).items())),
        "sample_sha256": hashlib.sha256(sample_bytes).hexdigest(),
        "limitations": [
            "One daily file, not the complete source population or the full provider-neutral query window",
            "CPV matching alone does not establish relevance, bid eligibility or a live deadline",
            "Record stage follows the published WebTruffle normalization",
            "No authenticated commercial-provider query was run for this comparison",
        ],
        "source_licenses": manifest.get("source_licenses", {}),
    }
    (output_dir / "sample-receipt.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8")
    return receipt


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--output-dir", type=Path, default=Path("tender-api-evaluation"))
    args = parser.parse_args()
    receipt = make_sample(fetch("manifest.json"), fetch("tenders.csv"), args.output_dir)
    print(json.dumps({key: receipt[key] for key in ["input_rows", "selected_rows", "by_source", "by_stage"]}, indent=2))


if __name__ == "__main__":
    main()