Does your town have a tower?

"""Rebuilds this insight's data files from HDB's public file.

    python prep/prep_tower.py              download HDB Property Information from data.gov.sg, then rebuild
    python prep/prep_tower.py path.csv     rebuild from a file you already have (no download)

Writes (next to this folder, in data/):
    raw/HDBPropertyInformation.csv   the file exactly as downloaded (only when downloading)
    dataset_metadata.json            data.gov.sg's own record of the file (name, last updated, coverage end)
    hdb_blocks.csv                   the residential blocks the page uses: one row per block
    prep_numbers.json                the headline numbers, counted here with pandas (the page counts them again
                                     with plain Python; build_page.py asserts the two agree)
Rule for "residential": the file's residential flag is Y and the block has at least one home.
"""
import csv, json, sys, urllib.request, time
from pathlib import Path
import pandas as pd

HERE = Path(__file__).resolve().parent
DATA = HERE.parent / "data"
DATASET = "d_17f5382f26140b1fdae0ba2ef6239d2f"
API = "https://api-open.data.gov.sg/v1/public/api/datasets/" + DATASET + "/poll-download"
META = "https://api-production.data.gov.sg/v2/public/api/datasets/" + DATASET + "/metadata"
# Town codes, from the legend in HDB's own description of the town column (data.gov.sg metadata)
TOWNS = {"AMK": "Ang Mo Kio", "BB": "Bukit Batok", "BD": "Bedok", "BH": "Bishan", "BM": "Bukit Merah",
         "BP": "Bukit Panjang", "BT": "Bukit Timah", "CCK": "Choa Chu Kang", "CL": "Clementi",
         "CT": "Central Area", "GL": "Geylang", "HG": "Hougang", "JE": "Jurong East", "JW": "Jurong West",
         "KWN": "Kallang/Whampoa", "MP": "Marine Parade", "PG": "Punggol", "PRC": "Pasir Ris",
         "QT": "Queenstown", "SB": "Sembawang", "SGN": "Serangoon", "SK": "Sengkang", "TAP": "Tampines",
         "TG": "Tengah", "TP": "Toa Payoh", "WL": "Woodlands", "YS": "Yishun"}
TOWER = 30


def get(url):
    # data.gov.sg refuses requests that carry no User-Agent (HTTP 403), so send a plain one
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (CompareSG insight prep)"})
    with urllib.request.urlopen(req, timeout=60) as r:
        return r.read()


def download():
    meta = json.loads(get(META))["data"]
    keep = {k: meta[k] for k in ("datasetId", "name", "managedBy", "lastUpdatedAt", "coverageStart", "coverageEnd", "format")}
    (DATA / "dataset_metadata.json").write_text(json.dumps(keep, indent=1), encoding="utf-8")
    for _ in range(10):
        j = json.loads(get(API))["data"]
        if j.get("status") == "DOWNLOAD_SUCCESS":
            break
        time.sleep(3)
    raw = get(j["url"])
    (DATA / "raw").mkdir(exist_ok=True)
    p = DATA / "raw" / "HDBPropertyInformation.csv"
    p.write_bytes(raw)
    return p


def main():
    src = Path(sys.argv[1]) if len(sys.argv) > 1 else download()
    d = pd.read_csv(src)
    r = d[(d.residential == "Y") & (d.total_dwelling_units > 0)].copy()
    assert r.bldg_contract_town.isin(TOWNS).all(), "a town code is missing from the legend"
    r = r.sort_values(["bldg_contract_town", "street", "blk_no"], kind="mergesort")
    with open(DATA / "hdb_blocks.csv", "w", newline="", encoding="utf-8") as f:
        w = csv.writer(f, lineterminator="\n")
        w.writerow(["blk_no", "street", "town_code", "max_floor_lvl", "year_completed", "total_dwelling_units"])
        for b in r.itertuples():
            w.writerow([b.blk_no, b.street, b.bldg_contract_town, int(b.max_floor_lvl), int(b.year_completed), int(b.total_dwelling_units)])
    towns = {}
    for code, x in r.groupby("bldg_contract_town"):
        top = x[x.max_floor_lvl == x.max_floor_lvl.max()]
        towns[TOWNS[code]] = dict(blocks=len(x), tallest=int(x.max_floor_lvl.max()), at_top=len(top),
                                  first_done=int(top.year_completed.min()), last_done=int(top.year_completed.max()),
                                  towers=int((x.max_floor_lvl >= TOWER).sum()))
    out = dict(records=len(d), residential=len(r), towers=int((r.max_floor_lvl >= TOWER).sum()),
               towns_with_towers=sum(1 for v in towns.values() if v["towers"] > 0),
               at_40_plus=int((r.max_floor_lvl >= 40).sum()), exactly_12=int((r.max_floor_lvl == 12).sum()),
               completed_2026=int((r.year_completed == 2026).sum()), towns=towns)
    (DATA / "prep_numbers.json").write_text(json.dumps(out, indent=1, sort_keys=True), encoding="utf-8")
    print("records", len(d), "residential", len(r), "towers", out["towers"], "wrote", DATA / "hdb_blocks.csv")


if __name__ == "__main__":
    main()