How many Singapore businesses use AI?

"""prep_ai.py - builds data/data.json for the CompareSG AI-by-sector page.
Run (inside the build folder):  python3 -I prep/prep_ai.py <out.json>
Inputs (all in data/):
  infocomm_usage.csv  SingStat table M651551 "Infocomm Usage By Enterprises, Annual" (IMDA data), downloaded from data.gov.sg on 13 Oct 2026.
  sgde2025.txt, sgde2026.txt  pdftotext -layout of IMDA's Singapore Digital Economy (SGDE) 2025 and 2026 publications (pages are split by form feed).
The sector rates and the other figures are typed below from the charts. Before anything is written, every typed number
is checked against the text of the page it comes from, the sector charts are checked in the order they print, the 2024 rates
are checked to be the same in both publications, and the all-business series is checked against the SingStat file. Standard library only."""
import csv, json, os, re, sys
from collections import Counter
HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
rd = lambda f: open(os.path.join(HERE, 'data', f), encoding='utf-8').read()
R25, R26 = rd('sgde2025.txt').split('\f'), rd('sgde2026.txt').split('\f')     # printed page n is R[n-1] in both
pg = lambda rep, n: rep[n - 1]
def has(rep, n, num, near=None):                    # the number is on that page (as a whole number, not part of a longer one)
    t = pg(rep, n)
    ok = re.search(r'(?<![\d.])' + re.escape(num) + r'(?![\d])', t) is not None
    assert ok and (near is None or near in t), ('not found', n, num)

# id, name, short name, "businesses in ...", 2023, 2024, 2025 (% of enterprises using AI). 2023 and 2024: SGDE 2025 Fig 8 p15. 2024 and 2025: SGDE 2026 Fig 11 p14.
SECTORS = [
 ('ic',     'Information & communications (I&C)', 'Info & comms',        'information and communications',  12.1, 35.9, 46.8),
 ('prof',   'Professional services',              'Professional',        'professional services',             5.1, 25.7, 34.5),
 ('fin',    'Finance & insurance',                'Finance',             'finance and insurance',            11.0, 22.6, 27.6),
 ('re',     'Real estate and admin & support',    'Real estate & admin', 'real estate and admin & support',   4.7, 18.0, 25.8),
 ('retail', 'Retail trade',                       'Retail',              'retail trade',                      4.4,  8.9, 18.9),
 ('mfg',    'Manufacturing',                      'Manufacturing',       'manufacturing',                     2.3,  8.9, 18.0),
 ('food',   'Accommodation & food services',      'Hotels & food',       'accommodation and food services',   2.3, 10.3, 17.7),
 ('whole',  'Wholesale trade',                    'Wholesale',           'wholesale trade',                   2.4,  6.6, 15.3),
 ('trans',  'Transportation & storage',           'Transport',           'transportation and storage',        1.8,  6.9, 11.4),
 ('constr', 'Construction',                       'Construction',        'construction',                      2.5,  7.4,  7.6),
]
# the order each chart prints its sectors, with the label the publication uses
ORDER26 = [('ic', 'I&C'), ('prof', 'Professional Services'), ('fin', 'Finance & Insurance'), ('re', 'Real Estate and Admin & Support'), ('retail', 'Retail Trade'),
           ('mfg', 'Manufacturing'), ('food', 'Accommodation & Food Services'), ('whole', 'Wholesale Trade'), ('trans', 'Transportation & Storage'), ('constr', 'Construction')]
ORDER25 = [('ic', 'I&C'), ('prof', 'Professional Services'), ('fin', 'Finance & Insurance'), ('re', 'Real Estate and Admin & Support'), ('food', 'Accommodation & Food Services'),
           ('mfg', 'Manufacturing'), ('retail', 'Retail Trade'), ('constr', 'Construction'), ('trans', 'Transport & Storage'), ('whole', 'Wholesale Trade')]
S = {s[0]: s for s in SECTORS}

def chart_numbers(rep, page, start, labels):        # the percentages in a chart, in print order, after checking the sector labels print in the expected order
    t = pg(rep, page); a = t.index(start); b = t.index('Source: IMDA', a); body = t[a:b]
    pos = [body.index(lab) for _, lab in labels]
    assert pos == sorted(pos), ('labels out of order', start)
    return [float(x) for x in re.findall(r'(\d+\.\d)%', body)]
n26 = chart_numbers(R26, 14, 'Figure 11:', ORDER26)
assert n26 == [x for k, _ in ORDER26 for x in (S[k][6], S[k][5])], ('Fig 11 (SGDE 2026) does not match', n26)       # each sector: 2025 then 2024
n25 = chart_numbers(R25, 15, 'FIGURE 8:', ORDER25)
assert n25 == [x for k, _ in ORDER25 for x in (S[k][5], S[k][4])], ('Fig 8 (SGDE 2025) does not match', n25)         # each sector: 2024 then 2023
assert Counter(n26[1::2]) == Counter(S[k][5] for k, _ in ORDER26)                                                    # 2024 rates: SGDE 2026 ...
assert Counter(n25[0::2]) == Counter(S[k][5] for k, _ in ORDER25)                                                    # ... equal SGDE 2025's

# all businesses: SingStat (IMDA data). Row "Business Usage Of Artificial Intelligence".
rows = list(csv.reader(open(os.path.join(HERE, 'data', 'infocomm_usage.csv'), newline='', encoding='utf-8')))
head = rows[0]; assert head[0] == 'DataSeries'
byname = {r[0]: dict(zip(head[1:], [float(x) for x in r[1:]])) for r in rows[1:]}
ai = {int(y): v for y, v in byname['Business Usage Of Artificial Intelligence'].items()}
assert sorted(ai) == list(range(2017, 2026))
# IMDA's SGDE 2026 (Fig 9, p12) agrees for 2024 and 2025. For 2023 it prints 4.3 where the SingStat file has 4.4; the page uses SingStat's.
has(R26, 12, '23.5'); has(R26, 12, '14.7'); has(R26, 12, '4.3'); assert ai[2025] == 23.5 and ai[2024] == 14.7 and ai[2023] == 4.4

size = {'sme': {2023: 4.2, 2024: 14.5, 2025: 23.4}, 'non': {2023: 44.0, 2024: 62.5, 2025: 70.4},
        'ten': {2021: 7.8, 2022: 8.8, 2023: 9.7, 2024: 21.4, 2025: 34.6}, 'all_imda': {2021: 3.3, 2022: 3.5, 2023: 4.3, 2024: 14.7, 2025: 23.5}}
for v in size['sme'].values(): has(R26, 11, '%.1f' % v)
for v in size['non'].values(): has(R26, 11, '%.1f' % v)
for v in list(size['ten'].values()) + list(size['all_imda'].values()): has(R26, 12, '%.1f' % v)
barriers = [('Do not see the need due to small business', 54.7, 57.4), ('No budget, or the investment is too high', 29.2, 35.1),
            ('IT infrastructure not ready for AI', 26.2, 39.5), ('Lack of skills or talent to use AI', 24.1, 30.8), ('Poor awareness or understanding of the technology', 19.1, 25.6)]
for lab, a, b in barriers: has(R26, 14, '%.1f' % a); has(R26, 14, '%.1f' % b)
plan = {2025: 12.1, 2023: 5.6}; has(R26, 15, '12.1'); has(R26, 15, '5.6')
tools = [('Commercial off-the-shelf AI tools, no customisation', 89.6), ('Commercial AI tools with some customisation', 28.3), ('AI tools built in-house or outsourced', 23.0)]
phases = [('Growing the use of AI across the company', 33.0), ('Fully deployed or integrated across the company', 6.6), ('Early testing', 37.9), ('Implementing a first use case', 22.6)]
benefits = [('Productivity and process improvement', 87.6), ('Cost reduction and resource optimisation', 44.0), ('Retaining and engaging existing customers', 28.7),
            ('Acquiring new customers', 27.8), ('Contribution to revenue increase', 24.5)]
for _, v in tools + phases: has(R26, 15, '%.1f' % v)
for _, v in benefits: has(R26, 16, '%.1f' % v)
prof = {'rows': [('Do not know how to use AI', 33.6, 28.2), ('Can use AI for general tasks', 43.5, 47.6), ('Can use AI for intermediate tasks', 17.8, 19.3), ('Can develop or deploy AI for complex tasks', 5.1, 5.0)], 'some': {2024: 66.4, 2025: 71.9}}
for _, a, b in prof['rows']: has(R26, 16, '%.1f' % a); has(R26, 16, '%.1f' % b)
has(R26, 16, '71.9'); has(R26, 16, '66.4')
oecd = [('Denmark', 42.0), ('Finland', 37.8), ('Sweden', 35.0), ('Singapore', 34.6), ('Belgium', 34.5), ('OECD average', 20.3)]
for name, v in oecd: assert re.search(name.split()[0] + r'\s+' + re.escape('%.1f' % v), pg(R26, 13)), name
digital = {k.replace('Business Usage Of ', ''): (byname[k]['2024'], byname[k]['2025']) for k in byname}

# checks on the shape of the data that the sentences on the page rely on
q = lambda x: round(x * 10)                          # tenths, so points are exact
gain = {k: q(S[k][6]) - q(S[k][5]) for k in S}
assert all(S[k][4] < S[k][5] < S[k][6] for k in S), 'every sector rose each year'
assert [s[0] for s in sorted(SECTORS, key=lambda s: -s[6])] == [s[0] for s in SECTORS], 'table is in 2025 order'
assert min(g for k, g in gain.items() if k != 'constr') == 45 and gain['constr'] == 2, 'nine sectors gained 4.5 points or more; construction gained 0.2'
doubled = [k for k in S if q(S[k][6]) >= 2 * q(S[k][5])]; assert sorted(doubled) == ['mfg', 'retail', 'whole'], doubled
close, cur = [], [SECTORS[0][0]]                    # sectors whose 2025 rate is within 1 point of the next one down form a group that is not ranked
for a, b in zip(SECTORS, SECTORS[1:]):
    if q(a[6]) - q(b[6]) < 10: cur.append(b[0])
    else: close.append(cur); cur = [b[0]]
close.append(cur); close = [c for c in close if len(c) > 1]; assert close == [['retail', 'mfg', 'food']], close
assert q(size['non'][2025]) > 2 * q(size['sme'][2025]) and q(ai[2025]) > q(byname['Business Usage Of Data Analytics']['2025']) > 0
assert q(byname['Business Usage Of Data Analytics']['2024']) > q(ai[2024]) and q(ai[2025]) > q(byname['Business Usage Of Data Analytics']['2025'])

out = {'sectors': [{'id': s[0], 'name': s[1], 'short': s[2], 'in': s[3], 'v': {'2023': s[4], '2024': s[5], '2025': s[6]}} for s in SECTORS],
       'all': {str(y): ai[y] for y in sorted(ai)}, 'size': {k: {str(y): v for y, v in d.items()} for k, d in size.items()},
       'barriers': barriers, 'plan': {str(k): v for k, v in plan.items()}, 'tools': tools, 'phases': phases, 'benefits': benefits,
       'prof': {'rows': prof['rows'], 'some': {str(k): v for k, v in prof['some'].items()}}, 'oecd': oecd, 'digital': digital,
       'facts': {'doubled': doubled, 'close': close[0], 'gain': gain}}
json.dump(out, open(sys.argv[1], 'w'), indent=1)
print('ok: sector charts match both publications in print order; 2024 rates agree; SingStat all-business series', [ai[y] for y in sorted(ai)])
print('gain, tenths of a point:', gain); print('more than doubled:', doubled, ' too close to order:', close[0])