#!/usr/bin/env python3 """Regenerate static catalogues and the NDSE2 companion from the current audited atlas. Python 3.10+, standard library only. No network or new research is performed by this builder. """ from __future__ import annotations import argparse, collections, csv, hashlib, html, io, json, os, re, shutil, unicodedata, zipfile from pathlib import Path from urllib.parse import quote, urlencode, urlsplit from build_ndse2_pages import BookCompanion, book_cover, publication_fields SOURCE=Path(__file__).resolve().parent ROOT=SOURCE.parent C=json.loads((SOURCE/'site_config.json').read_text(encoding='utf-8')) PUBLICATIONS=json.loads((ROOT/C['book_publications_json']).read_text(encoding='utf-8')) if C.get('book_publications_json') else {} PUBLICATIONS_BY_VOLUME={book['volume_id']:book for book in PUBLICATIONS.get('books',[])} BOOK_SERIES=PUBLICATIONS.get('series') or C.get('book_series',{}) ATLAS_SOURCE=SOURCE/C.get('atlas_source','atlas_v13_current.html') GRAPH_JSON=C.get('graph_json','data/graph/knowledge_graph_v13.json') GRAPH_GRAPHML=C.get('graph_graphml','data/graph/knowledge_graph_v13.graphml') BASE=ATLAS_SOURCE.read_text(encoding='utf-8') D=json.loads(re.search(r'',BASE,re.S).group(1)) BY={n['id']:n for n in D['nodes']} COLS={c['code']:c for c in C['collections']} SOURCES=[n for n in D['nodes'] if n.get('collection') in COLS] CITABLE_RECORDS=[n for n in SOURCES if n.get('collection')!='PAT'] PRACTICES=[n for n in D['nodes'] if n['kind']=='practice_descriptor'] OCCURRENCES=[n for n in D['nodes'] if n['kind']=='practice_occurrence'] FAMILIES=[n for n in D['nodes'] if n['kind']=='practice_family'] BOOKS=[n for n in D['nodes'] if n['kind']=='published_book'] COUNTS=collections.Counter(n.get('collection') for n in SOURCES) EXTERNAL_COUNT=sum(v for k,v in COUNTS.items() if k not in {'B','N','PAT'}) BOOK_VOLUME_IDS={n.get('attributes',{}).get('volume_id') for n in BOOKS} VOLUMES=[g for g in D['volumes'] if g['id'] in BOOK_VOLUME_IDS] EARLIER_VOLUME_IDS={v for n in SOURCES if n.get('collection')=='B' for v in n.get('volumes',[])} EARLIER_VOLUMES=[g for g in VOLUMES if g['id'] in EARLIER_VOLUME_IDS] BOOK_METADATA=C.get('book_json','source/ndse2_book.json') NDSE2=BookCompanion(ROOT/BOOK_METADATA,required='N' in COLS,metadata_link=BOOK_METADATA, collection_path='catalogues/'+COLS.get('N',{}).get('slug','ndse2')+'.html', publication=PUBLICATIONS_BY_VOLUME.get('V06')) RELEASE_NOTES=C.get('release_notes','UPDATE_20261018.html') RESEARCH_AUDIT=C.get('research_audit','data/updates/source_review_20261018.json') NAMES={n['id']:n['label'] for n in D['nodes']} NAMES.update({g['id']:g['label'] for k in ['themes','chapters','topics','volumes'] for g in D[k]}) ADJ=collections.defaultdict(list) for e in D['edges']: s=D['nodes'][e['source']]['id'];t=D['nodes'][e['target']]['id'];ADJ[s].append((t,e,'out')) if s!=t:ADJ[t].append((s,e,'in')) PAGES=[] def esc(x):return html.escape(str(x if x is not None else ''),quote=True) def slug(x): s=unicodedata.normalize('NFKD',x).encode('ascii','ignore').decode().lower() return re.sub(r'[^a-z0-9]+','-',s).strip('-') def rel(current,target): # target is a site-root-relative filename, possibly with query/fragment p,sep,f=target.partition('#');p,sepq,q=p.partition('?') out=os.path.relpath(p,os.path.dirname(current) or '.').replace(os.sep,'/') return out+('?' + q if sepq else '')+('#'+f if sep else '') def link(current,target,label,cls='',external=False): href=target if external else rel(current,target) return f'{label}' def safeurl(u): try:return u if urlsplit(u).scheme in ('http','https') and urlsplit(u).netloc else '' except:return '' def atlasurl(**params):return 'swe_knowledge_graph.html'+('?' + urlencode(params) if params else '') def collection_file(code):return f'catalogues/{COLS[code]["slug"]}.html' def record_file(n): if n.get('collection') in COLS:return collection_file(n['collection'])+'#'+quote(n['id'],safe='') if n['kind']=='published_book': g=next((g for g in VOLUMES if g['id']==n.get('attributes',{}).get('volume_id')),None) if g:return groupfile('volumes',g) if n['kind']=='practice_descriptor':return 'catalogues/practices.html#'+quote(n['id'],safe='') if n['kind']=='practice_occurrence':return 'catalogues/occurrences.html#'+quote(n['id'],safe='') if n['kind']=='practice_family':return atlasurl(collection='PAT',context='practices',grouping='family',node=n['id']) return atlasurl(node=n['id']) def groupfile(kind,g): return 'catalogues/'+kind+'/'+(g['id'].split(':')[-1].lower() if kind=='chapters' else g['id'].lower() if kind=='volumes' else slug(g['label']))+'.html' def named_group_link(page,gid,kind): groups=D[{'streams':'themes','chapters':'chapters','topics':'topics','volumes':'volumes'}[kind]] g=next((g for g in groups if g['id']==gid),None) return link(page,groupfile(kind,g),esc((f'Chapter {g["n"]}: ' if kind=='chapters' else '')+g['label'])) if g else esc(gid) def jsjson(obj):return json.dumps(obj,ensure_ascii=False).replace('<','\\u003c').replace('&','\\u0026') LOGO='' BOOK_CSS='''/* Book artwork and storefront links. Generated by source/build_site.py. */ .book-feature{display:grid;grid-template-columns:minmax(150px,235px) minmax(0,1fr);gap:34px;align-items:center;background:#fff;border:1px solid var(--line);border-radius:12px;padding:30px} .book-feature-copy{min-width:0}.book-feature h2{font-size:29px;line-height:1.2;max-width:750px}.book-feature .eyebrow{margin-bottom:12px}.book-feature p{font-size:14px;line-height:1.75}.book-feature .book-subtitle{font-size:17px;color:var(--muted)}.book-feature .book-author{font-size:12px;color:var(--muted)}.book-feature .note{font-size:12px} .book-cover{margin:0;text-align:center;min-width:0}.book-cover-link{display:block;border-radius:4px}.book-cover img{display:block;max-width:100%;width:auto;height:auto;max-height:370px;object-fit:contain;margin:0 auto}.book-cover figcaption{font-size:11px;color:var(--muted);line-height:1.6;margin-top:12px}.book-feature .actions{margin:22px 0 15px}.book-reference-links{display:flex;gap:10px 20px;flex-wrap:wrap;font-size:12px}.series-feature .book-cover img{max-height:250px}.book-feature .book-reference-links+p{margin-top:15px} .book-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}.book-listing{display:flex;flex-direction:column;min-width:0;background:#fff;border:1px solid var(--line);border-radius:9px;padding:23px}.book-listing h3{font-size:19px;line-height:1.4;margin:10px 0}.book-listing h3 a{color:var(--ink);text-decoration:none}.book-listing h3 a:hover{text-decoration:underline}.book-listing p{font-size:13px;line-height:1.7;color:var(--muted)}.book-listing .book-record-count{font-size:12px;margin-top:auto}.book-listing .actions{margin:18px 0 12px}.book-listing .book-reference-links{margin:0}.book-listing .pill{align-self:flex-start}.book-current{display:grid;grid-template-columns:minmax(130px,180px) minmax(0,1fr);gap:26px;align-items:center}.book-current .book-cover img{max-height:270px}.book-listing .book-copy{min-width:0} #book-volumes,#chapter-guides{scroll-margin-top:24px}.book-section-intro{font-size:14px;color:var(--muted);max-width:850px} .book-picker{border:1px solid var(--line);background:#fff;border-radius:8px;padding:16px 20px;margin:20px 0}.book-picker>summary{cursor:pointer;font-size:14px;font-weight:600;color:var(--accent)}.book-picker ul{list-style:none;padding:0;margin:15px 0 0}.book-picker li{display:flex;gap:12px 24px;justify-content:space-between;align-items:baseline;padding:12px 0;border-top:1px solid var(--line);font-size:13px}.book-picker .book-purchase-link{flex:none;font-weight:600}.book-picker .pill{margin-right:8px} @media(max-width:760px){.book-feature{grid-template-columns:170px minmax(0,1fr);gap:23px;padding:24px}.book-feature h2{font-size:25px}.book-feature .book-subtitle{font-size:15px}.book-grid{grid-template-columns:1fr}} @media(max-width:560px){.book-feature,.book-current{grid-template-columns:1fr;gap:24px;padding:22px}.book-feature .book-cover{max-width:210px;margin:0 auto}.series-feature .book-cover{max-width:245px}.book-current .book-cover{max-width:170px;margin:0 auto}.book-feature h2{font-size:26px}.book-feature .actions,.book-listing .actions{align-items:stretch}.book-feature .actions .button{flex:1 1 190px}.book-reference-links{row-gap:12px}.book-listing{padding:20px}.book-feature .book-cover img{max-height:315px}.book-picker li{flex-direction:column;gap:8px}} @media print{.book-feature,.book-listing{break-inside:avoid;box-shadow:none}.book-cover img{max-height:240px}.book-reference-links{display:none}} ''' def page(path,title,body,description='',active='Catalogues',wide=False): nav_items=[('Atlas','swe_knowledge_graph.html'),('Catalogues','catalogues/index.html')] if 'N' in COLS:nav_items.append(('NDSE2',collection_file('N'))) nav_items.extend([('About','about.html'),('Downloads','downloads.html')]) nav=''.join(link(path,t,esc(l)) if active!=l else f'{esc(l)}' for l,t in nav_items) s=f''' {esc(title)} · Knowledge Atlas
{link(path,'index.html',LOGO+'AI & Software EngineeringKnowledge Atlas','brand')}
{body}
''' p=ROOT/path;p.parent.mkdir(parents=True,exist_ok=True);p.write_text(s,encoding='utf-8');PAGES.append(path) def heading(title,kicker='Static catalogues',description=''): return f'

{esc(kicker)}

{esc(title)}

'+ (f'

{esc(description)}

' if description else '') def collection_nav(path,code=None): return '' def filter_bar(path,count,groups=None,field='streams',group_title='Stream'): opts='' if groups:opts=f'' return f'
{opts}

{count:,} records shown

' def fmt_meta(a,keys): vals=[] for key,title in keys: v=a.get(key) if v not in (None,'',[],{}):vals.append(f'
{esc(title)}
{esc(v if not isinstance(v,(dict,list)) else json.dumps(v,ensure_ascii=False))}
') return '
'+''.join(vals)+'
' if vals else '' def record(n,path): if n['kind']=='practice_descriptor':return practice_record(n,path) c=n.get('collection');a=n.get('attributes',{});col=COLS.get(c,{}) tone=f'var(--{c.lower()})' if c in COLS else 'var(--accent)' dat=' '.join(f'data-{key}=\'{esc(json.dumps(n.get(key,[]),ensure_ascii=False))}\'' for key in ['streams','topics','volumes','mapped','actual']) # Collection data are displayed as recorded. Never promote an inferred outcome or fill in a missing author. byline=n.get('authors') or a.get('authors') or a.get('author') or '' date=n.get('year') or a.get('publication_date') or a.get('year') or '' info=' · '.join(esc(x) for x in [byline,str(date)] if x) updated=a.get('updated_date') or a.get('revision_date') if updated and str(updated)!=str(date):info+=f' · Updated {esc(updated)}' evidence=a.get('reviewed_evidence_class') or a.get('evidence_class') or '' summary=n.get('summary') or a.get('contribution') or a.get('summary') or a.get('required_revision') or a.get('purpose') or '' limit=n.get('limitation') or a.get('limitation') or '' title=n['label'];anchor=quote(n['id'],safe='') pills=f'{esc(col.get("short",n["kind"].replace("_"," ")))}' if n.get('printed'):pills+='In the published NDSE2 bibliography' if a.get('access_status'):pills+=f'{esc(a["access_status"].replace("_"," "))}' top=f'
{esc(n["id"])}{pills}

{esc(title)}

'+ (f'

{info}

' if info else '') if c=='N' and a.get('title_it'):top+=f'

{esc(a["title_it"])}

' if summary:top+=f'

{esc(summary)}

' if n.get('streams'):top+='
'+''.join(named_group_link(path,g,'streams') for g in n['streams'])+'
' more='' if limit:more+=f'
Evidence boundary{esc(limit)}
' if a.get('summary') and a['summary']!=summary:more+=f'

Recorded summary. {esc(a["summary"])}

' if a.get('main_findings'):more+=f'

Reported findings / argument. {esc(a["main_findings"])}

' if evidence:more+=f'

Source type, as recorded: {esc(evidence)}

' if a.get('manual_use'):more+=f'

Editorial use. {esc(a["manual_use"])}

' if c=='W':more+='

Author names can be lead-author fields, not complete bylines. Follow the recorded source for the full citation. An index URL is labelled as an index, not silently replaced by an unverified primary-paper link.

' if c=='B': v=a.get('volume_id');more+='

Parent volume: '+(named_group_link(path,v,'volumes') if v else 'Not recorded')+'

' if c=='N': chapter=NDSE2.chapter(n['id']) if chapter: more+=NDSE2.chapter_content(chapter,path,link,BY,record_file,include_summary=False,include_evidence=False) top+='

'+link(path,NDSE2.chapter_path(chapter),'Chapter guide and cited sources →','card-link')+'

' if n['kind']=='practice_descriptor': more+=fmt_meta(a,[('required_revision','Recorded revision'),('disposition','Editorial disposition'),('family','Practice family'),('status','Record status')]) for occ in parsejson(a.get('occurrences','[]')): oid='OCC:'+occ if oid in BY:more+='

Historical occurrence: '+link(path,record_file(BY[oid]),esc(occ)+' — '+esc(BY[oid]['label']))+'

' more+=fmt_meta(a,[('source_scope','Collection scope'),('publication_status','Publication status'),('source_version_note','Source version'),('source_locator','Inspected passage'),('verification_scope','Inspection scope'),('source_review_scope','Review scope'),('review_scope','Mapping scope'),('access_note','Access note'),('source_path','Original source path'),('source_file','Original source file'),('source_line','Start line'),('source_end_line','End line'),('source_sha256','Original SHA-256'),('provenance','Provenance')]) if safeurl(a.get('latest_version_url','')):more+='

'+link(path,a['latest_version_url'],'Read the expanded source version ↗',external=True)+'

' if c in COLS and c not in {'N','PAT'}: actual=n.get('actual',[]);mapped=n.get('mapped',[]) more+='

Cited in published NDSE2 chapters

'+ ('
'.join(named_group_link(path,x,'chapters') for x in actual) if actual else 'No literal chapter citation recorded in the published edition.') more+='

Editorial chapter mappings

'+ ('
'.join(named_group_link(path,x,'chapters') for x in mapped) if mapped else 'None recorded.')+'

Editorial reading suggestions can extend beyond the published citations. Chapter guides show the outline and traceable source references.

' if n.get('topics'):more+='

Topics. '+' · '.join(named_group_link(path,g,'topics') for g in n['topics'])+'

' source_conn=[(BY[other],e,direction) for other,e,direction in ADJ[n['id']] if BY[other].get('collection') in COLS or (n['kind']=='practice_occurrence' and BY[other]['kind']=='practice_descriptor')] if source_conn: more+=f'
{len(source_conn)} recorded source relationships

These retain the graph’s exact relation labels. A connection is not automatically a causal claim or an independent replication.

' more+='
All preserved graph attributes
'+esc(json.dumps(a,ensure_ascii=False,indent=2))+'
' href=safeurl(n.get('url',''));actions=[] if href: lab='Open recorded index ↗' if href.startswith('https://www.sepolimi.com/p/se-ai-bibliography') else ('Open author’s companion ↗' if c=='B' else 'Open recorded source ↗') actions.append(link(path,href,lab,external=True)) args={'node':n['id']} if c in COLS:args={'collection':c,**args} elif n['kind']=='practice_occurrence':args={'collection':'PAT','context':'practices','grouping':'family',**args} actions.append(link(path,atlasurl(**args),'View in atlas →')) actions.append(link(path,record_file(n),'Permanent entry link')) return f'
{top}
Evidence, context & provenance{more}
'+''.join(actions)+'
' def parsejson(x): try:return json.loads(x) if isinstance(x,str) else x except:return [] def practice_family_id(n): families=n.get('families',[]) if families:return families[0] family=n.get('attributes',{}).get('family','') return family if family.startswith('FAMILY:') else 'FAMILY:'+family if family else '' def practice_list(title,values,ordered=False): if not values:return '' tag='ol' if ordered else 'ul' return f'

{esc(title)}

<{tag}>'+''.join('
  • '+esc(v)+'
    • ' for v in values)+f'
    ' def practice_record(n,path): """Render operational guidance without turning a descriptor into a cited article.""" a=n.get('attributes',{});p=a.get('practice',{});family=practice_family_id(n) family_node=BY.get(family,{}) family_name=a.get('family_name') or family_node.get('label') or family dat=' '.join(f'data-{key}=\'{esc(json.dumps(n.get(key,[]),ensure_ascii=False))}\'' for key in ['streams','topics','volumes','mapped','actual']) dat+=f' data-families=\'{esc(json.dumps([family] if family else []))}\'' top=f'
    {esc(n["id"])}PAT · PracticesEnglish guidance
    ' top+=f'

    {esc(n["label"])}

    ' top+=f'

    Family. '+link(path,'catalogues/practices.html#family-'+family.split(':')[-1],esc(family_name))+'

    ' if n.get('summary'):top+=f'

    {esc(n["summary"])}

    ' detail='' if p.get('purpose'):detail+=f'

    Purpose

    {esc(p["purpose"])}

    ' detail+='
    '+practice_list('When to use it',p.get('when_to_use',[]))+practice_list('When to avoid it',p.get('when_not_to_use',[]))+'
    ' detail+=practice_list('Before you start',p.get('prerequisites',[])) detail+=practice_list('How to apply it',p.get('steps',[]),ordered=True) if p.get('example'): example=re.sub(r'^Illustrative example:\s*','',p['example'],flags=re.I) detail+=f'

    Illustrative example

    {esc(example)}

    ' detail+='
    '+practice_list('What to check',p.get('verification',[]))+practice_list('Expected outputs',p.get('outputs',[]))+'
    ' detail+=practice_list('Risks and failure modes',p.get('risks',[])) chapters=[r for r in p.get('chapter_references',[]) if r.get('id') in BY] if chapters: detail+='

    Read with the published NDSE2 chapters

    These are editorial reading guides. A practice descriptor does not represent a literal source citation or an independent study.

    ' related=[BY[rid] for rid in p.get('related_practices',[]) if rid in BY and BY[rid]['kind']=='practice_descriptor' and rid!=n['id']] if related: detail+='

    Related practices

    Suggested companion techniques for comparison or combination; these links do not assert validated effectiveness.

    ' if p.get('editorial_note'):detail+=f'

    Editorial scope. {esc(p["editorial_note"])}

    ' operational='
    Apply this practice: steps, example and verification'+detail+'
    ' limit=f'
    Limits and evidence{esc(n.get("limitation",""))}
    ' if n.get('limitation') else '' provenance='

    The English guide expands the reconciled practice record. The example illustrates a possible application; it is not a reported experiment or an observed result. Historical occurrences trace the idea through the author’s books.

    ' provenance+=fmt_meta(a,[('required_revision','Editorial qualification'),('family_name','Practice family'),('practice_scope','Practice scope'),('source_scope','Source scope'),('source_version_note','Source version'),('source_locator','Source locator'),('source_path','Original source path'),('source_file','Original source file'),('source_line','Start line'),('source_end_line','End line'),('source_sha256','Original SHA-256'),('provenance','Provenance')]) occs=[] for raw in parsejson(a.get('occurrences','[]')): oid=raw if raw.startswith('OCC:') else 'OCC:'+raw if oid in BY:occs.append(BY[oid]) if occs: provenance+='

    Historical occurrences

    ' if n.get('streams'):provenance+='

    Editorial stream mappings. '+' · '.join(named_group_link(path,g,'streams') for g in n['streams'])+'

    ' if n.get('topics'):provenance+='

    Topics. '+' · '.join(named_group_link(path,g,'topics') for g in n['topics'])+'

    ' if a.get('historical_structural_chapters'):provenance+='

    Historical catalogue locators. '+esc(', '.join(a['historical_structural_chapters']))+'
    Retained structural IDs from the earlier catalogue; these are not literal citations in the current NDSE2 edition.

    ' provenance+='
    Complete graph attributes and retained history
    '+esc(json.dumps(a,ensure_ascii=False,indent=2))+'
    ' actions=link(path,atlasurl(collection='PAT',grouping='family',node=n['id']),'View in atlas →')+link(path,record_file(n),'Permanent entry link') return f'
    {top}{operational}{limit}
    Sources, scope and historical provenance{provenance}
    {actions}
    ' def practice_family_overview(path): out=f'

    Explore {len(FAMILIES)} practice families

    Choose a family to focus the catalogue, then open a practice for its procedure, example and checks. All entries remain readable without JavaScript.

    ' for family in FAMILIES: records=[n for n in PRACTICES if practice_family_id(n)==family['id']] first=records[0]['id'] if records else '' purpose=family.get('summary') or family.get('attributes',{}).get('purpose','') out+=f'
    {esc(family["id"].split(":")[-1])}{len(records)} practices

    {esc(family["label"])}

    {esc(purpose)}

    ' if records:out+=f'Explore this family →' out+='
    ' return out+'
    ' def build_practice_catalogue(): path='catalogues/practices.html';c=COLS.get('PAT',{}) title=c.get('title','Software Engineering Practices') desc=c.get('description',f'{len(PRACTICES)} English practice guides organized into {len(FAMILIES)} families, with practical steps, examples, checks, limits and traceable origins.') body='
    '+link(path,'index.html','Home')+' / '+link(path,'catalogues/index.html','Catalogues')+' / Practices
    ' body+=heading(title,f'PAT collection · {len(PRACTICES)} practices · {len(FAMILIES)} families',desc)+collection_nav(path,'PAT') body+='

    Each practice is an editorial engineering guide. Select a technique that addresses an observed need, adapt it to the available authority and environment, and use its verification criteria to judge the local result. Related techniques and repeated historical occurrences do not establish independent evidence of effectiveness.

    ' body+='
    '+link(path,atlasurl(collection='PAT',grouping='family'),'Explore the Practices collection','button primary')+link(path,'data/catalogues/practices.json','Complete JSON catalogue','button')+link(path,'data/catalogues/practices.csv','Detailed CSV catalogue','button')+link(path,'catalogues/occurrences.html','Historical occurrences','button')+'
    ' body+=practice_family_overview(path) body+='
    '+filter_bar(path,len(PRACTICES),FAMILIES,'families','Practice family')+'
    '+''.join(practice_record(n,path) for n in PRACTICES)+'
    ' page(path,title,body,desc) def write_dataset(name,records): obj={'meta':{'source':ATLAS_SOURCE.name,'snapshot':C['graph_snapshot'],'site_version':C['site_version'],'count':len(records),'note':'Materialized view of the current audited graph; complete node fields retained. Source inspection and publication status are recorded per item. The builder performs no new empirical validation.'},'records':records} p=ROOT/f'data/catalogues/{name}.json';p.write_text(json.dumps(obj,ensure_ascii=False,indent=2)+'\n',encoding='utf-8') keys=['id','title','authors_as_recorded','date_as_recorded','collection','kind','summary','limitation','recorded_url','streams','topics','actual_chapter_citations','editorial_chapter_mappings','source_path','source_sha256','title_it','publication_date','updated_date','publication_status','parent_volume'] practice_keys=['practice_family','practice_family_name','practice_language','purpose','when_to_use','when_not_to_use','prerequisites','steps','example','verification','outputs','risks','related_practices','editorial_note','chapter_references','historical_structural_chapters'] if any(n['kind']=='practice_descriptor' for n in records):keys+=practice_keys with (ROOT/f'data/catalogues/{name}.csv').open('w',encoding='utf-8-sig',newline='') as f: w=csv.DictWriter(f,fieldnames=keys);w.writeheader() for n in records: a=n.get('attributes',{});row=dict(zip(keys,[n['id'],n['label'],n.get('authors',''),n.get('year',''),n.get('collection',''),n['kind'],n.get('summary',''),n.get('limitation',''),n.get('url',''),'; '.join(n.get('streams',[])),'; '.join(n.get('topics',[])),'; '.join(n.get('actual',[])),'; '.join(n.get('mapped',[])),a.get('source_path',a.get('source_file','')),a.get('source_sha256',''),a.get('title_it',''),a.get('publication_date',''),a.get('updated_date',a.get('revision_date','')),a.get('publication_status',''),a.get('volume_id','')])) if n['kind']=='practice_descriptor': practice=a.get('practice',{}) row.update(practice_family=practice_family_id(n),practice_family_name=a.get('family_name',''),practice_language=practice.get('language','')) for key in ['purpose','when_to_use','when_not_to_use','prerequisites','steps','example','verification','outputs','risks','related_practices','editorial_note']: value=practice.get(key,'') row[key]='\n'.join(f'{i+1}. {v}' for i,v in enumerate(value)) if isinstance(value,list) else value row['chapter_references']='\n'.join(' · '.join(str(ref.get(k,'')) for k in ['id','title','role']) for ref in practice.get('chapter_references',[])) row['historical_structural_chapters']='; '.join(a.get('historical_structural_chapters',[])) # Protect spreadsheet users against formula interpretation; JSON is byte-faithful semantically. for k,v in row.items(): if isinstance(v,str) and v.startswith(('=','+','-','@','\t','\r')):row[k]="'"+v w.writerow(row) def collection_card(c,path): count=sum(n.get('collection')==c['code'] for n in SOURCES);tone=f'var(--{c["code"].lower()})' return f'
    {c["code"]} COLLECTION{count} records

    '+link(path,collection_file(c['code']),esc(c['title']))+f'

    {esc(c["description"])}

    '+link(path,collection_file(c['code']),'Read catalogue →','card-link')+'
    ' def profile(path,big=False): return f'
    Author photograph of Enrico Papalini from his published author biography

    About the author

    Enrico Papalini

    {esc(C["bio"])}

    {esc(C["bio_note"])}

    '+link(path,'about.html','About this project →','card-link')+'
    ' def ack(path): return '' def grouped_sources(path,records,kind=None): by=collections.defaultdict(list) for n in records:by[n.get('collection')].append(n) text='' for c in COLS: if not by[c]:continue text+=f'

    {esc(COLS[c]["title"])} {len(by[c])}

    ' contextual=[n for n in records if n.get('collection') not in COLS] if contextual: text+='

    Book and contextual references

    ' return text or '

    No source membership recorded in this view.

    ' def actual_sources_for_chapter(group): chapter=NDSE2.chapter(group['id']) if chapter: return [BY[sid] for sid in NDSE2.cited_ids(chapter) if sid in BY and BY[sid].get('collection') not in {'N','PAT'}] return [n for n in CITABLE_RECORDS if n.get('collection')!='N' and group['id'] in n.get('actual',[])] def book_attributes(book): # Storefront updates never replace titles, ISBNs or chapter/citation metadata. attributes=dict(book.get('attributes',{})) publication=PUBLICATIONS_BY_VOLUME.get(attributes.get('volume_id'),{}) if publication.get('node_id') and publication['node_id']!=book['id']: raise ValueError(f'Publication overlay node does not match volume: {book["id"]}') attributes.update(publication_fields(publication)) return attributes def book_actions(path,book,*,include_details=False): attributes=book_attributes(book);actions='' destination=safeurl(attributes.get('amazon_url','')) if destination:actions+=link(path,destination,'View this book on Amazon.com ↗','button primary',external=True) if include_details:actions+=link(path,record_file(book),'Chapter and source catalogue','button') result='
    '+actions+'
    ' if actions else '' companion=safeurl(attributes.get('companion_url','')) if not companion and attributes.get('url_role')=='author_companion':companion=safeurl(book.get('url','')) if companion:result+='
    '+link(path,companion,'Open the author’s companion ↗',external=True)+'
    ' return result def series_presentation(path): if not BOOK_SERIES:return '' result='
    ' result+=book_cover(path,link,BOOK_SERIES,caption='Series artwork · Non-Deterministic books by Enrico Papalini') result+='

    Non-Deterministic · Book series

    ' result+=f'

    {esc(BOOK_SERIES.get("title","The Non-Deterministic series"))}

    ' result+='

    Explore the books behind the Atlas: software engineering, spec-driven development, autonomous coding agents and loop engineering. Follow each volume into its chapter and source catalogue.

    ' result+=f'

    The Atlas distinguishes {len(EARLIER_VOLUMES)} earlier volumes from the published second edition of Non-Deterministic Software Engineering (NDSE2).

    ' if safeurl(BOOK_SERIES.get('amazon_url','')): result+=link(path,BOOK_SERIES['amazon_url'],'View the series on Amazon.com ↗','button primary',external=True) result+=link(path,'catalogues/volumes.html#book-volumes','Browse individual books','button') result+='
    ' return result def book_listing(path,book,display,count_text): attributes=book_attributes(book);current=attributes.get('volume_id')==NDSE2.volume_id result='
    ' if current:result+=book_cover(path,link,attributes) result+='
    ' if current else '' result+=''+('Current NDSE2 edition' if current else 'Earlier volume · '+esc(attributes.get('volume_id','')))+'' result+='

    '+link(path,record_file(book),esc(attributes.get('title') or display))+'

    ' if attributes.get('subtitle'):result+=f'

    {esc(attributes["subtitle"])}

    ' result+=f'

    {esc(count_text)}

    ' result+=book_actions(path,book,include_details=True) result+=('
    ' if current else '')+'
    ' return result def book_picker(path): result='
    Find an individual book · '+str(len(BOOKS))+' volumes
    ' def volume_presentation(path,book): attributes=book_attributes(book);current=attributes.get('volume_id')==NDSE2.volume_id individual_cover=bool(attributes.get('cover_image')) artwork=attributes if individual_cover else BOOK_SERIES caption=('Non-Deterministic Software Engineering · Second Edition' if current else attributes.get('title','Book cover')) if individual_cover else 'Series artwork · Non-Deterministic books by Enrico Papalini' result='
    ' result+=book_cover(path,link,artwork,caption=caption) result+='

    '+('NDSE2 · Published second edition' if current else esc(attributes.get('volume_id',''))+' · Earlier volume')+'

    ' result+=f'

    {esc(attributes.get("title") or book["label"])}

    ' if attributes.get('subtitle'):result+=f'

    {esc(attributes["subtitle"])}

    ' if attributes.get('author'):result+=f'

    By {esc(attributes["author"])}

    ' result+=book_actions(path,book) if not current:result+='

    This volume is part of the earlier-book collection. Its chapter and file records remain available below.

    ' else:result+='

    '+link(path,'catalogues/ndse2.html#chapter-guides','Explore the published NDSE2 chapter guides →','card-link')+'

    ' result+='
    ' return result def build_group_pages(): for kind,payloadkey,title in [ ('streams','themes','Thematic streams'),('topics','topics','Editorial topics'), ('chapters','chapters','NDSE2 chapter guides and sources'),('volumes','volumes','Published Non-Deterministic books')]: groups=VOLUMES if kind=='volumes' else D[payloadkey] index=f'catalogues/{kind}.html';cards='';current_book_cards='';earlier_book_cards='' for g in groups: path=groupfile(kind,g) chapter=NDSE2.chapter(g['id']) if kind=='chapters' else None if kind=='chapters': actual=actual_sources_for_chapter(g) mapped=[n for n in SOURCES if n.get('collection')!='N' and g['id'] in n.get('mapped',[])] records=mapped elif kind=='volumes': records=[n for n in SOURCES if n.get('collection') in {'B','N'} and g['id'] in n.get('volumes',[])] else: records=[n for n in SOURCES if g['id'] in n.get(kind,[])] display=(f'Chapter {g["n"]}: ' if kind=='chapters' else '')+(chapter['title'] if chapter else g['label']) if kind=='volumes' and g['id']=='V03': display='Non-Deterministic Spec-Driven Development — Enterprise / second edition' if kind=='chapters': count_text=f'{len(actual)} cited atlas references · {len(mapped)} editorial reading mappings' elif kind=='volumes': count_text=f'{len(records)} chapter or file records' count_text+=' · NDSE2 edition' if g['id']==NDSE2.volume_id else ' · Earlier volume' else: count_text=f'{len(records)} mapped records' book=next((n for n in BOOKS if n.get('attributes',{}).get('volume_id')==g['id']),None) if kind=='volumes' else None if book: card=book_listing(index,book,display,count_text) if g['id']==NDSE2.volume_id:current_book_cards+=card else:earlier_book_cards+=card cards+='

    '+link(index,path,esc(display))+'

    ' if chapter and chapter.get('title_it'): cards+=f'

    {esc(chapter["title_it"])}

    ' cards+=f'

    {esc(count_text)}

    ' descriptions={ 'streams':'Current source memberships across the atlas. A source can address more than one stream.', 'topics':'Editorial topic mappings connect related questions across sources and book chapters.', 'chapters':'A guide to the published second edition, with literal citations separated from wider editorial reading suggestions.', 'volumes':'Published book metadata and the corresponding chapter or file records. Volume IDs preserve the atlas history.'} desc=descriptions[kind] b='
    '+link(path,'catalogues/index.html','Catalogues')+' / '+link(path,index,esc(title))+'
    '+heading(display,g['id'],desc) grouping={'streams':'theme','topics':'topic','chapters':'chapter','volumes':'volume'}[kind] codes='B,N' if kind=='volumes' and 'N' in COLS else ('B' if kind=='volumes' else ','.join(COLS)) b+='
    '+link(path,atlasurl(collection=codes,grouping=grouping,group=g['id']),'Open this group in the atlas','button primary') if chapter: b+=link(path,record_file(BY[chapter['id']]),'Chapter catalogue entry','button') b+='
    ' if kind=='volumes': if book:b+=volume_presentation(path,book) if g['id']==NDSE2.volume_id and NDSE2.chapters: b+=NDSE2.edition_metadata(path,link) b+='

    '+link(path,collection_file('N'),'NDSE2 · published second edition →','card-link')+'

    ' else: if book: b+=fmt_meta(book['attributes'],[('title','Published title'),('subtitle','Subtitle'),('year','Year'),('edition_number','Edition'),('isbn','ISBN, as recorded'),('publisher','Publisher'),('metadata_verification','Metadata basis')]) if kind=='chapters': if chapter: b+='

    Chapter guide

    '+NDSE2.chapter_content(chapter,path,link,BY,record_file,include_evidence=False)+'
    ' b+='

    Cited in the published chapter

    ' b+=f'

    {len(actual)} linked atlas references from literal citations in the English and Italian editions. General bibliography membership and identifier-index entries are counted separately.

    ' b+=grouped_sources(path,actual)+'
    ' if chapter: b+=NDSE2.citation_evidence(chapter,path,link,BY,record_file) b+='

    Editorial reading mappings

    Broader topical suggestions can overlap the cited sources. A mapping does not add a citation to the published edition or establish that a practice is effective.

    '+grouped_sources(path,mapped)+'
    ' else: b+=grouped_sources(path,records) page(path,display,b,desc) descriptions={ 'streams':f'{len(D["themes"])} streams connect the original SE@PoliMi organization with the atlas’s additional collection mappings.', 'topics':f'{len(D["topics"])} editorial topics help follow engineering questions through the current graph.', 'chapters':f'{len(D["chapters"])} chapter guides for the published NDSE2 edition, with English and Italian titles, learning goals, outlines and traceable references.', 'volumes':f'{len(VOLUMES)} published books: {len(EARLIER_VOLUMES)} earlier volumes and the current second edition of Non-Deterministic Software Engineering.'} desc=descriptions[kind] body=heading(title,'Browse by group',desc) if kind=='volumes': body+=series_presentation(index)+'

    Current second edition

    '+current_book_cards+'
    ' body+='

    Earlier books

    Each book has its own Amazon.com destination and a catalogue of the chapter or file records preserved in the Atlas.

    '+earlier_book_cards+'
    ' else:body+='
    '+cards+'
    ' page(index,title,body,desc) def build_landing_pages(): path='index.html' hero='
    '+heading(C['title'],'A navigable evidence collection','Explore the research, practitioner writing and engineering ideas behind AI-assisted software development. Follow connections to their sources and read the limits of each claim.') hero+='
    '+link(path,'swe_knowledge_graph.html','Open the interactive atlas →','button primary')+link(path,'catalogues/index.html','Browse the catalogues','button') if 'N' in COLS: hero+=link(path,collection_file('N'),'NDSE2 · published second edition','button') hero+='

    Starts with the original PoliMi source collection. Select the other collections to expand the view.

    ' hero+='
    '+link(path,'swe_knowledge_graph.html',f'Knowledge Atlas with its original PoliMi starting collection')+'
    Explore the connections or read each source in a static catalogue.
    ' counts='
    '+''.join(f'
    {n:,}{esc(label)}
    ' for n,label in [ (EXTERNAL_COUNT,'External source records'),(COUNTS['B'],'Earlier-book records'),(COUNTS['N'],'Published NDSE2 chapters'),(len(PRACTICES),'Practice guides')])+'
    ' paths=[ ('streams',f'{len(D["themes"])} thematic streams','The original research organization, with additional source mappings.'), ('topics',f'{len(D["topics"])} editorial topics','Follow an engineering question across source collections.'), ('volumes',f'{len(VOLUMES)} published books','Distinguish the earlier volumes from the published NDSE2 edition.'), ('chapters',f'{len(D["chapters"])} chapter guides','Read chapter outlines and separate printed citations from suggested reading.')] grids='
    '+''.join('

    '+link(path,f'catalogues/{name}.html',esc(title))+'

    '+esc(desc)+'

    ' for name,title,desc in paths)+'
    ' body=hero+counts+NDSE2.introduction(path,link,heading=True)+ack(path) body+='

    Browse the Atlas collections

    '+link(path,'catalogues/index.html','All catalogues →')+'
    '+''.join(collection_card(c,path) for c in C['collections'])+'
    ' body+='

    Follow another route

    '+grids+'
    '+profile(path) page(path,C['title'],body,f'Explore {len(COLS)} Atlas collections, English practice guides, an interactive graph and the published NDSE2 chapter guides.',active='') page('knowledge-atlas.html',C['title'],body,'Project landing page, also suitable when preserving a separate existing homepage.',active='') path='catalogues/index.html' extra=paths+[ ('occurrences','Historical occurrences',f'{len(OCCURRENCES)} occurrences in earlier books, preserving conceptual genealogy.')] body=heading('Atlas catalogues',f'{len(COLS)} collections · one knowledge map','Read the source records, published chapters and English practice guides. All entries are present in static HTML, with search, filters, data downloads and permanent links.')+ack(path) body+='
    '+''.join(collection_card(c,path) for c in C['collections'])+'
    ' body+='

    Additional catalogue views

    '+''.join('

    '+link(path,f'catalogues/{name}.html',esc(title))+'

    '+esc(desc)+'

    ' for name,title,desc in extra)+'
    ' page(path,'Atlas catalogues',body,'Static source catalogues, published book chapters, English practice guides and evidence provenance.') def build_about_page(): path='about.html' labels=', '.join(COLS) body=heading('About this project','Independent work by Enrico Papalini','A companion for asking better questions about AI and software engineering, built around source provenance.')+profile(path,True) body+=f"""

    Why an atlas?

    The project connects research records, practitioner publications and the engineering questions explored in the Non-Deterministic book series. The interactive atlas exposes relationships; the catalogues make each record accessible as an ordinary web page.

    What the labels mean

    {esc(labels)} identify collections and their provenance. Papers, interviews, conceptual essays, book chapters and small experiments retain their distinct source types. Records show access restrictions, caveats and the scope of source inspection.

    The published second edition

    NDSE2 is represented by {COUNTS['N']} chapter records. Guides include English and Italian titles, learning goals, outlines and source references. {esc(NDSE2.publication_note())} Publication details are shown only when supplied in the edition metadata.

    """ body+=link(path,'catalogues/ndse2.html','NDSE2 · published second edition →')+'

    ' body+='

    Biography and photograph

    The biography and author photograph come from the author’s own Non-Deterministic Loop Engineering materials (About the Author, PDF page 277; printed page 265). The photograph was extracted, resized and saved locally for this project.

    '+link(path,'source/author_provenance.json','Photo and biography provenance')+'

    Connect through '+link(path,'https://github.com/popoloni','GitHub',external=True)+' or explore the '+link(path,'https://www.amazon.com/dp/B0GHCXPBRM','Non-Deterministic series',external=True)+'.

    '+ack(path) page(path,'About Enrico Papalini and the project',body,'Author biography and the purpose of the independent Knowledge Atlas.',active='About') def build_attribution(): path='attribution.html' f_printed=sum(bool(n.get('printed')) for n in SOURCES if n.get('collection')=='F') body=heading('Credits & provenance','Attribution is part of the model','The starting bibliography, source authors, editorial additions and evidence boundaries.') body+=f"""

    The starting collection: Software Engineering @ PoliMi

    The SE & AI Annotated Bibliography was curated by Alfonso Fuggetta, Damian Andrew Tamburri and Alessandro Margara, Software Engineering @ PoliMi. The retained original snapshot contains 193 works, dated 8 October 2026 and published on 9 October. Credit for the original collection, classification and annotations belongs to the curators.

    Original annotated bibliography ↗

    Independent editorial additions

    Enrico Papalini developed the graph, interface, independent summaries, cross-source comparisons, practice genealogy and book mappings with AI assistance. These are separate from the curators’ original work. No endorsement by Polimi, the source authors or their institutions is implied.

    Practitioner collections

    The Osmani collection credits Addy Osmani. The Pragmatic Engineer collection preserves named authors, interview participants and guest contributors. Publisher affiliation is not a substitute for an article byline.

    The Fowler website collection contains {COUNTS['F']} records. It preserves the original Exploring Generative AI series and explicitly labels the adjacent website articles added in this refresh. Individual bylines remain attached to each article; Birgitta Böckeler’s contribution as writer and editor is acknowledged. Hosting on martinfowler.com does not establish Martin Fowler authorship or Thoughtworks endorsement.

    Books and published citations

    The {COUNTS['B']} B records describe chapters and informative files in {len(EARLIER_VOLUMES)} earlier books. The {COUNTS['N']} N records describe the published second edition of Non-Deterministic Software Engineering. Its guides trace literal citations in the English and Italian chapter sources separately from general bibliography membership and the identifier index. {f_printed} Fowler records occur in the published bibliography; newly suggested reading is labelled according to its actual status.

    The {len(PRACTICES)} practice descriptors and {len(OCCURRENCES)} historical occurrences record conceptual genealogy. Repeated discussion of an idea is not independent empirical support.

    Evidence and access boundaries

    This release combines the retained historical graph with the documented {esc(C['graph_snapshot'])} source refresh and the published-book citation extraction. Inspection scope and access limitations are recorded per source. The builder reproduces those audited records; it does not execute studies, replicate statistical analyses or perform a production pilot. W records can contain lead-author-only metadata; consult the original publication for full bibliographic details.

    The current graph contains {len(D['nodes']):,} nodes, {len(D['edges']):,} keyed relationships and {len(D['relations']):,} relation types. Historical IDs and the original PoliMi starting collection remain available. Literal source citations, source-published links and editorial mappings have distinct meanings.

    """ body+=link(path,RESEARCH_AUDIT,'Read the source-review audit')+' · '+link(path,RELEASE_NOTES,'Read the update notes')+'

    ' body+='

    Rights and redistribution

    The inherited attribution record notes the SE@PoliMi publication’s stated CC BY-NC-ND 4.0 terms unless otherwise noted. Third-party works retain their own terms. This package contains metadata, concise paraphrases, chapter guides and provenance records. It grants no new rights to full articles, subscriber material, figures or datasets.

    No blanket licence is assigned to every source in the atlas. The author photograph is supplied for this project. Complete third-party articles and complete book manuscripts are not distributed in the website package.

    ' page(path,'Credits, provenance and evidence boundaries',body,'Credits to the original curators and source authors, with evidence and access boundaries.') text=f"""# Attribution and provenance The starting collection is **The SE & AI Annotated Bibliography**, curated by **Alfonso Fuggetta, Damian Andrew Tamburri and Alessandro Margara**, **Software Engineering @ PoliMi**: https://www.sepolimi.com/p/se-ai-annotated-bibliography. Its original snapshot contains 193 works (8 October 2026; published 9 October). The original collection, classification and annotations belong to the curators. The atlas, interface, independent summaries, additional collections, editorial mappings and book genealogy are Enrico Papalini’s independent work with AI assistance. No institutional endorsement is implied. Addy Osmani and all credited contributors to The Pragmatic Engineer retain their individual authorship. The Fowler website collection preserves article bylines and Birgitta Böckeler’s editorial contribution to [Exploring Generative AI](https://martinfowler.com/articles/exploring-gen-ai.html). Adjacent website articles added in this refresh are explicitly labelled. The current release combines the historical graph with the documented {C['graph_snapshot']} source refresh and published NDSE2 citation extraction. Source inspection does not amount to causal validation or independent replication. The {COUNTS['N']} N records are published-book chapter guides, not independent research studies. The {COUNTS['B']} B records preserve earlier-book genealogy. The inherited rights notice records SE@PoliMi’s stated CC BY-NC-ND 4.0 terms unless otherwise noted. No blanket licence for third-party content is granted. Full external manuscripts, subscriber article bodies and complete book texts are not bundled. See [the full notice](attribution.html), [source-review audit]({RESEARCH_AUDIT}) and [update notes]({RELEASE_NOTES}). """ (ROOT/'ATTRIBUTION.md').write_text(text,encoding='utf-8') def build_downloads(): path='downloads.html';rows='' datasets=[(COLS[c]['slug'],COLS[c]['title']) for c in COLS]+[ ('all-sources',f'All {len(CITABLE_RECORDS)} source and chapter records'), ('occurrences',f'{len(OCCURRENCES)} practice occurrences'), ('practice-families',f'{len(FAMILIES)} practice families'), ('published-books',f'{len(BOOKS)} published book records')] for name,title in datasets: rows+=''+esc(title)+''+link(path,f'data/catalogues/{name}.json','JSON')+''+link(path,f'data/catalogues/{name}.csv','CSV')+'' body=heading('Download the data','Static files · no API required','Current catalogues and graph exports represent the same audited release. JSON retains complete records; CSV provides a compact tabular view.') body+='
    '+rows+'
    Current catalogue exports
    CatalogueComplete recordsTabular export
    ' body+='

    The current complete graph

    '+link(path,GRAPH_GRAPHML,'Current GraphML','button')+link(path,GRAPH_JSON,'Current graph JSON','button')+link(path,'data/catalogue-data.zip','All current catalogue data (ZIP)','button primary')+'
    ' body+=f'

    Graph v{esc(C["graph_version"])} · {esc(C["graph_snapshot"])} · {len(D["nodes"]):,} nodes · {len(D["edges"]):,} keyed relationships · {len(D["relations"]):,} relation types.

    ' body+='

    Published NDSE2 evidence and this update

    ' body+='

    Historical graph and catalogue files

    These files retain their original version names and wording. Use the current graph above for the refreshed atlas. Relative links inside historical Markdown can refer to the original book archive.

    Project files

    ' page(path,'Downloads',body,'Current graph exports, static catalogues, published-book evidence, historical files and build sources.',active='Downloads') def write_readme_and_deployment(): rows='\n'.join(f'| {c["code"]} | [{c["title"]}]({collection_file(c["code"])}) | {COUNTS[c["code"]]} |' for c in C['collections']) readme=f"""# {C['title']} An independent, source-traceable map of research and practitioner writing about AI-assisted software development, by **Enrico Papalini**. The graph and static catalogues connect publications, topics, engineering practices and book chapters while retaining their evidence limits. **[Open the published atlas]({C['canonical_atlas']})** · **[Local atlas](swe_knowledge_graph.html)** · **[Catalogues](catalogues/index.html)** · **[NDSE2 · published second edition](catalogues/ndse2.html)** · **[Downloads](downloads.html)** · **[Update notes]({RELEASE_NOTES})** ## The starting collection The foundation is **[The SE & AI Annotated Bibliography](https://www.sepolimi.com/p/se-ai-annotated-bibliography)**, curated by **Alfonso Fuggetta, Damian Andrew Tamburri and Alessandro Margara**, **Software Engineering @ PoliMi**. Its original 8 October 2026 snapshot contains 193 works. The original classification and annotations are the curators’ contribution. This independent atlas and its editorial extensions do not imply institutional endorsement. The atlas opens with the original PoliMi collection. Additional collections, context and relationship controls are available through the interface. Source IDs and established links remain stable. ## Current catalogues | ID | Collection | Records | |---|---|---:| {rows} Also browse **[{len(D['themes'])} streams](catalogues/streams.html)**, **[{len(D['topics'])} topics](catalogues/topics.html)**, **[{len(VOLUMES)} published books](catalogues/volumes.html)**, **[{len(D['chapters'])} NDSE2 chapter guides](catalogues/chapters.html)**, **[{len(PRACTICES)} practice descriptors](catalogues/practices.html)** and **[{len(OCCURRENCES)} historical occurrences](catalogues/occurrences.html)**. Static entries work without JavaScript; optional filters improve navigation. ## Published NDSE2 second edition {NDSE2.publication_note()} ISBN and first-publication date are displayed only if supplied in the edition metadata. The **{COUNTS['N']} chapter records** contain concise English summaries, Italian titles, learning goals, outlines and source links. Literal citations from the English and Italian chapter files are distinguished from bibliography membership, the identifier index and broader editorial reading mappings. The full manuscripts are not included in the website. [Read the chapter guides](catalogues/ndse2.html) · [Edition and extraction provenance]({BOOK_METADATA}) · [Source-review audit]({RESEARCH_AUDIT}) ## Book covers and Amazon.com links The [publication registry](source/book_publications.json) keeps series and individual-book destinations together with the paths, descriptions and original dimensions of the supplied artwork in `assets/books/`. This presentation overlay is retained during future article and manuscript rebuilds; bibliographic titles, ISBNs and chapter citations remain in their own source metadata. After changing the registry or supplied artwork, run from the site root: ```sh python source/update_publications.py python source/build_site.py ``` The publication adapter applies book-link and cover metadata. The site builder then renders the catalogues. Cover updates do not require manuscript extraction and do not change chapter citations. See [the publication update guide](source/PUBLICATION_UPDATES.md) for the complete procedure. ## Authorship and evidence The Fowler collection credits each article’s actual author and acknowledges **Birgitta Böckeler** as contributor and editor of [Exploring Generative AI](https://martinfowler.com/articles/exploring-gen-ai.html). Its adjacent website additions are labelled separately. The Pragmatic Engineer catalogue preserves guest and interview roles. Collection membership does not establish peer review, causal support or independent replication. ## About Enrico Papalini Enrico Papalini — author photograph {C['bio']} Photo and biography are adapted from the author’s own *Non-Deterministic Loop Engineering* author page. [Photo provenance](source/author_provenance.json) · [About the project](about.html) · [GitHub](https://github.com/popoloni). This project expresses the author’s views, not those of his employers or cited institutions. ## Publish or update the static site Copy the package contents to the configured Pages publishing folder, preserving relative paths and `.nojekyll`. The homepage is `index.html`; the interactive atlas remains `swe_knowledge_graph.html`. A `knowledge-atlas.html` entrypoint is also supplied. A delta package contains only files to add or overwrite and is applied to the existing site using the same relative paths. No server application, API key, framework or CDN is required. See [DEPLOYMENT.md](DEPLOYMENT.md) for local preview and the complete update procedure. ## Repeat the update Use `python source/automation/atlas_cli.py` with the sequence **prepare → discover → build → check → package**. `prepare` requires the exact full ZIP confirmed as the currently published site and freezes its SHA-256. Complete the generated `review.json` after checking discovered source candidates, then build from those reviewed inputs. Book metadata remains unchanged unless a reviewed manuscript revision is supplied explicitly. `check` performs independent structural verification, real-browser checks and repeatable rebuilds. `package` creates the full archive and a delta containing only file additions or overwrites, and proves that baseline plus delta equals the full package. Neither command publishes to GitHub. [Commands, dependencies and book-update steps](DEPLOYMENT.md). **Graph v{C['graph_version']} · {C['graph_snapshot']}**. **Website v{C['site_version']} · {C['build_date']}**. The current graph contains **{len(D['nodes']):,} nodes**, **{len(D['edges']):,} keyed relationships** and **{len(D['relations']):,} relation types**. Retained earlier graph exports remain labelled as historical. [Credits and boundaries](attribution.html) · [Validation report](verification/report.html) """ (ROOT/'README.md').write_text(readme,encoding='utf-8') deployment=f"""# Publish or update this static site ## Apply the package Copy the extracted files to the folder already configured as the GitHub Pages publishing source. Preserve all relative paths and `.nojekyll`. The extracted `index.html` must be directly inside that folder. For a **full package**, use the complete site tree. For a **delta package**, add or overwrite only the included files at their existing relative paths. Follow the accompanying delta instructions for the baseline and file list. The homepage is `index.html`; the interactive atlas is `swe_knowledge_graph.html`. If an unrelated homepage must be retained, use `knowledge-atlas.html` as the project entrypoint. Local links work at a user-site root or within a project subdirectory. ## Preview locally Static catalogue contents remain readable with JavaScript disabled. To preview normal browser navigation, run from the site folder: ```sh python -m http.server 8000 ``` Open `http://localhost:8000/`. External source links still follow the publisher’s access conditions. ## Repeat an Atlas update Start from the **exact complete ZIP currently published**. The updater freezes its ZIP hash and file inventory, then works on an isolated copy. A previous delta or a live repository snapshot is not a substitute for that confirmed baseline. Run these commands from a complete package that contains `source/automation/`. Choose a fresh run directory outside the site and replace the path, date and release placeholders with this update’s values: ```sh python source/automation/atlas_cli.py prepare --baseline /path/to/published-complete.zip --run /path/to/atlas-update --date YYYY-MM-DD --release NEXT_VERSION python source/automation/atlas_cli.py discover --run /path/to/atlas-update ``` Discovery collects public publisher metadata and flags candidate additions or revisions. Review `/path/to/atlas-update/review.json`: resolve each candidate, retain actual bylines and original versus revision dates, identify access restrictions, and record source evidence before marking editorial review complete. Orosz blog and Deepdives remain in scope; The Pulse title-prefix and podcast exclusions remain in force. Canonical duplicates are revised under the existing ID; accepted additions receive new IDs. The discovery step does not create verified summaries or book citations. After that source review is complete: ```sh python source/automation/atlas_cli.py build --run /path/to/atlas-update python source/automation/atlas_cli.py check --run /path/to/atlas-update --browser /path/to/chromium --playwright /path/to/node_modules/playwright python source/automation/atlas_cli.py package --run /path/to/atlas-update --output /path/to/packages ``` Python 3.10+ uses only its standard library for graph updates, site rendering, structural checks and packaging. Real browser verification additionally requires Node.js, the Playwright package and a Chromium-family executable. The `--browser` and `--playwright` overrides can be omitted if those dependencies are discoverable in the environment. A missing browser is reported as an incomplete check and prevents packaging; it is not a successful browser result. The `check` command validates current pages, anchors, exact graph exports, stable IDs and historical keyed edges, literal book citations, and the preserved original starting view. It exercises every relation switch and the catalogues in a real local browser, then compares two rebuilds from the same frozen inputs. Verification records are bound to the current artifact’s file hashes. If reviewed inputs or candidate files change, build and check again before packaging. The `package` command calculates the delta by SHA-256 comparison of actual file contents against the frozen published ZIP. The delta includes only added or changed paths. Applying that delta to a fresh extraction of the original ZIP must reconstruct the complete package exactly. No file deletion is permitted by this copy-only procedure. An update with no approved changes is recorded as such instead of manufacturing a new release. ## Update the book only when its sources change Leave `book_policy` as `unchanged` for article-only updates. Current chapter citations and bibliography flags remain derived from the retained manuscript metadata. New article mappings are editorial reading suggestions unless the supplied book actually cites them. For a new final bilingual manuscript, `extract-book` can prepare a metadata draft for review. Use `source/automation/atlas_cli.py extract-book --help` for its required source paths. Review the extracted titles, summaries, goals and citation evidence, set `book_policy` to `reconcile`, and pass `--book /path/to/reviewed-book.json` to `build`. Chapter or section identity changes require an explicit migration and are blocked by the routine update workflow. ## Update publication links or supplied covers Keep book destinations and artwork metadata in `source/book_publications.json`, configured through `book_publications_json` in `source/site_config.json`. Store the original local cover and series images in `assets/books/` and retain their original proportions. The registry records the individual product destinations separately from the series page and author companion links. After editing that registry or replacing the supplied artwork, run from the site root: ```sh python source/update_publications.py python source/build_site.py ``` The publication adapter applies these presentation fields to the current book records. The renderer also reads the retained overlay, so future article and manuscript rebuilds keep the product links and covers. This adapter is separate from manuscript extraction: a cover or storefront-link update does not replace bibliographic titles or ISBNs, create source citations, or change chapter citation evidence. For an ordinary render with no publication changes, run only `python source/build_site.py`. See [source/PUBLICATION_UPDATES.md](source/PUBLICATION_UPDATES.md) for the registry fields and publication-update procedure. ## Render the frozen website without refreshing sources ```sh python source/build_site.py ``` This command rebuilds the static catalogues from the current configured template `source/{ATLAS_SOURCE.name}`, `source/site_config.json` and `{BOOK_METADATA}`. It does not discover articles, alter the graph, run verification or create a new approved package. The reusable `build` command above also performs this rendering step after applying the reviewed graph update. Current graph downloads are `{GRAPH_JSON}` and `{GRAPH_GRAPHML}`. Earlier exports remain historical artifacts. Release notes are `{RELEASE_NOTES}` and the source audit is `{RESEARCH_AUDIT}`. Use the existing GitHub Pages publishing workflow after reviewing the resulting full or delta package. """ (ROOT/'DEPLOYMENT.md').write_text(deployment,encoding='utf-8') def build(): # The graph engine is supplied by the audited current source; add only static-site navigation. PAGES.clear() (ROOT/'assets').mkdir(parents=True,exist_ok=True) (ROOT/'assets/books.css').write_text(BOOK_CSS,encoding='utf-8') patch=(SOURCE/'atlas_navigation_patch.js').read_text(encoding='utf-8') marker='buildControls();boardSize();rebuild();' patch_start='// ATLAS_STATIC_NAVIGATION_BEGIN' patch_end='// ATLAS_STATIC_NAVIGATION_END' if (patch_start in BASE) != (patch_end in BASE): raise ValueError('Current atlas contains an incomplete static-navigation block') if patch_start in BASE: block=re.compile(re.escape(patch_start)+r'.*?'+re.escape(patch_end),re.S) modified=block.sub(lambda _:patch_start+'\n'+patch.rstrip()+'\n'+patch_end,BASE,count=1) elif re.search(r'window\.atlasOpenCatalogueView\s*=\s*applyCatalogueURL',BASE) and re.search(r'const\s+catalogueFiles\s*=',BASE): # Legacy delivered templates already contain the navigation layer without # explicit block delimiters. Keeping it avoids duplicate const declarations. modified=BASE else: if marker not in BASE:raise ValueError('Current atlas is missing the expected navigation insertion point') modified=BASE.replace(marker,marker+'\n'+patch_start+'\n'+patch.rstrip()+'\n'+patch_end,1) nav='Catalogues' if not re.search(r'\bid=["\']siteCataloguesLink["\']',modified): modified=modified.replace('
    ','
    '+nav,1) (ROOT/'swe_knowledge_graph.html').write_text(modified,encoding='utf-8') (ROOT/'data/catalogues').mkdir(parents=True,exist_ok=True) for code,c in COLS.items():write_dataset(c['slug'],[n for n in SOURCES if n['collection']==code]) for name,records in [('all-sources',CITABLE_RECORDS),('practices',PRACTICES),('occurrences',OCCURRENCES),('practice-families',FAMILIES),('published-books',BOOKS)]: write_dataset(name,records) for code,c in COLS.items(): if code=='PAT':continue path=collection_file(code);records=[n for n in SOURCES if n['collection']==code] if code=='N':records.sort(key=lambda n:int(n.get('attributes',{}).get('chapter_number') or n['id'].split('C')[-1])) body='
    '+link(path,'index.html','Home')+' / '+link(path,'catalogues/index.html','Catalogues')+' / '+esc(c['short'])+'
    ' body+=heading(c['title'],f'{code} collection · {len(records)} records',c['description'])+collection_nav(path,code) if code=='W':body+=ack(path) if code=='F': printed=sum(bool(n.get('printed')) for n in records) body+='

    The collection follows the host website, with each article’s actual byline. The original Exploring Generative AI series credits Birgitta Böckeler as contributor and editor; adjacent website articles are labelled individually. ' body+=f'{printed} records are included in the published NDSE2 bibliography. Literal chapter citations and later editorial suggestions remain distinct.

    ' if code=='N':body+=NDSE2.introduction(path,link) if code=='B':body+=series_presentation(path)+book_picker(path) body+='
    '+link(path,atlasurl(collection=code),'Explore this collection in the atlas','button primary')+link(path,f'data/catalogues/{c["slug"]}.json','JSON catalogue','button')+link(path,f'data/catalogues/{c["slug"]}.csv','CSV catalogue','button')+'
    ' if code=='B':body+='

    '+link(path,'catalogues/volumes.html',f'Browse {len(EARLIER_VOLUMES)} earlier volumes and the published NDSE2 edition →')+'

    ' body+=filter_bar(path,len(records),D['themes'])+'
    '+''.join(record(n,path) for n in records)+'
    ' page(path,c['title'],body,c['description'],active='NDSE2' if code=='N' else 'Catalogues') build_practice_catalogue() for name,title,records,desc in [ ('occurrences','Historical practice occurrences',OCCURRENCES,f'{len(OCCURRENCES)} source occurrences across {len(EARLIER_VOLUMES)} earlier volumes. These trace where practices were discussed.')]: path=f'catalogues/{name}.html' body=heading(title,'Series practice catalogue',desc)+'
    '+link(path,'catalogues/practices.html','Descriptors','button')+link(path,'catalogues/occurrences.html','Occurrences','button')+link(path,f'data/catalogues/{name}.json','JSON','button')+link(path,f'data/catalogues/{name}.csv','CSV','button')+'
    ' body+=filter_bar(path,len(records),EARLIER_VOLUMES if name=='occurrences' else None,'volumes','Volume')+'
    '+''.join(record(n,path) for n in records)+'
    ' page(path,title,body,desc) build_group_pages() build_landing_pages() build_about_page() build_attribution() build_downloads() write_readme_and_deployment() date=tuple(int(part) for part in C['build_date'][:10].split('-'))+(0,0,0) with zipfile.ZipFile(ROOT/'data/catalogue-data.zip','w',compression=zipfile.ZIP_DEFLATED,compresslevel=9) as archive: for item in sorted(list((ROOT/'data/catalogues').glob('*'))+list((ROOT/'data/originals').glob('*'))): if not item.is_file():continue info=zipfile.ZipInfo(item.relative_to(ROOT/'data').as_posix(),date_time=date) info.compress_type=zipfile.ZIP_DEFLATED;info.external_attr=0o644<<16 archive.writestr(info,item.read_bytes(),compress_type=zipfile.ZIP_DEFLATED,compresslevel=9) (SOURCE/'page_inventory.json').write_text(json.dumps(sorted(set(PAGES+['swe_knowledge_graph.html'])),indent=2)+'\n',encoding='utf-8') summary={ 'site_version':C['site_version'],'graph_snapshot':C['graph_snapshot'],'graph_source':ATLAS_SOURCE.name, 'graph_nodes':len(D['nodes']),'graph_edges':len(D['edges']),'graph_relation_types':len(D['relations']), 'collection_counts':dict(COUNTS),'source_counts':{k:v for k,v in COUNTS.items() if k!='PAT'}, 'static_source_records':len(CITABLE_RECORDS),'static_collection_records':len(SOURCES),'practice_records':len(PRACTICES),'external_source_records':EXTERNAL_COUNT, 'practice_descriptors':len(PRACTICES),'practice_occurrences':len(OCCURRENCES), 'published_books':len(BOOKS),'ndse2_chapters':len(NDSE2.chapters), 'group_pages':{'streams':len(D['themes']),'topics':len(D['topics']),'chapters':len(D['chapters']),'volumes':len(VOLUMES)}, 'html_pages_built':len(PAGES)+1,'graph_payload_unchanged':True, 'graph_payload_matches_current_source':True,'new_empirical_verification':False} (SOURCE/'build_summary.json').write_text(json.dumps(summary,indent=2)+'\n',encoding='utf-8') print(f'Built {len(PAGES)+1} HTML pages, {len(CITABLE_RECORDS)} source/chapter records, {len(PRACTICES)} practices and {len(NDSE2.chapters)} NDSE2 guides from {ATLAS_SOURCE.name}.') if __name__=='__main__':build()