"""Offline pack builder. Spylls validates candidates without expanding every affix.
Dependencies: npm --prefix outputs/word-pack-tools install --ignore-scripts
dictionary-es dictionary-pt-br dictionary-fr dictionary-de
python -m pip install --target outputs/word-pack-tools/python spylls==0.1.7
"""
import sys,pathlib,json,unicodedata,urllib.request,hashlib,collections
sys.path.insert(0,str(pathlib.Path('outputs/word-pack-tools/python').resolve()))
from spylls.hunspell import Dictionary
out=pathlib.Path('lib/word-grid-data/packs');(out/'licenses').mkdir(parents=True,exist_ok=True)
def normalize(word,locale):
 if locale=='de':return unicodedata.normalize('NFC',word).replace('ß','ss').upper()
 if locale=='es':word=word.replace('ñ','\x01')
 return ''.join(c for c in unicodedata.normalize('NFD',word) if unicodedata.category(c)!='Mn').replace('\x01','ñ').replace('œ','oe').replace('æ','ae').upper()
for locale,pkg,code in [('es','es','es'),('pt-BR','pt-br','pt_br'),('fr','fr','fr'),('de','de','de')]:
 folder=pathlib.Path('outputs/word-pack-tools/node_modules/dictionary-'+pkg)
 dictionary=Dictionary.from_files(str(folder/'index'));print('Loaded',locale,flush=True)
 url=f'https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/{code}/{code}_50k.txt'
 with urllib.request.urlopen(url,timeout=30) as response:corpus=response.read()
 frequency=collections.Counter();weights=collections.Counter()
 for i,row in enumerate(corpus.decode().splitlines()):
  raw,count=row.rsplit(' ',1)
  if not raw.isalpha() or not raw.islower() or len(raw)<3 or len(raw)>24:continue
  if not dictionary.lookup(raw) and not (locale=='de' and dictionary.lookup(raw.capitalize())):continue
  word=normalize(raw,locale);alphabet='ABCDEFGHIJKLMNOPQRSTUVWXYZ'+('Ñ' if locale=='es' else 'ÄÖÜ' if locale=='de' else '')
  if any(c not in alphabet for c in word):continue
  frequency[word]+=int(count)
  for c in word:weights[c]+=int(count)
 words=sorted(frequency)
 if len(words)<10000:raise ValueError(f'{locale}: too few validated words: {len(words)}')
 manifest=json.loads((folder/'package.json').read_text(encoding='utf-8'));total=sum(weights.values())
 pack=dict(locale=locale,version='2026-09-1',minimumLength=3,maximumLength=24,alphabet=sorted(weights),weights={c:round(n/total,6) for c,n in sorted(weights.items())},normalization='accent-insensitive-preserve-enye' if locale=='es' else 'preserve-umlauts-sharp-s-as-ss' if locale=='de' else 'accent-insensitive',scoring='existing-partybox-rules',words=words,commonWords=sorted(w for w,n in frequency.most_common(6000)),provenance=dict(dictionary=manifest['name'],version=manifest['version'],license=manifest['license'],frequencySource=url,frequencySha256=hashlib.sha256(corpus).hexdigest(),validator='spylls 0.1.7'))
 (out/(locale+'.json')).write_text(json.dumps(pack,ensure_ascii=False,separators=(',',':'))+'\n',encoding='utf-8')
 (out/'licenses'/(locale+'.txt')).write_bytes((folder/'license').read_bytes())
 print(locale,len(words),'validated words',flush=True)
with urllib.request.urlopen('https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/LICENSE') as response:(out/'licenses/FrequencyWords.txt').write_bytes(response.read())