"""Collect Commons reference thumbnails; preserve attribution and download status."""
import json
from pathlib import Path
import time
import urllib.request

root = Path(__file__).resolve().parents[1] / 'data/references/summit/photos'
pages = json.loads((root / 'commons-metadata.json').read_text(encoding='utf-8-sig'))['query']['pages']
old = json.loads((root / 'manifest.json').read_text(encoding='utf-8-sig'))
by_title = {p['title']: p for p in pages.values()}
result = []
for index, entry in enumerate(old, 1):
    info = by_title[entry['title']].get('imageinfo', [])
    if not info:
        result.append({'title': entry['title'], 'status': 'missing metadata'})
        continue
    info = info[0]
    name = f'photo-{index:02d}.jpg'
    original = root / entry['file']
    target = root / name
    record = dict(title=entry['title'], file=name, source=info['descriptionurl'], metadata=info['extmetadata'])
    try:
        if original.exists():
            original.rename(target)
            record['download_url'] = info['url']
        elif not target.exists():
            time.sleep(3)
            url = info.get('thumburl', info['url'])
            request = urllib.request.Request(url, headers={'User-Agent': 'SunriseReferenceCollector/1.0 (local research)'})
            with urllib.request.urlopen(request, timeout=25) as response:
                data = response.read()
            if not data.startswith(b'\xff\xd8'):
                raise ValueError('Not JPEG')
            target.write_bytes(data)
            record['download_url'] = url
        record['status'] = 'downloaded'
    except Exception as error:
        record['status'] = str(error)
    result.append(record)
    print(name, record['status'], flush=True)
    (root / 'downloads.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')