42 lines
1.8 KiB
Python
42 lines
1.8 KiB
Python
"""Collect Commons reference thumbnails; preserve attribution and download status."""
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.request
|
|
|
|
root = Path(__file__).resolve().parents[1] / 'data/references/summit/photos'
|
|
pages = json.loads((root / 'commons-metadata.json').read_text(encoding='utf-8-sig'))['query']['pages']
|
|
old = json.loads((root / 'manifest.json').read_text(encoding='utf-8-sig'))
|
|
by_title = {p['title']: p for p in pages.values()}
|
|
result = []
|
|
for index, entry in enumerate(old, 1):
|
|
info = by_title[entry['title']].get('imageinfo', [])
|
|
if not info:
|
|
result.append({'title': entry['title'], 'status': 'missing metadata'})
|
|
continue
|
|
info = info[0]
|
|
name = f'photo-{index:02d}.jpg'
|
|
original = root / entry['file']
|
|
target = root / name
|
|
record = dict(title=entry['title'], file=name, source=info['descriptionurl'], metadata=info['extmetadata'])
|
|
try:
|
|
if original.exists():
|
|
original.rename(target)
|
|
record['download_url'] = info['url']
|
|
elif not target.exists():
|
|
time.sleep(3)
|
|
url = info.get('thumburl', info['url'])
|
|
request = urllib.request.Request(url, headers={'User-Agent': 'SunriseReferenceCollector/1.0 (local research)'})
|
|
with urllib.request.urlopen(request, timeout=25) as response:
|
|
data = response.read()
|
|
if not data.startswith(b'\xff\xd8'):
|
|
raise ValueError('Not JPEG')
|
|
target.write_bytes(data)
|
|
record['download_url'] = url
|
|
record['status'] = 'downloaded'
|
|
except Exception as error:
|
|
record['status'] = str(error)
|
|
result.append(record)
|
|
print(name, record['status'], flush=True)
|
|
(root / 'downloads.json').write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding='utf-8')
|