#!/usr/bin/env bash
# fetch.sh: download Ganjoor's per-poet SQLite databases (the ones its apps use) and unzip them.
#   bash research/the-word-that-comes-back/fetch.sh /some/dir      # then: measure.mjs /some/dir/x
# About 107 MB zipped, 290 MB unzipped. Ganjoor republishes these; the list this study used is
# committed as data/androidgdbs-2026-09-25.xml and the checksums of its zips as data/gdb-zips.sha256.
# A newer download may differ from ours where Ganjoor's editors have corrected a text since.
set -euo pipefail
DIR="${1:?usage: fetch.sh <dir>}"
mkdir -p "$DIR/x"
cd "$DIR"
curl -sS -m 120 -o androidgdbs.xml "https://i.ganjoor.net/android/androidgdbs.xml"
grep -o '<DownloadUrl>[^<]*' androidgdbs.xml | cut -d'>' -f2 > urls.txt
echo "$(wc -l < urls.txt) databases listed"
xargs -P 6 -I{} sh -c 'f=$(basename "{}"); [ -s "$f" ] || curl -sS -m 300 --retry 4 -o "$f" "{}"' < urls.txt
for f in *.zip; do unzip -o -q "$f" -d x; done
echo "unzipped $(ls x/*.gdb | wc -l) databases into $DIR/x"
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if sha256sum --quiet -c "$HERE/data/gdb-zips.sha256" 2>/dev/null; then echo "every zip matches the copy this study used"
else echo "some zips differ from the copy this study used (Ganjoor has republished); expect small differences"; fi