refactor(deploy): use date-stamped db assets and resolve latest at download time

Upload always creates a date-stamped asset (explainshell-{date}.db.zst) instead
of renaming the previous one. A shared download-latest-db.sh script queries the
GitHub API to find the newest asset and verifies its SHA256 from release metadata.
Both the Makefile target (renamed to download-latest-db) and Dockerfile use it.

Author: idank

AGENTS.md

@@ -247,5 +247,5 @@ fly deploy
 
 **Update the database:**
 
-1. `make upload-live-db` (uploads `explainshell.db.zst` to the GitHub release)
+1. `make upload-live-db` (uploads a date-stamped `explainshell-{date}.db.zst` to the GitHub release)
 2. `fly deploy` (rebuilds the image with the new DB)

Dockerfile

@@ -1,19 +1,15 @@
 FROM python:3.12-slim
 
-ARG DB_URL=https://github.com/idank/explainshell/releases/download/db-latest/explainshell.db.zst
-
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends wget zstd \
+  && apt-get install -y --no-install-recommends wget zstd jq \
   && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt/webapp
 COPY requirements.txt .
 RUN pip3 install --no-cache-dir --no-warn-script-location -r requirements.txt
 
-RUN wget -q -O explainshell.db.zst "$DB_URL" \
-  && sha256sum explainshell.db.zst | awk '{print $1}' > explainshell.db.sha256 \
-  && echo "db sha256: $(cat explainshell.db.sha256)" \
-  && zstd -d --rm explainshell.db.zst
+COPY tools/download-latest-db.sh tools/
+RUN tools/download-latest-db.sh explainshell.db
 
 COPY start.sh .
 COPY explainshell/ explainshell/

Makefile

@@ -73,16 +73,11 @@ arch-archive:
 		--distro arch --sections 1,1p,8 --output-dir manpages
 
 LIVE_DB := explainshell.db
-LIVE_DB_ASSET := $(LIVE_DB).zst
-LIVE_DB_RELEASE := db-latest
-LIVE_DB_REPO := idank/explainshell
-LIVE_DB_CDN_URL := https://github.com/$(LIVE_DB_REPO)/releases/download/$(LIVE_DB_RELEASE)/$(LIVE_DB_ASSET)
 
-download-live-db:
-	gh release download $(LIVE_DB_RELEASE) -R $(LIVE_DB_REPO) -p $(LIVE_DB_ASSET) -D . --clobber
-	zstd -d --rm $(LIVE_DB_ASSET)
+download-latest-db:
+	tools/download-latest-db.sh $(LIVE_DB)
 
 upload-live-db:
 	tools/upload-live-db.sh $(LIVE_DB)
 
-.PHONY: tests e2e e2e-db e2e-update test-llm tests-all tests-quick lint serve parsing-regression parsing-update db-check ubuntu-archive arch-archive download-live-db upload-live-db
+.PHONY: tests e2e e2e-db e2e-update test-llm tests-all tests-quick lint serve parsing-regression parsing-update db-check ubuntu-archive arch-archive download-latest-db upload-live-db

README.md

@@ -38,7 +38,7 @@ $ source .venv/bin/activate
 $ pip install -r requirements-dev.txt
 
 # Download the live db, or parse a manpage.
-$ make download-live-db
+$ make download-latest-db
 $ python -m explainshell.manager extract --mode source manpages/ubuntu/26.04/1/tar.1.gz
 
 # Run the web server

tools/download-latest-db.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -euo pipefail
+
+OUTPUT="${1:-explainshell.db}"
+REPO="idank/explainshell"
+RELEASE="db-latest"
+API_URL="https://api.github.com/repos/$REPO/releases/tags/$RELEASE"
+
+RELEASE_JSON=$(wget -qO- "$API_URL")
+ASSET=$(echo "$RELEASE_JSON" \
+    | jq -r '[.assets[] | select(.name | test("^explainshell-.*\\.db\\.zst$"))] | sort_by(.created_at) | last | .name')
+EXPECTED_SHA=$(echo "$RELEASE_JSON" \
+    | jq -r '[.assets[] | select(.name | test("^explainshell-.*\\.db\\.zst$"))] | sort_by(.created_at) | last | .digest' \
+    | sed 's/^sha256://')
+
+if [ -z "$ASSET" ] || [ "$ASSET" = "null" ]; then
+    echo "No db asset found in release $RELEASE" >&2
+    exit 1
+fi
+
+echo "Downloading $ASSET..."
+wget -q -O "$ASSET" "https://github.com/$REPO/releases/download/$RELEASE/$ASSET"
+
+ACTUAL_SHA=$(sha256sum "$ASSET" | awk '{print $1}')
+if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
+    echo "SHA256 mismatch: expected $EXPECTED_SHA, got $ACTUAL_SHA" >&2
+    rm -f "$ASSET"
+    exit 1
+fi
+echo "$ACTUAL_SHA" > "$OUTPUT.sha256"
+echo "SHA256 verified: $ACTUAL_SHA"
+
+zstd -d -f "$ASSET" -o "$OUTPUT"
+rm -f "$ASSET"
+echo "Wrote $OUTPUT"

tools/upload-live-db.sh

@@ -4,36 +4,32 @@ set -euo pipefail
 DB="${1:?Usage: $0 <db-file>}"
 REPO="idank/explainshell"
 RELEASE="db-latest"
-ASSET="explainshell.db.zst"
-CDN_URL="https://github.com/$REPO/releases/download/$RELEASE/$ASSET"
+DATE=$(date -u +%Y-%m-%d-%H%M%S)
+ASSET="explainshell-${DATE}.db.zst"
 
 test -f "$DB" || { echo "$DB not found"; exit 1; }
 
-# --- Compress and check if upload is needed ---
+# --- Compress ---
 zstd -1 -f "$DB" -o "$ASSET"
-local_sha=$(sha256sum "$ASSET" | awk '{print $1}')
-remote_sha=$(gh api "repos/$REPO/releases/tags/$RELEASE" \
-    --jq ".assets[] | select(.name == \"$ASSET\") | .digest" 2>/dev/null \
-    | sed 's/^sha256://')
-
-if [ "$local_sha" = "$remote_sha" ]; then
-    echo "Local DB digest ($local_sha) matches release. Nothing to upload."
-    rm -f "$ASSET"
-    exit 0
-fi
 
-echo "Local digest:  $local_sha"
-echo "Release digest: ${remote_sha:-<none>}"
-
-# --- Archive existing asset ---
-asset_id=$(gh api "repos/$REPO/releases/tags/$RELEASE" --jq ".assets[] | select(.name == \"$ASSET\") | .id")
-if [ -n "$asset_id" ]; then
-    upload_date=$(gh api "repos/$REPO/releases/tags/$RELEASE" \
-        --jq ".assets[] | select(.name == \"$ASSET\") | .updated_at" \
-        | tr -d 'Z' | tr 'T:' '-')
-    archive_name="explainshell-${upload_date}.db.zst"
-    echo "Renaming existing asset to $archive_name..."
-    gh api "repos/$REPO/releases/assets/$asset_id" -X PATCH -f name="$archive_name" --silent
+# --- Check if upload is needed by comparing to the newest existing asset ---
+newest_asset=$(gh api "repos/$REPO/releases/tags/$RELEASE" \
+    --jq '[.assets[] | select(.name | test("^explainshell-.*\\.db\\.zst$"))] | sort_by(.created_at) | last | .name' 2>/dev/null || true)
+
+if [ -n "$newest_asset" ]; then
+    remote_sha=$(gh api "repos/$REPO/releases/tags/$RELEASE" \
+        --jq ".assets[] | select(.name == \"$newest_asset\") | .digest" 2>/dev/null \
+        | sed 's/^sha256://')
+    local_sha=$(sha256sum "$ASSET" | awk '{print $1}')
+
+    if [ "$local_sha" = "$remote_sha" ]; then
+        echo "Local DB digest ($local_sha) matches latest release asset ($newest_asset). Nothing to upload."
+        rm -f "$ASSET"
+        exit 0
+    fi
+
+    echo "Local digest:  $local_sha"
+    echo "Latest asset:  $newest_asset (digest: ${remote_sha:-<none>})"
 fi
 
 # --- Upload ---
@@ -47,17 +43,5 @@ curl --progress-bar \
     --data-binary @"$ASSET" \
     "${upload_url}?name=$ASSET" | cat
 
-# --- Wait for CDN ---
-expected_size=$(wc -c < "$ASSET")
 rm -f "$ASSET"
-
-echo "Waiting for CDN to serve the new file ($expected_size bytes)..."
-while true; do
-    cdn_size=$(curl -sI -L "$CDN_URL" | grep -i content-length | tail -1 | tr -d '[:space:]' | cut -d: -f2)
-    if [ "$cdn_size" = "$expected_size" ]; then
-        echo "CDN updated."
-        break
-    fi
-    echo "  CDN still serving $cdn_size bytes, expected $expected_size. Retrying in 10s..."
-    sleep 10
-done
+echo "Uploaded $ASSET to release $RELEASE."