"""Fetch every live row for one country and write NDJSON.

    export JOA_KEY=...
    python3 daily_pull.py IE

Resumable: the cursor is persisted after each page, so an interrupted run
continues rather than restarting. Rows are keyed on id, so re-processing a
page costs records but corrupts nothing.
"""
import json, os, pathlib, sys, urllib.parse, urllib.request

API     = "https://api.jobopportunitiesapi.org"
KEY     = os.environ["JOA_KEY"]
country = sys.argv[1] if len(sys.argv) > 1 else "IE"
state   = pathlib.Path(f".joa-{country}.cursor")
out     = pathlib.Path(f"{country}.ndjson")

def get(params):
    req = urllib.request.Request(
        f"{API}/v1/jobs?" + urllib.parse.urlencode(params),
        headers={"Authorization": f"Bearer {KEY}"},
    )
    with urllib.request.urlopen(req, timeout=90) as r:
        return json.load(r)

cursor = state.read_text().strip() if state.exists() else ""
rows = 0
with out.open("a") as f:
    while True:
        params = {"country": country, "limit": 200}
        if cursor:
            params["cursor"] = cursor
        page = get(params)
        for job in page["data"]:
            f.write(json.dumps(job, separators=(",", ":")) + "\n")
            rows += 1
        f.flush()
        if not page.get("has_more") or not page.get("next_cursor"):
            state.unlink(missing_ok=True)   # finished: forget the cursor
            break
        cursor = page["next_cursor"]
        state.write_text(cursor)           # AFTER writing the page
print(f"{rows} rows -> {out}")

#!/bin/bash
# Mark rows stale in our own store. Idempotent; safe to run hourly.
set -euo pipefail
A=https://api.jobopportunitiesapi.org
STATE=${STATE:-.joa-expired-cursor}
since=$(cat "$STATE" 2>/dev/null || echo '2026-08-01T00:00:00Z')

while : ; do
  page=$(curl -s --fail-with-body -H "Authorization: Bearer $JOA_KEY" \
         --get "$A/v1/jobs/expired" \
         --data-urlencode "since=$since" --data-urlencode 'limit=1000')
  count=$(jq -r '.count' <<<"$page")
  jq -r '.data[] | "\(.id)\t\(.closed_at)\t\(.closed_reason)"' <<<"$page" \
    | while IFS=$'\t' read -r id at reason; do
        echo "UPDATE jobs SET closed_at='$at', closed_reason='$reason' WHERE id='$id';"
      done
  # Persist AFTER emitting, so a crash re-reads rather than skips.
  jq -r '.next_since' <<<"$page" > "$STATE"
  [ "$count" -eq 1000 ] || break   # short page: current
done

#!/bin/bash
set -euo pipefail
A=https://api.jobopportunitiesapi.org
out=${1:-corpus.ndjson}
touch "$out"

for attempt in 1 2 3 4 5; do
  after=$(tail -1 "$out" 2>/dev/null | jq -r '.id // empty' 2>/dev/null || true)
  if curl -sN --fail-with-body -H "Authorization: Bearer $JOA_KEY" \
       --get "$A/v1/export" \
       --data-urlencode 'country=IE' \
       ${after:+--data-urlencode "after=$after"} >> "$out"; then
    break
  fi
  echo "attempt $attempt interrupted after ${after:-start}" >&2
  sleep $((attempt * 5))
done

# The stream can end on an error object inside a 200. Silence here is a
# truncated dataset that looks complete.
if tail -1 "$out" | jq -e 'has("error")' >/dev/null 2>&1; then
  echo "INCOMPLETE: $(tail -1 "$out" | jq -r '.error')" >&2
  echo "resume with after=$(tail -1 "$out" | jq -r '.after')" >&2
  exit 1
fi
echo "$(wc -l < "$out") rows"