tas_councils/run_all.sh

#!/usr/bin/env bash
set -euo pipefail

echo "Starting run at $(date -u +"%Y-%m-%d %H:%M:%S")"

ONLY_LIST="${ONLY:-}"     # e.g. "meandervalley" or "kentish,break_oday"
SKIP_LIST="${SKIP:-}"     # e.g. "hobartcity,latrobe"
DEBUG_FLAG="${DEBUG:-}"   # pass through to scrapers
DRY_FLAG="${DRY_RUN:-}"   # pass through to scrapers

shopt -s nullglob
SCRIPTS=(/app/scrapers/*.rb)

should_run() {
  local name="$1"

  if [[ -n "$ONLY_LIST" ]]; then
    IFS=',' read -ra arr <<< "$ONLY_LIST"
    local pick
    for pick in "${arr[@]}"; do
      pick="${pick// /}"
      if [[ "$name" == "$pick" ]]; then
        echo "1"
        return
      fi
    done
    echo "0"
    return
  fi

  if [[ -n "$SKIP_LIST" ]]; then
    IFS=',' read -ra arr <<< "$SKIP_LIST"
    local skip
    for skip in "${arr[@]}"; do
      skip="${skip// /}"
      if [[ "$name" == "$skip" ]]; then
        echo "0"
        return
      fi
    done
  fi

  echo "1"
}

count=0
for f in "${SCRIPTS[@]}"; do
  name="$(basename "$f" .rb)"
  table="da_${name}"

  if [[ "$(should_run "$name")" != "1" ]]; then
    continue
  fi

  echo "Running ${name} -> table ${table}"
  TABLE_NAME="$table" DEBUG="$DEBUG_FLAG" DRY_RUN="$DRY_FLAG" ruby "$f" || echo "Error in $f"
  count=$((count+1))
done

echo "Finished run at $(date -u +"%Y-%m-%d %H:%M:%S"). Ran ${count} scraper(s)."