benjamin.harris
/
tas_councils


			
				
					
						
						
							123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
							# tools/send_summary_email.rb
#
# Sends an HTML summary email via SMTP when scraper errors are present.
# Called by run_all.sh; reads summary data from stdin in the format:
#
#   Line 1: finish timestamp (UTC)
#   Line 2: total scraper count
#   Line 3: total saved
#   Line 4: total warns
#   Lines 5+: pipe-delimited summary rows — name|saved|warns|status
#
# Required env vars (set via docker-compose.yml from .env):
#   SMTP_HOST, SMTP_PORT, SMTP_USERNAME, SMTP_PASSWORD,
#   SMTP_SMTPSecure, SMTP_SENTFROM, SMTP_ADDADDRESS

require "net/smtp"
require "openssl"

# ── Read SMTP config ────────────────────────────────────────────────────────
host     = ENV.fetch("SMTP_HOST",       "")
port     = ENV.fetch("SMTP_PORT",       "587").to_i
username = ENV.fetch("SMTP_USERNAME",   "")
password = ENV.fetch("SMTP_PASSWORD",   "")
secure   = ENV.fetch("SMTP_SMTPSecure", "tls").downcase   # "tls" or "ssl"
from     = ENV.fetch("SMTP_SENTFROM",   "")
to       = ENV.fetch("SMTP_ADDADDRESS", "")

if host.empty? || from.empty? || to.empty?
  warn "[send_summary_email] SMTP not configured — skipping email"
  exit 0
end

# ── Read stdin ───────────────────────────────────────────────────────────────
lines = $stdin.read.split("\n")
finish_time  = lines[0].to_s.strip
total_count  = lines[1].to_s.strip
total_saved  = lines[2].to_s.strip
total_warns  = lines[3].to_s.strip
entries      = lines[4..] || []

rows = entries.map do |e|
  parts = e.split("|")
  { name: parts[0].to_s, saved: parts[1].to_s, warns: parts[2].to_s, status: parts[3].to_s.strip }
end

error_rows   = rows.select { |r| r[:status] == "ERROR" }
blocked_rows = rows.select { |r| r[:status] == "blocked" }
warn_rows    = rows.select { |r| r[:status] == "warn" }

# ── Build subject ────────────────────────────────────────────────────────────
error_count = error_rows.size
subject = if error_count > 0
  "TAS Councils Scraper — #{error_count} error(s) — #{finish_time} UTC"
else
  "TAS Councils Scraper — completed with warnings — #{finish_time} UTC"
end

# ── Build HTML body ──────────────────────────────────────────────────────────
STATUS_COLOUR = {
  "ok"      => "#198754",
  "warn"    => "#856404",
  "blocked" => "#856404",
  "ERROR"   => "#dc3545"
}.freeze
STATUS_BG = {
  "ok"      => "#d1e7dd",
  "warn"    => "#fff3cd",
  "blocked" => "#fff3cd",
  "ERROR"   => "#f8d7da"
}.freeze

row_html = rows.map do |r|
  colour = STATUS_COLOUR.fetch(r[:status], "#333")
  bg     = STATUS_BG.fetch(r[:status], "#fff")
  <<~TR
    <tr style="background:#{bg}">
      <td style="padding:4px 10px;font-family:monospace">#{r[:name]}</td>
      <td style="padding:4px 10px;text-align:right">#{r[:saved]}</td>
      <td style="padding:4px 10px;text-align:right">#{r[:warns]}</td>
      <td style="padding:4px 10px;font-weight:bold;color:#{colour}">#{r[:status]}</td>
    </tr>
  TR
end.join

html_body = <<~HTML
  <html><body style="font-family:sans-serif;color:#333;max-width:700px">
    <h2 style="margin-bottom:4px">TAS Councils Scraper Summary</h2>
    <p style="color:#666;margin-top:0">Finished #{finish_time} UTC</p>

    #{error_rows.any? ? "<p style='color:#dc3545;font-weight:bold'>&#9888; #{error_rows.size} scraper(s) exited with errors: #{error_rows.map { |r| r[:name] }.join(', ')}</p>" : ""}
    #{blocked_rows.any? ? "<p style='color:#856404'>&#9888; #{blocked_rows.size} scraper(s) blocked by WAF/Cloudflare: #{blocked_rows.map { |r| r[:name] }.join(', ')}</p>" : ""}

    <table cellspacing="0" cellpadding="0" style="border-collapse:collapse;width:100%;margin-top:12px">
      <thead>
        <tr style="background:#343a40;color:#fff">
          <th style="padding:6px 10px;text-align:left">Council</th>
          <th style="padding:6px 10px;text-align:right">Saved</th>
          <th style="padding:6px 10px;text-align:right">Warns</th>
          <th style="padding:6px 10px;text-align:left">Status</th>
        </tr>
      </thead>
      <tbody>
        #{row_html}
      </tbody>
      <tfoot>
        <tr style="background:#f8f9fa;font-weight:bold">
          <td style="padding:6px 10px">TOTAL (#{total_count} scrapers)</td>
          <td style="padding:6px 10px;text-align:right">#{total_saved}</td>
          <td style="padding:6px 10px;text-align:right">#{total_warns}</td>
          <td></td>
        </tr>
      </tfoot>
    </table>
  </body></html>
HTML

# ── Compose RFC2822 message ──────────────────────────────────────────────────
boundary = "boundary_#{Time.now.to_i}"
message  = <<~MSG
  From: TAS Scraper <#{from}>
  To: #{to}
  Subject: #{subject}
  MIME-Version: 1.0
  Content-Type: multipart/alternative; boundary="#{boundary}"

  --#{boundary}
  Content-Type: text/plain; charset=UTF-8

  TAS Councils Scraper Summary — #{finish_time} UTC

  #{error_rows.any?  ? "ERRORS (#{error_rows.size}): #{error_rows.map { |r| r[:name] }.join(', ')}\n" : ""}#{blocked_rows.any? ? "BLOCKED (#{blocked_rows.size}): #{blocked_rows.map { |r| r[:name] }.join(', ')}\n" : ""}
  Council                           Saved  Warns  Status
  #{rows.map { |r| "%-34s %5s  %5s  %s" % [r[:name], r[:saved], r[:warns], r[:status]] }.join("\n  ")}

  TOTAL (#{total_count} scrapers)  #{total_saved} saved, #{total_warns} warns

  --#{boundary}
  Content-Type: text/html; charset=UTF-8

  #{html_body}
  --#{boundary}--
MSG

# ── Send ─────────────────────────────────────────────────────────────────────
begin
  smtp = Net::SMTP.new(host, port)

  if secure == "ssl"
    smtp.enable_tls(OpenSSL::SSL::SSLContext.new)
  else
    # "tls" → STARTTLS
    smtp.enable_starttls(OpenSSL::SSL::SSLContext.new)
  end

  smtp.start("localhost", username, password, :login) do |s|
    s.send_message(message, from, Array(to))
  end

  warn "[send_summary_email] Email sent to #{to}"
rescue StandardError => e
  warn "[send_summary_email] Failed to send email: #{e.class} #{e.message}"
  exit 1
end