tas_councils/scrapers/westtamar.rb

# West Tamar Council — Advertised Planning Applications

require "nokogiri"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/util"
require_relative "../lib/enrich"

TABLE    = ENV.fetch("TABLE_NAME")   # run_all.sh -> da_westtamar
URL = "https://www.wtc.tas.gov.au/advertised-planning-applications/"

DB.ensure_table!(TABLE)

begin
  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url TEXT NULL")
  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to DATE NULL")
  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to_raw VARCHAR(80) NULL")
  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS title_reference TEXT NULL")
rescue => e
  warn "Optional column add skipped: #{e.class} #{e.message}"
end

def abs_url(base, href)
  return "" if href.to_s.strip.empty?
  URI.join(base, href).to_s rescue href.to_s
end

REF_RX_SLASH  = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
REF_RX_HYPHEN = %r{\bDA\s*(\d{1,4})\s*-\s*(20\d{2})\b}i
def extract_ref(text)
  s = text.to_s
  if (m = s.match(REF_RX_SLASH))
    return "DA #{m[1]} / #{m[2]}"
  end
  if (m = s.match(REF_RX_HYPHEN))
    return "DA #{m[2]} / #{m[1]}"
  end
  if (m = s.match(/\bDA(20\d{2})(\d{3,})\b/i))
    return "DA #{m[1]} / #{m[2]}"
  end
  nil
end

def extract_date_like(str)
  s = str.to_s
  return $1 if s =~ /(\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/
  return $1 if s =~ /(\b\d{1,2}\s+[A-Za-z]{3,}\s+\d{4}\b)/
  return $1 if s =~ /(\b[A-Za-z]{3,}\s+\d{1,2},?\s+\d{4}\b)/
  ""
end

def extract_on_notice_raw(text)
  s = text.to_s.gsub(/\s+/, " ")
  if s =~ /\bon\s*notice\s*(until|to)\s*[:\-]?\s*([A-Za-z0-9\/ ,]+)/i
    d = extract_date_like($2)
    return d unless d.empty?
  end
  if s =~ /clos(?:e|ing|es)\s*(on)?\s*[:\-]?\s*([A-Za-z0-9\/ ,]+)/i
    d = extract_date_like($2)
    return d unless d.empty?
  end
  extract_date_like(s)
end

def parse_detail(url)
  html = Http.get(url)
  doc  = Nokogiri::HTML(html)

  # Try two-column detail table first
  kv = {}
  doc.css("table tr").each do |tr|
    cells = tr.css("th, td")
    next unless cells.length >= 2
    key = cells[0].text.strip
    val = cells[1].text.strip
    kv[key] = val unless key.empty?
  end

  find = ->(rx) { kv.find { |k,_| k =~ rx }&.last.to_s.strip }

  council_reference = find.call(/(Application\s*(No|Number|ID)|Reference)/i)
  address           = find.call(/(Address|Location|Property)/i)
  description       = find.call(/(Proposal|Description)/i)
  on_notice_raw     = find.call(/(On\s*Notice\s*(until|to)|Closing\s*Date|Closes)/i)
  on_notice         = Util.parse_aus_date(on_notice_raw)
  title_reference   = doc.at_css("h1, .entry-title")&.text&.strip.to_s

  # Fallbacks from page text if labels are missing
  if council_reference.empty?
    council_reference = extract_ref(title_reference) || extract_ref(doc.text)
  end
  address = title_reference if address.empty?
  description = "Development Application" if description.to_s.strip.empty?
  if on_notice.nil?
    guess = extract_on_notice_raw(doc.text)
    on_notice = Util.parse_aus_date(guess)
    on_notice_raw = guess if on_notice
  end

  pdf = doc.at_css("a[href$='.pdf'], a[href*='.pdf?']")&.[]("href")
  document_url = pdf ? abs_url(url, pdf) : ""

  return nil if council_reference.empty? || address.empty?

  {
    council_reference: council_reference,
    address: address,
    description: description,
    date_received: on_notice,
    date_received_raw: on_notice_raw.to_s,
    document_url: document_url,
    title_reference: title_reference
  }
end

list_html = Http.get(URL)
list_doc  = Nokogiri::HTML(list_html)

detail_links = list_doc.css("article h2 a, .entry-content a").map { |a|
  href = a["href"].to_s
  next if href.strip.empty? || href.start_with?("#")
  abs_url(URL, href)
}.compact.uniq

puts "Found #{detail_links.size} candidate link(s) for #{TABLE}"

saved = 0

detail_links.each do |u|
  begin
    item = parse_detail(u)
  rescue => e
    warn "Skip #{u}: #{e.class} #{e.message}"
    next
  end
  next unless item

  DB.upsert(TABLE, {
    description: item[:description],
    date_received: item[:date_received],
    date_received_raw: item[:date_received_raw],
    address: item[:address],
    council_reference: item[:council_reference],
    applicant: "",
    owner: ""
  })
  
  enrich_after_upsert!(
    table: TABLE,
    council_reference: council_reference,
    address: address
  )

  begin
    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ?, on_notice_to = ?, on_notice_to_raw = ?, title_reference = ? WHERE council_reference = ? AND address = ?")
    upd.execute(item[:document_url], item[:date_received], item[:date_received_raw], item[:title_reference], item[:council_reference], item[:address])
  rescue Mysql2::Error => e
    warn "[westtamar] db update skipped for #{item[:council_reference]}: #{e.message}"
  end

  puts "Upserted #{item[:council_reference]} -> #{item[:address]}"
  saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."