tas_councils/scrapers/westtamar.rb

# West Tamar Council — Advertised Planning Applications

require "nokogiri"
require_relative "../lib/http"
require_relative "../lib/util"
require_relative "../lib/scraper_helpers"

TABLE    = ENV.fetch("TABLE_NAME")   # run_all.sh -> da_westtamar
URL = "https://www.wtc.tas.gov.au/advertised-planning-applications/"

DB.ensure_table!(TABLE)

REF_RX_SLASH  = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
REF_RX_HYPHEN = %r{\bDA\s*(\d{1,4})\s*-\s*(20\d{2})\b}i
def extract_ref(text)
  s = text.to_s
  if (m = s.match(REF_RX_SLASH))
    return "DA #{m[1]} / #{m[2]}"
  end
  if (m = s.match(REF_RX_HYPHEN))
    return "DA #{m[2]} / #{m[1]}"
  end
  if (m = s.match(/\bDA(20\d{2})(\d{3,})\b/i))
    return "DA #{m[1]} / #{m[2]}"
  end
  nil
end

def extract_date_like(str)
  s = str.to_s
  return $1 if s =~ /(\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/
  return $1 if s =~ /(\b\d{1,2}\s+[A-Za-z]{3,}\s+\d{4}\b)/
  return $1 if s =~ /(\b[A-Za-z]{3,}\s+\d{1,2},?\s+\d{4}\b)/
  ""
end

def extract_on_notice_raw(text)
  s = text.to_s.gsub(/\s+/, " ")
  if s =~ /\bon\s*notice\s*(until|to)\s*[:\-]?\s*([A-Za-z0-9\/ ,]+)/i
    d = extract_date_like($2)
    return d unless d.empty?
  end
  if s =~ /clos(?:e|ing|es)\s*(on)?\s*[:\-]?\s*([A-Za-z0-9\/ ,]+)/i
    d = extract_date_like($2)
    return d unless d.empty?
  end
  extract_date_like(s)
end

def parse_detail(url)
  html = Http.get(url)
  doc  = Nokogiri::HTML(html)

  # Try two-column detail table first
  kv = {}
  doc.css("table tr").each do |tr|
    cells = tr.css("th, td")
    next unless cells.length >= 2
    key = cells[0].text.strip
    val = cells[1].text.strip
    kv[key] = val unless key.empty?
  end

  find = ->(rx) { kv.find { |k,_| k =~ rx }&.last.to_s.strip }

  council_reference = find.call(/(Application\s*(No|Number|ID)|Reference)/i)
  address           = find.call(/(Address|Location|Property)/i)
  description       = find.call(/(Proposal|Description)/i)
  on_notice_raw     = find.call(/(On\s*Notice\s*(until|to)|Closing\s*Date|Closes)/i)
  on_notice         = Util.parse_aus_date(on_notice_raw)
  title_reference   = doc.at_css("h1, .entry-title")&.text&.strip.to_s

  # Fallbacks from page text if labels are missing
  if council_reference.empty?
    council_reference = extract_ref(title_reference) || extract_ref(doc.text)
  end
  address = title_reference if address.empty?
  description = "Development Application" if description.to_s.strip.empty?
  if on_notice.nil?
    guess = extract_on_notice_raw(doc.text)
    on_notice = Util.parse_aus_date(guess)
    on_notice_raw = guess if on_notice
  end

  pdf = doc.at_css("a[href$='.pdf'], a[href*='.pdf?']")&.[]("href")
  document_url = pdf ? abs_url(url, pdf) : ""

  return nil if council_reference.empty? || address.empty?

  {
    council_reference: council_reference,
    address: address,
    description: description,
    date_received: on_notice,
    date_received_raw: on_notice_raw.to_s,
    document_url: document_url,
    title_reference: title_reference
  }
end

list_html = Http.get(URL)
list_doc  = Nokogiri::HTML(list_html)

detail_links = list_doc.css("article h2 a, .entry-content a").map { |a|
  href = a["href"].to_s
  next if href.strip.empty? || href.start_with?("#")
  abs_url(URL, href)
}.compact.uniq

puts "Found #{detail_links.size} candidate link(s) for #{TABLE}"

saved = 0

detail_links.each do |u|
  begin
    item = parse_detail(u)
  rescue => e
    warn "Skip #{u}: #{e.class} #{e.message}"
    next
  end
  next unless item

  upsert_and_enrich!(
    table: TABLE,
    row: {
      description: item[:description],
      date_received: item[:date_received],
      date_received_raw: item[:date_received_raw],
      address: item[:address],
      council_reference: item[:council_reference],
      applicant: "",
      owner: ""
    },
    extras: {
      document_url:    item[:document_url],
      on_notice_to:    item[:date_received],
      on_notice_to_raw: item[:date_received_raw],
      title_reference: item[:title_reference]
    }
  )
  saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."