tas_councils/scrapers/westcoast.rb

# West Coast Council — Advertised Development Applications
# Source list: https://www.westcoast.tas.gov.au/planning-and-development/planning/advertised-development-applications/
require "date"
require "nokogiri"
require "cgi"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/util"
require_relative "../lib/geocode"
require_relative "../lib/enrich"

TABLE    = ENV.fetch("TABLE_NAME")  # run_all.sh -> da_westcoast
URL = "https://www.westcoast.tas.gov.au/planning-and-development/planning/advertised-development-applications/"

DOWNLOAD_ATTACHMENTS = ENV["DOWNLOAD_ATTACHMENTS"] == "1"
DOWNLOAD_DIR         = ENV["DOWNLOAD_DIR"] || "/app/downloads"

DB.ensure_table!(TABLE)
ensure_extra_columns!(TABLE)  

def abs_url(base, href)
  return "" if href.to_s.strip.empty?
  URI.join(base, href).to_s rescue href.to_s
end

# Accepts DA2025-26, DA 2025/26, DA2025/026, DA 26-2025
REF_RXES = [
  %r{\bDA\s*(20\d{2})\s*[/\-]\s*([A-Za-z0-9\-_.]{1,})\b}i,  # DA 2025/26 or DA2025-26
  %r{\bDA\s*(20\d{2})([A-Za-z0-9]{3,})\b}i,                 # DA2025012
  %r{\bDA\s*([0-9]{1,4})\s*-\s*(20\d{2})\b}i               # DA 26-2025
]
def extract_ref(str)
  s = CGI.unescape(str.to_s)
  REF_RXES.each do |rx|
    if (m = s.match(rx))
      # Normalize to "DA YYYY / NNN"
      if rx.source.include?("\\s*\\-\\s*(20") && m[2] # hyphen reversed
        return "DA #{m[2]} / #{m[1]}"
      elsif rx.source.include?("(20\\d{2})([A-Za-z0-9]{3,})")
        return "DA #{m[1]} / #{m[2]}"
      else
        return "DA #{m[1]} / #{m[2]}"
      end
    end
  end
  nil
end

def extract_date_token(str)
  s = str.to_s
  return $1 if s =~ /(\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/
  return $1 if s =~ /(\b\d{1,2}\s+[A-Za-z]{3,}\s+\d{4}\b)/
  return $1 if s =~ /(\b[A-Za-z]{3,}\s+\d{1,2},?\s+\d{4}\b)/
  ""
end

def extract_on_notice_raw(text)
  s = text.to_s.gsub(/\s+/, " ")
  # West Coast detail pages usually say: "Representations must be received by 5pm on Monday 1 September 2025"
  if s =~ /representations? .*? (received|made) .*? by .*?([A-Za-z0-9\/ ,]+)/i
    d = extract_date_token($2)
    return d unless d.empty?
  end
  if s =~ /\bon\s*notice\s*(until|to)\s*[:\-]?\s*([A-Za-z0-9\/ ,]+)/i
    d = extract_date_token($2)
    return d unless d.empty?
  end
  extract_date_token(s)
end

def nearest_ctx(a)
  host = a.ancestors("article, li, p, div").first || a.parent
  host ? host.text.to_s.strip.gsub(/\s+/, " ") : ""
end

def parse_detail(url)
  html = Http.get(url)
  doc  = Nokogiri::HTML(html)

  title_reference = doc.at_css("h1, .entry-title")&.text&.strip.to_s
  page_text = doc.text.to_s.gsub(/\s+/, " ")

  # Reference from title or page body
  council_reference = extract_ref(title_reference) || extract_ref(page_text)

  # Address from title after ":" if present, else from first " at <address>" phrase
  address = if title_reference.include?(":")
    title_reference.split(":", 2)[1].to_s.strip
  elsif (m = page_text.match(/\bat\s+([A-Z0-9].+?)(?:\.\s|, Representations| In accordance|$)/i))
    m[1].strip
  else
    title_reference
  end
  address = address[0, 140] if address.length > 140

  # Description often appears near the top: "Residential – Outbuilding", etc
  description = if (m = page_text.match(/(Residential|Commercial|Industrial|Subdivision|Outbuilding|Dwelling|Multiple Dwelling|Alterations|Additions|Use|Development)\s*[–-]\s*([A-Za-z ]+)/i))
    [m[1], m[2]].join(" - ").strip
  else
    "Development Application"
  end

  # On-notice
  on_notice_raw = extract_on_notice_raw(page_text)
  on_notice     = Util.parse_aus_date(on_notice_raw)

  # First PDF on the page
  pdf = doc.at_css("a[href$='.pdf'], a[href*='.pdf?']")&.[]("href")
  document_url = pdf ? abs_url(url, pdf) : ""

  return nil if council_reference.to_s.strip.empty? || address.to_s.strip.empty?

  {
    council_reference: council_reference,
    address: address,
    description: description,
    on_notice: on_notice,
    on_notice_raw: on_notice_raw,
    document_url: document_url,
    title_reference: title_reference
  }
end

# 1) Open the list page and find links to individual Development Application posts
list_html = Http.get(URL)
list_doc  = Nokogiri::HTML(list_html)

detail_links = list_doc.css("a").map { |a|
  href = a["href"].to_s
  next if href.empty? || href.start_with?("#")
  u = abs_url(URL, href)
  u.include?("/development-application/")
}.compact

# Convert booleans from map to urls properly
detail_links = list_doc.css("a").map { |a|
  href = a["href"].to_s
  u = abs_url(URL, href)
  u if u.include?("/development-application/")
}.compact.uniq

puts "Found #{detail_links.size} candidate link(s) for #{TABLE}"

saved = 0

date_received	 = Date.today

detail_links.each do |u|
  begin
    item = parse_detail(u)
  rescue => e
    warn "Skip #{u}: #{e.class} #{e.message}"
    next
  end
  next unless item

  DB.upsert(TABLE, {
    description: item[:description],
    date_received: date_received,
	date_received_raw: date_received,
	on_notice: item[:date_received],       # store close date here to be consistent
    on_notice_raw: item[:date_received_raw],
    address: item[:address],
    council_reference: item[:council_reference],
    applicant: "",
    owner: ""
  })
  
  enrich_after_upsert!(
    table: TABLE,
    council_reference: item[:council_reference],
    address: address
  )

  begin
    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ?, on_notice_to = ?, on_notice_to_raw = ?, title_reference = ? WHERE council_reference = ? AND address = ?")
    upd.execute(item[:document_url], item[:date_received], item[:date_received_raw], item[:title_reference], item[:council_reference], item[:address])
  rescue => e
    warn "Extras update skipped for #{item[:council_reference]}: #{e.class} #{e.message}"
  end

  puts "Upserted #{item[:council_reference]} -> #{item[:address]}"
  saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."