tas_councils/scrapers/centralhighlands.rb

# Central Highlands Council — Development Applications (site page)

require "nokogiri"
require "cgi"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/util"
require_relative "../lib/enrich"

TABLE = ENV.fetch("TABLE_NAME") # run_all.sh -> da_centralhighlands
URL   = "https://centralhighlands.tas.gov.au/development-applications/"

DB.ensure_table!(TABLE)

# Optional: store the PDF URL
begin
  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
rescue => e
  warn "document_url add skipped: #{e.class} #{e.message}"
end

def abs_url(base, href)
  return "" if href.to_s.strip.empty?
  URI.join(base, href).to_s rescue href.to_s
end

# DA 2025/42, DA2025/00042, etc.
REF_RX = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
def extract_ref(str)
  s = CGI.unescape(str.to_s)
  if (m = s.match(%r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i))
    return "DA #{m[1]} / #{m[2]}"
  end
  if (m = s.match(%r{\bDA(20\d{2})\s*[-\/]?\s*([0-9]{3,})\b}i))
    return "DA #{m[1]} / #{m[2]}"
  end
  nil
end

def extract_addr(text)
  if (m = text.match(/Location:\s*(.+?)(?:\s*\/|\s{2,}|$)/i))
    m[1].strip
  else
    ""
  end
end

def extract_proposal(text)
  if (m = text.match(/Proposal:\s*(.+?)(?:\s{2,}|$)/i))
    m[1].strip
  else
    ""
  end
end

def extract_close_raw(text)
  s = text.gsub(/\s+/, " ")
  # “… until 20 August 2025”
  if (m = s.match(/\buntil\s+([0-9]{1,2}\s+[A-Za-z]{3,}\s+[0-9]{4})\b/i))
    return m[1]
  end
  # fallback date tokens
  return $1 if s =~ /(\b[0-9]{1,2}\/[0-9]{1,2}\/[0-9]{2,4}\b)/
  return $1 if s =~ /(\b[0-9]{1,2}\s+[A-Za-z]{3,}\s+[0-9]{4}\b)/
  ""
end

html = Http.get(URL)
doc  = Nokogiri::HTML(html)

container = doc.at_css("main, .entry-content, article") || doc

# Grab anchors that look like the advertised docs
links = container.css("a").select { |a|
  a.text =~ /click here to view application/i || a["href"].to_s.downcase.end_with?(".pdf")
}

puts "Found #{links.length} candidate link(s) for #{TABLE}"

saved = 0

links.each_with_index do |a, idx|
  pdf  = abs_url(URL, a["href"])
  # Walk up to a nearby block and use its text to find fields
  host = a.ancestors("p, li, div").first || a.parent
  text = host ? host.text.strip : ""
  text = text.gsub(/\s+/, " ")

  address       = extract_addr(text)
  description   = extract_proposal(text)
  on_notice_raw = extract_close_raw(text)
  on_notice     = Util.parse_aus_date(on_notice_raw)

  # Reference: proposal first, then file name, then surrounding text
  ref = extract_ref(description) || extract_ref(File.basename(pdf)) || extract_ref(text)

  # If we still have no address, fall back to a slice of the text
  address = text[0, 140] if address.empty?

  next if address.empty? || ref.nil?

  DB.upsert(TABLE, {
    description: description.empty? ? "Development Application" : description,
    date_received: on_notice,            # store close date here for consistency with others
    date_received_raw: on_notice_raw,
    address: address,
    council_reference: ref,
    applicant: "",
    owner: ""
  })
  
  enrich_after_upsert!(
    table: TABLE,
    council_reference: ref,
    address: address
  )

  begin
    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
    upd.execute(pdf, ref, address)
  rescue Mysql2::Error => e
    warn "[centralhighlands] db update skipped for #{ref}: #{e.message}"
  end

  puts "Upserted #{ref} -> #{address}"
  saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."