tas_councils/scrapers/flinders_council.rb

# Flinders Council — Current Advertising scraper (site page, not PlanBuild)

require "nokogiri"
require "cgi"

require_relative "../lib/enrich"
require_relative "../lib/log"
TABLE = ENV.fetch("TABLE_NAME") # run_all.sh sets this from filename: da_flinders
URL   = "https://www.flinders.tas.gov.au/current-advertising"

DB.ensure_table!(TABLE)

def abs_url(base, href)
  return "" if href.to_s.strip.empty?
  URI.join(base, href).to_s rescue href.to_s
end

def extract_ref_from(text)
  s = CGI.unescape(text.to_s)
  # Pattern 1: "DA 2025 / 00017" or "DA 2025/00017"
  if (m = s.match(/DA\s*20\d{2}\s*\/\s*\d{3,}/i))
    return m[0].gsub(/\s+/, " ").sub(" / ", " / ").strip
  end
  # Pattern 2: "DA202500017" -> "DA 2025 / 00017"
  if (m = s.match(/DA20\d{2}\d{5}/i))
    raw = m[0]
    year = raw[2,4]
    num  = raw[6,5]
    return "DA #{year} / #{num}"
  end
  nil
end

html = Http.get(URL)
doc  = Nokogiri::HTML(html)

# Pick all advertised PDFs listed on the page
links = doc.css("a").select { |a|
  href = a["href"].to_s
  href.match?(%r{/client-assets/images/Development/Downloads/Advertising/}i) && href.downcase.end_with?(".pdf")
}

puts "Found #{links.length} items for #{TABLE}"

saved = 0
links.each do |a|
  text = a.text.strip.gsub(/\s+/, " ")
  pdf  = abs_url(URL, a["href"])

  # Use link text first, fall back to file name as address
  address = text.empty? ? File.basename(pdf).sub(/\.pdf\z/i, "") : text

  # Council reference from link text or filename
  ref = extract_ref_from(text) || extract_ref_from(File.basename(pdf))

  # Description and dates are usually inside the PDF, so keep minimal fields here
  description = "Development Application"
  date_received_raw = ""
  date_received = nil

  next if address.empty? || ref.nil?

  DB.upsert(TABLE, {
    description:       description,
    date_received:     date_received,
    date_received_raw: date_received_raw,
    address:           address,
    council_reference: ref,
    document_url:      pdf,
    applicant:         "",
    owner:             ""
  })

  enrich_after_upsert!(
    table:             TABLE,
    council_reference: ref,
    address:           address
  )

  puts "Upserted #{ref} -> #{address}"
  saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."