tas_councils/scrapers/centralcoast.rb

# Central Coast Council — Current Planning Applications (site page)
# Source: https://www.centralcoast.tas.gov.au/current-planning-applications/

require "date"
require "nokogiri"
require "cgi"
require "fileutils"

require_relative "../lib/enrich"
require_relative "../lib/log"
require_relative "../lib/util"
TABLE = ENV.fetch("TABLE_NAME") # run_all.sh sets from filename: da_centralcoast
URL   = "https://www.centralcoast.tas.gov.au/current-planning-applications/"

DOWNLOAD_ATTACHMENTS = ENV["DOWNLOAD_ATTACHMENTS"] == "1"
DOWNLOAD_DIR         = ENV["DOWNLOAD_DIR"] || "/app/downloads"

DB.ensure_table!(TABLE)

def abs_url(base, href)
    return "" if href.to_s.strip.empty?
    URI.join(base, href).to_s
rescue URI::InvalidURIError
    href.to_s
end

def safe_name(s) = s.to_s.gsub(/[^\w\-.]+/, "_")

    def filename_from_response(res, fallback)
        cd = res.respond_to?(:[]) ? res["content-disposition"].to_s : ""
        if cd =~ /filename\*?=(?:UTF-8''|")?([^\";]+)/
            return safe_name($1)
        end
        base = safe_name(fallback || "document")
        ct   = res.respond_to?(:[]) ? res["content-type"].to_s.downcase : ""
        ext  = if base =~ /\.[A-Za-z0-9]{2,4}\z/
        ""
    elsif ct.include?("pdf")
        ".pdf"
    else
        ".bin"
    end
    "#{base}#{ext}"
end

def download_pdf(url, council_reference)
    return nil if url.to_s.strip.empty?

    dir = File.join(DOWNLOAD_DIR, "centralcoast", safe_name(council_reference))
    FileUtils.mkdir_p(dir)

    res  = (Http.get_response(url) rescue nil)
    body = if res && res.respond_to?(:body)
    res.body
else
    Http.get(url)
end

fname = filename_from_response(res, File.basename(URI.parse(url).path))
path  = File.join(dir, fname)
File.binwrite(path, body.to_s)
puts "  saved #{File.basename(path)} (#{body.to_s.bytesize} bytes)"

# adjust if your web container mounts differently
"/files/centralcoast/#{safe_name(council_reference)}/#{fname}"
rescue StandardError => e
Log.warn "scraper", "Download failed for #{url}: #{e.class} #{e.message}"
nil
end

# Normalize refs like "DA2025123" or "DA 2025/123" -> "DA 2025 / 123"
def normalize_ref(s)
    t = s.to_s
    if (m = t.match(/\bDA\s*([12]\d{3})\s*[-\/]?\s*0*([A-Za-z0-9]{2,})\b/i))
        return "DA #{m[1]} / #{m[2]}"
    elsif (m = t.match(/\bDA(20\d{2})\s*0*([A-Za-z0-9]{2,})\b/i))
        return "DA #{m[1]} / #{m[2]}"
    end
    t.split(/\s*-\s*/, 2).first # fallback
end

# Parse dd-mm-yyyy shown in "Date modified"
def parse_dd_mm_yyyy(s)
    s = s.to_s.strip
    if s =~ /\A\d{2}-\d{2}-\d{4}\z/
        Date.strptime(s, "%d-%m-%Y") rescue nil
    else
        # fall back to your common AUS parser
        Util.parse_aus_date(s)
    end
end

html = Http.get(URL)
doc  = Nokogiri::HTML(html)

rows = doc.css("table.wpfd-search-result tbody tr.file")
puts "Found #{rows.length} items for #{TABLE}"

saved = 0

rows.each do |tr|
    # Primary link + title from the first column
    a = tr.at_css("td.file_title a.wpfd_downloadlink")
    next unless a

    document_url    = abs_url(URL, a["href"].to_s)
    title_reference = a["title"].to_s.strip
    title_text      = tr.at_css("input.wpfd_file_preview_link_download")&.[]("data-filetitle").to_s.strip
    title_reference = title_reference.empty? ? title_text : title_reference

    # Title looks like: "DA2025123 - 148 Main Street, Ulverstone"
    council_reference_raw = title_reference.split(" - ").first.to_s.strip
    council_reference     = normalize_ref(council_reference_raw)

    address = if title_reference.include?(" - ")
    title_reference.split(" - ", 2)[1].to_s.strip
else
    # crude fallback to a street-like token
    title_reference[/\b\d+[A-Za-z]*\s[\w\s,]+/, 0].to_s.strip
end
address = title_reference[0, 140] if address.empty?

# Optional description column (often blank)
description = tr.at_css("td.file_desc")&.text.to_s.strip
description = "Development Application" if description.empty?

# Date modified -> date_received
date_received_raw = tr.at_css("td.file_modified")&.text.to_s.strip
date_received     = parse_dd_mm_yyyy(date_received_raw)

# Minimal required fields
next if council_reference.to_s.strip.empty? || address.to_s.strip.empty?

DB.upsert(TABLE, {
    description:        description,
    date_received:      date_received,
    date_received_raw:  date_received_raw,
    address:            address,
    council_reference:  council_reference,
    applicant:          "",
    owner:              ""
    })

enrich_after_upsert!(
    table: TABLE,
    council_reference: council_reference,
    address: address
    )

local_doc_url = nil
if DOWNLOAD_ATTACHMENTS && !document_url.empty?
    local_doc_url = download_pdf(document_url, council_reference)
end

begin
    upd = DB.client.prepare(
        "UPDATE `#{DB.client.escape(TABLE)}` " \
            "SET document_url = ?, local_document_url = ?, title_reference = ? " \
            "WHERE council_reference = ? AND address = ?"
        )
    upd.execute(document_url, local_doc_url, title_reference, council_reference, address)
rescue StandardError => e
    Log.warn "scraper", "Extras update skipped for #{council_reference}: #{e.class} #{e.message}"
end

puts "Upserted #{council_reference} -> #{address} #{local_doc_url ? '(downloaded)' : ''}"
saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."