tas_councils/scrapers/tasman.rb

# Tasman Council — Advertised Applications (site page, not PlanBuild)

require "nokogiri"
require "date"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/util"
require_relative "../lib/enrich"

TABLE = ENV.fetch("TABLE_NAME")  # run_all.sh sets from filename: da_tasman
URL   = "https://tasman.tas.gov.au/advertised-applications/"

DB.ensure_table!(TABLE)

# Optional: store the PDF link
begin
  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
rescue => e
  warn "document_url add skipped: #{e.class} #{e.message}"
end

def abs_url(base, href)
  return "" if href.to_s.strip.empty?
  URI.join(base, href).to_s rescue href.to_s
end

# Nokogiri CSS :contains(...) is a bit special. Use a safer find for the Date row.
def find_date_from_details(row)
  details = row.at_css(".details")
  return "" unless details
  details.css("tr").each do |tr|
    tds = tr.css("td")
    next unless tds.length >= 2
    key = tds[0].text.strip
    val = tds[1].text.strip
    return val if key =~ /\bDate\b/i
  end
  ""
end

html = Http.get(URL)
doc  = Nokogiri::HTML(html)

items = doc.css(".wpfilebase-file-default")
puts "Found #{items.length} items for #{TABLE}"

saved = 0

items.each_with_index do |row, idx|
  link = row.at_css(".filetitle a")
  next unless link

  title_text    = link.text.strip
  document_url  = abs_url(URL, link["href"])

  # Common pattern on this page is "REF - Address - On notice date"
  council_reference = title_text.split(" - ").first.to_s.strip
  council_reference = council_reference.empty? ? title_text[0,120] : council_reference

  # Use title as address if nothing cleaner is available
  address = title_text

  # On-notice date often appears inside the title
  on_notice_to_raw = if (m = title_text.match(/(\d{1,2}\s+[A-Za-z]+\s+\d{4})/))
    m[1]
  else
    ""
  end
  on_notice_to = Util.parse_aus_date(on_notice_to_raw)

  # Application date is shown inside the details table under "Date"
  date_received_raw = find_date_from_details(row)
  date_received =
    Util.parse_aus_date(date_received_raw) ||
    begin
      s = date_received_raw.to_s
      s.empty? ? nil : Date.strptime(s, "%A, %d %B, %Y") rescue nil
    end

  description = "Development Application"

  # Require core fields
  next if council_reference.empty? || address.empty?

  DB.upsert(TABLE, {
    description: description,
    date_received: date_received,
    date_received_raw: date_received_raw,
    address: address,
    council_reference: council_reference,
    applicant: "",
    owner: ""
  })
  
  enrich_after_upsert!(
    table: TABLE,
    council_reference: council_reference,
    address: address
  )

  # Save the PDF link if the column exists
  begin
    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
    upd.execute(document_url, council_reference, address)
  rescue Mysql2::Error => e
    warn "[tasman] db update skipped for #{council_reference}: #{e.message}"
  end

  puts "Upserted #{council_reference} -> #{address}"
  saved += 1
end

puts "Done #{TABLE}. Saved #{saved} item(s)."