tas_councils/scrapers/planbuild.rb

# scrapers/planbuild.rb
require "date"
require "json"
require "nokogiri"
require "uri"
require "net/http"
require "zlib"
require "stringio"
require "fileutils"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/log"
require_relative "../lib/util"
require_relative "../lib/geocode"
require_relative "../lib/enrich"

TABLE = ENV.fetch("TABLE_NAME")
BASE  = "https://portal.planbuild.tas.gov.au"
PAGE  = "#{BASE}/external/advertisement/search"

DOWNLOAD_ATTACHMENTS = ENV["DOWNLOAD_ATTACHMENTS"] == "1"
DOWNLOAD_DIR         = ENV["DOWNLOAD_DIR"] || "/app/downloads"

DB.ensure_table!(TABLE)

# --- cookie + csrf helpers ---
def merge_set_cookie!(jar, res)
    (res.get_fields("set-cookie") || []).each do |raw|
        raw.split(/,(?=[^;]+?=)/).each do |c|
            if c =~ /\A\s*([^=;,\s]+)\s*=\s*([^;,\s]+)/
                jar[$1] = $2
            end
        end
    end
end

def cookie_header(jar)
    base = "accepted=1; disclaimerAccepted=true; insecureSiteWideBanner=1"
    more = jar.map { |k, v| "#{k}=#{v}" }.join("; ")
    [base, more].reject(&:empty?).join("; ")
end

# --- fetch list of advertisements ---
def fetch_list
    jar = {}
    # 1) GET page to grab CSRF + SESSION
    res = Http.request(URI(PAGE), headers: { "Referer" => BASE }, jar: jar)
    merge_set_cookie!(jar, res)
    doc   = Nokogiri::HTML(res.body)
    token = doc.at(%{meta[name="_csrf"]})&.[]("content")
    hdr   = doc.at(%{meta[name="_csrf_header"]})&.[]("content") || "X-CSRF-TOKEN"
    raise "no CSRF token" unless token
    raise "no SESSION cookie" unless jar["SESSION"]

    # 2) POST listadvertisements
    uri = URI("#{BASE}/external/advertisement/search/listadvertisements")
    req = Net::HTTP::Post.new(uri)
    req["Content-Type"]     = "application/json"
    req["X-Requested-With"] = "XMLHttpRequest"
    req["Origin"]           = BASE
    req["Referer"]          = PAGE
    req[hdr]                = token
    req["Cookie"]           = cookie_header(jar)
    req.body = { lgas: [] }.to_json

    res = Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |h| h.request(req) }

    js = JSON.parse(res.body)
    items = js.is_a?(Array) ? js : js["items"]
    [items, jar, token, hdr]
end

# --- fetch details — always returns a Hash ---
def fetch_detail(uuid, jar, token, hdr)
    uri = URI("#{BASE}/external/advertisement/#{uuid}/get")
    req = Net::HTTP::Get.new(uri)
    req["X-Requested-With"] = "XMLHttpRequest"
    req["Referer"]          = PAGE
    req[hdr]                = token
    req["Cookie"]           = cookie_header(jar)

    res = Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |h| h.request(req) }

    body = res["Content-Encoding"] == "gzip" \
        ? Zlib::GzipReader.new(StringIO.new(res.body)).read \
        : res.body
    parsed = JSON.parse(body)
    parsed.is_a?(Hash) ? parsed : {}
rescue JSON::ParserError, Zlib::Error
    {}
end


puts "Fetching PlanBuild list…"

items, jar, token, hdr = fetch_list
puts "Found #{items.length} items for #{TABLE}"

items.each do |r|
    ref   = r["referenceNumber"]
    addr  = r["addressString"]
    desc  = r["description"]
    start = Util.parse_epoch_ms(r["startDate"])
    fin   = Util.parse_epoch_ms(r["endDate"])
    uuid  = r["uuid"]

    next if ref.to_s.strip.empty? || addr.to_s.strip.empty?

    begin
        # derive council table from reference number (e.g. PLN-HOB-xxxx)
        table        = Util.ref_to_table(ref)
        council_name = Util.ref_to_folder(ref).downcase
        DB.ensure_table!(table)

        # fetch detail
        detail = {}
        begin
            detail = fetch_detail(uuid, jar, token, hdr) if uuid
        rescue StandardError => e
            Log.warn "planbuild", "Detail fetch failed for #{ref}: #{e.class} #{e.message}"
        end

        Log.debug "planbuild", "#{ref} -> #{table}, detail keys: #{detail.keys.join(", ")}"

        # handle attachments
        saved_paths = []
        if DOWNLOAD_ATTACHMENTS && uuid && detail["attachments"]&.any?
            dir = File.join(DOWNLOAD_DIR, council_name, ref.gsub(/[^0-9a-zA-Z_-]/, "_"))
            FileUtils.mkdir_p(dir)

            (detail["attachments"] || []).each do |att|
                att_id = att["id"]
                title  = att["documentTitle"].to_s.gsub(/[^\w\-.]+/, "_")

                pdf_url = "#{BASE}/external/advertisement/#{uuid}/attachment/#{att_id}"
                path    = File.join(dir, "#{title}.pdf")

                att_uri = URI(pdf_url)
                att_req = Net::HTTP::Get.new(att_uri)
                att_req["Cookie"]  = cookie_header(jar)
                att_req["Referer"] = "#{BASE}/external/advertisement/#{uuid}"

                att_res = Net::HTTP.start(att_uri.host, att_uri.port, use_ssl: true) { |h| h.request(att_req) }
                File.binwrite(path, att_res.body)
                saved_paths << path
            rescue StandardError => e
                Log.warn "planbuild", "Attachment download failed for #{ref} att #{att["id"]}: #{e.class} #{e.message}"
            end
        end

        local_url = saved_paths.empty? ? nil : saved_paths.first.sub(DOWNLOAD_DIR, "/files")

        # upsert
        DB.upsert(table, {
            description:        desc,
            date_received:      start,
            date_received_raw:  start&.strftime("%Y-%m-%d"),
            on_notice_to:       fin,
            on_notice_to_raw:   fin&.strftime("%Y-%m-%d"),
            address:            addr[0, 255],
            council_reference:  ref[0, 100],
            applicant:          detail["applicant"],
            owner:              detail["owner"],
            local_document_url: local_url
        })

        enrich_after_upsert!(
            table:             table,
            council_reference: ref,
            address:           addr
        )

        Log.info "planbuild", "Upserted #{ref} -> #{addr} into #{table} (PDFs: #{saved_paths.length})"

    rescue StandardError => e
        Log.warn "planbuild", "Skipping #{ref}: #{e.class} #{e.message}"
    end
end

puts "Done #{TABLE}."