tas_councils/scrapers/glenorchy.rb

# Glenorchy City Council – Planning Applications (site page, not PlanBuild)

require "nokogiri"
require "date"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/util"
require_relative "../lib/enrich"


TABLE = ENV.fetch("TABLE_NAME") # run_all.sh sets from filename: da_glenorchy
URL   = "https://www.gcc.tas.gov.au/services/planning-and-building/planning-and-development/planning-applications/"

DOWNLOAD_ATTACHMENTS = ENV["DOWNLOAD_ATTACHMENTS"] == "1"
DOWNLOAD_DIR         = ENV["DOWNLOAD_DIR"] || "/app/downloads"

DB.ensure_table!(TABLE)
ensure_extra_columns!(TABLE)

# Optional: keep the document link with each row (adds a column if missing)
begin
    DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
rescue => e
    warn "Could not add document_url column: #{e.class} #{e.message}"
end

def text_or(node, default = "")
    node ? node.text.strip : default
end

def abs_url(href)
    return "" if href.to_s.strip.empty?
    URI.join(URL, href).to_s
rescue URI::InvalidURIError
    href.to_s
end

def safe_name(s) = s.to_s.gsub(/[^\w\-.]+/, "_")

    def filename_from_response(res, fallback_url)
        cd = res["content-disposition"].to_s
        name =
            if cd =~ /filename\*?=(?:UTF-8''|")?([^\";]+)/
                $1
            else
                File.basename(URI.parse(fallback_url).path)
            end
        name = "document.pdf" if name.to_s.strip.empty?
        name += ".pdf" unless name.downcase.end_with?(".pdf")
        safe_name(name)
    end

    def download_pdf(doc_url, council_reference)
        return "" unless DOWNLOAD_ATTACHMENTS && !doc_url.to_s.strip.empty?
        begin
            u   = abs_url(doc_url)
            res = Http.request(URI.parse(u), headers: {}, jar: {}, referer: URL)
            raise "HTTP #{res.code}" unless res.is_a?(Net::HTTPSuccess)

            dir   = File.join(DOWNLOAD_DIR, "glenorchy", safe_name(council_reference))
            FileUtils.mkdir_p(dir)
            fname = filename_from_response(res, u)
            path  = File.join(dir, fname)

            File.binwrite(path, res.body.to_s)
            puts "  saved #{path}"

            # web-facing relative path (match your nginx/apache mapping)
            "/downloads/glenorchy/#{safe_name(council_reference)}/#{fname}"
        rescue => e
            warn "Download failed for #{doc_url}: #{e.class} #{e.message}"
            ""
        end
    end

    html = Http.get(URL)
    doc  = Nokogiri::HTML(html)

    # Cards on this page use “content-block” classes (WordPress pattern).
    cards = doc.css(".content-block, .content-block--featured")

    puts "Found #{cards.length} items for #{TABLE}"

    saved = 0
    cards.each do |card|
        # Title / address: try specific title element, then fall back to header text or first link text
        title_el = card.at_css(".content-block__title, .content-block__heading, h3, h2, a")
        address  = text_or(title_el)

        # Closing date appears like: "Closes: 29 August 2025"
        date_el = card.at_css(".content-block__date")
        on_notice_to_raw = text_or(date_el).sub(/^Closes:\s*/i, "")
        on_notice_to     = Util.parse_aus_date(on_notice_to_raw)
		
		date_received	 = Date.today

        # Short description paragraph
        desc_el    = card.at_css(".content-block__description p, .content-block__description")
        description = text_or(desc_el)

        # Document link button (often a PDF)
        doc_link = card.at_css(".content-block__button a, a[href$='.pdf']")
        document_url = doc_link ? doc_link["href"].to_s.strip : ""

        # Council reference: prefer the document file name (e.g., ABC-123) if present
        council_reference =
            if document_url && !document_url.empty?
                File.basename(document_url).sub(/\.pdf\z/i, "").upcase
            else
                # fallback: compact title text
                address.gsub(/\s+/, " ")[0,120]
            end

        # Require key fields
        next if address.empty? || council_reference.empty?

        DB.upsert(TABLE, {
            description: description,
			date_received: date_received,
            on_notice_to: on_notice_to,                 # keep the closing date in the DATE column
            on_notice_to_raw: on_notice_to_raw,         # raw text as seen on page
            address: address,
            council_reference: council_reference,
            applicant: "",
            owner: ""
            })

        # store remote doc URL
        begin
            DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
            .execute(document_url, council_reference, address)
        rescue => e
            warn "document_url update failed: #{e.class} #{e.message}"
        end

        # download + store local_document_url
        local_rel = download_pdf(document_url, council_reference)
        if !local_rel.empty?
            begin
                DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET local_document_url = ? WHERE council_reference = ? AND address = ?")
                .execute(local_rel, council_reference, address)
            rescue => e
                warn "local_document_url update failed: #{e.class} #{e.message}"
            end
        end

        enrich_after_upsert!(
            table: TABLE,
            council_reference: council_reference,
            address: address
            )

        # If we added the optional column earlier, keep the link
        begin
            upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
            upd.execute(document_url, council_reference, address)
        rescue => e
            # ignore if column not present
        end

        puts "Upserted #{council_reference} -> #{address} (doc: #{document_url.empty? ? 'none' : 'remote'}#{local_rel.empty? ? '' : ', saved'})"
        saved += 1
    end

    puts "Done #{TABLE}. Saved #{saved} item(s)."