tas_councils/scrapers/meandervalley.rb

# Meander Valley Council — Advertised & Approved Planning Applications
# Source: https://www.meander.tas.gov.au/advertised-approved-planning-applications

require "nokogiri"
require "uri"
require "cgi"
require "date"
require_relative "../lib/http"
require_relative "../lib/db"
require_relative "../lib/util"
require_relative "../lib/geocode"
require_relative "../lib/enrich"

require "json"
DEBUG  = ENV["DEBUG"] == "1"
DRY_RUN = ENV["DRY_RUN"] == "1"

TABLE = ENV.fetch("TABLE_NAME")  # run_all.sh -> da_meandervalley
URL   = "https://www.meander.tas.gov.au/advertised-approved-planning-applications"

DOWNLOAD_ATTACHMENTS = ENV["DOWNLOAD_ATTACHMENTS"] == "1"
DOWNLOAD_DIR         = ENV["DOWNLOAD_DIR"] || "/app/downloads"

DB.ensure_table!(TABLE)
ensure_extra_columns!(TABLE)

# Pull nearest text around an anchor for parsing
def host_block_for(a)
    n = a.ancestors("li, p, div, article").first || a.parent
    n ? n.text.to_s.gsub(/\s+/, " ").strip : ""
end

# Extract value after a label up to the next label token
def field_value(text, key_rx, stops)
    s = text.to_s
    m = s.match(/#{key_rx.source}\s*:\s*(.+)/i)
    return "" unless m
    tail = m[1]
    cut = tail.length
    stops.each do |st|
        i = tail.index(st)
        cut = [cut, i].min if i
    end
    tail[0, cut].strip
end

# Normalize refs like "PA\26\0010", "PA.26.0010", "PA 26/0010" to "PA 26/0010"
def normalize_pa_ref(text, pdf = "")
    s = [text.to_s, File.basename(pdf.to_s)].join(" ")
    s = s.tr("\\", "/")
    if (m = s.match(/\bPA\s*([0-9]{2})[\/\.\s]+([0-9]{3,4})\b/i))
        return "PA #{m[1]}/#{m[2]}"
    end
    nil
end

def extract_between(text, key)
    text[/#{key}\s*:\s*(.+?)\s*$/i, 1].to_s.strip
end

def find_pdf_in(nodes, base)
    if (a = nodes.css('a[href$=".pdf"], a[href*=".pdf?"]').first)
        u = a["href"].to_s
        return URI.join(base, u).to_s rescue u
    end
    ""
end

def block_from_anchor(start_node)
    out = [start_node]
    cur = start_node
    20.times do
        cur = cur.next_element
        break if cur.nil?
        t = cur.text.to_s.strip
        break if t =~ /^\s*Application\s*:/i
        break if t =~ /^\s*(Approved|Refused)\b/i
        out << cur
    end
    out
end

def safe_name(s) = s.to_s.gsub(/[^\w\-.]+/, "_")

    # Download the PDF (if enabled) and return a web path:
    # /downloads/meandervalley/<council_reference>/<filename.pdf>
    def download_pdf(url, council_reference)
        return nil if DRY_RUN
        return nil unless DOWNLOAD_ATTACHMENTS && !url.to_s.strip.empty?

        folder = File.join(DOWNLOAD_DIR, "meandervalley", safe_name(council_reference))
        FileUtils.mkdir_p(folder)
        begin
            res  = Http.get_response(url) rescue Http.get(url)
            body = res.respond_to?(:body) ? res.body : res.to_s

            fname = safe_name(File.basename(URI.parse(url).path))
            fname += ".pdf" unless fname.downcase.end_with?(".pdf")
            path = File.join(folder, fname)

            File.binwrite(path, body)
            puts "Saved PDF #{path}"
            "/downloads/meandervalley/#{safe_name(council_reference)}/#{fname}"
        rescue => e
            warn "PDF download failed for #{url}: #{e.class} #{e.message}"
            nil
        end
    end

    html = Http.get(URL, referer: URL)
    doc  = Nokogiri::HTML(html)

    # Items are listed inline under the "Advertised" section with lines like:
    # Application: <link>  Applicant: ...  Property: ...  Proposal: ...  Closes: <date>
    items = []

    # Looser, case-insensitive match for the advertised PDFs
    all_anchors = doc.css('a[href]')
    if DEBUG
        puts "[DEBUG] anchors total=#{all_anchors.size}"
    end

    pdf_anchors = all_anchors.select do |a|
        href = a["href"].to_s
        hd = href.downcase
        hd.include?("/planning-applications/advertised/") && hd.include?(".pdf")
    end

    puts "[DEBUG] matched advertised pdf anchors=#{pdf_anchors.size}" if DEBUG

    pdf_anchors.each do |a|
        href = a["href"].to_s
        pdf  = begin
            URI.join(URL, href).to_s
        rescue URI::InvalidURIError
            href
        end

        blk_text = host_block_for(a)

        # Try to normalize reference from nearby text or the file name
        ref = normalize_pa_ref(blk_text, pdf)
        ref ||= a.text.to_s.gsub(/\s+/, " ")[0, 100]  # backstop so it never overflows

        address = field_value(blk_text, /Property/, [/Proposal/i, /Closes/i, /Application/i])
        desc    = field_value(blk_text, /Proposal/, [/Closes/i, /Application/i])
        close_r = field_value(blk_text, /Closes/,   [/Application/i])

        address = address[0, 255]
        desc    = desc.empty? ? "Development Application" : desc[0, 1000]

        items << {
            council_reference: ref,
            address: address,
            description: desc,
            on_notice_raw: close_r,
            on_notice: Util.parse_aus_date(close_r),
            pdf: pdf,
            title_reference: a.text.to_s.strip
            }
    end


    items.uniq! { |r| [r[:council_reference], r[:address]] }
    puts "Found #{items.length} item(s) for #{TABLE}"

    items.each_with_index do |r, idx|
        if DEBUG
            puts "[DEBUG] ##{idx+1} "\
                "ref=#{r[:council_reference].inspect} (#{r[:council_reference].to_s.length}) | "\
                "addr=#{r[:address].inspect} (#{r[:address].to_s.length}) | "\
                "desc.len=#{r[:description].to_s.length} | "\
                "date_raw=#{r[:on_notice_raw].inspect} (#{r[:on_notice_raw].to_s.length}) | "\
                "pdf=#{r[:pdf].to_s[0,120]}"
            # If you want to see the full source text that was parsed:
            # puts JSON.pretty_generate(raw: r[:_raw].to_s[0,2000])
        end

        if DRY_RUN
            next
        end

        #items.each do |r|
        ref = r[:council_reference].to_s.tr("\\", "/")[0, 100]
        addr = r[:address].to_s[0, 255]
        desc = r[:description].to_s[0, 1000]

        date_received	 = Date.today

        DB.upsert(TABLE, {
            description: r[:description],
            date_received: date_received,
            on_notice_to: r[:on_notice],        # store close date for consistency
            on_notice_to_raw: r[:on_notice_raw],
            address: r[:address],
            council_reference: r[:council_reference],
            applicant: "",
            owner: ""
            })

        enrich_after_upsert!(
            table: TABLE,
            council_reference: r[:council_reference],
            address: r[:address]
            )

        # Download & set local_document_url
        local_doc_url = download_pdf(r[:pdf], r[:council_reference])

        begin
            upd = DB.client.prepare(
                "UPDATE `#{DB.client.escape(TABLE)}` SET " \
                    "  document_url = ?, " \
                    "  local_document_url = COALESCE(?, local_document_url), " \
                    "  on_notice_to = ?, on_notice_to_raw = ?, title_reference = ? " \
                    "WHERE council_reference = ? AND address = ?"
                )
            upd.execute(
                r[:pdf],
                local_doc_url,
                r[:on_notice],
                r[:on_notice_raw],
                r[:title_reference],
                r[:council_reference],
                r[:address]
                )
        rescue => e
            warn "Extras update skipped for #{r[:council_reference]}: #{e.class} #{e.message}"
        end

        puts "Upserted #{r[:council_reference]} -> #{r[:address]}  saved: #{local_doc_url ? 1 : 0}"
    end

    puts "Done #{TABLE}."